Use the contributor_evaluation.question task type to ask contributors questions in the annotation UI.
Hide configuration properties
string
required
The ID of the question to include in the contributor annotation flow. Questions are most easily created through the UI here.
object
required
Controls the presentation of evaluation item data in the annotation UI.
Hide child attributes
object[]
required
A list of elements to be rendered within the layout container. Can be either an individual Component object, or a nested Container.
Component
Container
Hide child attributes
ItemLocator
required
A pointer to the data in each evaluation item to be displayed within the component.
string
An optional display name to be used at the top of the component. Defaults to the field name from the ItemLocator used for data.
Hide child attributes
object[]
required
A list of elements to be rendered within the layout container. Can be either an individual Component object, or a nested Container.
enum<'row' | 'column'>
default:"row"
The axis that children are placed in the container. Based on CSS flex-direction.
enum<'row' | 'column'>
default:"row"
The axis that children are placed in the container. Based on CSS flex-direction.
string
default:"default"
An optional group identifier. Any contributor_evaluation tasks that share the same queue_id will be grouped together in the annotation UI.
boolean
default:"false"
Whether contributors must answer the question.
string
The ID of the rubric to use for scoring this question.
string
A dataset column whose value prefills this question’s result. Prefill is off by default. Ask your Scale team to enable it for your account.
The column must exist in the evaluation data.
Every task in the evaluation must be a contributor task.
Rows with a null value in the column stay pending.
With skip_prefilled_rows set to True on the evaluation, a row isn’t queued when every required question on it is prefilled. If no question is required, or a required question has no prefill_from, every row is queued.
Example Usage
Assume we have a categorical question with ID a0b25d45-7c6c-4480-9871-61ae0fd0b819 configured as such:We can create an annotation queue as part of our evaluation to ask our contributors to answer this question in order to produce the same correctness result as the guided decoding example:
{"docsConfig":{"theme":"mint","$schema":"https://mintlify.com/docs.json","name":"Scale","colors":{"primary":"#3f2e7b","light":"#ca62ff","dark":"#3f2e7b"},"logo":{"light":"https://mintcdn.com/scalegp/YkQgtlZ7_jaCP1h0/logo/light.svg?fit=max\u0026auto=format\u0026n=YkQgtlZ7_jaCP1h0\u0026q=85\u0026s=3afcca56833a7bfb1c8bf1eec6176590","dark":"https://mintcdn.com/scalegp/YkQgtlZ7_jaCP1h0/logo/dark.svg?fit=max\u0026auto=format\u0026n=YkQgtlZ7_jaCP1h0\u0026q=85\u0026s=522975ce3d7a84c5946bc7f8a6702767"},"favicon":{"light":"https://mintcdn.com/scalegp/vT4TaGb2QZ2PoL0h/favicon-light.svg?fit=max\u0026auto=format\u0026n=vT4TaGb2QZ2PoL0h\u0026q=85\u0026s=9dcad04e938e90fdf4e9ecb28e5ad9f8","dark":"https://mintcdn.com/scalegp/vT4TaGb2QZ2PoL0h/favicon-dark.svg?fit=max\u0026auto=format\u0026n=vT4TaGb2QZ2PoL0h\u0026q=85\u0026s=f516033dd3f7c77f7f362c4f12a9508c"},"background":{"decoration":"windows"},"footer":{"socials":{"x":"https://x.com/scale_ai","facebook":"https://www.facebook.com/scaleapi","linkedin":"https://www.linkedin.com/company/scaleai"}},"styling":{"latex":false},"navigation":{"pages":[]}},"deployment":{"deploymentMetadataValue":{"subdomain":"scalegp","hasAudioNarration":false,"actualSubdomain":"scalegp","customDomains":["docs.gp.scale.com"],"gitSource":{"type":"github","owner":"scaleapi","repo":"sgp-docs","deployBranch":"main","contentDirectory":"","isPrivate":false,"mountPath":"","label":""},"entitlements":{"AI_CHAT":{"status":"ENABLED"},"REMOVE_BRANDING":{"status":"ENABLED"},"CONTEXTUAL_FEEDBACK":{"status":"ENABLED"},"CODE_SNIPPET_FEEDBACK":{"status":"ENABLED"},"WEB_MCP":{"status":"ENABLED"},"NARRATION":{"status":"ENABLED"}},"buildId":"6aff1ad442e7ae4824d5b99e","clientVersion":"0.0.4227","searchFilterCounts":[{"language":"default","tag":"API Reference","count":235},{"language":"default","tag":"AgentOps API","count":141},{"language":"default","tag":"Guides","count":131},{"language":"default","tag":"Evaluations","count":72},{"language":"default","tag":"Agentex API","count":60},{"language":"default","tag":"Dex API","count":48},{"language":"default","tag":"Agents","count":40},{"language":"default","tag":"Evaluation Datasets","count":34},{"language":"default","tag":"Models","count":32},{"language":"default","tag":"Knowledge Bases","count":31}]},"assistantConfig":{},"basePath":"","hasChatPermissions":true,"hasWebMcpPermissions":true,"toggles":[{"name":"kill-isr","enabled":true,"variant":{"name":"disabled","enabled":false,"feature_enabled":true},"impressionData":false},{"name":"custom-rbac-roles","enabled":true,"variant":{"name":"disabled","enabled":false,"feature_enabled":true},"impressionData":false},{"name":"agent-mintlify-slack","enabled":true,"variant":{"name":"disabled","enabled":false,"feature_enabled":true},"impressionData":false},{"name":"agent-integrations-settings","enabled":true,"variant":{"name":"disabled","enabled":false,"feature_enabled":true},"impressionData":false},{"name":"rbac-expansion","enabled":true,"variant":{"name":"disabled","enabled":false,"feature_enabled":true},"impressionData":false},{"name":"automations-pr-grouping","enabled":true,"variant":{"name":"disabled","enabled":false,"feature_enabled":true},"impressionData":false},{"name":"custom-prefix","enabled":true,"variant":{"name":"disabled","enabled":false,"feature_enabled":true},"impressionData":false},{"name":"analytics-v7","enabled":true,"variant":{"name":"disabled","enabled":false,"feature_enabled":true},"impressionData":false},{"name":"user-flows-analytics","enabled":true,"variant":{"name":"disabled","enabled":false,"feature_enabled":true},"impressionData":false},{"name":"no-publish-flow","enabled":true,"variant":{"name":"disabled","enabled":false,"feature_enabled":true},"impressionData":false},{"name":"workos-onboarding","enabled":true,"variant":{"name":"disabled","enabled":false,"feature_enabled":true},"impressionData":false},{"name":"snapshot-serving-live","enabled":true,"variant":{"name":"disabled","enabled":false,"feature_enabled":true},"impressionData":false},{"name":"stripTemperatureMiddleware","enabled":true,"variant":{"name":"disabled","enabled":false,"feature_enabled":true},"impressionData":false},{"name":"support-platform-pylon-zendesk","enabled":true,"variant":{"name":"disabled","enabled":false,"feature_enabled":true},"impressionData":false},{"name":"snapshot-serving","enabled":true,"variant":{"name":"disabled","enabled":false,"feature_enabled":true},"impressionData":false},{"name":"instant-rollback","enabled":true,"variant":{"name":"disabled","enabled":false,"feature_enabled":true},"impressionData":false},{"name":"granular-notifications-server","enabled":true,"variant":{"name":"disabled","enabled":false,"feature_enabled":true},"impressionData":false},{"name":"granular-notifications","enabled":true,"variant":{"name":"disabled","enabled":false,"feature_enabled":true},"impressionData":false},{"name":"automations-fable","enabled":true,"variant":{"name":"disabled","enabled":false,"feature_enabled":true},"impressionData":false},{"name":"fixed-ai-action-pricing","enabled":true,"variant":{"name":"disabled","enabled":false,"feature_enabled":true},"impressionData":false},{"name":"automation-resolve-merge-conflict","enabled":true,"variant":{"name":"disabled","enabled":false,"feature_enabled":true},"impressionData":false},{"name":"scoped-publishing","enabled":true,"variant":{"name":"disabled","enabled":false,"feature_enabled":true},"impressionData":false},{"name":"mintql","enabled":true,"variant":{"name":"disabled","enabled":false,"feature_enabled":true},"impressionData":false},{"name":"page-level-analytics","enabled":true,"variant":{"name":"disabled","enabled":false,"feature_enabled":true},"impressionData":false},{"name":"editor-version-history-v2","enabled":true,"variant":{"name":"disabled","enabled":false,"feature_enabled":true},"impressionData":false},{"name":"billing-page-v2","enabled":true,"variant":{"name":"disabled","enabled":false,"feature_enabled":true},"impressionData":false},{"name":"full-page-search","enabled":true,"variant":{"name":"disabled","enabled":false,"feature_enabled":true},"impressionData":false},{"name":"workspace-projects","enabled":true,"variant":{"name":"disabled","enabled":false,"feature_enabled":true},"impressionData":false},{"name":"assistant-v5","enabled":true,"variant":{"name":"disabled","enabled":false,"feature_enabled":true},"impressionData":false},{"name":"fork-pr-previews","enabled":true,"variant":{"name":"disabled","enabled":false,"feature_enabled":true},"impressionData":false},{"name":"revalidation-jwt","enabled":true,"variant":{"name":"disabled","enabled":false,"feature_enabled":true},"impressionData":false},{"name":"workos-sso-scim","enabled":true,"variant":{"name":"disabled","enabled":false,"feature_enabled":true},"impressionData":false},{"name":"hobby-abuse-scan","enabled":true,"variant":{"name":"disabled","enabled":false,"feature_enabled":true},"impressionData":false},{"name":"dashboard-inbox","enabled":true,"variant":{"name":"disabled","enabled":false,"feature_enabled":true},"impressionData":false},{"name":"workos-signup","enabled":true,"variant":{"name":"disabled","enabled":false,"feature_enabled":true},"impressionData":false},{"name":"turbopuffer-dual-write","enabled":true,"variant":{"name":"disabled","enabled":false,"feature_enabled":true},"impressionData":false},{"name":"workos-enabled","enabled":true,"variant":{"name":"disabled","enabled":false,"feature_enabled":true},"impressionData":false},{"name":"approve-review-flow","enabled":true,"variant":{"name":"disabled","enabled":false,"feature_enabled":true},"impressionData":false},{"name":"editor-metadata","enabled":true,"variant":{"name":"disabled","enabled":false,"feature_enabled":true},"impressionData":false},{"name":"sso-setup-deadline-banner","enabled":true,"variant":{"name":"disabled","enabled":false,"feature_enabled":true},"impressionData":false},{"name":"turbopuffer-serve","enabled":true,"variant":{"name":"disabled","enabled":false,"feature_enabled":true},"impressionData":false},{"name":"live-agents","enabled":true,"variant":{"name":"disabled","enabled":false,"feature_enabled":true},"impressionData":false},{"name":"starter-pricing","enabled":true,"variant":{"name":"treatment","enabled":true,"payload":{"type":"string","value":"treatment"},"feature_enabled":true},"impressionData":false},{"name":"llms-txt-v2","enabled":true,"variant":{"name":"disabled","enabled":false,"feature_enabled":true},"impressionData":false},{"name":"wildcard-email-auth","enabled":true,"variant":{"name":"disabled","enabled":false,"feature_enabled":true},"impressionData":false}],"googleFontsInlined":false},"customCss":[{"filePath":"styles.css","content":"@font-face{font-family:'Aeonik';src:url('https://fonts.gstatic.com/s/inter/v18/UcC73FwrK3iLTeHuS_nVMrMxCp50SjIa1ZL7W0Q5nw.woff2');font-weight:normal;font-style:normal;font-display:swap}h1,h2,h3{font-family:'aeonik';letter-spacing:normal!important}"}]}
{"page":{"pagePath":"docs/v5/next-gen-evaluation/tasks/contributor-evaluation","slug":"/docs/v5/next-gen-evaluation/tasks/contributor-evaluation","pageMetadata":{"title":"Contributor Evaluation","description":"Defer evaluation tasks to human contributors.","href":"/docs/v5/next-gen-evaluation/tasks/contributor-evaluation"},"description":{"compiledSource":"\"use strict\";\nconst {jsx: _jsx} = arguments[0];\nconst {useMDXComponents: _provideComponents} = arguments[0];\nfunction _createMdxContent(props) {\n const _components = {\n p: \"p\",\n ..._provideComponents(),\n ...props.components\n };\n return _jsx(_components.p, {\n children: \"Defer evaluation tasks to human contributors.\"\n });\n}\nfunction MDXContent(props = {}) {\n const {wrapper: MDXLayout} = {\n ..._provideComponents(),\n ...props.components\n };\n return MDXLayout ? _jsx(MDXLayout, {\n ...props,\n children: _jsx(_createMdxContent, {\n ...props\n })\n }) : _createMdxContent(props);\n}\nreturn {\n default: MDXContent\n};\n","frontmatter":{},"scope":{}},"pageType":"default","mdx":{"mdxSource":{"compiledSource":"\"use strict\";const{Fragment:_Fragment,jsx:_jsx,jsxs:_jsxs}=arguments[0];const{useMDXComponents:_provideComponents}=arguments[0];function _createMdxContent(props){const _components={a:`a`,code:`code`,li:`li`,p:`p`,pre:`pre`,span:`span`,ul:`ul`,..._provideComponents(),...props.components},{Card,CodeBlock,Expandable,Heading,ResponseField,Tab,Tabs}=_components;if(!Card)_missingMdxReference(`Card`,true);if(!CodeBlock)_missingMdxReference(`CodeBlock`,true);if(!Expandable)_missingMdxReference(`Expandable`,true);if(!Heading)_missingMdxReference(`Heading`,true);if(!ResponseField)_missingMdxReference(`ResponseField`,true);if(!Tab)_missingMdxReference(`Tab`,true);if(!Tabs)_missingMdxReference(`Tabs`,true);return _jsxs(_Fragment,{children:[_jsx(Heading,{level:`2`,id:`question`,children:`Question`}),`\n`,_jsxs(_components.p,{children:[`Use the `,_jsx(_components.code,{children:`contributor_evaluation.question`}),` task type to ask contributors `,_jsx(_components.a,{href:`https://egp.dashboard.scale.com/evaluation/questions/questions`,children:`questions`}),` in the annotation UI.`]}),`\n`,_jsxs(Expandable,{title:`configuration properties`,defaultOpen:true,children:[_jsx(ResponseField,{name:`question_id`,type:`string`,required:true,id:`param-question-id`,children:_jsxs(_components.p,{children:[`The ID of the question to include in the contributor annotation flow. Questions are most easily created through the UI `,_jsx(_components.a,{href:`https://egp.dashboard.scale.com/evaluation/questions/questions/new`,children:`here`}),`.`]})}),_jsxs(ResponseField,{name:`layout`,type:`object`,required:true,id:`param-layout`,children:[_jsx(_components.p,{children:`Controls the presentation of evaluation item data in the annotation UI.`}),_jsxs(Expandable,{title:`child attributes`,defaultOpen:true,children:[_jsxs(ResponseField,{name:`children`,type:`object[]`,required:true,id:`param-children`,children:[_jsxs(_components.p,{children:[`A list of elements to be rendered within the layout container. Can be either an individual `,_jsx(_components.code,{children:`Component`}),` object, or a nested `,_jsx(_components.code,{children:`Container`}),`.`]}),_jsx(Card,{title:``,children:_jsxs(Tabs,{children:[_jsx(Tab,{title:`Component`,id:`component`,children:_jsxs(Expandable,{title:`child attributes`,defaultOpen:true,children:[_jsx(ResponseField,{name:`data`,type:`ItemLocator`,required:true,id:`param-data`,children:_jsx(_components.p,{children:`A pointer to the data in each evaluation item to be displayed within the component.`})}),_jsx(ResponseField,{name:`label`,type:`string`,id:`param-label`,children:_jsxs(_components.p,{children:[`An optional display name to be used at the top of the component. Defaults to the field name from the `,_jsx(_components.code,{children:`ItemLocator`}),` used for `,_jsx(_components.code,{children:`data`}),`.`]})})]})}),_jsx(Tab,{title:`Container`,id:`container`,children:_jsxs(Expandable,{title:`child attributes`,defaultOpen:true,children:[_jsx(ResponseField,{name:`children`,type:`object[]`,required:true,id:`param-children-1`,children:_jsxs(_components.p,{children:[`A list of elements to be rendered within the layout container. Can be either an individual `,_jsx(_components.code,{children:`Component`}),` object, or a nested `,_jsx(_components.code,{children:`Container`}),`.`]})}),_jsx(ResponseField,{name:`direction`,type:`enum\u003c'row' | 'column'>`,default:`row`,id:`param-direction`,children:_jsxs(_components.p,{children:[`The axis that children are placed in the container. Based on CSS `,_jsx(_components.a,{href:`https://developer.mozilla.org/en-US/docs/Web/CSS/flex-direction`,children:_jsx(_components.code,{children:`flex-direction`})}),`.`]})})]})})]})})]}),_jsx(ResponseField,{name:`direction`,type:`enum\u003c'row' | 'column'>`,default:`row`,id:`param-direction-1`,children:_jsxs(_components.p,{children:[`The axis that children are placed in the container. Based on CSS `,_jsx(_components.a,{href:`https://developer.mozilla.org/en-US/docs/Web/CSS/flex-direction`,children:_jsx(_components.code,{children:`flex-direction`})}),`.`]})})]})]}),_jsx(ResponseField,{name:`queue_id`,type:`string`,default:`default`,id:`param-queue-id`,children:_jsxs(_components.p,{children:[`An optional group identifier. Any `,_jsx(_components.code,{children:`contributor_evaluation`}),` tasks that share the same `,_jsx(_components.code,{children:`queue_id`}),` will be grouped together in the annotation UI.`]})}),_jsx(ResponseField,{name:`required`,type:`boolean`,default:`false`,id:`param-required`,children:_jsx(_components.p,{children:`Whether contributors must answer the question.`})}),_jsx(ResponseField,{name:`rubric_id`,type:`string`,id:`param-rubric-id`,children:_jsx(_components.p,{children:`The ID of the rubric to use for scoring this question.`})}),_jsxs(ResponseField,{name:`prefill_from`,type:`string`,id:`param-prefill-from`,children:[_jsx(_components.p,{children:`A dataset column whose value prefills this question’s result. Prefill is off by default. Ask your Scale team to enable it for your account.`}),_jsxs(_components.ul,{children:[`\n`,_jsx(_components.li,{children:`The column must exist in the evaluation data.`}),`\n`,_jsx(_components.li,{children:`Every task in the evaluation must be a contributor task.`}),`\n`,_jsx(_components.li,{children:`Rows with a null value in the column stay pending.`}),`\n`,_jsxs(_components.li,{children:[`With `,_jsx(_components.code,{children:`skip_prefilled_rows`}),` set to `,_jsx(_components.code,{children:`True`}),` on the evaluation, a row isn’t queued when every required question on it is prefilled. If no question is required, or a required question has no `,_jsx(_components.code,{children:`prefill_from`}),`, every row is queued.`]}),`\n`]})]})]}),`\n`,_jsx(Heading,{level:`3`,id:`example-usage`,children:`Example Usage`}),`\n`,_jsxs(_components.p,{children:[`Assume we have a categorical question with ID `,_jsx(_components.code,{children:`a0b25d45-7c6c-4480-9871-61ae0fd0b819`}),` configured as such:`]}),`\n`,_jsx(`img`,{width:`75%`,className:`mint-mx-auto`,noZoom:true,src:`https://mintcdn.com/scalegp/2sSrpizRElJqluR6/images/nextgen-evaluation/correctness-question.png?fit=max\u0026auto=format\u0026n=2sSrpizRElJqluR6\u0026q=85\u0026s=4697cc6e2832b1266578deed4f5d489f`,\"data-og-width\":`1216`,\"data-og-height\":`794`,\"data-path\":`images/nextgen-evaluation/correctness-question.png`,\"data-optimize\":`true`,\"data-opv\":`3`,srcset:`https://mintcdn.com/scalegp/2sSrpizRElJqluR6/images/nextgen-evaluation/correctness-question.png?w=280\u0026fit=max\u0026auto=format\u0026n=2sSrpizRElJqluR6\u0026q=85\u0026s=5fb7299ac70976b3891759d46c66bfac 280w, https://mintcdn.com/scalegp/2sSrpizRElJqluR6/images/nextgen-evaluation/correctness-question.png?w=560\u0026fit=max\u0026auto=format\u0026n=2sSrpizRElJqluR6\u0026q=85\u0026s=18d948869e5955cb9b9396ec92d01493 560w, https://mintcdn.com/scalegp/2sSrpizRElJqluR6/images/nextgen-evaluation/correctness-question.png?w=840\u0026fit=max\u0026auto=format\u0026n=2sSrpizRElJqluR6\u0026q=85\u0026s=29eb24659bfbb75f9de3fede34115ab4 840w, https://mintcdn.com/scalegp/2sSrpizRElJqluR6/images/nextgen-evaluation/correctness-question.png?w=1100\u0026fit=max\u0026auto=format\u0026n=2sSrpizRElJqluR6\u0026q=85\u0026s=534a4c76fd68a33265f2d3821204bce4 1100w, https://mintcdn.com/scalegp/2sSrpizRElJqluR6/images/nextgen-evaluation/correctness-question.png?w=1650\u0026fit=max\u0026auto=format\u0026n=2sSrpizRElJqluR6\u0026q=85\u0026s=75cb14c1820da57db22c912d43336b9a 1650w, https://mintcdn.com/scalegp/2sSrpizRElJqluR6/images/nextgen-evaluation/correctness-question.png?w=2500\u0026fit=max\u0026auto=format\u0026n=2sSrpizRElJqluR6\u0026q=85\u0026s=49e142db35a402ca9fab0889f18cc18b 2500w`}),`\n`,_jsxs(_components.p,{children:[`We can create an annotation queue as part of our evaluation to ask our contributors to answer this question in order to produce the same `,_jsx(_components.code,{children:`correctness`}),` result as the `,_jsx(_components.a,{href:`/docs/v5/next-gen-evaluation/tasks/auto-evaluation#example-usage`,children:`guided decoding example`}),`:`]}),`\n`,_jsx(CodeBlock,{filename:``,numberOfLines:`36`,language:`python`,children:_jsx(_components.pre,{className:`shiki shiki-themes github-light-default dark-plus`,style:{backgroundColor:`#ffffff`,\"--shiki-dark-bg\":`#0B0C0E`,color:`#1f2328`,\"--shiki-dark\":`#D4D4D4`},language:`python`,children:_jsxs(_components.code,{language:`python`,numberOfLines:`36`,children:[_jsxs(_components.span,{className:`line`,children:[_jsx(_components.span,{style:{color:`#1F2328`,\"--shiki-dark\":`#D4D4D4`},children:`client.evaluations.create(`}),_jsx(_components.span,{style:{color:`#953800`,\"--shiki-dark\":`#9CDCFE`},children:`evaluation`}),_jsx(_components.span,{style:{color:`#CF222E`,\"--shiki-dark\":`#D4D4D4`},children:`=`}),_jsx(_components.span,{style:{color:`#1F2328`,\"--shiki-dark\":`#D4D4D4`},children:`{`})]}),`\n`,_jsxs(_components.span,{className:`line`,children:[_jsx(_components.span,{style:{color:`#0A3069`,\"--shiki-dark\":`#CE9178`},children:` \"name\"`}),_jsx(_components.span,{style:{color:`#1F2328`,\"--shiki-dark\":`#D4D4D4`},children:`: `}),_jsx(_components.span,{style:{color:`#0A3069`,\"--shiki-dark\":`#CE9178`},children:`\"Example Correctness Evaluation\"`}),_jsx(_components.span,{style:{color:`#1F2328`,\"--shiki-dark\":`#D4D4D4`},children:`,`})]}),`\n`,_jsxs(_components.span,{className:`line`,children:[_jsx(_components.span,{style:{color:`#0A3069`,\"--shiki-dark\":`#CE9178`},children:` \"data\"`}),_jsx(_components.span,{style:{color:`#1F2328`,\"--shiki-dark\":`#D4D4D4`},children:`: [`})]}),`\n`,_jsx(_components.span,{className:`line`,children:_jsx(_components.span,{style:{color:`#1F2328`,\"--shiki-dark\":`#D4D4D4`},children:` {`})}),`\n`,_jsxs(_components.span,{className:`line`,children:[_jsx(_components.span,{style:{color:`#0A3069`,\"--shiki-dark\":`#CE9178`},children:` \"input\"`}),_jsx(_components.span,{style:{color:`#1F2328`,\"--shiki-dark\":`#D4D4D4`},children:`: `}),_jsx(_components.span,{style:{color:`#0A3069`,\"--shiki-dark\":`#CE9178`},children:`\"What color is the sky?\"`}),_jsx(_components.span,{style:{color:`#1F2328`,\"--shiki-dark\":`#D4D4D4`},children:`,`})]}),`\n`,_jsxs(_components.span,{className:`line`,children:[_jsx(_components.span,{style:{color:`#0A3069`,\"--shiki-dark\":`#CE9178`},children:` \"expected_output\"`}),_jsx(_components.span,{style:{color:`#1F2328`,\"--shiki-dark\":`#D4D4D4`},children:`: `}),_jsx(_components.span,{style:{color:`#0A3069`,\"--shiki-dark\":`#CE9178`},children:`\"Blue\"`}),_jsx(_components.span,{style:{color:`#1F2328`,\"--shiki-dark\":`#D4D4D4`},children:`,`})]}),`\n`,_jsxs(_components.span,{className:`line`,children:[_jsx(_components.span,{style:{color:`#0A3069`,\"--shiki-dark\":`#CE9178`},children:` \"generated_output\"`}),_jsx(_components.span,{style:{color:`#1F2328`,\"--shiki-dark\":`#D4D4D4`},children:`: `}),_jsx(_components.span,{style:{color:`#0A3069`,\"--shiki-dark\":`#CE9178`},children:`\"The sky appears blue during ...\"`})]}),`\n`,_jsx(_components.span,{className:`line`,children:_jsx(_components.span,{style:{color:`#1F2328`,\"--shiki-dark\":`#D4D4D4`},children:` },`})}),`\n`,_jsx(_components.span,{className:`line`,children:_jsx(_components.span,{style:{color:`#0550AE`,\"--shiki-dark\":`#D4D4D4`},children:` ...`})}),`\n`,_jsx(_components.span,{className:`line`,children:_jsx(_components.span,{style:{color:`#1F2328`,\"--shiki-dark\":`#D4D4D4`},children:` ],`})}),`\n`,_jsxs(_components.span,{className:`line`,children:[_jsx(_components.span,{style:{color:`#0A3069`,\"--shiki-dark\":`#CE9178`},children:` \"tasks\"`}),_jsx(_components.span,{style:{color:`#1F2328`,\"--shiki-dark\":`#D4D4D4`},children:`: [`})]}),`\n`,_jsx(_components.span,{className:`line`,children:_jsx(_components.span,{style:{color:`#1F2328`,\"--shiki-dark\":`#D4D4D4`},children:` {`})}),`\n`,_jsxs(_components.span,{className:`line`,children:[_jsx(_components.span,{style:{color:`#0A3069`,\"--shiki-dark\":`#CE9178`},children:` \"task_type\"`}),_jsx(_components.span,{style:{color:`#1F2328`,\"--shiki-dark\":`#D4D4D4`},children:`: `}),_jsx(_components.span,{style:{color:`#0A3069`,\"--shiki-dark\":`#CE9178`},children:`\"contributor_evaluation.question\"`}),_jsx(_components.span,{style:{color:`#1F2328`,\"--shiki-dark\":`#D4D4D4`},children:`,`})]}),`\n`,_jsxs(_components.span,{className:`line`,children:[_jsx(_components.span,{style:{color:`#0A3069`,\"--shiki-dark\":`#CE9178`},children:` \"alias\"`}),_jsx(_components.span,{style:{color:`#1F2328`,\"--shiki-dark\":`#D4D4D4`},children:`: `}),_jsx(_components.span,{style:{color:`#0A3069`,\"--shiki-dark\":`#CE9178`},children:`\"correctness\"`}),_jsx(_components.span,{style:{color:`#1F2328`,\"--shiki-dark\":`#D4D4D4`},children:`,`})]}),`\n`,_jsxs(_components.span,{className:`line`,children:[_jsx(_components.span,{style:{color:`#0A3069`,\"--shiki-dark\":`#CE9178`},children:` \"configuration\"`}),_jsx(_components.span,{style:{color:`#1F2328`,\"--shiki-dark\":`#D4D4D4`},children:`: {`})]}),`\n`,_jsxs(_components.span,{className:`line`,children:[_jsx(_components.span,{style:{color:`#0A3069`,\"--shiki-dark\":`#CE9178`},children:` \"question_id\"`}),_jsx(_components.span,{style:{color:`#1F2328`,\"--shiki-dark\":`#D4D4D4`},children:`: `}),_jsx(_components.span,{style:{color:`#0A3069`,\"--shiki-dark\":`#CE9178`},children:`\"a0b25d45-7c6c-4480-9871-61ae0fd0b819\"`}),_jsx(_components.span,{style:{color:`#1F2328`,\"--shiki-dark\":`#D4D4D4`},children:`,`})]}),`\n`,_jsxs(_components.span,{className:`line`,children:[_jsx(_components.span,{style:{color:`#0A3069`,\"--shiki-dark\":`#CE9178`},children:` \"layout\"`}),_jsx(_components.span,{style:{color:`#1F2328`,\"--shiki-dark\":`#D4D4D4`},children:`: {`})]}),`\n`,_jsxs(_components.span,{className:`line`,children:[_jsx(_components.span,{style:{color:`#0A3069`,\"--shiki-dark\":`#CE9178`},children:` \"direction\"`}),_jsx(_components.span,{style:{color:`#1F2328`,\"--shiki-dark\":`#D4D4D4`},children:`: `}),_jsx(_components.span,{style:{color:`#0A3069`,\"--shiki-dark\":`#CE9178`},children:`\"column\"`}),_jsx(_components.span,{style:{color:`#1F2328`,\"--shiki-dark\":`#D4D4D4`},children:`,`})]}),`\n`,_jsxs(_components.span,{className:`line`,children:[_jsx(_components.span,{style:{color:`#0A3069`,\"--shiki-dark\":`#CE9178`},children:` \"children\"`}),_jsx(_components.span,{style:{color:`#1F2328`,\"--shiki-dark\":`#D4D4D4`},children:`: [`})]}),`\n`,_jsx(_components.span,{className:`line`,children:_jsx(_components.span,{style:{color:`#6E7781`,\"--shiki-dark\":`#6A9955`},children:` # Component:`})}),`\n`,_jsxs(_components.span,{className:`line`,children:[_jsx(_components.span,{style:{color:`#1F2328`,\"--shiki-dark\":`#D4D4D4`},children:` { `}),_jsx(_components.span,{style:{color:`#0A3069`,\"--shiki-dark\":`#CE9178`},children:`\"data\"`}),_jsx(_components.span,{style:{color:`#1F2328`,\"--shiki-dark\":`#D4D4D4`},children:`: `}),_jsx(_components.span,{style:{color:`#0A3069`,\"--shiki-dark\":`#CE9178`},children:`\"item.input\"`}),_jsx(_components.span,{style:{color:`#1F2328`,\"--shiki-dark\":`#D4D4D4`},children:`, `}),_jsx(_components.span,{style:{color:`#0A3069`,\"--shiki-dark\":`#CE9178`},children:`\"label\"`}),_jsx(_components.span,{style:{color:`#1F2328`,\"--shiki-dark\":`#D4D4D4`},children:`: `}),_jsx(_components.span,{style:{color:`#0A3069`,\"--shiki-dark\":`#CE9178`},children:`\"User Query\"`}),_jsx(_components.span,{style:{color:`#1F2328`,\"--shiki-dark\":`#D4D4D4`},children:` },`})]}),`\n`,_jsx(_components.span,{className:`line`,children:_jsx(_components.span,{style:{color:`#6E7781`,\"--shiki-dark\":`#6A9955`},children:` # Nested container:`})}),`\n`,_jsx(_components.span,{className:`line`,children:_jsx(_components.span,{style:{color:`#1F2328`,\"--shiki-dark\":`#D4D4D4`},children:` {`})}),`\n`,_jsxs(_components.span,{className:`line`,children:[_jsx(_components.span,{style:{color:`#0A3069`,\"--shiki-dark\":`#CE9178`},children:` \"direction\"`}),_jsx(_components.span,{style:{color:`#1F2328`,\"--shiki-dark\":`#D4D4D4`},children:`: `}),_jsx(_components.span,{style:{color:`#0A3069`,\"--shiki-dark\":`#CE9178`},children:`\"row\"`}),_jsx(_components.span,{style:{color:`#1F2328`,\"--shiki-dark\":`#D4D4D4`},children:`,`})]}),`\n`,_jsxs(_components.span,{className:`line`,children:[_jsx(_components.span,{style:{color:`#0A3069`,\"--shiki-dark\":`#CE9178`},children:` \"children\"`}),_jsx(_components.span,{style:{color:`#1F2328`,\"--shiki-dark\":`#D4D4D4`},children:`: [`})]}),`\n`,_jsxs(_components.span,{className:`line`,children:[_jsx(_components.span,{style:{color:`#1F2328`,\"--shiki-dark\":`#D4D4D4`},children:` { `}),_jsx(_components.span,{style:{color:`#0A3069`,\"--shiki-dark\":`#CE9178`},children:`\"data\"`}),_jsx(_components.span,{style:{color:`#1F2328`,\"--shiki-dark\":`#D4D4D4`},children:`: `}),_jsx(_components.span,{style:{color:`#0A3069`,\"--shiki-dark\":`#CE9178`},children:`\"item.expected_output\"`}),_jsx(_components.span,{style:{color:`#1F2328`,\"--shiki-dark\":`#D4D4D4`},children:`, `}),_jsx(_components.span,{style:{color:`#0A3069`,\"--shiki-dark\":`#CE9178`},children:`\"label\"`}),_jsx(_components.span,{style:{color:`#1F2328`,\"--shiki-dark\":`#D4D4D4`},children:`: `}),_jsx(_components.span,{style:{color:`#0A3069`,\"--shiki-dark\":`#CE9178`},children:`\"Expected Output\"`}),_jsx(_components.span,{style:{color:`#1F2328`,\"--shiki-dark\":`#D4D4D4`},children:` },`})]}),`\n`,_jsxs(_components.span,{className:`line`,children:[_jsx(_components.span,{style:{color:`#1F2328`,\"--shiki-dark\":`#D4D4D4`},children:` { `}),_jsx(_components.span,{style:{color:`#0A3069`,\"--shiki-dark\":`#CE9178`},children:`\"data\"`}),_jsx(_components.span,{style:{color:`#1F2328`,\"--shiki-dark\":`#D4D4D4`},children:`: `}),_jsx(_components.span,{style:{color:`#0A3069`,\"--shiki-dark\":`#CE9178`},children:`\"item.generated_output\"`}),_jsx(_components.span,{style:{color:`#1F2328`,\"--shiki-dark\":`#D4D4D4`},children:`, `}),_jsx(_components.span,{style:{color:`#0A3069`,\"--shiki-dark\":`#CE9178`},children:`\"label\"`}),_jsx(_components.span,{style:{color:`#1F2328`,\"--shiki-dark\":`#D4D4D4`},children:`: `}),_jsx(_components.span,{style:{color:`#0A3069`,\"--shiki-dark\":`#CE9178`},children:`\"Generated Output\"`}),_jsx(_components.span,{style:{color:`#1F2328`,\"--shiki-dark\":`#D4D4D4`},children:` }`})]}),`\n`,_jsx(_components.span,{className:`line`,children:_jsx(_components.span,{style:{color:`#1F2328`,\"--shiki-dark\":`#D4D4D4`},children:` ]`})}),`\n`,_jsx(_components.span,{className:`line`,children:_jsx(_components.span,{style:{color:`#1F2328`,\"--shiki-dark\":`#D4D4D4`},children:` }`})}),`\n`,_jsx(_components.span,{className:`line`,children:_jsx(_components.span,{style:{color:`#1F2328`,\"--shiki-dark\":`#D4D4D4`},children:` ]`})}),`\n`,_jsx(_components.span,{className:`line`,children:_jsx(_components.span,{style:{color:`#1F2328`,\"--shiki-dark\":`#D4D4D4`},children:` },`})}),`\n`,_jsxs(_components.span,{className:`line`,children:[_jsx(_components.span,{style:{color:`#0A3069`,\"--shiki-dark\":`#CE9178`},children:` \"queue_id\"`}),_jsx(_components.span,{style:{color:`#1F2328`,\"--shiki-dark\":`#D4D4D4`},children:`: `}),_jsx(_components.span,{style:{color:`#0A3069`,\"--shiki-dark\":`#CE9178`},children:`\"default\"`})]}),`\n`,_jsx(_components.span,{className:`line`,children:_jsx(_components.span,{style:{color:`#1F2328`,\"--shiki-dark\":`#D4D4D4`},children:` }`})}),`\n`,_jsx(_components.span,{className:`line`,children:_jsx(_components.span,{style:{color:`#1F2328`,\"--shiki-dark\":`#D4D4D4`},children:` }`})}),`\n`,_jsx(_components.span,{className:`line`,children:_jsx(_components.span,{style:{color:`#1F2328`,\"--shiki-dark\":`#D4D4D4`},children:` ]`})}),`\n`,_jsx(_components.span,{className:`line`,children:_jsx(_components.span,{style:{color:`#1F2328`,\"--shiki-dark\":`#D4D4D4`},children:`})`})}),`\n`]})})})]})}function MDXContent(props={}){const{wrapper:MDXLayout}={..._provideComponents(),...props.components};return MDXLayout?_jsx(MDXLayout,{...props,children:_jsx(_createMdxContent,{...props})}):_createMdxContent(props)}return{default:MDXContent};function _missingMdxReference(id,component){throw new Error(`Expected `+(component?`component`:`object`)+\" `\"+id+\"` to be defined: you likely forgot to import, pass, or provide it.\")}","frontmatter":{},"scope":{"config":{},"pageMetadata":{"title":"Contributor Evaluation","description":"Defer evaluation tasks to human contributors.","href":"/docs/v5/next-gen-evaluation/tasks/contributor-evaluation"}}},"mdxSourceWithNoJs":null},"mdxExtracts":{"tableOfContents":[{"title":"Question","slug":"question","depth":2,"children":[{"title":"Example Usage","slug":"example-usage","depth":3,"children":[]}]}],"codeExamples":{}},"jsLinks":[],"dynamicCss":".mint-mx-auto {\n margin-left: auto;\n margin-right: auto\n}","subdomain":"scalegp","siteName":"Scale","title":"Contributor Evaluation - Scale","navMaps":{"firstHrefInVersion":{"V5":{"href":"/docs/v5/next-gen-evaluation/tasks/contributor-evaluation","matchLevel":0},"V4 (legacy)":{"href":"/docs/how-to-run-an-evaluation","matchLevel":2}},"firstHrefInLanguage":{}},"lastModified":"2026-11-19T15:51:11.546Z","originalFileLocation":"docs/v5/next-gen-evaluation/tasks/contributor-evaluation.mdx"},"headHtml":"\u003cscript data-mint-head>(function(){var d=document.documentElement,p=null;if(!p){try{p=localStorage.getItem(\"isDarkMode\")}catch(e){}if(p==='true')p='dark';else if(p==='false')p='light';if(p!=='light'\u0026\u0026p!=='dark'\u0026\u0026p!=='system')p=\"system\"}var r=p==='system'?(matchMedia(\"(prefers-color-scheme: dark)\").matches?'dark':'light'):p;d.classList.remove(r==='dark'?'light':'dark');d.classList.add(r);d.setAttribute(\"data-theme-preference\",p)})();\u003c/script>\u003cmeta data-mint-head name=\"description\" content=\"Defer evaluation tasks to human contributors.\">\u003cmeta data-mint-head name=\"application-name\" content=\"Scale\">\u003cmeta data-mint-head name=\"generator\" content=\"Mintlify\">\u003clink data-mint-head rel=\"icon\" type=\"image/png\" sizes=\"192x192\" href=\"/mintlify-assets/_mintlify/favicons/scalegp/KjwNgkAP0XyDRKtA/_generated/favicon/android-chrome-192x192.png\" media=\"(prefers-color-scheme: light)\">\u003clink data-mint-head rel=\"apple-touch-icon\" type=\"image/png\" sizes=\"180x180\" href=\"/mintlify-assets/_mintlify/favicons/scalegp/KjwNgkAP0XyDRKtA/_generated/favicon/apple-touch-icon.png\">\u003clink data-mint-head rel=\"icon\" type=\"image/png\" sizes=\"16x16\" href=\"/mintlify-assets/_mintlify/favicons/scalegp/KjwNgkAP0XyDRKtA/_generated/favicon/favicon-16x16.png\" media=\"(prefers-color-scheme: light)\">\u003clink data-mint-head rel=\"icon\" type=\"image/png\" sizes=\"32x32\" href=\"/mintlify-assets/_mintlify/favicons/scalegp/KjwNgkAP0XyDRKtA/_generated/favicon/favicon-32x32.png\" media=\"(prefers-color-scheme: light)\">\u003clink data-mint-head rel=\"shortcut icon\" type=\"image/x-icon\" href=\"/mintlify-assets/_mintlify/favicons/scalegp/KjwNgkAP0XyDRKtA/_generated/favicon/favicon.ico\" media=\"(prefers-color-scheme: light)\">\u003clink data-mint-head rel=\"icon\" type=\"image/png\" sizes=\"192x192\" href=\"/mintlify-assets/_mintlify/favicons/scalegp/KjwNgkAP0XyDRKtA/_generated/favicon-dark/android-chrome-192x192.png\" media=\"(prefers-color-scheme: dark)\">\u003clink data-mint-head rel=\"icon\" type=\"image/png\" sizes=\"16x16\" href=\"/mintlify-assets/_mintlify/favicons/scalegp/KjwNgkAP0XyDRKtA/_generated/favicon-dark/favicon-16x16.png\" media=\"(prefers-color-scheme: dark)\">\u003clink data-mint-head rel=\"icon\" type=\"image/png\" sizes=\"32x32\" href=\"/mintlify-assets/_mintlify/favicons/scalegp/KjwNgkAP0XyDRKtA/_generated/favicon-dark/favicon-32x32.png\" media=\"(prefers-color-scheme: dark)\">\u003clink data-mint-head rel=\"shortcut icon\" type=\"image/x-icon\" href=\"/mintlify-assets/_mintlify/favicons/scalegp/KjwNgkAP0XyDRKtA/_generated/favicon-dark/favicon.ico\" media=\"(prefers-color-scheme: dark)\">\u003cmeta data-mint-head property=\"og:title\" content=\"Contributor Evaluation - Scale\">\u003cmeta data-mint-head property=\"og:description\" content=\"Defer evaluation tasks to human contributors.\">\u003cmeta data-mint-head property=\"og:site_name\" content=\"Scale\">\u003cmeta data-mint-head property=\"og:url\" content=\"https://docs.gp.scale.com/docs/v5/next-gen-evaluation/tasks/contributor-evaluation\">\u003cmeta data-mint-head property=\"og:type\" content=\"website\">\u003cmeta data-mint-head property=\"og:image\" content=\"https://scalegp.mintlify.site/_mintlify/api/og/v2?division=Task+Types\u0026amp;title=Contributor+Evaluation\u0026amp;description=Defer+evaluation+tasks+to+human+contributors.\u0026amp;theme=4eb366a505141b7fa10bcb74\u0026amp;signature=0a0b1f71cfe450d57b7b31a8dec6dbfe4638d93135772a7129c77fe50834d95b\">\u003cmeta data-mint-head property=\"og:image:width\" content=\"1200\">\u003cmeta data-mint-head property=\"og:image:height\" content=\"630\">\u003cmeta data-mint-head name=\"twitter:card\" content=\"summary_large_image\">\u003cmeta data-mint-head name=\"twitter:title\" content=\"Contributor Evaluation - Scale\">\u003cmeta data-mint-head name=\"twitter:description\" content=\"Defer evaluation tasks to human contributors.\">\u003cmeta data-mint-head name=\"twitter:image\" content=\"https://scalegp.mintlify.site/_mintlify/api/og/v2?division=Task+Types\u0026amp;title=Contributor+Evaluation\u0026amp;description=Defer+evaluation+tasks+to+human+contributors.\u0026amp;theme=4eb366a505141b7fa10bcb74\u0026amp;signature=0a0b1f71cfe450d57b7b31a8dec6dbfe4638d93135772a7129c77fe50834d95b\">\u003cmeta data-mint-head name=\"twitter:image:width\" content=\"1200\">\u003cmeta data-mint-head name=\"twitter:image:height\" content=\"630\">\u003cmeta data-mint-head name=\"msapplication-config\" content=\"/mintlify-assets/_mintlify/favicons/scalegp/KjwNgkAP0XyDRKtA/_generated/favicon/browserconfig.xml\">\u003cmeta data-mint-head name=\"apple-mobile-web-app-title\" content=\"Scale\">\u003cmeta data-mint-head name=\"msapplication-TileColor\" content=\"#3f2e7b\">\u003clink data-mint-head rel=\"canonical\" href=\"https://docs.gp.scale.com/docs/v5/next-gen-evaluation/tasks/contributor-evaluation\">\u003clink data-mint-head rel=\"alternate\" type=\"application/xml\" href=\"/sitemap.xml\">\u003clink data-mint-head rel=\"alternate\" type=\"text/markdown\" href=\"/docs/v5/next-gen-evaluation/tasks/contributor-evaluation.md\">\u003cscript data-mint-head type=\"application/ld+json\">{\"@context\":\"https://schema.org\",\"@graph\":[{\"@type\":\"Organization\",\"@id\":\"https://docs.gp.scale.com/#organization\",\"name\":\"Scale\",\"url\":\"https://docs.gp.scale.com\",\"logo\":{\"@type\":\"ImageObject\",\"url\":\"https://mintcdn.com/scalegp/YkQgtlZ7_jaCP1h0/logo/light.svg?fit=max\u0026auto=format\u0026n=YkQgtlZ7_jaCP1h0\u0026q=85\u0026s=3afcca56833a7bfb1c8bf1eec6176590\"}},{\"@type\":\"WebSite\",\"@id\":\"https://docs.gp.scale.com#website\",\"name\":\"Scale\",\"url\":\"https://docs.gp.scale.com\",\"publisher\":{\"@id\":\"https://docs.gp.scale.com/#organization\"}},{\"@type\":\"WebPage\",\"@id\":\"https://docs.gp.scale.com/docs/v5/next-gen-evaluation/tasks/contributor-evaluation#webpage\",\"url\":\"https://docs.gp.scale.com/docs/v5/next-gen-evaluation/tasks/contributor-evaluation\",\"name\":\"Contributor Evaluation\",\"description\":\"Defer evaluation tasks to human contributors.\",\"dateModified\":\"2026-11-19T15:51:11.546Z\",\"isPartOf\":{\"@id\":\"https://docs.gp.scale.com#website\"},\"breadcrumb\":{\"@id\":\"https://docs.gp.scale.com/docs/v5/next-gen-evaluation/tasks/contributor-evaluation#breadcrumb\"}},{\"@type\":\"BreadcrumbList\",\"@id\":\"https://docs.gp.scale.com/docs/v5/next-gen-evaluation/tasks/contributor-evaluation#breadcrumb\",\"itemListElement\":[{\"@type\":\"ListItem\",\"position\":1,\"name\":\"Evaluations\",\"item\":\"https://docs.gp.scale.com/docs/v5/next-gen-evaluation/overview\"},{\"@type\":\"ListItem\",\"position\":2,\"name\":\"Task Types\",\"item\":\"https://docs.gp.scale.com/docs/v5/next-gen-evaluation/tasks/auto-evaluation\"},{\"@type\":\"ListItem\",\"position\":3,\"name\":\"Contributor Evaluation\",\"item\":\"https://docs.gp.scale.com/docs/v5/next-gen-evaluation/tasks/contributor-evaluation\"}]},{\"@type\":[\"Article\",\"TechArticle\"],\"@id\":\"https://docs.gp.scale.com/docs/v5/next-gen-evaluation/tasks/contributor-evaluation#article\",\"headline\":\"Contributor Evaluation\",\"name\":\"Contributor Evaluation\",\"description\":\"Defer evaluation tasks to human contributors.\",\"url\":\"https://docs.gp.scale.com/docs/v5/next-gen-evaluation/tasks/contributor-evaluation\",\"mainEntityOfPage\":{\"@id\":\"https://docs.gp.scale.com/docs/v5/next-gen-evaluation/tasks/contributor-evaluation#webpage\"},\"dateModified\":\"2026-11-19T15:51:11.546Z\",\"publisher\":{\"@id\":\"https://docs.gp.scale.com/#organization\"},\"isPartOf\":{\"@id\":\"https://docs.gp.scale.com#website\"}}]}\u003c/script>","agentPointerHref":"/docs/v5/next-gen-evaluation/tasks/contributor-evaluation.md","siteName":"Scale","nav":{"global":null,"versions":[{"version":"V5","tabs":[{"tab":"Guides","groups":[{"group":"Overview","pages":[{"title":"SGP Overview","href":"/docs/v5/overview"}]},{"group":"Getting Started","pages":[{"title":"Authentication","href":"/docs/v5/authentication"},{"title":"Quick Start - AgentOps SDK","href":"/docs/v5/next-gen-evaluation/getting-started"}]},{"group":"Infrastructure","pages":[{"group":"AWS Infrastructure","pages":[{"title":"AWS SGP Deployment","href":"/docs/infrastructure/aws/aws-sgp-deployment"},{"title":"AWS Architecture Reference","href":"/docs/infrastructure/aws/aws-architecture-reference"}]},{"group":"Azure Infrastructure","pages":[{"title":"Azure SGP Deployment","href":"/docs/infrastructure/azure/azure-sgp-deployment"},{"title":"Azure Architecture Reference","href":"/docs/infrastructure/azure/azure-architecture-reference"}]},{"group":"GCP Infrastructure","pages":[{"title":"GCP SGP Deployment","href":"/docs/infrastructure/gcp/gcp-sgp-deployment"},{"title":"GCP Architecture Reference","href":"/docs/infrastructure/gcp/gcp-architecture-reference"}]},{"title":"System Manager","href":"/docs/infrastructure/system-manager"}]},{"group":"Evaluations","pages":[{"title":"Introduction to Next Gen Evals","href":"/docs/v5/next-gen-evaluation/overview"},{"group":"Task Types","pages":[{"title":"Auto Evaluation","href":"/docs/v5/next-gen-evaluation/tasks/auto-evaluation"},{"title":"Agent Evaluation","href":"/docs/v5/next-gen-evaluation/tasks/agent-evaluation"},{"title":"Agentex Evaluation","description":"Generate outputs using external Agentex agents.","href":"/docs/v5/next-gen-evaluation/tasks/agentex-evaluation"},{"title":"Contributor Evaluation","href":"/docs/v5/next-gen-evaluation/tasks/contributor-evaluation"},{"title":"Custom Function Evaluation","description":"Define your own Python scoring functions and run them as evaluation tasks.","href":"/docs/v5/next-gen-evaluation/tasks/custom-function"}]},{"group":"Evaluations in the UI","pages":[{"title":"LLM as a Judge","href":"/docs/v5/next-gen-evaluation/ui-evaluations/auto-evaluation"},{"title":"Agent as a Judge","href":"/docs/v5/next-gen-evaluation/ui-evaluations/agent-evaluation"},{"title":"Agentex Agent Output","href":"/docs/v5/next-gen-evaluation/ui-evaluations/agentex-evaluation"},{"title":"Contributor Evaluation","href":"/docs/v5/next-gen-evaluation/ui-evaluations/contributor-evaluation"}]},{"title":"Conditional Rubrics","href":"/docs/v5/next-gen-evaluation/conditional-rubrics"},{"title":"Voice Evaluations","href":"/docs/v5/next-gen-evaluation/voice-evaluations"},{"group":"Completing Contributor Evaluations","pages":[{"title":"Managing Contributor Evaluations","href":"/docs/v5/next-gen-evaluation/labeling-evaluations/managing-contributor-evaluations"},{"title":"Labeling Evaluations","href":"/docs/v5/next-gen-evaluation/labeling-evaluations/labeling-evaluations"},{"title":"Auditing Evaluations","href":"/docs/v5/next-gen-evaluation/labeling-evaluations/auditing-evaluations"},{"title":"Contributor and Auditor Metrics","href":"/docs/v5/next-gen-evaluation/labeling-evaluations/contributor-auditor-metrics"}]}]},{"group":"Evaluation Dashboards","pages":[{"title":"Introduction to Evaluation Dashboards","href":"/docs/v5/evaluation-dashboards/overview"},{"title":"Getting Started with Dashboards","href":"/docs/v5/evaluation-dashboards/getting-started"},{"group":"Evaluation Groups","pages":[{"title":"Evaluation Group Dashboards","href":"/docs/v5/evaluation-dashboards/eval-group-dashboards"},{"title":"Latest Snapshots","href":"/docs/v5/evaluation-dashboards/eval-group-latest-snapshots"}]},{"group":"Widget Types","pages":[{"title":"Widget Types Overview","href":"/docs/v5/evaluation-dashboards/widget-types/overview"},{"title":"Metric Widget","href":"/docs/v5/evaluation-dashboards/widget-types/metric"},{"title":"Table Widget","href":"/docs/v5/evaluation-dashboards/widget-types/table"},{"title":"Bar Chart Widget","href":"/docs/v5/evaluation-dashboards/widget-types/bar"},{"title":"Histogram Widget","href":"/docs/v5/evaluation-dashboards/widget-types/histogram"},{"title":"Donut Chart Widget","href":"/docs/v5/evaluation-dashboards/widget-types/donut"},{"title":"Scatter Plot Widget","href":"/docs/v5/evaluation-dashboards/widget-types/scatter"},{"title":"Timeseries Widget","href":"/docs/v5/evaluation-dashboards/widget-types/timeseries"},{"title":"Markdown Widget","href":"/docs/v5/evaluation-dashboards/widget-types/markdown"},{"title":"Heading Widget","href":"/docs/v5/evaluation-dashboards/widget-types/heading"}]},{"title":"Query Language Reference","href":"/docs/v5/evaluation-dashboards/query-language"}]},{"group":"Tracing","pages":[{"title":"Introduction to Tracing","href":"/docs/v5/tracing/overview"},{"group":"Tracing SDK (Python)","pages":[{"title":"Initialization","href":"/docs/v5/tracing/sdk/initialization"},{"title":"Creating Traces and Spans","href":"/docs/v5/tracing/sdk/creating-traces-spans"},{"title":"Helper Methods","href":"/docs/v5/tracing/sdk/helper-methods"},{"title":"Flushing Tracing Data","href":"/docs/v5/tracing/sdk/flushing-data"},{"title":"Configuration Options","href":"/docs/v5/tracing/sdk/configuration-options"},{"title":"Distributed Tracing","href":"/docs/v5/tracing/sdk/multi-process"}]},{"group":"Integrations","pages":[{"group":"Agentex","pages":[{"title":"Agentex Tracing Overview","href":"/docs/v5/tracing/integrations/agentex/overview"},{"title":"Built-in Tracing Providers","href":"/docs/v5/tracing/integrations/agentex/built-in-providers"},{"title":"Span Hierarchy \u0026 Best Practices","href":"/docs/v5/tracing/integrations/agentex/span-hierarchy"},{"title":"Multi-Agent Tracing","href":"/docs/v5/tracing/integrations/agentex/multi-agent-tracing"}]},{"title":"OpenAI Agents Integration","href":"/docs/v5/tracing/integrations/openai-agents"},{"title":"OpenTelemetry Integration","href":"/docs/v5/tracing/integrations/opentelemetry"},{"title":"Temporal Integration","href":"/docs/v5/tracing/integrations/temporal"}]},{"group":"Tracing in the UI","pages":[{"title":"Traces","href":"/docs/v5/tracing/tracing-ui/traces-table"},{"title":"Profiler","href":"/docs/v5/tracing/tracing-ui/profiler"},{"title":"Trace Detail View","href":"/docs/v5/tracing/tracing-ui/trace-detail-view"}]}]},{"group":"Agents","pages":[{"title":"Introduction to Agents","href":"/docs/v5/agents/overview"},{"group":"Agentex","pages":[{"title":"Introduction to Agentex","href":"/docs/v5/agents/agentex/overview"},{"title":"Agentex CI/CD","href":"/docs/v5/agents/agentex/agentex-monitoring"},{"group":"Cloud Build and Deploy","pages":[{"title":"Cloud Build and Deploy","href":"/docs/v5/agents/agentex/cloud-build-and-deploy/overview"},{"title":"UI Walkthrough","href":"/docs/v5/agents/agentex/cloud-build-and-deploy/ui-walkthrough"},{"title":"CI/CD with Gitea Actions","href":"/docs/v5/agents/agentex/cloud-build-and-deploy/ci-cd"},{"title":"sgpctl CLI","href":"/docs/v5/agents/agentex/cloud-build-and-deploy/sgpctl"},{"title":"API and SDK","href":"/docs/v5/agents/agentex/cloud-build-and-deploy/api"},{"title":"Secrets","href":"/docs/v5/agents/agentex/cloud-build-and-deploy/secrets"},{"title":"Preview Deployments","href":"/docs/v5/agents/agentex/cloud-build-and-deploy/preview-deployments"}]},{"group":"Agentex Types","pages":[{"title":"Sync Agent","href":"/docs/v5/agents/agentex/types/sync-agent"},{"title":"Agentic Agent","href":"/docs/v5/agents/agentex/types/agentic-agent"}]}]},{"group":"Agent Service","pages":[{"title":"Introduction to Agent Service","href":"/docs/v5/agents/agent-service/overview"},{"group":"Lifecycle","pages":[{"title":"Creating an Agent","href":"/docs/v5/agents/agent-service/lifecycle/creating-agent-service-agents"},{"title":"Deploying an Agent","href":"/docs/v5/agents/agent-service/lifecycle/deploying-agent-service-agents"},{"title":"Monitoring an Agent","href":"/docs/v5/agents/agent-service/lifecycle/monitoring-agent-service-agents"}]}]}]}]},{"tab":"AgentOps API","pages":[{"title":"AgentOps API","href":"/reference/v5/inference/run-inference-with-free-form-payload"}]},{"tab":"AgentOps SDK","href":"https://scaleapi.github.io/sgp-sdk-docs/api/python/"},{"tab":"Agentex API","pages":[{"title":"Agentex API","href":"/reference/agentex/agents/get-agent-by-id"}]},{"tab":"Agentex SDK","href":"https://scaleapi.github.io/agentex-sdk-docs/api/python/"},{"tab":"Capabilities","pages":[{"title":"Capabilities","href":"/docs/capabilities/overview"}]},{"tab":"Dex API","pages":[{"title":"Dex API","href":"/reference/dex/projects/list-projects"}]}]},{"version":"V4 (legacy)","pages":[{"title":"V4 (legacy)","href":"/docs/how-to-run-an-evaluation"}]}]},"basePath":"","title":"Contributor Evaluation - Scale","deploymentId":"dpl_vSEmv961Ncyvz1psVwgUnTXQMSYk","buildId":"dpl_vSEmv961Ncyvz1psVwgUnTXQMSYk","dynamics":[],"chunks":[]}