Add Test Criteria to Evaluation
curl --request POST \
--url https://api.egp.scale.com/v5/evaluations/{evaluation_id}/tasks \
--header 'Content-Type: application/json' \
--header 'x-api-key: <api-key>' \
--data '
{
"task": {
"configuration": {
"model": "<string>",
"messages": [
{}
],
"top_k": 123,
"frequency_penalty": 123,
"function_call": {},
"functions": [
{}
],
"logit_bias": {},
"logprobs": true,
"max_completion_tokens": 123,
"max_tokens": 123,
"metadata": {},
"modalities": [
"<string>"
],
"n": 123,
"parallel_tool_calls": true,
"prediction": {},
"presence_penalty": 123,
"reasoning_effort": "<string>",
"response_format": {},
"seed": 123,
"stop": "<string>",
"store": true,
"temperature": 123,
"tool_choice": "<string>",
"tools": [
{}
],
"top_logprobs": 123,
"top_p": 123,
"audio": {}
},
"task_type": "chat_completion",
"alias": "chat_completion"
}
}
'import requests
url = "https://api.egp.scale.com/v5/evaluations/{evaluation_id}/tasks"
payload = { "task": {
"configuration": {
"model": "<string>",
"messages": [{}],
"top_k": 123,
"frequency_penalty": 123,
"function_call": {},
"functions": [{}],
"logit_bias": {},
"logprobs": True,
"max_completion_tokens": 123,
"max_tokens": 123,
"metadata": {},
"modalities": ["<string>"],
"n": 123,
"parallel_tool_calls": True,
"prediction": {},
"presence_penalty": 123,
"reasoning_effort": "<string>",
"response_format": {},
"seed": 123,
"stop": "<string>",
"store": True,
"temperature": 123,
"tool_choice": "<string>",
"tools": [{}],
"top_logprobs": 123,
"top_p": 123,
"audio": {}
},
"task_type": "chat_completion",
"alias": "chat_completion"
} }
headers = {
"x-api-key": "<api-key>",
"Content-Type": "application/json"
}
response = requests.post(url, json=payload, headers=headers)
print(response.text)const options = {
method: 'POST',
headers: {'x-api-key': '<api-key>', 'Content-Type': 'application/json'},
body: JSON.stringify({
task: {
configuration: {
model: '<string>',
messages: [{}],
top_k: 123,
frequency_penalty: 123,
function_call: {},
functions: [{}],
logit_bias: {},
logprobs: true,
max_completion_tokens: 123,
max_tokens: 123,
metadata: {},
modalities: ['<string>'],
n: 123,
parallel_tool_calls: true,
prediction: {},
presence_penalty: 123,
reasoning_effort: '<string>',
response_format: {},
seed: 123,
stop: '<string>',
store: true,
temperature: 123,
tool_choice: '<string>',
tools: [{}],
top_logprobs: 123,
top_p: 123,
audio: {}
},
task_type: 'chat_completion',
alias: 'chat_completion'
}
})
};
fetch('https://api.egp.scale.com/v5/evaluations/{evaluation_id}/tasks', options)
.then(res => res.json())
.then(res => console.log(res))
.catch(err => console.error(err));<?php
$curl = curl_init();
curl_setopt_array($curl, [
CURLOPT_URL => "https://api.egp.scale.com/v5/evaluations/{evaluation_id}/tasks",
CURLOPT_RETURNTRANSFER => true,
CURLOPT_ENCODING => "",
CURLOPT_MAXREDIRS => 10,
CURLOPT_TIMEOUT => 30,
CURLOPT_HTTP_VERSION => CURL_HTTP_VERSION_1_1,
CURLOPT_CUSTOMREQUEST => "POST",
CURLOPT_POSTFIELDS => json_encode([
'task' => [
'configuration' => [
'model' => '<string>',
'messages' => [
[
]
],
'top_k' => 123,
'frequency_penalty' => 123,
'function_call' => [
],
'functions' => [
[
]
],
'logit_bias' => [
],
'logprobs' => true,
'max_completion_tokens' => 123,
'max_tokens' => 123,
'metadata' => [
],
'modalities' => [
'<string>'
],
'n' => 123,
'parallel_tool_calls' => true,
'prediction' => [
],
'presence_penalty' => 123,
'reasoning_effort' => '<string>',
'response_format' => [
],
'seed' => 123,
'stop' => '<string>',
'store' => true,
'temperature' => 123,
'tool_choice' => '<string>',
'tools' => [
[
]
],
'top_logprobs' => 123,
'top_p' => 123,
'audio' => [
]
],
'task_type' => 'chat_completion',
'alias' => 'chat_completion'
]
]),
CURLOPT_HTTPHEADER => [
"Content-Type: application/json",
"x-api-key: <api-key>"
],
]);
$response = curl_exec($curl);
$err = curl_error($curl);
curl_close($curl);
if ($err) {
echo "cURL Error #:" . $err;
} else {
echo $response;
}package main
import (
"fmt"
"strings"
"net/http"
"io"
)
func main() {
url := "https://api.egp.scale.com/v5/evaluations/{evaluation_id}/tasks"
payload := strings.NewReader("{\n \"task\": {\n \"configuration\": {\n \"model\": \"<string>\",\n \"messages\": [\n {}\n ],\n \"top_k\": 123,\n \"frequency_penalty\": 123,\n \"function_call\": {},\n \"functions\": [\n {}\n ],\n \"logit_bias\": {},\n \"logprobs\": true,\n \"max_completion_tokens\": 123,\n \"max_tokens\": 123,\n \"metadata\": {},\n \"modalities\": [\n \"<string>\"\n ],\n \"n\": 123,\n \"parallel_tool_calls\": true,\n \"prediction\": {},\n \"presence_penalty\": 123,\n \"reasoning_effort\": \"<string>\",\n \"response_format\": {},\n \"seed\": 123,\n \"stop\": \"<string>\",\n \"store\": true,\n \"temperature\": 123,\n \"tool_choice\": \"<string>\",\n \"tools\": [\n {}\n ],\n \"top_logprobs\": 123,\n \"top_p\": 123,\n \"audio\": {}\n },\n \"task_type\": \"chat_completion\",\n \"alias\": \"chat_completion\"\n }\n}")
req, _ := http.NewRequest("POST", url, payload)
req.Header.Add("x-api-key", "<api-key>")
req.Header.Add("Content-Type", "application/json")
res, _ := http.DefaultClient.Do(req)
defer res.Body.Close()
body, _ := io.ReadAll(res.Body)
fmt.Println(string(body))
}HttpResponse<String> response = Unirest.post("https://api.egp.scale.com/v5/evaluations/{evaluation_id}/tasks")
.header("x-api-key", "<api-key>")
.header("Content-Type", "application/json")
.body("{\n \"task\": {\n \"configuration\": {\n \"model\": \"<string>\",\n \"messages\": [\n {}\n ],\n \"top_k\": 123,\n \"frequency_penalty\": 123,\n \"function_call\": {},\n \"functions\": [\n {}\n ],\n \"logit_bias\": {},\n \"logprobs\": true,\n \"max_completion_tokens\": 123,\n \"max_tokens\": 123,\n \"metadata\": {},\n \"modalities\": [\n \"<string>\"\n ],\n \"n\": 123,\n \"parallel_tool_calls\": true,\n \"prediction\": {},\n \"presence_penalty\": 123,\n \"reasoning_effort\": \"<string>\",\n \"response_format\": {},\n \"seed\": 123,\n \"stop\": \"<string>\",\n \"store\": true,\n \"temperature\": 123,\n \"tool_choice\": \"<string>\",\n \"tools\": [\n {}\n ],\n \"top_logprobs\": 123,\n \"top_p\": 123,\n \"audio\": {}\n },\n \"task_type\": \"chat_completion\",\n \"alias\": \"chat_completion\"\n }\n}")
.asString();require 'uri'
require 'net/http'
url = URI("https://api.egp.scale.com/v5/evaluations/{evaluation_id}/tasks")
http = Net::HTTP.new(url.host, url.port)
http.use_ssl = true
request = Net::HTTP::Post.new(url)
request["x-api-key"] = '<api-key>'
request["Content-Type"] = 'application/json'
request.body = "{\n \"task\": {\n \"configuration\": {\n \"model\": \"<string>\",\n \"messages\": [\n {}\n ],\n \"top_k\": 123,\n \"frequency_penalty\": 123,\n \"function_call\": {},\n \"functions\": [\n {}\n ],\n \"logit_bias\": {},\n \"logprobs\": true,\n \"max_completion_tokens\": 123,\n \"max_tokens\": 123,\n \"metadata\": {},\n \"modalities\": [\n \"<string>\"\n ],\n \"n\": 123,\n \"parallel_tool_calls\": true,\n \"prediction\": {},\n \"presence_penalty\": 123,\n \"reasoning_effort\": \"<string>\",\n \"response_format\": {},\n \"seed\": 123,\n \"stop\": \"<string>\",\n \"store\": true,\n \"temperature\": 123,\n \"tool_choice\": \"<string>\",\n \"tools\": [\n {}\n ],\n \"top_logprobs\": 123,\n \"top_p\": 123,\n \"audio\": {}\n },\n \"task_type\": \"chat_completion\",\n \"alias\": \"chat_completion\"\n }\n}"
response = http.request(request)
puts response.read_body{
"id": "<string>",
"name": "<string>",
"created_at": "2023-11-07T05:31:56Z",
"created_by": {
"id": "<string>",
"object": "identity"
},
"tags": [
"<string>"
],
"datasets": [
{
"id": "<string>",
"created_by": {
"id": "<string>",
"object": "identity"
},
"name": "<string>",
"created_at": "2023-11-07T05:31:56Z",
"tags": [
"<string>"
],
"current_version_num": 123,
"object": "dataset",
"description": "<string>",
"archived_at": "2023-11-07T05:31:56Z"
}
],
"object": "evaluation",
"description": "<string>",
"archived_at": "2023-11-07T05:31:56Z",
"status_reason": "<string>",
"progress": {
"workflows": {
"total": 123,
"completed": 123,
"failed": 123,
"pending": 123
},
"items": {
"total": 123,
"successful": 123,
"failed": 123,
"pending": 123,
"failed_items": []
}
},
"tasks": [
{
"configuration": {
"model": "<string>",
"messages": [
{}
],
"top_k": 123,
"frequency_penalty": 123,
"function_call": {},
"functions": [
{}
],
"logit_bias": {},
"logprobs": true,
"max_completion_tokens": 123,
"max_tokens": 123,
"metadata": {},
"modalities": [
"<string>"
],
"n": 123,
"parallel_tool_calls": true,
"prediction": {},
"presence_penalty": 123,
"reasoning_effort": "<string>",
"response_format": {},
"seed": 123,
"stop": "<string>",
"store": true,
"temperature": 123,
"tool_choice": "<string>",
"tools": [
{}
],
"top_logprobs": 123,
"top_p": 123,
"audio": {}
},
"task_type": "chat_completion",
"alias": "chat_completion"
}
],
"metadata": {},
"error_count": 123
}{
"detail": [
{
"loc": [
"<string>"
],
"msg": "<string>",
"type": "<string>",
"input": "<unknown>",
"ctx": {}
}
]
}Evaluations
Add Test Criteria to Evaluation
Add a new test criteria to an existing evaluation.
Narrowed to contributor question tasks (contributor_evaluation.question); other task types
must be configured when the evaluation is first created and are rejected here. The request is
also rejected if the evaluation is archived, if a test criteria with the same alias already
exists, or if any contributor annotation task for the evaluation has already been claimed or
completed. Because only contributor question tasks are accepted, the added criteria is applied
synchronously and contributors answer it against the evaluation’s existing items — no async job
or Temporal workflow is started.
POST
/
v5
/
evaluations
/
{evaluation_id}
/
tasks
Add Test Criteria to Evaluation
curl --request POST \
--url https://api.egp.scale.com/v5/evaluations/{evaluation_id}/tasks \
--header 'Content-Type: application/json' \
--header 'x-api-key: <api-key>' \
--data '
{
"task": {
"configuration": {
"model": "<string>",
"messages": [
{}
],
"top_k": 123,
"frequency_penalty": 123,
"function_call": {},
"functions": [
{}
],
"logit_bias": {},
"logprobs": true,
"max_completion_tokens": 123,
"max_tokens": 123,
"metadata": {},
"modalities": [
"<string>"
],
"n": 123,
"parallel_tool_calls": true,
"prediction": {},
"presence_penalty": 123,
"reasoning_effort": "<string>",
"response_format": {},
"seed": 123,
"stop": "<string>",
"store": true,
"temperature": 123,
"tool_choice": "<string>",
"tools": [
{}
],
"top_logprobs": 123,
"top_p": 123,
"audio": {}
},
"task_type": "chat_completion",
"alias": "chat_completion"
}
}
'import requests
url = "https://api.egp.scale.com/v5/evaluations/{evaluation_id}/tasks"
payload = { "task": {
"configuration": {
"model": "<string>",
"messages": [{}],
"top_k": 123,
"frequency_penalty": 123,
"function_call": {},
"functions": [{}],
"logit_bias": {},
"logprobs": True,
"max_completion_tokens": 123,
"max_tokens": 123,
"metadata": {},
"modalities": ["<string>"],
"n": 123,
"parallel_tool_calls": True,
"prediction": {},
"presence_penalty": 123,
"reasoning_effort": "<string>",
"response_format": {},
"seed": 123,
"stop": "<string>",
"store": True,
"temperature": 123,
"tool_choice": "<string>",
"tools": [{}],
"top_logprobs": 123,
"top_p": 123,
"audio": {}
},
"task_type": "chat_completion",
"alias": "chat_completion"
} }
headers = {
"x-api-key": "<api-key>",
"Content-Type": "application/json"
}
response = requests.post(url, json=payload, headers=headers)
print(response.text)const options = {
method: 'POST',
headers: {'x-api-key': '<api-key>', 'Content-Type': 'application/json'},
body: JSON.stringify({
task: {
configuration: {
model: '<string>',
messages: [{}],
top_k: 123,
frequency_penalty: 123,
function_call: {},
functions: [{}],
logit_bias: {},
logprobs: true,
max_completion_tokens: 123,
max_tokens: 123,
metadata: {},
modalities: ['<string>'],
n: 123,
parallel_tool_calls: true,
prediction: {},
presence_penalty: 123,
reasoning_effort: '<string>',
response_format: {},
seed: 123,
stop: '<string>',
store: true,
temperature: 123,
tool_choice: '<string>',
tools: [{}],
top_logprobs: 123,
top_p: 123,
audio: {}
},
task_type: 'chat_completion',
alias: 'chat_completion'
}
})
};
fetch('https://api.egp.scale.com/v5/evaluations/{evaluation_id}/tasks', options)
.then(res => res.json())
.then(res => console.log(res))
.catch(err => console.error(err));<?php
$curl = curl_init();
curl_setopt_array($curl, [
CURLOPT_URL => "https://api.egp.scale.com/v5/evaluations/{evaluation_id}/tasks",
CURLOPT_RETURNTRANSFER => true,
CURLOPT_ENCODING => "",
CURLOPT_MAXREDIRS => 10,
CURLOPT_TIMEOUT => 30,
CURLOPT_HTTP_VERSION => CURL_HTTP_VERSION_1_1,
CURLOPT_CUSTOMREQUEST => "POST",
CURLOPT_POSTFIELDS => json_encode([
'task' => [
'configuration' => [
'model' => '<string>',
'messages' => [
[
]
],
'top_k' => 123,
'frequency_penalty' => 123,
'function_call' => [
],
'functions' => [
[
]
],
'logit_bias' => [
],
'logprobs' => true,
'max_completion_tokens' => 123,
'max_tokens' => 123,
'metadata' => [
],
'modalities' => [
'<string>'
],
'n' => 123,
'parallel_tool_calls' => true,
'prediction' => [
],
'presence_penalty' => 123,
'reasoning_effort' => '<string>',
'response_format' => [
],
'seed' => 123,
'stop' => '<string>',
'store' => true,
'temperature' => 123,
'tool_choice' => '<string>',
'tools' => [
[
]
],
'top_logprobs' => 123,
'top_p' => 123,
'audio' => [
]
],
'task_type' => 'chat_completion',
'alias' => 'chat_completion'
]
]),
CURLOPT_HTTPHEADER => [
"Content-Type: application/json",
"x-api-key: <api-key>"
],
]);
$response = curl_exec($curl);
$err = curl_error($curl);
curl_close($curl);
if ($err) {
echo "cURL Error #:" . $err;
} else {
echo $response;
}package main
import (
"fmt"
"strings"
"net/http"
"io"
)
func main() {
url := "https://api.egp.scale.com/v5/evaluations/{evaluation_id}/tasks"
payload := strings.NewReader("{\n \"task\": {\n \"configuration\": {\n \"model\": \"<string>\",\n \"messages\": [\n {}\n ],\n \"top_k\": 123,\n \"frequency_penalty\": 123,\n \"function_call\": {},\n \"functions\": [\n {}\n ],\n \"logit_bias\": {},\n \"logprobs\": true,\n \"max_completion_tokens\": 123,\n \"max_tokens\": 123,\n \"metadata\": {},\n \"modalities\": [\n \"<string>\"\n ],\n \"n\": 123,\n \"parallel_tool_calls\": true,\n \"prediction\": {},\n \"presence_penalty\": 123,\n \"reasoning_effort\": \"<string>\",\n \"response_format\": {},\n \"seed\": 123,\n \"stop\": \"<string>\",\n \"store\": true,\n \"temperature\": 123,\n \"tool_choice\": \"<string>\",\n \"tools\": [\n {}\n ],\n \"top_logprobs\": 123,\n \"top_p\": 123,\n \"audio\": {}\n },\n \"task_type\": \"chat_completion\",\n \"alias\": \"chat_completion\"\n }\n}")
req, _ := http.NewRequest("POST", url, payload)
req.Header.Add("x-api-key", "<api-key>")
req.Header.Add("Content-Type", "application/json")
res, _ := http.DefaultClient.Do(req)
defer res.Body.Close()
body, _ := io.ReadAll(res.Body)
fmt.Println(string(body))
}HttpResponse<String> response = Unirest.post("https://api.egp.scale.com/v5/evaluations/{evaluation_id}/tasks")
.header("x-api-key", "<api-key>")
.header("Content-Type", "application/json")
.body("{\n \"task\": {\n \"configuration\": {\n \"model\": \"<string>\",\n \"messages\": [\n {}\n ],\n \"top_k\": 123,\n \"frequency_penalty\": 123,\n \"function_call\": {},\n \"functions\": [\n {}\n ],\n \"logit_bias\": {},\n \"logprobs\": true,\n \"max_completion_tokens\": 123,\n \"max_tokens\": 123,\n \"metadata\": {},\n \"modalities\": [\n \"<string>\"\n ],\n \"n\": 123,\n \"parallel_tool_calls\": true,\n \"prediction\": {},\n \"presence_penalty\": 123,\n \"reasoning_effort\": \"<string>\",\n \"response_format\": {},\n \"seed\": 123,\n \"stop\": \"<string>\",\n \"store\": true,\n \"temperature\": 123,\n \"tool_choice\": \"<string>\",\n \"tools\": [\n {}\n ],\n \"top_logprobs\": 123,\n \"top_p\": 123,\n \"audio\": {}\n },\n \"task_type\": \"chat_completion\",\n \"alias\": \"chat_completion\"\n }\n}")
.asString();require 'uri'
require 'net/http'
url = URI("https://api.egp.scale.com/v5/evaluations/{evaluation_id}/tasks")
http = Net::HTTP.new(url.host, url.port)
http.use_ssl = true
request = Net::HTTP::Post.new(url)
request["x-api-key"] = '<api-key>'
request["Content-Type"] = 'application/json'
request.body = "{\n \"task\": {\n \"configuration\": {\n \"model\": \"<string>\",\n \"messages\": [\n {}\n ],\n \"top_k\": 123,\n \"frequency_penalty\": 123,\n \"function_call\": {},\n \"functions\": [\n {}\n ],\n \"logit_bias\": {},\n \"logprobs\": true,\n \"max_completion_tokens\": 123,\n \"max_tokens\": 123,\n \"metadata\": {},\n \"modalities\": [\n \"<string>\"\n ],\n \"n\": 123,\n \"parallel_tool_calls\": true,\n \"prediction\": {},\n \"presence_penalty\": 123,\n \"reasoning_effort\": \"<string>\",\n \"response_format\": {},\n \"seed\": 123,\n \"stop\": \"<string>\",\n \"store\": true,\n \"temperature\": 123,\n \"tool_choice\": \"<string>\",\n \"tools\": [\n {}\n ],\n \"top_logprobs\": 123,\n \"top_p\": 123,\n \"audio\": {}\n },\n \"task_type\": \"chat_completion\",\n \"alias\": \"chat_completion\"\n }\n}"
response = http.request(request)
puts response.read_body{
"id": "<string>",
"name": "<string>",
"created_at": "2023-11-07T05:31:56Z",
"created_by": {
"id": "<string>",
"object": "identity"
},
"tags": [
"<string>"
],
"datasets": [
{
"id": "<string>",
"created_by": {
"id": "<string>",
"object": "identity"
},
"name": "<string>",
"created_at": "2023-11-07T05:31:56Z",
"tags": [
"<string>"
],
"current_version_num": 123,
"object": "dataset",
"description": "<string>",
"archived_at": "2023-11-07T05:31:56Z"
}
],
"object": "evaluation",
"description": "<string>",
"archived_at": "2023-11-07T05:31:56Z",
"status_reason": "<string>",
"progress": {
"workflows": {
"total": 123,
"completed": 123,
"failed": 123,
"pending": 123
},
"items": {
"total": 123,
"successful": 123,
"failed": 123,
"pending": 123,
"failed_items": []
}
},
"tasks": [
{
"configuration": {
"model": "<string>",
"messages": [
{}
],
"top_k": 123,
"frequency_penalty": 123,
"function_call": {},
"functions": [
{}
],
"logit_bias": {},
"logprobs": true,
"max_completion_tokens": 123,
"max_tokens": 123,
"metadata": {},
"modalities": [
"<string>"
],
"n": 123,
"parallel_tool_calls": true,
"prediction": {},
"presence_penalty": 123,
"reasoning_effort": "<string>",
"response_format": {},
"seed": 123,
"stop": "<string>",
"store": true,
"temperature": 123,
"tool_choice": "<string>",
"tools": [
{}
],
"top_logprobs": 123,
"top_p": 123,
"audio": {}
},
"task_type": "chat_completion",
"alias": "chat_completion"
}
],
"metadata": {},
"error_count": 123
}{
"detail": [
{
"loc": [
"<string>"
],
"msg": "<string>",
"type": "<string>",
"input": "<unknown>",
"ctx": {}
}
]
}Authorizations
Headers
Path Parameters
Body
application/json
New test criteria to add to the evaluation. Rejected when contributor annotation tasks for this evaluation have already been claimed or completed. Triggers a rerun so the new task executes against existing items.
- ChatCompletionEvaluationTask
- GenericInferenceEvaluationTask
- ApplicationVariantV1EvaluationTask
- AgentexOutputEvaluationTask
- MetricEvaluationTask
- AutoEvaluationQuestionTask
- AutoEvaluationGuidedDecodingEvaluationTask
- AutoEvaluationAgentEvaluationTask
- ContributorEvaluationQuestionTask
- CustomFunctionEvaluationTask
Show child attributes
Show child attributes
Response
Successful Response
The identity that created the entity.
Show child attributes
Show child attributes
The tags associated with the entity
Show child attributes
Show child attributes
Available options:
failed, completed, running Allowed value:
"evaluation"Reason for evaluation status
Progress of the evaluation's underlying async job
Show child attributes
Show child attributes
tasks
(ChatCompletionEvaluationTask · object | GenericInferenceEvaluationTask · object | ApplicationVariantV1EvaluationTask · object | AgentexOutputEvaluationTask · object | MetricEvaluationTask · object | AutoEvaluationQuestionTask · object | AutoEvaluationGuidedDecodingEvaluationTask · object | AutoEvaluationAgentEvaluationTask · object | ContributorEvaluationQuestionTask · object | CustomFunctionEvaluationTask · object)[]
Tasks executed during evaluation. Populated with optional task view.
- ChatCompletionEvaluationTask
- GenericInferenceEvaluationTask
- ApplicationVariantV1EvaluationTask
- AgentexOutputEvaluationTask
- MetricEvaluationTask
- AutoEvaluationQuestionTask
- AutoEvaluationGuidedDecodingEvaluationTask
- AutoEvaluationAgentEvaluationTask
- ContributorEvaluationQuestionTask
- CustomFunctionEvaluationTask
Show child attributes
Show child attributes
Metadata key-value pairs for the evaluation
Number of task errors across all items in this evaluation.
⌘I