Python
import os
from scale_gp import SGPClient
client = SGPClient(
api_key=os.environ.get("SGP_API_KEY"), # This is the default and can be omitted
)
evaluation = client.evaluations.create(
account_id="account_id",
application_spec_id="application_spec_id",
application_variant_id="application_variant_id",
description="description",
evaluation_dataset_id="evaluation_dataset_id",
name="name",
)
print(evaluation.id)package main
import (
"context"
"fmt"
"github.com/stainless-sdks/sgp-go"
"github.com/stainless-sdks/sgp-go/option"
)
func main() {
client := sgp.NewClient(
option.WithAPIKey("My API Key"),
)
evaluation, err := client.Evaluations.New(context.TODO(), sgp.EvaluationNewParams{
Body: sgp.EvaluationNewParamsBodyEvaluationBuilderRequest{
AccountID: sgp.F("account_id"),
ApplicationSpecID: sgp.F("application_spec_id"),
ApplicationVariantID: sgp.F("application_variant_id"),
Description: sgp.F("description"),
EvaluationDatasetID: sgp.F("evaluation_dataset_id"),
Name: sgp.F("name"),
Type: sgp.F(sgp.EvaluationNewParamsBodyEvaluationBuilderRequestTypeBuilder),
},
})
if err != nil {
panic(err.Error())
}
fmt.Printf("%+v\n", evaluation.ID)
}curl --request POST \
--url https://api.egp.scale.com/v4/evaluations \
--header 'Content-Type: application/json' \
--header 'x-api-key: <api-key>' \
--data '
{
"name": "<string>",
"description": "<string>",
"application_spec_id": "<string>",
"application_variant_id": "<string>",
"account_id": "<string>",
"evaluation_dataset_id": "<string>",
"tags": {},
"evaluation_config": {},
"evaluation_config_id": "<string>",
"question_id_to_annotation_config": {},
"annotation_config": {
"components": [
[
{
"data_loc": [
"<string>"
],
"optional": false,
"label": "<string>"
}
]
],
"direction": "row"
},
"type": "builder",
"evaluation_dataset_version": 123,
"application_test_case_output_group_id": "<string>",
"inline_evaluation_config": {
"question_set_id": "<string>",
"account_id": "<string>",
"evaluation_type": "llm_auto",
"studio_project_id": "<string>",
"auto_evaluation_model": "gpt-4-turbo-2024-04-09",
"auto_evaluation_parameters": {
"temperature": 1,
"batch_size": 13
}
}
}
'const options = {
method: 'POST',
headers: {'x-api-key': '<api-key>', 'Content-Type': 'application/json'},
body: JSON.stringify({
name: '<string>',
description: '<string>',
application_spec_id: '<string>',
application_variant_id: '<string>',
account_id: '<string>',
evaluation_dataset_id: '<string>',
tags: {},
evaluation_config: {},
evaluation_config_id: '<string>',
question_id_to_annotation_config: {},
annotation_config: {
components: [[{data_loc: ['<string>'], optional: false, label: '<string>'}]],
direction: 'row'
},
type: 'builder',
evaluation_dataset_version: 123,
application_test_case_output_group_id: '<string>',
inline_evaluation_config: {
question_set_id: '<string>',
account_id: '<string>',
evaluation_type: 'llm_auto',
studio_project_id: '<string>',
auto_evaluation_model: 'gpt-4-turbo-2024-04-09',
auto_evaluation_parameters: {temperature: 1, batch_size: 13}
}
})
};
fetch('https://api.egp.scale.com/v4/evaluations', options)
.then(res => res.json())
.then(res => console.log(res))
.catch(err => console.error(err));<?php
$curl = curl_init();
curl_setopt_array($curl, [
CURLOPT_URL => "https://api.egp.scale.com/v4/evaluations",
CURLOPT_RETURNTRANSFER => true,
CURLOPT_ENCODING => "",
CURLOPT_MAXREDIRS => 10,
CURLOPT_TIMEOUT => 30,
CURLOPT_HTTP_VERSION => CURL_HTTP_VERSION_1_1,
CURLOPT_CUSTOMREQUEST => "POST",
CURLOPT_POSTFIELDS => json_encode([
'name' => '<string>',
'description' => '<string>',
'application_spec_id' => '<string>',
'application_variant_id' => '<string>',
'account_id' => '<string>',
'evaluation_dataset_id' => '<string>',
'tags' => [
],
'evaluation_config' => [
],
'evaluation_config_id' => '<string>',
'question_id_to_annotation_config' => [
],
'annotation_config' => [
'components' => [
[
[
'data_loc' => [
'<string>'
],
'optional' => false,
'label' => '<string>'
]
]
],
'direction' => 'row'
],
'type' => 'builder',
'evaluation_dataset_version' => 123,
'application_test_case_output_group_id' => '<string>',
'inline_evaluation_config' => [
'question_set_id' => '<string>',
'account_id' => '<string>',
'evaluation_type' => 'llm_auto',
'studio_project_id' => '<string>',
'auto_evaluation_model' => 'gpt-4-turbo-2024-04-09',
'auto_evaluation_parameters' => [
'temperature' => 1,
'batch_size' => 13
]
]
]),
CURLOPT_HTTPHEADER => [
"Content-Type: application/json",
"x-api-key: <api-key>"
],
]);
$response = curl_exec($curl);
$err = curl_error($curl);
curl_close($curl);
if ($err) {
echo "cURL Error #:" . $err;
} else {
echo $response;
}HttpResponse<String> response = Unirest.post("https://api.egp.scale.com/v4/evaluations")
.header("x-api-key", "<api-key>")
.header("Content-Type", "application/json")
.body("{\n \"name\": \"<string>\",\n \"description\": \"<string>\",\n \"application_spec_id\": \"<string>\",\n \"application_variant_id\": \"<string>\",\n \"account_id\": \"<string>\",\n \"evaluation_dataset_id\": \"<string>\",\n \"tags\": {},\n \"evaluation_config\": {},\n \"evaluation_config_id\": \"<string>\",\n \"question_id_to_annotation_config\": {},\n \"annotation_config\": {\n \"components\": [\n [\n {\n \"data_loc\": [\n \"<string>\"\n ],\n \"optional\": false,\n \"label\": \"<string>\"\n }\n ]\n ],\n \"direction\": \"row\"\n },\n \"type\": \"builder\",\n \"evaluation_dataset_version\": 123,\n \"application_test_case_output_group_id\": \"<string>\",\n \"inline_evaluation_config\": {\n \"question_set_id\": \"<string>\",\n \"account_id\": \"<string>\",\n \"evaluation_type\": \"llm_auto\",\n \"studio_project_id\": \"<string>\",\n \"auto_evaluation_model\": \"gpt-4-turbo-2024-04-09\",\n \"auto_evaluation_parameters\": {\n \"temperature\": 1,\n \"batch_size\": 13\n }\n }\n}")
.asString();require 'uri'
require 'net/http'
url = URI("https://api.egp.scale.com/v4/evaluations")
http = Net::HTTP.new(url.host, url.port)
http.use_ssl = true
request = Net::HTTP::Post.new(url)
request["x-api-key"] = '<api-key>'
request["Content-Type"] = 'application/json'
request.body = "{\n \"name\": \"<string>\",\n \"description\": \"<string>\",\n \"application_spec_id\": \"<string>\",\n \"application_variant_id\": \"<string>\",\n \"account_id\": \"<string>\",\n \"evaluation_dataset_id\": \"<string>\",\n \"tags\": {},\n \"evaluation_config\": {},\n \"evaluation_config_id\": \"<string>\",\n \"question_id_to_annotation_config\": {},\n \"annotation_config\": {\n \"components\": [\n [\n {\n \"data_loc\": [\n \"<string>\"\n ],\n \"optional\": false,\n \"label\": \"<string>\"\n }\n ]\n ],\n \"direction\": \"row\"\n },\n \"type\": \"builder\",\n \"evaluation_dataset_version\": 123,\n \"application_test_case_output_group_id\": \"<string>\",\n \"inline_evaluation_config\": {\n \"question_set_id\": \"<string>\",\n \"account_id\": \"<string>\",\n \"evaluation_type\": \"llm_auto\",\n \"studio_project_id\": \"<string>\",\n \"auto_evaluation_model\": \"gpt-4-turbo-2024-04-09\",\n \"auto_evaluation_parameters\": {\n \"temperature\": 1,\n \"batch_size\": 13\n }\n }\n}"
response = http.request(request)
puts response.read_body{
"name": "<string>",
"description": "<string>",
"application_spec_id": "<string>",
"total_test_case_result_count": 123,
"completed_test_case_result_count": 123,
"id": "<string>",
"created_at": "2023-11-07T05:31:56Z",
"account_id": "<string>",
"created_by_user_id": "<string>",
"application_variant_id": "<string>",
"tags": {},
"evaluation_config": {},
"evaluation_config_id": "<string>",
"completed_at": "2023-11-07T05:31:56Z",
"annotation_config": {
"annotation_config_type": "flexible",
"components": [
[
{
"data_loc": [
"<string>"
],
"optional": false,
"label": "<string>"
}
]
],
"direction": "row",
"llm_prompt": {
"variables": [
{
"name": "<string>",
"data_loc": [
"<string>"
],
"optional": false
}
],
"template": "<string>"
}
},
"question_id_to_annotation_config": {},
"metric_config": {
"components": [
{
"name": "<string>",
"mappings": {},
"params": {}
}
]
},
"archived_at": "2023-11-07T05:31:56Z"
}{
"detail": [
{
"loc": [
"<string>"
],
"msg": "<string>",
"type": "<string>",
"input": "<unknown>",
"ctx": {}
}
]
}Evaluations
Create Evaluation
Description
Creates a evaluation
Details
This API can be used to create a evaluation. To use this API, review the request schema and pass in all fields that are required to create a evaluation.
POST
/
v4
/
evaluations
Python
import os
from scale_gp import SGPClient
client = SGPClient(
api_key=os.environ.get("SGP_API_KEY"), # This is the default and can be omitted
)
evaluation = client.evaluations.create(
account_id="account_id",
application_spec_id="application_spec_id",
application_variant_id="application_variant_id",
description="description",
evaluation_dataset_id="evaluation_dataset_id",
name="name",
)
print(evaluation.id)package main
import (
"context"
"fmt"
"github.com/stainless-sdks/sgp-go"
"github.com/stainless-sdks/sgp-go/option"
)
func main() {
client := sgp.NewClient(
option.WithAPIKey("My API Key"),
)
evaluation, err := client.Evaluations.New(context.TODO(), sgp.EvaluationNewParams{
Body: sgp.EvaluationNewParamsBodyEvaluationBuilderRequest{
AccountID: sgp.F("account_id"),
ApplicationSpecID: sgp.F("application_spec_id"),
ApplicationVariantID: sgp.F("application_variant_id"),
Description: sgp.F("description"),
EvaluationDatasetID: sgp.F("evaluation_dataset_id"),
Name: sgp.F("name"),
Type: sgp.F(sgp.EvaluationNewParamsBodyEvaluationBuilderRequestTypeBuilder),
},
})
if err != nil {
panic(err.Error())
}
fmt.Printf("%+v\n", evaluation.ID)
}curl --request POST \
--url https://api.egp.scale.com/v4/evaluations \
--header 'Content-Type: application/json' \
--header 'x-api-key: <api-key>' \
--data '
{
"name": "<string>",
"description": "<string>",
"application_spec_id": "<string>",
"application_variant_id": "<string>",
"account_id": "<string>",
"evaluation_dataset_id": "<string>",
"tags": {},
"evaluation_config": {},
"evaluation_config_id": "<string>",
"question_id_to_annotation_config": {},
"annotation_config": {
"components": [
[
{
"data_loc": [
"<string>"
],
"optional": false,
"label": "<string>"
}
]
],
"direction": "row"
},
"type": "builder",
"evaluation_dataset_version": 123,
"application_test_case_output_group_id": "<string>",
"inline_evaluation_config": {
"question_set_id": "<string>",
"account_id": "<string>",
"evaluation_type": "llm_auto",
"studio_project_id": "<string>",
"auto_evaluation_model": "gpt-4-turbo-2024-04-09",
"auto_evaluation_parameters": {
"temperature": 1,
"batch_size": 13
}
}
}
'const options = {
method: 'POST',
headers: {'x-api-key': '<api-key>', 'Content-Type': 'application/json'},
body: JSON.stringify({
name: '<string>',
description: '<string>',
application_spec_id: '<string>',
application_variant_id: '<string>',
account_id: '<string>',
evaluation_dataset_id: '<string>',
tags: {},
evaluation_config: {},
evaluation_config_id: '<string>',
question_id_to_annotation_config: {},
annotation_config: {
components: [[{data_loc: ['<string>'], optional: false, label: '<string>'}]],
direction: 'row'
},
type: 'builder',
evaluation_dataset_version: 123,
application_test_case_output_group_id: '<string>',
inline_evaluation_config: {
question_set_id: '<string>',
account_id: '<string>',
evaluation_type: 'llm_auto',
studio_project_id: '<string>',
auto_evaluation_model: 'gpt-4-turbo-2024-04-09',
auto_evaluation_parameters: {temperature: 1, batch_size: 13}
}
})
};
fetch('https://api.egp.scale.com/v4/evaluations', options)
.then(res => res.json())
.then(res => console.log(res))
.catch(err => console.error(err));<?php
$curl = curl_init();
curl_setopt_array($curl, [
CURLOPT_URL => "https://api.egp.scale.com/v4/evaluations",
CURLOPT_RETURNTRANSFER => true,
CURLOPT_ENCODING => "",
CURLOPT_MAXREDIRS => 10,
CURLOPT_TIMEOUT => 30,
CURLOPT_HTTP_VERSION => CURL_HTTP_VERSION_1_1,
CURLOPT_CUSTOMREQUEST => "POST",
CURLOPT_POSTFIELDS => json_encode([
'name' => '<string>',
'description' => '<string>',
'application_spec_id' => '<string>',
'application_variant_id' => '<string>',
'account_id' => '<string>',
'evaluation_dataset_id' => '<string>',
'tags' => [
],
'evaluation_config' => [
],
'evaluation_config_id' => '<string>',
'question_id_to_annotation_config' => [
],
'annotation_config' => [
'components' => [
[
[
'data_loc' => [
'<string>'
],
'optional' => false,
'label' => '<string>'
]
]
],
'direction' => 'row'
],
'type' => 'builder',
'evaluation_dataset_version' => 123,
'application_test_case_output_group_id' => '<string>',
'inline_evaluation_config' => [
'question_set_id' => '<string>',
'account_id' => '<string>',
'evaluation_type' => 'llm_auto',
'studio_project_id' => '<string>',
'auto_evaluation_model' => 'gpt-4-turbo-2024-04-09',
'auto_evaluation_parameters' => [
'temperature' => 1,
'batch_size' => 13
]
]
]),
CURLOPT_HTTPHEADER => [
"Content-Type: application/json",
"x-api-key: <api-key>"
],
]);
$response = curl_exec($curl);
$err = curl_error($curl);
curl_close($curl);
if ($err) {
echo "cURL Error #:" . $err;
} else {
echo $response;
}HttpResponse<String> response = Unirest.post("https://api.egp.scale.com/v4/evaluations")
.header("x-api-key", "<api-key>")
.header("Content-Type", "application/json")
.body("{\n \"name\": \"<string>\",\n \"description\": \"<string>\",\n \"application_spec_id\": \"<string>\",\n \"application_variant_id\": \"<string>\",\n \"account_id\": \"<string>\",\n \"evaluation_dataset_id\": \"<string>\",\n \"tags\": {},\n \"evaluation_config\": {},\n \"evaluation_config_id\": \"<string>\",\n \"question_id_to_annotation_config\": {},\n \"annotation_config\": {\n \"components\": [\n [\n {\n \"data_loc\": [\n \"<string>\"\n ],\n \"optional\": false,\n \"label\": \"<string>\"\n }\n ]\n ],\n \"direction\": \"row\"\n },\n \"type\": \"builder\",\n \"evaluation_dataset_version\": 123,\n \"application_test_case_output_group_id\": \"<string>\",\n \"inline_evaluation_config\": {\n \"question_set_id\": \"<string>\",\n \"account_id\": \"<string>\",\n \"evaluation_type\": \"llm_auto\",\n \"studio_project_id\": \"<string>\",\n \"auto_evaluation_model\": \"gpt-4-turbo-2024-04-09\",\n \"auto_evaluation_parameters\": {\n \"temperature\": 1,\n \"batch_size\": 13\n }\n }\n}")
.asString();require 'uri'
require 'net/http'
url = URI("https://api.egp.scale.com/v4/evaluations")
http = Net::HTTP.new(url.host, url.port)
http.use_ssl = true
request = Net::HTTP::Post.new(url)
request["x-api-key"] = '<api-key>'
request["Content-Type"] = 'application/json'
request.body = "{\n \"name\": \"<string>\",\n \"description\": \"<string>\",\n \"application_spec_id\": \"<string>\",\n \"application_variant_id\": \"<string>\",\n \"account_id\": \"<string>\",\n \"evaluation_dataset_id\": \"<string>\",\n \"tags\": {},\n \"evaluation_config\": {},\n \"evaluation_config_id\": \"<string>\",\n \"question_id_to_annotation_config\": {},\n \"annotation_config\": {\n \"components\": [\n [\n {\n \"data_loc\": [\n \"<string>\"\n ],\n \"optional\": false,\n \"label\": \"<string>\"\n }\n ]\n ],\n \"direction\": \"row\"\n },\n \"type\": \"builder\",\n \"evaluation_dataset_version\": 123,\n \"application_test_case_output_group_id\": \"<string>\",\n \"inline_evaluation_config\": {\n \"question_set_id\": \"<string>\",\n \"account_id\": \"<string>\",\n \"evaluation_type\": \"llm_auto\",\n \"studio_project_id\": \"<string>\",\n \"auto_evaluation_model\": \"gpt-4-turbo-2024-04-09\",\n \"auto_evaluation_parameters\": {\n \"temperature\": 1,\n \"batch_size\": 13\n }\n }\n}"
response = http.request(request)
puts response.read_body{
"name": "<string>",
"description": "<string>",
"application_spec_id": "<string>",
"total_test_case_result_count": 123,
"completed_test_case_result_count": 123,
"id": "<string>",
"created_at": "2023-11-07T05:31:56Z",
"account_id": "<string>",
"created_by_user_id": "<string>",
"application_variant_id": "<string>",
"tags": {},
"evaluation_config": {},
"evaluation_config_id": "<string>",
"completed_at": "2023-11-07T05:31:56Z",
"annotation_config": {
"annotation_config_type": "flexible",
"components": [
[
{
"data_loc": [
"<string>"
],
"optional": false,
"label": "<string>"
}
]
],
"direction": "row",
"llm_prompt": {
"variables": [
{
"name": "<string>",
"data_loc": [
"<string>"
],
"optional": false
}
],
"template": "<string>"
}
},
"question_id_to_annotation_config": {},
"metric_config": {
"components": [
{
"name": "<string>",
"mappings": {},
"params": {}
}
]
},
"archived_at": "2023-11-07T05:31:56Z"
}{
"detail": [
{
"loc": [
"<string>"
],
"msg": "<string>",
"type": "<string>",
"input": "<unknown>",
"ctx": {}
}
]
}Authorizations
Body
application/json
- EvaluationBuilderRequest
- DefaultEvaluationRequest
The ID of the account that owns the given entity.
The ID of the associated evaluation config.
Specifies the annotation configuration to use for specific questions.
Show child attributes
Show child attributes
Specifies the config for the metrics to be computed.
Show child attributes
Show child attributes
Annotation configuration for tasking
- AnnotationConfigRequestBase
- AnnotationConfigGenerationRequest
- AnnotationConfigMultiturnUseCaseRequest
- AnnotationConfigSummarizationUseCaseRequest
- AnnotationConfigTranslationUseCaseRequest
Show child attributes
Show child attributes
create standalone evaluation or build evaluation which will auto generate test case results
Allowed value:
"builder"Inline evaluation config data to create atomically with the evaluation. Provide this OR evaluation_config_id, not both.
- AutoEvalEvaluationConfigRequest
- ManualEvaluationConfigRequest
Show child attributes
Show child attributes
Response
Successful Response
Available options:
PENDING, COMPLETED, FAILED The total number of test case results for the evaluation
The number of test case results that have been completed for the evaluation
The unique identifier of the entity.
The date and time when the entity was created in ISO format.
The ID of the account that owns the given entity.
The user who originally created the entity.
The type of identity that created the entity.
Available options:
user, service_account The ID of the associated evaluation config.
The date and time that all test case results for the evaluation were completed for the evaluation in ISO format.
Annotation configuration for tasking
Show child attributes
Show child attributes
Specifies the annotation configuration to use for specific questions.
Show child attributes
Show child attributes
Specifies the config for the metrics to be computed.
Show child attributes
Show child attributes
The date and time when the entity was archived in ISO format.
⌘I