1. Instantiate Client
1. Instantiate Client
Follow the instructions in the Quickstart Guide to setup the SGP Client
from scale_gp import SGPClient
client = SGPClient(api_key=api_key)
2. Create a dataset
2. Create a dataset
Create a dataset that we will manually upload test cases into
manual_evaluation_dataset = client.evaluation_datasets.create(
account_id=account_id,
name="manual_evaluation_dataset",
schema_type="GENERATION",
type="manual",
)
3. Define and upload test cases
3. Define and upload test cases
Define and batch upload desired test cases into the dataset
DATASET: List[TestCaseData]= [
{
"input": "What is the capital of France?",
"expected_output": "The capital of France is Paris.",
"expected_extra_info": {
"info": "Paris, located in the northern part of France along the Seine River, is not only the country’s capital but also its largest city. Known for its art, fashion, gastronomy, and culture, Paris has a significant influence both in France and globally.",
"schema_type": "STRING",
},
},
{
"input": "What color is an apple?",
"expected_output": "An apple is typically red when ripe, though green and yellow varieties also exist depending on the species and maturity.",
"expected_extra_info": {
"info": "Apples come in various colors including red, green, and yellow. Red apples are one of the most common, with varieties like Red Delicious being very popular. Green apples, like Granny Smith, are tart and used often in baking. Yellow apples such as Golden Delicious are sweet and softer.",
"schema_type": "STRING",
},
},
{
"input": "Who is the first president of the USA?",
"expected_output": "The first president of the USA is George Washington.",
"expected_extra_info": {
"info": "George Washington served as the first president of the United States from 1789 to 1797. He is a pivotal figure in American history, recognized for his leadership during the Revolutionary War and setting many precedents for the national government.",
"schema_type": "STRING",
},
},
]
items: List[Item] = [Item({"account_id": account_id, "test_case_data": item}) for item in DATASET]
test_cases: List[TestCase] = []
uploaded_test_cases = client.evaluation_datasets.test_cases.batch(
evaluation_dataset_id=manual_evaluation_dataset.id,
items=items,
)
test_cases.extend(uploaded_test_cases)
4. Publish the dataset
4. Publish the dataset
Publishing the dataset allows it to be available for use in evaluations
published_dataset_response = client.evaluation_datasets.publish(
evaluation_dataset_id=manual_evaluation_dataset.id,
)
import os
from typing import List
from scale_gp import SGPClient
from scale_gp.types.evaluation_datasets.test_case import TestCase
from scale_gp.types.evaluation_datasets.test_case_batch_params import Item
from scale_gp.types.evaluation_datasets.test_case_create_params import TestCaseData
client = SGPClient(api_key=api_key)
manual_evaluation_dataset = client.evaluation_datasets.create(
account_id=account_id,
name="manual_evaluation_dataset",
schema_type="GENERATION",
type="manual",
)
print(manual_evaluation_dataset)
DATASET: List[TestCaseData]= [
{
"input": "What is the capital of France?",
"expected_output": "The capital of France is Paris.",
"expected_extra_info": {
"info": "Paris, located in the northern part of France along the Seine River, is not only the country’s capital but also its largest city. Known for its art, fashion, gastronomy, and culture, Paris has a significant influence both in France and globally.",
"schema_type": "STRING",
},
},
{
"input": "What color is an apple?",
"expected_output": "An apple is typically red when ripe, though green and yellow varieties also exist depending on the species and maturity.",
"expected_extra_info": {
"info": "Apples come in various colors including red, green, and yellow. Red apples are one of the most common, with varieties like Red Delicious being very popular. Green apples, like Granny Smith, are tart and used often in baking. Yellow apples such as Golden Delicious are sweet and softer.",
"schema_type": "STRING",
},
},
{
"input": "Who is the first president of the USA?",
"expected_output": "The first president of the USA is George Washington.",
"expected_extra_info": {
"info": "George Washington served as the first president of the United States from 1789 to 1797. He is a pivotal figure in American history, recognized for his leadership during the Revolutionary War and setting many precedents for the national government.",
"schema_type": "STRING",
},
},
]
items: List[Item] = [Item({"account_id": account_id, "test_case_data": item}) for item in DATASET]
test_cases: List[TestCase] = []
uploaded_test_cases = client.evaluation_datasets.test_cases.batch(
evaluation_dataset_id=manual_evaluation_dataset.id,
items=items,
)
test_cases.extend(uploaded_test_cases)
print(uploaded_test_cases)
published_dataset_response = client.evaluation_datasets.publish(
evaluation_dataset_id=manual_evaluation_dataset.id,
)
print(published_dataset_response)
EvaluationDataset(
id='16a576f3-31f7-4f2d-b7bc-778e87c12e10',
account_id='66049ada2fc77c99ef015be7',
created_at=datetime.datetime(2024, 9, 26, 20, 4, 6, 600998),
created_by_user_id='42a5c8af-f698-43d0-923e-ba70102a2887',
draft=None,
name='manual_evaluation_dataset',
schema_type='GENERATION',
updated_at=datetime.datetime(2024, 9, 26, 20, 4, 6, 600998),
archived_at=None,
evaluation_dataset_metadata=None,
knowledge_base_id=None,
out_of_date=None,
vendor=None
)
[
GenerationTestCaseVersionResponse(
id='75449ec1-b908-4107-bd45-12218ef74805',
account_id='66049ada2fc77c99ef015be7',
autogenerated=False,
created_at=datetime.datetime(2024, 9, 26, 20, 4, 56, 410892),
created_by_user_id='42a5c8af-f698-43d0-923e-ba70102a2887',
evaluation_dataset_id='16a576f3-31f7-4f2d-b7bc-778e87c12e10',
test_case_data= GenerationTestCaseSchema(
input='What is the capital of France?',
expected_extra_info=StringExtraInfoSchema(info='Paris, located in the northern part of France along the Seine River, is not only the country’s capital but also its largest city. Known for its art, fashion, gastronomy, and culture, Paris has a significant influence both in France and globally.', schema_type='STRING'),
expected_output='The capital of France is Paris.'
),
archived_at=None,
chat_history=None,
schema_type='GENERATION',
test_case_metadata=None
),
GenerationTestCaseVersionResponse(
id='0d897e10-5dcf-4b9f-9cf3-d57d08cc2b67',
account_id='66049ada2fc77c99ef015be7',
autogenerated=False,
created_at=datetime.datetime(2024, 9, 26, 20, 4, 56, 410892),
created_by_user_id='42a5c8af-f698-43d0-923e-ba70102a2887',
evaluation_dataset_id='16a576f3-31f7-4f2d-b7bc-778e87c12e10',
test_case_data=GenerationTestCaseSchema(
input='What color is an apple?',
expected_extra_info=StringExtraInfoSchema(info='Apples come in various colors including red, green, and yellow. Red apples are one of the most common, with varieties like Red Delicious being very popular. Green apples, like Granny Smith, are tart and used often in baking. Yellow apples such as Golden Delicious are sweet and softer.', schema_type='STRING'),
expected_output='An apple is typically red when ripe, though green and yellow varieties also exist depending on the species and maturity.'
),
archived_at=None,
chat_history=None,
schema_type='GENERATION',
test_case_metadata=None
),
GenerationTestCaseVersionResponse(
id='dcf17ad8-68c0-45e8-a564-b4f7b576eed4'
account_id='66049ada2fc77c99ef015be7',
autogenerated=False,
created_at=datetime.datetime(2024, 9, 26, 20, 4, 56, 410892),
created_by_user_id='42a5c8af-f698-43d0-923e-ba70102a2887',
evaluation_dataset_id='16a576f3-31f7-4f2d-b7bc-778e87c12e10',
test_case_data=GenerationTestCaseSchema(
input='Who is the first president of the USA?',
expected_extra_info=StringExtraInfoSchema(info='George Washington served as the first president of the United States from 1789 to 1797. He is a pivotal figure in American history, recognized for his leadership during the Revolutionary War and setting many precedents for the national government.', schema_type='STRING'),
expected_output='The first president of the USA is George Washington.'
),
archived_at=None,
chat_history=None,
schema_type='GENERATION',
test_case_metadata=None
)
]
PublishEvaluationDatasetDraftResponse(
autogenerated_draft_test_cases=[],
success=True
)