1. Instantiate Client
1. Instantiate Client
Follow the instructions in the Quickstart Guide to setup the SGP Client
from scale_gp import SGPClient
client = SGPClient(api_key=api_key)
2. Create autogenerated dataset
2. Create autogenerated dataset
For autogenerated evaluation datasets, a generation job workflow is created to generate test cases.
Test cases are generated based on the knowledge base provided, and must be approved before the dataset can be published.
Evaluation datasets, once published, can be used for application variant runs and report card generation
autogenerated_evaluation_dataset = client.evaluation_datasets.create(
account_id=account_id,
name="autogenerated_eval_dataset",
schema_type="GENERATION",
type="autogenerated",
knowledge_base_id=knowledge_base_id,
)
dataset = client.evaluation_datasets.retrieve(evaluation_dataset_id=autogenerated_evaluation_dataset.id)
3. Start generation job
3. Start generation job
Start the generation job. This job will generate test cases based on the chunks/data present inside the specified knowledge base.
In this example, we used a knowledge base with Legend of Zelda playthrough guides.
generation_job = client.evaluation_datasets.generation_jobs.create(
evaluation_dataset_id=autogenerated_evaluation_dataset.id,
num_test_cases=3,
group_by_artifact_id=False,
)
while True:
generation_job = client.evaluation_datasets.generation_jobs.retrieve(
generation_job_id=generation_job.generation_job_id,
evaluation_dataset_id=autogenerated_evaluation_dataset.id
)
if generation_job.status == "Pending":
print("generating test cases...")
time.sleep(5)
else:
break
# view autogenerated test cases
test_cases = client.evaluation_datasets.autogenerated_draft_test_cases.list(
evaluation_dataset_id=autogenerated_evaluation_dataset.id
)
4. Approve auto-generated test cases
4. Approve auto-generated test cases
Before publishing the dataset, review the auto-generated test cases and approve/decline each test case. Publishing is blocked until
all test cases are reviewed.
for test_case in test_cases.items:
client.evaluation_datasets.autogenerated_draft_test_cases.approve(
evaluation_dataset_id=autogenerated_evaluation_dataset.id,
autogenerated_draft_test_case_id=test_case.id,
)
5. Publish the dataset
5. Publish the dataset
Publishing the dataset allows it to be available for use in evaluations
published_dataset_response = client.evaluation_datasets.publish(
evaluation_dataset_id=autogenerated_evaluation_dataset.id,
)
import os
import time
from scale_gp import SGPClient
client = SGPClient(api_key=api_key)
autogenerated_evaluation_dataset = client.evaluation_datasets.create(
account_id=account_id,
name="autogenerated_eval_dataset",
schema_type="GENERATION",
type="autogenerated",
knowledge_base_id=knowledge_base_id,
)
print(autogenerated_evaluation_dataset)
generation_job = client.evaluation_datasets.generation_jobs.create(
evaluation_dataset_id=autogenerated_evaluation_dataset.id,
num_test_cases=3,
group_by_artifact_id=False,
)
while True:
generation_job = client.evaluation_datasets.generation_jobs.retrieve(
generation_job_id=generation_job.generation_job_id,
evaluation_dataset_id=autogenerated_evaluation_dataset.id
)
if generation_job.status == "Pending":
print("generating test cases...")
time.sleep(5)
else:
break
print(generation_job)
# view autogenerated test cases
test_cases = client.evaluation_datasets.autogenerated_draft_test_cases.list(
evaluation_dataset_id=autogenerated_evaluation_dataset.id
)
print(test_cases.items)
for test_case in test_cases.items:
client.evaluation_datasets.autogenerated_draft_test_cases.approve(
evaluation_dataset_id=autogenerated_evaluation_dataset.id,
autogenerated_draft_test_case_id=test_case.id,
)
published_dataset_response = client.evaluation_datasets.publish(
evaluation_dataset_id=autogenerated_evaluation_dataset.id,
)
print(published_dataset_response)
EvaluationDataset(
id='da20895d-1784-4d40-a403-e7b5f8a2ce78',
account_id='66049ada2fc77c99ef015be7',
created_at=datetime.datetime(2024, 9, 26, 20, 6, 46, 689131),
created_by_user_id='42a5c8af-f698-43d0-923e-ba70102a2887',
draft=None,
name='autogenerated_eval_dataset',
schema_type='GENERATION',
updated_at=datetime.datetime(2024, 9, 26, 20, 6, 46, 689131),
archived_at=None,
evaluation_dataset_metadata={
'knowledge_base_name': 'Zelda OOT Walkthroughs'
},
knowledge_base_id='78447679-4cbf-4c27-af40-d4f1ec9f6911',
out_of_date=None,
vendor=None
)
EvaluationDatasetGenerationJob(
created_at=datetime.datetime(2024, 9, 26, 20, 7, 48, 457437),
generation_job_id='e508c2f1-1b4b-4be9-b296-3c52c0df4375',
status='Running',
updated_at=datetime.datetime(2024, 9, 26, 20, 7, 49, 346074),
failure_reason=None,
num_completed_test_cases=None,
num_test_cases=3,
total_chunk_count=34214
)
SyncPageResponse[AutogeneratedDraftTestCaseListResponse](
items=[
AutogeneratedDraftTestCaseListResponse(
id='d8043540-5d50-46e3-a3b4-db1eb1103355',
account_id='66049ada2fc77c99ef015be7',
approved=False,
content_metadata={
'270af972-825c-4391-903c-c728aa55a2cc': {
'artifact_id': '19ddfc0c-2a23-400d-8aa4-2a8fd902e503',
'artifact_name': 'The Legend of Zelda: Ocarina of Time - Guide and Walkthrough - Nintendo 64 - By BakonBitz - GameFAQs.pdf',
'artifact_content_modification_identifier': None,
'chunk_text': ' target twice.'
},
'47483b14-818a-4f33-92eb-1ba47de36d99': {
'artifact_id': '19ddfc0c-2a23-400d-8aa4-2a8fd902e503',
'artifact_name': 'The Legend of Zelda: Ocarina of Time - Guide and Walkthrough - Nintendo 64 - By BakonBitz - GameFAQs.pdf',
'artifact_content_modification_identifier': None,
'chunk_text': '- As you turn around, aim at the line of targets and hit the center of each'
},
'4f3fb66e-9e22-4c0f-9cdd-c12a3a73e449': {
'artifact_id': '9624c5fb-f672-46c9-b64c-49d2c061d171',
'artifact_name': 'The Legend of Zelda: Ocarina of Time - Guide and Walkthrough - Nintendo 64 - By A_I_e_x - GameFAQs.pdf',
'artifact_content_modification_identifier': None,
'chunk_text': "Here's how it works, there are a number of targets and pots along the path."
},
'55312dff-5331-4f18-a0c6-69a0df85dc69': {
'artifact_id': '00003d33-7763-46ab-9005-9aaad947ba25',
'artifact_name': 'The Legend of Zelda: Ocarina of Time - Guide and Walkthrough - Nintendo 64 - By MrShotgun - GameFAQs.pdf',
'artifact_content_modification_identifier': None,
'chunk_text': 'down the middle. Absolutely never play anywhere except in the exact center of'
},
'621c6ba5-5b6f-495e-a789-f065795d47e4': {
'artifact_id': '00003d33-7763-46ab-9005-9aaad947ba25',
'artifact_name': 'The Legend of Zelda: Ocarina of Time - Guide and Walkthrough - Nintendo 64 - By MrShotgun - GameFAQs.pdf',
'artifact_content_modification_identifier': None,
'chunk_text': 'depending upon how close to the center you are (the exact middle of the target'
},
'81a0ccee-d062-4718-8d32-84d249a34ba1': {
'artifact_id': '9624c5fb-f672-46c9-b64c-49d2c061d171',
'artifact_name': 'The Legend of Zelda: Ocarina of Time - Guide and Walkthrough - Nintendo 64 - By A_I_e_x - GameFAQs.pdf',
'artifact_content_modification_identifier': None,
'chunk_text': " it's extremely difficult and not worth doing. Instead stand exactly where"
},
'b09478e9-487b-4caf-93ca-07255f4abb66': {
'artifact_id': '9624c5fb-f672-46c9-b64c-49d2c061d171',
'artifact_name': 'The Legend of Zelda: Ocarina of Time - Guide and Walkthrough - Nintendo 64 - By A_I_e_x - GameFAQs.pdf',
'artifact_content_modification_identifier': None,
'chunk_text': 'large target symbol and arrow through it, this place should not be too'
}
},
created_at=datetime.datetime(2024, 9, 26, 20, 8, 27, 396367),
created_by_user_id='42a5c8af-f698-43d0-923e-ba70102a2887',
evaluation_dataset_id='da20895d-1784-4d40-a403-e7b5f8a2ce78',
schema_type='GENERATION',
test_case_data=GenerationTestCaseSchema(
input='Where should you aim and stand to hit each target effectively?',
expected_extra_info=StringExtraInfoSchema(
info="Here's how it works, there are a number of targets and pots along the path.\n"
"CHUNK_SEPARATING_STRING\ndown the middle. Absolutely never play anywhere except in the exact center of\n"
"CHUNK_SEPARATING_STRING\n target twice.\n"
"CHUNK_SEPARATING_STRING\nlarge target symbol and arrow through it, this place should not be too\n"
"CHUNK_SEPARATING_STRING\ndepending upon how close to the center you are (the exact middle of the target\n"
"CHUNK_SEPARATING_STRING\n it's extremely difficult and not worth doing. Instead stand exactly where\n"
"CHUNK_SEPARATING_STRING\n- As you turn around, aim at the line of targets and hit the center of each",
schema_type='STRING'
),
expected_output="You should aim at the line of targets and hit the center of each target twice. Make sure to always play in the exact center of the field. Depending upon how close you are to the center, your success rate may vary. When there is a large target symbol with an arrow through it, aim to hit its center."
),
updated_at=datetime.datetime(2024, 9, 26, 20, 8, 27, 396367),
missing_chunks=[],
topic_str='targeting and aiming adjustments on a course'
),
AutogeneratedDraftTestCaseListResponse(
id='15bdc65d-5223-4ce0-948f-9cb6b3b24f1a',
account_id='66049ada2fc77c99ef015be7',
approved=False,
content_metadata={
'00d41d43-a857-4ee7-a7a4-4ad35249d11c': {
'artifact_id': '19ddfc0c-2a23-400d-8aa4-2a8fd902e503',
'artifact_name': 'The Legend of Zelda: Ocarina of Time - Guide and Walkthrough - Nintendo 64 - By BakonBitz - GameFAQs.pdf',
'artifact_content_modification_identifier': None,
'chunk_text': 'for it as well. To the left of the chest is a ladder. Climb it to reach'
}
},
created_at=datetime.datetime(2024, 9, 26, 20, 8, 14, 253887),
created_by_user_id='42a5c8af-f698-43d0-923e-ba70102a2887',
evaluation_dataset_id='da20895d-1784-4d40-a403-e7b5f8a2ce78',
schema_type='GENERATION',
test_case_data=ArtifactSchemaGeneration(
artifact_ids_filter=['19ddfc0c-2a23-400d-8aa4-2a8fd902e503'],
input='How would you proceed if you encounter a chest and want to climb further?',
expected_extra_info=StringExtraInfoSchema(
info='for it as well. To the left of the chest is a ladder. Climb it to reach',
schema_type='STRING'
),
expected_output='If you encounter a chest, look to the left of it where you will find a ladder. Climb the ladder to proceed further.'
),
updated_at=datetime.datetime(2024, 9, 26, 20, 8, 14, 253887),
missing_chunks=[],
topic_str='Climbing and Navigation in Structures'
),
AutogeneratedDraftTestCaseListResponse(
id='2d5d8282-32fb-4a84-83cd-87b03c3c80d8',
account_id='66049ada2fc77c99ef015be7',
approved=False,
content_metadata={
'ae01cc98-4af6-44d0-9560-a7ba9fe9434e': {
'artifact_id': '00003d33-7763-46ab-9005-9aaad947ba25',
'artifact_name': 'The Legend of Zelda: Ocarina of Time - Guide and Walkthrough - Nintendo 64 - By MrShotgun - GameFAQs.pdf',
'artifact_content_modification_identifier': None,
'chunk_text': '02. Enter the Great Deku Tree and defeat Gohma.'
},
'e28cf8ae-3b44-4f2b-ae54-7669245a056b': {
'artifact_id': '9624c5fb-f672-46c9-b64c-49d2c061d171',
'artifact_name': 'The Legend of Zelda: Ocarina of Time - Guide and Walkthrough - Nintendo 64 - By A_I_e_x - GameFAQs.pdf',
'artifact_content_modification_identifier': None,
'chunk_text': 'He will knock you down and then always turn around. Get up quickly and throw'
},
'f8090dd2-c9a0-4f3b-b53c-8c0d660ed445': {
'artifact_id': '9624c5fb-f672-46c9-b64c-49d2c061d171',
'artifact_name': 'The Legend of Zelda: Ocarina of Time - Guide and Walkthrough - Nintendo 64 - By A_I_e_x - GameFAQs.pdf',
'artifact_content_modification_identifier': None,
'chunk_text': "This enemy moves exactly as you do so you're going to have to find a way to"
}
},
created_at=datetime.datetime(2024, 9, 26, 20, 8, 4, 692678),
created_by_user_id='42a5c8af-f698-43d0-923e-ba70102a2887',
evaluation_dataset_id='da20895d-1784-4d40-a403-e7b5f8a2ce78',
schema_type='GENERATION',
test_case_data=GenerationTestCaseSchema(
input='What steps should you follow to defeat Gohma inside the Great Deku Tree?',
expected_extra_info=StringExtraInfoSchema(
info="This enemy moves exactly as you do so you're going to have to find a way to\n"
"CHUNK_SEPARATING_STRING\nHe will knock you down and then always turn around. Get up quickly and throw\n"
"CHUNK_SEPARATING_STRING\n02. Enter the Great Deku Tree and defeat Gohma.",
schema_type='STRING'
),
expected_output="To defeat Gohma inside the Great Deku Tree, you should follow these steps:\n"
"1. Enter the Great Deku Tree.\n"
"2. Make your way to the room where Gohma is located.\n"
"3. Gohma will mirror your movements as you approach. Find a way to outmaneuver it to avoid its attacks.\n"
"4. When it knocks you down, be prepared to get up quickly and throw something at it before it turns around.\n"
"5. Look for visual cues such as Gohma on the ceiling or walls to track its movements and attack accordingly.\n"
"6. Continue to attack until you defeat Gohma."
),
updated_at=datetime.datetime(2024, 9, 26, 20, 8, 4, 692678),
missing_chunks=[],
topic_str='Gold Skulltula Locations Guide'
)
],
current_page=1,
total_item_count=3,
items_per_page=100
)
PublishEvaluationDatasetDraftResponse(
autogenerated_draft_test_cases=[
ApproveAutoGeneratedDraftTestCaseResponse(
autogenerated_draft_test_case_id='2d5d8282-32fb-4a84-83cd-87b03c3c80d8',
failed_chunks=[],
success=True
),
ApproveAutoGeneratedDraftTestCaseResponse(
autogenerated_draft_test_case_id='d8043540-5d50-46e3-a3b4-db1eb1103355',
failed_chunks=[],
success=True
),
ApproveAutoGeneratedDraftTestCaseResponse(
autogenerated_draft_test_case_id='15bdc65d-5223-4ce0-948f-9cb6b3b24f1a',
failed_chunks=[],
success=True
)
],
success=True
)