1. Instantiate Client
1. Instantiate Client
Follow the instructions in the Quickstart Guide to setup the SGP Client
from scale_gp import SGPClient
client = SGPClient(api_key=api_key)
2. Create an evaluation
2. Create an evaluation
Create an evaluation by following any previous evaluation recipe such as Multistage Evaluation
evaluation = client.evaluations.create(
type="builder",
account_id=account_id,
application_spec_id=spec_id,
application_variant_id=variant_id,
description="description",
evaluation_dataset_id=evaluation_dataset.id,
name="Report card evaluation",
evaluation_config_id=evaluation_config.id,
)
3. Generate a report card for the application variant
3. Generate a report card for the application variant
A report card will provide a summary of the performance of the application variant against an evaluation dataset
Three main scores will be produced: Accuracy, Quality and Trust & Safety.
The performance in each category will contribute to an overall Scale Confidence Score for the variant.You can view generated report cards in the UI at:
https://egp.dashboard.scale.com/applications/{application_spec_id}/{application_variant_id}/report-card/overview
create_response = client.application_variant_reports.create(
application_variant_id=variant_id,
evaluation_dataset_ids=[manual_evaluation_dataset.id],
account_id=account_id,
)
# retrieve the application variant report, will still be PENDING if immediately retrieved
application_variant_report = client.application_variant_reports.retrieve(
application_variant_report_id=create_response.id,
view=["AsyncJobs"],
)
import os
import time
from scale_gp import SGPClient
client = SGPClient(api_key=api_key)
evaluation = client.evaluations.create(
type="builder",
account_id=account_id,
application_spec_id=spec_id,
application_variant_id=variant_id,
description="description",
evaluation_dataset_id=manual_evaluation_dataset.id,
name="Report card evaluation",
evaluation_config_id=evaluation_config.id,
)
create_response = client.application_variant_reports.create(
application_variant_id=builder.application.variant_id,
evaluation_dataset_ids=[manual_evaluation_dataset.id],
account_id=account_id,
)
print(create_response)
# retrieve the application variant report, will still be PENDING if immediately retrieved
application_variant_report = client.application_variant_reports.retrieve(
application_variant_report_id=create_response.id,
view=["AsyncJobs"],
)
print(application_variant_report)
ApplicationVariantWithScores(
id='cba0f08d-cb98-4221-a96d-d3eacb1c7a38',
account_id='66049ada2fc77c99ef015be7',
application_spec_id='ef0f2d50-1304-4b40-a0cc-1faf449b9555',
application_variant_id='37d7d6ed-b7e8-4c3b-a488-03c96bb23d5d',
created_at=datetime.datetime(2024, 9, 26, 20, 34, 6, 45000),
updated_at=datetime.datetime(2024, 9, 26, 20, 34, 6, 45000),
category_scores=None,
evaluation_datasets=None,
score=None
)
ApplicationVariantWithScoresAndViews(
id='cba0f08d-cb98-4221-a96d-d3eacb1c7a38',
account_id='66049ada2fc77c99ef015be7',
application_spec_id='ef0f2d50-1304-4b40-a0cc-1faf449b9555',
application_variant_id='37d7d6ed-b7e8-4c3b-a488-03c96bb23d5d',
created_at=datetime.datetime(2024, 9, 26, 20, 34, 6, 45000),
updated_at=datetime.datetime(2024, 9, 26, 20, 34, 6, 45000),
async_jobs=[
AsyncJob(
id='ddc6d3fd-bc23-4ee1-a84b-77cc485b5b31',
created_at=datetime.datetime(2024, 9, 26, 20, 34, 6, 45479),
job_type='application-variant-report-generation',
status='Running',
updated_at=datetime.datetime(2024, 9, 26, 20, 35, 12, 963088),
job_metadata=None,
progress=AsyncJobProgress(
completed=166,
failed=0,
pending=400
)
)
],
category_scores=[
CategoryScoreApplicationCategoryScoreQuality(
category='quality',
metric_scores=[
CategoryScoreApplicationCategoryScoreQualityMetricScoreApplicationMetricScoreCoherence(
category='quality',
metric_type='coherence',
score=100.0
),
CategoryScoreApplicationCategoryScoreQualityMetricScoreApplicationMetricScoreGrammar(
category='quality',
metric_type='grammar',
score=100.0
)
],
score=100.0
),
CategoryScoreApplicationCategoryScoreTrustAndSafety(
category='trust-and-safety',
metric_scores=[
CategoryScoreApplicationCategoryScoreTrustAndSafetyMetricScoreApplicationMetricScoreModeration(
category='trust-and-safety',
metric_type='moderation',
score=100.0
)
],
score=100.0
),
CategoryScoreApplicationCategoryScoreAccuracy(
category='accuracy',
metric_scores=[
CategoryScoreApplicationCategoryScoreAccuracyMetricScoreApplicationMetricScoreAnswerRelevance(
category='accuracy',
metric_type='answer-relevance',
score=79.33333333333333
),
CategoryScoreApplicationCategoryScoreAccuracyMetricScoreApplicationMetricScoreAnswerCorrectness(
category='accuracy',
metric_type='answer-correctness',
score=83.33333333333333
)
],
score=81.33333333333333
)
],
evaluation_datasets=[
EvaluationDataset(
evaluation_dataset=EvaluationDataset(
id='04f7fdf5-cdb6-490b-ae59-b1e14659e064',
account_id='66049ada2fc77c99ef015be7',
created_at=datetime.datetime(2024, 9, 26, 20, 31, 28, 553621),
created_by_user_id='42a5c8af-f698-43d0-923e-ba70102a2887',
draft=None,
name='manual_evaluation_dataset',
schema_type='GENERATION',
updated_at=datetime.datetime(2024, 9, 26, 20, 31, 28, 553621),
archived_at=None,
evaluation_dataset_metadata=None,
knowledge_base_id=None,
out_of_date=None,
vendor=None
),
evaluation_dataset_version_num=1,
generation_status='Completed',
scored_test_case_count=3
),
EvaluationDataset(
evaluation_dataset=EvaluationDataset(
id='89154549-6867-4d84-8bfd-435a3769e675',
account_id='66049ada2fc77c99ef015be7',
created_at=datetime.datetime(2024, 7, 11, 8, 38, 40),
created_by_user_id='5c102f95-7142-4cdc-899e-823aae3fdb9e',
draft=None,
name='Trust and Safety Evaluation Dataset v0.0.0',
schema_type='GENERATION',
updated_at=datetime.datetime(2024, 7, 11, 8, 38, 40),
archived_at=None,
evaluation_dataset_metadata=None,
knowledge_base_id=None,
out_of_date=None,
vendor='scale'
),
evaluation_dataset_version_num=1,
generation_status='Pending',
scored_test_case_count=0
)
],
score=93.40938696795908
)