import os
import time
from scale_gp import SGPClient
client = SGPClient(api_key=api_key)
evaluation = client.evaluations.create(
type="builder",
account_id=account_id,
application_spec_id=spec_id,
application_variant_id=variant_id,
description="description",
evaluation_dataset_id=manual_evaluation_dataset.id,
name="Report card evaluation",
evaluation_config_id=evaluation_config.id,
)
create_response = client.application_variant_reports.create(
application_variant_id=builder.application.variant_id,
evaluation_dataset_ids=[manual_evaluation_dataset.id],
account_id=account_id,
)
print(create_response)
# retrieve the application variant report, will still be PENDING if immediately retrieved
application_variant_report = client.application_variant_reports.retrieve(
application_variant_report_id=create_response.id,
view=["AsyncJobs"],
)
print(application_variant_report)
ApplicationVariantWithScores(
id='cba0f08d-cb98-4221-a96d-d3eacb1c7a38',
account_id='66049ada2fc77c99ef015be7',
application_spec_id='ef0f2d50-1304-4b40-a0cc-1faf449b9555',
application_variant_id='37d7d6ed-b7e8-4c3b-a488-03c96bb23d5d',
created_at=datetime.datetime(2024, 9, 26, 20, 34, 6, 45000),
updated_at=datetime.datetime(2024, 9, 26, 20, 34, 6, 45000),
category_scores=None,
evaluation_datasets=None,
score=None
)
Generate a report card for an application variant
import os
import time
from scale_gp import SGPClient
client = SGPClient(api_key=api_key)
evaluation = client.evaluations.create(
type="builder",
account_id=account_id,
application_spec_id=spec_id,
application_variant_id=variant_id,
description="description",
evaluation_dataset_id=manual_evaluation_dataset.id,
name="Report card evaluation",
evaluation_config_id=evaluation_config.id,
)
create_response = client.application_variant_reports.create(
application_variant_id=builder.application.variant_id,
evaluation_dataset_ids=[manual_evaluation_dataset.id],
account_id=account_id,
)
print(create_response)
# retrieve the application variant report, will still be PENDING if immediately retrieved
application_variant_report = client.application_variant_reports.retrieve(
application_variant_report_id=create_response.id,
view=["AsyncJobs"],
)
print(application_variant_report)
ApplicationVariantWithScores(
id='cba0f08d-cb98-4221-a96d-d3eacb1c7a38',
account_id='66049ada2fc77c99ef015be7',
application_spec_id='ef0f2d50-1304-4b40-a0cc-1faf449b9555',
application_variant_id='37d7d6ed-b7e8-4c3b-a488-03c96bb23d5d',
created_at=datetime.datetime(2024, 9, 26, 20, 34, 6, 45000),
updated_at=datetime.datetime(2024, 9, 26, 20, 34, 6, 45000),
category_scores=None,
evaluation_datasets=None,
score=None
)
1. Instantiate Client
from scale_gp import SGPClient
client = SGPClient(api_key=api_key)
2. Create an evaluation
evaluation = client.evaluations.create(
type="builder",
account_id=account_id,
application_spec_id=spec_id,
application_variant_id=variant_id,
description="description",
evaluation_dataset_id=evaluation_dataset.id,
name="Report card evaluation",
evaluation_config_id=evaluation_config.id,
)
3. Generate a report card for the application variant
create_response = client.application_variant_reports.create(
application_variant_id=variant_id,
evaluation_dataset_ids=[manual_evaluation_dataset.id],
account_id=account_id,
)
# retrieve the application variant report, will still be PENDING if immediately retrieved
application_variant_report = client.application_variant_reports.retrieve(
application_variant_report_id=create_response.id,
view=["AsyncJobs"],
)
import os
import time
from scale_gp import SGPClient
client = SGPClient(api_key=api_key)
evaluation = client.evaluations.create(
type="builder",
account_id=account_id,
application_spec_id=spec_id,
application_variant_id=variant_id,
description="description",
evaluation_dataset_id=manual_evaluation_dataset.id,
name="Report card evaluation",
evaluation_config_id=evaluation_config.id,
)
create_response = client.application_variant_reports.create(
application_variant_id=builder.application.variant_id,
evaluation_dataset_ids=[manual_evaluation_dataset.id],
account_id=account_id,
)
print(create_response)
# retrieve the application variant report, will still be PENDING if immediately retrieved
application_variant_report = client.application_variant_reports.retrieve(
application_variant_report_id=create_response.id,
view=["AsyncJobs"],
)
print(application_variant_report)
ApplicationVariantWithScores(
id='cba0f08d-cb98-4221-a96d-d3eacb1c7a38',
account_id='66049ada2fc77c99ef015be7',
application_spec_id='ef0f2d50-1304-4b40-a0cc-1faf449b9555',
application_variant_id='37d7d6ed-b7e8-4c3b-a488-03c96bb23d5d',
created_at=datetime.datetime(2024, 9, 26, 20, 34, 6, 45000),
updated_at=datetime.datetime(2024, 9, 26, 20, 34, 6, 45000),
category_scores=None,
evaluation_datasets=None,
score=None
)