Customize Annotation Config for Evaluations

external_application = ExternalApplication(client).initialize(
    application_variant_id=external_application_variant,
    application=application,
)

external_application.generate_outputs(
    evaluation_dataset_id=flexible_evaluation_dataset.id,
    evaluation_dataset_version=1,
)

annotation_config_dict: AnnotationConfigParam = {
    "direction": "row",  # this is by default - try switching to "col"
    "components": [
        [
            {
                "data_loc": ["test_case_data", "input"],
            },
            {
                "data_loc": ["test_case_data", "expected_output"],
            },
        ],
        [
            {
                "data_loc": ["test_case_output", "output"],
            },
        ],
    ],
}

question_id_to_annotation_config_dict: Dict[str, AnnotationConfigParam] = {
    question_ids[1]: {
        "components": [
            [
                {
                    "data_loc": ["test_case_data", "input"],
                },
                {
                    "data_loc":  ["trace", "completion", "output", "completion_output"], # show a trace span of format ["trace", <node_id>, "output", <key, optionally>]
                }
            ]
        ],
    }
}

evaluation_with_annotation_config = client.evaluations.create(
    type="builder",
    account_id=account_id,
    application_spec_id=external_application_spec,
    application_variant_id=external_application_variant,
    description="description",
    evaluation_dataset_id=flexible_evaluation_dataset.id,
    annotation_config=annotation_config_dict,
    question_id_to_annotation_config=question_id_to_annotation_config_dict,
    name="Flexible eval",
    evaluation_config_id=evaluation_config.id,
)
print(evaluation_with_annotation_config)

Evaluation(
    id='5d459525-68de-4a42-bf2f-7b65348c7d25',
    account_id='66049ada2fc77c99ef015be7',
    application_spec_id='c8033f8f-a9f3-4081-9de0-307d955f36fc',
    completed_test_case_result_count=0,
    created_at=datetime.datetime(2024, 9, 26, 20, 25, 1, 346790),
    created_by_user_id='42a5c8af-f698-43d0-923e-ba70102a2887',
    description='description',
    name='Flexible eval',
    status='PENDING',
    total_test_case_result_count=0,
    annotation_config=AnnotationConfig(
        components=[
            [Component(data_loc=['test_case_data', 'input'], label=None), 
             Component(data_loc=['test_case_data', 'expected_output'], label=None)],
            [Component(data_loc=['test_case_output', 'output'], label=None)]
        ],
        annotation_config_type='flexible',
        direction='row'
    ),
    application_variant_id='61ba83b9-4b95-439c-9922-c918e6d07c13',
    archived_at=None,
    completed_at=None,
    evaluation_config=None,
    evaluation_config_id='952b4868-a71a-454f-a397-dbe16e241dc0',
    question_id_to_annotation_config={
        'd9dc6331-4054-4424-b7f6-b35520aa04a6': AnnotationConfig(
            components=[
                [Component(data_loc=['test_case_data', 'input'], label=None), 
                 Component(data_loc=['trace', 'completion', 'output', 'completion_output'], label=None)]
            ],
            annotation_config_type='flexible',
            direction='row'
        )
    },
    tags=None
)

external_application = ExternalApplication(client).initialize(
    application_variant_id=external_application_variant,
    application=application,
)

external_application.generate_outputs(
    evaluation_dataset_id=flexible_evaluation_dataset.id,
    evaluation_dataset_version=1,
)

annotation_config_dict: AnnotationConfigParam = {
    "direction": "row",  # this is by default - try switching to "col"
    "components": [
        [
            {
                "data_loc": ["test_case_data", "input"],
            },
            {
                "data_loc": ["test_case_data", "expected_output"],
            },
        ],
        [
            {
                "data_loc": ["test_case_output", "output"],
            },
        ],
    ],
}

question_id_to_annotation_config_dict: Dict[str, AnnotationConfigParam] = {
    question_ids[1]: {
        "components": [
            [
                {
                    "data_loc": ["test_case_data", "input"],
                },
                {
                    "data_loc":  ["trace", "completion", "output", "completion_output"], # show a trace span of format ["trace", <node_id>, "output", <key, optionally>]
                }
            ]
        ],
    }
}

evaluation_with_annotation_config = client.evaluations.create(
    type="builder",
    account_id=account_id,
    application_spec_id=external_application_spec,
    application_variant_id=external_application_variant,
    description="description",
    evaluation_dataset_id=flexible_evaluation_dataset.id,
    annotation_config=annotation_config_dict,
    question_id_to_annotation_config=question_id_to_annotation_config_dict,
    name="Flexible eval",
    evaluation_config_id=evaluation_config.id,
)
print(evaluation_with_annotation_config)

Evaluation(
    id='5d459525-68de-4a42-bf2f-7b65348c7d25',
    account_id='66049ada2fc77c99ef015be7',
    application_spec_id='c8033f8f-a9f3-4081-9de0-307d955f36fc',
    completed_test_case_result_count=0,
    created_at=datetime.datetime(2024, 9, 26, 20, 25, 1, 346790),
    created_by_user_id='42a5c8af-f698-43d0-923e-ba70102a2887',
    description='description',
    name='Flexible eval',
    status='PENDING',
    total_test_case_result_count=0,
    annotation_config=AnnotationConfig(
        components=[
            [Component(data_loc=['test_case_data', 'input'], label=None), 
             Component(data_loc=['test_case_data', 'expected_output'], label=None)],
            [Component(data_loc=['test_case_output', 'output'], label=None)]
        ],
        annotation_config_type='flexible',
        direction='row'
    ),
    application_variant_id='61ba83b9-4b95-439c-9922-c918e6d07c13',
    archived_at=None,
    completed_at=None,
    evaluation_config=None,
    evaluation_config_id='952b4868-a71a-454f-a397-dbe16e241dc0',
    question_id_to_annotation_config={
        'd9dc6331-4054-4424-b7f6-b35520aa04a6': AnnotationConfig(
            components=[
                [Component(data_loc=['test_case_data', 'input'], label=None), 
                 Component(data_loc=['trace', 'completion', 'output', 'completion_output'], label=None)]
            ],
            annotation_config_type='flexible',
            direction='row'
        )
    },
    tags=None
)

1. Perform the steps outlined in the Multi-Stage Evaluation Recipe

Following the previous recipe, create an evaluation dataset, question set, application variant etc. Stop after generating test case outputs for your application.


from scale_gp.types.annotation_config_param import AnnotationConfigParam

external_application = ExternalApplication(client).initialize(
  application_variant_id=external_application_variant,
  application=application,
)

external_application.generate_outputs(
  evaluation_dataset_id=flexible_evaluation_dataset.id,
  evaluation_dataset_version=1,
)

2. Create an annotation config

An annotation configuration is used to configure what the annotation UI will look like. The annotation UI is displayed as a two dimensional grid. There are configurations used to indicate the annotation experience.First we will define the “annotation_config_dict”. This is used to indicate the layout of the test case on the annotations page. This dictionary has three keys: “annotation_config_type”, “direction”, and “components”. For all flexible evaluations, set the “annotation_config_type” to “flexible”. The “direction” allows users to configure their UI in two ways. If the user selects “row” for this configuration, then the UI will allow for a max number of 2 items in each row and users can create as many rows as desired. If the user selects “col” for this configuration, then the UI will allow for a max number of 2 items in each column, and the users can choose to have as many columns as possible. Overflow for both will allow for either horizontal or vertical scrolling. The “components” section indicates which components in the application variant to show for annotations in the UI.


annotation_config_dict: AnnotationConfigParam = {
    "direction": "row",  # this is by default - try switching to "col"
    "components": [
        [
            {
                "data_loc": ["test_case_data", "input"],
            },
            {
                "data_loc": ["test_case_data", "expected_output"],
            },
        ],
        [
            {
                "data_loc": ["test_case_output", "output"],
            },
        ],
    ],
}

3. Create question specific configs

If desired, you can configure specific views per question. For each question that needs to be annotated, users can indicate which components of the test case are relevant to the question. For example, if the question is focused on evaluating the retrieval part of an application test case, indicate here to only show the traces that are relevant to the retrieval part of the test case in the annotation UI.

# try clicking on the second question to see the annotation UI change
question_id_to_annotation_config_dict: Dict[str, AnnotationConfigParam] = {
    question_ids[1]: {
        "components": [
            [
                {
                    "data_loc": ["test_case_data", "input"],
                },
                {
                    "data_loc":  ["trace", "completion", "output", "completion_output"], # show a trace span of format ["trace", <node_id>, "output", <key, optionally>]
                }
            ]
        ],
    }
}

4. Create an evaluation with the annotation config

You can now create an evaluation with your specifiec annotation configs. If you take a look at the annotation view, you will see it follow the row and column format you specified, along with showing any special information such as traces.If you click on the second question in the annotation view, you will see it change according to the question specific config defined in question_id_to_annotation_config_dict.

evaluation = client.evaluations.create(
    type="builder",
    account_id=account_id,
    application_spec_id=external_application_spec,
    application_variant_id=external_application_variant,
    description="description",
    evaluation_dataset_id=flexible_evaluation_dataset.id,
    annotation_config=annotation_config_dict,
    question_id_to_annotation_config=question_id_to_annotation_config_dict,
    name="Flexible eval",
    evaluation_config_id=evaluation_config.id,
)

external_application = ExternalApplication(client).initialize(
    application_variant_id=external_application_variant,
    application=application,
)

external_application.generate_outputs(
    evaluation_dataset_id=flexible_evaluation_dataset.id,
    evaluation_dataset_version=1,
)

annotation_config_dict: AnnotationConfigParam = {
    "direction": "row",  # this is by default - try switching to "col"
    "components": [
        [
            {
                "data_loc": ["test_case_data", "input"],
            },
            {
                "data_loc": ["test_case_data", "expected_output"],
            },
        ],
        [
            {
                "data_loc": ["test_case_output", "output"],
            },
        ],
    ],
}

question_id_to_annotation_config_dict: Dict[str, AnnotationConfigParam] = {
    question_ids[1]: {
        "components": [
            [
                {
                    "data_loc": ["test_case_data", "input"],
                },
                {
                    "data_loc":  ["trace", "completion", "output", "completion_output"], # show a trace span of format ["trace", <node_id>, "output", <key, optionally>]
                }
            ]
        ],
    }
}

evaluation_with_annotation_config = client.evaluations.create(
    type="builder",
    account_id=account_id,
    application_spec_id=external_application_spec,
    application_variant_id=external_application_variant,
    description="description",
    evaluation_dataset_id=flexible_evaluation_dataset.id,
    annotation_config=annotation_config_dict,
    question_id_to_annotation_config=question_id_to_annotation_config_dict,
    name="Flexible eval",
    evaluation_config_id=evaluation_config.id,
)
print(evaluation_with_annotation_config)

Evaluation(
    id='5d459525-68de-4a42-bf2f-7b65348c7d25',
    account_id='66049ada2fc77c99ef015be7',
    application_spec_id='c8033f8f-a9f3-4081-9de0-307d955f36fc',
    completed_test_case_result_count=0,
    created_at=datetime.datetime(2024, 9, 26, 20, 25, 1, 346790),
    created_by_user_id='42a5c8af-f698-43d0-923e-ba70102a2887',
    description='description',
    name='Flexible eval',
    status='PENDING',
    total_test_case_result_count=0,
    annotation_config=AnnotationConfig(
        components=[
            [Component(data_loc=['test_case_data', 'input'], label=None), 
             Component(data_loc=['test_case_data', 'expected_output'], label=None)],
            [Component(data_loc=['test_case_output', 'output'], label=None)]
        ],
        annotation_config_type='flexible',
        direction='row'
    ),
    application_variant_id='61ba83b9-4b95-439c-9922-c918e6d07c13',
    archived_at=None,
    completed_at=None,
    evaluation_config=None,
    evaluation_config_id='952b4868-a71a-454f-a397-dbe16e241dc0',
    question_id_to_annotation_config={
        'd9dc6331-4054-4424-b7f6-b35520aa04a6': AnnotationConfig(
            components=[
                [Component(data_loc=['test_case_data', 'input'], label=None), 
                 Component(data_loc=['trace', 'completion', 'output', 'completion_output'], label=None)]
            ],
            annotation_config_type='flexible',
            direction='row'
        )
    },
    tags=None
)

Create a Flexible Evaluation Generate an Application Report Card

Recipes