external_application = ExternalApplication(client).initialize(
    application_variant_id=external_application_variant,
    application=application,
)

external_application.generate_outputs(
    evaluation_dataset_id=flexible_evaluation_dataset.id,
    evaluation_dataset_version=1,
)

annotation_config_dict: AnnotationConfigParam = {
    "direction": "row",  # this is by default - try switching to "col"
    "components": [
        [
            {
                "data_loc": ["test_case_data", "input"],
            },
            {
                "data_loc": ["test_case_data", "expected_output"],
            },
        ],
        [
            {
                "data_loc": ["test_case_output", "output"],
            },
        ],
    ],
}

question_id_to_annotation_config_dict: Dict[str, AnnotationConfigParam] = {
    question_ids[1]: {
        "components": [
            [
                {
                    "data_loc": ["test_case_data", "input"],
                },
                {
                    "data_loc":  ["trace", "completion", "output", "completion_output"], # show a trace span of format ["trace", <node_id>, "output", <key, optionally>]
                }
            ]
        ],
    }
}

evaluation_with_annotation_config = client.evaluations.create(
    type="builder",
    account_id=account_id,
    application_spec_id=external_application_spec,
    application_variant_id=external_application_variant,
    description="description",
    evaluation_dataset_id=flexible_evaluation_dataset.id,
    annotation_config=annotation_config_dict,
    question_id_to_annotation_config=question_id_to_annotation_config_dict,
    name="Flexible eval",
    evaluation_config_id=evaluation_config.id,
)
print(evaluation_with_annotation_config)
Evaluation(
    id='5d459525-68de-4a42-bf2f-7b65348c7d25',
    account_id='66049ada2fc77c99ef015be7',
    application_spec_id='c8033f8f-a9f3-4081-9de0-307d955f36fc',
    completed_test_case_result_count=0,
    created_at=datetime.datetime(2024, 9, 26, 20, 25, 1, 346790),
    created_by_user_id='42a5c8af-f698-43d0-923e-ba70102a2887',
    description='description',
    name='Flexible eval',
    status='PENDING',
    total_test_case_result_count=0,
    annotation_config=AnnotationConfig(
        components=[
            [Component(data_loc=['test_case_data', 'input'], label=None), 
             Component(data_loc=['test_case_data', 'expected_output'], label=None)],
            [Component(data_loc=['test_case_output', 'output'], label=None)]
        ],
        annotation_config_type='flexible',
        direction='row'
    ),
    application_variant_id='61ba83b9-4b95-439c-9922-c918e6d07c13',
    archived_at=None,
    completed_at=None,
    evaluation_config=None,
    evaluation_config_id='952b4868-a71a-454f-a397-dbe16e241dc0',
    question_id_to_annotation_config={
        'd9dc6331-4054-4424-b7f6-b35520aa04a6': AnnotationConfig(
            components=[
                [Component(data_loc=['test_case_data', 'input'], label=None), 
                 Component(data_loc=['trace', 'completion', 'output', 'completion_output'], label=None)]
            ],
            annotation_config_type='flexible',
            direction='row'
        )
    },
    tags=None
)
external_application = ExternalApplication(client).initialize(
    application_variant_id=external_application_variant,
    application=application,
)

external_application.generate_outputs(
    evaluation_dataset_id=flexible_evaluation_dataset.id,
    evaluation_dataset_version=1,
)

annotation_config_dict: AnnotationConfigParam = {
    "direction": "row",  # this is by default - try switching to "col"
    "components": [
        [
            {
                "data_loc": ["test_case_data", "input"],
            },
            {
                "data_loc": ["test_case_data", "expected_output"],
            },
        ],
        [
            {
                "data_loc": ["test_case_output", "output"],
            },
        ],
    ],
}

question_id_to_annotation_config_dict: Dict[str, AnnotationConfigParam] = {
    question_ids[1]: {
        "components": [
            [
                {
                    "data_loc": ["test_case_data", "input"],
                },
                {
                    "data_loc":  ["trace", "completion", "output", "completion_output"], # show a trace span of format ["trace", <node_id>, "output", <key, optionally>]
                }
            ]
        ],
    }
}

evaluation_with_annotation_config = client.evaluations.create(
    type="builder",
    account_id=account_id,
    application_spec_id=external_application_spec,
    application_variant_id=external_application_variant,
    description="description",
    evaluation_dataset_id=flexible_evaluation_dataset.id,
    annotation_config=annotation_config_dict,
    question_id_to_annotation_config=question_id_to_annotation_config_dict,
    name="Flexible eval",
    evaluation_config_id=evaluation_config.id,
)
print(evaluation_with_annotation_config)
Evaluation(
    id='5d459525-68de-4a42-bf2f-7b65348c7d25',
    account_id='66049ada2fc77c99ef015be7',
    application_spec_id='c8033f8f-a9f3-4081-9de0-307d955f36fc',
    completed_test_case_result_count=0,
    created_at=datetime.datetime(2024, 9, 26, 20, 25, 1, 346790),
    created_by_user_id='42a5c8af-f698-43d0-923e-ba70102a2887',
    description='description',
    name='Flexible eval',
    status='PENDING',
    total_test_case_result_count=0,
    annotation_config=AnnotationConfig(
        components=[
            [Component(data_loc=['test_case_data', 'input'], label=None), 
             Component(data_loc=['test_case_data', 'expected_output'], label=None)],
            [Component(data_loc=['test_case_output', 'output'], label=None)]
        ],
        annotation_config_type='flexible',
        direction='row'
    ),
    application_variant_id='61ba83b9-4b95-439c-9922-c918e6d07c13',
    archived_at=None,
    completed_at=None,
    evaluation_config=None,
    evaluation_config_id='952b4868-a71a-454f-a397-dbe16e241dc0',
    question_id_to_annotation_config={
        'd9dc6331-4054-4424-b7f6-b35520aa04a6': AnnotationConfig(
            components=[
                [Component(data_loc=['test_case_data', 'input'], label=None), 
                 Component(data_loc=['trace', 'completion', 'output', 'completion_output'], label=None)]
            ],
            annotation_config_type='flexible',
            direction='row'
        )
    },
    tags=None
)