Summarization Evaluation

import os
from typing import List
from datetime import datetime
from uuid import uuid4

from scale_gp import SGPClient
from scale_gp.lib.types.summarization import SummarizationTestCaseSchema
from scale_gp.lib.dataset_builder import DatasetBuilder
from scale_gp.lib.external_applications import ExternalApplicationOutputFlexible, ExternalApplication
from scale_gp.lib.types import data_locator
from scale_gp.types import SummarizationAnnotationConfigParam

client = SGPClient(base_url="https://api.egp.scale.com")

document_data = [
    {
        "document": """The Industrial Revolution, which took place from the 18th to 19th centuries, was a period during which predominantly agrarian, rural societies in Europe and America became industrial and urban. Prior to the Industrial Revolution, which began in Britain in the late 1700s, manufacturing was often done in people's homes, using hand tools or basic machines. Industrialization marked a shift to powered, special-purpose machinery, factories and mass production.

The iron and textile industries, along with the development of the steam engine, played central roles in the Industrial Revolution, which also saw improved systems of transportation, communication and banking. While industrialization brought about an increased volume and variety of manufactured goods and an improved standard of living for some, it also resulted in often grim employment and living conditions for the poor and working classes.

The Industrial Revolution led to unprecedented population growth and rapid urbanization, as people moved from rural areas to cities in search of work. This led to overcrowding and poor living conditions for many urban workers. Child labor became prevalent, and working conditions were often hazardous with long hours in dangerous environments.

However, the Industrial Revolution also brought about many positive changes. It led to the growth of the middle class, increased literacy rates, and improvements in healthcare and life expectancy. The increased production of goods made many items more affordable and accessible to the general population. Additionally, the technological advancements of this period laid the groundwork for future innovations and economic growth.""",
        "expected_summary": "The Industrial Revolution was a transformative period from the 18th to 19th centuries, marked by a shift from agrarian to industrial societies. It brought about technological advancements, increased production, and economic growth, but also led to challenging working and living conditions for many. While it had negative impacts such as child labor and urban overcrowding, it also resulted in the growth of the middle class, improved literacy, and better healthcare."
    },
    {
        "document": """Quantum computing is an area of computing focused on developing computer technology based on the principles of quantum theory. Quantum computers use quantum bits or qubits, which can exist in multiple states simultaneously, unlike classical bits that are either 0 or 1. This property, known as superposition, allows quantum computers to perform certain calculations much faster than classical computers.

One of the key principles in quantum computing is entanglement, where qubits can be correlated with each other in such a way that the state of one qubit can depend on the state of another, regardless of the distance between them. This property enables quantum computers to process a vast number of possibilities simultaneously.

Quantum computers have the potential to revolutionize various fields, including cryptography, drug discovery, financial modeling, and climate change prediction. In cryptography, quantum computers could potentially break many of the encryption systems currently in use, which rely on the difficulty of factoring large numbers. This has led to the development of quantum-resistant cryptography.

In the field of drug discovery, quantum computers could simulate complex molecular interactions more accurately than classical computers, potentially speeding up the process of developing new medications. In financial modeling, quantum computers could optimize trading strategies and risk management by processing vast amounts of data more efficiently.

However, quantum computing faces several challenges. Quantum systems are highly sensitive to their environment, and maintaining quantum coherence (the ability of quantum systems to maintain their quantum state) is difficult. Error correction in quantum systems is also a significant challenge, as measuring a qubit can cause it to lose its quantum state.

Despite these challenges, many tech giants and startups are investing heavily in quantum computing research and development. As the field progresses, we may see quantum computers solving problems that are currently intractable for classical computers, opening up new frontiers in science and technology.""",
        "expected_summary": "Quantum computing is an emerging field that uses quantum mechanics principles to process information. It utilizes qubits, which can exist in multiple states simultaneously, enabling faster computation for certain problems. While promising for various applications like cryptography and drug discovery, quantum computing faces challenges such as maintaining quantum coherence and error correction. Despite these hurdles, significant investment and research continue in this potentially revolutionary technology."
    },
    {
        "document": """Climate change refers to long-term shifts in global weather patterns and average temperatures. The main driver of current climate change is human activity, particularly the burning of fossil fuels, which releases greenhouse gases into the atmosphere. These gases, including carbon dioxide and methane, trap heat and cause the Earth's average temperature to rise.

The effects of climate change are far-reaching and complex. Rising global temperatures lead to melting polar ice caps and glaciers, causing sea levels to rise. This threatens coastal communities and low-lying islands with flooding and erosion. Climate change also results in more frequent and intense extreme weather events, such as hurricanes, heatwaves, and droughts.

Ecosystems worldwide are being affected by climate change. Many plant and animal species are struggling to adapt to rapidly changing conditions, leading to shifts in their geographic ranges and, in some cases, extinction. Coral reefs, which are particularly sensitive to changes in ocean temperature and acidity, are experiencing widespread bleaching events.

The impacts of climate change extend to human societies as well. Changes in precipitation patterns and temperature affect agriculture, potentially leading to food insecurity in some regions. Water scarcity is becoming a growing concern in many parts of the world. Climate change can also exacerbate social and economic inequalities, as vulnerable populations often bear the brunt of its impacts.

Efforts to address climate change focus on two main strategies: mitigation and adaptation. Mitigation involves reducing greenhouse gas emissions through measures such as transitioning to renewable energy sources, improving energy efficiency, and protecting and restoring natural carbon sinks like forests. Adaptation strategies aim to build resilience to the impacts of climate change that are already occurring or are inevitable.

International cooperation is crucial in tackling climate change. The Paris Agreement, adopted in 2015, aims to limit global temperature increase to well below 2 degrees Celsius above pre-industrial levels. However, current national commitments are not sufficient to meet this goal, and there is an urgent need for more ambitious action.

As the effects of climate change become more apparent, there is growing public awareness and demand for action. Many cities and businesses are taking steps to reduce their carbon footprint and prepare for climate impacts. Technological innovations, such as renewable energy and carbon capture systems, are also playing a role in addressing the challenge. However, addressing climate change will require sustained effort and collaboration at all levels of society.""",
        "expected_summary": "Climate change is a global phenomenon primarily driven by human activities, especially the emission of greenhouse gases. It leads to rising temperatures, sea-level rise, extreme weather events, and ecosystem disruptions. The impacts affect both natural systems and human societies, potentially exacerbating inequalities. Addressing climate change involves mitigation (reducing emissions) and adaptation (building resilience) strategies. While international efforts like the Paris Agreement exist, more ambitious action is needed to tackle this complex, far-reaching issue."
    }
]

test_cases = []
for data in document_data * 10:
    tc = SummarizationTestCaseSchema(
        document=data["document"],
        expected_summary=data["expected_summary"]
    )
    test_cases.append(tc)
    print(tc)

dataset = DatasetBuilder(client).initialize(
    account_id="your_account_id",
    name=f"Summarization Dataset {timestamp()}",
    test_cases=test_cases
)
print(dataset)

def my_summarization_app(prompt, test_case):
    document = prompt['document']
    start = datetime.now().replace(microsecond=5000)

    # generate summary here
    summary = "GENERATED OUTPUT SUMMARY"

    return ExternalApplicationOutputFlexible(
        generation_output={
            "generated_summary": summary
        },
        trace_spans=[
            {
                "node_id": "formatting",
                "start_timestamp": str(start.isoformat()),
                "operation_input": {
                    "document": "EXAMPLE INPUT DOCUMENT"
                },
                "operation_output": {
                    "formatted_document": "EXAMPLE OUTPUT DOCUMENT FORMATTED"
                },
                "duration_ms": 1000,
            }
        ],
        metrics={"grammar": 0.5}
    )

app = ExternalApplication(client)
app.initialize(application_variant_id="your_variant_id", application=my_summarization_app)
app.generate_outputs(evaluation_dataset_id=dataset.id, evaluation_dataset_version='1')

question_requests = [
    {
        "type": "categorical",
        "title": "Test Question 1",
        "prompt": "Test Prompt",
        "choices": [{"label": "No", "value": 0}, {"label": "Yes", "value": 1}]
    },
    {
        "type": "categorical",
        "title": "Test Question 2",
        "prompt": "Was the summary concise?",
        "choices": [{"label": "No", "value": 0}, {"label": "Yes", "value": 1}]
    },
    {
        "type": "free_text",
        "title": "Test Question 3",
        "prompt": "List relevant information the summary cut out"
    }
]

question_ids = []
for question in question_requests:
    q = client.questions.create(
        **question
    )
    question_ids.append(q.id)
    print(q)

q_set = client.question_sets.create(
    name="summarization question set",
    question_ids=question_ids
)
print(q_set)

config = client.evaluation_configs.create(
    question_set_id=q_set.id,
    evaluation_type='human'
)
print(config)

annotation_config_dict = SummarizationAnnotationConfigParam(
    document_loc=data_locator.test_case_data.input["document"],
    summary_loc=data_locator.test_case_output.output["generated_summary"],
    expected_summary_loc=data_locator.test_case_data.expected_output["expected_summary"]
)

evaluation = client.evaluations.create(
    application_variant_id="your_variant_id",
    application_spec_id="your_spec_id",
    description="Demo Evaluation",
    name="Summarization Evaluation",
    evaluation_config_id=config.id,
    annotation_config=annotation_config_dict,
    evaluation_dataset_id=dataset.id,
    type="builder"
)

print(evaluation)

EvaluationDataset(
    id='3fa40b7a-7b77-487f-b3dd-7d75246ae3dc',
    account_id='6630377a5a7b09c735cfeebb',
    created_at=datetime.datetime(2024, 9, 30, 23, 12, 9, 947509),
    created_by_user_id='d6dee46a-25e2-45af-bf89-eb6ae949ac1a',
    name='Summarization Dataset 2024-09-30 16:12:09 d0e432a6-fd5d-46e5-bd07-af44b638d5f3',
    schema_type='FLEXIBLE',
    updated_at=datetime.datetime(2024, 9, 30, 23, 12, 9, 947509),
    archived_at=None,
    evaluation_dataset_metadata=None,
    knowledge_base_id=None,
    out_of_date=None,
    vendor=None
)