Create agent evaluation

curl --request POST \
  --url https://api.egp.scale.com/v5/agent-evaluations \
  --header 'Content-Type: application/json' \
  --header 'x-api-key: <api-key>' \
  --data '
{
  "account_id": "acct_123",
  "data_plane_dataset_id": "dp_dataset_123",
  "description": "Metadata-only control-plane request",
  "metadata": {
    "suite": "golden"
  },
  "name": "Agent judge eval",
  "projection_policy_id": "policy_eval_projection_v1",
  "tags": [
    "two-plane",
    "mvp"
  ],
  "tasks": [
    {
      "alias": "mock_judge",
      "config": {
        "judge_adapter": "mock"
      },
      "type": "auto_evaluation.agent"
    }
  ],
  "tenant_id": "tenant_123"
}
'

import requests

url = "https://api.egp.scale.com/v5/agent-evaluations"

payload = {
    "account_id": "acct_123",
    "data_plane_dataset_id": "dp_dataset_123",
    "description": "Metadata-only control-plane request",
    "metadata": { "suite": "golden" },
    "name": "Agent judge eval",
    "projection_policy_id": "policy_eval_projection_v1",
    "tags": ["two-plane", "mvp"],
    "tasks": [
        {
            "alias": "mock_judge",
            "config": { "judge_adapter": "mock" },
            "type": "auto_evaluation.agent"
        }
    ],
    "tenant_id": "tenant_123"
}
headers = {
    "x-api-key": "<api-key>",
    "Content-Type": "application/json"
}

response = requests.post(url, json=payload, headers=headers)

print(response.text)

const options = {
  method: 'POST',
  headers: {'x-api-key': '<api-key>', 'Content-Type': 'application/json'},
  body: JSON.stringify({
    account_id: 'acct_123',
    data_plane_dataset_id: 'dp_dataset_123',
    description: 'Metadata-only control-plane request',
    metadata: {suite: 'golden'},
    name: 'Agent judge eval',
    projection_policy_id: 'policy_eval_projection_v1',
    tags: ['two-plane', 'mvp'],
    tasks: [
      {
        alias: 'mock_judge',
        config: {judge_adapter: 'mock'},
        type: 'auto_evaluation.agent'
      }
    ],
    tenant_id: 'tenant_123'
  })
};

fetch('https://api.egp.scale.com/v5/agent-evaluations', options)
  .then(res => res.json())
  .then(res => console.log(res))
  .catch(err => console.error(err));

<?php

$curl = curl_init();

curl_setopt_array($curl, [
  CURLOPT_URL => "https://api.egp.scale.com/v5/agent-evaluations",
  CURLOPT_RETURNTRANSFER => true,
  CURLOPT_ENCODING => "",
  CURLOPT_MAXREDIRS => 10,
  CURLOPT_TIMEOUT => 30,
  CURLOPT_HTTP_VERSION => CURL_HTTP_VERSION_1_1,
  CURLOPT_CUSTOMREQUEST => "POST",
  CURLOPT_POSTFIELDS => json_encode([
    'account_id' => 'acct_123',
    'data_plane_dataset_id' => 'dp_dataset_123',
    'description' => 'Metadata-only control-plane request',
    'metadata' => [
        'suite' => 'golden'
    ],
    'name' => 'Agent judge eval',
    'projection_policy_id' => 'policy_eval_projection_v1',
    'tags' => [
        'two-plane',
        'mvp'
    ],
    'tasks' => [
        [
                'alias' => 'mock_judge',
                'config' => [
                                'judge_adapter' => 'mock'
                ],
                'type' => 'auto_evaluation.agent'
        ]
    ],
    'tenant_id' => 'tenant_123'
  ]),
  CURLOPT_HTTPHEADER => [
    "Content-Type: application/json",
    "x-api-key: <api-key>"
  ],
]);

$response = curl_exec($curl);
$err = curl_error($curl);

curl_close($curl);

if ($err) {
  echo "cURL Error #:" . $err;
} else {
  echo $response;
}

package main

import (
	"fmt"
	"strings"
	"net/http"
	"io"
)

func main() {

	url := "https://api.egp.scale.com/v5/agent-evaluations"

	payload := strings.NewReader("{\n  \"account_id\": \"acct_123\",\n  \"data_plane_dataset_id\": \"dp_dataset_123\",\n  \"description\": \"Metadata-only control-plane request\",\n  \"metadata\": {\n    \"suite\": \"golden\"\n  },\n  \"name\": \"Agent judge eval\",\n  \"projection_policy_id\": \"policy_eval_projection_v1\",\n  \"tags\": [\n    \"two-plane\",\n    \"mvp\"\n  ],\n  \"tasks\": [\n    {\n      \"alias\": \"mock_judge\",\n      \"config\": {\n        \"judge_adapter\": \"mock\"\n      },\n      \"type\": \"auto_evaluation.agent\"\n    }\n  ],\n  \"tenant_id\": \"tenant_123\"\n}")

	req, _ := http.NewRequest("POST", url, payload)

	req.Header.Add("x-api-key", "<api-key>")
	req.Header.Add("Content-Type", "application/json")

	res, _ := http.DefaultClient.Do(req)

	defer res.Body.Close()
	body, _ := io.ReadAll(res.Body)

	fmt.Println(string(body))

}

HttpResponse<String> response = Unirest.post("https://api.egp.scale.com/v5/agent-evaluations")
  .header("x-api-key", "<api-key>")
  .header("Content-Type", "application/json")
  .body("{\n  \"account_id\": \"acct_123\",\n  \"data_plane_dataset_id\": \"dp_dataset_123\",\n  \"description\": \"Metadata-only control-plane request\",\n  \"metadata\": {\n    \"suite\": \"golden\"\n  },\n  \"name\": \"Agent judge eval\",\n  \"projection_policy_id\": \"policy_eval_projection_v1\",\n  \"tags\": [\n    \"two-plane\",\n    \"mvp\"\n  ],\n  \"tasks\": [\n    {\n      \"alias\": \"mock_judge\",\n      \"config\": {\n        \"judge_adapter\": \"mock\"\n      },\n      \"type\": \"auto_evaluation.agent\"\n    }\n  ],\n  \"tenant_id\": \"tenant_123\"\n}")
  .asString();

require 'uri'
require 'net/http'

url = URI("https://api.egp.scale.com/v5/agent-evaluations")

http = Net::HTTP.new(url.host, url.port)
http.use_ssl = true

request = Net::HTTP::Post.new(url)
request["x-api-key"] = '<api-key>'
request["Content-Type"] = 'application/json'
request.body = "{\n  \"account_id\": \"acct_123\",\n  \"data_plane_dataset_id\": \"dp_dataset_123\",\n  \"description\": \"Metadata-only control-plane request\",\n  \"metadata\": {\n    \"suite\": \"golden\"\n  },\n  \"name\": \"Agent judge eval\",\n  \"projection_policy_id\": \"policy_eval_projection_v1\",\n  \"tags\": [\n    \"two-plane\",\n    \"mvp\"\n  ],\n  \"tasks\": [\n    {\n      \"alias\": \"mock_judge\",\n      \"config\": {\n        \"judge_adapter\": \"mock\"\n      },\n      \"type\": \"auto_evaluation.agent\"\n    }\n  ],\n  \"tenant_id\": \"tenant_123\"\n}"

response = http.request(request)
puts response.read_body

{
  "id": "<string>",
  "run_id": "<string>",
  "name": "<string>",
  "progress": {
    "workflows": [
      {
        "name": "<string>",
        "status": "pending"
      }
    ],
    "total_items": 0,
    "completed_items": 0
  },
  "tasks": [
    {
      "alias": "<string>",
      "type": "<string>",
      "config": {},
      "depends_on": [
        "<string>"
      ]
    }
  ],
  "account_id": "<string>",
  "tenant_id": "<string>",
  "data_plane_dataset_id": "<string>",
  "projection_policy_id": "<string>",
  "created_at": "2023-11-07T05:31:56Z",
  "updated_at": "2023-11-07T05:31:56Z",
  "description": "<string>",
  "metadata": {},
  "tags": [
    "<string>"
  ],
  "data_plane_version": "<string>",
  "projection_freshness_seconds": 123,
  "error_count": 0
}

{
  "detail": [
    {
      "loc": [
        "<string>"
      ],
      "msg": "<string>",
      "type": "<string>",
      "input": "<unknown>",
      "ctx": {}
    }
  ]
}

Agent Evaluations

Create agent evaluation

This is the control-plane entry point for the two-plane agent-eval system and is gated by the SGP_TWO_PLANE_AGENT_EVALS_ENABLED feature flag. The request is metadata-only: it must reference data that already lives in the data plane via data_plane_dataset_id and must supply at least one task and a projection_policy_id. Inline raw evaluation data is rejected — if the payload contains any forbidden raw-data field (e.g. prompt, output, response, dataset, rows, traces, credentials), validation fails before the request is forwarded. Use this instead of embedding evaluation rows in the request.

The call is proxied to the agent-eval control service; if that service is unreachable the endpoint returns 503. On success the returned record includes a run_id, a status, and a progress block (per-workflow status and item completion counts) that reflect the run as it executes over time — poll the get or results endpoints to observe completion rather than expecting a finished run on this response.

POST

agent-evaluations

curl --request POST \
  --url https://api.egp.scale.com/v5/agent-evaluations \
  --header 'Content-Type: application/json' \
  --header 'x-api-key: <api-key>' \
  --data '
{
  "account_id": "acct_123",
  "data_plane_dataset_id": "dp_dataset_123",
  "description": "Metadata-only control-plane request",
  "metadata": {
    "suite": "golden"
  },
  "name": "Agent judge eval",
  "projection_policy_id": "policy_eval_projection_v1",
  "tags": [
    "two-plane",
    "mvp"
  ],
  "tasks": [
    {
      "alias": "mock_judge",
      "config": {
        "judge_adapter": "mock"
      },
      "type": "auto_evaluation.agent"
    }
  ],
  "tenant_id": "tenant_123"
}
'

import requests

url = "https://api.egp.scale.com/v5/agent-evaluations"

payload = {
    "account_id": "acct_123",
    "data_plane_dataset_id": "dp_dataset_123",
    "description": "Metadata-only control-plane request",
    "metadata": { "suite": "golden" },
    "name": "Agent judge eval",
    "projection_policy_id": "policy_eval_projection_v1",
    "tags": ["two-plane", "mvp"],
    "tasks": [
        {
            "alias": "mock_judge",
            "config": { "judge_adapter": "mock" },
            "type": "auto_evaluation.agent"
        }
    ],
    "tenant_id": "tenant_123"
}
headers = {
    "x-api-key": "<api-key>",
    "Content-Type": "application/json"
}

response = requests.post(url, json=payload, headers=headers)

print(response.text)

const options = {
  method: 'POST',
  headers: {'x-api-key': '<api-key>', 'Content-Type': 'application/json'},
  body: JSON.stringify({
    account_id: 'acct_123',
    data_plane_dataset_id: 'dp_dataset_123',
    description: 'Metadata-only control-plane request',
    metadata: {suite: 'golden'},
    name: 'Agent judge eval',
    projection_policy_id: 'policy_eval_projection_v1',
    tags: ['two-plane', 'mvp'],
    tasks: [
      {
        alias: 'mock_judge',
        config: {judge_adapter: 'mock'},
        type: 'auto_evaluation.agent'
      }
    ],
    tenant_id: 'tenant_123'
  })
};

fetch('https://api.egp.scale.com/v5/agent-evaluations', options)
  .then(res => res.json())
  .then(res => console.log(res))
  .catch(err => console.error(err));

<?php

$curl = curl_init();

curl_setopt_array($curl, [
  CURLOPT_URL => "https://api.egp.scale.com/v5/agent-evaluations",
  CURLOPT_RETURNTRANSFER => true,
  CURLOPT_ENCODING => "",
  CURLOPT_MAXREDIRS => 10,
  CURLOPT_TIMEOUT => 30,
  CURLOPT_HTTP_VERSION => CURL_HTTP_VERSION_1_1,
  CURLOPT_CUSTOMREQUEST => "POST",
  CURLOPT_POSTFIELDS => json_encode([
    'account_id' => 'acct_123',
    'data_plane_dataset_id' => 'dp_dataset_123',
    'description' => 'Metadata-only control-plane request',
    'metadata' => [
        'suite' => 'golden'
    ],
    'name' => 'Agent judge eval',
    'projection_policy_id' => 'policy_eval_projection_v1',
    'tags' => [
        'two-plane',
        'mvp'
    ],
    'tasks' => [
        [
                'alias' => 'mock_judge',
                'config' => [
                                'judge_adapter' => 'mock'
                ],
                'type' => 'auto_evaluation.agent'
        ]
    ],
    'tenant_id' => 'tenant_123'
  ]),
  CURLOPT_HTTPHEADER => [
    "Content-Type: application/json",
    "x-api-key: <api-key>"
  ],
]);

$response = curl_exec($curl);
$err = curl_error($curl);

curl_close($curl);

if ($err) {
  echo "cURL Error #:" . $err;
} else {
  echo $response;
}

package main

import (
	"fmt"
	"strings"
	"net/http"
	"io"
)

func main() {

	url := "https://api.egp.scale.com/v5/agent-evaluations"

	payload := strings.NewReader("{\n  \"account_id\": \"acct_123\",\n  \"data_plane_dataset_id\": \"dp_dataset_123\",\n  \"description\": \"Metadata-only control-plane request\",\n  \"metadata\": {\n    \"suite\": \"golden\"\n  },\n  \"name\": \"Agent judge eval\",\n  \"projection_policy_id\": \"policy_eval_projection_v1\",\n  \"tags\": [\n    \"two-plane\",\n    \"mvp\"\n  ],\n  \"tasks\": [\n    {\n      \"alias\": \"mock_judge\",\n      \"config\": {\n        \"judge_adapter\": \"mock\"\n      },\n      \"type\": \"auto_evaluation.agent\"\n    }\n  ],\n  \"tenant_id\": \"tenant_123\"\n}")

	req, _ := http.NewRequest("POST", url, payload)

	req.Header.Add("x-api-key", "<api-key>")
	req.Header.Add("Content-Type", "application/json")

	res, _ := http.DefaultClient.Do(req)

	defer res.Body.Close()
	body, _ := io.ReadAll(res.Body)

	fmt.Println(string(body))

}

HttpResponse<String> response = Unirest.post("https://api.egp.scale.com/v5/agent-evaluations")
  .header("x-api-key", "<api-key>")
  .header("Content-Type", "application/json")
  .body("{\n  \"account_id\": \"acct_123\",\n  \"data_plane_dataset_id\": \"dp_dataset_123\",\n  \"description\": \"Metadata-only control-plane request\",\n  \"metadata\": {\n    \"suite\": \"golden\"\n  },\n  \"name\": \"Agent judge eval\",\n  \"projection_policy_id\": \"policy_eval_projection_v1\",\n  \"tags\": [\n    \"two-plane\",\n    \"mvp\"\n  ],\n  \"tasks\": [\n    {\n      \"alias\": \"mock_judge\",\n      \"config\": {\n        \"judge_adapter\": \"mock\"\n      },\n      \"type\": \"auto_evaluation.agent\"\n    }\n  ],\n  \"tenant_id\": \"tenant_123\"\n}")
  .asString();

require 'uri'
require 'net/http'

url = URI("https://api.egp.scale.com/v5/agent-evaluations")

http = Net::HTTP.new(url.host, url.port)
http.use_ssl = true

request = Net::HTTP::Post.new(url)
request["x-api-key"] = '<api-key>'
request["Content-Type"] = 'application/json'
request.body = "{\n  \"account_id\": \"acct_123\",\n  \"data_plane_dataset_id\": \"dp_dataset_123\",\n  \"description\": \"Metadata-only control-plane request\",\n  \"metadata\": {\n    \"suite\": \"golden\"\n  },\n  \"name\": \"Agent judge eval\",\n  \"projection_policy_id\": \"policy_eval_projection_v1\",\n  \"tags\": [\n    \"two-plane\",\n    \"mvp\"\n  ],\n  \"tasks\": [\n    {\n      \"alias\": \"mock_judge\",\n      \"config\": {\n        \"judge_adapter\": \"mock\"\n      },\n      \"type\": \"auto_evaluation.agent\"\n    }\n  ],\n  \"tenant_id\": \"tenant_123\"\n}"

response = http.request(request)
puts response.read_body

{
  "id": "<string>",
  "run_id": "<string>",
  "name": "<string>",
  "progress": {
    "workflows": [
      {
        "name": "<string>",
        "status": "pending"
      }
    ],
    "total_items": 0,
    "completed_items": 0
  },
  "tasks": [
    {
      "alias": "<string>",
      "type": "<string>",
      "config": {},
      "depends_on": [
        "<string>"
      ]
    }
  ],
  "account_id": "<string>",
  "tenant_id": "<string>",
  "data_plane_dataset_id": "<string>",
  "projection_policy_id": "<string>",
  "created_at": "2023-11-07T05:31:56Z",
  "updated_at": "2023-11-07T05:31:56Z",
  "description": "<string>",
  "metadata": {},
  "tags": [
    "<string>"
  ],
  "data_plane_version": "<string>",
  "projection_freshness_seconds": 123,
  "error_count": 0
}

{
  "detail": [
    {
      "loc": [
        "<string>"
      ],
      "msg": "<string>",
      "type": "<string>",
      "input": "<unknown>",
      "ctx": {}
    }
  ]
}

Authorizations

x-api-key

string

header

required

Headers

x-selected-account-id

string | null

Body

application/json

name

string

required

Minimum string length: 1

data_plane_dataset_id

string

required

Minimum string length: 1

tasks

AgentEvaluationTask · object[]

required

Minimum array length: 1

Show child attributes

projection_policy_id

string

required

Minimum string length: 1

description

string

Response

Successful Response

string

required

run_id

string

required

name

string

required

status

enum<string>

required

Available options:

pending,

running,

completed,

failed

progress

AgentEvaluationProgress · object

required

Show child attributes

tasks

AgentEvaluationTask · object[]

required

Show child attributes

account_id

string

required

tenant_id

string

required

data_plane_dataset_id

string

required

projection_policy_id

string

required

created_at

string<date-time>

required

updated_at

string<date-time>

required

description

string

metadata

Metadata · object