Create Eval Run

POST

eval-runs

Create Eval Run

curl --request POST \
  --url https://api.example.com/v1/eval-runs \
  --header 'Content-Type: application/json' \
  --data '
{
  "target": {
    "type": "<string>",
    "agent_id": "3c90c3cc-0d44-4b50-8888-8dd25736052a",
    "name": "<string>",
    "environment": "<string>",
    "agent": {}
  },
  "scenario_id": "3c90c3cc-0d44-4b50-8888-8dd25736052a",
  "tag": "<string>",
  "scenario": {
    "name": "<string>",
    "definition": {},
    "tool_mocks": {},
    "judge": {
      "provider": "ollama",
      "model": "<string>",
      "temperature": 1,
      "endpoint": "<string>"
    },
    "simulator": {
      "provider": "ollama",
      "model": "<string>",
      "temperature": 1,
      "endpoint": "<string>"
    }
  },
  "modality": "text",
  "iterations": 1
}
'

import requests

url = "https://api.example.com/v1/eval-runs"

payload = {
    "target": {
        "type": "<string>",
        "agent_id": "3c90c3cc-0d44-4b50-8888-8dd25736052a",
        "name": "<string>",
        "environment": "<string>",
        "agent": {}
    },
    "scenario_id": "3c90c3cc-0d44-4b50-8888-8dd25736052a",
    "tag": "<string>",
    "scenario": {
        "name": "<string>",
        "definition": {},
        "tool_mocks": {},
        "judge": {
            "provider": "ollama",
            "model": "<string>",
            "temperature": 1,
            "endpoint": "<string>"
        },
        "simulator": {
            "provider": "ollama",
            "model": "<string>",
            "temperature": 1,
            "endpoint": "<string>"
        }
    },
    "modality": "text",
    "iterations": 1
}
headers = {"Content-Type": "application/json"}

response = requests.post(url, json=payload, headers=headers)

print(response.text)

const options = {
  method: 'POST',
  headers: {'Content-Type': 'application/json'},
  body: JSON.stringify({
    target: {
      type: '<string>',
      agent_id: '3c90c3cc-0d44-4b50-8888-8dd25736052a',
      name: '<string>',
      environment: '<string>',
      agent: {}
    },
    scenario_id: '3c90c3cc-0d44-4b50-8888-8dd25736052a',
    tag: '<string>',
    scenario: {
      name: '<string>',
      definition: {},
      tool_mocks: {},
      judge: {provider: 'ollama', model: '<string>', temperature: 1, endpoint: '<string>'},
      simulator: {provider: 'ollama', model: '<string>', temperature: 1, endpoint: '<string>'}
    },
    modality: 'text',
    iterations: 1
  })
};

fetch('https://api.example.com/v1/eval-runs', options)
  .then(res => res.json())
  .then(res => console.log(res))
  .catch(err => console.error(err));

<?php

$curl = curl_init();

curl_setopt_array($curl, [
  CURLOPT_URL => "https://api.example.com/v1/eval-runs",
  CURLOPT_RETURNTRANSFER => true,
  CURLOPT_ENCODING => "",
  CURLOPT_MAXREDIRS => 10,
  CURLOPT_TIMEOUT => 30,
  CURLOPT_HTTP_VERSION => CURL_HTTP_VERSION_1_1,
  CURLOPT_CUSTOMREQUEST => "POST",
  CURLOPT_POSTFIELDS => json_encode([
    'target' => [
        'type' => '<string>',
        'agent_id' => '3c90c3cc-0d44-4b50-8888-8dd25736052a',
        'name' => '<string>',
        'environment' => '<string>',
        'agent' => [
                
        ]
    ],
    'scenario_id' => '3c90c3cc-0d44-4b50-8888-8dd25736052a',
    'tag' => '<string>',
    'scenario' => [
        'name' => '<string>',
        'definition' => [
                
        ],
        'tool_mocks' => [
                
        ],
        'judge' => [
                'provider' => 'ollama',
                'model' => '<string>',
                'temperature' => 1,
                'endpoint' => '<string>'
        ],
        'simulator' => [
                'provider' => 'ollama',
                'model' => '<string>',
                'temperature' => 1,
                'endpoint' => '<string>'
        ]
    ],
    'modality' => 'text',
    'iterations' => 1
  ]),
  CURLOPT_HTTPHEADER => [
    "Content-Type: application/json"
  ],
]);

$response = curl_exec($curl);
$err = curl_error($curl);

curl_close($curl);

if ($err) {
  echo "cURL Error #:" . $err;
} else {
  echo $response;
}

package main

import (
	"fmt"
	"strings"
	"net/http"
	"io"
)

func main() {

	url := "https://api.example.com/v1/eval-runs"

	payload := strings.NewReader("{\n  \"target\": {\n    \"type\": \"<string>\",\n    \"agent_id\": \"3c90c3cc-0d44-4b50-8888-8dd25736052a\",\n    \"name\": \"<string>\",\n    \"environment\": \"<string>\",\n    \"agent\": {}\n  },\n  \"scenario_id\": \"3c90c3cc-0d44-4b50-8888-8dd25736052a\",\n  \"tag\": \"<string>\",\n  \"scenario\": {\n    \"name\": \"<string>\",\n    \"definition\": {},\n    \"tool_mocks\": {},\n    \"judge\": {\n      \"provider\": \"ollama\",\n      \"model\": \"<string>\",\n      \"temperature\": 1,\n      \"endpoint\": \"<string>\"\n    },\n    \"simulator\": {\n      \"provider\": \"ollama\",\n      \"model\": \"<string>\",\n      \"temperature\": 1,\n      \"endpoint\": \"<string>\"\n    }\n  },\n  \"modality\": \"text\",\n  \"iterations\": 1\n}")

	req, _ := http.NewRequest("POST", url, payload)

	req.Header.Add("Content-Type", "application/json")

	res, _ := http.DefaultClient.Do(req)

	defer res.Body.Close()
	body, _ := io.ReadAll(res.Body)

	fmt.Println(string(body))

}

HttpResponse<String> response = Unirest.post("https://api.example.com/v1/eval-runs")
  .header("Content-Type", "application/json")
  .body("{\n  \"target\": {\n    \"type\": \"<string>\",\n    \"agent_id\": \"3c90c3cc-0d44-4b50-8888-8dd25736052a\",\n    \"name\": \"<string>\",\n    \"environment\": \"<string>\",\n    \"agent\": {}\n  },\n  \"scenario_id\": \"3c90c3cc-0d44-4b50-8888-8dd25736052a\",\n  \"tag\": \"<string>\",\n  \"scenario\": {\n    \"name\": \"<string>\",\n    \"definition\": {},\n    \"tool_mocks\": {},\n    \"judge\": {\n      \"provider\": \"ollama\",\n      \"model\": \"<string>\",\n      \"temperature\": 1,\n      \"endpoint\": \"<string>\"\n    },\n    \"simulator\": {\n      \"provider\": \"ollama\",\n      \"model\": \"<string>\",\n      \"temperature\": 1,\n      \"endpoint\": \"<string>\"\n    }\n  },\n  \"modality\": \"text\",\n  \"iterations\": 1\n}")
  .asString();

require 'uri'
require 'net/http'

url = URI("https://api.example.com/v1/eval-runs")

http = Net::HTTP.new(url.host, url.port)
http.use_ssl = true

request = Net::HTTP::Post.new(url)
request["Content-Type"] = 'application/json'
request.body = "{\n  \"target\": {\n    \"type\": \"<string>\",\n    \"agent_id\": \"3c90c3cc-0d44-4b50-8888-8dd25736052a\",\n    \"name\": \"<string>\",\n    \"environment\": \"<string>\",\n    \"agent\": {}\n  },\n  \"scenario_id\": \"3c90c3cc-0d44-4b50-8888-8dd25736052a\",\n  \"tag\": \"<string>\",\n  \"scenario\": {\n    \"name\": \"<string>\",\n    \"definition\": {},\n    \"tool_mocks\": {},\n    \"judge\": {\n      \"provider\": \"ollama\",\n      \"model\": \"<string>\",\n      \"temperature\": 1,\n      \"endpoint\": \"<string>\"\n    },\n    \"simulator\": {\n      \"provider\": \"ollama\",\n      \"model\": \"<string>\",\n      \"temperature\": 1,\n      \"endpoint\": \"<string>\"\n    }\n  },\n  \"modality\": \"text\",\n  \"iterations\": 1\n}"

response = http.request(request)
puts response.read_body

{}

{
  "detail": [
    {
      "loc": [
        "<string>"
      ],
      "msg": "<string>",
      "type": "<string>",
      "input": "<unknown>",
      "ctx": {}
    }
  ]
}

Headers

authorization

string | null

Body

application/json

Run one stored scenario, every scenario carrying a tag (#75), or a scenario supplied inline (#77).

Exactly one of the three. A tag fans out to one run per matching scenario, all sharing a batch id, so one request has one readable verdict — which is what the CLI's single exit code is built on.

extra="forbid" is what makes "no run-level judge" a rule (#119). A run is what it was queued as: a request that could swap the judge would make two runs of one scenario incomparable with nothing on either row explaining why, and quietly ignoring the field is the same outcome with the author believing otherwise. Name it on the scenario — including the inline one above, which is where a run-shaped judge: was aiming.

target

EvalTarget · object

required

What a run points at (#74).

Three forms, and the difference between the first two matters: an agent row is one immutable version, so agent pins a version forever — a scenario targeting it silently stops testing production the moment the next version is published. agent_name resolves to whatever is published at run time. inline has no row at all, which is how a prompt or model change is evaluated before publishing it, and the sandbox a scenario is pointed at when its tools have real side effects.

Show child attributes

scenario_id

string<uuid> | null

tag

string | null

Required string length: 1 - 64

scenario

InlineScenario · object | null

A scenario supplied on the run instead of stored first (#77).

This is what a local file holds, and why the CLI needs no format of its own: the file is the API request body, validated by the same parser a stored scenario is. Nothing is written to eval_scenarios — the run's scenario_id is null and its resolved_scenario snapshot is the record, which is the same shape ADR-0017 uses for an inline agent.

Show child attributes

modality

enum<string>

default:text

Whether the conversation is spoken or typed.

text skips STT and TTS entirely, so it is fast and free and covers the smallest share of a voice platform's risk; audio is the only mode that reaches the provider-connect path where most regressions have lived.

Available options:

text,

audio

iterations

integer

default:1

Required range: x >= 1

Response

Successful Response

The response is of type Response Create Eval Run V1 Eval Runs Post · object.

Create Eval Run

curl --request POST \
  --url https://api.example.com/v1/eval-runs \
  --header 'Content-Type: application/json' \
  --data '
{
  "target": {
    "type": "<string>",
    "agent_id": "3c90c3cc-0d44-4b50-8888-8dd25736052a",
    "name": "<string>",
    "environment": "<string>",
    "agent": {}
  },
  "scenario_id": "3c90c3cc-0d44-4b50-8888-8dd25736052a",
  "tag": "<string>",
  "scenario": {
    "name": "<string>",
    "definition": {},
    "tool_mocks": {},
    "judge": {
      "provider": "ollama",
      "model": "<string>",
      "temperature": 1,
      "endpoint": "<string>"
    },
    "simulator": {
      "provider": "ollama",
      "model": "<string>",
      "temperature": 1,
      "endpoint": "<string>"
    }
  },
  "modality": "text",
  "iterations": 1
}
'

import requests

url = "https://api.example.com/v1/eval-runs"

payload = {
    "target": {
        "type": "<string>",
        "agent_id": "3c90c3cc-0d44-4b50-8888-8dd25736052a",
        "name": "<string>",
        "environment": "<string>",
        "agent": {}
    },
    "scenario_id": "3c90c3cc-0d44-4b50-8888-8dd25736052a",
    "tag": "<string>",
    "scenario": {
        "name": "<string>",
        "definition": {},
        "tool_mocks": {},
        "judge": {
            "provider": "ollama",
            "model": "<string>",
            "temperature": 1,
            "endpoint": "<string>"
        },
        "simulator": {
            "provider": "ollama",
            "model": "<string>",
            "temperature": 1,
            "endpoint": "<string>"
        }
    },
    "modality": "text",
    "iterations": 1
}
headers = {"Content-Type": "application/json"}

response = requests.post(url, json=payload, headers=headers)

print(response.text)

const options = {
  method: 'POST',
  headers: {'Content-Type': 'application/json'},
  body: JSON.stringify({
    target: {
      type: '<string>',
      agent_id: '3c90c3cc-0d44-4b50-8888-8dd25736052a',
      name: '<string>',
      environment: '<string>',
      agent: {}
    },
    scenario_id: '3c90c3cc-0d44-4b50-8888-8dd25736052a',
    tag: '<string>',
    scenario: {
      name: '<string>',
      definition: {},
      tool_mocks: {},
      judge: {provider: 'ollama', model: '<string>', temperature: 1, endpoint: '<string>'},
      simulator: {provider: 'ollama', model: '<string>', temperature: 1, endpoint: '<string>'}
    },
    modality: 'text',
    iterations: 1
  })
};

fetch('https://api.example.com/v1/eval-runs', options)
  .then(res => res.json())
  .then(res => console.log(res))
  .catch(err => console.error(err));

<?php

$curl = curl_init();

curl_setopt_array($curl, [
  CURLOPT_URL => "https://api.example.com/v1/eval-runs",
  CURLOPT_RETURNTRANSFER => true,
  CURLOPT_ENCODING => "",
  CURLOPT_MAXREDIRS => 10,
  CURLOPT_TIMEOUT => 30,
  CURLOPT_HTTP_VERSION => CURL_HTTP_VERSION_1_1,
  CURLOPT_CUSTOMREQUEST => "POST",
  CURLOPT_POSTFIELDS => json_encode([
    'target' => [
        'type' => '<string>',
        'agent_id' => '3c90c3cc-0d44-4b50-8888-8dd25736052a',
        'name' => '<string>',
        'environment' => '<string>',
        'agent' => [
                
        ]
    ],
    'scenario_id' => '3c90c3cc-0d44-4b50-8888-8dd25736052a',
    'tag' => '<string>',
    'scenario' => [
        'name' => '<string>',
        'definition' => [
                
        ],
        'tool_mocks' => [
                
        ],
        'judge' => [
                'provider' => 'ollama',
                'model' => '<string>',
                'temperature' => 1,
                'endpoint' => '<string>'
        ],
        'simulator' => [
                'provider' => 'ollama',
                'model' => '<string>',
                'temperature' => 1,
                'endpoint' => '<string>'
        ]
    ],
    'modality' => 'text',
    'iterations' => 1
  ]),
  CURLOPT_HTTPHEADER => [
    "Content-Type: application/json"
  ],
]);

$response = curl_exec($curl);
$err = curl_error($curl);

curl_close($curl);

if ($err) {
  echo "cURL Error #:" . $err;
} else {
  echo $response;
}

package main

import (
	"fmt"
	"strings"
	"net/http"
	"io"
)

func main() {

	url := "https://api.example.com/v1/eval-runs"

	payload := strings.NewReader("{\n  \"target\": {\n    \"type\": \"<string>\",\n    \"agent_id\": \"3c90c3cc-0d44-4b50-8888-8dd25736052a\",\n    \"name\": \"<string>\",\n    \"environment\": \"<string>\",\n    \"agent\": {}\n  },\n  \"scenario_id\": \"3c90c3cc-0d44-4b50-8888-8dd25736052a\",\n  \"tag\": \"<string>\",\n  \"scenario\": {\n    \"name\": \"<string>\",\n    \"definition\": {},\n    \"tool_mocks\": {},\n    \"judge\": {\n      \"provider\": \"ollama\",\n      \"model\": \"<string>\",\n      \"temperature\": 1,\n      \"endpoint\": \"<string>\"\n    },\n    \"simulator\": {\n      \"provider\": \"ollama\",\n      \"model\": \"<string>\",\n      \"temperature\": 1,\n      \"endpoint\": \"<string>\"\n    }\n  },\n  \"modality\": \"text\",\n  \"iterations\": 1\n}")

	req, _ := http.NewRequest("POST", url, payload)

	req.Header.Add("Content-Type", "application/json")

	res, _ := http.DefaultClient.Do(req)

	defer res.Body.Close()
	body, _ := io.ReadAll(res.Body)

	fmt.Println(string(body))

}

HttpResponse<String> response = Unirest.post("https://api.example.com/v1/eval-runs")
  .header("Content-Type", "application/json")
  .body("{\n  \"target\": {\n    \"type\": \"<string>\",\n    \"agent_id\": \"3c90c3cc-0d44-4b50-8888-8dd25736052a\",\n    \"name\": \"<string>\",\n    \"environment\": \"<string>\",\n    \"agent\": {}\n  },\n  \"scenario_id\": \"3c90c3cc-0d44-4b50-8888-8dd25736052a\",\n  \"tag\": \"<string>\",\n  \"scenario\": {\n    \"name\": \"<string>\",\n    \"definition\": {},\n    \"tool_mocks\": {},\n    \"judge\": {\n      \"provider\": \"ollama\",\n      \"model\": \"<string>\",\n      \"temperature\": 1,\n      \"endpoint\": \"<string>\"\n    },\n    \"simulator\": {\n      \"provider\": \"ollama\",\n      \"model\": \"<string>\",\n      \"temperature\": 1,\n      \"endpoint\": \"<string>\"\n    }\n  },\n  \"modality\": \"text\",\n  \"iterations\": 1\n}")
  .asString();

require 'uri'
require 'net/http'

url = URI("https://api.example.com/v1/eval-runs")

http = Net::HTTP.new(url.host, url.port)
http.use_ssl = true

request = Net::HTTP::Post.new(url)
request["Content-Type"] = 'application/json'
request.body = "{\n  \"target\": {\n    \"type\": \"<string>\",\n    \"agent_id\": \"3c90c3cc-0d44-4b50-8888-8dd25736052a\",\n    \"name\": \"<string>\",\n    \"environment\": \"<string>\",\n    \"agent\": {}\n  },\n  \"scenario_id\": \"3c90c3cc-0d44-4b50-8888-8dd25736052a\",\n  \"tag\": \"<string>\",\n  \"scenario\": {\n    \"name\": \"<string>\",\n    \"definition\": {},\n    \"tool_mocks\": {},\n    \"judge\": {\n      \"provider\": \"ollama\",\n      \"model\": \"<string>\",\n      \"temperature\": 1,\n      \"endpoint\": \"<string>\"\n    },\n    \"simulator\": {\n      \"provider\": \"ollama\",\n      \"model\": \"<string>\",\n      \"temperature\": 1,\n      \"endpoint\": \"<string>\"\n    }\n  },\n  \"modality\": \"text\",\n  \"iterations\": 1\n}"

response = http.request(request)
puts response.read_body

{}

{
  "detail": [
    {
      "loc": [
        "<string>"
      ],
      "msg": "<string>",
      "type": "<string>",
      "input": "<unknown>",
      "ctx": {}
    }
  ]
}

Overview

Projects

API Keys

Agents

Phone Numbers

Calls

Call Control

Chat

Knowledge Bases

Takeaways

Evals

Tools

Webhooks

WebRTC

Health

Headers

Body

Response