Prepare and queue test run

Create a Supabase-backed v1 test run, create linked evaluation_runs, and enqueue judge evaluation work.

POST/v1/projects/{projectId}/test-runs

Authorization

AuthorizationPersonal API key · headerrequired

Your personal API key from Settings > Personal API Keys, sent as Bearer <key>.

X-Organization-IdOrganization ID · headerrequired

The organization to act in. Copy it with Copy organization ID in the organization menu or the settings sidebar.

Path parameters

projectIdstringrequired

Request body

requiredapplication/json

test_run_idstring<uuid>

Reuse an owned, pending platform agent run instead of creating a new run.

test_config_idstringrequired

namestring

dataset_version_idstring

dataset_version_numbernumber

judge_versionsobject[]

Show properties

Array of object

judge_idstring

namestring

versionstring

major_versionnumber

minor_versionnumber

sourcestring

Allowed:sdkclimcpplatformlegacy

agent_errorsobject[]

Show properties

Array of object

example_idstringrequired

errorstringrequired

min length 1

agent_tracesobject[]

Show properties

Array of object

example_idstringrequired

agent_offline_trace_idstringrequired

Responses

200

Response for status 200

test_runobjectrequired

Show properties

idstringrequired

organization_idstring

project_idstringrequired

user_idstringrequired

test_config_idstringrequired

dataset_idstringrequired

dataset_version_idstringrequired

sourcestringrequired

Allowed:sdkclimcpplatformlegacy

statusstringrequired

Allowed:pendingrunningcompletederrorcancelled

namestringrequired

error_messagestring | null

Show properties

Any of:

string

string

null

null

created_atstring

updated_atstring

started_atstring | null

Show properties

Any of:

string

string

null

null

completed_atstring | null

Show properties

Any of:

string

string

null

null

datasetobjectrequired

Show properties

dataset_idstringrequired

organization_idstring

project_idstring

namestringrequired

schemaobject | null

Show properties

Any of:

object

object

null

null

current_versionnumber

created_atstring

updated_atstring

creator_idstring | nullrequired

Show properties

Any of:

string

string

null

null

entriesnumber

test_config_countnumber

dataset_versionobjectrequired

Show properties

version_idstringrequired

dataset_idstringrequired

organization_idstringrequired

project_idstringrequired

version_numbernumberrequired

created_atstringrequired

user_idstring | nullrequired

Show properties

Any of:

string

string

null

null

item_countnumberrequired

examplesobject[]required

Show properties

Array of object

example_idstringrequired

organization_idstring

project_idstring

dataobjectrequired

metadataobject | null

Show properties

Any of:

object

object

null

null

managed_metadataobjectrequired

created_atstring

user_idstringrequired

judgesobject[]required

Show properties

Array of object

judge_idstringrequired

judge_namestringrequired

judge_typestringrequired

score_typestringrequired

judge_major_versionnumberrequired

judge_minor_versionnumberrequired

evaluation_runsobject[]required

Show properties

Array of object

run_idstringrequired

judge_namestringrequired

example_idstringrequired

test_run_idstringrequired

judge_idstringrequired

judge_major_versionnumberrequired

judge_minor_versionnumberrequired

ui_results_urlstringrequired

402

Response for status 402

errorstringrequired

messagestring

codestring

detailsstring[]

404

Response for status 404

errorstringrequired

messagestring

codestring

detailsstring[]

409

Response for status 409

errorstringrequired

messagestring

codestring

detailsstring[]

422

Response for status 422

errorstringrequired

messagestring

codestring

detailsstring[]

429

Response for status 429

errorstringrequired

messagestring

codestring

detailsstring[]

Try it

Server

Base URLCustom base URLOverrides the selected server when set.

Authorization

Personal API keyOrganization IDRemember on this deviceInclude my values in samples

Parameters

projectIdpath*

Bodyapplication/json

Request body

Request

curl -X POST 'https://api.judgmentlabs.ai/v1/projects/YOUR_PROJECT_ID/test-runs' \
  -H 'Authorization: Bearer YOUR_API_KEY' \
  -H 'X-Organization-Id: YOUR_ORGANIZATION_ID' \
  -H 'Accept: application/json' \
  -H 'Content-Type: application/json' \
  -d '{
  "test_run_id": "01dbc221-8963-4bd7-8591-4a1f9d286862",
  "test_config_id": "string",
  "name": "string",
  "dataset_version_id": "string",
  "dataset_version_number": 0,
  "judge_versions": [
    {
      "judge_id": "string",
      "name": "string",
      "version": "string",
      "major_version": 0,
      "minor_version": 0
    }
  ],
  "source": "sdk",
  "agent_errors": [
    {
      "example_id": "string",
      "error": "string"
    }
  ],
  "agent_traces": [
    {
      "example_id": "string",
      "agent_offline_trace_id": "string"
    }
  ]
}'

const response = await fetch("https://api.judgmentlabs.ai/v1/projects/YOUR_PROJECT_ID/test-runs", {
  method: "POST",
  headers: {
    "Authorization": "Bearer YOUR_API_KEY",
    "X-Organization-Id": "YOUR_ORGANIZATION_ID",
    "Accept": "application/json",
    "Content-Type": "application/json"
  },
  body: "{\n  \"test_run_id\": \"01dbc221-8963-4bd7-8591-4a1f9d286862\",\n  \"test_config_id\": \"string\",\n  \"name\": \"string\",\n  \"dataset_version_id\": \"string\",\n  \"dataset_version_number\": 0,\n  \"judge_versions\": [\n    {\n      \"judge_id\": \"string\",\n      \"name\": \"string\",\n      \"version\": \"string\",\n      \"major_version\": 0,\n      \"minor_version\": 0\n    }\n  ],\n  \"source\": \"sdk\",\n  \"agent_errors\": [\n    {\n      \"example_id\": \"string\",\n      \"error\": \"string\"\n    }\n  ],\n  \"agent_traces\": [\n    {\n      \"example_id\": \"string\",\n      \"agent_offline_trace_id\": \"string\"\n    }\n  ]\n}"
});

import requests

response = requests.post(
    "https://api.judgmentlabs.ai/v1/projects/YOUR_PROJECT_ID/test-runs",
    headers={
        "Authorization": "Bearer YOUR_API_KEY",
        "X-Organization-Id": "YOUR_ORGANIZATION_ID",
        "Accept": "application/json",
        "Content-Type": "application/json"
    },
    data="{\n  \"test_run_id\": \"01dbc221-8963-4bd7-8591-4a1f9d286862\",\n  \"test_config_id\": \"string\",\n  \"name\": \"string\",\n  \"dataset_version_id\": \"string\",\n  \"dataset_version_number\": 0,\n  \"judge_versions\": [\n    {\n      \"judge_id\": \"string\",\n      \"name\": \"string\",\n      \"version\": \"string\",\n      \"major_version\": 0,\n      \"minor_version\": 0\n    }\n  ],\n  \"source\": \"sdk\",\n  \"agent_errors\": [\n    {\n      \"example_id\": \"string\",\n      \"error\": \"string\"\n    }\n  ],\n  \"agent_traces\": [\n    {\n      \"example_id\": \"string\",\n      \"agent_offline_trace_id\": \"string\"\n    }\n  ]\n}",
)

Response

{
  "test_run": {
    "id": "string",
    "organization_id": "string",
    "project_id": "string",
    "user_id": "string",
    "test_config_id": "string",
    "dataset_id": "string",
    "dataset_version_id": "string",
    "source": "sdk",
    "status": "pending",
    "name": "string",
    "error_message": "string",
    "created_at": "string",
    "updated_at": "string",
    "started_at": "string",
    "completed_at": "string"
  },
  "dataset": {
    "dataset_id": "string",
    "organization_id": "string",
    "project_id": "string",
    "name": "string",
    "schema": {},
    "current_version": 0,
    "created_at": "string",
    "updated_at": "string",
    "creator_id": "string",
    "entries": 0,
    "test_config_count": 0
  },
  "dataset_version": {
    "version_id": "string",
    "dataset_id": "string",
    "organization_id": "string",
    "project_id": "string",
    "version_number": 0,
    "created_at": "string",
    "user_id": "string",
    "item_count": 0
  },
  "examples": [
    {
      "example_id": "string",
      "organization_id": "string",
      "project_id": "string",
      "data": {},
      "metadata": {},
      "managed_metadata": {},
      "created_at": "string",
      "user_id": "string"
    }
  ],
  "judges": [
    {
      "judge_id": "string",
      "judge_name": "string",
      "judge_type": "string",
      "score_type": "string",
      "judge_major_version": 0,
      "judge_minor_version": 0
    }
  ],
  "evaluation_runs": [
    {
      "run_id": "string",
      "judge_name": "string",
      "example_id": "string",
      "test_run_id": "string",
      "judge_id": "string",
      "judge_major_version": 0,
      "judge_minor_version": 0
    }
  ],
  "ui_results_url": "string"
}

{
  "error": "string",
  "message": "string",
  "code": "string",
  "details": [
    "string"
  ]
}

{
  "error": "string",
  "message": "string",
  "code": "string",
  "details": [
    "string"
  ]
}

{
  "error": "string",
  "message": "string",
  "code": "string",
  "details": [
    "string"
  ]
}

{
  "error": "string",
  "message": "string",
  "code": "string",
  "details": [
    "string"
  ]
}

{
  "error": "string",
  "message": "string",
  "code": "string",
  "details": [
    "string"
  ]
}