JudgmentApiClient

Client for the public /v1 API: one method per Judgment MCP tool.

Reach it as client.api on a Judgeval client, which supplies the credentials. Methods on project resources take the project’s ID, which list_projects() returns. Request bodies are dicts and responses are dicts.

from judgeval import Judgeval

client = Judgeval(project_name="search-assistant")

projects = client.api.list_projects()["projects"]
project_id = next(
    p["project_id"]
    for p in projects
    if p["project_name"] == "search-assistant"
)

judges = client.api.list_judges(project_id)
result = client.api.sql_query(
    project_id,
    {"sql": "SELECT trace_id, status FROM telemetry.traces LIMIT 5"},
)
print(result["rows"])

list_organizations()

List organization memberships (GET /v1/organizations/memberships).

client.api.list_organizations()

def list_organizations() -> ListOrganizationsResponse:

Returns

ListOrganizationsResponse

create_project()

Add new project (POST /v1/projects).

client.api.create_project({"project_name": ...})

def create_project(payload) -> AddProjectResponse:

Parameters

Returns

AddProjectResponse

get_artifact_revision()

Get artifact revision (GET /v1/projects/{projectId}/artifact-revisions).

client.api.get_artifact_revision(project_id)

def get_artifact_revision(project_id, *, revision_id=None, artifact_id=None, version=None) -> GetArtifactRevisionResponse:

Parameters

Returns

GetArtifactRevisionResponse

create_artifact()

Create artifact (POST /v1/projects/{projectId}/artifacts).

client.api.create_artifact(project_id, {"scope": ..., "name": ..., "filename": ..., "source_code": ...})

def create_artifact(project_id, payload) -> CreateArtifactResponse:

Parameters

Returns

CreateArtifactResponse

create_supabase_artifact_revision()

Create artifact revision (POST /v1/projects/{projectId}/artifacts/{artifactId}/revisions).

client.api.create_supabase_artifact_revision(project_id, artifact_id, {"version": ..., "filename": ..., "source_code": ...})

def create_supabase_artifact_revision(project_id, artifact_id, payload) -> CreateSupabaseArtifactRevisionResponse:

Parameters

Returns

CreateSupabaseArtifactRevisionResponse

test_optimistic_eval_candidate()

Test optimistic eval candidate (POST /v1/projects/{projectId}/artifacts/optimistic-eval-tests).

client.api.test_optimistic_eval_candidate(project_id, {"trace_id": ..., "source_code": ..., "judge_id": ..., "judge_version_id": ...})

def test_optimistic_eval_candidate(project_id, payload) -> TestOptimisticEvalCandidateResponse:

Parameters

Returns

TestOptimisticEvalCandidateResponse

test_preprocessing_candidate()

Test preprocessing candidate (POST /v1/projects/{projectId}/artifacts/preprocessing-tests).

client.api.test_preprocessing_candidate(project_id, {"trace_id": ..., "source_code": ...})

def test_preprocessing_candidate(project_id, payload) -> TestPreprocessingCandidateResponse:

Parameters

Returns

TestPreprocessingCandidateResponse

list_mappers()

List attribute mappers (GET /v1/projects/{projectId}/attribute-mappers).

client.api.list_mappers(project_id)

def list_mappers(project_id) -> ListMappersResponse:

Parameters

Returns

ListMappersResponse

create_mapper()

Create attribute mapper (POST /v1/projects/{projectId}/attribute-mappers).

client.api.create_mapper(project_id, {"name": ..., "rules": ...})

def create_mapper(project_id, payload) -> CreateMapperResponse:

Parameters

Returns

CreateMapperResponse

get_mapper()

Get attribute mapper (GET /v1/projects/{projectId}/attribute-mappers/{configId}).

client.api.get_mapper(project_id, config_id)

def get_mapper(project_id, config_id) -> GetMapperResponse:

Parameters

Returns

GetMapperResponse

update_mapper()

Update attribute mapper (PATCH /v1/projects/{projectId}/attribute-mappers/{configId}).

client.api.update_mapper(project_id, config_id, {})

def update_mapper(project_id, config_id, payload) -> UpdateMapperResponse:

Parameters

Returns

UpdateMapperResponse

delete_mapper()

Delete attribute mapper (DELETE /v1/projects/{projectId}/attribute-mappers/{configId}).

client.api.delete_mapper(project_id, config_id)

def delete_mapper(project_id, config_id) -> DeleteMapperResponse:

Parameters

Returns

DeleteMapperResponse

update_judge_label()

Update judge label (PATCH /v1/projects/{projectId}/behaviors/{behaviorId}).

client.api.update_judge_label(project_id, behavior_id, {})

def update_judge_label(project_id, behavior_id, payload) -> UpdateJudgeLabelResponse:

Parameters

Returns

UpdateJudgeLabelResponse

get_judge_label()

Get judge label (GET /v1/projects/{projectId}/behaviors/{behaviorId}/analytics).

client.api.get_judge_label(project_id, behavior_id)

def get_judge_label(project_id, behavior_id, *, start_date=None, end_date=None) -> GetJudgeLabelResponse:

Parameters

Returns

GetJudgeLabelResponse

get_conversation_display_rules()

Get conversation display rules (GET /v1/projects/{projectId}/conversation-display-rules).

client.api.get_conversation_display_rules(project_id)

def get_conversation_display_rules(project_id) -> GetConversationDisplayRulesResponse:

Parameters

Returns

GetConversationDisplayRulesResponse

set_conversation_display_rules()

Replace conversation display rules (PUT /v1/projects/{projectId}/conversation-display-rules).

client.api.set_conversation_display_rules(project_id, {"rules": ...})

def set_conversation_display_rules(project_id, payload) -> SetConversationDisplayRulesResponse:

Parameters

Returns

SetConversationDisplayRulesResponse

create_dataset()

Create offline-test dataset (POST /v1/projects/{projectId}/datasets).

client.api.create_dataset(project_id, {"name": ..., "schema": ...})

def create_dataset(project_id, payload) -> CreateOfflineDatasetResponse:

Parameters

Returns

CreateOfflineDatasetResponse

list_datasets()

Fetch offline-test datasets (GET /v1/projects/{projectId}/datasets).

client.api.list_datasets(project_id)

def list_datasets(project_id) -> PullAllOfflineDatasetsResponse:

Parameters

Returns

PullAllOfflineDatasetsResponse

delete_dataset()

Delete offline-test dataset (DELETE /v1/projects/{projectId}/datasets/{datasetIdentifier}).

client.api.delete_dataset(project_id, dataset_identifier)

def delete_dataset(project_id, dataset_identifier) -> DeleteOfflineDatasetResponse:

Parameters

Returns

DeleteOfflineDatasetResponse

add_examples_to_dataset()

Insert examples to offline-test dataset (POST /v1/projects/{projectId}/datasets/{datasetIdentifier}/examples).

client.api.add_examples_to_dataset(project_id, dataset_identifier, {"examples": ...})

def add_examples_to_dataset(project_id, dataset_identifier, payload) -> AddDatasetItemsResponse:

Parameters

Returns

AddDatasetItemsResponse

remove_examples_from_dataset()

Remove examples from offline-test dataset (POST /v1/projects/{projectId}/datasets/{datasetIdentifier}/examples/remove).

client.api.remove_examples_from_dataset(project_id, dataset_identifier, {"example_ids": ...})

def remove_examples_from_dataset(project_id, dataset_identifier, payload) -> RemoveExamplesFromDatasetResponse:

Parameters

Returns

RemoveExamplesFromDatasetResponse

get_dataset()

Fetch dataset page (GET /v1/projects/{projectId}/datasets/{datasetIdentifier}/page).

client.api.get_dataset(project_id, dataset_identifier)

def get_dataset(project_id, dataset_identifier, *, version=None, limit=None, cursor_created_at=None, cursor_example_id=None) -> OfflineDatasetPageResponse:

Parameters

Returns

OfflineDatasetPageResponse

list_issues()

List issues (GET /v1/projects/{projectId}/issues).

client.api.list_issues(project_id)

def list_issues(project_id, *, user_status=None, production_activity=None, priority=None) -> ListIssuesResponse:

Parameters

Returns

ListIssuesResponse

create_issue()

Create issue (POST /v1/projects/{projectId}/issues).

client.api.create_issue(project_id, {"title": ..., "diagnosis": ..., "cited_items": ...})

def create_issue(project_id, payload) -> CreateIssueResponse:

Parameters

Returns

CreateIssueResponse

get_issue()

Get issue (GET /v1/projects/{projectId}/issues/{issueId}).

client.api.get_issue(project_id, issue_id)

def get_issue(project_id, issue_id) -> GetIssueResponse:

Parameters

Returns

GetIssueResponse

update_issue()

Update issue (PATCH /v1/projects/{projectId}/issues/{issueId}).

client.api.update_issue(project_id, issue_id, {})

def update_issue(project_id, issue_id, payload) -> UpdateIssueResponse:

Parameters

Returns

UpdateIssueResponse

list_judges()

List judges (GET /v1/projects/{projectId}/judges).

client.api.list_judges(project_id)

def list_judges(project_id, *, version=None) -> ListJudgesResponse:

Parameters

Returns

ListJudgesResponse

create_judge()

Create judge (POST /v1/projects/{projectId}/judges).

client.api.create_judge(project_id, {"judge": ...})

def create_judge(project_id, payload) -> CreateJudgeResponse:

Parameters

Returns

CreateJudgeResponse

update_judge()

Update judge (PATCH /v1/projects/{projectId}/judges/{judgeId}).

client.api.update_judge(project_id, judge_id, {"update": ...})

def update_judge(project_id, judge_id, payload) -> UpdateJudgeResponse:

Parameters

Returns

UpdateJudgeResponse

delete_judge()

Delete judge (DELETE /v1/projects/{projectId}/judges/{judgeId}).

client.api.delete_judge(project_id, judge_id)

def delete_judge(project_id, judge_id) -> DeleteJudgeResponse:

Parameters

Returns

DeleteJudgeResponse

set_judge_action_groups()

Set judge action groups (POST /v1/projects/{projectId}/judges/{judgeId}/action-groups).

client.api.set_judge_action_groups(project_id, judge_id, {"source_version": ..., "target_version": ..., "promote_to_production": ..., "action_groups": ...})

def set_judge_action_groups(project_id, judge_id, payload) -> SetJudgeActionGroupsResponse:

Parameters

Returns

SetJudgeActionGroupsResponse

get_judge()

Get judge (POST /v1/projects/{projectId}/judges/{judgeId}/lookup).

client.api.get_judge(project_id, judge_id, {})

def get_judge(project_id, judge_id, payload) -> GetJudgeResponse:

Parameters

Returns

GetJudgeResponse

set_judge_optimistic_eval_enabled()

Set judge version optimistic eval (PUT /v1/projects/{projectId}/judges/{judgeId}/versions/{judgeVersionId}/optimistic-eval).

client.api.set_judge_optimistic_eval_enabled(project_id, judge_id, judge_version_id, {"enabled": ...})

def set_judge_optimistic_eval_enabled(project_id, judge_id, judge_version_id, payload) -> SetJudgeOptimisticEvalEnabledResponse:

Parameters

Returns

SetJudgeOptimisticEvalEnabledResponse

set_judge_preprocessor_artifact_revision()

Set judge version preprocessing artifact (PUT /v1/projects/{projectId}/judges/{judgeId}/versions/{judgeVersionId}/preprocessor-artifact-revision).

client.api.set_judge_preprocessor_artifact_revision(project_id, judge_id, judge_version_id, {"preprocessor_artifact_revision_id": ...})

def set_judge_preprocessor_artifact_revision(project_id, judge_id, judge_version_id, payload) -> SetJudgePreprocessorArtifactRevisionResponse:

Parameters

Returns

SetJudgePreprocessorArtifactRevisionResponse

list_judge_action_groups()

List judge action groups (GET /v1/projects/{projectId}/judges/action-groups).

client.api.list_judge_action_groups(project_id)

def list_judge_action_groups(project_id, *, version=None) -> ListJudgeActionGroupsResponse:

Parameters

Returns

ListJudgeActionGroupsResponse

list_memory_entries()

List memory entries (GET /v1/projects/{projectId}/memory/entries).

client.api.list_memory_entries(project_id)

def list_memory_entries(project_id) -> ListMemoryEntriesResponse:

Parameters

Returns

ListMemoryEntriesResponse

write_memory_file()

Write a memory file (PUT /v1/projects/{projectId}/memory/files).

client.api.write_memory_file(project_id, {"name": ..., "body": ...})

def write_memory_file(project_id, payload) -> WriteMemoryFileResponse:

Parameters

Returns

WriteMemoryFileResponse

read_memory_files()

Fetch memory files (POST /v1/projects/{projectId}/memory/files/fetch).

client.api.read_memory_files(project_id, {})

def read_memory_files(project_id, payload) -> ReadMemoryFilesResponse:

Parameters

Returns

ReadMemoryFilesResponse

search_memory_files()

Search memory files (POST /v1/projects/{projectId}/memory/search).

client.api.search_memory_files(project_id, {"query": ...})

def search_memory_files(project_id, payload) -> SearchMemoryFilesResponse:

Parameters

Returns

SearchMemoryFilesResponse

list_scheduled_tasks()

List scheduled tasks (GET /v1/projects/{projectId}/scheduled-tasks).

client.api.list_scheduled_tasks(project_id)

def list_scheduled_tasks(project_id) -> ListScheduledTasksResponse:

Parameters

Returns

ListScheduledTasksResponse

create_scheduled_task()

Create a scheduled task (POST /v1/projects/{projectId}/scheduled-tasks).

client.api.create_scheduled_task(project_id, {"name": ...})

def create_scheduled_task(project_id, payload) -> CreateScheduledTaskResponse:

Parameters

Returns

CreateScheduledTaskResponse

get_scheduled_task()

Get a scheduled task (GET /v1/projects/{projectId}/scheduled-tasks/{taskId}).

client.api.get_scheduled_task(project_id, task_id)

def get_scheduled_task(project_id, task_id) -> GetScheduledTaskResponse:

Parameters

Returns

GetScheduledTaskResponse

update_scheduled_task()

Update a scheduled task (PATCH /v1/projects/{projectId}/scheduled-tasks/{taskId}).

client.api.update_scheduled_task(project_id, task_id, {})

def update_scheduled_task(project_id, task_id, payload) -> UpdateScheduledTaskResponse:

Parameters

Returns

UpdateScheduledTaskResponse

sql_query()

Run a tenant-scoped virtual SQL query (POST /v1/projects/{projectId}/sql).

client.api.sql_query(project_id, {"sql": ...})

def sql_query(project_id, payload) -> PublicSqlResponse:

Parameters

Returns

PublicSqlResponse

list_test_configs()

Fetch test configs (GET /v1/projects/{projectId}/test-configs).

client.api.list_test_configs(project_id)

def list_test_configs(project_id, *, dataset_id=None) -> TestConfigsResponse:

Parameters

Returns

TestConfigsResponse

create_test_config()

Create test config (POST /v1/projects/{projectId}/test-configs).

client.api.create_test_config(project_id, {"name": ...})

def create_test_config(project_id, payload) -> TestConfigResponse:

Parameters

Returns

TestConfigResponse

update_test_config()

Update test config (PATCH /v1/projects/{projectId}/test-configs/{testConfigId}).

client.api.update_test_config(project_id, test_config_id, {})

def update_test_config(project_id, test_config_id, payload) -> TestConfigResponse:

Parameters

Returns

TestConfigResponse

delete_test_config()

Delete test config (DELETE /v1/projects/{projectId}/test-configs/{testConfigId}).

client.api.delete_test_config(project_id, test_config_id)

def delete_test_config(project_id, test_config_id) -> DeleteTestConfigResponse:

Parameters

Returns

DeleteTestConfigResponse

list_test_runs()

Fetch test runs (GET /v1/projects/{projectId}/test-runs).

client.api.list_test_runs(project_id)

def list_test_runs(project_id, *, test_config_id=None, dataset_id=None, status=None, limit=None) -> TestRunsResponse:

Parameters

Returns

TestRunsResponse

run_test()

Prepare and queue test run (POST /v1/projects/{projectId}/test-runs).

client.api.run_test(project_id, {"test_config_id": ...})

def run_test(project_id, payload) -> PreparedTestRunResponse:

Parameters

Returns

PreparedTestRunResponse

get_test_run()

Fetch test run (GET /v1/projects/{projectId}/test-runs/{testRunId}).

client.api.get_test_run(project_id, test_run_id)

def get_test_run(project_id, test_run_id) -> TestRunResponse:

Parameters

Returns

TestRunResponse

cancel_test_run()

Cancel test run (POST /v1/projects/{projectId}/test-runs/{testRunId}/cancel).

client.api.cancel_test_run(project_id, test_run_id)

def cancel_test_run(project_id, test_run_id) -> CancelTestRunResponse:

Parameters

Returns

CancelTestRunResponse

get_test_run_items()

Fetch test run items (GET /v1/projects/{projectId}/test-runs/{testRunId}/items).

client.api.get_test_run_items(project_id, test_run_id)

def get_test_run_items(project_id, test_run_id, *, limit=None, cursor=None) -> TestRunItemsResponse:

Parameters

Returns

TestRunItemsResponse

create_test_suite()

Create a test suite (POST /v1/projects/{projectId}/test-suites).

client.api.create_test_suite(project_id, {"name": ...})

def create_test_suite(project_id, payload) -> TestSuite:

Parameters

Returns

TestSuite

list_test_suites()

List test suites (GET /v1/projects/{projectId}/test-suites).

client.api.list_test_suites(project_id)

def list_test_suites(project_id, *, offset=None, limit=None) -> ListTestSuitesResponse:

Parameters

Returns

ListTestSuitesResponse

delete_test_suite()

Delete a test suite with its runs (DELETE /v1/projects/{projectId}/test-suites/{testSuiteId}).

client.api.delete_test_suite(project_id, test_suite_id)

def delete_test_suite(project_id, test_suite_id) -> DeleteTestSuiteResponse:

Parameters

Returns

DeleteTestSuiteResponse

list_test_cases()

List test cases in a version or the draft (GET /v1/projects/{projectId}/test-suites/{testSuiteId}/cases).

client.api.list_test_cases(project_id, test_suite_id)

def list_test_cases(project_id, test_suite_id, *, version=None, offset=None, limit=None) -> ListTestCasesResponse:

Parameters

Returns

ListTestCasesResponse

add_test_cases()

Add cases to a test suite’s draft (POST /v1/projects/{projectId}/test-suites/{testSuiteId}/cases).

client.api.add_test_cases(project_id, test_suite_id, {"cases": ...})

def add_test_cases(project_id, test_suite_id, payload) -> AddTestCasesResponse:

Parameters

Returns

AddTestCasesResponse

update_test_case()

Replace a case in the draft (PUT /v1/projects/{projectId}/test-suites/{testSuiteId}/cases/{caseId}).

client.api.update_test_case(project_id, test_suite_id, case_id, {"input": ...})

def update_test_case(project_id, test_suite_id, case_id, payload) -> UpdateTestCaseResponse:

Parameters

Returns

UpdateTestCaseResponse

remove_test_case()

Remove a case from the draft (DELETE /v1/projects/{projectId}/test-suites/{testSuiteId}/cases/{caseId}).

client.api.remove_test_case(project_id, test_suite_id, case_id)

def remove_test_case(project_id, test_suite_id, case_id) -> RemoveTestCaseResponse:

Parameters

Returns

RemoveTestCaseResponse

discard_test_suite_draft()

Throw away the draft’s changes (POST /v1/projects/{projectId}/test-suites/{testSuiteId}/discard).

client.api.discard_test_suite_draft(project_id, test_suite_id)

def discard_test_suite_draft(project_id, test_suite_id) -> DiscardTestSuiteDraftResponse:

Parameters

Returns

DiscardTestSuiteDraftResponse

create_test_suite_draft()

Copy the published cases into the draft to edit (POST /v1/projects/{projectId}/test-suites/{testSuiteId}/draft).

client.api.create_test_suite_draft(project_id, test_suite_id)

def create_test_suite_draft(project_id, test_suite_id) -> CreateTestSuiteDraftResponse:

Parameters

Returns

CreateTestSuiteDraftResponse

publish_test_suite()

Publish the draft as the next version (POST /v1/projects/{projectId}/test-suites/{testSuiteId}/publish).

client.api.publish_test_suite(project_id, test_suite_id)

def publish_test_suite(project_id, test_suite_id) -> PublishTestSuiteResponse:

Parameters

Returns

PublishTestSuiteResponse

start_test_suite_run()

Start a test run (POST /v1/projects/{projectId}/test-suites/{testSuiteId}/runs).

client.api.start_test_suite_run(project_id, test_suite_id, {})

def start_test_suite_run(project_id, test_suite_id, payload) -> StartTestRunResponse:

Parameters

Returns

StartTestRunResponse

list_test_suite_runs()

List test runs (GET /v1/projects/{projectId}/test-suites/{testSuiteId}/runs).

client.api.list_test_suite_runs(project_id, test_suite_id)

def list_test_suite_runs(project_id, test_suite_id, *, offset=None, limit=None) -> ListTestRunsResponse:

Parameters

Returns

ListTestRunsResponse

get_test_suite_run_results()

Get test run results (GET /v1/projects/{projectId}/test-suites/runs/{runId}/results).

client.api.get_test_suite_run_results(project_id, run_id)

def get_test_suite_run_results(project_id, run_id) -> TestResultsResponse:

Parameters

Returns

TestResultsResponse

score_test_run()

Score a test run (POST /v1/projects/{projectId}/test-suites/runs/{runId}/score).

client.api.score_test_run(project_id, run_id)

def score_test_run(project_id, run_id) -> ScoreTestRunResponse:

Parameters

Returns

ScoreTestRunResponse

list_trace_health_failures()

List traces failing a trace health check (GET /v1/projects/{projectId}/trace-health/checks/{checkId}/traces).

client.api.list_trace_health_failures(project_id, check_id, start_time=..., end_time=...)

def list_trace_health_failures(project_id, check_id, *, start_time, end_time, category=None, limit=None, cursor_trace_started_at=None, cursor_trace_id=None) -> ListTraceHealthFailuresResponse:

Parameters

Returns

ListTraceHealthFailuresResponse

get_trace_health_summary()

Get trace health summary (GET /v1/projects/{projectId}/trace-health/summary).

client.api.get_trace_health_summary(project_id, start_time=..., end_time=...)

def get_trace_health_summary(project_id, *, start_time, end_time, category=None) -> GetTraceHealthSummaryResponse:

Parameters

Returns

GetTraceHealthSummaryResponse

get_trace_health()

Get trace health for one trace (GET /v1/projects/{projectId}/trace-health/traces/{traceId}).

client.api.get_trace_health(project_id, trace_id)

def get_trace_health(project_id, trace_id, *, category=None) -> GetTraceHealthResponse:

Parameters

Returns

GetTraceHealthResponse

evaluate_traces()

Evaluate stored traces (POST /v1/projects/{projectId}/traces/evaluate).

client.api.evaluate_traces(project_id, {"trace_ids": ...})

def evaluate_traces(project_id, payload) -> EvaluateTracesResponse:

Parameters

Returns

EvaluateTracesResponse

evaluate_all_traces()

Evaluate recent traces (POST /v1/projects/{projectId}/traces/evaluate-all).

client.api.evaluate_all_traces(project_id, {})

def evaluate_all_traces(project_id, payload) -> EvaluateAllTracesResponse:

Parameters

Returns

EvaluateAllTracesResponse

list_projects()

List project summaries (GET /v1/projects/summaries).

client.api.list_projects()

def list_projects() -> ListProjectsResponse:

Returns

ListProjectsResponse

discover_schema()

Discover the virtual SQL schema (GET /v1/sql/schema).

client.api.discover_schema()

def discover_schema() -> PublicSqlSchemaResponse:

Returns

PublicSqlSchemaResponse