PromptScorer

A PromptScorer is a powerful tool for evaluating your LLM system using use-case specific, natural language rubrics. Prompt Scorers make it easy to prototype your evaluation rubrics—you can easily set up new criteria and test them on a few examples in the scorer playground, then evaluate your agents' behavior in production with real customer usage.

from openai import OpenAI
from judgeval.scorers import PromptScorer
from judgeval.tracer import Tracer, wrap
from judgeval.data import Example

judgment = Tracer(project_name="default_project")
client = wrap(OpenAI())

# Create or retrieve a PromptScorer
scorer = PromptScorer.create(
    name="PositivityScorer",
    prompt="Is the response positive or negative? Response: {{actual_output}}",
    options={"positive": 1, "negative": 0}
)

class QAAgent:
    def __init__(self, client):
        self.client = client

    @judgment.observe(span_type="tool")
    def process_query(self, query):
        response = self.client.chat.completions.create(
            model="gpt-5",
            messages=[
                {"role": "system", "content": "You are a helpful assistant"},
                {"role": "user", "content": query}
            ]
        )
        return response.choices[0].message.content

    @judgment.agent()
    @judgment.observe(span_type="agent")
    def invoke_agent(self, query):
        result = self.process_query(query)
        judgment.async_evaluate(
            scorer=scorer,
            example=Example(input=query, actual_output=result),
            model="gpt-5"
        )
        return result

if __name__ == "__main__":
    agent = QAAgent(client)
    print(agent.invoke_agent("What is the capital of the United States?"))

from openai import OpenAI
from judgeval.scorers import TracePromptScorer
from judgeval.tracer import Tracer, wrap, TraceScorerConfig

# Initialize tracer
judgment = Tracer(
    project_name="default_project"
)

# Auto-trace LLM calls
client = wrap(OpenAI())

# Retrieve your TracePromptScorer, created either through the platform or with the "create" class method
scorer = TracePromptScorer.get(
    name="PositivityScorer"
)

class QAAgent:
    def __init__(self, client):
        self.client = client

    @judgment.observe(span_type="tool")
    def process_query(self, query):
        response = self.client.chat.completions.create(
            model="gpt-5",
            messages=[
                {"role": "system", "content": "You are a helpful assitant"},
                {"role": "user", "content": f"I have a query: {query}"}]
        )  # Automatically traced
        return f"Response: {response.choices[0].message.content}"

    # Basic function tracing
    @judgment.agent()
    @judgment.observe(span_type="agent", scorer_config=TraceScorerConfig(scorer=scorer))
    def invoke_agent(self, query):
        result = self.process_query(query)
        return result


if __name__ == "__main__":
    agent = QAAgent(client)
    print(agent.invoke_agent("What is the capital of the United States?"))

All PromptScorer methods automatically sync changes with the Judgment platform.

Static Method

PromptScorer.create() | TracePromptScorer.create()

Initialize a PromptScorer or TracePromptScorer object.

Parameters

namerequired:str

The name of the PromptScorer

promptrequired:str

The prompt used by the LLM judge to make an evaluation

options:dict

If specified, the LLM judge will pick from one of the choices, and the score will be the one corresponding to the choice

judgment_api_key:str

Recommended - set using the JUDGMENT_API_KEY environment variable

organization_id:str

Recommended - set using the JUDGMENT_ORG_ID environment variable

Returns

A PromptScorer instance

Example

create_prompt_scorer.py

from judgeval.scorers import PromptScorer

scorer = PromptScorer.create(
    name="Test Scorer",
    prompt="Is the response positive or negative? Response: {{actual_output}}",
    options={"positive" : 1, "negative" : 0}
)