Get Started

from judgeval.tracer import Tracer, wrap
from openai import OpenAI

client = wrap(OpenAI())  # tracks all LLM calls
judgment = Tracer(project_name="my_project")

@judgment.observe(span_type="tool")
def format_question(question: str) -> str:
    # dummy tool
    return f"Question : {question}"

@judgment.observe(span_type="function")
def run_agent(prompt: str) -> str:
    task = format_question(prompt)
    response = client.chat.completions.create(
        model="gpt-4.1",
        messages=[{"role": "user", "content": task}]
    )
    return response.choices[0].message.content

run_agent("What is the capital of the United States?")

from judgeval import JudgmentClient
from judgeval.data import Example
from judgeval.scorers import FaithfulnessScorer

client = JudgmentClient()

task = "What is the capital of the United States?"
example = Example(
    input=task,
    actual_output=run_agent(task),  # e.g. "The capital of the U.S. is Washington, D.C."
    retrieval_context=["Washington D.C. was founded in 1790 and became the capital of the U.S."],
)

scorer = FaithfulnessScorer(threshold=0.5)
client.assert_test(
    examples=[example],
    scorers=[scorer],
    model="gpt-4.1",
)