Judge

Base class for building custom evaluation scorers.

Subclass Judge and implement the score method to create your own evaluation logic.

class ContainsAnswer extends Judge<BinaryResponse> {
  async score(data: Example): Promise<BinaryResponse> {
    const expected = (data.get("expected_output") as string).toLowerCase();
    const actual = (data.get("actual_output") as string).toLowerCase();
    return {
      value: actual.includes(expected),
      reason: actual.includes(expected) ? "Found" : "Not found",
    };
  }
}