From 28674bdeff76922f55e834a0975e5cb5aef7ebe3 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 5 Aug 2026 11:36:08 +0000 Subject: [PATCH] feat: Score workflow evals on a 6-dimension rubric MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Replaces the single PASS/FAIL judge verdict in evals/workflows with a fixed 6-dimension rubric (toolSelection, argumentCorrectness, resultUtilization, taskCompletion, errorRecovery, planEfficiency), so a reviewer can see which capability regressed instead of one opaque verdict. Two checks are computed in code, not by the LLM: - toolSelection: exact set-match against a new expectedTools field on the test case, overriding the judge's own verdict when set. Ported from the toolsExactMatch evaluator in evals/run_evaluation.ts. - schemaValidity: every tool call's arguments validated with AJV against that tool's declared inputSchema. Reported separately from argumentCorrectness. The judge now also sees tool results (truncated to 2000 chars per result, errors shown as errors) — resultUtilization and errorRecovery are unjudgeable without them. overallVerdict = taskCompletion, purely informational. Exit-code behavior is unchanged and nothing here is gated in CI. results.json keeps verdict/reason as mirrors of taskCompletion so existing readers don't break, and gains rubric, toolSelectionCheck and schemaValidityCheck. Baseline comparison prints per-dimension pass-rate deltas alongside the byte/token ones. expectedTools backfilled on 25 of the 30 test cases; left unset on the 5 where multiple tool paths are genuinely valid. Assisted-by: Claude Code Claude-Session: https://claude.ai/code/session_0136aUA25ZgcWmM7RMDq3iar --- evals/shared/types.ts | 7 + evals/workflows/README.md | 76 ++++++- evals/workflows/config.ts | 43 ++-- evals/workflows/deterministic_checks.ts | 130 ++++++++++++ evals/workflows/output_formatter.ts | 110 +++++++++- evals/workflows/results_writer.ts | 7 +- evals/workflows/run_workflow_evals.ts | 18 +- evals/workflows/test_cases.json | 43 +++- evals/workflows/workflow_judge.ts | 197 +++++++++++++----- tests/unit/evals.deterministic_checks.test.ts | 156 ++++++++++++++ tests/unit/evals.workflow_judge.test.ts | 123 +++++++++++ tests/unit/evals.workflow_output.test.ts | 159 +++++++++++++- 12 files changed, 957 insertions(+), 112 deletions(-) create mode 100644 evals/workflows/deterministic_checks.ts create mode 100644 tests/unit/evals.deterministic_checks.test.ts create mode 100644 tests/unit/evals.workflow_judge.test.ts diff --git a/evals/shared/types.ts b/evals/shared/types.ts index 62002b7bf..89f29f8ce 100644 --- a/evals/shared/types.ts +++ b/evals/shared/types.ts @@ -49,6 +49,13 @@ export type WorkflowTestCase = { * that the live server + API cannot reproduce on demand. See mcp_client.ts. */ failTools?: string[]; + /** + * Exact tool-call sequence (by name, order-independent, dedup'd) that constitutes correct tool + * selection for this test. When set, the `toolSelection` rubric dimension is scored + * deterministically (code exact-match) instead of by the judge. Omit only when multiple tool + * paths are genuinely valid. + */ + expectedTools?: string[]; } & BaseTestCase; /** diff --git a/evals/workflows/README.md b/evals/workflows/README.md index a3e2aa7dc..b516e8120 100644 --- a/evals/workflows/README.md +++ b/evals/workflows/README.md @@ -62,7 +62,7 @@ Tests AI agents executing tasks using Apify MCP server tools through multi-turn - Multi-turn conversations with tool calling - Dynamic tool discovery during execution - MCP server instructions automatically added to agent system prompt -- LLM-based evaluation against requirements +- LLM judge scoring a fixed 6-dimension rubric, plus two code-based checks - Isolated MCP server per test - Configurable tool call timeout (default: 60 seconds) - Strict pass/fail (all tests must pass) @@ -139,23 +139,51 @@ while (turnNumber < maxTurns) { **Location:** `run-workflow-evals.ts` -### 4. Judge Sees Tool Calls, Not Results +### 4. Six-Dimension Rubric, Backed by Deterministic Checks -**Decision:** Judge sees tool calls with arguments and agent responses, but NOT raw tool results. +**Decision:** The judge scores 6 fixed dimensions instead of one PASS/FAIL, and it sees truncated +tool results alongside the calls. Two checks are computed in code, not by the LLM. **Why:** -- Evaluates agent behavior (tool selection, arguments) -- Tool results are often very long and noisy -- Agent should summarize results, judge evaluates the summary +- One opaque verdict doesn't say *which* capability regressed in a PR +- `resultUtilization` and `errorRecovery` are unjudgeable without seeing what tools returned +- Tool selection and schema validity have knowable answers; an LLM adds noise, not signal + +**Dimensions** (same 6 for every test case, scored independently): + +| Dimension | Question | +|-----------|----------| +| `toolSelection` | Right tools, no unnecessary or missing calls? | +| `argumentCorrectness` | Arguments semantically correct (right IDs, filters, values)? | +| `resultUtilization` | Did the agent read and use what the tools actually returned? | +| `taskCompletion` | Does the final response satisfy the test case's `reference`? | +| `errorRecovery` | On a failed or unexpected result, did the agent respond sensibly? | +| `planEfficiency` | Reasonably direct path — no redundant calls or excessive turns? | + +**Deterministic checks** (`deterministic_checks.ts`): +- `toolSelectionCheck` — exact set-match (deduped, order-independent) of the tools called against + the test case's `expectedTools`. When `expectedTools` is set, this **overrides** the judge's + `toolSelection` verdict; when unset, the judge's own verdict stands. +- `schemaValidityCheck` — every tool call's arguments validated with AJV against that tool's + declared `inputSchema`. Reported separately from `argumentCorrectness`: schema-valid arguments + can still be semantically wrong. Calls naming a tool absent from the final tool list are skipped. + +**Overall verdict:** `overallVerdict` = the `taskCompletion` verdict. No separate aggregation rule. +The rubric is **reporting only** — the exit code still reflects errors and task completion, and +nothing here is gated in CI. **Judge input format:** ``` USER: Find actors for Google Maps AGENT: [Called tool: search-actors with args: {"keywords":"google maps","limit":5}] +TOOL RESULT (search-actors): {"actors":[...]} [truncated at 2000 chars, 4213 total] AGENT: I found 5 actors: 1. Google Maps Scraper... 2. ... ``` -**Location:** `workflow-judge.ts` +Each result is truncated to 2000 chars. Failed calls show the error text, not "no result" — that +is exactly what `errorRecovery` is scored on. + +**Location:** `workflow_judge.ts`, `deterministic_checks.ts`, `config.ts` (`JUDGE_PROMPT_TEMPLATE`) ### 5. LLM Client Shared, MCP Client Isolated @@ -221,6 +249,7 @@ const conversation = await executeConversation({ - `convert-mcp-tools.ts` - MCP → OpenAI tool format - `conversation-executor.ts` - Multi-turn loop with dynamic tools and server instructions - `workflow-judge.ts` - Judge evaluation +- `deterministic_checks.ts` - Code-based tool-selection and schema-validity checks - `test-cases-loader.ts` - Load/filter test cases - `output-formatter.ts` - Results formatting - `run-workflow-evals.ts` - Main CLI entry @@ -362,6 +391,21 @@ The `--output` (or `-o`) option saves test results to `evals/workflows/results.j "testId": "search-google-maps", "verdict": "PASS", "reason": "Agent successfully searched for Google Maps actors", + "rubric": { + "toolSelection": { "verdict": "PASS", "reason": "Deterministic: expected [search-actors], got [search-actors]" }, + "argumentCorrectness": { "verdict": "PASS", "reason": "Keywords matched the request" }, + "resultUtilization": { "verdict": "PASS", "reason": "Reported the actors the search returned" }, + "taskCompletion": { "verdict": "PASS", "reason": "Agent successfully searched for Google Maps actors" }, + "errorRecovery": { "verdict": "PASS", "reason": "No tool call failed" }, + "planEfficiency": { "verdict": "PASS", "reason": "One search, one answer" } + }, + "toolSelectionCheck": { + "checked": true, + "verdict": "PASS", + "expected": ["search-actors"], + "actual": ["search-actors"] + }, + "schemaValidityCheck": { "verdict": "PASS", "invalidCalls": [] }, "durationMs": 5234, "turns": 3, "resultBytes": 18452, @@ -379,8 +423,15 @@ The `--output` (or `-o`) option saves test results to `evals/workflows/results.j - `agentModel` - LLM model used for the agent - `judgeModel` - LLM model used for judging - `testId` - Test case identifier -- `verdict` - `PASS` or `FAIL` -- `reason` - Judge reasoning or error message +- `verdict` - `PASS` or `FAIL`. Mirrors `rubric.taskCompletion.verdict`; kept under the old name so + existing readers don't break +- `reason` - Mirrors `rubric.taskCompletion.reason`, or the error message on an errored test +- `rubric` - All 6 dimension verdicts and reasons (see design decision #4). Absent on errored tests + and in records written before the rubric existed +- `toolSelectionCheck` - Deterministic `expectedTools` exact-match: `checked`, `verdict`, `expected`, + `actual`. `checked` is `false` (and `verdict` `null`) when the test case declares no `expectedTools` +- `schemaValidityCheck` - Deterministic per-call `inputSchema` validation: `verdict` plus + `invalidCalls` (`{ toolName, errors }`, empty on PASS) - `durationMs` - Test duration in milliseconds - `turns` - Number of conversation turns - `resultBytes` - Total UTF-8 bytes of tool results returned to the agent across the conversation (measured at the point each result is fed to the LLM, so it reflects what the agent actually receives). Compare across branches to quantify byte savings. @@ -415,9 +466,11 @@ The `results.json` file is tracked in git, allowing you to: - Compare results across branches - Track performance regressions in PRs -### Comparing against a baseline (byte/token deltas) +### Comparing against a baseline (byte/token and rubric deltas) + +Every run automatically compares against a baseline and prints per-test and aggregate **deltas** for tool bytes, tokens, and each rubric dimension's pass rate — no manual file diffing. This is how you answer "did this change grow the response size?" and "which capability regressed?". -Every run automatically compares against a baseline and prints per-test and aggregate **deltas** for tool bytes and tokens — no manual file diffing. This is how you answer "did this change grow the response size?". +Dimension deltas read as `Result utilization (30/30): 27/30 passed (▼ -2 vs baseline 29/30)`. Polarity is the opposite of the byte/token metrics: more passes is better, so ▲ is an improvement and ▼ a regression. They are matched only against baseline records that carry a `rubric` field. - **Default baseline** is the committed `evals/workflows/results.json`. Each test is matched by its `agentModel:judgeModel:testId` key. - **Custom baseline:** `--baseline ` compares against any saved results file. @@ -462,6 +515,7 @@ File: `test-cases.json` - `maxTurns` - Override default (10) - `tools` - List of tools to enable for this test (e.g., `["actors", "docs", "apify/rag-web-browser"]`). If omitted, all default tools are enabled. Passed to MCP server as `--tools` argument. - `failTools` - Tool names the harness force-fails with a synthetic `INTERNAL_ERROR` result carrying the real `report-problem` nudge, instead of calling the server (e.g. `["call-actor"]`). Use it to deterministically throw a nudge-eligible error that the live server + API cannot reproduce on demand, e.g. to test that the agent proactively calls `report-problem` after a failure. See `mcp_client.ts`. +- `expectedTools` - Exact set of tool names the agent should call (order-independent, deduped), e.g. `["call-actor", "get-dataset-items"]`. When set, the `toolSelection` dimension is scored in code by exact match and the judge's own verdict for it is discarded. Omit it only when multiple tool paths are genuinely valid — 5 of the 30 test cases do (the "which is the best scraper" searches, where fetching details on top of searching is legitimate, and the cases reachable either via a dedicated Actor or `apify/rag-web-browser`). Use the names as they appear in `HELPER_TOOLS` (`src/const.ts`). ## Performance diff --git a/evals/workflows/config.ts b/evals/workflows/config.ts index 414e7687d..3ce762203 100644 --- a/evals/workflows/config.ts +++ b/evals/workflows/config.ts @@ -61,26 +61,31 @@ export const DEFAULT_TOOL_TIMEOUT_SECONDS = 60; * - {{reference}}: The requirements the agent should meet * - {{conversation}}: The formatted conversation to evaluate */ -export const JUDGE_PROMPT_TEMPLATE = `You are evaluating whether an AI agent successfully completed a user's task using available tools. +export const JUDGE_PROMPT_TEMPLATE = `You are evaluating an AI agent that used tools to complete a user's task. + +Evaluate the agent's conversation against these 6 dimensions. Score each independently — +do not let a strong result on one dimension excuse a weak result on another. Give each a +verdict (PASS or FAIL) and a brief explanation (1-2 sentences). + +1. toolSelection: Did the agent call appropriate tool(s) for the task (right tools, no + unnecessary or missing calls)? A different tool than expected is fine if it accomplishes + the same goal. [Note: for some test cases this is overridden by a deterministic check + after your response — score it anyway, your answer may be discarded.] +2. argumentCorrectness: Were tool call arguments semantically correct and complete for + the task (right IDs, filters, values) — not just schema-valid? +3. resultUtilization: Did the agent correctly read and use what each tool actually + returned — not ignore an error, not misreport data, not claim something the result + didn't say? +4. taskCompletion: Did the final response fully satisfy the requirements below? Judge the + requirements, not the writing style. +5. errorRecovery: When a tool call failed or returned something unexpected, did the + agent respond sensibly (retry, use an alternative, explain to the user) rather than + stall or hallucinate? PASS when nothing went wrong and there was nothing to recover from. +6. planEfficiency: Was the path to the answer reasonably direct — no redundant calls, + no excessive turns? Minor inefficiencies are acceptable. TASK REQUIREMENTS: {{reference}} -AGENT CONVERSATION: -{{conversation}} - -Your task is to evaluate if the agent met ALL the requirements listed above. - -Evaluation criteria: -1. Did the agent use appropriate tools to accomplish the task? -2. Were the tool calls made with correct arguments? -3. Did the agent provide a clear, helpful final response to the user? -4. Did the agent fully address all requirements? - -Important notes: -- Focus on whether requirements were met, not on writing style -- The agent may use different tools than expected if they accomplish the same goal -- Tool results are not shown (only tool calls and agent responses) -- Minor inefficiencies are acceptable if the task was completed - -Provide your evaluation with a verdict (PASS or FAIL) and a brief explanation (1-2 sentences).`; +AGENT CONVERSATION (includes tool results): +{{conversation}}`; diff --git a/evals/workflows/deterministic_checks.ts b/evals/workflows/deterministic_checks.ts new file mode 100644 index 000000000..8ea64770b --- /dev/null +++ b/evals/workflows/deterministic_checks.ts @@ -0,0 +1,130 @@ +/** + * Code-based checks for the parts of an evaluation whose answer is knowable without an LLM. + * Their results sit next to the judge's rubric in JudgeResult (see workflow_judge.ts). + */ + +import type { ValidateFunction } from 'ajv'; +import Ajv from 'ajv'; + +import type { ConversationHistory, McpTool } from './types.js'; + +/** + * Result of comparing the tools the agent called against the test case's expectedTools. + */ +export type ToolSelectionCheck = { + /** False when the test case declares no expectedTools, so nothing was checked */ + checked: boolean; + /** Null when checked is false */ + verdict: 'PASS' | 'FAIL' | null; + /** Expected tool names, deduped and sorted (empty when checked is false) */ + expected: string[]; + /** Tool names the agent actually called, deduped and sorted */ + actual: string[]; +}; + +/** + * Result of validating every tool call's arguments against that tool's declared inputSchema. + */ +export type SchemaValidityCheck = { + verdict: 'PASS' | 'FAIL'; + /** Empty array when verdict is PASS */ + invalidCalls: { toolName: string; errors: string[] }[]; +}; + +/** + * Dedupe and sort so the comparison is order-independent and repeated calls to the same tool + * count once. Same semantics as the toolsExactMatch evaluator in evals/run_evaluation.ts. + */ +function normalizeToolNames(names: string[]): string[] { + return Array.from(new Set(names)).sort(); +} + +function collectCalledToolNames(conversation: ConversationHistory): string[] { + return normalizeToolNames(conversation.turns.flatMap((turn) => turn.toolCalls.map((call) => call.name))); +} + +/** + * Exact-match the set of tools the agent called against the test case's expectedTools. + * Returns `checked: false` when the test case declares none, leaving the dimension to the judge. + */ +export function checkToolSelection( + expectedTools: string[] | undefined, + conversation: ConversationHistory, +): ToolSelectionCheck { + const actual = collectCalledToolNames(conversation); + + if (!expectedTools || expectedTools.length === 0) { + return { checked: false, verdict: null, expected: [], actual }; + } + + const expected = normalizeToolNames(expectedTools); + return { + checked: true, + verdict: JSON.stringify(expected) === JSON.stringify(actual) ? 'PASS' : 'FAIL', + expected, + actual, + }; +} + +/** + * Format one AJV error as a single readable line, e.g. `/limit must be integer`. + */ +function formatAjvError(error: { instancePath?: string; message?: string }): string { + return `${error.instancePath || '/'} ${error.message ?? 'is invalid'}`.trim(); +} + +/** + * Validate each tool call's arguments against the declared inputSchema of the tool it named. + * + * Known limitation: validation uses the latest snapshot of each tool's schema, not the schema as + * it was at the moment of that specific call. Dynamic tool schemas do not change mid-run in + * practice; revisit only if this ever produces a false positive. + * + * @param conversation - Conversation whose tool calls are validated + * @param tools - Tool list snapshot taken after the conversation finished (mcpClient.getTools()) + */ +export function checkSchemaValidity(conversation: ConversationHistory, tools: McpTool[]): SchemaValidityCheck { + // strict: false tolerates the loose schema shapes MCP tools declare (unknown keywords, etc.). + const ajv = new Ajv({ strict: false }); + const schemaByName = new Map(tools.map((tool) => [tool.name, tool.inputSchema])); + // Compile once per tool; tool schemas carry a $id, which AJV rejects on a second compile. + const validators = new Map(); + + function getValidator(toolName: string): ValidateFunction | null { + if (validators.has(toolName)) return validators.get(toolName) ?? null; + + const schema = schemaByName.get(toolName); + let validate: ValidateFunction | null = null; + if (schema) { + try { + validate = ajv.compile(schema); + } catch { + // An uncompilable schema is a server-side problem, not an agent mistake — skip it. + validate = null; + } + } + validators.set(toolName, validate); + return validate; + } + + const invalidCalls: SchemaValidityCheck['invalidCalls'] = []; + + for (const turn of conversation.turns) { + for (const call of turn.toolCalls) { + // A tool absent from the final list (e.g. a since-deregistered dynamic tool) is not a + // schema question — skip rather than flag it. + const validate = getValidator(call.name); + if (!validate || validate(call.arguments)) continue; + + invalidCalls.push({ + toolName: call.name, + errors: (validate.errors ?? []).map(formatAjvError), + }); + } + } + + return { + verdict: invalidCalls.length === 0 ? 'PASS' : 'FAIL', + invalidCalls, + }; +} diff --git a/evals/workflows/output_formatter.ts b/evals/workflows/output_formatter.ts index 53f642b09..abceb1809 100644 --- a/evals/workflows/output_formatter.ts +++ b/evals/workflows/output_formatter.ts @@ -2,9 +2,11 @@ * Output formatter for evaluation results */ +import type { SchemaValidityCheck, ToolSelectionCheck } from './deterministic_checks.js'; import type { WorkflowTestCase } from './test_cases_loader.js'; import type { ConversationHistory } from './types.js'; -import type { JudgeResult } from './workflow_judge.js'; +import type { JudgeResult, RubricResult } from './workflow_judge.js'; +import { RUBRIC_DIMENSIONS } from './workflow_judge.js'; /** * Single evaluation result @@ -64,11 +66,34 @@ export function formatWithDelta(current: number, baseline: number | undefined, f return `${format(current)} (${arrow} ${sign}${format(Math.abs(diff))} / ${pct})`; } +/** + * Render a pass count followed by its change vs a baseline pass count over the same test subset. + * More passes is better here, so ▲ marks an improvement and ▼ a regression — the opposite reading + * of the byte/token deltas, where lower is better. + */ +export function formatPassRateWithDelta(current: number, baseline: number, total: number): string { + const diff = current - baseline; + if (diff === 0) return `${current}/${total} passed (= baseline)`; + + const arrow = diff > 0 ? '▲' : '▼'; + const sign = diff > 0 ? '+' : '-'; + return `${current}/${total} passed (${arrow} ${sign}${Math.abs(diff)} vs baseline ${baseline}/${total})`; +} + +/** + * Render the rubric as one compact line of per-dimension glyphs, e.g. + * `tool✓ args✓ result✗ complete✓ recover✓ eff✓`. + */ +export function formatRubricGlyphs(rubric: RubricResult): string { + return RUBRIC_DIMENSIONS.map(({ key, glyph }) => `${glyph}${rubric[key].verdict === 'PASS' ? '✓' : '✗'}`).join(' '); +} + /** * Format results as a table. * * @param results - Evaluation results to render - * @param baseline - Optional prior results keyed by test ID; when present, byte/token deltas are shown + * @param baseline - Optional prior results keyed by test ID; when present, byte/token and + * per-dimension pass-rate deltas are shown */ export function formatResultsTable(results: EvaluationResult[], baseline?: Map): string { const lines: string[] = []; @@ -84,7 +109,7 @@ export function formatResultsTable(results: EvaluationResult[], baseline?: Map 0) { + const summary = invalidCalls.map((call) => `${call.toolName} (${call.errors.join('; ')})`).join(', '); + lines.push(` Schema-invalid calls: ${summary}`); + } } lines.push(''); @@ -113,8 +145,8 @@ export function formatResultsTable(results: EvaluationResult[], baseline?: Map !r.error && r.judgeResult.verdict === 'PASS').length; - const failedTests = results.filter((r) => !r.error && r.judgeResult.verdict === 'FAIL').length; + const passedTests = results.filter((r) => !r.error && r.judgeResult.overallVerdict === 'PASS').length; + const failedTests = results.filter((r) => !r.error && r.judgeResult.overallVerdict === 'FAIL').length; const errorTests = results.filter((r) => r.error).length; const totalBytes = results.reduce((sum, r) => sum + sumResultBytes(r.conversation), 0); @@ -129,10 +161,26 @@ export function formatResultsTable(results: EvaluationResult[], baseline?: Map(); + const dimensionBaseline = new Map(); if (baseline) { for (const result of results) { const prior = baseline.get(result.testCase.id); if (!prior) continue; + if (prior.rubric) { + rubricMatched++; + for (const { key } of RUBRIC_DIMENSIONS) { + if (result.judgeResult.rubric[key].verdict === 'PASS') { + dimensionCurrent.set(key, (dimensionCurrent.get(key) ?? 0) + 1); + } + if (prior.rubric[key]?.verdict === 'PASS') { + dimensionBaseline.set(key, (dimensionBaseline.get(key) ?? 0) + 1); + } + } + } // Records written before these metrics existed lack the field, so match each independently. const priorBytes = prior.resultBytes; if (priorBytes !== undefined) { @@ -161,8 +209,14 @@ export function formatResultsTable(results: EvaluationResult[], baseline?: Map r.judgeResult.rubric[key].verdict === 'PASS').length; + lines.push(` ${label}: ${passed}/${totalTests} passed`); + } } - if (bytesMatched > 0 || tokensMatched > 0) { + if (bytesMatched > 0 || tokensMatched > 0 || rubricMatched > 0) { lines.push(''); lines.push(` vs baseline:`); if (bytesMatched > 0) { @@ -175,6 +229,15 @@ export function formatResultsTable(results: EvaluationResult[], baseline?: Map 0) { + for (const { key, label } of RUBRIC_DIMENSIONS) { + const current = dimensionCurrent.get(key) ?? 0; + const prior = dimensionBaseline.get(key) ?? 0; + lines.push( + ` ${label} (${rubricMatched}/${totalTests}): ${formatPassRateWithDelta(current, prior, rubricMatched)}`, + ); + } + } } lines.push(''); @@ -252,8 +315,27 @@ export function formatDetailedResult(result: EvaluationResult): string { } lines.push(''); - lines.push(`⚖️ Judge Verdict: ${result.judgeResult.verdict}`); - lines.push(` Reason: ${result.judgeResult.reason}`); + lines.push(`⚖️ Overall verdict (= task completion): ${result.judgeResult.overallVerdict}`); + for (const { key, label } of RUBRIC_DIMENSIONS) { + const dimension = result.judgeResult.rubric[key]; + lines.push(` ${dimension.verdict === 'PASS' ? '✓' : '✗'} ${label}: ${dimension.reason}`); + } + lines.push(''); + + const { toolSelectionCheck, schemaValidityCheck } = result.judgeResult; + lines.push(`🧮 Deterministic checks:`); + if (toolSelectionCheck.checked) { + lines.push(` Tool selection: ${toolSelectionCheck.verdict}`); + lines.push(` Expected: [${toolSelectionCheck.expected.join(', ')}]`); + lines.push(` Actual: [${toolSelectionCheck.actual.join(', ')}]`); + } else { + lines.push(` Tool selection: not checked (no expectedTools on this test case)`); + lines.push(` Actual: [${toolSelectionCheck.actual.join(', ')}]`); + } + lines.push(` Schema validity: ${schemaValidityCheck.verdict}`); + for (const invalidCall of schemaValidityCheck.invalidCalls) { + lines.push(` ${invalidCall.toolName}: ${invalidCall.errors.join('; ')}`); + } lines.push(''); lines.push(`⏱️ Duration: ${result.durationMs}ms`); @@ -276,10 +358,16 @@ export type TestResultRecord = { judgeModel: string; /** Test case ID */ testId: string; - /** Test verdict (PASS or FAIL) */ + /** Test verdict (PASS or FAIL) — mirrors `rubric.taskCompletion.verdict`, kept for readers of the old field */ verdict: 'PASS' | 'FAIL'; - /** Judge reasoning or error message */ + /** Judge reasoning or error message — mirrors `rubric.taskCompletion.reason` */ reason: string; + /** All 6 rubric dimensions (absent in records written before the rubric existed) */ + rubric?: RubricResult; + /** Deterministic expectedTools exact-match (absent in records written before it existed, and on errored tests) */ + toolSelectionCheck?: ToolSelectionCheck; + /** Deterministic per-call inputSchema validation (absent in records written before it existed, and on errored tests) */ + schemaValidityCheck?: SchemaValidityCheck; /** Test duration in milliseconds */ durationMs: number; /** Number of conversation turns */ diff --git a/evals/workflows/results_writer.ts b/evals/workflows/results_writer.ts index 1161ec249..5c561e498 100644 --- a/evals/workflows/results_writer.ts +++ b/evals/workflows/results_writer.ts @@ -127,8 +127,11 @@ export function convertEvaluationResultToRecord( agentModel, judgeModel, testId: result.testCase.id, - verdict: result.judgeResult.verdict, - reason: result.judgeResult.reason, + verdict: result.judgeResult.overallVerdict, + reason: result.judgeResult.rubric.taskCompletion.reason, + rubric: result.judgeResult.rubric, + toolSelectionCheck: result.judgeResult.toolSelectionCheck, + schemaValidityCheck: result.judgeResult.schemaValidityCheck, durationMs: result.durationMs, turns: result.conversation.totalTurns, resultBytes: sumResultBytes(result.conversation), diff --git a/evals/workflows/run_workflow_evals.ts b/evals/workflows/run_workflow_evals.ts index a6b265fc4..f879248e3 100644 --- a/evals/workflows/run_workflow_evals.ts +++ b/evals/workflows/run_workflow_evals.ts @@ -35,7 +35,7 @@ import { } from './results_writer.js'; import type { WorkflowTestCase, WorkflowTestCaseWithLineNumbers } from './test_cases_loader.js'; import { filterTestCases, loadTestCases, loadTestCasesWithLineNumbers } from './test_cases_loader.js'; -import { evaluateConversation } from './workflow_judge.js'; +import { createErrorJudgeResult, evaluateConversation } from './workflow_judge.js'; type CliArgs = { category?: string; @@ -98,8 +98,10 @@ async function runSingleTest( serverInstructions, }); - // Judge conversation - const judgeResult = await evaluateConversation(testCase, conversation, llmClient, argv.judgeModel); + // Judge conversation. The tool list is snapshotted now (before cleanup clears it) so the + // schema-validity check has the inputSchema of every tool the agent could have called. + const mcpTools = mcpClient.getTools(); + const judgeResult = await evaluateConversation(testCase, conversation, llmClient, argv.judgeModel, mcpTools); const durationMs = Date.now() - startTime; @@ -112,7 +114,7 @@ async function runSingleTest( logWithPrefix( testId, - ` ${judgeResult.verdict === 'PASS' ? '✅' : '❌'} ${judgeResult.verdict} (${durationMs}ms)`, + ` ${judgeResult.overallVerdict === 'PASS' ? '✅' : '❌'} ${judgeResult.overallVerdict} (${durationMs}ms)`, ); } catch (error) { const durationMs = Date.now() - startTime; @@ -126,11 +128,7 @@ async function runSingleTest( hitMaxTurns: false, totalTurns: 0, }, - judgeResult: { - verdict: 'FAIL', - reason: 'Error during execution', - rawResponse: '', - }, + judgeResult: createErrorJudgeResult('Error during execution'), durationMs, error: error instanceof Error ? error.message : String(error), }; @@ -393,7 +391,7 @@ async function main() { // Exit with appropriate code - ALL tests must pass const totalTests = results.length; - const passedTests = results.filter((r) => !r.error && r.judgeResult.verdict === 'PASS').length; + const passedTests = results.filter((r) => !r.error && r.judgeResult.overallVerdict === 'PASS').length; const errorTests = results.filter((r) => r.error).length; // Exit 0 only if ALL tests passed with no errors diff --git a/evals/workflows/test_cases.json b/evals/workflows/test_cases.json index 53b4fcceb..2fe7ce484 100644 --- a/evals/workflows/test_cases.json +++ b/evals/workflows/test_cases.json @@ -5,14 +5,16 @@ "id": "search-google-maps", "category": "search", "query": "Is there any Google Maps scraping tool on Apify?", - "reference": "The agent must search for actors related to Google Maps and return at least 3 results with their names and descriptions." + "reference": "The agent must search for actors related to Google Maps and return at least 3 results with their names and descriptions.", + "expectedTools": ["search-actors"] }, { "id": "search-add-call-python-actor", "tools": ["actors"], "category": "call", "query": "Find example Python Actor and call it with random sample input and present the results", - "reference": "The agent must find apify/python-example Actor or any similar and call it using the call-actor tool with sample input, and return the output." + "reference": "The agent must find apify/python-example Actor or any similar and call it using the call-actor tool with sample input, and return the output.", + "expectedTools": ["search-actors", "call-actor"] }, { "id": "instagram-posts", @@ -48,49 +50,57 @@ "id": "call-google-maps-restaurants", "category": "call", "query": "Find the top 3 restaurants in Prague using the Google Maps Scraper, I want to know their names, ratings and addresses", - "reference": "The agent must find Google Maps scraper, call it with 'restaurants Prague' query, and return a filtered list of restaurant names with ratings and addresses." + "reference": "The agent must find Google Maps scraper, call it with 'restaurants Prague' query, and return a filtered list of restaurant names with ratings and addresses.", + "expectedTools": ["search-actors", "call-actor"] }, { "id": "call-tiktok-scraper", "category": "call", "query": "Show me 1 latest post from natgeo tik tok using Apify", - "reference": "The agent must find tik tok scraper, call it for natgeo profile, return the latest post and present it to the user." + "reference": "The agent must find tik tok scraper, call it for natgeo profile, return the latest post and present it to the user.", + "expectedTools": ["search-actors", "call-actor"] }, { "id": "fetch-details-mcp-tools-list", "category": "mcp", "query": "Show me what tools are available in the apify/example-mcp-server MCP server", - "reference": "The agent must use fetch-actor-details with output={ mcpTools: true } to list available tools. The response should present the list of tools with their names and descriptions." + "reference": "The agent must use fetch-actor-details with output={ mcpTools: true } to list available tools. The response should present the list of tools with their names and descriptions.", + "expectedTools": ["fetch-actor-details"] }, { "id": "fetch-details-mcp-tools-then-call", "category": "mcp", "query": "List tools from apify/actors-mcp-server and then call the fetch-apify-docs tool from the Actor MCP server apify/actors-mcp-server to get docs from https://docs.apify.com", - "reference": "The agent must first use fetch-actor-details with output={ mcpTools: true } to discover tools, then use call-actor with actor='apify/actors-mcp-server:fetch-apify-docs' and proper input to call the specific tool." + "reference": "The agent must first use fetch-actor-details with output={ mcpTools: true } to discover tools, then use call-actor with actor='apify/actors-mcp-server:fetch-apify-docs' and proper input to call the specific tool.", + "expectedTools": ["fetch-actor-details", "call-actor"] }, { "id": "fetch-details-non-mcp-actor-graceful", "category": "fetch", "query": "Show me MCP tools for apify/instagram-scraper", - "reference": "The agent must use fetch-actor-details with output={ mcpTools: true } for a regular (non-MCP) Actor. The response should gracefully indicate that this Actor is not an MCP server, not throw an error." + "reference": "The agent must use fetch-actor-details with output={ mcpTools: true } for a regular (non-MCP) Actor. The response should gracefully indicate that this Actor is not an MCP server, not throw an error.", + "expectedTools": ["fetch-actor-details"] }, { "id": "workflow-search-fetch-schema-call", "category": "call", "query": "Find a TikTok scraper, check what input it needs, and run it to get 5 videos from user @example", - "reference": "The agent must: 1) use search-actors to find TikTok scraper, 2) use fetch-actor-details with output={ inputSchema: true } to get required input, 3) use call-actor with proper input. This tests the complete new workflow pattern." + "reference": "The agent must: 1) use search-actors to find TikTok scraper, 2) use fetch-actor-details with output={ inputSchema: true } to get required input, 3) use call-actor with proper input. This tests the complete new workflow pattern.", + "expectedTools": ["search-actors", "fetch-actor-details", "call-actor"] }, { "id": "workflow-mcp-discover-and-execute", "category": "mcp", "query": "Use the apify/example-mcp-server to execute one of its tools", - "reference": "The agent must: 1) use fetch-actor-details to list tools, 2) use call-actor to execute. Tests full MCP server discovery and execution workflow." + "reference": "The agent must: 1) use fetch-actor-details to list tools, 2) use call-actor to execute. Tests full MCP server discovery and execution workflow.", + "expectedTools": ["fetch-actor-details", "call-actor"] }, { "id": "storage-dataset-items-seeded", "category": "storage", "query": "Run the apify/rag-web-browser Actor on the URL https://docs.apify.com/ with maxResults set to 1, waiting up to 45 seconds for it to finish. Then tell me the title and the URL of the page that was scraped.", "reference": "The agent must call call-actor for apify/rag-web-browser, then call get-dataset-items with the datasetId from that run, and finally report a page title and a URL. PASS only if get-dataset-items was called with a datasetId argument and the final answer contains a title and a URL.", + "expectedTools": ["call-actor", "get-dataset-items"], "tools": ["actors", "get-dataset-items"], "maxTurns": 12 }, @@ -99,6 +109,7 @@ "category": "storage", "query": "Run the apify/rag-web-browser Actor on the URL https://docs.apify.com/ with maxResults set to 1, waiting up to 45 seconds for it to finish. Then generate a JSON schema describing the structure of that run's dataset items.", "reference": "The agent must call call-actor for apify/rag-web-browser, then call get-dataset-schema with the datasetId from that run. PASS only if get-dataset-schema was called with a datasetId argument.", + "expectedTools": ["call-actor", "get-dataset-schema"], "tools": ["actors", "get-dataset-schema"], "maxTurns": 12 }, @@ -107,6 +118,7 @@ "category": "storage", "query": "Run the apify/rag-web-browser Actor on the URL https://docs.apify.com/ waiting up to 45 seconds for it to finish. When the run finishes call get-dataset to tell me how many items the dataset contains.", "reference": "The agent must call call-actor for apify/rag-web-browser, then call get-dataset with the datasetId from that run. PASS only if get-dataset was called with a datasetId argument and the final answer states an item count.", + "expectedTools": ["call-actor", "get-dataset"], "tools": ["actors", "get-dataset"], "maxTurns": 12 }, @@ -115,6 +127,7 @@ "category": "storage", "query": "Run the apify/rag-web-browser Actor on the URL https://docs.apify.com/ with maxResults set to 1, waiting up to 45 seconds for it to finish. Then use the key-value store keys tool to list the keys stored in that run's key-value store (limit 10) and tell me how many keys there are and what they are named.", "reference": "The agent must call call-actor for apify/rag-web-browser, then call get-key-value-store-keys with the keyValueStoreId from that run. PASS only if get-key-value-store-keys was actually called with a keyValueStoreId argument and the final answer lists one or more key names.", + "expectedTools": ["call-actor", "get-key-value-store-keys"], "tools": ["actors", "get-key-value-store-keys"], "maxTurns": 12 }, @@ -123,6 +136,7 @@ "category": "storage", "query": "Run the apify/rag-web-browser Actor on the URL https://docs.apify.com/ with maxResults set to 1, waiting up to 45 seconds for it to finish. Then read one record from that run's key-value store and tell me what kind of content that record holds.", "reference": "The agent must call call-actor for apify/rag-web-browser, then call get-key-value-store-record with the keyValueStoreId from that run and a recordKey. PASS only if get-key-value-store-record was called with both a keyValueStoreId and a recordKey argument, and the final answer describes the record's content.", + "expectedTools": ["call-actor", "get-key-value-store-record"], "tools": ["actors", "get-key-value-store-record"], "maxTurns": 12 }, @@ -131,6 +145,7 @@ "category": "storage", "query": "Run the apify/rag-web-browser Actor on the URL https://docs.apify.com/ with maxResults set to 1, waiting up to 45 seconds for it to finish. Then look at my recent Actor runs (newest first) and confirm that the run you just started appears, reporting its status and run ID.", "reference": "The agent must call call-actor for apify/rag-web-browser, then call get-actor-run-list. PASS only if get-actor-run-list was actually called and the final answer reports a run ID and a status matching the run the agent just started.", + "expectedTools": ["call-actor", "get-actor-run-list"], "tools": ["actors", "get-actor-run-list"], "maxTurns": 12 }, @@ -139,6 +154,7 @@ "category": "storage", "query": "Use the Google Maps Scraper to find the best 20 restaurants in Prague. When the run finishes, retrieve all of them in a single call. Then tell me how many restaurants were returned and list the names and ratings of the first five.", "reference": "The agent must find the Google Maps scraper and call-actor it for restaurants in Prague, then call get-dataset-items with the datasetId from that run and a limit high enough to return every row in one call. PASS only if get-dataset-items was called with a datasetId argument and the final answer states a restaurant count and lists at least five restaurant names with ratings.", + "expectedTools": ["search-actors", "call-actor", "get-dataset-items"], "tools": ["actors", "get-dataset-items"], "maxTurns": 20 }, @@ -147,6 +163,7 @@ "category": "storage", "query": "Use the Google Maps Scraper to find at least 20 hotels in Vienna. When the run finishes, retrieve all the results and tell me how many hotels were returned and the names and addresses of the first five.", "reference": "The agent must find the Google Maps scraper and call-actor it for hotels in Vienna, then call get-dataset-items with the datasetId from that run and a limit high enough to return every row in one call. PASS only if get-dataset-items was called with a datasetId argument and the final answer states a hotel count and lists at least five hotel names with addresses.", + "expectedTools": ["search-actors", "call-actor", "get-dataset-items"], "tools": ["actors", "get-dataset-items"], "maxTurns": 20 }, @@ -155,6 +172,7 @@ "category": "storage", "query": "Run the apify/rag-web-browser Actor on the URL https://docs.apify.com/ with maxResults set to 1, waiting up to 45 seconds for it to finish. Then look up that specific run by its run ID to confirm its status, and retrieve its dataset items. Tell me the run status and the title of the scraped page.", "reference": "The agent must call call-actor for apify/rag-web-browser, then call get-actor-run with the runId from that call, and call get-dataset-items with the datasetId from that run. PASS only if get-actor-run was called with a runId argument, get-dataset-items was called with a datasetId argument, and the final answer states the run status and a page title.", + "expectedTools": ["call-actor", "get-actor-run", "get-dataset-items"], "tools": ["actors", "get-actor-run", "get-dataset-items"], "maxTurns": 14 }, @@ -163,6 +181,7 @@ "category": "storage", "query": "List my 10 most recent Actor runs, newest first. For each run report its run ID, the Actor it ran, and its status.", "reference": "The agent must call get-actor-run-list (newest first, limit 10). PASS only if get-actor-run-list was actually called and if there are runs, the final answer reports the runs, each with a run ID and a status.", + "expectedTools": ["get-actor-run-list"], "tools": ["get-actor-run-list"], "maxTurns": 6 }, @@ -171,6 +190,7 @@ "category": "storage", "query": "Run a Google search via Apify for 'apify web scraping' and request the top 15 organic results. When the run finishes, retrieve only the title and URL of each result and list them.", "reference": "The agent must find the Google Search Results scraper and call-actor it for the query 'apify web scraping', then call get-dataset-items with the datasetId from that run. PASS only if get-dataset-items was called with a datasetId argument and the final answer lists at least five results, each with a title and a URL.", + "expectedTools": ["search-actors", "call-actor", "get-dataset-items"], "tools": ["actors", "get-dataset-items"], "maxTurns": 16 }, @@ -179,6 +199,7 @@ "category": "storage", "query": "Scrape the 15 latest posts from the natgeo Instagram profile using Apify. When the run finishes, retrieve only the caption and like count of each post and list them.", "reference": "The agent must find an Instagram scraper and call-actor it for the natgeo profile, then call get-dataset-items with the datasetId from that run. PASS only if get-dataset-items was called with a datasetId argument and the final answer lists several posts, each with a caption and a like count.", + "expectedTools": ["search-actors", "call-actor", "get-dataset-items"], "tools": ["actors", "get-dataset-items"], "maxTurns": 18 }, @@ -187,6 +208,7 @@ "category": "storage", "query": "Scrape the 15 latest TikTok videos from the @natgeo profile using Apify. When the run finishes, retrieve only each video's URL and view count and list them.", "reference": "The agent must find a TikTok scraper and call-actor it for the @natgeo profile, then call get-dataset-items with the datasetId from that run. PASS only if get-dataset-items was called with a datasetId argument and the final answer lists several videos, each with a URL and a view count.", + "expectedTools": ["search-actors", "call-actor", "get-dataset-items"], "tools": ["actors", "get-dataset-items"], "maxTurns": 18 }, @@ -195,6 +217,7 @@ "category": "storage", "query": "Use Apify to scrape 15 videos from the Apify YouTube channel. When the run finishes, retrieve only the title and view count of each video and list them.", "reference": "The agent must find a YouTube scraper and call-actor it for the Apify channel, then call get-dataset-items with the datasetId from that run. PASS only if get-dataset-items was called with a datasetId argument and the final answer lists several videos, each with a title and a view count.", + "expectedTools": ["search-actors", "call-actor", "get-dataset-items"], "tools": ["actors", "get-dataset-items"], "maxTurns": 18 }, @@ -203,6 +226,7 @@ "category": "storage", "query": "Scrape 15 posts from the r/technology subreddit using Apify. When the run finishes, retrieve only the title and upvote count of each post and list them.", "reference": "The agent must find a Reddit scraper and call-actor it for the r/technology subreddit, then call get-dataset-items with the datasetId from that run. PASS only if get-dataset-items was called with a datasetId argument and the final answer lists several posts, each with a title and an upvote count.", + "expectedTools": ["search-actors", "call-actor", "get-dataset-items"], "tools": ["actors", "get-dataset-items"], "maxTurns": 18 }, @@ -211,6 +235,7 @@ "category": "report-problem", "query": "Use the apify/rag-web-browser Actor to scrape https://example.com and tell me the page title.", "reference": "While running the Actor, the call-actor tool fails with an internal error, so the task cannot be completed. The user did NOT ask for any problem to be reported. This tests whether the agent proactively reports the blocker after the error. PASS only if the agent calls the report-problem tool with a non-empty message describing what it was trying to do and what went wrong. FAIL if report-problem is not called (e.g., the agent only tells the user it could not complete the task).", + "expectedTools": ["call-actor", "report-problem"], "tools": ["actors", "report-problem"], "failTools": ["call-actor"], "maxTurns": 8 diff --git a/evals/workflows/workflow_judge.ts b/evals/workflows/workflow_judge.ts index a07dc5eba..947467b4e 100644 --- a/evals/workflows/workflow_judge.ts +++ b/evals/workflows/workflow_judge.ts @@ -8,25 +8,80 @@ import type { ResponseFormatJSONSchema } from 'openai/resources/shared'; import type { WorkflowTestCase } from '../shared/types.js'; import { JUDGE_PROMPT_TEMPLATE, MODELS } from './config.js'; +import type { SchemaValidityCheck, ToolSelectionCheck } from './deterministic_checks.js'; +import { checkSchemaValidity, checkToolSelection } from './deterministic_checks.js'; import type { LlmClient } from './llm_client.js'; -import type { ConversationHistory } from './types.js'; +import type { ConversationHistory, McpTool } from './types.js'; + +/** + * Verdict for one rubric dimension + */ +export type DimensionResult = { verdict: 'PASS' | 'FAIL'; reason: string }; + +/** + * The fixed 6-dimension rubric, scored independently for every test case + */ +export type RubricResult = { + toolSelection: DimensionResult; + argumentCorrectness: DimensionResult; + resultUtilization: DimensionResult; + taskCompletion: DimensionResult; + errorRecovery: DimensionResult; + planEfficiency: DimensionResult; +}; + +/** + * Rubric dimensions in display order, with the short glyph label used in the compact results table. + */ +export const RUBRIC_DIMENSIONS = [ + { key: 'toolSelection', glyph: 'tool', label: 'Tool selection' }, + { key: 'argumentCorrectness', glyph: 'args', label: 'Argument correctness' }, + { key: 'resultUtilization', glyph: 'result', label: 'Result utilization' }, + { key: 'taskCompletion', glyph: 'complete', label: 'Task completion' }, + { key: 'errorRecovery', glyph: 'recover', label: 'Error recovery' }, + { key: 'planEfficiency', glyph: 'eff', label: 'Plan efficiency' }, +] as const satisfies readonly { key: keyof RubricResult; glyph: string; label: string }[]; /** * Judge evaluation result */ export type JudgeResult = { - /** PASS or FAIL verdict */ - verdict: 'PASS' | 'FAIL'; - /** Explanation from judge */ - reason: string; + /** Mirrors rubric.taskCompletion.verdict — informational, the harness exit code does not use it */ + overallVerdict: 'PASS' | 'FAIL'; + /** The 6 rubric dimensions (toolSelection may be overridden by the deterministic check) */ + rubric: RubricResult; + /** Deterministic expectedTools exact-match */ + toolSelectionCheck: ToolSelectionCheck; + /** Deterministic per-call inputSchema validation */ + schemaValidityCheck: SchemaValidityCheck; /** Raw response from judge (for debugging) */ rawResponse: string; }; +/** Per-result cap on tool output shown to the judge, so a huge dataset can't crowd out the rest. */ +const MAX_RESULT_CHARS = 2000; + /** * JSON schema for structured judge output * Guarantees the LLM returns valid JSON matching this schema */ +const DIMENSION_SCHEMA = { + type: 'object', + properties: { + verdict: { + type: 'string', + enum: ['PASS', 'FAIL'], + description: 'Whether the agent passed or failed this dimension', + }, + reason: { + type: 'string', + description: 'Brief explanation in 1-2 sentences', + }, + }, + required: ['verdict', 'reason'], + additionalProperties: false, +}; + const JUDGE_RESPONSE_SCHEMA: ResponseFormatJSONSchema = { type: 'json_schema', json_schema: { @@ -34,26 +89,27 @@ const JUDGE_RESPONSE_SCHEMA: ResponseFormatJSONSchema = { strict: true, schema: { type: 'object', - properties: { - verdict: { - type: 'string', - enum: ['PASS', 'FAIL'], - description: 'Whether the agent passed or failed the evaluation', - }, - reason: { - type: 'string', - description: 'Brief explanation in 1-2 sentences explaining why the agent passed or failed', - }, - }, - required: ['verdict', 'reason'], + properties: Object.fromEntries(RUBRIC_DIMENSIONS.map(({ key }) => [key, DIMENSION_SCHEMA])), + required: RUBRIC_DIMENSIONS.map(({ key }) => key), additionalProperties: false, }, }, }; +/** + * Serialize a tool result and cut it to `maxChars`, noting the cut so the judge knows the output + * continued rather than ended. + */ +export function truncateResult(result: unknown, maxChars = MAX_RESULT_CHARS): string { + const serialized = typeof result === 'string' ? result : JSON.stringify(result); + if (serialized === undefined) return 'undefined'; + if (serialized.length <= maxChars) return serialized; + return `${serialized.slice(0, maxChars)} [truncated at ${maxChars} chars, ${serialized.length} total]`; +} + /** * Format conversation for judge evaluation - * Judge sees: tool calls + arguments + final responses (NOT tool results) + * Judge sees: tool calls + arguments + truncated tool results + final responses */ function formatConversationForJudge(conversation: ConversationHistory): string { const lines: string[] = []; @@ -64,10 +120,17 @@ function formatConversationForJudge(conversation: ConversationHistory): string { // Each turn for (const turn of conversation.turns) { - // Show tool calls (if any) - if (turn.toolCalls.length > 0) { - for (const toolCall of turn.toolCalls) { - lines.push(`AGENT: [Called tool: ${toolCall.name} with args: ${JSON.stringify(toolCall.arguments)}]`); + // Show tool calls paired with their results (both arrays are filled in call order) + for (const [index, toolCall] of turn.toolCalls.entries()) { + lines.push(`AGENT: [Called tool: ${toolCall.name} with args: ${JSON.stringify(toolCall.arguments)}]`); + + const toolResult = turn.toolResults[index]; + if (!toolResult) continue; + if (toolResult.success) { + lines.push(`TOOL RESULT (${toolResult.toolName}): ${truncateResult(toolResult.result)}`); + } else { + // Show the error text, not "no result" — errorRecovery is scored on what the agent saw. + lines.push(`TOOL ERROR (${toolResult.toolName}): ${truncateResult(toolResult.error)}`); } } @@ -85,36 +148,64 @@ function formatConversationForJudge(conversation: ConversationHistory): string { /** * Parse structured JSON response from judge */ -function parseJudgeResponse(response: string): { verdict: 'PASS' | 'FAIL'; reason: string } { +function parseJudgeResponse(response: string): RubricResult { + let parsed: Partial>; try { - const parsed = JSON.parse(response) as { verdict: 'PASS' | 'FAIL'; reason: string }; - - // Validate the structure (should be guaranteed by schema, but double-check) - if (!parsed.verdict || (parsed.verdict !== 'PASS' && parsed.verdict !== 'FAIL')) { - throw new Error(`Invalid verdict: ${parsed.verdict}`); - } - - if (!parsed.reason || typeof parsed.reason !== 'string') { - throw new Error(`Invalid reason: ${parsed.reason}`); - } - - return parsed; + parsed = JSON.parse(response) as Partial>; } catch (error) { throw new Error( `Failed to parse judge JSON response: ${error instanceof Error ? error.message : String(error)}\n` + `Raw response: ${response}`, ); } + + // Validate the structure (should be guaranteed by schema, but double-check) + for (const { key } of RUBRIC_DIMENSIONS) { + const dimension = parsed[key]; + if (!dimension || (dimension.verdict !== 'PASS' && dimension.verdict !== 'FAIL')) { + throw new Error(`Invalid verdict for dimension ${key}: ${JSON.stringify(dimension)}`); + } + if (!dimension.reason || typeof dimension.reason !== 'string') { + throw new Error(`Invalid reason for dimension ${key}: ${JSON.stringify(dimension.reason)}`); + } + } + + return parsed as RubricResult; } /** - * Evaluate a conversation using the judge LLM + * Build a JudgeResult for a test that errored before it could be judged. + */ +export function createErrorJudgeResult(reason: string): JudgeResult { + const dimension: DimensionResult = { verdict: 'FAIL', reason }; + return { + overallVerdict: 'FAIL', + rubric: { + toolSelection: dimension, + argumentCorrectness: dimension, + resultUtilization: dimension, + taskCompletion: dimension, + errorRecovery: dimension, + planEfficiency: dimension, + }, + toolSelectionCheck: { checked: false, verdict: null, expected: [], actual: [] }, + schemaValidityCheck: { verdict: 'PASS', invalidCalls: [] }, + rawResponse: '', + }; +} + +/** + * Evaluate a conversation using the judge LLM plus the deterministic checks. + * + * @param mcpTools - Tool list snapshot taken after the conversation finished, used to validate + * each tool call's arguments against the tool's declared inputSchema */ export async function evaluateConversation( testCase: WorkflowTestCase, conversation: ConversationHistory, llmClient: LlmClient, judgeModel: string = MODELS.judge, + mcpTools: McpTool[] = [], ): Promise { // Format conversation for judge const formattedConversation = formatConversationForJudge(conversation); @@ -134,19 +225,27 @@ export async function evaluateConversation( ); const rawResponse = response.content || ''; - - // Parse response - try { - const { verdict, reason } = parseJudgeResponse(rawResponse); - return { - verdict, - reason, - rawResponse, + const rubric = parseJudgeResponse(rawResponse); + + // Deterministic checks. When expectedTools is set, code decides toolSelection and the judge's + // own answer for that dimension is discarded. + const toolSelectionCheck = checkToolSelection(testCase.expectedTools, conversation); + if (toolSelectionCheck.checked) { + rubric.toolSelection = { + verdict: toolSelectionCheck.verdict as 'PASS' | 'FAIL', + reason: + `Deterministic: expected [${toolSelectionCheck.expected.join(', ')}], ` + + `got [${toolSelectionCheck.actual.join(', ')}]`, }; - } catch (error) { - throw new Error( - `Failed to parse judge response: ${error instanceof Error ? error.message : String(error)}\n` + - `Raw response: ${rawResponse}`, - ); } + + const schemaValidityCheck = checkSchemaValidity(conversation, mcpTools); + + return { + overallVerdict: rubric.taskCompletion.verdict, + rubric, + toolSelectionCheck, + schemaValidityCheck, + rawResponse, + }; } diff --git a/tests/unit/evals.deterministic_checks.test.ts b/tests/unit/evals.deterministic_checks.test.ts new file mode 100644 index 000000000..ff562c39a --- /dev/null +++ b/tests/unit/evals.deterministic_checks.test.ts @@ -0,0 +1,156 @@ +import { describe, expect, it } from 'vitest'; + +import { checkSchemaValidity, checkToolSelection } from '../../evals/workflows/deterministic_checks.js'; +import type { ConversationHistory, McpTool } from '../../evals/workflows/types.js'; + +function makeConversation( + toolCallsPerTurn: { name: string; arguments: Record }[][], +): ConversationHistory { + return { + userPrompt: 'test', + turns: toolCallsPerTurn.map((toolCalls, index) => ({ + turnNumber: index + 1, + toolCalls, + toolResults: toolCalls.map((call) => ({ toolName: call.name, success: true, result: {} })), + })), + completed: true, + hitMaxTurns: false, + totalTurns: toolCallsPerTurn.length, + }; +} + +describe('checkToolSelection()', () => { + it('reports not-checked when the test case declares no expectedTools', () => { + const check = checkToolSelection(undefined, makeConversation([[{ name: 'search-actors', arguments: {} }]])); + expect(check).toEqual({ checked: false, verdict: null, expected: [], actual: ['search-actors'] }); + }); + + it('reports not-checked for an empty expectedTools array', () => { + expect(checkToolSelection([], makeConversation([])).checked).toBe(false); + }); + + it('passes on an exact match regardless of call order', () => { + const conversation = makeConversation([ + [{ name: 'call-actor', arguments: {} }], + [{ name: 'get-dataset-items', arguments: {} }], + ]); + const check = checkToolSelection(['get-dataset-items', 'call-actor'], conversation); + expect(check.verdict).toBe('PASS'); + expect(check.expected).toEqual(['call-actor', 'get-dataset-items']); + expect(check.actual).toEqual(['call-actor', 'get-dataset-items']); + }); + + it('passes when the agent calls the same expected tool several times', () => { + const conversation = makeConversation([ + [{ name: 'get-dataset-items', arguments: {} }], + [{ name: 'get-dataset-items', arguments: {} }], + ]); + expect(checkToolSelection(['get-dataset-items'], conversation).verdict).toBe('PASS'); + }); + + it('fails on an extra tool call', () => { + const conversation = makeConversation([ + [ + { name: 'call-actor', arguments: {} }, + { name: 'search-actors', arguments: {} }, + ], + ]); + const check = checkToolSelection(['call-actor'], conversation); + expect(check.verdict).toBe('FAIL'); + expect(check.actual).toEqual(['call-actor', 'search-actors']); + }); + + it('fails on a missing tool call', () => { + const conversation = makeConversation([[{ name: 'call-actor', arguments: {} }]]); + expect(checkToolSelection(['call-actor', 'get-dataset-items'], conversation).verdict).toBe('FAIL'); + }); + + it('fails when the agent called nothing at all', () => { + const check = checkToolSelection(['search-actors'], makeConversation([])); + expect(check.verdict).toBe('FAIL'); + expect(check.actual).toEqual([]); + }); +}); + +describe('checkSchemaValidity()', () => { + const tools: McpTool[] = [ + { + name: 'get-dataset-items', + inputSchema: { + $id: 'get-dataset-items', + type: 'object', + properties: { datasetId: { type: 'string' }, limit: { type: 'integer' } }, + required: ['datasetId'], + }, + }, + ]; + + it('passes when every call matches its tool schema', () => { + const conversation = makeConversation([ + [{ name: 'get-dataset-items', arguments: { datasetId: 'd1', limit: 5 } }], + ]); + expect(checkSchemaValidity(conversation, tools)).toEqual({ verdict: 'PASS', invalidCalls: [] }); + }); + + it('passes for a conversation with no tool calls', () => { + expect(checkSchemaValidity(makeConversation([]), tools).verdict).toBe('PASS'); + }); + + it('flags a call missing a required property', () => { + const conversation = makeConversation([[{ name: 'get-dataset-items', arguments: { limit: 5 } }]]); + const check = checkSchemaValidity(conversation, tools); + expect(check.verdict).toBe('FAIL'); + expect(check.invalidCalls).toHaveLength(1); + expect(check.invalidCalls[0].toolName).toBe('get-dataset-items'); + expect(check.invalidCalls[0].errors.join()).toContain('datasetId'); + }); + + it('flags a call with a wrongly typed property', () => { + const conversation = makeConversation([ + [{ name: 'get-dataset-items', arguments: { datasetId: 'd1', limit: 'five' } }], + ]); + const check = checkSchemaValidity(conversation, tools); + expect(check.verdict).toBe('FAIL'); + expect(check.invalidCalls[0].errors.join()).toContain('/limit'); + }); + + it('validates repeat calls to the same tool against one cached validator', () => { + const conversation = makeConversation([ + [{ name: 'get-dataset-items', arguments: { datasetId: 'd1' } }], + [{ name: 'get-dataset-items', arguments: {} }], + ]); + const check = checkSchemaValidity(conversation, tools); + expect(check.invalidCalls).toHaveLength(1); + }); + + it('skips a call to a tool that is not in the final tool list', () => { + const conversation = makeConversation([[{ name: 'since-deregistered-tool', arguments: { anything: 1 } }]]); + expect(checkSchemaValidity(conversation, tools).verdict).toBe('PASS'); + }); + + it('skips a tool whose schema cannot be compiled', () => { + const brokenTools: McpTool[] = [ + { name: 'broken', inputSchema: { type: 'object', properties: { a: { type: 'not-a-type' } } } }, + ]; + const conversation = makeConversation([[{ name: 'broken', arguments: { a: 1 } }]]); + expect(checkSchemaValidity(conversation, brokenTools).verdict).toBe('PASS'); + }); + + it('does not flag a failed tool call whose arguments were schema-valid', () => { + // failTools makes the harness synthesize an error result; the call itself is still valid. + const conversation: ConversationHistory = { + userPrompt: 'test', + turns: [ + { + turnNumber: 1, + toolCalls: [{ name: 'get-dataset-items', arguments: { datasetId: 'd1' } }], + toolResults: [{ toolName: 'get-dataset-items', success: false, error: 'internal error' }], + }, + ], + completed: true, + hitMaxTurns: false, + totalTurns: 1, + }; + expect(checkSchemaValidity(conversation, tools).verdict).toBe('PASS'); + }); +}); diff --git a/tests/unit/evals.workflow_judge.test.ts b/tests/unit/evals.workflow_judge.test.ts new file mode 100644 index 000000000..99a1c960c --- /dev/null +++ b/tests/unit/evals.workflow_judge.test.ts @@ -0,0 +1,123 @@ +import { describe, expect, it, vi } from 'vitest'; + +import type { WorkflowTestCase } from '../../evals/shared/types.js'; +import type { LlmClient } from '../../evals/workflows/llm_client.js'; +import type { ConversationHistory, McpTool } from '../../evals/workflows/types.js'; +import { evaluateConversation, truncateResult } from '../../evals/workflows/workflow_judge.js'; + +/** Judge response with every dimension passing, so a FAIL can only come from a deterministic check. */ +const ALL_PASS_RESPONSE = JSON.stringify({ + toolSelection: { verdict: 'PASS', reason: 'judge says tools were fine' }, + argumentCorrectness: { verdict: 'PASS', reason: 'args ok' }, + resultUtilization: { verdict: 'PASS', reason: 'results used' }, + taskCompletion: { verdict: 'PASS', reason: 'task done' }, + errorRecovery: { verdict: 'PASS', reason: 'nothing to recover from' }, + planEfficiency: { verdict: 'PASS', reason: 'direct path' }, +}); + +function makeLlmClient(content: string): { client: LlmClient; callLlm: ReturnType } { + const callLlm = vi.fn().mockResolvedValue({ content }); + return { client: { callLlm } as unknown as LlmClient, callLlm }; +} + +function makeConversation(): ConversationHistory { + return { + userPrompt: 'run the actor', + turns: [ + { + turnNumber: 1, + toolCalls: [{ name: 'call-actor', arguments: { actor: 'apify/rag-web-browser' } }], + toolResults: [{ toolName: 'call-actor', success: false, error: 'internal error' }], + }, + { turnNumber: 2, toolCalls: [], toolResults: [], finalResponse: 'It failed.' }, + ], + completed: true, + hitMaxTurns: false, + totalTurns: 2, + }; +} + +const TEST_CASE: WorkflowTestCase = { id: 't', category: 'call', query: 'q', reference: 'requirements' }; + +describe('truncateResult()', () => { + it('returns short results unchanged', () => { + expect(truncateResult({ a: 1 })).toBe('{"a":1}'); + }); + + it('keeps a result exactly at the cap unchanged', () => { + expect(truncateResult('x'.repeat(10), 10)).toBe('x'.repeat(10)); + }); + + it('slices past the cap and notes the original length', () => { + expect(truncateResult('x'.repeat(12), 10)).toBe(`${'x'.repeat(10)} [truncated at 10 chars, 12 total]`); + }); + + it('passes strings through without re-quoting them', () => { + expect(truncateResult('plain error text')).toBe('plain error text'); + }); +}); + +describe('evaluateConversation()', () => { + it('sets overallVerdict from the taskCompletion dimension', async () => { + const response = JSON.parse(ALL_PASS_RESPONSE); + response.taskCompletion = { verdict: 'FAIL', reason: 'never answered' }; + const { client } = makeLlmClient(JSON.stringify(response)); + + const result = await evaluateConversation(TEST_CASE, makeConversation(), client); + expect(result.overallVerdict).toBe('FAIL'); + expect(result.rubric.planEfficiency.verdict).toBe('PASS'); + }); + + it('overrides the judge toolSelection verdict when expectedTools is set', async () => { + const { client } = makeLlmClient(ALL_PASS_RESPONSE); + const testCase: WorkflowTestCase = { ...TEST_CASE, expectedTools: ['call-actor', 'get-dataset-items'] }; + + const result = await evaluateConversation(testCase, makeConversation(), client); + expect(result.toolSelectionCheck.checked).toBe(true); + // Judge said PASS; the deterministic mismatch wins. + expect(result.rubric.toolSelection.verdict).toBe('FAIL'); + expect(result.rubric.toolSelection.reason).toBe( + 'Deterministic: expected [call-actor, get-dataset-items], got [call-actor]', + ); + }); + + it('keeps the judge toolSelection verdict when expectedTools is unset', async () => { + const { client } = makeLlmClient(ALL_PASS_RESPONSE); + + const result = await evaluateConversation(TEST_CASE, makeConversation(), client); + expect(result.toolSelectionCheck.checked).toBe(false); + expect(result.rubric.toolSelection.reason).toBe('judge says tools were fine'); + }); + + it('does not flag a forced tool failure as a schema violation', async () => { + const { client } = makeLlmClient(ALL_PASS_RESPONSE); + const tools: McpTool[] = [ + { + name: 'call-actor', + inputSchema: { type: 'object', properties: { actor: { type: 'string' } }, required: ['actor'] }, + }, + ]; + + const result = await evaluateConversation(TEST_CASE, makeConversation(), client, undefined, tools); + expect(result.schemaValidityCheck).toEqual({ verdict: 'PASS', invalidCalls: [] }); + }); + + it('shows tool results and errors to the judge', async () => { + const { client, callLlm } = makeLlmClient(ALL_PASS_RESPONSE); + + await evaluateConversation(TEST_CASE, makeConversation(), client); + const prompt = callLlm.mock.calls[0][0][0].content as string; + expect(prompt).toContain('AGENT: [Called tool: call-actor with args: {"actor":"apify/rag-web-browser"}]'); + expect(prompt).toContain('TOOL ERROR (call-actor): internal error'); + }); + + it('rejects a judge response missing a dimension', async () => { + const partial = JSON.parse(ALL_PASS_RESPONSE); + delete partial.errorRecovery; + const { client } = makeLlmClient(JSON.stringify(partial)); + + await expect(evaluateConversation(TEST_CASE, makeConversation(), client)).rejects.toThrow( + /Invalid verdict for dimension errorRecovery/, + ); + }); +}); diff --git a/tests/unit/evals.workflow_output.test.ts b/tests/unit/evals.workflow_output.test.ts index 793708212..11bf18314 100644 --- a/tests/unit/evals.workflow_output.test.ts +++ b/tests/unit/evals.workflow_output.test.ts @@ -3,12 +3,41 @@ import { describe, expect, it } from 'vitest'; import type { EvaluationResult, TestResultRecord } from '../../evals/workflows/output_formatter.js'; import { formatBytes, + formatDetailedResult, + formatPassRateWithDelta, formatResultsTable, + formatRubricGlyphs, formatTokens, formatWithDelta, sumResultBytes, } from '../../evals/workflows/output_formatter.js'; import type { ConversationHistory } from '../../evals/workflows/types.js'; +import type { JudgeResult, RubricResult } from '../../evals/workflows/workflow_judge.js'; + +function makeRubric(overrides: Partial = {}): RubricResult { + const pass = { verdict: 'PASS' as const, reason: 'ok' }; + return { + toolSelection: pass, + argumentCorrectness: pass, + resultUtilization: pass, + taskCompletion: pass, + errorRecovery: pass, + planEfficiency: pass, + ...overrides, + }; +} + +function makeJudgeResult(overrides: Partial = {}): JudgeResult { + const rubric = overrides.rubric ?? makeRubric(); + return { + overallVerdict: rubric.taskCompletion.verdict, + rubric, + toolSelectionCheck: { checked: false, verdict: null, expected: [], actual: [] }, + schemaValidityCheck: { verdict: 'PASS', invalidCalls: [] }, + rawResponse: '', + ...overrides, + }; +} function makeConversation(turns: ConversationHistory['turns']): ConversationHistory { return { @@ -123,7 +152,7 @@ describe('formatResultsTable()', () => { ]), totalTokens: tokens, }, - judgeResult: { verdict: 'PASS', reason: 'ok', rawResponse: '' }, + judgeResult: makeJudgeResult(), durationMs: 100, }; } @@ -169,4 +198,132 @@ describe('formatResultsTable()', () => { const table = formatResultsTable([makeResult('b', 1000, 500)], baseline); expect(table).toContain('(no baseline)'); }); + + it('renders the compact rubric glyph line and per-dimension pass rates', () => { + const result = makeResult('a', 1000, 500); + result.judgeResult = makeJudgeResult({ + rubric: makeRubric({ resultUtilization: { verdict: 'FAIL', reason: 'ignored an error' } }), + }); + const table = formatResultsTable([result]); + expect(table).toContain('Rubric: tool✓ args✓ result✗ complete✓ recover✓ eff✓'); + expect(table).toContain('Result utilization: 0/1 passed'); + expect(table).toContain('Task completion: 1/1 passed'); + }); + + it('derives the per-test status from overallVerdict, not from the other dimensions', () => { + const result = makeResult('a', 1000, 500); + result.judgeResult = makeJudgeResult({ + rubric: makeRubric({ taskCompletion: { verdict: 'FAIL', reason: 'no answer' } }), + }); + const table = formatResultsTable([result]); + expect(table).toContain('❌ FAIL | a'); + expect(table).toContain('Reason: no answer'); + }); + + it('lists schema-invalid calls only when the schema check fails', () => { + const passing = formatResultsTable([makeResult('a', 1000, 500)]); + expect(passing).not.toContain('Schema-invalid calls'); + + const result = makeResult('a', 1000, 500); + result.judgeResult = makeJudgeResult({ + schemaValidityCheck: { + verdict: 'FAIL', + invalidCalls: [{ toolName: 'get-dataset-items', errors: ['/limit must be integer'] }], + }, + }); + expect(formatResultsTable([result])).toContain( + 'Schema-invalid calls: get-dataset-items (/limit must be integer)', + ); + }); + + it('shows per-dimension baseline deltas only for baselines that carry a rubric', () => { + const withoutRubric = new Map([['a', makeRecord('a', 2000, 800)]]); + expect(formatResultsTable([makeResult('a', 1000, 500)], withoutRubric)).not.toContain('Task completion ('); + + const record = makeRecord('a', 2000, 800); + record.rubric = makeRubric(); + const withRubric = new Map([['a', record]]); + const result = makeResult('a', 1000, 500); + result.judgeResult = makeJudgeResult({ + rubric: makeRubric({ planEfficiency: { verdict: 'FAIL', reason: 'looped' } }), + }); + const table = formatResultsTable([result], withRubric); + expect(table).toContain('Task completion (1/1): 1/1 passed (= baseline)'); + expect(table).toContain('Plan efficiency (1/1): 0/1 passed (▼ -1 vs baseline 1/1)'); + }); +}); + +describe('formatRubricGlyphs()', () => { + it('renders one glyph per dimension in a fixed order', () => { + expect(formatRubricGlyphs(makeRubric())).toBe('tool✓ args✓ result✓ complete✓ recover✓ eff✓'); + expect(formatRubricGlyphs(makeRubric({ errorRecovery: { verdict: 'FAIL', reason: 'stalled' } }))).toBe( + 'tool✓ args✓ result✓ complete✓ recover✗ eff✓', + ); + }); +}); + +describe('formatPassRateWithDelta()', () => { + it('marks an unchanged pass count', () => { + expect(formatPassRateWithDelta(27, 27, 30)).toBe('27/30 passed (= baseline)'); + }); + + it('marks more passes with ▲ (better)', () => { + expect(formatPassRateWithDelta(29, 27, 30)).toBe('29/30 passed (▲ +2 vs baseline 27/30)'); + }); + + it('marks fewer passes with ▼ (worse)', () => { + expect(formatPassRateWithDelta(27, 29, 30)).toBe('27/30 passed (▼ -2 vs baseline 29/30)'); + }); +}); + +describe('formatDetailedResult()', () => { + function makeVerboseResult(judgeResult: JudgeResult): EvaluationResult { + return { + testCase: { id: 'a', category: 'basic', query: 'q', reference: 'r' } as EvaluationResult['testCase'], + conversation: makeConversation([{ turnNumber: 1, toolCalls: [], toolResults: [] }]), + judgeResult, + durationMs: 100, + }; + } + + it('prints every dimension verdict with its reason', () => { + const output = formatDetailedResult( + makeVerboseResult( + makeJudgeResult({ + rubric: makeRubric({ resultUtilization: { verdict: 'FAIL', reason: 'misread the dataset' } }), + }), + ), + ); + expect(output).toContain('✓ Tool selection: ok'); + expect(output).toContain('✗ Result utilization: misread the dataset'); + }); + + it('prints both deterministic check results', () => { + const output = formatDetailedResult( + makeVerboseResult( + makeJudgeResult({ + toolSelectionCheck: { + checked: true, + verdict: 'FAIL', + expected: ['call-actor'], + actual: ['call-actor', 'search-actors'], + }, + schemaValidityCheck: { + verdict: 'FAIL', + invalidCalls: [{ toolName: 'call-actor', errors: ["must have required property 'actor'"] }], + }, + }), + ), + ); + expect(output).toContain('Tool selection: FAIL'); + expect(output).toContain('Expected: [call-actor]'); + expect(output).toContain('Actual: [call-actor, search-actors]'); + expect(output).toContain('Schema validity: FAIL'); + expect(output).toContain("call-actor: must have required property 'actor'"); + }); + + it('says tool selection was not checked when the test case declares no expectedTools', () => { + const output = formatDetailedResult(makeVerboseResult(makeJudgeResult())); + expect(output).toContain('not checked (no expectedTools on this test case)'); + }); });