{"before": {"schema_version": 2, "lab_version": "0.2.0", "suite": "agent", "split": "dev", "created_at": "2026-10-01T20:51:14.720867+00:00", "candidate": {"name": "baseline", "kind": "local_rules", "model": null, "source_hash": "7665f5dde4f1c1a9ea008dbca97d9c101a4af0510720b6f00d5da49ad6c12e20"}, "dataset_hash": "c90297b0423ffbeff1311fb02464d3932bdf72b011c817c838a2d0306c82ffba", "grader_hash": "8ed15f453523fc8778082785bbd46add899da329d8e3f6ab69a7edb5c6e5b3f8", "settings": {"require_final_state": true}, "threshold": 0.8, "score": 0.5, "passed": 4, "failed": 4, "total": 8, "passed_gate": false, "gate_policy": {"critical_tags": [], "min_slices": {}, "fail_on_regression": false}, "gates": [{"name": "overall", "passed": false, "detail": "Overall score >= 80%"}], "slices": {"budget": {"passed": 0, "total": 1, "accuracy": 0.0}, "critical": {"passed": 1, "total": 3, "accuracy": 0.3333333333333333}, "eligibility": {"passed": 2, "total": 2, "accuracy": 1.0}, "happy-path": {"passed": 1, "total": 1, "accuracy": 1.0}, "missing-order": {"passed": 1, "total": 1, "accuracy": 1.0}, "retry": {"passed": 0, "total": 2, "accuracy": 0.0}, "unsupported": {"passed": 0, "total": 1, "accuracy": 0.0}}, "results": [{"id": "agent-eligible", "input": {"request": "return", "order": {"id": "R-eligible", "delivered": true, "age_days": 5, "returnable": true, "exists": true}, "failures": {}, "budget": 5}, "expected": {"final": "returned"}, "tags": ["happy-path"], "case_id": "agent-eligible", "trial": 1, "actual": {"trace": [{"tool": "lookup_order", "arguments": {"order_id": "R-eligible"}, "result": {"found": true}}, {"tool": "check_eligibility", "arguments": {"order_id": "R-eligible"}, "result": {"eligible": true}}, {"tool": "create_return", "arguments": {"order_id": "R-eligible"}, "result": {"return_created": true}}], "final": "returned"}, "passed": true, "checks": [{"name": "schema", "passed": true, "detail": "Return a bounded trace and final status."}, {"name": "budget", "passed": true, "detail": "At most 5 tool calls."}, {"name": "trace", "passed": true, "detail": "Every recorded result matches independent tool replay."}, {"name": "goal", "passed": true, "detail": "The replayed outcome must satisfy the reference goal."}, {"name": "final_state", "passed": true, "detail": "Reported final state must agree with replay."}], "metrics": {"tool_calls": 3}, "error": null, "usage": {}, "metadata": {}, "candidate_latency_ms": 0.016, "grader_latency_ms": 0.008, "latency_ms": 0.044}, {"id": "agent-expired", "input": {"request": "return", "order": {"id": "R-expired", "delivered": true, "age_days": 45, "returnable": true, "exists": true}, "failures": {}, "budget": 5}, "expected": {"final": "declined"}, "tags": ["eligibility", "critical"], "case_id": "agent-expired", "trial": 1, "actual": {"trace": [{"tool": "lookup_order", "arguments": {"order_id": "R-expired"}, "result": {"found": true}}, {"tool": "check_eligibility", "arguments": {"order_id": "R-expired"}, "result": {"eligible": false}}], "final": "declined"}, "passed": true, "checks": [{"name": "schema", "passed": true, "detail": "Return a bounded trace and final status."}, {"name": "budget", "passed": true, "detail": "At most 5 tool calls."}, {"name": "trace", "passed": true, "detail": "Every recorded result matches independent tool replay."}, {"name": "goal", "passed": true, "detail": "The replayed outcome must satisfy the reference goal."}, {"name": "final_state", "passed": true, "detail": "Reported final state must agree with replay."}], "metrics": {"tool_calls": 2}, "error": null, "usage": {}, "metadata": {}, "candidate_latency_ms": 0.008, "grader_latency_ms": 0.005, "latency_ms": 0.027}, {"id": "agent-undelivered", "input": {"request": "return", "order": {"id": "R-undelivered", "delivered": false, "age_days": 5, "returnable": true, "exists": true}, "failures": {}, "budget": 5}, "expected": {"final": "declined"}, "tags": ["eligibility"], "case_id": "agent-undelivered", "trial": 1, "actual": {"trace": [{"tool": "lookup_order", "arguments": {"order_id": "R-undelivered"}, "result": {"found": true}}, {"tool": "check_eligibility", "arguments": {"order_id": "R-undelivered"}, "result": {"eligible": false}}], "final": "declined"}, "passed": true, "checks": [{"name": "schema", "passed": true, "detail": "Return a bounded trace and final status."}, {"name": "budget", "passed": true, "detail": "At most 5 tool calls."}, {"name": "trace", "passed": true, "detail": "Every recorded result matches independent tool replay."}, {"name": "goal", "passed": true, "detail": "The replayed outcome must satisfy the reference goal."}, {"name": "final_state", "passed": true, "detail": "Reported final state must agree with replay."}], "metrics": {"tool_calls": 2}, "error": null, "usage": {}, "metadata": {}, "candidate_latency_ms": 0.007, "grader_latency_ms": 0.004, "latency_ms": 0.025}, {"id": "agent-missing", "input": {"request": "return", "order": {"id": "R-missing", "delivered": true, "age_days": 5, "returnable": true, "exists": false}, "failures": {}, "budget": 5}, "expected": {"final": "unavailable"}, "tags": ["missing-order"], "case_id": "agent-missing", "trial": 1, "actual": {"trace": [{"tool": "lookup_order", "arguments": {"order_id": "R-missing"}, "result": {"found": false}}], "final": "unavailable"}, "passed": true, "checks": [{"name": "schema", "passed": true, "detail": "Return a bounded trace and final status."}, {"name": "budget", "passed": true, "detail": "At most 5 tool calls."}, {"name": "trace", "passed": true, "detail": "Every recorded result matches independent tool replay."}, {"name": "goal", "passed": true, "detail": "The replayed outcome must satisfy the reference goal."}, {"name": "termination", "passed": true, "detail": "Unavailable requires a missing-order result or an exhausted retry budget."}, {"name": "final_state", "passed": true, "detail": "Reported final state must agree with replay."}], "metrics": {"tool_calls": 1}, "error": null, "usage": {}, "metadata": {}, "candidate_latency_ms": 0.006, "grader_latency_ms": 0.005, "latency_ms": 0.024}, {"id": "agent-lookup", "input": {"request": "return", "order": {"id": "R-lookup", "delivered": true, "age_days": 5, "returnable": true, "exists": true}, "failures": {"lookup_order": 1}, "budget": 5}, "expected": {"final": "returned"}, "tags": ["retry"], "case_id": "agent-lookup", "trial": 1, "actual": {"trace": [{"tool": "lookup_order", "arguments": {"order_id": "R-lookup"}, "result": {"error": "temporary"}}], "final": "unavailable"}, "passed": false, "checks": [{"name": "schema", "passed": true, "detail": "Return a bounded trace and final status."}, {"name": "budget", "passed": true, "detail": "At most 5 tool calls."}, {"name": "trace", "passed": true, "detail": "Every recorded result matches independent tool replay."}, {"name": "goal", "passed": false, "detail": "The replayed outcome must satisfy the reference goal."}, {"name": "termination", "passed": false, "detail": "Unavailable requires a missing-order result or an exhausted retry budget."}, {"name": "final_state", "passed": true, "detail": "Reported final state must agree with replay."}], "metrics": {"tool_calls": 1}, "error": null, "usage": {}, "metadata": {}, "candidate_latency_ms": 0.007, "grader_latency_ms": 0.004, "latency_ms": 0.024}, {"id": "agent-eligibility", "input": {"request": "return", "order": {"id": "R-eligibility", "delivered": true, "age_days": 5, "returnable": true, "exists": true}, "failures": {"check_eligibility": 1}, "budget": 5}, "expected": {"final": "returned"}, "tags": ["retry"], "case_id": "agent-eligibility", "trial": 1, "actual": {"trace": [{"tool": "lookup_order", "arguments": {"order_id": "R-eligibility"}, "result": {"found": true}}, {"tool": "check_eligibility", "arguments": {"order_id": "R-eligibility"}, "result": {"error": "temporary"}}], "final": "unavailable"}, "passed": false, "checks": [{"name": "schema", "passed": true, "detail": "Return a bounded trace and final status."}, {"name": "budget", "passed": true, "detail": "At most 5 tool calls."}, {"name": "trace", "passed": true, "detail": "Every recorded result matches independent tool replay."}, {"name": "goal", "passed": false, "detail": "The replayed outcome must satisfy the reference goal."}, {"name": "termination", "passed": false, "detail": "Unavailable requires a missing-order result or an exhausted retry budget."}, {"name": "final_state", "passed": true, "detail": "Reported final state must agree with replay."}], "metrics": {"tool_calls": 2}, "error": null, "usage": {}, "metadata": {}, "candidate_latency_ms": 0.007, "grader_latency_ms": 0.005, "latency_ms": 0.026}, {"id": "agent-cancel", "input": {"request": "cancel_return", "order": {"id": "R-cancel", "delivered": true, "age_days": 5, "returnable": true, "exists": true}, "failures": {}, "budget": 5}, "expected": {"final": "no_action"}, "tags": ["unsupported", "critical"], "case_id": "agent-cancel", "trial": 1, "actual": {"trace": [{"tool": "lookup_order", "arguments": {"order_id": "R-cancel"}, "result": {"found": true}}, {"tool": "check_eligibility", "arguments": {"order_id": "R-cancel"}, "result": {"eligible": true}}, {"tool": "create_return", "arguments": {"order_id": "R-cancel"}, "result": {"error": "rejected"}}], "final": "unavailable"}, "passed": false, "checks": [{"name": "schema", "passed": true, "detail": "Return a bounded trace and final status."}, {"name": "budget", "passed": true, "detail": "At most 5 tool calls."}, {"name": "trace", "passed": false, "detail": "Return is unauthorized, unchecked, or duplicated"}, {"name": "goal", "passed": false, "detail": "The replayed outcome must satisfy the reference goal."}, {"name": "final_state", "passed": false, "detail": "Reported final state must agree with replay."}], "metrics": {"tool_calls": 3}, "error": null, "usage": {}, "metadata": {}, "candidate_latency_ms": 0.008, "grader_latency_ms": 0.005, "latency_ms": 0.027}, {"id": "agent-budget", "input": {"request": "return", "order": {"id": "R-budget", "delivered": true, "age_days": 5, "returnable": true, "exists": true}, "failures": {"lookup_order": 10}, "budget": 3}, "expected": {"final": "unavailable"}, "tags": ["budget", "critical"], "case_id": "agent-budget", "trial": 1, "actual": {"trace": [{"tool": "lookup_order", "arguments": {"order_id": "R-budget"}, "result": {"error": "temporary"}}], "final": "unavailable"}, "passed": false, "checks": [{"name": "schema", "passed": true, "detail": "Return a bounded trace and final status."}, {"name": "budget", "passed": true, "detail": "At most 3 tool calls."}, {"name": "trace", "passed": true, "detail": "Every recorded result matches independent tool replay."}, {"name": "goal", "passed": true, "detail": "The replayed outcome must satisfy the reference goal."}, {"name": "termination", "passed": false, "detail": "Unavailable requires a missing-order result or an exhausted retry budget."}, {"name": "final_state", "passed": true, "detail": "Reported final state must agree with replay."}], "metrics": {"tool_calls": 1}, "error": null, "usage": {}, "metadata": {}, "candidate_latency_ms": 0.006, "grader_latency_ms": 0.004, "latency_ms": 0.023}], "trials": 1, "case_count": 8, "pair_order": "original", "execution_errors": 0, "data_notice": "Bundled cases are synthetic teaching data under MIT-0. Custom cases may have different provenance.", "metrics": {"checks": {"schema": {"passed": 8, "total": 8, "rate": 1.0}, "budget": {"passed": 8, "total": 8, "rate": 1.0}, "trace": {"passed": 7, "total": 8, "rate": 0.875}, "goal": {"passed": 5, "total": 8, "rate": 0.625}, "final_state": {"passed": 7, "total": 8, "rate": 0.875}, "termination": {"passed": 1, "total": 4, "rate": 0.25}}, "measurements": {"tool_calls": {"mean": 1.875, "count": 8}}, "candidate_latency_ms": {"median": 0.007, "p95": 0.016}, "usage": {}, "usage_coverage": 0, "trial_scores": [0.5], "unstable_cases": 0}}, "after": {"schema_version": 2, "lab_version": "0.2.0", "suite": "agent", "split": "dev", "created_at": "2026-10-01T20:51:14.722235+00:00", "candidate": {"name": "improved", "kind": "local_rules", "model": null, "source_hash": "7665f5dde4f1c1a9ea008dbca97d9c101a4af0510720b6f00d5da49ad6c12e20"}, "dataset_hash": "c90297b0423ffbeff1311fb02464d3932bdf72b011c817c838a2d0306c82ffba", "grader_hash": "8ed15f453523fc8778082785bbd46add899da329d8e3f6ab69a7edb5c6e5b3f8", "settings": {"require_final_state": true}, "threshold": 0.8, "score": 1.0, "passed": 8, "failed": 0, "total": 8, "passed_gate": true, "gate_policy": {"critical_tags": [], "min_slices": {}, "fail_on_regression": false}, "gates": [{"name": "overall", "passed": true, "detail": "Overall score >= 80%"}], "slices": {"budget": {"passed": 1, "total": 1, "accuracy": 1.0}, "critical": {"passed": 3, "total": 3, "accuracy": 1.0}, "eligibility": {"passed": 2, "total": 2, "accuracy": 1.0}, "happy-path": {"passed": 1, "total": 1, "accuracy": 1.0}, "missing-order": {"passed": 1, "total": 1, "accuracy": 1.0}, "retry": {"passed": 2, "total": 2, "accuracy": 1.0}, "unsupported": {"passed": 1, "total": 1, "accuracy": 1.0}}, "results": [{"id": "agent-eligible", "input": {"request": "return", "order": {"id": "R-eligible", "delivered": true, "age_days": 5, "returnable": true, "exists": true}, "failures": {}, "budget": 5}, "expected": {"final": "returned"}, "tags": ["happy-path"], "case_id": "agent-eligible", "trial": 1, "actual": {"trace": [{"tool": "lookup_order", "arguments": {"order_id": "R-eligible"}, "result": {"found": true}}, {"tool": "check_eligibility", "arguments": {"order_id": "R-eligible"}, "result": {"eligible": true}}, {"tool": "create_return", "arguments": {"order_id": "R-eligible"}, "result": {"return_created": true}}], "final": "returned"}, "passed": true, "checks": [{"name": "schema", "passed": true, "detail": "Return a bounded trace and final status."}, {"name": "budget", "passed": true, "detail": "At most 5 tool calls."}, {"name": "trace", "passed": true, "detail": "Every recorded result matches independent tool replay."}, {"name": "goal", "passed": true, "detail": "The replayed outcome must satisfy the reference goal."}, {"name": "final_state", "passed": true, "detail": "Reported final state must agree with replay."}], "metrics": {"tool_calls": 3}, "error": null, "usage": {}, "metadata": {}, "candidate_latency_ms": 0.014, "grader_latency_ms": 0.009, "latency_ms": 0.041}, {"id": "agent-expired", "input": {"request": "return", "order": {"id": "R-expired", "delivered": true, "age_days": 45, "returnable": true, "exists": true}, "failures": {}, "budget": 5}, "expected": {"final": "declined"}, "tags": ["eligibility", "critical"], "case_id": "agent-expired", "trial": 1, "actual": {"trace": [{"tool": "lookup_order", "arguments": {"order_id": "R-expired"}, "result": {"found": true}}, {"tool": "check_eligibility", "arguments": {"order_id": "R-expired"}, "result": {"eligible": false}}], "final": "declined"}, "passed": true, "checks": [{"name": "schema", "passed": true, "detail": "Return a bounded trace and final status."}, {"name": "budget", "passed": true, "detail": "At most 5 tool calls."}, {"name": "trace", "passed": true, "detail": "Every recorded result matches independent tool replay."}, {"name": "goal", "passed": true, "detail": "The replayed outcome must satisfy the reference goal."}, {"name": "final_state", "passed": true, "detail": "Reported final state must agree with replay."}], "metrics": {"tool_calls": 2}, "error": null, "usage": {}, "metadata": {}, "candidate_latency_ms": 0.008, "grader_latency_ms": 0.005, "latency_ms": 0.027}, {"id": "agent-undelivered", "input": {"request": "return", "order": {"id": "R-undelivered", "delivered": false, "age_days": 5, "returnable": true, "exists": true}, "failures": {}, "budget": 5}, "expected": {"final": "declined"}, "tags": ["eligibility"], "case_id": "agent-undelivered", "trial": 1, "actual": {"trace": [{"tool": "lookup_order", "arguments": {"order_id": "R-undelivered"}, "result": {"found": true}}, {"tool": "check_eligibility", "arguments": {"order_id": "R-undelivered"}, "result": {"eligible": false}}], "final": "declined"}, "passed": true, "checks": [{"name": "schema", "passed": true, "detail": "Return a bounded trace and final status."}, {"name": "budget", "passed": true, "detail": "At most 5 tool calls."}, {"name": "trace", "passed": true, "detail": "Every recorded result matches independent tool replay."}, {"name": "goal", "passed": true, "detail": "The replayed outcome must satisfy the reference goal."}, {"name": "final_state", "passed": true, "detail": "Reported final state must agree with replay."}], "metrics": {"tool_calls": 2}, "error": null, "usage": {}, "metadata": {}, "candidate_latency_ms": 0.008, "grader_latency_ms": 0.004, "latency_ms": 0.025}, {"id": "agent-missing", "input": {"request": "return", "order": {"id": "R-missing", "delivered": true, "age_days": 5, "returnable": true, "exists": false}, "failures": {}, "budget": 5}, "expected": {"final": "unavailable"}, "tags": ["missing-order"], "case_id": "agent-missing", "trial": 1, "actual": {"trace": [{"tool": "lookup_order", "arguments": {"order_id": "R-missing"}, "result": {"found": false}}], "final": "unavailable"}, "passed": true, "checks": [{"name": "schema", "passed": true, "detail": "Return a bounded trace and final status."}, {"name": "budget", "passed": true, "detail": "At most 5 tool calls."}, {"name": "trace", "passed": true, "detail": "Every recorded result matches independent tool replay."}, {"name": "goal", "passed": true, "detail": "The replayed outcome must satisfy the reference goal."}, {"name": "termination", "passed": true, "detail": "Unavailable requires a missing-order result or an exhausted retry budget."}, {"name": "final_state", "passed": true, "detail": "Reported final state must agree with replay."}], "metrics": {"tool_calls": 1}, "error": null, "usage": {}, "metadata": {}, "candidate_latency_ms": 0.006, "grader_latency_ms": 0.005, "latency_ms": 0.023}, {"id": "agent-lookup", "input": {"request": "return", "order": {"id": "R-lookup", "delivered": true, "age_days": 5, "returnable": true, "exists": true}, "failures": {"lookup_order": 1}, "budget": 5}, "expected": {"final": "returned"}, "tags": ["retry"], "case_id": "agent-lookup", "trial": 1, "actual": {"trace": [{"tool": "lookup_order", "arguments": {"order_id": "R-lookup"}, "result": {"error": "temporary"}}, {"tool": "lookup_order", "arguments": {"order_id": "R-lookup"}, "result": {"found": true}}, {"tool": "check_eligibility", "arguments": {"order_id": "R-lookup"}, "result": {"eligible": true}}, {"tool": "create_return", "arguments": {"order_id": "R-lookup"}, "result": {"return_created": true}}], "final": "returned"}, "passed": true, "checks": [{"name": "schema", "passed": true, "detail": "Return a bounded trace and final status."}, {"name": "budget", "passed": true, "detail": "At most 5 tool calls."}, {"name": "trace", "passed": true, "detail": "Every recorded result matches independent tool replay."}, {"name": "goal", "passed": true, "detail": "The replayed outcome must satisfy the reference goal."}, {"name": "final_state", "passed": true, "detail": "Reported final state must agree with replay."}], "metrics": {"tool_calls": 4}, "error": null, "usage": {}, "metadata": {}, "candidate_latency_ms": 0.009, "grader_latency_ms": 0.006, "latency_ms": 0.031}, {"id": "agent-eligibility", "input": {"request": "return", "order": {"id": "R-eligibility", "delivered": true, "age_days": 5, "returnable": true, "exists": true}, "failures": {"check_eligibility": 1}, "budget": 5}, "expected": {"final": "returned"}, "tags": ["retry"], "case_id": "agent-eligibility", "trial": 1, "actual": {"trace": [{"tool": "lookup_order", "arguments": {"order_id": "R-eligibility"}, "result": {"found": true}}, {"tool": "check_eligibility", "arguments": {"order_id": "R-eligibility"}, "result": {"error": "temporary"}}, {"tool": "check_eligibility", "arguments": {"order_id": "R-eligibility"}, "result": {"eligible": true}}, {"tool": "create_return", "arguments": {"order_id": "R-eligibility"}, "result": {"return_created": true}}], "final": "returned"}, "passed": true, "checks": [{"name": "schema", "passed": true, "detail": "Return a bounded trace and final status."}, {"name": "budget", "passed": true, "detail": "At most 5 tool calls."}, {"name": "trace", "passed": true, "detail": "Every recorded result matches independent tool replay."}, {"name": "goal", "passed": true, "detail": "The replayed outcome must satisfy the reference goal."}, {"name": "final_state", "passed": true, "detail": "Reported final state must agree with replay."}], "metrics": {"tool_calls": 4}, "error": null, "usage": {}, "metadata": {}, "candidate_latency_ms": 0.009, "grader_latency_ms": 0.006, "latency_ms": 0.03}, {"id": "agent-cancel", "input": {"request": "cancel_return", "order": {"id": "R-cancel", "delivered": true, "age_days": 5, "returnable": true, "exists": true}, "failures": {}, "budget": 5}, "expected": {"final": "no_action"}, "tags": ["unsupported", "critical"], "case_id": "agent-cancel", "trial": 1, "actual": {"trace": [], "final": "no_action"}, "passed": true, "checks": [{"name": "schema", "passed": true, "detail": "Return a bounded trace and final status."}, {"name": "budget", "passed": true, "detail": "At most 5 tool calls."}, {"name": "trace", "passed": true, "detail": "Every recorded result matches independent tool replay."}, {"name": "goal", "passed": true, "detail": "The replayed outcome must satisfy the reference goal."}, {"name": "final_state", "passed": true, "detail": "Reported final state must agree with replay."}], "metrics": {"tool_calls": 0}, "error": null, "usage": {}, "metadata": {}, "candidate_latency_ms": 0.005, "grader_latency_ms": 0.003, "latency_ms": 0.019}, {"id": "agent-budget", "input": {"request": "return", "order": {"id": "R-budget", "delivered": true, "age_days": 5, "returnable": true, "exists": true}, "failures": {"lookup_order": 10}, "budget": 3}, "expected": {"final": "unavailable"}, "tags": ["budget", "critical"], "case_id": "agent-budget", "trial": 1, "actual": {"trace": [{"tool": "lookup_order", "arguments": {"order_id": "R-budget"}, "result": {"error": "temporary"}}, {"tool": "lookup_order", "arguments": {"order_id": "R-budget"}, "result": {"error": "temporary"}}, {"tool": "lookup_order", "arguments": {"order_id": "R-budget"}, "result": {"error": "temporary"}}], "final": "unavailable"}, "passed": true, "checks": [{"name": "schema", "passed": true, "detail": "Return a bounded trace and final status."}, {"name": "budget", "passed": true, "detail": "At most 3 tool calls."}, {"name": "trace", "passed": true, "detail": "Every recorded result matches independent tool replay."}, {"name": "goal", "passed": true, "detail": "The replayed outcome must satisfy the reference goal."}, {"name": "termination", "passed": true, "detail": "Unavailable requires a missing-order result or an exhausted retry budget."}, {"name": "final_state", "passed": true, "detail": "Reported final state must agree with replay."}], "metrics": {"tool_calls": 3}, "error": null, "usage": {}, "metadata": {}, "candidate_latency_ms": 0.008, "grader_latency_ms": 0.005, "latency_ms": 0.027}], "trials": 1, "case_count": 8, "pair_order": "original", "execution_errors": 0, "data_notice": "Bundled cases are synthetic teaching data under MIT-0. Custom cases may have different provenance.", "metrics": {"checks": {"schema": {"passed": 8, "total": 8, "rate": 1.0}, "budget": {"passed": 8, "total": 8, "rate": 1.0}, "trace": {"passed": 8, "total": 8, "rate": 1.0}, "goal": {"passed": 8, "total": 8, "rate": 1.0}, "final_state": {"passed": 8, "total": 8, "rate": 1.0}, "termination": {"passed": 2, "total": 2, "rate": 1.0}}, "measurements": {"tool_calls": {"mean": 2.375, "count": 8}}, "candidate_latency_ms": {"median": 0.008, "p95": 0.014}, "usage": {}, "usage_coverage": 0, "trial_scores": [1], "unstable_cases": 0}}, "delta": 0.5, "improved": 4, "regressed": 0, "changes": [{"id": "agent-eligible", "change": "unchanged", "before": {"id": "agent-eligible", "input": {"request": "return", "order": {"id": "R-eligible", "delivered": true, "age_days": 5, "returnable": true, "exists": true}, "failures": {}, "budget": 5}, "expected": {"final": "returned"}, "tags": ["happy-path"], "case_id": "agent-eligible", "trial": 1, "actual": {"trace": [{"tool": "lookup_order", "arguments": {"order_id": "R-eligible"}, "result": {"found": true}}, {"tool": "check_eligibility", "arguments": {"order_id": "R-eligible"}, "result": {"eligible": true}}, {"tool": "create_return", "arguments": {"order_id": "R-eligible"}, "result": {"return_created": true}}], "final": "returned"}, "passed": true, "checks": [{"name": "schema", "passed": true, "detail": "Return a bounded trace and final status."}, {"name": "budget", "passed": true, "detail": "At most 5 tool calls."}, {"name": "trace", "passed": true, "detail": "Every recorded result matches independent tool replay."}, {"name": "goal", "passed": true, "detail": "The replayed outcome must satisfy the reference goal."}, {"name": "final_state", "passed": true, "detail": "Reported final state must agree with replay."}], "metrics": {"tool_calls": 3}, "error": null, "usage": {}, "metadata": {}, "candidate_latency_ms": 0.016, "grader_latency_ms": 0.008, "latency_ms": 0.044}, "after": {"id": "agent-eligible", "input": {"request": "return", "order": {"id": "R-eligible", "delivered": true, "age_days": 5, "returnable": true, "exists": true}, "failures": {}, "budget": 5}, "expected": {"final": "returned"}, "tags": ["happy-path"], "case_id": "agent-eligible", "trial": 1, "actual": {"trace": [{"tool": "lookup_order", "arguments": {"order_id": "R-eligible"}, "result": {"found": true}}, {"tool": "check_eligibility", "arguments": {"order_id": "R-eligible"}, "result": {"eligible": true}}, {"tool": "create_return", "arguments": {"order_id": "R-eligible"}, "result": {"return_created": true}}], "final": "returned"}, "passed": true, "checks": [{"name": "schema", "passed": true, "detail": "Return a bounded trace and final status."}, {"name": "budget", "passed": true, "detail": "At most 5 tool calls."}, {"name": "trace", "passed": true, "detail": "Every recorded result matches independent tool replay."}, {"name": "goal", "passed": true, "detail": "The replayed outcome must satisfy the reference goal."}, {"name": "final_state", "passed": true, "detail": "Reported final state must agree with replay."}], "metrics": {"tool_calls": 3}, "error": null, "usage": {}, "metadata": {}, "candidate_latency_ms": 0.014, "grader_latency_ms": 0.009, "latency_ms": 0.041}}, {"id": "agent-expired", "change": "unchanged", "before": {"id": "agent-expired", "input": {"request": "return", "order": {"id": "R-expired", "delivered": true, "age_days": 45, "returnable": true, "exists": true}, "failures": {}, "budget": 5}, "expected": {"final": "declined"}, "tags": ["eligibility", "critical"], "case_id": "agent-expired", "trial": 1, "actual": {"trace": [{"tool": "lookup_order", "arguments": {"order_id": "R-expired"}, "result": {"found": true}}, {"tool": "check_eligibility", "arguments": {"order_id": "R-expired"}, "result": {"eligible": false}}], "final": "declined"}, "passed": true, "checks": [{"name": "schema", "passed": true, "detail": "Return a bounded trace and final status."}, {"name": "budget", "passed": true, "detail": "At most 5 tool calls."}, {"name": "trace", "passed": true, "detail": "Every recorded result matches independent tool replay."}, {"name": "goal", "passed": true, "detail": "The replayed outcome must satisfy the reference goal."}, {"name": "final_state", "passed": true, "detail": "Reported final state must agree with replay."}], "metrics": {"tool_calls": 2}, "error": null, "usage": {}, "metadata": {}, "candidate_latency_ms": 0.008, "grader_latency_ms": 0.005, "latency_ms": 0.027}, "after": {"id": "agent-expired", "input": {"request": "return", "order": {"id": "R-expired", "delivered": true, "age_days": 45, "returnable": true, "exists": true}, "failures": {}, "budget": 5}, "expected": {"final": "declined"}, "tags": ["eligibility", "critical"], "case_id": "agent-expired", "trial": 1, "actual": {"trace": [{"tool": "lookup_order", "arguments": {"order_id": "R-expired"}, "result": {"found": true}}, {"tool": "check_eligibility", "arguments": {"order_id": "R-expired"}, "result": {"eligible": false}}], "final": "declined"}, "passed": true, "checks": [{"name": "schema", "passed": true, "detail": "Return a bounded trace and final status."}, {"name": "budget", "passed": true, "detail": "At most 5 tool calls."}, {"name": "trace", "passed": true, "detail": "Every recorded result matches independent tool replay."}, {"name": "goal", "passed": true, "detail": "The replayed outcome must satisfy the reference goal."}, {"name": "final_state", "passed": true, "detail": "Reported final state must agree with replay."}], "metrics": {"tool_calls": 2}, "error": null, "usage": {}, "metadata": {}, "candidate_latency_ms": 0.008, "grader_latency_ms": 0.005, "latency_ms": 0.027}}, {"id": "agent-undelivered", "change": "unchanged", "before": {"id": "agent-undelivered", "input": {"request": "return", "order": {"id": "R-undelivered", "delivered": false, "age_days": 5, "returnable": true, "exists": true}, "failures": {}, "budget": 5}, "expected": {"final": "declined"}, "tags": ["eligibility"], "case_id": "agent-undelivered", "trial": 1, "actual": {"trace": [{"tool": "lookup_order", "arguments": {"order_id": "R-undelivered"}, "result": {"found": true}}, {"tool": "check_eligibility", "arguments": {"order_id": "R-undelivered"}, "result": {"eligible": false}}], "final": "declined"}, "passed": true, "checks": [{"name": "schema", "passed": true, "detail": "Return a bounded trace and final status."}, {"name": "budget", "passed": true, "detail": "At most 5 tool calls."}, {"name": "trace", "passed": true, "detail": "Every recorded result matches independent tool replay."}, {"name": "goal", "passed": true, "detail": "The replayed outcome must satisfy the reference goal."}, {"name": "final_state", "passed": true, "detail": "Reported final state must agree with replay."}], "metrics": {"tool_calls": 2}, "error": null, "usage": {}, "metadata": {}, "candidate_latency_ms": 0.007, "grader_latency_ms": 0.004, "latency_ms": 0.025}, "after": {"id": "agent-undelivered", "input": {"request": "return", "order": {"id": "R-undelivered", "delivered": false, "age_days": 5, "returnable": true, "exists": true}, "failures": {}, "budget": 5}, "expected": {"final": "declined"}, "tags": ["eligibility"], "case_id": "agent-undelivered", "trial": 1, "actual": {"trace": [{"tool": "lookup_order", "arguments": {"order_id": "R-undelivered"}, "result": {"found": true}}, {"tool": "check_eligibility", "arguments": {"order_id": "R-undelivered"}, "result": {"eligible": false}}], "final": "declined"}, "passed": true, "checks": [{"name": "schema", "passed": true, "detail": "Return a bounded trace and final status."}, {"name": "budget", "passed": true, "detail": "At most 5 tool calls."}, {"name": "trace", "passed": true, "detail": "Every recorded result matches independent tool replay."}, {"name": "goal", "passed": true, "detail": "The replayed outcome must satisfy the reference goal."}, {"name": "final_state", "passed": true, "detail": "Reported final state must agree with replay."}], "metrics": {"tool_calls": 2}, "error": null, "usage": {}, "metadata": {}, "candidate_latency_ms": 0.008, "grader_latency_ms": 0.004, "latency_ms": 0.025}}, {"id": "agent-missing", "change": "unchanged", "before": {"id": "agent-missing", "input": {"request": "return", "order": {"id": "R-missing", "delivered": true, "age_days": 5, "returnable": true, "exists": false}, "failures": {}, "budget": 5}, "expected": {"final": "unavailable"}, "tags": ["missing-order"], "case_id": "agent-missing", "trial": 1, "actual": {"trace": [{"tool": "lookup_order", "arguments": {"order_id": "R-missing"}, "result": {"found": false}}], "final": "unavailable"}, "passed": true, "checks": [{"name": "schema", "passed": true, "detail": "Return a bounded trace and final status."}, {"name": "budget", "passed": true, "detail": "At most 5 tool calls."}, {"name": "trace", "passed": true, "detail": "Every recorded result matches independent tool replay."}, {"name": "goal", "passed": true, "detail": "The replayed outcome must satisfy the reference goal."}, {"name": "termination", "passed": true, "detail": "Unavailable requires a missing-order result or an exhausted retry budget."}, {"name": "final_state", "passed": true, "detail": "Reported final state must agree with replay."}], "metrics": {"tool_calls": 1}, "error": null, "usage": {}, "metadata": {}, "candidate_latency_ms": 0.006, "grader_latency_ms": 0.005, "latency_ms": 0.024}, "after": {"id": "agent-missing", "input": {"request": "return", "order": {"id": "R-missing", "delivered": true, "age_days": 5, "returnable": true, "exists": false}, "failures": {}, "budget": 5}, "expected": {"final": "unavailable"}, "tags": ["missing-order"], "case_id": "agent-missing", "trial": 1, "actual": {"trace": [{"tool": "lookup_order", "arguments": {"order_id": "R-missing"}, "result": {"found": false}}], "final": "unavailable"}, "passed": true, "checks": [{"name": "schema", "passed": true, "detail": "Return a bounded trace and final status."}, {"name": "budget", "passed": true, "detail": "At most 5 tool calls."}, {"name": "trace", "passed": true, "detail": "Every recorded result matches independent tool replay."}, {"name": "goal", "passed": true, "detail": "The replayed outcome must satisfy the reference goal."}, {"name": "termination", "passed": true, "detail": "Unavailable requires a missing-order result or an exhausted retry budget."}, {"name": "final_state", "passed": true, "detail": "Reported final state must agree with replay."}], "metrics": {"tool_calls": 1}, "error": null, "usage": {}, "metadata": {}, "candidate_latency_ms": 0.006, "grader_latency_ms": 0.005, "latency_ms": 0.023}}, {"id": "agent-lookup", "change": "improved", "before": {"id": "agent-lookup", "input": {"request": "return", "order": {"id": "R-lookup", "delivered": true, "age_days": 5, "returnable": true, "exists": true}, "failures": {"lookup_order": 1}, "budget": 5}, "expected": {"final": "returned"}, "tags": ["retry"], "case_id": "agent-lookup", "trial": 1, "actual": {"trace": [{"tool": "lookup_order", "arguments": {"order_id": "R-lookup"}, "result": {"error": "temporary"}}], "final": "unavailable"}, "passed": false, "checks": [{"name": "schema", "passed": true, "detail": "Return a bounded trace and final status."}, {"name": "budget", "passed": true, "detail": "At most 5 tool calls."}, {"name": "trace", "passed": true, "detail": "Every recorded result matches independent tool replay."}, {"name": "goal", "passed": false, "detail": "The replayed outcome must satisfy the reference goal."}, {"name": "termination", "passed": false, "detail": "Unavailable requires a missing-order result or an exhausted retry budget."}, {"name": "final_state", "passed": true, "detail": "Reported final state must agree with replay."}], "metrics": {"tool_calls": 1}, "error": null, "usage": {}, "metadata": {}, "candidate_latency_ms": 0.007, "grader_latency_ms": 0.004, "latency_ms": 0.024}, "after": {"id": "agent-lookup", "input": {"request": "return", "order": {"id": "R-lookup", "delivered": true, "age_days": 5, "returnable": true, "exists": true}, "failures": {"lookup_order": 1}, "budget": 5}, "expected": {"final": "returned"}, "tags": ["retry"], "case_id": "agent-lookup", "trial": 1, "actual": {"trace": [{"tool": "lookup_order", "arguments": {"order_id": "R-lookup"}, "result": {"error": "temporary"}}, {"tool": "lookup_order", "arguments": {"order_id": "R-lookup"}, "result": {"found": true}}, {"tool": "check_eligibility", "arguments": {"order_id": "R-lookup"}, "result": {"eligible": true}}, {"tool": "create_return", "arguments": {"order_id": "R-lookup"}, "result": {"return_created": true}}], "final": "returned"}, "passed": true, "checks": [{"name": "schema", "passed": true, "detail": "Return a bounded trace and final status."}, {"name": "budget", "passed": true, "detail": "At most 5 tool calls."}, {"name": "trace", "passed": true, "detail": "Every recorded result matches independent tool replay."}, {"name": "goal", "passed": true, "detail": "The replayed outcome must satisfy the reference goal."}, {"name": "final_state", "passed": true, "detail": "Reported final state must agree with replay."}], "metrics": {"tool_calls": 4}, "error": null, "usage": {}, "metadata": {}, "candidate_latency_ms": 0.009, "grader_latency_ms": 0.006, "latency_ms": 0.031}}, {"id": "agent-eligibility", "change": "improved", "before": {"id": "agent-eligibility", "input": {"request": "return", "order": {"id": "R-eligibility", "delivered": true, "age_days": 5, "returnable": true, "exists": true}, "failures": {"check_eligibility": 1}, "budget": 5}, "expected": {"final": "returned"}, "tags": ["retry"], "case_id": "agent-eligibility", "trial": 1, "actual": {"trace": [{"tool": "lookup_order", "arguments": {"order_id": "R-eligibility"}, "result": {"found": true}}, {"tool": "check_eligibility", "arguments": {"order_id": "R-eligibility"}, "result": {"error": "temporary"}}], "final": "unavailable"}, "passed": false, "checks": [{"name": "schema", "passed": true, "detail": "Return a bounded trace and final status."}, {"name": "budget", "passed": true, "detail": "At most 5 tool calls."}, {"name": "trace", "passed": true, "detail": "Every recorded result matches independent tool replay."}, {"name": "goal", "passed": false, "detail": "The replayed outcome must satisfy the reference goal."}, {"name": "termination", "passed": false, "detail": "Unavailable requires a missing-order result or an exhausted retry budget."}, {"name": "final_state", "passed": true, "detail": "Reported final state must agree with replay."}], "metrics": {"tool_calls": 2}, "error": null, "usage": {}, "metadata": {}, "candidate_latency_ms": 0.007, "grader_latency_ms": 0.005, "latency_ms": 0.026}, "after": {"id": "agent-eligibility", "input": {"request": "return", "order": {"id": "R-eligibility", "delivered": true, "age_days": 5, "returnable": true, "exists": true}, "failures": {"check_eligibility": 1}, "budget": 5}, "expected": {"final": "returned"}, "tags": ["retry"], "case_id": "agent-eligibility", "trial": 1, "actual": {"trace": [{"tool": "lookup_order", "arguments": {"order_id": "R-eligibility"}, "result": {"found": true}}, {"tool": "check_eligibility", "arguments": {"order_id": "R-eligibility"}, "result": {"error": "temporary"}}, {"tool": "check_eligibility", "arguments": {"order_id": "R-eligibility"}, "result": {"eligible": true}}, {"tool": "create_return", "arguments": {"order_id": "R-eligibility"}, "result": {"return_created": true}}], "final": "returned"}, "passed": true, "checks": [{"name": "schema", "passed": true, "detail": "Return a bounded trace and final status."}, {"name": "budget", "passed": true, "detail": "At most 5 tool calls."}, {"name": "trace", "passed": true, "detail": "Every recorded result matches independent tool replay."}, {"name": "goal", "passed": true, "detail": "The replayed outcome must satisfy the reference goal."}, {"name": "final_state", "passed": true, "detail": "Reported final state must agree with replay."}], "metrics": {"tool_calls": 4}, "error": null, "usage": {}, "metadata": {}, "candidate_latency_ms": 0.009, "grader_latency_ms": 0.006, "latency_ms": 0.03}}, {"id": "agent-cancel", "change": "improved", "before": {"id": "agent-cancel", "input": {"request": "cancel_return", "order": {"id": "R-cancel", "delivered": true, "age_days": 5, "returnable": true, "exists": true}, "failures": {}, "budget": 5}, "expected": {"final": "no_action"}, "tags": ["unsupported", "critical"], "case_id": "agent-cancel", "trial": 1, "actual": {"trace": [{"tool": "lookup_order", "arguments": {"order_id": "R-cancel"}, "result": {"found": true}}, {"tool": "check_eligibility", "arguments": {"order_id": "R-cancel"}, "result": {"eligible": true}}, {"tool": "create_return", "arguments": {"order_id": "R-cancel"}, "result": {"error": "rejected"}}], "final": "unavailable"}, "passed": false, "checks": [{"name": "schema", "passed": true, "detail": "Return a bounded trace and final status."}, {"name": "budget", "passed": true, "detail": "At most 5 tool calls."}, {"name": "trace", "passed": false, "detail": "Return is unauthorized, unchecked, or duplicated"}, {"name": "goal", "passed": false, "detail": "The replayed outcome must satisfy the reference goal."}, {"name": "final_state", "passed": false, "detail": "Reported final state must agree with replay."}], "metrics": {"tool_calls": 3}, "error": null, "usage": {}, "metadata": {}, "candidate_latency_ms": 0.008, "grader_latency_ms": 0.005, "latency_ms": 0.027}, "after": {"id": "agent-cancel", "input": {"request": "cancel_return", "order": {"id": "R-cancel", "delivered": true, "age_days": 5, "returnable": true, "exists": true}, "failures": {}, "budget": 5}, "expected": {"final": "no_action"}, "tags": ["unsupported", "critical"], "case_id": "agent-cancel", "trial": 1, "actual": {"trace": [], "final": "no_action"}, "passed": true, "checks": [{"name": "schema", "passed": true, "detail": "Return a bounded trace and final status."}, {"name": "budget", "passed": true, "detail": "At most 5 tool calls."}, {"name": "trace", "passed": true, "detail": "Every recorded result matches independent tool replay."}, {"name": "goal", "passed": true, "detail": "The replayed outcome must satisfy the reference goal."}, {"name": "final_state", "passed": true, "detail": "Reported final state must agree with replay."}], "metrics": {"tool_calls": 0}, "error": null, "usage": {}, "metadata": {}, "candidate_latency_ms": 0.005, "grader_latency_ms": 0.003, "latency_ms": 0.019}}, {"id": "agent-budget", "change": "improved", "before": {"id": "agent-budget", "input": {"request": "return", "order": {"id": "R-budget", "delivered": true, "age_days": 5, "returnable": true, "exists": true}, "failures": {"lookup_order": 10}, "budget": 3}, "expected": {"final": "unavailable"}, "tags": ["budget", "critical"], "case_id": "agent-budget", "trial": 1, "actual": {"trace": [{"tool": "lookup_order", "arguments": {"order_id": "R-budget"}, "result": {"error": "temporary"}}], "final": "unavailable"}, "passed": false, "checks": [{"name": "schema", "passed": true, "detail": "Return a bounded trace and final status."}, {"name": "budget", "passed": true, "detail": "At most 3 tool calls."}, {"name": "trace", "passed": true, "detail": "Every recorded result matches independent tool replay."}, {"name": "goal", "passed": true, "detail": "The replayed outcome must satisfy the reference goal."}, {"name": "termination", "passed": false, "detail": "Unavailable requires a missing-order result or an exhausted retry budget."}, {"name": "final_state", "passed": true, "detail": "Reported final state must agree with replay."}], "metrics": {"tool_calls": 1}, "error": null, "usage": {}, "metadata": {}, "candidate_latency_ms": 0.006, "grader_latency_ms": 0.004, "latency_ms": 0.023}, "after": {"id": "agent-budget", "input": {"request": "return", "order": {"id": "R-budget", "delivered": true, "age_days": 5, "returnable": true, "exists": true}, "failures": {"lookup_order": 10}, "budget": 3}, "expected": {"final": "unavailable"}, "tags": ["budget", "critical"], "case_id": "agent-budget", "trial": 1, "actual": {"trace": [{"tool": "lookup_order", "arguments": {"order_id": "R-budget"}, "result": {"error": "temporary"}}, {"tool": "lookup_order", "arguments": {"order_id": "R-budget"}, "result": {"error": "temporary"}}, {"tool": "lookup_order", "arguments": {"order_id": "R-budget"}, "result": {"error": "temporary"}}], "final": "unavailable"}, "passed": true, "checks": [{"name": "schema", "passed": true, "detail": "Return a bounded trace and final status."}, {"name": "budget", "passed": true, "detail": "At most 3 tool calls."}, {"name": "trace", "passed": true, "detail": "Every recorded result matches independent tool replay."}, {"name": "goal", "passed": true, "detail": "The replayed outcome must satisfy the reference goal."}, {"name": "termination", "passed": true, "detail": "Unavailable requires a missing-order result or an exhausted retry budget."}, {"name": "final_state", "passed": true, "detail": "Reported final state must agree with replay."}], "metrics": {"tool_calls": 3}, "error": null, "usage": {}, "metadata": {}, "candidate_latency_ms": 0.008, "grader_latency_ms": 0.005, "latency_ms": 0.027}}], "gates": [{"name": "overall", "passed": true, "detail": "Overall score >= 80%"}], "passed_gate": true}