diff --git a/docs/OVERLAY_BENCHMARK_CONTRACT.md b/docs/OVERLAY_BENCHMARK_CONTRACT.md index 1c15e1fe5..d6f3cc38e 100644 --- a/docs/OVERLAY_BENCHMARK_CONTRACT.md +++ b/docs/OVERLAY_BENCHMARK_CONTRACT.md @@ -1,4 +1,4 @@ -# Overlay benchmark behavior contract v2 +# Overlay benchmark behavior contract v4 The paid overlay fixtures verify supported behavior, with comparative efficacy reported separately as research evidence. A passing case does **not** establish @@ -7,7 +7,7 @@ blocking efficacy contract; historical failures retain their original verdicts. The contract is defined by `OVERLAY_CONTRACT` in `test/helpers/overlay-case-policy.ts`. Every new measurement and aggregate record -identifies version 3. Eval entry names include `contract-v3` so automated result +identifies version 4. Eval entry names include `contract-v4` so automated result matching cannot silently compare the old and new contracts by the same name. Selection IDs for the six retained paid cases stay unchanged. @@ -20,9 +20,17 @@ OFF does not. Each retained fixture runs ten samples per arm, separately for Every planned sample in both arms must execute successfully, produce exactly one successful native terminal result without an error flag, preserve workspace scope, and yield a finite metric within the fixture's bounds. Missing reasoning -metadata, failed output validation, oracle process errors, partial sampling, -deadlines, cleanup failures, or recording failures keep the case failed. Missing -observations are never replaced with zero or discarded to obtain a passing arm. +metadata, oracle process errors, partial sampling, deadlines, cleanup failures, +or recording failures keep the case failed. Missing observations are never +replaced with zero or discarded to obtain a passing arm. + +A completed answer that fails the fixture's output check is a measured +correctness result, not an invalid measurement, in either arm: its record has +`measurementPassed: true`, `taskCorrect: false` and the check's message in +`answerError`. Only ON correctness gates the case, so an OFF wrong answer stays +in the comparison and an ON wrong answer fails the case. Version 3 counted an +OFF wrong answer as an invalid measurement (census 37198445662); v3 verdicts +are not reinterpreted. | Fixture family | Exact task and scope contract | Additional ON requirement | Measurement | | --- | --- | --- | --- | diff --git a/test/fixtures/overlay-v4-replay-37198445662.json b/test/fixtures/overlay-v4-replay-37198445662.json new file mode 100644 index 000000000..33e3a7d5c --- /dev/null +++ b/test/fixtures/overlay-v4-replay-37198445662.json @@ -0,0 +1,396 @@ +{ + "source": { + "run": "37198445662", + "sha": "fd6854b", + "tier": "periodic", + "artifact": "paid-slice-26-a1", + "case": "overlay-harness-claude-dedicated-tools-vs-bash-sonnet", + "fixture": "claude-dedicated-tools-vs-bash-sonnet", + "attempt": 1, + "contract": 3 + }, + "excerpt": "Per trial: tool_use names in order and the native terminal result string. Everything else in the SDK stream is omitted.", + "originalOutcome": "FAIL under contract v3: overlay-off-0 answered src/api.ts: [] (wrong), was recorded as an invalid measurement, and the comparison became incomplete.", + "expected": { + "3": "fail", + "4": "pass" + }, + "justification": "Contract v4 (D1, approved 2026-10-04) scores a completed OFF wrong answer as a correctness result; every ON trial is correct and the zero-Bash baseline gate holds.", + "trials": [ + { + "arm": "overlay-on", + "trial": 0, + "tools": [ + "Glob", + "Glob", + "Grep", + "Grep", + "Grep", + "Grep", + "Grep" + ], + "answer": "{\"src/index.ts\":[\"x\"],\"src/util.ts\":[\"util\"],\"src/config.ts\":[\"c\"],\"src/api.ts\":[\"fetchFoo\"],\"src/types.ts\":[\"Foo\"]}", + "v3": { + "measurementPassed": true, + "taskCorrect": true, + "exitReason": "success" + } + }, + { + "arm": "overlay-on", + "trial": 1, + "tools": [ + "Glob", + "Glob", + "Grep", + "Grep", + "Grep", + "Grep", + "Grep", + "Read" + ], + "answer": "{\"src/index.ts\":[\"x\"],\"src/util.ts\":[\"util\"],\"src/config.ts\":[\"c\"],\"src/api.ts\":[\"fetchFoo\"],\"src/types.ts\":[\"Foo\"]}", + "v3": { + "measurementPassed": true, + "taskCorrect": true, + "exitReason": "success" + } + }, + { + "arm": "overlay-on", + "trial": 2, + "tools": [ + "Glob", + "Glob", + "Read", + "Read", + "Read", + "Read", + "Read" + ], + "answer": "{\"src/index.ts\":[\"x\"],\"src/util.ts\":[\"util\"],\"src/config.ts\":[\"c\"],\"src/api.ts\":[\"fetchFoo\"],\"src/types.ts\":[\"Foo\"]}", + "v3": { + "measurementPassed": true, + "taskCorrect": true, + "exitReason": "success" + } + }, + { + "arm": "overlay-on", + "trial": 3, + "tools": [ + "Glob", + "Glob", + "Grep", + "Grep", + "Grep", + "Grep", + "Grep", + "Read" + ], + "answer": "{\"src/index.ts\":[\"x\"],\"src/util.ts\":[\"util\"],\"src/config.ts\":[\"c\"],\"src/api.ts\":[\"fetchFoo\"],\"src/types.ts\":[\"Foo\"]}", + "v3": { + "measurementPassed": true, + "taskCorrect": true, + "exitReason": "success" + } + }, + { + "arm": "overlay-on", + "trial": 4, + "tools": [ + "Glob", + "Glob", + "Grep" + ], + "answer": "{\"src/index.ts\":[\"x\"],\"src/util.ts\":[\"util\"],\"src/config.ts\":[\"c\"],\"src/api.ts\":[\"fetchFoo\"],\"src/types.ts\":[\"Foo\"]}", + "v3": { + "measurementPassed": true, + "taskCorrect": true, + "exitReason": "success" + } + }, + { + "arm": "overlay-on", + "trial": 5, + "tools": [ + "Glob", + "Glob", + "Grep", + "Grep", + "Grep", + "Grep", + "Grep" + ], + "answer": "{\"src/index.ts\":[\"x\"],\"src/util.ts\":[\"util\"],\"src/config.ts\":[\"c\"],\"src/api.ts\":[\"fetchFoo\"],\"src/types.ts\":[\"Foo\"]}", + "v3": { + "measurementPassed": true, + "taskCorrect": true, + "exitReason": "success" + } + }, + { + "arm": "overlay-on", + "trial": 6, + "tools": [ + "Glob", + "Glob", + "Grep", + "Grep", + "Grep", + "Grep", + "Grep" + ], + "answer": "{\"src/index.ts\":[\"x\"],\"src/util.ts\":[\"util\"],\"src/config.ts\":[\"c\"],\"src/api.ts\":[\"fetchFoo\"],\"src/types.ts\":[\"Foo\"]}", + "v3": { + "measurementPassed": true, + "taskCorrect": true, + "exitReason": "success" + } + }, + { + "arm": "overlay-on", + "trial": 7, + "tools": [ + "Glob", + "Glob", + "Grep", + "Grep", + "Grep", + "Grep", + "Grep", + "Read" + ], + "answer": "{\"src/index.ts\":[\"x\"],\"src/util.ts\":[\"util\"],\"src/config.ts\":[\"c\"],\"src/api.ts\":[\"fetchFoo\"],\"src/types.ts\":[\"Foo\"]}", + "v3": { + "measurementPassed": true, + "taskCorrect": true, + "exitReason": "success" + } + }, + { + "arm": "overlay-on", + "trial": 8, + "tools": [ + "Glob", + "Grep" + ], + "answer": "{\"src/index.ts\":[\"x\"],\"src/util.ts\":[\"util\"],\"src/config.ts\":[\"c\"],\"src/api.ts\":[\"fetchFoo\"],\"src/types.ts\":[\"Foo\"]}", + "v3": { + "measurementPassed": true, + "taskCorrect": true, + "exitReason": "success" + } + }, + { + "arm": "overlay-on", + "trial": 9, + "tools": [ + "Glob", + "Glob", + "Grep", + "Grep", + "Grep", + "Grep", + "Grep", + "Read" + ], + "answer": "{\"src/index.ts\":[\"x\"],\"src/util.ts\":[\"util\"],\"src/config.ts\":[\"c\"],\"src/api.ts\":[\"fetchFoo\"],\"src/types.ts\":[\"Foo\"]}", + "v3": { + "measurementPassed": true, + "taskCorrect": true, + "exitReason": "success" + } + }, + { + "arm": "overlay-off", + "trial": 0, + "tools": [ + "Glob", + "Glob", + "Grep", + "Grep", + "Grep", + "Grep", + "Grep" + ], + "answer": "{\"src/index.ts\":[\"x\"],\"src/util.ts\":[\"util\"],\"src/config.ts\":[\"c\"],\"src/api.ts\":[],\"src/types.ts\":[\"Foo\"]}", + "v3": { + "measurementPassed": false, + "taskCorrect": false, + "exitReason": "validation_failed" + } + }, + { + "arm": "overlay-off", + "trial": 1, + "tools": [ + "Glob", + "Glob", + "Grep", + "Grep", + "Grep", + "Grep", + "Grep" + ], + "answer": "{\"src/index.ts\":[\"x\"],\"src/util.ts\":[\"util\"],\"src/config.ts\":[\"c\"],\"src/types.ts\":[\"Foo\"],\"src/api.ts\":[\"fetchFoo\"]}", + "v3": { + "measurementPassed": true, + "taskCorrect": true, + "exitReason": "success" + } + }, + { + "arm": "overlay-off", + "trial": 2, + "tools": [ + "Glob", + "Glob", + "Grep", + "Grep", + "Grep", + "Grep", + "Grep" + ], + "answer": "{\"src/index.ts\":[\"x\"],\"src/util.ts\":[\"util\"],\"src/config.ts\":[\"c\"],\"src/api.ts\":[\"fetchFoo\"],\"src/types.ts\":[\"Foo\"]}", + "v3": { + "measurementPassed": true, + "taskCorrect": true, + "exitReason": "success" + } + }, + { + "arm": "overlay-off", + "trial": 3, + "tools": [ + "Glob", + "Glob", + "Grep", + "Grep", + "Grep", + "Grep", + "Grep" + ], + "answer": "{\"src/index.ts\":[\"x\"],\"src/util.ts\":[\"util\"],\"src/config.ts\":[\"c\"],\"src/api.ts\":[\"fetchFoo\"],\"src/types.ts\":[\"Foo\"]}", + "v3": { + "measurementPassed": true, + "taskCorrect": true, + "exitReason": "success" + } + }, + { + "arm": "overlay-off", + "trial": 4, + "tools": [ + "Glob", + "Glob", + "Grep", + "Grep", + "Grep", + "Grep", + "Grep" + ], + "answer": "{\"src/index.ts\":[\"x\"],\"src/util.ts\":[\"util\"],\"src/config.ts\":[\"c\"],\"src/api.ts\":[\"fetchFoo\"],\"src/types.ts\":[\"Foo\"]}", + "v3": { + "measurementPassed": true, + "taskCorrect": true, + "exitReason": "success" + } + }, + { + "arm": "overlay-off", + "trial": 5, + "tools": [ + "Glob", + "Glob", + "Grep", + "Grep", + "Grep", + "Grep", + "Grep" + ], + "answer": "{\"src/index.ts\":[\"x\"],\"src/util.ts\":[\"util\"],\"src/config.ts\":[\"c\"],\"src/api.ts\":[\"fetchFoo\"],\"src/types.ts\":[\"Foo\"]}", + "v3": { + "measurementPassed": true, + "taskCorrect": true, + "exitReason": "success" + } + }, + { + "arm": "overlay-off", + "trial": 6, + "tools": [ + "Glob", + "Glob", + "Grep", + "Grep", + "Grep", + "Grep", + "Grep" + ], + "answer": "{\"src/index.ts\":[\"x\"],\"src/util.ts\":[\"util\"],\"src/config.ts\":[\"c\"],\"src/api.ts\":[\"fetchFoo\"],\"src/types.ts\":[\"Foo\"]}", + "v3": { + "measurementPassed": true, + "taskCorrect": true, + "exitReason": "success" + } + }, + { + "arm": "overlay-off", + "trial": 7, + "tools": [ + "Glob", + "Glob", + "Grep", + "Grep", + "Grep", + "Grep", + "Grep" + ], + "answer": "{\"src/index.ts\":[\"x\"],\"src/util.ts\":[\"util\"],\"src/config.ts\":[\"c\"],\"src/api.ts\":[\"fetchFoo\"],\"src/types.ts\":[\"Foo\"]}", + "v3": { + "measurementPassed": true, + "taskCorrect": true, + "exitReason": "success" + } + }, + { + "arm": "overlay-off", + "trial": 8, + "tools": [ + "Glob", + "Glob", + "Grep", + "Grep", + "Grep", + "Grep", + "Grep" + ], + "answer": "{\"src/index.ts\":[\"x\"],\"src/util.ts\":[\"util\"],\"src/config.ts\":[\"c\"],\"src/api.ts\":[\"fetchFoo\"],\"src/types.ts\":[\"Foo\"]}", + "v3": { + "measurementPassed": true, + "taskCorrect": true, + "exitReason": "success" + } + }, + { + "arm": "overlay-off", + "trial": 9, + "tools": [ + "Glob", + "Glob", + "Grep", + "Grep", + "Grep", + "Grep", + "Grep" + ], + "answer": "{\"src/index.ts\":[\"x\"],\"src/util.ts\":[\"util\"],\"src/config.ts\":[\"c\"],\"src/api.ts\":[\"fetchFoo\"],\"src/types.ts\":[\"Foo\"]}", + "v3": { + "measurementPassed": true, + "taskCorrect": true, + "exitReason": "success" + } + } + ] +} \ No newline at end of file diff --git a/test/helpers/overlay-attempt.ts b/test/helpers/overlay-attempt.ts index cfe5c59d8..741bd6d04 100644 --- a/test/helpers/overlay-attempt.ts +++ b/test/helpers/overlay-attempt.ts @@ -10,6 +10,8 @@ export interface OverlayTrialOutcome { passed: boolean; taskCorrect: boolean; metric?: number; + /** Contract v4: a completed answer that failed the fixture's output check. */ + answerError?: string; result?: AgentSdkResult; error?: string; exitReason: string; @@ -57,13 +59,20 @@ export async function runOverlayTrial(options: { (fixture.comparison.maximum !== undefined && metric > fixture.comparison.maximum)))) { throw new Error(`invalid fixture metric: ${metric}`); } - fixture.verify?.(result, directory, metric); + let answerError: string | undefined; + try { fixture.verify?.(result, directory, metric); } + catch (error) { answerError = error instanceof Error ? `${error.name}: ${error.message}` : String(error); } checkActive(); // Implementation oracles execute agent-written code. Their side effects // must obey the same scope contract and appear in the retained snapshot. after = snapshotWorkspace(directory); assertWorkspaceChanges(before, after, fixture.allowedChanges ?? []); - outcome = { passed: true, taskCorrect: fixture.taskCorrect?.(metric) ?? true, metric, result, exitReason: result.exitReason, before, after }; + // Contract v4: a wrong completed answer is a measured correctness result in + // either arm, not an invalid measurement. + outcome = { + passed: true, taskCorrect: !answerError && (fixture.taskCorrect?.(metric) ?? true), + metric, result, exitReason: result.exitReason, before, after, ...(answerError ? { answerError } : {}), + }; } catch (error) { // Capture post-failure files when possible; preserve the first failure if // the workspace is absent or evidence capture itself cannot complete. @@ -98,11 +107,11 @@ export function assessOverlayArms(fixture: OverlayFixture, overlay: OverlayTrial off: off.filter((trial) => trial?.passed).map((trial) => trial.metric!), }; const comparison = assessComparison(metrics, fixture.trials, fixture.comparison, fixture.pass); + // Only execution, scope and metric failures invalidate a trial, in both arms. const measurementsValid = comparison.status !== 'incomplete' && [overlay, off].every((arm) => arm.length === fixture.trials && Array.from(arm).every((trial) => trial?.passed)); - // Baseline completion is an experimental variable for literal scope. A valid - // OFF sample may score 0..3; ON must complete all three. Every ON trial has - // already passed its fixture's output check to count as a valid measurement. + // Task correctness is a measured result in both arms (OFF may score 0..3 or + // answer wrongly); only ON correctness gates the case. const correctnessPassed = measurementsValid && overlay.every((trial) => trial.taskCorrect); // A fixture with a gate also blocks on its comparison (dedicated tools: 20% // fewer Bash calls, or none at a zero baseline). Without a gate the diff --git a/test/helpers/overlay-case-policy.ts b/test/helpers/overlay-case-policy.ts index 6316c3c5d..444413fec 100644 --- a/test/helpers/overlay-case-policy.ts +++ b/test/helpers/overlay-case-policy.ts @@ -1,11 +1,14 @@ -/** Version 3 gates supported behavior, plus the comparison for fixtures that +/** Version 4 keeps v3's gate but scores a completed wrong answer in either arm + * as a correctness result; only execution, scope and metric failures invalidate + * a trial. An OFF wrong answer no longer fails the comparison. + * Version 3 gates supported behavior, plus the comparison for fixtures that * declare a gate (dedicated tools: correct output and 20% fewer Bash calls, * replacing v2's zero Bash in every ON trial). Other comparisons stay research-only. * Version 1 required efficacy lift. Never reinterpret recorded failures of earlier versions. */ export const OVERLAY_CONTRACT = { name: 'overlay-behavior', - version: 3, + version: 4, releaseGate: 'execution_scope_complete_sampling_on_correctness_and_fixture_gate', comparisonRole: 'research_only_unless_fixture_gate', resourceNonRegression: 'not_established', diff --git a/test/helpers/overlay-case.ts b/test/helpers/overlay-case.ts index 3d9c7c856..ddef0c127 100644 --- a/test/helpers/overlay-case.ts +++ b/test/helpers/overlay-case.ts @@ -46,7 +46,7 @@ function saveTrial(fixture: OverlayFixture, attempt: number, arm: Arm, n: number // this JSONL remains the final SDK result used for the trial measurement. rateLimitAttemptStreamsRetained: true, measurementPassed: outcome.passed, taskCorrect: outcome.taskCorrect, metric: outcome.metric, exitReason: outcome.exitReason, - error: outcome.error, before: outcome.before, after: outcome.after, + error: outcome.error, answerError: outcome.answerError, before: outcome.before, after: outcome.after, }, null, 2) + '\n', { flag: 'wx' }); } @@ -143,7 +143,7 @@ export function registerOverlayCase(fixtureId: string): void { name: `${fixture.id}-contract-v${OVERLAY_CONTRACT.version}-${key}`, suite: 'overlay-harness-measurement', tier: 'e2e', passed: outcome.passed, duration_ms: sdk?.durationMs ?? 0, cost_usd: sdk?.costUsd ?? 0, transcript: sdk?.events, prompt: fixture.userPrompt, - output: JSON.stringify({ contract: OVERLAY_CONTRACT, measurementPassed: outcome.passed, taskCorrect: outcome.taskCorrect, metric: outcome.metric, assistantOutput: sdk?.output }), + output: JSON.stringify({ contract: OVERLAY_CONTRACT, measurementPassed: outcome.passed, taskCorrect: outcome.taskCorrect, answerError: outcome.answerError, metric: outcome.metric, assistantOutput: sdk?.output }), turns_used: sdk?.turnsUsed, browse_errors: sdk?.browseErrors, exit_reason: outcome.exitReason, error: outcome.error, model: sdk?.model ?? fixture.model, first_response_ms: sdk?.firstResponseMs, max_inter_turn_ms: sdk?.maxInterTurnMs, diff --git a/test/overlay-measurement.test.ts b/test/overlay-measurement.test.ts index c7e875e83..273647113 100644 --- a/test/overlay-measurement.test.ts +++ b/test/overlay-measurement.test.ts @@ -5,6 +5,7 @@ import * as path from 'path'; import type { AgentSdkResult, QueryProvider } from './helpers/agent-sdk-runner'; import { firstAssistantMessageToolCount, reportedThinkingTokens, assessComparison, trialArtifactStem } from './helpers/overlay-measurement'; import { runOverlayTrial, awaitOverlayWorkers, assessOverlayArms, captureOverlayQueryAttempts, type OverlayTrialOutcome } from './helpers/overlay-attempt'; +import { OVERLAY_CONTRACT } from './helpers/overlay-case-policy'; import { setupLiteralWorkspace, correctLiteralTargets, snapshotWorkspace, assertReadOnlyWorkspace } from './helpers/overlay-workspace'; import { fanoutPass, higherIsBetter20Pct, lowerIsBetter20Pct, lowerIsBetter20PctOrZeroBaseline, OVERLAY_FIXTURES, type OverlayFixture } from './fixtures/overlay-nudges'; @@ -66,9 +67,10 @@ describe('SDK overlay measurements', () => { describe('correctness versus efficacy', () => { describe.each(['claude-opus-4-7', 'claude-sonnet-4-6'])('dedicated-tool gate for %s', (model) => { const f = { ...OVERLAY_FIXTURES.find(f => f.metricName === 'bash_tool_calls' && f.model === model)!, trials: 3 }; - // A trial whose final JSON fails the fixture's verify is recorded as a failed measurement. + // Contract v4: a trial whose final JSON fails the fixture's verify is a valid + // measurement with a wrong answer, not an invalid measurement. const correct = (metric: number): OverlayTrialOutcome => ({ passed: true, taskCorrect: true, metric, exitReason: 'success' }); - const incorrect = (metric: number): OverlayTrialOutcome => ({ passed: false, taskCorrect: false, metric, exitReason: 'validation_failed' }); + const incorrect = (metric: number): OverlayTrialOutcome => ({ passed: true, taskCorrect: false, metric, exitReason: 'success', answerError: 'AssertionError: final answer does not match the fixture task' }); const arm = (...trials: OverlayTrialOutcome[]) => trials; test('fixture gates on the comparison and on output correctness, not on zero ON Bash', () => { expect(f.taskCorrect).toBeUndefined(); @@ -88,9 +90,25 @@ describe('correctness versus efficacy', () => { gatePassed: false, passed: false, comparison: { status: 'no_measured_improvement' }, }); }); - test('a 20% reduction with incorrect output fails', () => { + test('a 20% reduction with incorrect ON output fails', () => { expect(assessOverlayArms(f, arm(correct(0), incorrect(0), correct(0)), arm(correct(5), correct(5), correct(5)))).toMatchObject({ - measurementsValid: false, passed: false, + measurementsValid: true, correctnessPassed: false, passed: false, + }); + }); + test('an OFF wrong answer is a comparison result, not an invalid measurement', () => { + expect(assessOverlayArms(f, arm(correct(0), correct(0), correct(0)), arm(incorrect(0), correct(0), correct(0)))).toMatchObject({ + measurementsValid: true, correctnessPassed: true, gatePassed: true, passed: true, comparison: { status: 'baseline_saturated' }, + }); + }); + test('an execution failure invalidates the measurement in either arm', () => { + const failed: OverlayTrialOutcome = { passed: false, taskCorrect: false, metric: 0, exitReason: 'error_max_turns', error: 'Error: SDK execution failed: error_max_turns' }; + for (const [on, off] of [[arm(failed, correct(0), correct(0)), arm(correct(0), correct(0), correct(0))], [arm(correct(0), correct(0), correct(0)), arm(failed, correct(0), correct(0))]]) { + expect(assessOverlayArms(f, on, off)).toMatchObject({ measurementsValid: false, correctnessPassed: false, gatePassed: false, passed: false, comparison: { status: 'incomplete' } }); + } + }); + test('an OFF wrong answer does not rescue a failed comparison gate', () => { + expect(assessOverlayArms(f, arm(correct(6), correct(6), correct(6)), arm(incorrect(5), correct(5), correct(5)))).toMatchObject({ + measurementsValid: true, correctnessPassed: true, gatePassed: false, passed: false, comparison: { status: 'regressed' }, }); }); test('a zero-Bash baseline passes only when every ON trial is also zero', () => { @@ -288,10 +306,13 @@ describe('complete trial lifecycle', () => { }, record: (entry) => records.push(entry) }); expect(records).toHaveLength(1); expect(records[0]).toBe(outcome); - expect(outcome.passed).toBe(scenario === 'success'); + // Contract v4: a failed output check is a completed wrong answer, not an invalid measurement. + expect(outcome.passed).toBe(scenario === 'success' || scenario === 'assertion'); + expect(outcome.taskCorrect).toBe(scenario === 'success'); + if (scenario === 'assertion') expect(outcome).toMatchObject({ exitReason: 'success', answerError: 'Error: required output absent' }); if (scenario === 'runner_error' || scenario === 'setup_error') expect(outcome.exitReason).toBe('harness_error'); if (scenario === 'max_turns') expect(outcome.exitReason).toBe('error_max_turns'); - if (scenario !== 'success') expect(outcome.error).toBeTruthy(); + if (scenario !== 'success' && scenario !== 'assertion') expect(outcome.error).toBeTruthy(); if (scenario !== 'runner_error' && scenario !== 'setup_error') expect(outcome.result?.costUsd).toBe(0.03); } finally { fs.rmSync(dir, { recursive: true, force: true }); } }); @@ -311,6 +332,40 @@ describe('complete trial lifecycle', () => { }); }); +describe('contract v4 replay of census 37198445662', () => { + const corpus = JSON.parse(fs.readFileSync(path.join(import.meta.dir, 'fixtures', 'overlay-v4-replay-37198445662.json'), 'utf8')) as { + source: { fixture: string }; trials: Array<{ arm: 'overlay-on' | 'overlay-off'; trial: number; tools: string[]; answer: string; v3: { measurementPassed: boolean } }>; + }; + const f = OVERLAY_FIXTURES.find(candidate => candidate.id === corpus.source.fixture)!; + const replay = async (trial: (typeof corpus.trials)[number]) => { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'overlay-v4-replay-')); + try { + return await runOverlayTrial({ fixture: f, directory: dir, record: () => {}, invoke: async () => result({ + events: [systemInit(), { type: 'result', subtype: 'success', result: trial.answer }] as AgentSdkResult['events'], + toolCalls: trial.tools.map(tool => ({ tool, input: {}, output: '' })), output: trial.answer, model: f.model, + }) }); + } finally { fs.rmSync(dir, { recursive: true, force: true }); } + }; + test('the stored OFF wrong answer is a measured result and the case passes under v4', async () => { + const outcomes = await Promise.all(corpus.trials.map(replay)); + const arm = (name: string) => outcomes.filter((_, index) => corpus.trials[index].arm === name); + expect(corpus.trials.filter(trial => !trial.v3.measurementPassed).map(trial => `${trial.arm}-${trial.trial}`)).toEqual(['overlay-off-0']); + expect(outcomes.every(outcome => outcome.passed)).toBe(true); + expect(arm('overlay-off').map(outcome => outcome.taskCorrect)).toEqual([false, ...Array(9).fill(true)]); + expect(arm('overlay-off')[0].answerError).toContain('does not match the fixture task'); + expect(assessOverlayArms(f, arm('overlay-on'), arm('overlay-off'))).toMatchObject({ + measurementsValid: true, correctnessPassed: true, gatePassed: true, passed: true, comparison: { status: 'baseline_saturated', meanOn: 0, meanOff: 0 }, + }); + expect(OVERLAY_CONTRACT.version).toBe(4); + }); + test('the same wrong answer in the ON arm still fails the case', async () => { + const swapped = corpus.trials.map(trial => ({ ...trial, arm: trial.arm === 'overlay-on' ? 'overlay-off' as const : 'overlay-on' as const })); + const outcomes = await Promise.all(swapped.map(replay)); + const arm = (name: string) => outcomes.filter((_, index) => swapped[index].arm === name); + expect(assessOverlayArms(f, arm('overlay-on'), arm('overlay-off'))).toMatchObject({ measurementsValid: true, correctnessPassed: false, passed: false }); + }); +}); + describe('retry-specific native evidence', () => { test('retains events preceding an exception and a separate successful retry stream', async () => { const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'overlay-capture-unit-'));