D1: score overlay wrong answers as comparison results (contract v4)

A completed answer that fails the fixture's output check is now a valid
measurement with taskCorrect false and answerError set, in either arm.
Only execution, scope and metric failures invalidate a trial. ON
correctness still gates the case, so an OFF wrong answer stays in the
comparison. OVERLAY_CONTRACT moves to v4; v3 verdicts are not
reinterpreted. Replays the 37198445662 sonnet trials (OFF-0 wrong).

docs/OVERLAY_BENCHMARK_CONTRACT.md

@@ -1,4 +1,4 @@
-# Overlay benchmark behavior contract v2
+# Overlay benchmark behavior contract v4
 
 The paid overlay fixtures verify supported behavior, with comparative efficacy
 reported separately as research evidence. A passing case does **not** establish
@@ -7,7 +7,7 @@ blocking efficacy contract; historical failures retain their original verdicts.
 
 The contract is defined by `OVERLAY_CONTRACT` in
 `test/helpers/overlay-case-policy.ts`. Every new measurement and aggregate record
-identifies version 3. Eval entry names include `contract-v3` so automated result
+identifies version 4. Eval entry names include `contract-v4` so automated result
 matching cannot silently compare the old and new contracts by the same name.
 Selection IDs for the six retained paid cases stay unchanged.
 
@@ -20,9 +20,17 @@ OFF does not. Each retained fixture runs ten samples per arm, separately for
 Every planned sample in both arms must execute successfully, produce exactly
 one successful native terminal result without an error flag, preserve workspace
 scope, and yield a finite metric within the fixture's bounds. Missing reasoning
-metadata, failed output validation, oracle process errors, partial sampling,
-deadlines, cleanup failures, or recording failures keep the case failed. Missing
-observations are never replaced with zero or discarded to obtain a passing arm.
+metadata, oracle process errors, partial sampling, deadlines, cleanup failures,
+or recording failures keep the case failed. Missing observations are never
+replaced with zero or discarded to obtain a passing arm.
+
+A completed answer that fails the fixture's output check is a measured
+correctness result, not an invalid measurement, in either arm: its record has
+`measurementPassed: true`, `taskCorrect: false` and the check's message in
+`answerError`. Only ON correctness gates the case, so an OFF wrong answer stays
+in the comparison and an ON wrong answer fails the case. Version 3 counted an
+OFF wrong answer as an invalid measurement (census 37198445662); v3 verdicts
+are not reinterpreted.
 
 | Fixture family | Exact task and scope contract | Additional ON requirement | Measurement |
 | --- | --- | --- | --- |

test/fixtures/overlay-v4-replay-37198445662.json

@@ -0,0 +1,396 @@
+{
+ "source": {
+  "run": "37198445662",
+  "sha": "fd6854b",
+  "tier": "periodic",
+  "artifact": "paid-slice-26-a1",
+  "case": "overlay-harness-claude-dedicated-tools-vs-bash-sonnet",
+  "fixture": "claude-dedicated-tools-vs-bash-sonnet",
+  "attempt": 1,
+  "contract": 3
+ },
+ "excerpt": "Per trial: tool_use names in order and the native terminal result string. Everything else in the SDK stream is omitted.",
+ "originalOutcome": "FAIL under contract v3: overlay-off-0 answered src/api.ts: [] (wrong), was recorded as an invalid measurement, and the comparison became incomplete.",
+ "expected": {
+  "3": "fail",
+  "4": "pass"
+ },
+ "justification": "Contract v4 (D1, approved 2026-10-04) scores a completed OFF wrong answer as a correctness result; every ON trial is correct and the zero-Bash baseline gate holds.",
+ "trials": [
+  {
+   "arm": "overlay-on",
+   "trial": 0,
+   "tools": [
+    "Glob",
+    "Glob",
+    "Grep",
+    "Grep",
+    "Grep",
+    "Grep",
+    "Grep"
+   ],
+   "answer": "{\"src/index.ts\":[\"x\"],\"src/util.ts\":[\"util\"],\"src/config.ts\":[\"c\"],\"src/api.ts\":[\"fetchFoo\"],\"src/types.ts\":[\"Foo\"]}",
+   "v3": {
+    "measurementPassed": true,
+    "taskCorrect": true,
+    "exitReason": "success"
+   }
+  },
+  {
+   "arm": "overlay-on",
+   "trial": 1,
+   "tools": [
+    "Glob",
+    "Glob",
+    "Grep",
+    "Grep",
+    "Grep",
+    "Grep",
+    "Grep",
+    "Read"
+   ],
+   "answer": "{\"src/index.ts\":[\"x\"],\"src/util.ts\":[\"util\"],\"src/config.ts\":[\"c\"],\"src/api.ts\":[\"fetchFoo\"],\"src/types.ts\":[\"Foo\"]}",
+   "v3": {
+    "measurementPassed": true,
+    "taskCorrect": true,
+    "exitReason": "success"
+   }
+  },
+  {
+   "arm": "overlay-on",
+   "trial": 2,
+   "tools": [
+    "Glob",
+    "Glob",
+    "Read",
+    "Read",
+    "Read",
+    "Read",
+    "Read"
+   ],
+   "answer": "{\"src/index.ts\":[\"x\"],\"src/util.ts\":[\"util\"],\"src/config.ts\":[\"c\"],\"src/api.ts\":[\"fetchFoo\"],\"src/types.ts\":[\"Foo\"]}",
+   "v3": {
+    "measurementPassed": true,
+    "taskCorrect": true,
+    "exitReason": "success"
+   }
+  },
+  {
+   "arm": "overlay-on",
+   "trial": 3,
+   "tools": [
+    "Glob",
+    "Glob",
+    "Grep",
+    "Grep",
+    "Grep",
+    "Grep",
+    "Grep",
+    "Read"
+   ],
+   "answer": "{\"src/index.ts\":[\"x\"],\"src/util.ts\":[\"util\"],\"src/config.ts\":[\"c\"],\"src/api.ts\":[\"fetchFoo\"],\"src/types.ts\":[\"Foo\"]}",
+   "v3": {
+    "measurementPassed": true,
+    "taskCorrect": true,
+    "exitReason": "success"
+   }
+  },
+  {
+   "arm": "overlay-on",
+   "trial": 4,
+   "tools": [
+    "Glob",
+    "Glob",
+    "Grep"
+   ],
+   "answer": "{\"src/index.ts\":[\"x\"],\"src/util.ts\":[\"util\"],\"src/config.ts\":[\"c\"],\"src/api.ts\":[\"fetchFoo\"],\"src/types.ts\":[\"Foo\"]}",
+   "v3": {
+    "measurementPassed": true,
+    "taskCorrect": true,
+    "exitReason": "success"
+   }
+  },
+  {
+   "arm": "overlay-on",
+   "trial": 5,
+   "tools": [
+    "Glob",
+    "Glob",
+    "Grep",
+    "Grep",
+    "Grep",
+    "Grep",
+    "Grep"
+   ],
+   "answer": "{\"src/index.ts\":[\"x\"],\"src/util.ts\":[\"util\"],\"src/config.ts\":[\"c\"],\"src/api.ts\":[\"fetchFoo\"],\"src/types.ts\":[\"Foo\"]}",
+   "v3": {
+    "measurementPassed": true,
+    "taskCorrect": true,
+    "exitReason": "success"
+   }
+  },
+  {
+   "arm": "overlay-on",
+   "trial": 6,
+   "tools": [
+    "Glob",
+    "Glob",
+    "Grep",
+    "Grep",
+    "Grep",
+    "Grep",
+    "Grep"
+   ],
+   "answer": "{\"src/index.ts\":[\"x\"],\"src/util.ts\":[\"util\"],\"src/config.ts\":[\"c\"],\"src/api.ts\":[\"fetchFoo\"],\"src/types.ts\":[\"Foo\"]}",
+   "v3": {
+    "measurementPassed": true,
+    "taskCorrect": true,
+    "exitReason": "success"
+   }
+  },
+  {
+   "arm": "overlay-on",
+   "trial": 7,
+   "tools": [
+    "Glob",
+    "Glob",
+    "Grep",
+    "Grep",
+    "Grep",
+    "Grep",
+    "Grep",
+    "Read"
+   ],
+   "answer": "{\"src/index.ts\":[\"x\"],\"src/util.ts\":[\"util\"],\"src/config.ts\":[\"c\"],\"src/api.ts\":[\"fetchFoo\"],\"src/types.ts\":[\"Foo\"]}",
+   "v3": {
+    "measurementPassed": true,
+    "taskCorrect": true,
+    "exitReason": "success"
+   }
+  },
+  {
+   "arm": "overlay-on",
+   "trial": 8,
+   "tools": [
+    "Glob",
+    "Grep"
+   ],
+   "answer": "{\"src/index.ts\":[\"x\"],\"src/util.ts\":[\"util\"],\"src/config.ts\":[\"c\"],\"src/api.ts\":[\"fetchFoo\"],\"src/types.ts\":[\"Foo\"]}",
+   "v3": {
+    "measurementPassed": true,
+    "taskCorrect": true,
+    "exitReason": "success"
+   }
+  },
+  {
+   "arm": "overlay-on",
+   "trial": 9,
+   "tools": [
+    "Glob",
+    "Glob",
+    "Grep",
+    "Grep",
+    "Grep",
+    "Grep",
+    "Grep",
+    "Read"
+   ],
+   "answer": "{\"src/index.ts\":[\"x\"],\"src/util.ts\":[\"util\"],\"src/config.ts\":[\"c\"],\"src/api.ts\":[\"fetchFoo\"],\"src/types.ts\":[\"Foo\"]}",
+   "v3": {
+    "measurementPassed": true,
+    "taskCorrect": true,
+    "exitReason": "success"
+   }
+  },
+  {
+   "arm": "overlay-off",
+   "trial": 0,
+   "tools": [
+    "Glob",
+    "Glob",
+    "Grep",
+    "Grep",
+    "Grep",
+    "Grep",
+    "Grep"
+   ],
+   "answer": "{\"src/index.ts\":[\"x\"],\"src/util.ts\":[\"util\"],\"src/config.ts\":[\"c\"],\"src/api.ts\":[],\"src/types.ts\":[\"Foo\"]}",
+   "v3": {
+    "measurementPassed": false,
+    "taskCorrect": false,
+    "exitReason": "validation_failed"
+   }
+  },
+  {
+   "arm": "overlay-off",
+   "trial": 1,
+   "tools": [
+    "Glob",
+    "Glob",
+    "Grep",
+    "Grep",
+    "Grep",
+    "Grep",
+    "Grep"
+   ],
+   "answer": "{\"src/index.ts\":[\"x\"],\"src/util.ts\":[\"util\"],\"src/config.ts\":[\"c\"],\"src/types.ts\":[\"Foo\"],\"src/api.ts\":[\"fetchFoo\"]}",
+   "v3": {
+    "measurementPassed": true,
+    "taskCorrect": true,
+    "exitReason": "success"
+   }
+  },
+  {
+   "arm": "overlay-off",
+   "trial": 2,
+   "tools": [
+    "Glob",
+    "Glob",
+    "Grep",
+    "Grep",
+    "Grep",
+    "Grep",
+    "Grep"
+   ],
+   "answer": "{\"src/index.ts\":[\"x\"],\"src/util.ts\":[\"util\"],\"src/config.ts\":[\"c\"],\"src/api.ts\":[\"fetchFoo\"],\"src/types.ts\":[\"Foo\"]}",
+   "v3": {
+    "measurementPassed": true,
+    "taskCorrect": true,
+    "exitReason": "success"
+   }
+  },
+  {
+   "arm": "overlay-off",
+   "trial": 3,
+   "tools": [
+    "Glob",
+    "Glob",
+    "Grep",
+    "Grep",
+    "Grep",
+    "Grep",
+    "Grep"
+   ],
+   "answer": "{\"src/index.ts\":[\"x\"],\"src/util.ts\":[\"util\"],\"src/config.ts\":[\"c\"],\"src/api.ts\":[\"fetchFoo\"],\"src/types.ts\":[\"Foo\"]}",
+   "v3": {
+    "measurementPassed": true,
+    "taskCorrect": true,
+    "exitReason": "success"
+   }
+  },
+  {
+   "arm": "overlay-off",
+   "trial": 4,
+   "tools": [
+    "Glob",
+    "Glob",
+    "Grep",
+    "Grep",
+    "Grep",
+    "Grep",
+    "Grep"
+   ],
+   "answer": "{\"src/index.ts\":[\"x\"],\"src/util.ts\":[\"util\"],\"src/config.ts\":[\"c\"],\"src/api.ts\":[\"fetchFoo\"],\"src/types.ts\":[\"Foo\"]}",
+   "v3": {
+    "measurementPassed": true,
+    "taskCorrect": true,
+    "exitReason": "success"
+   }
+  },
+  {
+   "arm": "overlay-off",
+   "trial": 5,
+   "tools": [
+    "Glob",
+    "Glob",
+    "Grep",
+    "Grep",
+    "Grep",
+    "Grep",
+    "Grep"
+   ],
+   "answer": "{\"src/index.ts\":[\"x\"],\"src/util.ts\":[\"util\"],\"src/config.ts\":[\"c\"],\"src/api.ts\":[\"fetchFoo\"],\"src/types.ts\":[\"Foo\"]}",
+   "v3": {
+    "measurementPassed": true,
+    "taskCorrect": true,
+    "exitReason": "success"
+   }
+  },
+  {
+   "arm": "overlay-off",
+   "trial": 6,
+   "tools": [
+    "Glob",
+    "Glob",
+    "Grep",
+    "Grep",
+    "Grep",
+    "Grep",
+    "Grep"
+   ],
+   "answer": "{\"src/index.ts\":[\"x\"],\"src/util.ts\":[\"util\"],\"src/config.ts\":[\"c\"],\"src/api.ts\":[\"fetchFoo\"],\"src/types.ts\":[\"Foo\"]}",
+   "v3": {
+    "measurementPassed": true,
+    "taskCorrect": true,
+    "exitReason": "success"
+   }
+  },
+  {
+   "arm": "overlay-off",
+   "trial": 7,
+   "tools": [
+    "Glob",
+    "Glob",
+    "Grep",
+    "Grep",
+    "Grep",
+    "Grep",
+    "Grep"
+   ],
+   "answer": "{\"src/index.ts\":[\"x\"],\"src/util.ts\":[\"util\"],\"src/config.ts\":[\"c\"],\"src/api.ts\":[\"fetchFoo\"],\"src/types.ts\":[\"Foo\"]}",
+   "v3": {
+    "measurementPassed": true,
+    "taskCorrect": true,
+    "exitReason": "success"
+   }
+  },
+  {
+   "arm": "overlay-off",
+   "trial": 8,
+   "tools": [
+    "Glob",
+    "Glob",
+    "Grep",
+    "Grep",
+    "Grep",
+    "Grep",
+    "Grep"
+   ],
+   "answer": "{\"src/index.ts\":[\"x\"],\"src/util.ts\":[\"util\"],\"src/config.ts\":[\"c\"],\"src/api.ts\":[\"fetchFoo\"],\"src/types.ts\":[\"Foo\"]}",
+   "v3": {
+    "measurementPassed": true,
+    "taskCorrect": true,
+    "exitReason": "success"
+   }
+  },
+  {
+   "arm": "overlay-off",
+   "trial": 9,
+   "tools": [
+    "Glob",
+    "Glob",
+    "Grep",
+    "Grep",
+    "Grep",
+    "Grep",
+    "Grep"
+   ],
+   "answer": "{\"src/index.ts\":[\"x\"],\"src/util.ts\":[\"util\"],\"src/config.ts\":[\"c\"],\"src/api.ts\":[\"fetchFoo\"],\"src/types.ts\":[\"Foo\"]}",
+   "v3": {
+    "measurementPassed": true,
+    "taskCorrect": true,
+    "exitReason": "success"
+   }
+  }
+ ]
+}

test/helpers/overlay-attempt.ts

@@ -10,6 +10,8 @@ export interface OverlayTrialOutcome {
   passed: boolean;
   taskCorrect: boolean;
   metric?: number;
+  /** Contract v4: a completed answer that failed the fixture's output check. */
+  answerError?: string;
   result?: AgentSdkResult;
   error?: string;
   exitReason: string;
@@ -57,13 +59,20 @@ export async function runOverlayTrial(options: {
       (fixture.comparison.maximum !== undefined && metric > fixture.comparison.maximum)))) {
       throw new Error(`invalid fixture metric: ${metric}`);
     }
-    fixture.verify?.(result, directory, metric);
+    let answerError: string | undefined;
+    try { fixture.verify?.(result, directory, metric); }
+    catch (error) { answerError = error instanceof Error ? `${error.name}: ${error.message}` : String(error); }
     checkActive();
     // Implementation oracles execute agent-written code. Their side effects
     // must obey the same scope contract and appear in the retained snapshot.
     after = snapshotWorkspace(directory);
     assertWorkspaceChanges(before, after, fixture.allowedChanges ?? []);
-    outcome = { passed: true, taskCorrect: fixture.taskCorrect?.(metric) ?? true, metric, result, exitReason: result.exitReason, before, after };
+    // Contract v4: a wrong completed answer is a measured correctness result in
+    // either arm, not an invalid measurement.
+    outcome = {
+      passed: true, taskCorrect: !answerError && (fixture.taskCorrect?.(metric) ?? true),
+      metric, result, exitReason: result.exitReason, before, after, ...(answerError ? { answerError } : {}),
+    };
   } catch (error) {
     // Capture post-failure files when possible; preserve the first failure if
     // the workspace is absent or evidence capture itself cannot complete.
@@ -98,11 +107,11 @@ export function assessOverlayArms(fixture: OverlayFixture, overlay: OverlayTrial
     off: off.filter((trial) => trial?.passed).map((trial) => trial.metric!),
   };
   const comparison = assessComparison(metrics, fixture.trials, fixture.comparison, fixture.pass);
+  // Only execution, scope and metric failures invalidate a trial, in both arms.
   const measurementsValid = comparison.status !== 'incomplete' && [overlay, off].every((arm) =>
     arm.length === fixture.trials && Array.from(arm).every((trial) => trial?.passed));
-  // Baseline completion is an experimental variable for literal scope. A valid
-  // OFF sample may score 0..3; ON must complete all three. Every ON trial has
-  // already passed its fixture's output check to count as a valid measurement.
+  // Task correctness is a measured result in both arms (OFF may score 0..3 or
+  // answer wrongly); only ON correctness gates the case.
   const correctnessPassed = measurementsValid && overlay.every((trial) => trial.taskCorrect);
   // A fixture with a gate also blocks on its comparison (dedicated tools: 20%
   // fewer Bash calls, or none at a zero baseline). Without a gate the

test/helpers/overlay-case-policy.ts

@@ -1,11 +1,14 @@
-/** Version 3 gates supported behavior, plus the comparison for fixtures that
+/** Version 4 keeps v3's gate but scores a completed wrong answer in either arm
+ * as a correctness result; only execution, scope and metric failures invalidate
+ * a trial. An OFF wrong answer no longer fails the comparison.
+ * Version 3 gates supported behavior, plus the comparison for fixtures that
  * declare a gate (dedicated tools: correct output and 20% fewer Bash calls,
  * replacing v2's zero Bash in every ON trial). Other comparisons stay research-only.
  * Version 1 required efficacy lift. Never reinterpret recorded failures of earlier versions.
  */
 export const OVERLAY_CONTRACT = {
   name: 'overlay-behavior',
-  version: 3,
+  version: 4,
   releaseGate: 'execution_scope_complete_sampling_on_correctness_and_fixture_gate',
   comparisonRole: 'research_only_unless_fixture_gate',
   resourceNonRegression: 'not_established',

test/helpers/overlay-case.ts

@@ -46,7 +46,7 @@ function saveTrial(fixture: OverlayFixture, attempt: number, arm: Arm, n: number
     // this JSONL remains the final SDK result used for the trial measurement.
     rateLimitAttemptStreamsRetained: true,
     measurementPassed: outcome.passed, taskCorrect: outcome.taskCorrect, metric: outcome.metric, exitReason: outcome.exitReason,
-    error: outcome.error, before: outcome.before, after: outcome.after,
+    error: outcome.error, answerError: outcome.answerError, before: outcome.before, after: outcome.after,
   }, null, 2) + '\n', { flag: 'wx' });
 }
 
@@ -143,7 +143,7 @@ export function registerOverlayCase(fixtureId: string): void {
               name: `${fixture.id}-contract-v${OVERLAY_CONTRACT.version}-${key}`, suite: 'overlay-harness-measurement', tier: 'e2e',
               passed: outcome.passed, duration_ms: sdk?.durationMs ?? 0, cost_usd: sdk?.costUsd ?? 0,
               transcript: sdk?.events, prompt: fixture.userPrompt,
-              output: JSON.stringify({ contract: OVERLAY_CONTRACT, measurementPassed: outcome.passed, taskCorrect: outcome.taskCorrect, metric: outcome.metric, assistantOutput: sdk?.output }),
+              output: JSON.stringify({ contract: OVERLAY_CONTRACT, measurementPassed: outcome.passed, taskCorrect: outcome.taskCorrect, answerError: outcome.answerError, metric: outcome.metric, assistantOutput: sdk?.output }),
               turns_used: sdk?.turnsUsed, browse_errors: sdk?.browseErrors,
               exit_reason: outcome.exitReason, error: outcome.error,
               model: sdk?.model ?? fixture.model, first_response_ms: sdk?.firstResponseMs, max_inter_turn_ms: sdk?.maxInterTurnMs,

test/overlay-measurement.test.ts

@@ -5,6 +5,7 @@ import * as path from 'path';
 import type { AgentSdkResult, QueryProvider } from './helpers/agent-sdk-runner';
 import { firstAssistantMessageToolCount, reportedThinkingTokens, assessComparison, trialArtifactStem } from './helpers/overlay-measurement';
 import { runOverlayTrial, awaitOverlayWorkers, assessOverlayArms, captureOverlayQueryAttempts, type OverlayTrialOutcome } from './helpers/overlay-attempt';
+import { OVERLAY_CONTRACT } from './helpers/overlay-case-policy';
 import { setupLiteralWorkspace, correctLiteralTargets, snapshotWorkspace, assertReadOnlyWorkspace } from './helpers/overlay-workspace';
 import { fanoutPass, higherIsBetter20Pct, lowerIsBetter20Pct, lowerIsBetter20PctOrZeroBaseline, OVERLAY_FIXTURES, type OverlayFixture } from './fixtures/overlay-nudges';
 
@@ -66,9 +67,10 @@ describe('SDK overlay measurements', () => {
 describe('correctness versus efficacy', () => {
   describe.each(['claude-opus-4-7', 'claude-sonnet-4-6'])('dedicated-tool gate for %s', (model) => {
     const f = { ...OVERLAY_FIXTURES.find(f => f.metricName === 'bash_tool_calls' && f.model === model)!, trials: 3 };
-    // A trial whose final JSON fails the fixture's verify is recorded as a failed measurement.
+    // Contract v4: a trial whose final JSON fails the fixture's verify is a valid
+    // measurement with a wrong answer, not an invalid measurement.
     const correct = (metric: number): OverlayTrialOutcome => ({ passed: true, taskCorrect: true, metric, exitReason: 'success' });
-    const incorrect = (metric: number): OverlayTrialOutcome => ({ passed: false, taskCorrect: false, metric, exitReason: 'validation_failed' });
+    const incorrect = (metric: number): OverlayTrialOutcome => ({ passed: true, taskCorrect: false, metric, exitReason: 'success', answerError: 'AssertionError: final answer does not match the fixture task' });
     const arm = (...trials: OverlayTrialOutcome[]) => trials;
     test('fixture gates on the comparison and on output correctness, not on zero ON Bash', () => {
       expect(f.taskCorrect).toBeUndefined();
@@ -88,9 +90,25 @@ describe('correctness versus efficacy', () => {
         gatePassed: false, passed: false, comparison: { status: 'no_measured_improvement' },
       });
     });
-    test('a 20% reduction with incorrect output fails', () => {
+    test('a 20% reduction with incorrect ON output fails', () => {
       expect(assessOverlayArms(f, arm(correct(0), incorrect(0), correct(0)), arm(correct(5), correct(5), correct(5)))).toMatchObject({
-        measurementsValid: false, passed: false,
+        measurementsValid: true, correctnessPassed: false, passed: false,
+      });
+    });
+    test('an OFF wrong answer is a comparison result, not an invalid measurement', () => {
+      expect(assessOverlayArms(f, arm(correct(0), correct(0), correct(0)), arm(incorrect(0), correct(0), correct(0)))).toMatchObject({
+        measurementsValid: true, correctnessPassed: true, gatePassed: true, passed: true, comparison: { status: 'baseline_saturated' },
+      });
+    });
+    test('an execution failure invalidates the measurement in either arm', () => {
+      const failed: OverlayTrialOutcome = { passed: false, taskCorrect: false, metric: 0, exitReason: 'error_max_turns', error: 'Error: SDK execution failed: error_max_turns' };
+      for (const [on, off] of [[arm(failed, correct(0), correct(0)), arm(correct(0), correct(0), correct(0))], [arm(correct(0), correct(0), correct(0)), arm(failed, correct(0), correct(0))]]) {
+        expect(assessOverlayArms(f, on, off)).toMatchObject({ measurementsValid: false, correctnessPassed: false, gatePassed: false, passed: false, comparison: { status: 'incomplete' } });
+      }
+    });
+    test('an OFF wrong answer does not rescue a failed comparison gate', () => {
+      expect(assessOverlayArms(f, arm(correct(6), correct(6), correct(6)), arm(incorrect(5), correct(5), correct(5)))).toMatchObject({
+        measurementsValid: true, correctnessPassed: true, gatePassed: false, passed: false, comparison: { status: 'regressed' },
       });
     });
     test('a zero-Bash baseline passes only when every ON trial is also zero', () => {
@@ -288,10 +306,13 @@ describe('complete trial lifecycle', () => {
       }, record: (entry) => records.push(entry) });
       expect(records).toHaveLength(1);
       expect(records[0]).toBe(outcome);
-      expect(outcome.passed).toBe(scenario === 'success');
+      // Contract v4: a failed output check is a completed wrong answer, not an invalid measurement.
+      expect(outcome.passed).toBe(scenario === 'success' || scenario === 'assertion');
+      expect(outcome.taskCorrect).toBe(scenario === 'success');
+      if (scenario === 'assertion') expect(outcome).toMatchObject({ exitReason: 'success', answerError: 'Error: required output absent' });
       if (scenario === 'runner_error' || scenario === 'setup_error') expect(outcome.exitReason).toBe('harness_error');
       if (scenario === 'max_turns') expect(outcome.exitReason).toBe('error_max_turns');
-      if (scenario !== 'success') expect(outcome.error).toBeTruthy();
+      if (scenario !== 'success' && scenario !== 'assertion') expect(outcome.error).toBeTruthy();
       if (scenario !== 'runner_error' && scenario !== 'setup_error') expect(outcome.result?.costUsd).toBe(0.03);
     } finally { fs.rmSync(dir, { recursive: true, force: true }); }
   });
@@ -311,6 +332,40 @@ describe('complete trial lifecycle', () => {
   });
 });
 
+describe('contract v4 replay of census 37198445662', () => {
+  const corpus = JSON.parse(fs.readFileSync(path.join(import.meta.dir, 'fixtures', 'overlay-v4-replay-37198445662.json'), 'utf8')) as {
+    source: { fixture: string }; trials: Array<{ arm: 'overlay-on' | 'overlay-off'; trial: number; tools: string[]; answer: string; v3: { measurementPassed: boolean } }>;
+  };
+  const f = OVERLAY_FIXTURES.find(candidate => candidate.id === corpus.source.fixture)!;
+  const replay = async (trial: (typeof corpus.trials)[number]) => {
+    const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'overlay-v4-replay-'));
+    try {
+      return await runOverlayTrial({ fixture: f, directory: dir, record: () => {}, invoke: async () => result({
+        events: [systemInit(), { type: 'result', subtype: 'success', result: trial.answer }] as AgentSdkResult['events'],
+        toolCalls: trial.tools.map(tool => ({ tool, input: {}, output: '' })), output: trial.answer, model: f.model,
+      }) });
+    } finally { fs.rmSync(dir, { recursive: true, force: true }); }
+  };
+  test('the stored OFF wrong answer is a measured result and the case passes under v4', async () => {
+    const outcomes = await Promise.all(corpus.trials.map(replay));
+    const arm = (name: string) => outcomes.filter((_, index) => corpus.trials[index].arm === name);
+    expect(corpus.trials.filter(trial => !trial.v3.measurementPassed).map(trial => `${trial.arm}-${trial.trial}`)).toEqual(['overlay-off-0']);
+    expect(outcomes.every(outcome => outcome.passed)).toBe(true);
+    expect(arm('overlay-off').map(outcome => outcome.taskCorrect)).toEqual([false, ...Array(9).fill(true)]);
+    expect(arm('overlay-off')[0].answerError).toContain('does not match the fixture task');
+    expect(assessOverlayArms(f, arm('overlay-on'), arm('overlay-off'))).toMatchObject({
+      measurementsValid: true, correctnessPassed: true, gatePassed: true, passed: true, comparison: { status: 'baseline_saturated', meanOn: 0, meanOff: 0 },
+    });
+    expect(OVERLAY_CONTRACT.version).toBe(4);
+  });
+  test('the same wrong answer in the ON arm still fails the case', async () => {
+    const swapped = corpus.trials.map(trial => ({ ...trial, arm: trial.arm === 'overlay-on' ? 'overlay-off' as const : 'overlay-on' as const }));
+    const outcomes = await Promise.all(swapped.map(replay));
+    const arm = (name: string) => outcomes.filter((_, index) => swapped[index].arm === name);
+    expect(assessOverlayArms(f, arm('overlay-on'), arm('overlay-off'))).toMatchObject({ measurementsValid: true, correctnessPassed: false, passed: false });
+  });
+});
+
 describe('retry-specific native evidence', () => {
   test('retains events preceding an exception and a separate successful retry stream', async () => {
     const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'overlay-capture-unit-'));