mirror of
https://github.com/garrytan/gstack.git
synced 2026-11-02 15:04:37 +01:00
D1: score overlay wrong answers as comparison results (contract v4)
A completed answer that fails the fixture's output check is now a valid measurement with taskCorrect false and answerError set, in either arm. Only execution, scope and metric failures invalidate a trial. ON correctness still gates the case, so an OFF wrong answer stays in the comparison. OVERLAY_CONTRACT moves to v4; v3 verdicts are not reinterpreted. Replays the 37198445662 sonnet trials (OFF-0 wrong).
This commit is contained in:
garrytan committed
1 parent
e54923176f
commit
c02efeb57f
6 files changed
+491
-20
No files matched your search
+396
@@ -0,0 +1,396 @@
|
||||
{
|
||||
"source": {
|
||||
"run": "37198445662",
|
||||
"sha": "fd6854b",
|
||||
"tier": "periodic",
|
||||
"artifact": "paid-slice-26-a1",
|
||||
"case": "overlay-harness-claude-dedicated-tools-vs-bash-sonnet",
|
||||
"fixture": "claude-dedicated-tools-vs-bash-sonnet",
|
||||
"attempt": 1,
|
||||
"contract": 3
|
||||
},
|
||||
"excerpt": "Per trial: tool_use names in order and the native terminal result string. Everything else in the SDK stream is omitted.",
|
||||
"originalOutcome": "FAIL under contract v3: overlay-off-0 answered src/api.ts: [] (wrong), was recorded as an invalid measurement, and the comparison became incomplete.",
|
||||
"expected": {
|
||||
"3": "fail",
|
||||
"4": "pass"
|
||||
},
|
||||
"justification": "Contract v4 (D1, approved 2026-10-04) scores a completed OFF wrong answer as a correctness result; every ON trial is correct and the zero-Bash baseline gate holds.",
|
||||
"trials": [
|
||||
{
|
||||
"arm": "overlay-on",
|
||||
"trial": 0,
|
||||
"tools": [
|
||||
"Glob",
|
||||
"Glob",
|
||||
"Grep",
|
||||
"Grep",
|
||||
"Grep",
|
||||
"Grep",
|
||||
"Grep"
|
||||
],
|
||||
"answer": "{\"src/index.ts\":[\"x\"],\"src/util.ts\":[\"util\"],\"src/config.ts\":[\"c\"],\"src/api.ts\":[\"fetchFoo\"],\"src/types.ts\":[\"Foo\"]}",
|
||||
"v3": {
|
||||
"measurementPassed": true,
|
||||
"taskCorrect": true,
|
||||
"exitReason": "success"
|
||||
}
|
||||
},
|
||||
{
|
||||
"arm": "overlay-on",
|
||||
"trial": 1,
|
||||
"tools": [
|
||||
"Glob",
|
||||
"Glob",
|
||||
"Grep",
|
||||
"Grep",
|
||||
"Grep",
|
||||
"Grep",
|
||||
"Grep",
|
||||
"Read"
|
||||
],
|
||||
"answer": "{\"src/index.ts\":[\"x\"],\"src/util.ts\":[\"util\"],\"src/config.ts\":[\"c\"],\"src/api.ts\":[\"fetchFoo\"],\"src/types.ts\":[\"Foo\"]}",
|
||||
"v3": {
|
||||
"measurementPassed": true,
|
||||
"taskCorrect": true,
|
||||
"exitReason": "success"
|
||||
}
|
||||
},
|
||||
{
|
||||
"arm": "overlay-on",
|
||||
"trial": 2,
|
||||
"tools": [
|
||||
"Glob",
|
||||
"Glob",
|
||||
"Read",
|
||||
"Read",
|
||||
"Read",
|
||||
"Read",
|
||||
"Read"
|
||||
],
|
||||
"answer": "{\"src/index.ts\":[\"x\"],\"src/util.ts\":[\"util\"],\"src/config.ts\":[\"c\"],\"src/api.ts\":[\"fetchFoo\"],\"src/types.ts\":[\"Foo\"]}",
|
||||
"v3": {
|
||||
"measurementPassed": true,
|
||||
"taskCorrect": true,
|
||||
"exitReason": "success"
|
||||
}
|
||||
},
|
||||
{
|
||||
"arm": "overlay-on",
|
||||
"trial": 3,
|
||||
"tools": [
|
||||
"Glob",
|
||||
"Glob",
|
||||
"Grep",
|
||||
"Grep",
|
||||
"Grep",
|
||||
"Grep",
|
||||
"Grep",
|
||||
"Read"
|
||||
],
|
||||
"answer": "{\"src/index.ts\":[\"x\"],\"src/util.ts\":[\"util\"],\"src/config.ts\":[\"c\"],\"src/api.ts\":[\"fetchFoo\"],\"src/types.ts\":[\"Foo\"]}",
|
||||
"v3": {
|
||||
"measurementPassed": true,
|
||||
"taskCorrect": true,
|
||||
"exitReason": "success"
|
||||
}
|
||||
},
|
||||
{
|
||||
"arm": "overlay-on",
|
||||
"trial": 4,
|
||||
"tools": [
|
||||
"Glob",
|
||||
"Glob",
|
||||
"Grep"
|
||||
],
|
||||
"answer": "{\"src/index.ts\":[\"x\"],\"src/util.ts\":[\"util\"],\"src/config.ts\":[\"c\"],\"src/api.ts\":[\"fetchFoo\"],\"src/types.ts\":[\"Foo\"]}",
|
||||
"v3": {
|
||||
"measurementPassed": true,
|
||||
"taskCorrect": true,
|
||||
"exitReason": "success"
|
||||
}
|
||||
},
|
||||
{
|
||||
"arm": "overlay-on",
|
||||
"trial": 5,
|
||||
"tools": [
|
||||
"Glob",
|
||||
"Glob",
|
||||
"Grep",
|
||||
"Grep",
|
||||
"Grep",
|
||||
"Grep",
|
||||
"Grep"
|
||||
],
|
||||
"answer": "{\"src/index.ts\":[\"x\"],\"src/util.ts\":[\"util\"],\"src/config.ts\":[\"c\"],\"src/api.ts\":[\"fetchFoo\"],\"src/types.ts\":[\"Foo\"]}",
|
||||
"v3": {
|
||||
"measurementPassed": true,
|
||||
"taskCorrect": true,
|
||||
"exitReason": "success"
|
||||
}
|
||||
},
|
||||
{
|
||||
"arm": "overlay-on",
|
||||
"trial": 6,
|
||||
"tools": [
|
||||
"Glob",
|
||||
"Glob",
|
||||
"Grep",
|
||||
"Grep",
|
||||
"Grep",
|
||||
"Grep",
|
||||
"Grep"
|
||||
],
|
||||
"answer": "{\"src/index.ts\":[\"x\"],\"src/util.ts\":[\"util\"],\"src/config.ts\":[\"c\"],\"src/api.ts\":[\"fetchFoo\"],\"src/types.ts\":[\"Foo\"]}",
|
||||
"v3": {
|
||||
"measurementPassed": true,
|
||||
"taskCorrect": true,
|
||||
"exitReason": "success"
|
||||
}
|
||||
},
|
||||
{
|
||||
"arm": "overlay-on",
|
||||
"trial": 7,
|
||||
"tools": [
|
||||
"Glob",
|
||||
"Glob",
|
||||
"Grep",
|
||||
"Grep",
|
||||
"Grep",
|
||||
"Grep",
|
||||
"Grep",
|
||||
"Read"
|
||||
],
|
||||
"answer": "{\"src/index.ts\":[\"x\"],\"src/util.ts\":[\"util\"],\"src/config.ts\":[\"c\"],\"src/api.ts\":[\"fetchFoo\"],\"src/types.ts\":[\"Foo\"]}",
|
||||
"v3": {
|
||||
"measurementPassed": true,
|
||||
"taskCorrect": true,
|
||||
"exitReason": "success"
|
||||
}
|
||||
},
|
||||
{
|
||||
"arm": "overlay-on",
|
||||
"trial": 8,
|
||||
"tools": [
|
||||
"Glob",
|
||||
"Grep"
|
||||
],
|
||||
"answer": "{\"src/index.ts\":[\"x\"],\"src/util.ts\":[\"util\"],\"src/config.ts\":[\"c\"],\"src/api.ts\":[\"fetchFoo\"],\"src/types.ts\":[\"Foo\"]}",
|
||||
"v3": {
|
||||
"measurementPassed": true,
|
||||
"taskCorrect": true,
|
||||
"exitReason": "success"
|
||||
}
|
||||
},
|
||||
{
|
||||
"arm": "overlay-on",
|
||||
"trial": 9,
|
||||
"tools": [
|
||||
"Glob",
|
||||
"Glob",
|
||||
"Grep",
|
||||
"Grep",
|
||||
"Grep",
|
||||
"Grep",
|
||||
"Grep",
|
||||
"Read"
|
||||
],
|
||||
"answer": "{\"src/index.ts\":[\"x\"],\"src/util.ts\":[\"util\"],\"src/config.ts\":[\"c\"],\"src/api.ts\":[\"fetchFoo\"],\"src/types.ts\":[\"Foo\"]}",
|
||||
"v3": {
|
||||
"measurementPassed": true,
|
||||
"taskCorrect": true,
|
||||
"exitReason": "success"
|
||||
}
|
||||
},
|
||||
{
|
||||
"arm": "overlay-off",
|
||||
"trial": 0,
|
||||
"tools": [
|
||||
"Glob",
|
||||
"Glob",
|
||||
"Grep",
|
||||
"Grep",
|
||||
"Grep",
|
||||
"Grep",
|
||||
"Grep"
|
||||
],
|
||||
"answer": "{\"src/index.ts\":[\"x\"],\"src/util.ts\":[\"util\"],\"src/config.ts\":[\"c\"],\"src/api.ts\":[],\"src/types.ts\":[\"Foo\"]}",
|
||||
"v3": {
|
||||
"measurementPassed": false,
|
||||
"taskCorrect": false,
|
||||
"exitReason": "validation_failed"
|
||||
}
|
||||
},
|
||||
{
|
||||
"arm": "overlay-off",
|
||||
"trial": 1,
|
||||
"tools": [
|
||||
"Glob",
|
||||
"Glob",
|
||||
"Grep",
|
||||
"Grep",
|
||||
"Grep",
|
||||
"Grep",
|
||||
"Grep"
|
||||
],
|
||||
"answer": "{\"src/index.ts\":[\"x\"],\"src/util.ts\":[\"util\"],\"src/config.ts\":[\"c\"],\"src/types.ts\":[\"Foo\"],\"src/api.ts\":[\"fetchFoo\"]}",
|
||||
"v3": {
|
||||
"measurementPassed": true,
|
||||
"taskCorrect": true,
|
||||
"exitReason": "success"
|
||||
}
|
||||
},
|
||||
{
|
||||
"arm": "overlay-off",
|
||||
"trial": 2,
|
||||
"tools": [
|
||||
"Glob",
|
||||
"Glob",
|
||||
"Grep",
|
||||
"Grep",
|
||||
"Grep",
|
||||
"Grep",
|
||||
"Grep"
|
||||
],
|
||||
"answer": "{\"src/index.ts\":[\"x\"],\"src/util.ts\":[\"util\"],\"src/config.ts\":[\"c\"],\"src/api.ts\":[\"fetchFoo\"],\"src/types.ts\":[\"Foo\"]}",
|
||||
"v3": {
|
||||
"measurementPassed": true,
|
||||
"taskCorrect": true,
|
||||
"exitReason": "success"
|
||||
}
|
||||
},
|
||||
{
|
||||
"arm": "overlay-off",
|
||||
"trial": 3,
|
||||
"tools": [
|
||||
"Glob",
|
||||
"Glob",
|
||||
"Grep",
|
||||
"Grep",
|
||||
"Grep",
|
||||
"Grep",
|
||||
"Grep"
|
||||
],
|
||||
"answer": "{\"src/index.ts\":[\"x\"],\"src/util.ts\":[\"util\"],\"src/config.ts\":[\"c\"],\"src/api.ts\":[\"fetchFoo\"],\"src/types.ts\":[\"Foo\"]}",
|
||||
"v3": {
|
||||
"measurementPassed": true,
|
||||
"taskCorrect": true,
|
||||
"exitReason": "success"
|
||||
}
|
||||
},
|
||||
{
|
||||
"arm": "overlay-off",
|
||||
"trial": 4,
|
||||
"tools": [
|
||||
"Glob",
|
||||
"Glob",
|
||||
"Grep",
|
||||
"Grep",
|
||||
"Grep",
|
||||
"Grep",
|
||||
"Grep"
|
||||
],
|
||||
"answer": "{\"src/index.ts\":[\"x\"],\"src/util.ts\":[\"util\"],\"src/config.ts\":[\"c\"],\"src/api.ts\":[\"fetchFoo\"],\"src/types.ts\":[\"Foo\"]}",
|
||||
"v3": {
|
||||
"measurementPassed": true,
|
||||
"taskCorrect": true,
|
||||
"exitReason": "success"
|
||||
}
|
||||
},
|
||||
{
|
||||
"arm": "overlay-off",
|
||||
"trial": 5,
|
||||
"tools": [
|
||||
"Glob",
|
||||
"Glob",
|
||||
"Grep",
|
||||
"Grep",
|
||||
"Grep",
|
||||
"Grep",
|
||||
"Grep"
|
||||
],
|
||||
"answer": "{\"src/index.ts\":[\"x\"],\"src/util.ts\":[\"util\"],\"src/config.ts\":[\"c\"],\"src/api.ts\":[\"fetchFoo\"],\"src/types.ts\":[\"Foo\"]}",
|
||||
"v3": {
|
||||
"measurementPassed": true,
|
||||
"taskCorrect": true,
|
||||
"exitReason": "success"
|
||||
}
|
||||
},
|
||||
{
|
||||
"arm": "overlay-off",
|
||||
"trial": 6,
|
||||
"tools": [
|
||||
"Glob",
|
||||
"Glob",
|
||||
"Grep",
|
||||
"Grep",
|
||||
"Grep",
|
||||
"Grep",
|
||||
"Grep"
|
||||
],
|
||||
"answer": "{\"src/index.ts\":[\"x\"],\"src/util.ts\":[\"util\"],\"src/config.ts\":[\"c\"],\"src/api.ts\":[\"fetchFoo\"],\"src/types.ts\":[\"Foo\"]}",
|
||||
"v3": {
|
||||
"measurementPassed": true,
|
||||
"taskCorrect": true,
|
||||
"exitReason": "success"
|
||||
}
|
||||
},
|
||||
{
|
||||
"arm": "overlay-off",
|
||||
"trial": 7,
|
||||
"tools": [
|
||||
"Glob",
|
||||
"Glob",
|
||||
"Grep",
|
||||
"Grep",
|
||||
"Grep",
|
||||
"Grep",
|
||||
"Grep"
|
||||
],
|
||||
"answer": "{\"src/index.ts\":[\"x\"],\"src/util.ts\":[\"util\"],\"src/config.ts\":[\"c\"],\"src/api.ts\":[\"fetchFoo\"],\"src/types.ts\":[\"Foo\"]}",
|
||||
"v3": {
|
||||
"measurementPassed": true,
|
||||
"taskCorrect": true,
|
||||
"exitReason": "success"
|
||||
}
|
||||
},
|
||||
{
|
||||
"arm": "overlay-off",
|
||||
"trial": 8,
|
||||
"tools": [
|
||||
"Glob",
|
||||
"Glob",
|
||||
"Grep",
|
||||
"Grep",
|
||||
"Grep",
|
||||
"Grep",
|
||||
"Grep"
|
||||
],
|
||||
"answer": "{\"src/index.ts\":[\"x\"],\"src/util.ts\":[\"util\"],\"src/config.ts\":[\"c\"],\"src/api.ts\":[\"fetchFoo\"],\"src/types.ts\":[\"Foo\"]}",
|
||||
"v3": {
|
||||
"measurementPassed": true,
|
||||
"taskCorrect": true,
|
||||
"exitReason": "success"
|
||||
}
|
||||
},
|
||||
{
|
||||
"arm": "overlay-off",
|
||||
"trial": 9,
|
||||
"tools": [
|
||||
"Glob",
|
||||
"Glob",
|
||||
"Grep",
|
||||
"Grep",
|
||||
"Grep",
|
||||
"Grep",
|
||||
"Grep"
|
||||
],
|
||||
"answer": "{\"src/index.ts\":[\"x\"],\"src/util.ts\":[\"util\"],\"src/config.ts\":[\"c\"],\"src/api.ts\":[\"fetchFoo\"],\"src/types.ts\":[\"Foo\"]}",
|
||||
"v3": {
|
||||
"measurementPassed": true,
|
||||
"taskCorrect": true,
|
||||
"exitReason": "success"
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -10,6 +10,8 @@ export interface OverlayTrialOutcome {
|
||||
passed: boolean;
|
||||
taskCorrect: boolean;
|
||||
metric?: number;
|
||||
/** Contract v4: a completed answer that failed the fixture's output check. */
|
||||
answerError?: string;
|
||||
result?: AgentSdkResult;
|
||||
error?: string;
|
||||
exitReason: string;
|
||||
@@ -57,13 +59,20 @@ export async function runOverlayTrial(options: {
|
||||
(fixture.comparison.maximum !== undefined && metric > fixture.comparison.maximum)))) {
|
||||
throw new Error(`invalid fixture metric: ${metric}`);
|
||||
}
|
||||
fixture.verify?.(result, directory, metric);
|
||||
let answerError: string | undefined;
|
||||
try { fixture.verify?.(result, directory, metric); }
|
||||
catch (error) { answerError = error instanceof Error ? `${error.name}: ${error.message}` : String(error); }
|
||||
checkActive();
|
||||
// Implementation oracles execute agent-written code. Their side effects
|
||||
// must obey the same scope contract and appear in the retained snapshot.
|
||||
after = snapshotWorkspace(directory);
|
||||
assertWorkspaceChanges(before, after, fixture.allowedChanges ?? []);
|
||||
outcome = { passed: true, taskCorrect: fixture.taskCorrect?.(metric) ?? true, metric, result, exitReason: result.exitReason, before, after };
|
||||
// Contract v4: a wrong completed answer is a measured correctness result in
|
||||
// either arm, not an invalid measurement.
|
||||
outcome = {
|
||||
passed: true, taskCorrect: !answerError && (fixture.taskCorrect?.(metric) ?? true),
|
||||
metric, result, exitReason: result.exitReason, before, after, ...(answerError ? { answerError } : {}),
|
||||
};
|
||||
} catch (error) {
|
||||
// Capture post-failure files when possible; preserve the first failure if
|
||||
// the workspace is absent or evidence capture itself cannot complete.
|
||||
@@ -98,11 +107,11 @@ export function assessOverlayArms(fixture: OverlayFixture, overlay: OverlayTrial
|
||||
off: off.filter((trial) => trial?.passed).map((trial) => trial.metric!),
|
||||
};
|
||||
const comparison = assessComparison(metrics, fixture.trials, fixture.comparison, fixture.pass);
|
||||
// Only execution, scope and metric failures invalidate a trial, in both arms.
|
||||
const measurementsValid = comparison.status !== 'incomplete' && [overlay, off].every((arm) =>
|
||||
arm.length === fixture.trials && Array.from(arm).every((trial) => trial?.passed));
|
||||
// Baseline completion is an experimental variable for literal scope. A valid
|
||||
// OFF sample may score 0..3; ON must complete all three. Every ON trial has
|
||||
// already passed its fixture's output check to count as a valid measurement.
|
||||
// Task correctness is a measured result in both arms (OFF may score 0..3 or
|
||||
// answer wrongly); only ON correctness gates the case.
|
||||
const correctnessPassed = measurementsValid && overlay.every((trial) => trial.taskCorrect);
|
||||
// A fixture with a gate also blocks on its comparison (dedicated tools: 20%
|
||||
// fewer Bash calls, or none at a zero baseline). Without a gate the
|
||||
|
||||
@@ -1,11 +1,14 @@
|
||||
/** Version 3 gates supported behavior, plus the comparison for fixtures that
|
||||
/** Version 4 keeps v3's gate but scores a completed wrong answer in either arm
|
||||
* as a correctness result; only execution, scope and metric failures invalidate
|
||||
* a trial. An OFF wrong answer no longer fails the comparison.
|
||||
* Version 3 gates supported behavior, plus the comparison for fixtures that
|
||||
* declare a gate (dedicated tools: correct output and 20% fewer Bash calls,
|
||||
* replacing v2's zero Bash in every ON trial). Other comparisons stay research-only.
|
||||
* Version 1 required efficacy lift. Never reinterpret recorded failures of earlier versions.
|
||||
*/
|
||||
export const OVERLAY_CONTRACT = {
|
||||
name: 'overlay-behavior',
|
||||
version: 3,
|
||||
version: 4,
|
||||
releaseGate: 'execution_scope_complete_sampling_on_correctness_and_fixture_gate',
|
||||
comparisonRole: 'research_only_unless_fixture_gate',
|
||||
resourceNonRegression: 'not_established',
|
||||
|
||||
@@ -46,7 +46,7 @@ function saveTrial(fixture: OverlayFixture, attempt: number, arm: Arm, n: number
|
||||
// this JSONL remains the final SDK result used for the trial measurement.
|
||||
rateLimitAttemptStreamsRetained: true,
|
||||
measurementPassed: outcome.passed, taskCorrect: outcome.taskCorrect, metric: outcome.metric, exitReason: outcome.exitReason,
|
||||
error: outcome.error, before: outcome.before, after: outcome.after,
|
||||
error: outcome.error, answerError: outcome.answerError, before: outcome.before, after: outcome.after,
|
||||
}, null, 2) + '\n', { flag: 'wx' });
|
||||
}
|
||||
|
||||
@@ -143,7 +143,7 @@ export function registerOverlayCase(fixtureId: string): void {
|
||||
name: `${fixture.id}-contract-v${OVERLAY_CONTRACT.version}-${key}`, suite: 'overlay-harness-measurement', tier: 'e2e',
|
||||
passed: outcome.passed, duration_ms: sdk?.durationMs ?? 0, cost_usd: sdk?.costUsd ?? 0,
|
||||
transcript: sdk?.events, prompt: fixture.userPrompt,
|
||||
output: JSON.stringify({ contract: OVERLAY_CONTRACT, measurementPassed: outcome.passed, taskCorrect: outcome.taskCorrect, metric: outcome.metric, assistantOutput: sdk?.output }),
|
||||
output: JSON.stringify({ contract: OVERLAY_CONTRACT, measurementPassed: outcome.passed, taskCorrect: outcome.taskCorrect, answerError: outcome.answerError, metric: outcome.metric, assistantOutput: sdk?.output }),
|
||||
turns_used: sdk?.turnsUsed, browse_errors: sdk?.browseErrors,
|
||||
exit_reason: outcome.exitReason, error: outcome.error,
|
||||
model: sdk?.model ?? fixture.model, first_response_ms: sdk?.firstResponseMs, max_inter_turn_ms: sdk?.maxInterTurnMs,
|
||||
|
||||
@@ -5,6 +5,7 @@ import * as path from 'path';
|
||||
import type { AgentSdkResult, QueryProvider } from './helpers/agent-sdk-runner';
|
||||
import { firstAssistantMessageToolCount, reportedThinkingTokens, assessComparison, trialArtifactStem } from './helpers/overlay-measurement';
|
||||
import { runOverlayTrial, awaitOverlayWorkers, assessOverlayArms, captureOverlayQueryAttempts, type OverlayTrialOutcome } from './helpers/overlay-attempt';
|
||||
import { OVERLAY_CONTRACT } from './helpers/overlay-case-policy';
|
||||
import { setupLiteralWorkspace, correctLiteralTargets, snapshotWorkspace, assertReadOnlyWorkspace } from './helpers/overlay-workspace';
|
||||
import { fanoutPass, higherIsBetter20Pct, lowerIsBetter20Pct, lowerIsBetter20PctOrZeroBaseline, OVERLAY_FIXTURES, type OverlayFixture } from './fixtures/overlay-nudges';
|
||||
|
||||
@@ -66,9 +67,10 @@ describe('SDK overlay measurements', () => {
|
||||
describe('correctness versus efficacy', () => {
|
||||
describe.each(['claude-opus-4-7', 'claude-sonnet-4-6'])('dedicated-tool gate for %s', (model) => {
|
||||
const f = { ...OVERLAY_FIXTURES.find(f => f.metricName === 'bash_tool_calls' && f.model === model)!, trials: 3 };
|
||||
// A trial whose final JSON fails the fixture's verify is recorded as a failed measurement.
|
||||
// Contract v4: a trial whose final JSON fails the fixture's verify is a valid
|
||||
// measurement with a wrong answer, not an invalid measurement.
|
||||
const correct = (metric: number): OverlayTrialOutcome => ({ passed: true, taskCorrect: true, metric, exitReason: 'success' });
|
||||
const incorrect = (metric: number): OverlayTrialOutcome => ({ passed: false, taskCorrect: false, metric, exitReason: 'validation_failed' });
|
||||
const incorrect = (metric: number): OverlayTrialOutcome => ({ passed: true, taskCorrect: false, metric, exitReason: 'success', answerError: 'AssertionError: final answer does not match the fixture task' });
|
||||
const arm = (...trials: OverlayTrialOutcome[]) => trials;
|
||||
test('fixture gates on the comparison and on output correctness, not on zero ON Bash', () => {
|
||||
expect(f.taskCorrect).toBeUndefined();
|
||||
@@ -88,9 +90,25 @@ describe('correctness versus efficacy', () => {
|
||||
gatePassed: false, passed: false, comparison: { status: 'no_measured_improvement' },
|
||||
});
|
||||
});
|
||||
test('a 20% reduction with incorrect output fails', () => {
|
||||
test('a 20% reduction with incorrect ON output fails', () => {
|
||||
expect(assessOverlayArms(f, arm(correct(0), incorrect(0), correct(0)), arm(correct(5), correct(5), correct(5)))).toMatchObject({
|
||||
measurementsValid: false, passed: false,
|
||||
measurementsValid: true, correctnessPassed: false, passed: false,
|
||||
});
|
||||
});
|
||||
test('an OFF wrong answer is a comparison result, not an invalid measurement', () => {
|
||||
expect(assessOverlayArms(f, arm(correct(0), correct(0), correct(0)), arm(incorrect(0), correct(0), correct(0)))).toMatchObject({
|
||||
measurementsValid: true, correctnessPassed: true, gatePassed: true, passed: true, comparison: { status: 'baseline_saturated' },
|
||||
});
|
||||
});
|
||||
test('an execution failure invalidates the measurement in either arm', () => {
|
||||
const failed: OverlayTrialOutcome = { passed: false, taskCorrect: false, metric: 0, exitReason: 'error_max_turns', error: 'Error: SDK execution failed: error_max_turns' };
|
||||
for (const [on, off] of [[arm(failed, correct(0), correct(0)), arm(correct(0), correct(0), correct(0))], [arm(correct(0), correct(0), correct(0)), arm(failed, correct(0), correct(0))]]) {
|
||||
expect(assessOverlayArms(f, on, off)).toMatchObject({ measurementsValid: false, correctnessPassed: false, gatePassed: false, passed: false, comparison: { status: 'incomplete' } });
|
||||
}
|
||||
});
|
||||
test('an OFF wrong answer does not rescue a failed comparison gate', () => {
|
||||
expect(assessOverlayArms(f, arm(correct(6), correct(6), correct(6)), arm(incorrect(5), correct(5), correct(5)))).toMatchObject({
|
||||
measurementsValid: true, correctnessPassed: true, gatePassed: false, passed: false, comparison: { status: 'regressed' },
|
||||
});
|
||||
});
|
||||
test('a zero-Bash baseline passes only when every ON trial is also zero', () => {
|
||||
@@ -288,10 +306,13 @@ describe('complete trial lifecycle', () => {
|
||||
}, record: (entry) => records.push(entry) });
|
||||
expect(records).toHaveLength(1);
|
||||
expect(records[0]).toBe(outcome);
|
||||
expect(outcome.passed).toBe(scenario === 'success');
|
||||
// Contract v4: a failed output check is a completed wrong answer, not an invalid measurement.
|
||||
expect(outcome.passed).toBe(scenario === 'success' || scenario === 'assertion');
|
||||
expect(outcome.taskCorrect).toBe(scenario === 'success');
|
||||
if (scenario === 'assertion') expect(outcome).toMatchObject({ exitReason: 'success', answerError: 'Error: required output absent' });
|
||||
if (scenario === 'runner_error' || scenario === 'setup_error') expect(outcome.exitReason).toBe('harness_error');
|
||||
if (scenario === 'max_turns') expect(outcome.exitReason).toBe('error_max_turns');
|
||||
if (scenario !== 'success') expect(outcome.error).toBeTruthy();
|
||||
if (scenario !== 'success' && scenario !== 'assertion') expect(outcome.error).toBeTruthy();
|
||||
if (scenario !== 'runner_error' && scenario !== 'setup_error') expect(outcome.result?.costUsd).toBe(0.03);
|
||||
} finally { fs.rmSync(dir, { recursive: true, force: true }); }
|
||||
});
|
||||
@@ -311,6 +332,40 @@ describe('complete trial lifecycle', () => {
|
||||
});
|
||||
});
|
||||
|
||||
describe('contract v4 replay of census 37198445662', () => {
|
||||
const corpus = JSON.parse(fs.readFileSync(path.join(import.meta.dir, 'fixtures', 'overlay-v4-replay-37198445662.json'), 'utf8')) as {
|
||||
source: { fixture: string }; trials: Array<{ arm: 'overlay-on' | 'overlay-off'; trial: number; tools: string[]; answer: string; v3: { measurementPassed: boolean } }>;
|
||||
};
|
||||
const f = OVERLAY_FIXTURES.find(candidate => candidate.id === corpus.source.fixture)!;
|
||||
const replay = async (trial: (typeof corpus.trials)[number]) => {
|
||||
const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'overlay-v4-replay-'));
|
||||
try {
|
||||
return await runOverlayTrial({ fixture: f, directory: dir, record: () => {}, invoke: async () => result({
|
||||
events: [systemInit(), { type: 'result', subtype: 'success', result: trial.answer }] as AgentSdkResult['events'],
|
||||
toolCalls: trial.tools.map(tool => ({ tool, input: {}, output: '' })), output: trial.answer, model: f.model,
|
||||
}) });
|
||||
} finally { fs.rmSync(dir, { recursive: true, force: true }); }
|
||||
};
|
||||
test('the stored OFF wrong answer is a measured result and the case passes under v4', async () => {
|
||||
const outcomes = await Promise.all(corpus.trials.map(replay));
|
||||
const arm = (name: string) => outcomes.filter((_, index) => corpus.trials[index].arm === name);
|
||||
expect(corpus.trials.filter(trial => !trial.v3.measurementPassed).map(trial => `${trial.arm}-${trial.trial}`)).toEqual(['overlay-off-0']);
|
||||
expect(outcomes.every(outcome => outcome.passed)).toBe(true);
|
||||
expect(arm('overlay-off').map(outcome => outcome.taskCorrect)).toEqual([false, ...Array(9).fill(true)]);
|
||||
expect(arm('overlay-off')[0].answerError).toContain('does not match the fixture task');
|
||||
expect(assessOverlayArms(f, arm('overlay-on'), arm('overlay-off'))).toMatchObject({
|
||||
measurementsValid: true, correctnessPassed: true, gatePassed: true, passed: true, comparison: { status: 'baseline_saturated', meanOn: 0, meanOff: 0 },
|
||||
});
|
||||
expect(OVERLAY_CONTRACT.version).toBe(4);
|
||||
});
|
||||
test('the same wrong answer in the ON arm still fails the case', async () => {
|
||||
const swapped = corpus.trials.map(trial => ({ ...trial, arm: trial.arm === 'overlay-on' ? 'overlay-off' as const : 'overlay-on' as const }));
|
||||
const outcomes = await Promise.all(swapped.map(replay));
|
||||
const arm = (name: string) => outcomes.filter((_, index) => swapped[index].arm === name);
|
||||
expect(assessOverlayArms(f, arm('overlay-on'), arm('overlay-off'))).toMatchObject({ measurementsValid: true, correctnessPassed: false, passed: false });
|
||||
});
|
||||
});
|
||||
|
||||
describe('retry-specific native evidence', () => {
|
||||
test('retains events preceding an exception and a separate successful retry stream', async () => {
|
||||
const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'overlay-capture-unit-'));
|
||||
|
||||
Reference in new issue
Block a user