test(ship-docsync): seed fault cases at their gate instead of replaying attempt 1

The post-dispatch fault cases (missing-marker, launch-failure, timeout-unsettled,
late-result, stale-before, stale-after, recovery) now start from a fixture-owned
attempt 1: the real actor prepares and dispatches it, its verbatim output is saved
once, and the invocation journal carries its pre-dispatch entry with the child
asset hashes. The model resumes at Parent processing with a trimmed read list,
inspect named as the authoritative repository observation, and recovery's
intermediate checkpoint folded into the next attempt's pre-dispatch entry.
Assertions count only parent-issued transport events and require a read of the
saved attempt-1 output; missing-asset and the legacy failure case keep the full
model-driven first attempt, and their prompts are byte-identical.
This commit is contained in:
garrytan committed 2026-09-30 16:30:43 +00:00
1 parent dd5708e6e8
commit fb52689822
3 files changed
+112 -20

No files matched your search

+61 -10
View File
@@ -4,7 +4,7 @@ import * as os from 'node:os';
import * as path from 'node:path';
import { createHash } from 'node:crypto';
import { DOC_PATH, docsCandidate, fixtureDocs, repoSnapshot } from './helpers/docsync-fixture';
import { DOCS_CHECKPOINT_MARKER, docsActorCommand, docsActorHook, installDocsActor, type DocsActorState } from './helpers/docsync-fault-actor';
import { DOCS_CHECKPOINT_MARKER, DOCS_SEEDED_AUDIT_ID, docsActorCommand, docsActorHook, docsActorSeeded, installDocsActor, seedDocsFirstAttempt, type DocsActorState } from './helpers/docsync-fault-actor';
import { docsActorVerdict } from './helpers/docsync-fault-eval';
import { extractDocsDispatch, parseDocsCompletion } from './helpers/docsync-contract';
import { docsNativeInterface } from './helpers/docsync-observer';
@@ -211,6 +211,44 @@ test('inspect grants no repair, attempt, or missing-asset bypass and no lifecycl
} finally { fixture.clean(); }
});
test('seeded attempt 1 runs the real prepare and dispatch, saves verbatim output once and journals its pre-dispatch entry', () => {
expect((['missing-asset', 'legacy-completion'] as const).filter(docsActorSeeded)).toEqual([]);
for (const scenario of ['stale-after', 'launch-failure', 'late-result'] as const) {
expect(docsActorSeeded(scenario)).toBe(true);
const fixture = fixtureDocs('current');
try {
const stateFile = installDocsActor(fixture, scenario);
const seed = seedDocsFirstAttempt(fixture, stateFile);
const state = JSON.parse(fs.readFileSync(stateFile, 'utf8')) as DocsActorState;
expect(state.events.map(e => e.action)).toEqual(scenario === 'stale-after' ? ['prepare', 'dispatch', 'completion'] : ['prepare', 'dispatch']);
expect(seed.events).toBe(state.events.length);
expect(state.tasks).toHaveLength(1);
expect(state.tasks[0].audit_id).toBe(DOCS_SEEDED_AUDIT_ID);
expect(state.armed).toBe(scenario === 'stale-after');
expect(repoSnapshot(fixture.repo)).toEqual(fixture.before);
expect(fs.readFileSync(seed.completion, 'utf8')).toBe(seed.text);
expect(seed.exit).toBe(scenario === 'launch-failure' ? 23 : 0);
expect(JSON.parse(fs.readFileSync(seed.candidate, 'utf8'))).toEqual(state.tasks[0].observed_candidate);
const record = fs.readFileSync(fixture.invocation, 'utf8');
expect(record.split(DOCS_CHECKPOINT_MARKER)).toHaveLength(2);
const entry = record.slice(record.indexOf('### Checkpoint 1'));
expect(entry).toContain('Attempts used: 1.');
for (const value of [seed.candidate, seed.prompt, seed.completion, 'dispatch exit code ' + seed.exit, 'run_in_background=false']) expect(entry).toContain(value);
for (const asset of ['SKILL.md', 'sections/audit-scope.md', 'sections/release-body.md']) {
const file = path.join(fixture.skills, 'document-release', asset);
expect(entry).toContain(file + ' ' + createHash('sha256').update(fs.readFileSync(file)).digest('hex'));
}
expect(entry.includes('Returned child handle: fixture-child-1.')).toBe(scenario === 'late-result');
expect(entry).not.toMatch(/stale|blocked|current|accepted|repair/i);
expect(() => seedDocsFirstAttempt(fixture, stateFile)).toThrow();
} finally { fixture.clean(); }
}
const missing = fixtureDocs('current');
try {
expect(() => seedDocsFirstAttempt(missing, installDocsActor(missing, 'missing-asset'))).toThrow('seeded attempt requires installed document-release/sections/audit-scope.md');
} finally { missing.clean(); }
});
test('legacy completion is deterministic data with a real preserved partial edit, not instructions to a model', () => {
const fixture = fixtureDocs('legacy');
try {
@@ -292,6 +330,7 @@ const fixtures = { ...await import(fixtureModule) };
const observers = { ...await import(observerModule) };
const { CAPTURE_MS, CAPTURE_LONG_MS } = await import(path.join(root, 'test/helpers/eval-budgets.ts'));
const { parseDocsCompletion } = await import(path.join(root, 'test/helpers/docsync-contract.ts'));
const { docsActorSeeded, DOCS_SEEDED_AUDIT_ID } = await import(path.join(root, 'test/helpers/docsync-fault-actor.ts'));
const callbacks = new Map();
let fixture, control = '', launches = 0, recorded, legacyReaudit = false;
let returnedResult: SkillTestResult | undefined;
@@ -422,7 +461,15 @@ mock.module(path.join(root, 'test/helpers/session-runner.ts'), () => ({ async ru
expect(initialRecord.split(marker)).toHaveLength(2);
const recordPrefix = initialRecord.split(marker)[0];
let checkpointCount = 0;
let attemptsUsed = 0;
const seeded = docsActorSeeded(scenario);
let attemptsUsed = seeded ? 1 : 0;
let seedOutput = '';
if (seeded) {
expect(initialRecord).toContain('Attempts used: 1');
read(path.join(fixture.home, 'candidate-' + DOCS_SEEDED_AUDIT_ID + '.json'));
const saved = path.join(fixture.home, 'completion-' + DOCS_SEEDED_AUDIT_ID + '.md');
seedOutput = control === 'skip-seeded-output' ? fs.readFileSync(saved, 'utf8') : read(saved);
}
const checkpoint = (details, finished = false) => {
const before = fs.readFileSync(fixture.invocation, 'utf8');
expect(before.split(marker)).toHaveLength(2);
@@ -468,16 +515,18 @@ mock.module(path.join(root, 'test/helpers/session-runner.ts'), () => ({ async ru
expect(saved.input.new_string).toContain(prepared.prompt);
return result;
};
let output = '', accepted = null;
let output = '', accepted = null, first;
if (scenario !== 'missing-asset') {
const first = prepare('ship-docs-20260926-a1');
const initial = dispatch(first);
first = seeded ? { audit_id: DOCS_SEEDED_AUDIT_ID, candidate: path.join(fixture.home, 'candidate-' + DOCS_SEEDED_AUDIT_ID + '.json'),
prompt: path.join(fixture.home, 'prompt-' + DOCS_SEEDED_AUDIT_ID + '.md') } : prepare('ship-docs-20260926-a1');
const initial = seeded ? { text: seedOutput, output: seedOutput } : dispatch(first);
output = initial.text;
if (scenario === 'missing-marker') {
expect(initial.output).toBe('SESSION_KIND: interactive\\n{"schema_version":1,"audit_id":"ship-docs-20260926-a1","status":"blocked","files_updated":[],"files_reviewed":[],"documentation_section":"blocked — fixture child audit ship-docs-20260926-a1; Missing spawned marker.","blockers":["Missing spawned marker"],"decisions":[]}');
expect(initial.output).toBe('SESSION_KIND: interactive\\n{"schema_version":1,"audit_id":"' + first.audit_id + '","status":"blocked","files_updated":[],"files_reviewed":[],"documentation_section":"blocked — fixture child audit ' + first.audit_id + '; Missing spawned marker.","blockers":["Missing spawned marker"],"decisions":[]}');
}
if (scenario === 'launch-failure') {
expect(initial.output).toBe('Exit code 23\\nChild launch failed: injected unavailable worker. No child was started.');
expect(initial.text).toBe('Child launch failed: injected unavailable worker. No child was started.');
expect(seeded ? initialRecord.includes('dispatch exit code 23') : initial.output.startsWith('Exit code 23\\n')).toBe(true);
const attempts = JSON.parse(fs.readFileSync(stateFile, 'utf8')).tasks;
expect(attempts).toHaveLength(1);
expect(attempts[0].id).toBeNull();
@@ -486,7 +535,8 @@ mock.module(path.join(root, 'test/helpers/session-runner.ts'), () => ({ async ru
if (['timeout-unsettled', 'late-result'].includes(scenario)) {
expect(initial.output).toBe('{"task_id":"fixture-child-1","status":"running","elapsed_ms":0,"virtual_clock":true}');
const task_id = JSON.parse(output).task_id;
checkpoint('Running child handle: ' + task_id);
if (seeded) expect(initialRecord).toContain('Returned child handle: ' + task_id + '.');
else checkpoint('Running child handle: ' + task_id);
expect(invoke('status', { task_id }).output).toBe('{"task_id":"fixture-child-1","status":"running","settled":false,"elapsed_ms":600001,"virtual_clock":true}');
const stop = invoke('stop', { task_id });
expect(JSON.parse(stop.text).settled).toBe(scenario === 'late-result');
@@ -570,7 +620,7 @@ mock.module(path.join(root, 'test/helpers/session-runner.ts'), () => ({ async ru
if (control !== 'missing-report') write(report, finalReport);
expect(fs.readFileSync(fixture.invocation, 'utf8')).toContain(finalReport);
expect(fs.readFileSync(fixture.invocation, 'utf8')).toContain('Attempts used: ' + attemptsUsed);
if (attemptsUsed > 1) expect(fs.readFileSync(fixture.invocation, 'utf8')).toContain('ship-docs-20260926-a1');
if (attemptsUsed > 1) expect(fs.readFileSync(fixture.invocation, 'utf8')).toContain(first.audit_id);
if (control === '') {
expect(calls.slice(-2).map(call => [call.tool, call.input.file_path])).toEqual([
['Edit', fixture.invocation], ['Write', report],
@@ -623,6 +673,7 @@ for (const [name, mutation] of [
['ship-docsync-launch-failure', 'invented-handle'], ['ship-docsync-timeout-unsettled', 'skip-post-stop-status'],
['ship-docsync-late-result', 'missing-report'],
['ship-docsync-stale-before', 'stale-candidate'],
['ship-docsync-recovery', 'skip-seeded-output'],
['ship-docsync-failure', 'legacy-unchanged'], ['ship-docsync-failure', 'legacy-unsettled'],
['ship-docsync-failure', 'legacy-stale'], ['ship-docsync-failure', 'legacy-third'],
['ship-docsync-failure', 'legacy-fake-repair'],
@@ -649,7 +700,7 @@ for (const [name, mutation] of [
});
const output = result.stdout.toString() + result.stderr.toString();
expect(result.exitCode, output).toBe(0);
expect(output).toContain('35 pass');
expect(output).toContain('36 pass');
expect(output).toContain('0 fail');
} finally { fs.rmSync(dir, { recursive: true, force: true }); }
}, 120_000);