diff --git a/TODOS.md b/TODOS.md index 804ba365e..39faf5109 100644 --- a/TODOS.md +++ b/TODOS.md @@ -30,6 +30,14 @@ Deferred from the approved reliability follow-ups plan (docs/designs/RELIABILITY diagnostic run on the pre-verbatim wording paraphrased it; one on the verbatim wording passed. Watch the next censuses; a red here is a template miss, not a detector false red. **Effort:** S. **Priority:** P2. +- **Named red: ship-docsync-late-result on PR run 37237194905** — the model + wrote its report to a mistyped shard root (`…-3GanyY` for `…-3GanyU`) at + call 20, disclosed it, then wrote the real report at the right path. The + authority detector is correct (the file was created outside the fixture); + `test/docsync-authority.test.ts` replays the capture. If any docsync case + writes to a near-miss shard path again, give the model a short stable + fixture root (env var or fixed short symlink) instead of random tokens to + copy, without loosening the authority check. **Effort:** S. **Priority:** P2. - **PTY sessions are cost-unknown** — `test/helpers/pricing.ts` excludes cache-write charges, so PTY trials stay `cost_known: false`. Price them from native transcript usage once pricing covers cache writes. **Effort:** S. **Priority:** P3. diff --git a/test/docsync-authority.test.ts b/test/docsync-authority.test.ts index 9991ca091..79cc2dc5c 100644 --- a/test/docsync-authority.test.ts +++ b/test/docsync-authority.test.ts @@ -105,3 +105,22 @@ test('completed docs review accepts literal cat but not an unrelated command or expect(docsCompletedRead(calls({ tool: 'Read', input: { file_path: doc, limit: 1 }, output: original.slice(0, 20) }), doc, fixture)).toBe(false); } finally { fixture.clean(); } }); + +test('replay of run 37237194905 late-result: only the write to a mistyped shard root is outside authority', () => { + const fixture = fixtureDocs('current'); + try { + const actor = path.join(import.meta.dir, 'helpers', 'docsync-fault-actor.ts'); + const typoHome = path.join(path.dirname(fixture.home), `typo-${path.basename(fixture.home)}`); + const replay = JSON.parse(fs.readFileSync(path.join(import.meta.dir, 'fixtures/docsync-replay/37237194905-late-result-calls.json'), 'utf8')); + const toolCalls = replay.calls.map((call: { tool: string; input: Record }) => ({ tool: call.tool, output: '', + input: Object.fromEntries(Object.entries(call.input).map(([key, value]) => [key, + value.replaceAll('', typoHome).replaceAll('', fixture.home).replaceAll('', actor)])) })); + expect(toolCalls).toHaveLength(21); + expect(docsToolFailures(calls(...toolCalls), fixture, [actor])).toEqual(['write outside docs fixture authority']); + expect(toolCalls[19].input.file_path.startsWith(typoHome)).toBe(true); + expect(docsToolFailures(calls(...toolCalls.filter((_: unknown, index: number) => index !== 19)), fixture, [actor])).toEqual([]); + const snapshot = [path.join(fixture.skills, 'bin/gstack-docs-candidate'), 'snapshot', '--out', path.join(fixture.home, 'ship-docs-a2-post.json'), + '--audit-id', 'ship-docs-a2', '--mode', 'edit', '--base', 'c7e8a106ea1279ee9efc96c79dad732efbc745aa', '--docs', 'handbook'].join(' '); + expect(docsToolFailures(calls({ tool: 'Bash', input: { command: snapshot }, output: '' }), fixture, [actor])).toEqual([]); + } finally { fixture.clean(); } +}); diff --git a/test/fixtures/docsync-replay/37237194905-late-result-calls.json b/test/fixtures/docsync-replay/37237194905-late-result-calls.json new file mode 100644 index 000000000..1cddde214 --- /dev/null +++ b/test/fixtures/docsync-replay/37237194905-late-result-calls.json @@ -0,0 +1,136 @@ +{ + "source": { + "run": "37237194905", + "case": "ship-docsync-late-result", + "sha": "c245d1bc228a38f004927a8ef5e63f1b6320c761" + }, + "note": "Tool calls of the failed trial, reduced to the fields the authority detector reads. Call 20 wrote to a mistyped shard root (3GanyY for 3GanyU); the agent disclosed it and rewrote the report at the right path.", + "calls": [ + { + "tool": "Read", + "input": { + "file_path": "/phase.md" + } + }, + { + "tool": "Read", + "input": { + "file_path": "/.claude/skills/gstack/ship/sections/documentation.md" + } + }, + { + "tool": "Read", + "input": { + "file_path": "/ship-invocation.md" + } + }, + { + "tool": "Read", + "input": { + "file_path": "/candidate-ship-docs-a1.json" + } + }, + { + "tool": "Read", + "input": { + "file_path": "/completion-ship-docs-a1.md" + } + }, + { + "tool": "Bash", + "input": { + "command": "bun status /actor-state.json task_id=fixture-child-1" + } + }, + { + "tool": "Bash", + "input": { + "command": "bun stop /actor-state.json task_id=fixture-child-1" + } + }, + { + "tool": "Bash", + "input": { + "command": "bun status /actor-state.json task_id=fixture-child-1" + } + }, + { + "tool": "Bash", + "input": { + "command": "bun inspect /actor-state.json" + } + }, + { + "tool": "Bash", + "input": { + "command": "bun repair /actor-state.json" + } + }, + { + "tool": "Bash", + "input": { + "command": "sha256sum /.claude/skills/gstack/document-release/SKILL.md /.claude/skills/gstack/document-release/sections/audit-scope.md /.claude/skills/gstack/document-release/sections/release-body.md" + } + }, + { + "tool": "Bash", + "input": { + "command": "bun prepare /actor-state.json audit_id=ship-docs-a2" + } + }, + { + "tool": "Read", + "input": { + "file_path": "/candidate-ship-docs-a2.json" + } + }, + { + "tool": "Read", + "input": { + "file_path": "/prompt-ship-docs-a2.md" + } + }, + { + "tool": "Edit", + "input": { + "file_path": "/ship-invocation.md" + } + }, + { + "tool": "Bash", + "input": { + "command": "bun dispatch /actor-state.json audit_id=ship-docs-a2 candidate=/candidate-ship-docs-a2.json prompt=/prompt-ship-docs-a2.md run_in_background=false" + } + }, + { + "tool": "Write", + "input": { + "file_path": "/completion-ship-docs-a2.md" + } + }, + { + "tool": "Bash", + "input": { + "command": "bun inspect /actor-state.json" + } + }, + { + "tool": "Edit", + "input": { + "file_path": "/ship-invocation.md" + } + }, + { + "tool": "Write", + "input": { + "file_path": "/ship-report.md" + } + }, + { + "tool": "Write", + "input": { + "file_path": "/ship-report.md" + } + } + ] +} \ No newline at end of file