diff --git a/test/ceo-mode-handoff.test.ts b/test/ceo-mode-handoff.test.ts index 2b89e855f..4b0de176e 100644 --- a/test/ceo-mode-handoff.test.ts +++ b/test/ceo-mode-handoff.test.ts @@ -65,7 +65,9 @@ describe('native handoff evidence', () => { } }); - test('an auto-decided handoff stands in for the unsent annotation, unless later withdrawn', () => { + // The helper's Bash result is collapsed in the terminal (Claude Code 2.1.284), so it locates the + // witness but is not one: the chat must begin with the printed line (integration decision 2026-10-04). + test('an auto-decided handoff counts only once the chat begins with its line, unless later withdrawn', () => { const captured = JSON.parse(fs.readFileSync(path.join(import.meta.dir, 'fixtures/native-auto-decide-ag.json'), 'utf8')); const attempt = () => { const f = structuredClone(captured.attempts[0]); @@ -79,19 +81,24 @@ describe('native handoff evidence', () => { expect(verdict(unsent)).toBeNull(); const sessionId = unsent.options.sessionId; const after = Date.parse('2026-09-10T02:19:00.000Z'); + const helperOnly = attempt(); + helperOnly.tools.push(...handoff(sessionId, after, `${CMD} "HOLD SCOPE" --auto`, AUTO_LINE)); + expect(verdict(helperOnly)).toBeNull(); const sent = attempt(); sent.tools.push(...handoff(sessionId, after, `${CMD} "HOLD SCOPE" --auto`, AUTO_LINE)); + sent.transcript.assistantMessages.push({ sessionId, timestamp: '2026-09-10T02:19:02.000Z', text: `${AUTO_LINE}\n\nContinuing with the review.` }); expect(verdict(sent)).toMatchObject({ option: 'HOLD SCOPE', annotation: AUTO_LINE }); const asked = attempt(); asked.tools.push(...handoff(sessionId, after, `${CMD} "HOLD SCOPE"`, 'Mode: HOLD SCOPE; approved decisions: none.')); expect(verdict(asked)).toBeNull(); const withdrawn = attempt(); withdrawn.tools.push(...handoff(sessionId, after, `${CMD} "HOLD SCOPE" --auto`, AUTO_LINE)); + withdrawn.transcript.assistantMessages.push({ sessionId, timestamp: '2026-09-10T02:19:02.000Z', text: AUTO_LINE }); withdrawn.transcript.assistantMessages.push({ sessionId, timestamp: '2026-09-10T02:19:05.000Z', text: 'Correction: I withdraw this auto-decision.' }); expect(verdict(withdrawn)).toBeNull(); }); - test('with the owned decision record, the handoff is the declaration the census run never sent', () => { + test('with the owned decision record, a handoff the chat never repeated is still a miss', () => { const attempt = () => { const f = structuredClone(stateCapture) as any; const use = f.tools.find((e: any) => e.input?.command?.includes('gstack-question-log')); @@ -105,8 +112,12 @@ describe('native handoff evidence', () => { const decide = (f: any) => findNativeAutoDecision(f.transcript, f.tools, f.options); const { f: unsent } = attempt(); expect(decide(unsent)).toBeNull(); + const { f: helperOnly, loggedAt: helperAt } = attempt(); + helperOnly.tools.push(...handoff(helperOnly.options.sessionId, helperAt + 1000, `${CMD} "HOLD SCOPE" --auto`, AUTO_LINE)); + expect(decide(helperOnly)).toBeNull(); const { f: sent, loggedAt } = attempt(); sent.tools.push(...handoff(sent.options.sessionId, loggedAt + 1000, `${CMD} "HOLD SCOPE" --auto`, AUTO_LINE)); + sent.transcript.assistantMessages.push({ sessionId: sent.options.sessionId, timestamp: new Date(loggedAt + 2000).toISOString(), text: AUTO_LINE }); expect(decide(sent)).toMatchObject({ option: 'HOLD SCOPE', annotation: AUTO_LINE, stateRecord: sent.options.stateEvidence.records[0] }); const { f: early } = attempt(); const started = Date.parse(early.tools.find((e: any) => e.kind === 'result' && String(e.content).includes('SKILL_START_PROTO')).timestamp); @@ -144,7 +155,7 @@ describe('native handoff evidence', () => { } }); - test('census 37182865432: the printed handoff before the provenance log is the declaration', () => { + test('census 37182865432: the handoff ran but its line never reached the chat, so it is a miss', () => { const captured = JSON.parse(fs.readFileSync(path.join(import.meta.dir, 'fixtures/auto-decide-handoff-before-log-37182865432.json'), 'utf8')); const attempt = () => { const f = structuredClone(captured); @@ -156,21 +167,22 @@ describe('native handoff evidence', () => { return f; }; const decide = (f: any) => findNativeAutoDecision(f.transcript, f.tools, f.options); - // As captured, the closing "**Mode decided: HOLD SCOPE.**" chat also declares the mode. - expect(decide(attempt())).toMatchObject({ option: 'HOLD SCOPE' }); - const handoffOnly = () => { + // As captured: the helper printed the line in a collapsed Bash result and the chat said + // "I'm defaulting to HOLD SCOPE…" then "**Mode decided: HOLD SCOPE.**"; neither is the line. + expect(decide(attempt())).toBeNull(); + const shown = () => { const f = attempt(); - f.transcript.assistantMessages.find((m: any) => m.text.startsWith('**Mode decided:')).text = 'Wrapping up this invocation.'; + f.transcript.assistantMessages.find((m: any) => m.text.startsWith('**Mode decided:')).text = `${AUTO_LINE}\n\nWrapping up this invocation.`; return f; }; - expect(decide(handoffOnly())).toMatchObject({ option: 'HOLD SCOPE', annotation: AUTO_LINE }); + expect(decide(shown())).toMatchObject({ option: 'HOLD SCOPE', annotation: AUTO_LINE }); for (const [name, mutate] of Object.entries({ - 'handoff output altered': (f: any) => { f.tools.find((e: any) => e.kind === 'result' && e.content === AUTO_LINE).content = AUTO_LINE.replace('HOLD SCOPE', 'SCOPE EXPANSION'); }, + 'line not at the start of the chat': (f: any) => { const m = f.transcript.assistantMessages.find((x: any) => x.text.startsWith(AUTO_LINE)); m.text = `Note: ${m.text}`; }, 'record names another mode': (f: any) => { f.options.stateEvidence.records[0].user_choice = 'SCOPE EXPANSION'; f.options.stateEvidence.records[0].recommended = 'SCOPE EXPANSION'; }, 'later withdrawal': (f: any) => { f.transcript.assistantMessages.push({ sessionId: f.options.sessionId, timestamp: new Date(f.options.now - 1000).toISOString(), text: 'Correction: I withdraw this decision.' }); }, 'missing record': (f: any) => { f.options.stateEvidence.records = []; }, })) { - const f = handoffOnly(); mutate(f); expect(decide(f), name).toBeNull(); + const f = shown(); mutate(f); expect(decide(f), name).toBeNull(); } const pending = attempt(); pending.transcript.assistantMessages.find((m: any) => m.text.startsWith('**Mode decided:')).text = '**Mode pending: HOLD SCOPE.**'; diff --git a/test/detector-corpus-auto-decide-preserved.test.ts b/test/detector-corpus-auto-decide-preserved.test.ts index 86d88ae57..bf5646ba2 100644 --- a/test/detector-corpus-auto-decide-preserved.test.ts +++ b/test/detector-corpus-auto-decide-preserved.test.ts @@ -62,3 +62,18 @@ describe('auto-decide-preserved detector corpus', () => { } }); }); + +describe('auto-decide handoff needs a user-visible witness', () => { + test('37182865432: the helper result alone is refused; the same session is credited once its chat begins with the line', () => { + const entry = load('37182865432-t1-fail.json'); + const line = 'Auto-decided review mode → HOLD SCOPE (your preference). Change with /plan-tune. Approved decisions: none.'; + expect(entry.input.native.tools.some((t: { kind: string; content?: unknown }) => t.kind === 'result' && t.content === line)).toBe(true); + expect(native(entry)).toBeNull(); + const last = entry.input.native.transcript.assistantMessages.at(-1); + const original = last.text; + last.text = `${line}\n\n${original}`; + expect(native(entry)).toMatchObject({ option: 'HOLD SCOPE', annotation: line }); + last.text = `Note: ${line}\n\n${original}`; + expect(native(entry)).toBeNull(); + }); +}); diff --git a/test/fixtures/detector-corpora/auto-decide-preserved/37182865432-t1-fail.json b/test/fixtures/detector-corpora/auto-decide-preserved/37182865432-t1-fail.json index 6fd82f061..a595f1287 100644 --- a/test/fixtures/detector-corpora/auto-decide-preserved/37182865432-t1-fail.json +++ b/test/fixtures/detector-corpora/auto-decide-preserved/37182865432-t1-fail.json @@ -13,10 +13,10 @@ }, "census_outcome": "failed", "census_error": "AUTO_DECIDE preserve test inconclusive: outcome=timeout", - "expected": "pass", - "justification": "Fixed false red: the model ran gstack-ceo-mode-handoff --auto (printing the AUTO_DECIDE line) before its provenance log, as Step 0E orders; the detector then required the handoff after the log. Repaired by d89dc32.", + "expected": "refuse", + "justification": "Genuine miss (integration decision 2026-10-04, lane S over lane D): the model ran gstack-ceo-mode-handoff --auto, but Claude Code collapses that Bash result, and its chat never began with the printed \"Auto-decided review mode → …\" line (\"I'm defaulting to HOLD SCOPE…\", then \"Mode decided: HOLD SCOPE\"), so the user never saw it. The helper's tool result locates the witness; it does not replace it. Same class as 37176837432 and 37179171083; the template now asks for the line verbatim (b8ec190).", "screen_expected": "refuse", - "native_expected": "pass", + "native_expected": "refuse", "derived": "options.stateEvidence rebuilt from the session's gstack-question-log argument with source \"agent\" and ts = that call's tool-result timestamp (the bound question-log.jsonl itself is not archived)", "input": { "native": { diff --git a/test/helpers/native-auto-decide.ts b/test/helpers/native-auto-decide.ts index de9ba7c24..7e55b830f 100644 --- a/test/helpers/native-auto-decide.ts +++ b/test/helpers/native-auto-decide.ts @@ -258,6 +258,13 @@ export function ceoModeHandoffs(tools: ReadonlyArray, ses }); } +/** The helper's tool result is collapsed in the terminal, so the user sees the + * AUTO_DECIDE line only when a later chat message of the session begins with it. */ +function handoffShown(handoff: CeoModeHandoff, messages: ReadonlyArray<{ sessionId: string; timestamp: string; text: string }>): boolean { + return messages.some(m => m.sessionId === handoff.sessionId && Date.parse(m.timestamp) >= Date.parse(handoff.timestamp) && + plain(m.text.split(/\r?\n/).find(line => line.trim()) ?? '').startsWith(handoff.line)); +} + function currentModeStatement(text: string, questionSummary?: string): { option: string; statement: string } | null { const prose = publicProse(text); const lines = prose.split('\n'); @@ -318,11 +325,14 @@ function structuredModeDecision(transcript: PlanCountTranscript, tools: NativePu if (starts.length !== 1) return null; const start = starts[0]!, questionId = `${opts.skillName}-mode`; // The handoff helper's printed AUTO_DECIDE line is the current declaration - // when it follows the preference check and nothing later withdraws or changes - // it. Step 0E records provenance after the handoff, so the log may follow it. + // when it follows the preference check, the chat repeats it to the user, and + // nothing later withdraws or changes it. Step 0E records provenance after + // the handoff, so the log may follow it. A handoff the chat never showed is + // a miss: another mode statement cannot stand in for the AUTO_DECIDE line. + const autoHandoffs = (after: number, option: string) => ceoModeHandoffs(owned, opts.sessionId).filter(h => h.auto && + h.option === option && timely(h.timestamp) && time(h.timestamp) >= after); const autoHandoff = (after: number, option: string, summary: string) => { - const handoff = ceoModeHandoffs(owned, opts.sessionId).find(h => h.auto && h.option === option && - timely(h.timestamp) && time(h.timestamp) >= after); + const handoff = autoHandoffs(after, option).find(h => handoffShown(h, current)); if (!handoff) return null; const later = current.filter(m => time(m.timestamp) >= time(handoff.timestamp)); if (withdrawn(later.map(m => m.text).join('\n\n'), option, summary) || @@ -343,6 +353,7 @@ function structuredModeDecision(transcript: PlanCountTranscript, tools: NativePu loggedAt >= time(start.result.timestamp) && loggedAt <= opts.now) { const handoff = autoHandoff(time(start.result.timestamp), modeValue(row.user_choice)!, row.question_summary); if (handoff) return { ...handoff, summary: row.question_summary, preambleToolUseId: start.use.toolUseId, stateRecord: row }; + if (autoHandoffs(time(start.result.timestamp), modeValue(row.user_choice)!).length) return null; for (const message of current) { const declared = currentModeStatement(message.text, row.question_summary); if (time(message.timestamp) < loggedAt || !declared || declared.option !== modeValue(row.user_choice)) continue; @@ -394,6 +405,7 @@ function structuredModeDecision(transcript: PlanCountTranscript, tools: NativePu const handoff = autoHandoff(time(check.result!.timestamp), modeValue(logged.log.user_choice)!, logged.log.question_summary); if (handoff) return { ...handoff, summary: logged.log.question_summary, preambleToolUseId: start.use.toolUseId, preferenceToolUseId: check.use.toolUseId, questionLogToolUseId: logged.use.toolUseId }; + if (autoHandoffs(time(check.result!.timestamp), modeValue(logged.log.user_choice)!).length) return null; for (const message of current) { if (time(message.timestamp) < time(logged.result!.timestamp)) continue; const declared = currentModeStatement(message.text, logged.log.question_summary); @@ -432,7 +444,7 @@ export function findNativeAutoDecision( if (messages.some(m => !Number.isFinite(at(m.timestamp)) || at(m.timestamp) > opts.now)) return null; const loadedAt = at(results[0]!.timestamp); for (const handoff of ceoModeHandoffs(tools, opts.sessionId)) { - if (!handoff.auto || !timely(handoff.timestamp) || at(handoff.timestamp) < loadedAt) continue; + if (!handoff.auto || !timely(handoff.timestamp) || at(handoff.timestamp) < loadedAt || !handoffShown(handoff, messages)) continue; const later = messages.filter(m => at(m.timestamp) >= at(handoff.timestamp)).map(m => m.text).join('\n\n'); if (withdrawn(later, handoff.option)) continue; return { sessionId: opts.sessionId, skillToolUseId: use.toolUseId, timestamp: handoff.timestamp,