diff --git a/changelog.d/auto-decide-check-reads-load-time-start.md b/changelog.d/auto-decide-check-reads-load-time-start.md new file mode 100644 index 000000000..c16083221 --- /dev/null +++ b/changelog.d/auto-decide-check-reads-load-time-start.md @@ -0,0 +1 @@ +- **For contributors: the paid auto-decide check counts a start step that ran while the skill loaded.** Since Claude Code skills started running their start step as they load, a /plan-ceo-review run that auto-decided its review mode left no start command in its tool calls, so the check that reads the session journal could never credit it. In the captured runs, one of the five passes was credited that way. The check now also reads the skill text Claude Code loaded and accepts the start output it finds there. It still requires exactly one start in the session, and refuses a failed or disabled load or a second start through Bash. The plan-mode and auto-decide paid cases now also run when the load-time helper changes. diff --git a/lib/claude-public-transcript.ts b/lib/claude-public-transcript.ts index e01aec458..fded7514b 100644 --- a/lib/claude-public-transcript.ts +++ b/lib/claude-public-transcript.ts @@ -34,6 +34,8 @@ export interface PlanCountTranscript { assistantMessages: Array<{ sessionId: string; text: string; timestamp: string }>; /** Actual native plan-mode approval requests; pending is the UI gate, never an AUQ. */ planReadyRequests?: Array<{ sessionId: string; toolUseId: string; timestamp: string; failed: boolean; source?: 'pre_tool_use' }>; + /** Skill text Claude Code loaded for a slash command or the Skill tool, after its load-time commands ran. */ + skillLoads?: Array<{ sessionId: string; skill: string; text: string; timestamp: string }>; error?: string; /** Owned reads only: why the exact parent journal supplied no owned lines. */ reason?: OwnedTranscriptReason; @@ -80,6 +82,7 @@ export function readPlanCountTranscript(configDir: string, cwd: string, const calls = new Map(); const assistantMessages: PlanCountTranscript['assistantMessages'] = []; const planReadyRequests = new Map[number]>(); + const skillLoads: NonNullable = []; let matched = false; let bytes = 0; let files = 0; @@ -164,6 +167,9 @@ export function readPlanCountTranscript(configDir: string, cwd: string, } for (const block of record.message.content) { if (!object(block)) continue; + const loadedFrom = record.message.role === 'user' && record.isMeta === true && block.type === 'text' && + typeof block.text === 'string' && validTimestamp(record.timestamp) ? /^Base directory for this skill: ([^\n]+)\n/.exec(block.text)?.[1] : undefined; + if (loadedFrom) skillLoads.push({ sessionId: record.sessionId, skill: path.basename(loadedFrom.trim()), text: block.text, timestamp: record.timestamp }); if (record.message.role === 'assistant' && block.type === 'tool_use' && block.name === 'ExitPlanMode' && typeof block.id === 'string' && validTimestamp(record.timestamp)) { const key = `${record.sessionId}:${block.id}`; @@ -208,7 +214,8 @@ export function readPlanCountTranscript(configDir: string, cwd: string, } } return { status: matched ? 'ready' : 'missing', calls: [...calls.values()], assistantMessages, - ...(planReadyRequests.size ? { planReadyRequests: [...planReadyRequests.values()] } : {}) }; + ...(planReadyRequests.size ? { planReadyRequests: [...planReadyRequests.values()] } : {}), + ...(skillLoads.length ? { skillLoads } : {}) }; } catch (error) { // A failed read cannot silently turn an incomplete transcript into a // complete review. Keep the diagnostic explicit and return no coverage. diff --git a/test/detector-corpus-auto-decide-preserved.test.ts b/test/detector-corpus-auto-decide-preserved.test.ts index bf5646ba2..7ec117f34 100644 --- a/test/detector-corpus-auto-decide-preserved.test.ts +++ b/test/detector-corpus-auto-decide-preserved.test.ts @@ -77,3 +77,56 @@ describe('auto-decide handoff needs a user-visible witness', () => { expect(native(entry)).toBeNull(); }); }); + +describe('a start that ran while the skill loaded (GST-3157)', () => { + const START_LINE = '!`~/.claude/skills/gstack/bin/gstack-skill-load start --skill "plan-ceo-review" --model "claude"`'; + const rendered = fs.readFileSync(path.join(import.meta.dir, '../plan-ceo-review/SKILL.md'), 'utf8'); + const moved = () => { + const entry = load('37195203538-t1-pass.json'); + const tools: any[] = entry.input.native.tools; + const use = tools.find(t => t.kind === 'use' && String(t.input?.command ?? '').includes('gstack-skill-start --skill')); + const result = tools.find(t => t.kind === 'result' && t.toolUseId === use.toolUseId); + entry.input.native.tools = tools.filter(t => t !== use && t !== result); + const text = (output: string, skill = 'plan-ceo-review') => + `Base directory for this skill: /fixture/.claude/skills/${skill}\n\n${rendered.replace(START_LINE, () => output)}`; + const skillLoad = (output = result.content as string, extra: object = {}) => + ({ sessionId: entry.input.native.options.sessionId, skill: 'plan-ceo-review', text: text(output), timestamp: use.timestamp, ...extra }); + return { entry, use, result, text, skillLoad, loads: (...loads: object[]) => { entry.input.native.transcript.skillLoads = loads; } }; + }; + + test('the rendered skill still carries the injected start line this replay replaces', () => { + expect(rendered.split(START_LINE)).toHaveLength(2); + }); + + test('37195203538 with its start moved to load is credited from the loaded skill text alone', () => { + const m = moved(); + expect(native(m.entry)).toBeNull(); + m.loads(m.skillLoad()); + const decision = native(m.entry); + expect(decision).toMatchObject({ option: 'HOLD SCOPE', sessionId: m.entry.input.native.options.sessionId }); + expect(decision!.preambleToolUseId).toBeUndefined(); + }); + + for (const [name, mutate] of Object.entries({ + 'a second start through Bash': (m: ReturnType) => { m.loads(m.skillLoad()); m.entry.input.native.tools.push(m.use, m.result); }, + 'two loads': (m: ReturnType) => m.loads(m.skillLoad(), m.skillLoad()), + 'another skill': (m: ReturnType) => m.loads(m.skillLoad(undefined, { skill: 'plan-eng-review' })), + 'another session': (m: ReturnType) => m.loads(m.skillLoad(undefined, { sessionId: 'foreign' })), + 'a load before the command': (m: ReturnType) => m.loads(m.skillLoad(undefined, { timestamp: '2026-10-04T10:27:00.000Z' })), + 'question tuning off': (m: ReturnType) => m.loads(m.skillLoad((m.result.content as string).replace('QUESTION_TUNING: true', 'QUESTION_TUNING: false'))), + 'a start that failed at load': (m: ReturnType) => m.loads(m.skillLoad('SKILL_START: failed at load (exit 1); run the start command below yourself with Bash, then continue.')), + 'shell execution disabled by policy': (m: ReturnType) => m.loads(m.skillLoad('[shell command execution disabled by policy]')), + 'an un-run injection line': (m: ReturnType) => m.loads(m.skillLoad(START_LINE)), + })) test(`a load-time start is refused with ${name}`, () => { + const m = moved(); + mutate(m); + expect(native(m.entry)).toBeNull(); + }); + + test('a start that failed at load and then ran through Bash is the one start', () => { + const m = moved(); + m.loads(m.skillLoad('SKILL_START: failed at load (exit 1); run the start command below yourself with Bash, then continue.')); + m.entry.input.native.tools.push(m.use, m.result); + expect(native(m.entry)).toMatchObject({ option: 'HOLD SCOPE', preambleToolUseId: m.use.toolUseId }); + }); +}); diff --git a/test/helpers/native-auto-decide.ts b/test/helpers/native-auto-decide.ts index 35e0e17c5..39293f108 100644 --- a/test/helpers/native-auto-decide.ts +++ b/test/helpers/native-auto-decide.ts @@ -284,6 +284,14 @@ function currentModeStatement(text: string, questionSummary?: string): { option: return null; } +/** The output Claude Code put in place of the injected start line, or null when the text has no single such slot. */ +function loadedStartOutput(text: string): string | null { + const head = '\n## Preamble (run first)\n\n', tail = '\n\nIf the line above is not the start output, run this once; otherwise it already ran:'; + const from = text.indexOf(head), to = text.indexOf(tail); + if (from < 0 || to < from + head.length || text.indexOf(head, from + 1) >= 0 || text.indexOf(tail, to + 1) >= 0) return null; + return text.slice(from + head.length, to); +} + /** A native slash expansion need not produce a Skill tool call. Its completed * preamble → preference check → decision log → current public mode provides * the alternative evidence, without attributing a synthetic Skill event. */ @@ -308,7 +316,12 @@ function structuredModeDecision(transcript: PlanCountTranscript, tools: NativePu return result && result.isError === false && timely(result.timestamp) && time(result.timestamp) >= time(use.timestamp) ? result : null; }; const bash = owned.filter(e => e.kind === 'use' && e.name === 'Bash' && timely(e.timestamp) && typeof e.input?.command === 'string'); - const starts = bash.flatMap(use => { + const startSession = (output: string) => { + const status = output.split('GSTACK_INSTRUCTION_BEGIN:')[0]!; + const sessions = [...status.matchAll(/^SESSION_ID: ([A-Za-z0-9-]+)$/gm)]; + return sessions.length === 1 && /^SKILL_START_PROTO: 1$/m.test(status) && /^QUESTION_TUNING: true$/m.test(status) ? sessions[0]![1]! : null; + }; + const shellStarts = bash.flatMap(use => { const args = preambleArgs(use.input!.command as string), result = successful(use); if (!args || !result || args.length % 2 || args.length < 2) return []; const pairs = new Map(); @@ -317,11 +330,17 @@ function structuredModeDecision(transcript: PlanCountTranscript, tools: NativePu pairs.set(args[i]!, args[i + 1]!); } if (pairs.get('--skill') !== opts.skillName || typeof result.content !== 'string') return []; - const status = result.content.split('GSTACK_INSTRUCTION_BEGIN:')[0]!; - const sessions = [...status.matchAll(/^SESSION_ID: ([A-Za-z0-9-]+)$/gm)]; - if (sessions.length !== 1 || !/^SKILL_START_PROTO: 1$/m.test(status) || !/^QUESTION_TUNING: true$/m.test(status)) return []; - return [{ use, result, session: sessions[0]![1]! }]; + const session = startSession(result.content); + return session ? [{ toolUseId: use.toolUseId, at: result.timestamp, session }] : []; }); + // Claude Code runs the start while the skill loads and puts its output in the + // loaded text (GST-3157); that load counts as the session's one start. + const loadStarts = (transcript.skillLoads ?? []).flatMap(load => { + if (load.sessionId !== opts.sessionId || load.skill !== opts.skillName || !timely(load.timestamp)) return []; + const output = loadedStartOutput(load.text), session = output === null ? null : startSession(output); + return session ? [{ toolUseId: undefined, at: load.timestamp, session }] : []; + }); + const starts = [...shellStarts, ...loadStarts]; if (starts.length !== 1) return null; const start = starts[0]!, questionId = `${opts.skillName}-mode`; // The handoff helper's printed AUTO_DECIDE line is the current declaration @@ -350,17 +369,17 @@ function structuredModeDecision(transcript: PlanCountTranscript, tools: NativePu if (row.skill === opts.skillName && row.session_id === start.session && row.source === 'agent' && row.auto_decided === true && modeValue(row.user_choice) && modeValue(row.user_choice) === modeValue(row.recommended) && typeof row.question_summary === 'string' && row.question_summary.trim() && - loggedAt >= time(start.result.timestamp) && loggedAt <= opts.now) { - const handoff = autoHandoff(time(start.result.timestamp), modeValue(row.user_choice)!, row.question_summary); - if (handoff) return { ...handoff, summary: row.question_summary, preambleToolUseId: start.use.toolUseId, stateRecord: row }; - if (autoHandoffs(time(start.result.timestamp), modeValue(row.user_choice)!).length) return null; + loggedAt >= time(start.at) && loggedAt <= opts.now) { + const handoff = autoHandoff(time(start.at), modeValue(row.user_choice)!, row.question_summary); + if (handoff) return { ...handoff, summary: row.question_summary, preambleToolUseId: start.toolUseId, stateRecord: row }; + if (autoHandoffs(time(start.at), modeValue(row.user_choice)!).length) return null; for (const message of current) { const declared = currentModeStatement(message.text, row.question_summary); if (time(message.timestamp) < loggedAt || !declared || declared.option !== modeValue(row.user_choice)) continue; const after = current.filter(m => time(m.timestamp) >= time(message.timestamp)).map(m => m.text).join('\n\n'); if (withdrawn(after, declared.option, row.question_summary)) continue; return { sessionId: opts.sessionId, timestamp: message.timestamp, summary: row.question_summary, - option: declared.option, annotation: message.text, preambleToolUseId: start.use.toolUseId, stateRecord: row }; + option: declared.option, annotation: message.text, preambleToolUseId: start.toolUseId, stateRecord: row }; } } } @@ -368,7 +387,7 @@ function structuredModeDecision(transcript: PlanCountTranscript, tools: NativePu } const checks = bash.flatMap(use => { let command = (use.input!.command as string).trim(); - const result = successful(use); if (time(use.timestamp) < time(start.result.timestamp)) return []; + const result = successful(use); if (time(use.timestamp) < time(start.at)) return []; const status = '; echo "EXIT: $?"', hasStatus = command.endsWith(status); if (hasStatus) command = command.slice(0, -status.length); const pipe = /^printf\s+(?:'%s'|"%s")\s+(?:'[^']*'|"[^"$`\\]*")\s*\|\s*/.exec(command); @@ -406,7 +425,7 @@ function structuredModeDecision(transcript: PlanCountTranscript, tools: NativePu if (logs.length !== 1 || !logs[0]!.result) return null; const logged = logs[0]!; const handoff = autoHandoff(time(check.result!.timestamp), modeValue(logged.log.user_choice)!, logged.log.question_summary); - if (handoff) return { ...handoff, summary: logged.log.question_summary, preambleToolUseId: start.use.toolUseId, + if (handoff) return { ...handoff, summary: logged.log.question_summary, preambleToolUseId: start.toolUseId, preferenceToolUseId: check.use.toolUseId, questionLogToolUseId: logged.use.toolUseId }; if (autoHandoffs(time(check.result!.timestamp), modeValue(logged.log.user_choice)!).length) return null; for (const message of current) { @@ -417,7 +436,7 @@ function structuredModeDecision(transcript: PlanCountTranscript, tools: NativePu if (withdrawn(after, declared.option, logged.log.question_summary) || current.some(m => time(m.timestamp) >= time(message.timestamp) && currentModeStatement(m.text, logged.log.question_summary)?.option !== undefined && currentModeStatement(m.text, logged.log.question_summary)!.option !== declared.option)) continue; return { sessionId: opts.sessionId, timestamp: message.timestamp, summary: logged.log.question_summary, - option: declared.option, annotation: message.text, preambleToolUseId: start.use.toolUseId, + option: declared.option, annotation: message.text, preambleToolUseId: start.toolUseId, preferenceToolUseId: check.use.toolUseId, questionLogToolUseId: logged.use.toolUseId }; } return null; diff --git a/test/helpers/touchfiles-data.ts b/test/helpers/touchfiles-data.ts index efee9c93d..f11850fd6 100644 --- a/test/helpers/touchfiles-data.ts +++ b/test/helpers/touchfiles-data.ts @@ -205,7 +205,7 @@ export const E2E_TOUCHFILES: Record = { // include question-tuning.ts and generate-ask-user-format.ts because the // AUTO_DECIDE preamble injection lives there and changes can flip the // regression test outcome between 'asked' and 'auto_decided'. - 'plan-ceo-review-plan-mode': [ 'ETHOS.md','bin/gstack-state-root.sh', 'lib/state-root.ts', 'test/helpers/plan-count-fixture.ts', + 'plan-ceo-review-plan-mode': [ 'ETHOS.md', 'bin/gstack-skill-load', 'scripts/resolvers/load-injection.ts','bin/gstack-state-root.sh', 'lib/state-root.ts', 'test/helpers/plan-count-fixture.ts', 'test/fixtures/auto-decide-recommendation-361c.json', @@ -313,7 +313,7 @@ export const E2E_TOUCHFILES: Record = { // written a never-ask preference, AUQ should still auto-decide rather than // surfacing the question. Touches the question-tuning + preference // infrastructure plus the resolvers that own the AUTO_DECIDE preamble. - 'auto-decide-preserved': [ 'ETHOS.md','bin/gstack-ceo-mode-handoff', 'test/fixtures/auto-decide-handoff-line-e354.json', 'test/fixtures/auto-decide-handoff-before-log-37182865432.json', 'bin/gstack-state-root.sh', 'lib/state-root.ts', 'lib/claude-public-transcript.ts', 'lib/claude-journal-records.ts', 'lib/claude-owned-journal.ts', + 'auto-decide-preserved': [ 'ETHOS.md', 'bin/gstack-skill-load', 'scripts/resolvers/load-injection.ts','bin/gstack-ceo-mode-handoff', 'test/fixtures/auto-decide-handoff-line-e354.json', 'test/fixtures/auto-decide-handoff-before-log-37182865432.json', 'bin/gstack-state-root.sh', 'lib/state-root.ts', 'lib/claude-public-transcript.ts', 'lib/claude-journal-records.ts', 'lib/claude-owned-journal.ts', 'test/fixtures/auto-decide-recommendation-361c.json', diff --git a/test/plan-count-transcript.test.ts b/test/plan-count-transcript.test.ts index 0a9996a63..d6b61f0de 100644 --- a/test/plan-count-transcript.test.ts +++ b/test/plan-count-transcript.test.ts @@ -407,3 +407,28 @@ describe('native journal roots: SessionStart preamble, resumed genesis and reaso }); }); }); + +describe('skill text Claude Code loaded', () => { + test('keeps only scoped, timestamped skill expansions with their load-time output', () => { + const f = fixture(); + const text = 'Base directory for this skill: /fixture/.claude/skills/plan-ceo-review\n\n## Preamble (run first)\n\nSKILL_START_PROTO: 1\n'; + const other = (name: string) => text.replace('plan-ceo-review', name); + f.append( + f.record('user', [{ type: 'text', text: 'plan-ceo-review' }]), + f.record('user', [{ type: 'text', text }], { isMeta: true }), + f.record('user', [{ type: 'text', text: other('typed-by-user') }]), + f.record('user', [{ type: 'text', text: other('sidechain') }], { isMeta: true, isSidechain: true }), + f.record('user', [{ type: 'text', text: other('untimed') }], { isMeta: true, timestamp: 'later' }), + f.record('user', [{ type: 'text', text: other('foreign') }], { isMeta: true, cwd: '/elsewhere' }), + f.record('user', [{ type: 'text', text: `Quoted: ${text}` }], { isMeta: true }), + f.record('assistant', [{ type: 'text', text: other('assistant') }], { isMeta: true }), + ); + expect(f.read().skillLoads).toEqual([{ sessionId: 'session-a', skill: 'plan-ceo-review', text, timestamp: '2026-09-08T15:31:13.607Z' }]); + }); + + test('a transcript with no skill expansion reports none', () => { + const f = fixture(); + f.append(f.ask('one')); + expect(f.read().skillLoads).toBeUndefined(); + }); +});