mirror of
https://github.com/garrytan/gstack.git
synced 2026-11-22 08:01:32 +01:00
The paid auto-decide check counts a start step that ran while the skill loaded (#4543)
Since #4509, Claude Code skills run their start step as they load, so the paid /plan-ceo-review auto-decide check could miss a run that really did auto-decide. It now accepts the start output in the loaded skill text. Why: one of the five passing runs captured for that check was credited only through the session journal, and that path needed a start command in the tool calls. Nothing to do: this changes test code and the journal reader the tests use; no skill text changes. Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>
This commit is contained in:
1 parent
7f9c1bc52a
commit
db2a56e679
6 files changed
+121
-16
No files matched your search
@@ -77,3 +77,56 @@ describe('auto-decide handoff needs a user-visible witness', () => {
|
||||
expect(native(entry)).toBeNull();
|
||||
});
|
||||
});
|
||||
|
||||
describe('a start that ran while the skill loaded (GST-3157)', () => {
|
||||
const START_LINE = '!`~/.claude/skills/gstack/bin/gstack-skill-load start --skill "plan-ceo-review" --model "claude"`';
|
||||
const rendered = fs.readFileSync(path.join(import.meta.dir, '../plan-ceo-review/SKILL.md'), 'utf8');
|
||||
const moved = () => {
|
||||
const entry = load('37195203538-t1-pass.json');
|
||||
const tools: any[] = entry.input.native.tools;
|
||||
const use = tools.find(t => t.kind === 'use' && String(t.input?.command ?? '').includes('gstack-skill-start --skill'));
|
||||
const result = tools.find(t => t.kind === 'result' && t.toolUseId === use.toolUseId);
|
||||
entry.input.native.tools = tools.filter(t => t !== use && t !== result);
|
||||
const text = (output: string, skill = 'plan-ceo-review') =>
|
||||
`Base directory for this skill: /fixture/.claude/skills/${skill}\n\n${rendered.replace(START_LINE, () => output)}`;
|
||||
const skillLoad = (output = result.content as string, extra: object = {}) =>
|
||||
({ sessionId: entry.input.native.options.sessionId, skill: 'plan-ceo-review', text: text(output), timestamp: use.timestamp, ...extra });
|
||||
return { entry, use, result, text, skillLoad, loads: (...loads: object[]) => { entry.input.native.transcript.skillLoads = loads; } };
|
||||
};
|
||||
|
||||
test('the rendered skill still carries the injected start line this replay replaces', () => {
|
||||
expect(rendered.split(START_LINE)).toHaveLength(2);
|
||||
});
|
||||
|
||||
test('37195203538 with its start moved to load is credited from the loaded skill text alone', () => {
|
||||
const m = moved();
|
||||
expect(native(m.entry)).toBeNull();
|
||||
m.loads(m.skillLoad());
|
||||
const decision = native(m.entry);
|
||||
expect(decision).toMatchObject({ option: 'HOLD SCOPE', sessionId: m.entry.input.native.options.sessionId });
|
||||
expect(decision!.preambleToolUseId).toBeUndefined();
|
||||
});
|
||||
|
||||
for (const [name, mutate] of Object.entries({
|
||||
'a second start through Bash': (m: ReturnType<typeof moved>) => { m.loads(m.skillLoad()); m.entry.input.native.tools.push(m.use, m.result); },
|
||||
'two loads': (m: ReturnType<typeof moved>) => m.loads(m.skillLoad(), m.skillLoad()),
|
||||
'another skill': (m: ReturnType<typeof moved>) => m.loads(m.skillLoad(undefined, { skill: 'plan-eng-review' })),
|
||||
'another session': (m: ReturnType<typeof moved>) => m.loads(m.skillLoad(undefined, { sessionId: 'foreign' })),
|
||||
'a load before the command': (m: ReturnType<typeof moved>) => m.loads(m.skillLoad(undefined, { timestamp: '2026-10-04T10:27:00.000Z' })),
|
||||
'question tuning off': (m: ReturnType<typeof moved>) => m.loads(m.skillLoad((m.result.content as string).replace('QUESTION_TUNING: true', 'QUESTION_TUNING: false'))),
|
||||
'a start that failed at load': (m: ReturnType<typeof moved>) => m.loads(m.skillLoad('SKILL_START: failed at load (exit 1); run the start command below yourself with Bash, then continue.')),
|
||||
'shell execution disabled by policy': (m: ReturnType<typeof moved>) => m.loads(m.skillLoad('[shell command execution disabled by policy]')),
|
||||
'an un-run injection line': (m: ReturnType<typeof moved>) => m.loads(m.skillLoad(START_LINE)),
|
||||
})) test(`a load-time start is refused with ${name}`, () => {
|
||||
const m = moved();
|
||||
mutate(m);
|
||||
expect(native(m.entry)).toBeNull();
|
||||
});
|
||||
|
||||
test('a start that failed at load and then ran through Bash is the one start', () => {
|
||||
const m = moved();
|
||||
m.loads(m.skillLoad('SKILL_START: failed at load (exit 1); run the start command below yourself with Bash, then continue.'));
|
||||
m.entry.input.native.tools.push(m.use, m.result);
|
||||
expect(native(m.entry)).toMatchObject({ option: 'HOLD SCOPE', preambleToolUseId: m.use.toolUseId });
|
||||
});
|
||||
});
|
||||
@@ -284,6 +284,14 @@ function currentModeStatement(text: string, questionSummary?: string): { option:
|
||||
return null;
|
||||
}
|
||||
|
||||
/** The output Claude Code put in place of the injected start line, or null when the text has no single such slot. */
|
||||
function loadedStartOutput(text: string): string | null {
|
||||
const head = '\n## Preamble (run first)\n\n', tail = '\n\nIf the line above is not the start output, run this once; otherwise it already ran:';
|
||||
const from = text.indexOf(head), to = text.indexOf(tail);
|
||||
if (from < 0 || to < from + head.length || text.indexOf(head, from + 1) >= 0 || text.indexOf(tail, to + 1) >= 0) return null;
|
||||
return text.slice(from + head.length, to);
|
||||
}
|
||||
|
||||
/** A native slash expansion need not produce a Skill tool call. Its completed
|
||||
* preamble → preference check → decision log → current public mode provides
|
||||
* the alternative evidence, without attributing a synthetic Skill event. */
|
||||
@@ -308,7 +316,12 @@ function structuredModeDecision(transcript: PlanCountTranscript, tools: NativePu
|
||||
return result && result.isError === false && timely(result.timestamp) && time(result.timestamp) >= time(use.timestamp) ? result : null;
|
||||
};
|
||||
const bash = owned.filter(e => e.kind === 'use' && e.name === 'Bash' && timely(e.timestamp) && typeof e.input?.command === 'string');
|
||||
const starts = bash.flatMap(use => {
|
||||
const startSession = (output: string) => {
|
||||
const status = output.split('GSTACK_INSTRUCTION_BEGIN:')[0]!;
|
||||
const sessions = [...status.matchAll(/^SESSION_ID: ([A-Za-z0-9-]+)$/gm)];
|
||||
return sessions.length === 1 && /^SKILL_START_PROTO: 1$/m.test(status) && /^QUESTION_TUNING: true$/m.test(status) ? sessions[0]![1]! : null;
|
||||
};
|
||||
const shellStarts = bash.flatMap(use => {
|
||||
const args = preambleArgs(use.input!.command as string), result = successful(use);
|
||||
if (!args || !result || args.length % 2 || args.length < 2) return [];
|
||||
const pairs = new Map<string, string>();
|
||||
@@ -317,11 +330,17 @@ function structuredModeDecision(transcript: PlanCountTranscript, tools: NativePu
|
||||
pairs.set(args[i]!, args[i + 1]!);
|
||||
}
|
||||
if (pairs.get('--skill') !== opts.skillName || typeof result.content !== 'string') return [];
|
||||
const status = result.content.split('GSTACK_INSTRUCTION_BEGIN:')[0]!;
|
||||
const sessions = [...status.matchAll(/^SESSION_ID: ([A-Za-z0-9-]+)$/gm)];
|
||||
if (sessions.length !== 1 || !/^SKILL_START_PROTO: 1$/m.test(status) || !/^QUESTION_TUNING: true$/m.test(status)) return [];
|
||||
return [{ use, result, session: sessions[0]![1]! }];
|
||||
const session = startSession(result.content);
|
||||
return session ? [{ toolUseId: use.toolUseId, at: result.timestamp, session }] : [];
|
||||
});
|
||||
// Claude Code runs the start while the skill loads and puts its output in the
|
||||
// loaded text (GST-3157); that load counts as the session's one start.
|
||||
const loadStarts = (transcript.skillLoads ?? []).flatMap(load => {
|
||||
if (load.sessionId !== opts.sessionId || load.skill !== opts.skillName || !timely(load.timestamp)) return [];
|
||||
const output = loadedStartOutput(load.text), session = output === null ? null : startSession(output);
|
||||
return session ? [{ toolUseId: undefined, at: load.timestamp, session }] : [];
|
||||
});
|
||||
const starts = [...shellStarts, ...loadStarts];
|
||||
if (starts.length !== 1) return null;
|
||||
const start = starts[0]!, questionId = `${opts.skillName}-mode`;
|
||||
// The handoff helper's printed AUTO_DECIDE line is the current declaration
|
||||
@@ -350,17 +369,17 @@ function structuredModeDecision(transcript: PlanCountTranscript, tools: NativePu
|
||||
if (row.skill === opts.skillName && row.session_id === start.session && row.source === 'agent' &&
|
||||
row.auto_decided === true && modeValue(row.user_choice) && modeValue(row.user_choice) === modeValue(row.recommended) &&
|
||||
typeof row.question_summary === 'string' && row.question_summary.trim() &&
|
||||
loggedAt >= time(start.result.timestamp) && loggedAt <= opts.now) {
|
||||
const handoff = autoHandoff(time(start.result.timestamp), modeValue(row.user_choice)!, row.question_summary);
|
||||
if (handoff) return { ...handoff, summary: row.question_summary, preambleToolUseId: start.use.toolUseId, stateRecord: row };
|
||||
if (autoHandoffs(time(start.result.timestamp), modeValue(row.user_choice)!).length) return null;
|
||||
loggedAt >= time(start.at) && loggedAt <= opts.now) {
|
||||
const handoff = autoHandoff(time(start.at), modeValue(row.user_choice)!, row.question_summary);
|
||||
if (handoff) return { ...handoff, summary: row.question_summary, preambleToolUseId: start.toolUseId, stateRecord: row };
|
||||
if (autoHandoffs(time(start.at), modeValue(row.user_choice)!).length) return null;
|
||||
for (const message of current) {
|
||||
const declared = currentModeStatement(message.text, row.question_summary);
|
||||
if (time(message.timestamp) < loggedAt || !declared || declared.option !== modeValue(row.user_choice)) continue;
|
||||
const after = current.filter(m => time(m.timestamp) >= time(message.timestamp)).map(m => m.text).join('\n\n');
|
||||
if (withdrawn(after, declared.option, row.question_summary)) continue;
|
||||
return { sessionId: opts.sessionId, timestamp: message.timestamp, summary: row.question_summary,
|
||||
option: declared.option, annotation: message.text, preambleToolUseId: start.use.toolUseId, stateRecord: row };
|
||||
option: declared.option, annotation: message.text, preambleToolUseId: start.toolUseId, stateRecord: row };
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -368,7 +387,7 @@ function structuredModeDecision(transcript: PlanCountTranscript, tools: NativePu
|
||||
}
|
||||
const checks = bash.flatMap(use => {
|
||||
let command = (use.input!.command as string).trim();
|
||||
const result = successful(use); if (time(use.timestamp) < time(start.result.timestamp)) return [];
|
||||
const result = successful(use); if (time(use.timestamp) < time(start.at)) return [];
|
||||
const status = '; echo "EXIT: $?"', hasStatus = command.endsWith(status);
|
||||
if (hasStatus) command = command.slice(0, -status.length);
|
||||
const pipe = /^printf\s+(?:'%s'|"%s")\s+(?:'[^']*'|"[^"$`\\]*")\s*\|\s*/.exec(command);
|
||||
@@ -406,7 +425,7 @@ function structuredModeDecision(transcript: PlanCountTranscript, tools: NativePu
|
||||
if (logs.length !== 1 || !logs[0]!.result) return null;
|
||||
const logged = logs[0]!;
|
||||
const handoff = autoHandoff(time(check.result!.timestamp), modeValue(logged.log.user_choice)!, logged.log.question_summary);
|
||||
if (handoff) return { ...handoff, summary: logged.log.question_summary, preambleToolUseId: start.use.toolUseId,
|
||||
if (handoff) return { ...handoff, summary: logged.log.question_summary, preambleToolUseId: start.toolUseId,
|
||||
preferenceToolUseId: check.use.toolUseId, questionLogToolUseId: logged.use.toolUseId };
|
||||
if (autoHandoffs(time(check.result!.timestamp), modeValue(logged.log.user_choice)!).length) return null;
|
||||
for (const message of current) {
|
||||
@@ -417,7 +436,7 @@ function structuredModeDecision(transcript: PlanCountTranscript, tools: NativePu
|
||||
if (withdrawn(after, declared.option, logged.log.question_summary) || current.some(m => time(m.timestamp) >= time(message.timestamp) &&
|
||||
currentModeStatement(m.text, logged.log.question_summary)?.option !== undefined && currentModeStatement(m.text, logged.log.question_summary)!.option !== declared.option)) continue;
|
||||
return { sessionId: opts.sessionId, timestamp: message.timestamp, summary: logged.log.question_summary,
|
||||
option: declared.option, annotation: message.text, preambleToolUseId: start.use.toolUseId,
|
||||
option: declared.option, annotation: message.text, preambleToolUseId: start.toolUseId,
|
||||
preferenceToolUseId: check.use.toolUseId, questionLogToolUseId: logged.use.toolUseId };
|
||||
}
|
||||
return null;
|
||||
|
||||
@@ -205,7 +205,7 @@ export const E2E_TOUCHFILES: Record<string, string[]> = {
|
||||
// include question-tuning.ts and generate-ask-user-format.ts because the
|
||||
// AUTO_DECIDE preamble injection lives there and changes can flip the
|
||||
// regression test outcome between 'asked' and 'auto_decided'.
|
||||
'plan-ceo-review-plan-mode': [ 'ETHOS.md','bin/gstack-state-root.sh', 'lib/state-root.ts', 'test/helpers/plan-count-fixture.ts',
|
||||
'plan-ceo-review-plan-mode': [ 'ETHOS.md', 'bin/gstack-skill-load', 'scripts/resolvers/load-injection.ts','bin/gstack-state-root.sh', 'lib/state-root.ts', 'test/helpers/plan-count-fixture.ts',
|
||||
|
||||
|
||||
'test/fixtures/auto-decide-recommendation-361c.json',
|
||||
@@ -313,7 +313,7 @@ export const E2E_TOUCHFILES: Record<string, string[]> = {
|
||||
// written a never-ask preference, AUQ should still auto-decide rather than
|
||||
// surfacing the question. Touches the question-tuning + preference
|
||||
// infrastructure plus the resolvers that own the AUTO_DECIDE preamble.
|
||||
'auto-decide-preserved': [ 'ETHOS.md','bin/gstack-ceo-mode-handoff', 'test/fixtures/auto-decide-handoff-line-e354.json', 'test/fixtures/auto-decide-handoff-before-log-37182865432.json', 'bin/gstack-state-root.sh', 'lib/state-root.ts', 'lib/claude-public-transcript.ts', 'lib/claude-journal-records.ts', 'lib/claude-owned-journal.ts',
|
||||
'auto-decide-preserved': [ 'ETHOS.md', 'bin/gstack-skill-load', 'scripts/resolvers/load-injection.ts','bin/gstack-ceo-mode-handoff', 'test/fixtures/auto-decide-handoff-line-e354.json', 'test/fixtures/auto-decide-handoff-before-log-37182865432.json', 'bin/gstack-state-root.sh', 'lib/state-root.ts', 'lib/claude-public-transcript.ts', 'lib/claude-journal-records.ts', 'lib/claude-owned-journal.ts',
|
||||
|
||||
'test/fixtures/auto-decide-recommendation-361c.json',
|
||||
|
||||
|
||||
@@ -407,3 +407,28 @@ describe('native journal roots: SessionStart preamble, resumed genesis and reaso
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe('skill text Claude Code loaded', () => {
|
||||
test('keeps only scoped, timestamped skill expansions with their load-time output', () => {
|
||||
const f = fixture();
|
||||
const text = 'Base directory for this skill: /fixture/.claude/skills/plan-ceo-review\n\n## Preamble (run first)\n\nSKILL_START_PROTO: 1\n';
|
||||
const other = (name: string) => text.replace('plan-ceo-review', name);
|
||||
f.append(
|
||||
f.record('user', [{ type: 'text', text: '<command-message>plan-ceo-review</command-message>' }]),
|
||||
f.record('user', [{ type: 'text', text }], { isMeta: true }),
|
||||
f.record('user', [{ type: 'text', text: other('typed-by-user') }]),
|
||||
f.record('user', [{ type: 'text', text: other('sidechain') }], { isMeta: true, isSidechain: true }),
|
||||
f.record('user', [{ type: 'text', text: other('untimed') }], { isMeta: true, timestamp: 'later' }),
|
||||
f.record('user', [{ type: 'text', text: other('foreign') }], { isMeta: true, cwd: '/elsewhere' }),
|
||||
f.record('user', [{ type: 'text', text: `Quoted: ${text}` }], { isMeta: true }),
|
||||
f.record('assistant', [{ type: 'text', text: other('assistant') }], { isMeta: true }),
|
||||
);
|
||||
expect(f.read().skillLoads).toEqual([{ sessionId: 'session-a', skill: 'plan-ceo-review', text, timestamp: '2026-09-08T15:31:13.607Z' }]);
|
||||
});
|
||||
|
||||
test('a transcript with no skill expansion reports none', () => {
|
||||
const f = fixture();
|
||||
f.append(f.ask('one'));
|
||||
expect(f.read().skillLoads).toBeUndefined();
|
||||
});
|
||||
});
|
||||
Reference in new issue
Block a user