mirror of
https://github.com/garrytan/gstack.git
synced 2026-11-18 22:24:17 +01:00
Unattended /autoplan on Claude Code says its phase guard is on, and a report hidden in reasoning gets a clear fix (GST-3062) (#4481)
An unattended /autoplan run on Claude Code now says its phase guard is on, instead of saying it is off while the guard blocks it. A phase report written only in reasoning now gets a denial that says so and names the exact command to send. Why: in the day-one eval, two runs were blocked by a guard their own record called off, and one run ended blocked after four publish attempts whose report was invisible. Nothing to do. Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>
This commit is contained in:
1 parent
f87d4325a3
commit
44170d6791
19 files changed
+244
-34
No files matched your search
@@ -115,7 +115,7 @@ describe('Cause B: the current call is not journaled at hook time (payload path)
|
||||
expect(codeOf(await f.hook(f.input('b', 'Agent', dispatch(f, { model: 'haiku' }))))).toBe('agent_key');
|
||||
const next = await f.hook(f.input('c', 'Read', { file_path: section('design-phase.md') }));
|
||||
expect(codeOf(next)).toBe('publication_missing');
|
||||
expect(denial(next)).toContain('true autoplan-published <phase>');
|
||||
expect(denial(next)).toContain('true autoplan-published ceo');
|
||||
});
|
||||
});
|
||||
|
||||
@@ -126,7 +126,7 @@ describe('Publication flush: a report counts only once a later journaled record
|
||||
const f = make(); f.report(f.nextMessage()); f.journal();
|
||||
const output = await f.hook(next(f));
|
||||
expect(codeOf(output)).toBe('publication_unflushed');
|
||||
expect(denial(output)).toContain('`true autoplan-published <phase>`');
|
||||
expect(denial(output)).toContain('`true autoplan-published ceo`');
|
||||
expect(denial(output)).not.toMatch(/\bretry\b/i);
|
||||
});
|
||||
|
||||
|
||||
@@ -0,0 +1,84 @@
|
||||
/**
|
||||
* GST-3062 (GEVA-549): an unattended /autoplan session sent four publish messages whose
|
||||
* only content was reasoning plus `true autoplan-published dx`. The guard denied the next
|
||||
* phase each time with a generic publication_missing and the run ended blocked. A no-op
|
||||
* publish with no visible text now gets its own corrective code that names the exact next
|
||||
* step, and the guard still denies until a visible report is really published.
|
||||
*/
|
||||
import { afterEach, describe, expect, test } from 'bun:test';
|
||||
import * as fs from 'node:fs';
|
||||
import * as path from 'node:path';
|
||||
import { REASONS, anchor } from '../autoplan/bin/guard-reasons';
|
||||
import { guardFixture, section, denial, codeOf, type GuardFixture } from './helpers/autoplan-guard-fixture';
|
||||
|
||||
const fixtures: GuardFixture[] = [];
|
||||
afterEach(() => { for (const f of fixtures.splice(0)) f.cleanup(); });
|
||||
const make = () => { const f = guardFixture(); fixtures.push(f); return f; };
|
||||
const nextPhase = (f: GuardFixture, id = 'next') => f.input(id, 'Read', { file_path: section('design-phase.md') });
|
||||
|
||||
/** Phase-close step 6 as the stuck session journaled it: reasoning, the no-op and its result; no text block. */
|
||||
function reasoningOnlyPublish(f: GuardFixture) {
|
||||
const id = f.nextMessage();
|
||||
f.say('**Phase 1 complete.**\nConsensus: 3/3 confirmed.', id);
|
||||
f.use(`noop-${id}`, 'Bash', { command: 'true autoplan-published ceo' }, id);
|
||||
f.result(`noop-${id}`, { content: '' });
|
||||
return id;
|
||||
}
|
||||
/** Turn every text block that holds `marker` into a private (untagged) thinking block, as Claude Code journals reasoning. */
|
||||
function intoReasoning(f: GuardFixture, marker: string) {
|
||||
const rows = fs.readFileSync(f.transcript, 'utf8').trim().split('\n').map(line => JSON.parse(line));
|
||||
for (const row of rows) if (row.message?.content?.[0]?.type === 'text' && row.message.content[0].text.includes(marker))
|
||||
row.message.content = [{ type: 'thinking', thinking: row.message.content[0].text, signature: 'c2lnbmF0dXJl' }];
|
||||
fs.writeFileSync(f.transcript, rows.map(r => JSON.stringify(r) + '\n').join(''));
|
||||
}
|
||||
|
||||
describe('a publish message whose report is only in reasoning', () => {
|
||||
test('is denied as publication_not_visible with the exact next step, on every attempt', async () => {
|
||||
const f = make();
|
||||
for (let attempt = 0; attempt < 4; attempt++) reasoningOnlyPublish(f);
|
||||
f.journal(); intoReasoning(f, 'Phase 1 complete');
|
||||
const output = await f.hook(nextPhase(f));
|
||||
expect(codeOf(output)).toBe('publication_not_visible');
|
||||
const text = denial(output)!;
|
||||
expect(text).toContain('carried no visible text');
|
||||
expect(text).toContain('reasoning');
|
||||
expect(text).toContain('starting with the line **Phase 1 complete.**');
|
||||
expect(text).toContain('`true autoplan-published ceo`');
|
||||
expect(REASONS.publication_not_visible.disposition).toBe('corrective');
|
||||
expect(text).toContain(`docs/autoplan-guard-troubleshooting.md#${anchor('publication_not_visible')}`);
|
||||
expect(text).not.toMatch(/\bretry\b/i);
|
||||
});
|
||||
|
||||
test('a visible report afterwards lets the next phase start', async () => {
|
||||
const f = make();
|
||||
reasoningOnlyPublish(f); f.journal(); intoReasoning(f, 'Phase 1 complete');
|
||||
expect(codeOf(await f.hook(nextPhase(f)))).toBe('publication_not_visible');
|
||||
f.publish(); f.journal(); intoReasoning(f, 'Consensus: 3/3');
|
||||
expect(denial(await f.hook(nextPhase(f, 'next-2')))).toBeUndefined();
|
||||
});
|
||||
});
|
||||
|
||||
describe('negative controls: the guard still denies a phase whose report is really missing', () => {
|
||||
test('no publish message at all stays publication_missing', async () => {
|
||||
const f = make(); f.journal();
|
||||
const output = await f.hook(nextPhase(f));
|
||||
expect(codeOf(output)).toBe('publication_missing');
|
||||
expect(denial(output)).toContain('`true autoplan-published ceo`');
|
||||
});
|
||||
|
||||
test('a visible publish message without the phase declaration stays publication_missing', async () => {
|
||||
const f = make();
|
||||
const id = f.nextMessage();
|
||||
f.say('Done with the review.', id);
|
||||
f.use(`noop-${id}`, 'Bash', { command: 'true autoplan-published ceo' }, id);
|
||||
f.result(`noop-${id}`, { content: '' }); f.journal();
|
||||
expect(codeOf(await f.hook(nextPhase(f)))).toBe('publication_missing');
|
||||
});
|
||||
|
||||
test('a report saved to a file, with a reasoning-only publish, does not publish it', async () => {
|
||||
const f = make();
|
||||
fs.writeFileSync(path.join(f.cwd, 'ceo-report.md'), '**Phase 1 complete.**\nConsensus: 3/3 confirmed.\n');
|
||||
reasoningOnlyPublish(f); f.journal(); intoReasoning(f, 'Phase 1 complete');
|
||||
expect(codeOf(await f.hook(nextPhase(f)))).toBe('publication_not_visible');
|
||||
});
|
||||
});
|
||||
@@ -24,6 +24,7 @@ import * as path from 'path';
|
||||
import { generatePreamble } from '../scripts/resolvers/preamble';
|
||||
import { HOST_PATHS, type TemplateContext } from '../scripts/resolvers/types';
|
||||
import { expectAbsent, expectMentions } from './helpers/prompt-structure';
|
||||
import { GUARD_ENFORCED_LINE, GUARD_NOT_INSTALLED_LINE, autoplanGuardLine } from '../lib/autoplan-guard-status';
|
||||
|
||||
const ROOT = path.resolve(import.meta.dir, '..');
|
||||
const START = path.join(ROOT, 'bin', 'gstack-skill-start');
|
||||
@@ -388,6 +389,22 @@ describe('gstack-skill-start behavior', () => {
|
||||
}
|
||||
});
|
||||
|
||||
// GST-3062 (GEVA-549): an unattended Claude Code run printed GUARD_NOT_INSTALLED while autoplan's
|
||||
// PreToolUse hook denied its phase Reads. The line follows the host and the installed skill, not the session kind.
|
||||
test('the autoplan guard line says enforced whenever Claude Code runs the declared hook, unattended included', () => {
|
||||
const line = (env: Record<string, string>) => (runStart(['--skill', 'autoplan'], env).match(/^autoplan guard: .*$/gm) ?? []);
|
||||
const unattendedClaude = line({ CLAUDECODE: '1', GSTACK_SESSION_KIND: 'unattended' });
|
||||
expect(unattendedClaude).toEqual([GUARD_ENFORCED_LINE]);
|
||||
expect(unattendedClaude.join('\n')).not.toContain('GUARD_NOT_INSTALLED');
|
||||
const matrix: Record<string, string>[] = [
|
||||
{ CLAUDECODE: '1' }, { CLAUDECODE: '1', GSTACK_ACTIVE_HOST: 'claude' },
|
||||
{ CLAUDECODE: '1', GSTACK_ACTIVE_HOST: 'codex' }, { GSTACK_SESSION_KIND: 'unattended' }, {},
|
||||
];
|
||||
for (const env of matrix) expect({ env, line: line(env) }).toEqual({ env, line: [autoplanGuardLine(env, ROOT)] });
|
||||
expect(line({ CLAUDECODE: '1', GSTACK_ACTIVE_HOST: 'codex', GSTACK_SESSION_KIND: 'unattended' })).toEqual([GUARD_NOT_INSTALLED_LINE]);
|
||||
expect(runStart(['--skill', 'review'], { CLAUDECODE: '1' })).not.toContain('autoplan guard:');
|
||||
});
|
||||
|
||||
test('unattended block promises GSTACK_RESULT only to skills that write it; the rest name their record (#3281)', () => {
|
||||
const block = (skill: string) => {
|
||||
const out = runStart(['--skill', skill], { GSTACK_SESSION_KIND: 'unattended' });
|
||||
|
||||
@@ -14,6 +14,7 @@ import {
|
||||
renderUrgentBlock, urgentBlockPrecedes, validateRun, validateFile,
|
||||
} from '../lib/headless-artifacts';
|
||||
import { RESULT_CODES } from '../lib/result-codes';
|
||||
import { GUARD_ENFORCED_LINE } from '../lib/autoplan-guard-status';
|
||||
|
||||
const ROOT = path.resolve(import.meta.dir, '..');
|
||||
const BIN = path.join(ROOT, 'bin', 'gstack-artifact');
|
||||
@@ -186,6 +187,21 @@ describe('validate fails on', () => {
|
||||
expect(r.out.trim()).toBe('ARTIFACT_INVALID: 1 error(s)');
|
||||
});
|
||||
|
||||
test('an unattended run whose review-record.md lacks its guard line; GUARD_ENFORCED counts as one (GST-3062)', () => {
|
||||
const enforced = copyRun((m, dir) => {
|
||||
m.session_kind = 'unattended';
|
||||
fs.writeFileSync(path.join(dir, 'review-record.md'), `# Review record\nDeviation: ${GUARD_ENFORCED_LINE}\n`);
|
||||
m.artifacts['review-record.md'] = { path: 'review-record.md', sha256: sha(path.join(dir, 'review-record.md')) };
|
||||
});
|
||||
expect(codes(enforced)).toEqual([]);
|
||||
const bareCode = copyRun((m, dir) => {
|
||||
m.session_kind = 'unattended';
|
||||
fs.writeFileSync(path.join(dir, 'review-record.md'), '# Review record\nThe GUARD_ENFORCED_LINE constant is defined in lib.\n');
|
||||
m.artifacts['review-record.md'] = { path: 'review-record.md', sha256: sha(path.join(dir, 'review-record.md')) };
|
||||
});
|
||||
expect(codes(bareCode)).toEqual(['ARTIFACT_GUARD_LINE_MISSING']);
|
||||
});
|
||||
|
||||
test('an unattended run whose review-record.md lacks the GUARD_NOT_INSTALLED line', () => {
|
||||
const without = copyRun((m, dir) => {
|
||||
m.session_kind = 'unattended';
|
||||
|
||||
@@ -29,6 +29,8 @@ export function env(extra: Record<string, string> = {}): Record<string, string>
|
||||
const out: Record<string, string> = {};
|
||||
for (const [k, v] of Object.entries(process.env)) if (v !== undefined) out[k] = v;
|
||||
out.GSTACK_EPHEMERAL = '1';
|
||||
// The run record's guard line follows the host (lib/autoplan-guard-status.ts); keep it the CI host's.
|
||||
delete out.CLAUDECODE; delete out.GSTACK_ACTIVE_HOST;
|
||||
// The runner's default kind; spelled through the bin's own flag table so the
|
||||
// free-suite environment scan (test/gstack-session-kind.test.ts) stays clean.
|
||||
out[SESSION_KIND_VAR] = RUNNER_KIND;
|
||||
|
||||
Reference in new issue
Block a user