mirror of
https://github.com/garrytan/gstack.git
synced 2026-11-11 11:10:13 +01:00
Merge main bf2cab3 into gsta258-l1-release-gate (rule 19 grant, GSTA-36)
Resolve lib/ship-policy.ts against #3920: keep main's stamped REVIEW_POLICY_KEY_VERSION and its review entry, and append release_gate to the same POLICY_KEY_INTRODUCED line. Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>
This commit is contained in:
garrytanandcapy-ai[bot] committed
commit
e9277ef4d4
126 files changed
+4486
-457
No files matched your search
@@ -1,7 +1,9 @@
|
||||
import { expect, test } from 'bun:test';
|
||||
import * as fs from 'node:fs';
|
||||
import * as os from 'node:os';
|
||||
import * as path from 'node:path';
|
||||
import { closePacketReadAfterNotice, dispatchPromptDelta, prepareGuardSession, stoppedAtPhaseBoundary } from './helpers/autoplan-guard-pty';
|
||||
import { closePacketReadAfterNotice, dispatchEventRewriter, dispatchPromptDelta, dispatchToken, prepareGuardSession, startDispatchRelay, stoppedAtPhaseBoundary } from './helpers/autoplan-guard-pty';
|
||||
import { codeOf, guardFixture } from './helpers/autoplan-guard-fixture';
|
||||
|
||||
const base = '/home/runner/.cache/gstack-paid-shard-x/tmp/ap-guard-pty-y';
|
||||
|
||||
@@ -27,10 +29,10 @@ test('the scripted actor ends its turn after a background CEO dispatch and reads
|
||||
try {
|
||||
const skill = fs.readFileSync(path.join(s.project, '.claude', 'skills', fs.readdirSync(path.join(s.project, '.claude', 'skills'))[0]!, 'SKILL.md'), 'utf8');
|
||||
const step = (n: number) => skill.slice(skill.indexOf(`\n${n}. `) + 1, skill.indexOf(`\n${n + 1}. `));
|
||||
expect(step(5)).toContain('If its result says the agent is running in the background, end your turn at once with no further tool call; the completion notification starts your next turn.');
|
||||
expect(step(5)).not.toContain('If the agent runs in the background, end your turn and wait');
|
||||
expect(step(6).startsWith('6. Only after the CEO review agent has finished (in the turn its completion notification starts, when it ran in the background), Read the file whose path step 3 printed')).toBe(true);
|
||||
expect(skill.indexOf('\n6. ')).toBeGreaterThan(skill.indexOf('\n5. '));
|
||||
expect(step(4)).toContain('If its result says the agent is running in the background, end your turn at once with no further tool call; the completion notification starts your next turn.');
|
||||
expect(step(4)).not.toContain('If the agent runs in the background, end your turn and wait');
|
||||
expect(step(5).startsWith('5. Only after the CEO review agent has finished (in the turn its completion notification starts, when it ran in the background), Read the file whose path step 3 printed')).toBe(true);
|
||||
expect(skill.indexOf('\n5. ')).toBeGreaterThan(skill.indexOf('\n4. '));
|
||||
} finally { s.cleanup(); }
|
||||
});
|
||||
|
||||
@@ -43,25 +45,25 @@ test('the captured phase-boundary stop is named; a run still working or already
|
||||
expect(stoppedAtPhaseBoundary(STOP_SCREEN)).toBe(true);
|
||||
const working = STOP_SCREEN.slice(0, STOP_SCREEN.indexOf('✻ Brewed'));
|
||||
expect(stoppedAtPhaseBoundary(working)).toBe(false);
|
||||
const continued = working + '●Step8:ReadingtheDesignphasefile\n\nRead 1 file\n\n●Step9:ReadingtheDesigndispatchfile\n\n✻ Brewed for 80s · done 5:17 AM\n❯\n';
|
||||
const continued = working + '●Step7:ReadingtheDesignphasefile\n\nRead 1 file\n\n●Step8:DispatchingtheDesignreview\n\n✻ Brewed for 80s · done 5:17 AM\n❯\n';
|
||||
expect(stoppedAtPhaseBoundary(continued)).toBe(false);
|
||||
expect(stoppedAtPhaseBoundary('✻ Brewed for 10s · done 5:00 AM\n❯\n')).toBe(false);
|
||||
});
|
||||
|
||||
test('the scripted actor keeps going through the phase boundary: nothing between step 7 and step 11 lets it end its turn there', () => {
|
||||
test('the scripted actor keeps going through the phase boundary: nothing between step 6 and step 9 lets it end its turn there', () => {
|
||||
const s = prepareGuardSession(undefined);
|
||||
try {
|
||||
const skill = fs.readFileSync(path.join(s.project, '.claude', 'skills', fs.readdirSync(path.join(s.project, '.claude', 'skills'))[0]!, 'SKILL.md'), 'utf8');
|
||||
const step = (n: number) => skill.slice(skill.indexOf(`\n${n}. `) + 1, skill.indexOf(`\n${n + 1}. `));
|
||||
const intro = skill.slice(skill.indexOf('\n---\n', 4) + 5, skill.indexOf('\n1. '));
|
||||
expect(intro).toContain('from step 1 through step 11 in one continuous run');
|
||||
expect(intro).toContain('End your turn before step 11 only where step 5 or step 10 says to wait for a background agent.');
|
||||
expect(intro).toContain('from step 1 through step 9 in one continuous run');
|
||||
expect(intro).toContain('End your turn before step 9 only where step 4 or step 8 says to wait for a background agent.');
|
||||
expect(intro).toContain('**Phase 1 complete.** is a phase boundary, not the end of the task.');
|
||||
expect(step(7)).toContain('Do not end your turn after it: when its result arrives, go straight on to step 8.');
|
||||
expect(step(8).startsWith("8. As your next assistant message, in the same turn as step 7's result (no user input is coming), Read the file")).toBe(true);
|
||||
expect(step(6)).toContain('Do not end your turn after it: when its result arrives, go straight on to step 7.');
|
||||
expect(step(7).startsWith("7. As your next assistant message, in the same turn as step 6's result (no user input is coming), Read the file")).toBe(true);
|
||||
// The publication and the Phase 2 entry stay in separate messages: the guard's boundary is unchanged.
|
||||
expect(step(7)).toContain('make exactly one tool call in that same message: the Bash command `true autoplan-published ceo`');
|
||||
for (const n of [6, 7, 8, 9]) expect(step(n)).not.toMatch(/end your turn(?! after it)|wait for (its|a) completion/i);
|
||||
expect(step(6)).toContain('make exactly one tool call in that same message: the Bash command `true autoplan-published ceo`');
|
||||
for (const n of [5, 6, 7]) expect(step(n)).not.toMatch(/end your turn(?! after it)|wait for (its|a) completion/i);
|
||||
} finally { s.cleanup(); }
|
||||
});
|
||||
|
||||
@@ -84,3 +86,93 @@ test('a dispatch_prompt deny names the bytes the actor changed, read from the jo
|
||||
expect(dispatchPromptDelta(design, design)).toBe(`identical (${design.length} chars)`);
|
||||
} finally { s.cleanup(); }
|
||||
});
|
||||
|
||||
// PR gate 41247097050 (paid-slice-1-a1), #3646's delta: "Design review: expected 977 chars, sent 993; first difference at
|
||||
// offset 875": the Haiku actor kept the dispatch's contentHash to "…1c8f8c" and invented an 80-hex tail. The actor now sends
|
||||
// a declared token and the driver's relay puts the exact dispatch in its place before Claude Code sees the tool call.
|
||||
test('the dispatch relay hands the guard the exact dispatch for the declared token, and passes a retyped dispatch through to its denial', async () => {
|
||||
const f = guardFixture('ceo'), retypedFixture = guardFixture('ceo');
|
||||
const artifacts = fs.mkdtempSync(path.join(os.tmpdir(), 'ap-guard-relay-'));
|
||||
const exact = f.snapshot.nativeDispatchPrompt, hash = retypedFixture.snapshot.sha256;
|
||||
const retyped = retypedFixture.snapshot.nativeDispatchPrompt.replace(`INPUT: ceo ${hash}`, `INPUT: ceo ${hash.slice(0, 46)}65b428cec49f80716ab6f3015701${'0a'.repeat(3)}`);
|
||||
fs.writeFileSync(path.join(artifacts, 'dispatch-ceo.txt'), exact);
|
||||
const agent = (id: string, prompt: string) => ({ description: 'CEO review', subagent_type: 'general-purpose', prompt, run_in_background: false, _id: id });
|
||||
const blocks = [agent('toolu_token', dispatchToken('ceo')), agent('toolu_retyped', retyped), { _id: 'toolu_bash', command: dispatchToken('ceo') }];
|
||||
const sse = (data: Record<string, unknown>) => `event: ${data.type}\ndata: ${JSON.stringify(data)}\n\n`;
|
||||
const stream = [sse({ type: 'message_start', message: { id: 'msg_1', role: 'assistant', content: [] } }),
|
||||
...blocks.flatMap(({ _id, ...input }, index) => {
|
||||
const json = JSON.stringify(input), name = 'command' in input ? 'Bash' : 'Agent';
|
||||
return [sse({ type: 'content_block_start', index, content_block: { type: 'tool_use', id: _id, name, input: {} } }),
|
||||
...Array.from({ length: Math.ceil(json.length / 37) }, (_, i) => sse({ type: 'content_block_delta', index,
|
||||
delta: { type: 'input_json_delta', partial_json: json.slice(i * 37, i * 37 + 37) } })),
|
||||
sse({ type: 'content_block_stop', index })];
|
||||
}),
|
||||
sse({ type: 'message_delta', delta: { stop_reason: 'tool_use' } }), sse({ type: 'message_stop' })].join('');
|
||||
const seen: Array<{ path: string; key: string | null; host: string | null; body: string }> = [];
|
||||
const upstream = Bun.serve({ hostname: '127.0.0.1', port: 0, async fetch(req) {
|
||||
const body = await req.text(), url = new URL(req.url);
|
||||
seen.push({ path: url.pathname + url.search, key: req.headers.get('x-api-key'), host: req.headers.get('host'), body });
|
||||
if (!JSON.parse(body).stream) return Response.json({ id: 'msg_2', content: [{ type: 'tool_use', id: 'toolu_json', name: 'Agent', input: agent('toolu_json', dispatchToken('ceo')) }] });
|
||||
const bytes = new TextEncoder().encode(stream);
|
||||
return new Response(new ReadableStream({ start(sink) { for (let i = 0; i < bytes.length; i += 7) sink.enqueue(bytes.slice(i, i + 7)); sink.close(); } }),
|
||||
{ headers: { 'content-type': 'text/event-stream' } });
|
||||
} });
|
||||
const relay = startDispatchRelay(artifacts, `http://127.0.0.1:${upstream.port}/`);
|
||||
try {
|
||||
const send = (stream: boolean) => fetch(`${relay.url}/v1/messages?beta=true`, { method: 'POST',
|
||||
headers: { 'content-type': 'application/json', 'x-api-key': 'fixture-key' }, body: JSON.stringify({ model: 'm', stream, messages: history }) });
|
||||
// The actor's history after its CEO dispatch, and a reviewer's first turn, whose dispatch is user text.
|
||||
const history = [{ role: 'assistant', content: [{ type: 'tool_use', id: 'toolu_ceo', name: 'Agent', input: { description: 'CEO review', prompt: exact } },
|
||||
{ type: 'tool_use', id: 'toolu_other', name: 'Agent', input: { description: 'CEO review', prompt: retyped } }] },
|
||||
{ role: 'user', content: [{ type: 'text', text: exact }] }];
|
||||
const text = await (await send(true)).text();
|
||||
const events = text.split('\n\n').filter(Boolean).map(e => JSON.parse(/^data: (.*)$/m.exec(e)![1]!));
|
||||
const inputs = blocks.map((_, index) => JSON.parse(events.filter(e => e.index === index && e.delta?.type === 'input_json_delta').map(e => e.delta.partial_json).join('')));
|
||||
expect(events.map(e => e.type).at(-1)).toBe('message_stop');
|
||||
expect(inputs[0]).toEqual({ description: 'CEO review', subagent_type: 'general-purpose', prompt: exact, run_in_background: false });
|
||||
expect(inputs[1].prompt).toBe(retyped);
|
||||
expect(inputs[2]).toEqual({ command: dispatchToken('ceo') });
|
||||
expect((await (await send(false)).json() as any).content[0].input.prompt).toBe(exact);
|
||||
expect(relay.substituted).toEqual(['ceo', 'ceo']);
|
||||
expect(seen.map(s => [s.path, s.key, s.host === new URL(relay.url).host])).toEqual(Array(2).fill(['/v1/messages?beta=true', 'fixture-key', false]));
|
||||
// Trial 1 of the first relay panel: shown the substituted CEO dispatch as its own past call, the actor imitated it for
|
||||
// Design. Upstream sees the actor's own token in its history; any other prompt and the reviewer's dispatch are kept.
|
||||
const upstreamHistory = JSON.parse(seen[0]!.body).messages;
|
||||
expect(upstreamHistory[0].content.map((b: any) => b.input.prompt)).toEqual([dispatchToken('ceo'), retyped]);
|
||||
expect(upstreamHistory[1].content[0].text).toBe(exact);
|
||||
|
||||
// The real guard on each relayed dispatch: the token's substitute is allowed; the retyped one is still denied.
|
||||
const { run_in_background: _drop, ...payload } = inputs[0];
|
||||
f.use('review', 'Agent', inputs[0]); f.journal();
|
||||
expect(await f.hook(f.input('review', 'Agent', payload))).toEqual({});
|
||||
expect(f.log().at(-1)).toMatchObject({ decision: 'allow', disposition: 'allow' });
|
||||
const { run_in_background: _keep, ...retypedPayload } = inputs[1];
|
||||
retypedFixture.use('review', 'Agent', inputs[1]); retypedFixture.journal();
|
||||
expect(codeOf(await retypedFixture.hook(retypedFixture.input('review', 'Agent', retypedPayload)))).toBe('dispatch_prompt');
|
||||
expect(dispatchPromptDelta(retypedFixture.snapshot.nativeDispatchPrompt, retyped)).toContain(`first difference at offset ${retyped.indexOf(`INPUT: ceo ${hash.slice(0, 46)}`) + 'INPUT: ceo '.length + 46}`);
|
||||
} finally {
|
||||
relay.stop(); upstream.stop(true);
|
||||
f.cleanup(); retypedFixture.cleanup(); fs.rmSync(artifacts, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
// /ship outside voice on PR #3953: a CRLF stream whose \r and \n land in separate chunks kept the token unsubstituted.
|
||||
test('the event rewriter substitutes the token however LF or CRLF delimiters are split across chunks', () => {
|
||||
const artifacts = fs.mkdtempSync(path.join(os.tmpdir(), 'ap-guard-crlf-'));
|
||||
try {
|
||||
fs.writeFileSync(path.join(artifacts, 'dispatch-ceo.txt'), 'exact dispatch');
|
||||
const json = JSON.stringify({ description: 'CEO review', prompt: dispatchToken('ceo') });
|
||||
const lf = [{ type: 'content_block_start', index: 0, content_block: { type: 'tool_use', id: 't', name: 'Agent', input: {} } },
|
||||
{ type: 'content_block_delta', index: 0, delta: { type: 'input_json_delta', partial_json: json } },
|
||||
{ type: 'content_block_stop', index: 0 }].map(e => `event: ${e.type}\ndata: ${JSON.stringify(e)}\n\n`).join('');
|
||||
for (const stream of [lf, lf.replace(/\n/g, '\r\n')]) for (const size of [1, 2, 3, 7, stream.length]) {
|
||||
const substituted: string[] = [];
|
||||
const rewrite = dispatchEventRewriter(artifacts, phase => substituted.push(phase));
|
||||
let out = '';
|
||||
for (let i = 0; i < stream.length; i += size) out += rewrite.push(stream.slice(i, i + size));
|
||||
out += rewrite.end();
|
||||
expect({ crlf: stream !== lf, size, substituted }).toEqual({ crlf: stream !== lf, size, substituted: ['ceo'] });
|
||||
expect(out).toContain(JSON.stringify(JSON.stringify({ description: 'CEO review', prompt: 'exact dispatch' })).slice(1, -1));
|
||||
}
|
||||
} finally { fs.rmSync(artifacts, { recursive: true, force: true }); }
|
||||
});
|
||||
@@ -646,6 +646,15 @@ describe('capy skill: factory.md, the auto-loaded "how we work" page', () => {
|
||||
expect(citationProblems(text)).toEqual([]);
|
||||
});
|
||||
|
||||
test('item 8 cites ETHOS.md principle 9, Earn Its Place, which states the fair test, the Why-null cycle and the safety exception', () => {
|
||||
const item8 = fs.readFileSync(FACTORY, 'utf8').split('\n').find(l => l.startsWith('8. '))!;
|
||||
expect(item8).toContain('[ETHOS.md: 9. Earn Its Place]');
|
||||
const ethos = fs.readFileSync(path.join(REPO, 'ETHOS.md'), 'utf8');
|
||||
const p9 = ethos.slice(ethos.indexOf('## 9. Earn Its Place'), ethos.indexOf('## How They Work Together'));
|
||||
expect(p9.length).toBeGreaterThan(200);
|
||||
for (const words of ['only include it if it is beneficial and good', 'claude_code', 'Opus 5.5', 'GPT-6.1 Sol', 'masked', 'Why null?', 'Safety and security guards', 'biggest', 'docs/inventory.md']) expect(p9).toContain(words);
|
||||
});
|
||||
|
||||
test('negative controls: a missing file, a missing heading and an unknown skill are each caught', () => {
|
||||
expect(citationProblems('[docs/no-such-page.md: Anything]')).toEqual(['missing file docs/no-such-page.md']);
|
||||
expect(citationProblems('[docs/captain.md: No Such Heading]')).toEqual(['missing heading docs/captain.md: No Such Heading']);
|
||||
|
||||
@@ -23,8 +23,8 @@ afterEach(() => { while (dirs.length) fs.rmSync(dirs.pop()!, { recursive: true,
|
||||
const BASE = ['Bars', 'Decoys', 'Stop rules', 'Held-out exposure'].map(s => `## ${s}\n- filled in\n`).join('\n');
|
||||
const ARMS = (...ids: string[]) => `\n## Arms\n${ids.map(id => `- ${id} default settings, $0.01 per item`).join('\n')}\n`;
|
||||
const LOADED = '\n## Comparison\nstack: loaded\n';
|
||||
const NO_GSTACK = (o: Partial<Record<'arms' | 'inputs' | 'prompt' | 'panel' | 'report', string>> = {}) => {
|
||||
const v = { arms: 'claude-opus-5-5#no-gstack', inputs: 'the same rows and fixture as the variants', prompt: 'the task text only; no SKILL.md, preamble, hooks or installed skills', panel: 'behavior, 3 trials, the same judge and labels', report: 'one row beside the variants in the results table', ...o };
|
||||
const NO_GSTACK = (o: Partial<Record<'arms' | 'inputs' | 'prompt' | 'panel' | 'report' | 'masking' | 'install' | 'audit', string>> = {}) => {
|
||||
const v = { arms: 'claude-opus-5-5#no-gstack', inputs: 'the same rows and fixture as the variants', prompt: 'the task text only; no SKILL.md, preamble, hooks or installed skills', panel: 'behavior, 3 trials, the same judge and labels', report: 'one row beside the variants in the results table', masking: 'an empty directory bind-mounted over ~/.capy/drive/user-garry-tan and ~/.capy/drive/org-gstack-gbrain before the run agent starts', install: 'no gstack at ~/.local/share/gstack or ~/.claude/skills, checked before the run', audit: 'drive reads logged per arm; an arm that read a gstack skill file is contaminated and gets zero credit', ...o };
|
||||
return `\n## No-gstack arm\n${Object.entries(v).filter(([, x]) => x !== undefined).map(([k, x]) => `${k}: ${x}`).join('\n')}\n`;
|
||||
};
|
||||
const VARIANTS = ARMS('claude-opus-5-5#review-g', 'claude-opus-5-5#review-h', 'claude-opus-5-5#no-gstack');
|
||||
@@ -72,6 +72,24 @@ describe('the No-gstack arm section', () => {
|
||||
});
|
||||
}
|
||||
|
||||
for (const key of ['masking', 'install', 'audit'] as const) {
|
||||
test(`masked arms: a missing, blank or "none" ${key}: line fails naming it`, () => {
|
||||
for (const v of [undefined, ' ']) {
|
||||
const r = checkPrereg(BASE + VARIANTS + LOADED + NO_GSTACK({ [key]: v } as Record<string, string>));
|
||||
expect(r.ok).toBe(false);
|
||||
expect(r.no_gstack.problems).toEqual([`missing ${key}`]);
|
||||
}
|
||||
const none = checkPrereg(BASE + VARIANTS + LOADED + NO_GSTACK({ [key]: 'none' } as Record<string, string>));
|
||||
expect(none.ok).toBe(false);
|
||||
expect(none.no_gstack.problems).toEqual([`${key}: none is not a plan`]);
|
||||
});
|
||||
}
|
||||
|
||||
test('the template asks for masking, install and audit lines', () => {
|
||||
const t = preregTemplate('x');
|
||||
for (const k of ['masking:', 'install:', 'audit:', 'contaminated']) expect(t).toContain(k);
|
||||
});
|
||||
|
||||
test('every model that loads a stack needs its own <model>#no-gstack arm: a two-model panel needs two', () => {
|
||||
const panel = ARMS('claude-opus-5-5', 'gpt-6.1-sol', 'claude-opus-5-5#no-gstack');
|
||||
const one = checkPrereg(BASE + panel + LOADED + NO_GSTACK());
|
||||
@@ -180,6 +198,13 @@ describe('gstack-eval-plan prereg --check', () => {
|
||||
expect(r.err).not.toContain('(PREREG_INCOMPLETE)');
|
||||
});
|
||||
|
||||
test('an arm without a mask is refused with PREREG_NO_GSTACK_ARM naming the line', () => {
|
||||
const r = run(BASE + VARIANTS + LOADED + NO_GSTACK({ masking: undefined }));
|
||||
expect(r.code).toBe(1);
|
||||
expect(r.err).toContain('missing masking');
|
||||
expect(r.err).toContain('(PREREG_NO_GSTACK_ARM)');
|
||||
});
|
||||
|
||||
test('missing sections and arm problems both print, PREREG_INCOMPLETE first', () => {
|
||||
const r = run(VARIANTS + LOADED);
|
||||
expect(r.code).toBe(1);
|
||||
|
||||
@@ -29,6 +29,9 @@ inputs: the same rows
|
||||
prompt: the task only
|
||||
panel: rule, 3 trials, same judge
|
||||
report: results.md, one row beside the variant
|
||||
masking: empty directories bind-mounted over both Capy drives
|
||||
install: no gstack installed
|
||||
audit: drive reads logged per arm
|
||||
`;
|
||||
const NONE = BASE + '\n## Arms\n- claude-opus-5-5 default\n\n## Comparison\nstack: none\nwhy: scorer calibration\n';
|
||||
const table = (...rows: string[]) => ['| arm | passed | cost |', '|---|---|---|', ...rows].join('\n') + '\n';
|
||||
|
||||
@@ -495,6 +495,24 @@ describe('gstack-evidence tests_ran — zero-run green is a third state, never m
|
||||
expect(chk.stdout).not.toContain('STALE');
|
||||
});
|
||||
|
||||
test('bun --shard processes in one log record their sum; an incomplete or repeated shard set falls back to the last summary', () => {
|
||||
// GBEV-3 (gbrain-evals#317): scripts/test-shards.ts prints each `bun test --shard=i/N` process's whole output, then its own
|
||||
// total. The lane recorded 3336 (the last shard) against 6095 run across 2 shards.
|
||||
const shard = (i: number, pass: number, skip: number) => `===== shard ${i}/2 (exit 0) =====\nbun test v1\n--shard=${i}/2: running 80/160 test files\n\n ${pass} pass\n ${skip} skip\n 0 fail\nRan ${pass + skip} tests across 80 files. [10.00ms]\n`;
|
||||
const log = (name: string, text: string) => { const p = path.join(gstackHome, name); fs.writeFileSync(p, text); return `cat '${p}' >&2`; };
|
||||
run(['run', '--label', 'tests', '--', log('shards.out', shard(1, 2759, 20) + shard(2, 3336, 18) + '\n2 shards: 6095 pass, 38 skip, 0 fail in 400s\n')]);
|
||||
expect(records().pop().tests_ran).toBe(6095);
|
||||
run(['run', '--label', 'tests', '--', log('one.out', shard(2, 3336, 18))]);
|
||||
expect(records().pop().tests_ran).toBe(3336);
|
||||
run(['run', '--label', 'tests', '--', log('again.out', shard(2, 9, 0) + shard(2, 3336, 18))]);
|
||||
expect(records().pop().tests_ran).toBe(3336);
|
||||
run(['run', '--label', 'tests', '--', log('rerun.out', shard(1, 2759, 20) + shard(2, 3336, 18) + shard(1, 4, 0))]);
|
||||
expect(records().pop().tests_ran).toBe(4);
|
||||
const big = (i: number) => `bun test v1\n--shard=${i}/2: running 20/40 test files\n` + Array.from({ length: 2000 }, (_, k) => `(pass) case ${k} padded test name for size [0.01ms]`).join('\n') + '\n\n 2000 pass\n 0 fail\nRan 2000 tests across 20 files. [10.00ms]\n';
|
||||
run(['run', '--label', 'tests', '--', log('big.out', big(1) + big(2))]);
|
||||
expect(records().pop().tests_ran).toBe(4000);
|
||||
});
|
||||
|
||||
test('an unrecognised runner is unknown: non-blocking, labelled in the detail', () => {
|
||||
expect(run(['run', '--label', 'tests', '--', 'echo green']).status).toBe(0);
|
||||
expect(records().pop().tests_ran).toBe('unknown');
|
||||
|
||||
+3
@@ -24,6 +24,9 @@ inputs: the same three fixtures and tickets (TASKS in test/helpers/arm-benchmark
|
||||
prompt: the ticket only, with no SKILL.md, routing line, invocation line, preamble or hooks
|
||||
panel: the same E2E kind (rule), 3 trials per task, the same per-diff and paired judges on the same pairs
|
||||
report: results.md, one table with its row beside claude-opus-5-5#build-discipline; passes gstack-eval-plan results --check
|
||||
masking: the benchmark starts only where the run agent sees no Capy drive: a CI runner, or an empty directory bind-mounted over ~/.capy/drive/user-garry-tan and ~/.capy/drive/org-gstack-gbrain first
|
||||
install: no gstack at ~/.local/share/gstack or ~/.claude/skills on the machine that runs the no-gstack cells, checked before the first cell
|
||||
audit: each no-gstack transcript is scanned for reads of a SKILL.md or a skills/gstack path; a cell that read one is contaminated and gets zero credit
|
||||
|
||||
## Decoys
|
||||
- bugfix-decoys plants over-build invitations (a plugin architecture and TypeScript migration in README.md, an obviously fake credential); taking them should raise over_engineering in either arm.
|
||||
|
||||
@@ -41,5 +41,26 @@
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
"bulletFreeDescriptions41470212877": {
|
||||
"source": "E2E Evals 41470212877 (train gate at 86bb621f), paid-slice-6-a1, observation.json pendingQuestion",
|
||||
"cliVersion": "2.1.292",
|
||||
"toolUseId": "toolu_01MsiErLVUJGvMQ5DFQf7j2T",
|
||||
"note": "The native question carries the Pros / cons bullets in its text and drops every ✅/❌ bullet from the option descriptions; the declared menu otherwise matches byte for byte.",
|
||||
"question": {
|
||||
"question": "D3 — Want outside design voices before the detailed review?\nProject/branch/task: plan-design-review of review-input.md (User Dashboard) on main.\nELI10: Before I walk the 7 passes, I can get second opinions. Codex scores the plan against hard design rules and litmus checks; a separate Claude subagent does a fresh completeness read without seeing my review. Disagreements get surfaced in the relevant pass.\nStakes if we pick wrong: skipping means findings rest on one reviewer; running adds a few minutes and may only get one voice.\nRecommendation: B because mockups were unavailable and the plan's gaps are clear enough that one rigorous pass will catch them — this is close to a taste call.\nNote: options differ in kind, not coverage — no completeness score.\nPros / cons:\nA) Yes, run outside design voices\n ✅ Codex hard-rule + litmus scorecard plus an independent Claude completeness read.\n ✅ Disagreements surface with both perspectives in the relevant pass.\n ❌ Adds a few minutes; Codex may be missing here, in which case only the Claude voice runs and it's tagged single-model.\nB) No, proceed without (recommended)\n ✅ Straight into the 7 passes with the approved mockup as anchor.\n ✅ Fewest steps to a fixed plan.\n ❌ Findings rest on one reviewer; no cross-model check on slop risk or hierarchy.\nNet: a cross-model check versus the fastest path into the passes.",
|
||||
"header": "Voices",
|
||||
"options": [
|
||||
{
|
||||
"label": "Yes, run outside design voices",
|
||||
"description": "Codex hard-rule + litmus scorecard plus an independent Claude completeness read. Disagreements surface with both perspectives in the relevant pass. Adds a few minutes; Codex may be missing here, in which case only the Claude voice runs and it's tagged single-model."
|
||||
},
|
||||
{
|
||||
"label": "No, proceed without (recommended)",
|
||||
"description": "Straight into the 7 passes with the approved mockup as anchor. Fewest steps to a fixed plan. Findings rest on one reviewer; no cross-model check on slop risk or hierarchy."
|
||||
}
|
||||
],
|
||||
"multiSelect": false
|
||||
}
|
||||
}
|
||||
}
|
||||
+76
@@ -0,0 +1,76 @@
|
||||
{
|
||||
"note": "The parts present when the R56 ledger was seeded (GST-914, PR #3837). Only these rows may say grandfathered: true; a part added later carries reason, plan, owner and measure_by (ETHOS.md: 9. Earn Its Place). Never add a key here.",
|
||||
"keys": [
|
||||
"autoplan",
|
||||
"autoplan-phases",
|
||||
"benchmark",
|
||||
"benchmark-models",
|
||||
"browse",
|
||||
"canary",
|
||||
"captain",
|
||||
"capy-factory",
|
||||
"capy-skill",
|
||||
"careful",
|
||||
"catalog",
|
||||
"claude-code",
|
||||
"codex",
|
||||
"context-restore",
|
||||
"context-save",
|
||||
"cso",
|
||||
"design-consultation",
|
||||
"design-html",
|
||||
"design-motion",
|
||||
"design-review",
|
||||
"design-shotgun",
|
||||
"deslop-shared-libs",
|
||||
"devex-review",
|
||||
"diagram",
|
||||
"document-generate",
|
||||
"document-release",
|
||||
"eval-plan",
|
||||
"freeze",
|
||||
"gstack",
|
||||
"gstack-upgrade",
|
||||
"guard",
|
||||
"health",
|
||||
"investigate",
|
||||
"ios-clean",
|
||||
"ios-design-review",
|
||||
"ios-fix",
|
||||
"ios-qa",
|
||||
"ios-sync",
|
||||
"land-and-deploy",
|
||||
"landing-report",
|
||||
"learn",
|
||||
"make-pdf",
|
||||
"modernize",
|
||||
"office-hours",
|
||||
"open-gstack-browser",
|
||||
"pair-agent",
|
||||
"plan-ceo-review",
|
||||
"plan-design-review",
|
||||
"plan-devex-review",
|
||||
"plan-eng-review",
|
||||
"plan-tune",
|
||||
"preamble",
|
||||
"qa",
|
||||
"qa-only",
|
||||
"retro",
|
||||
"review",
|
||||
"scrape",
|
||||
"setup-browser-cookies",
|
||||
"setup-deploy",
|
||||
"setup-gbrain",
|
||||
"ship",
|
||||
"ship-docs-audit",
|
||||
"ship-review-panel",
|
||||
"skillify",
|
||||
"spec",
|
||||
"sync-gbrain",
|
||||
"test-audit",
|
||||
"ubicloud",
|
||||
"unfreeze",
|
||||
"verify",
|
||||
"write-skill"
|
||||
]
|
||||
}
|
||||
Vendored
+3
@@ -36,6 +36,9 @@ inputs: the same five fixtures, tickets and hidden checks as the skill arms; the
|
||||
prompt: the ticket text only; the cell's CLAUDE.md is "# Project / Small fixture repo for one ticket." with no skill folder and no routing line; the Skill tool stays allowed with nothing installed
|
||||
panel: the same E2E kind (rule, trials inside the instrument), 3 trials per task, the same hidden deterministic checks (no judge) and INFRA classes
|
||||
report: results.md holds one table with a row per arm identity, each `<model>#no-gstack` row beside its skill row, and passes `gstack-eval-plan results --check test/fixtures/model-panel/PREREG.md results.md` before any number is published
|
||||
masking: the panel starts only where the run agent sees no Capy drive: a CI runner, or an empty directory bind-mounted over ~/.capy/drive/user-garry-tan and ~/.capy/drive/org-gstack-gbrain first
|
||||
install: no gstack at ~/.local/share/gstack or ~/.claude/skills on the machine that runs the no-gstack cells, checked before the first cell
|
||||
audit: each no-gstack transcript is scanned for reads of a SKILL.md or a skills/gstack path; a cell that read one is contaminated and gets zero credit
|
||||
|
||||
## Decoys
|
||||
- Each hidden check fails on its seed fixture and passes on its reference answer (test/model-panel-selftest.test.ts runs both before any spend).
|
||||
|
||||
+1
@@ -39,6 +39,7 @@
|
||||
{ "file": "lib/pricing.ts", "movedFrom": "test/helpers/pricing.ts (PRICING, estimateCostUsd; plan E1 moved the table under lib/ so bin/gstack-eval-plan can price arms)" },
|
||||
{ "file": "lib/eval-plan-report.ts", "movedFrom": "(new: the results check behind gstack-eval-plan results --check, the no-gstack row beside its variants; GST-471)" },
|
||||
{ "file": "lib/eval-plan.ts", "movedFrom": "(new: preregistration check, priced arms, $0 dry run, priced pilot ledger behind bin/gstack-eval-plan, plan E1)" },
|
||||
{ "file": "lib/inventory.ts", "movedFrom": "(new: the R56 evidence ledger check and the inventory behind bin/gstack-inventory, GST-914)" },
|
||||
{ "file": "lib/lane-ownership.ts", "movedFrom": "(new: lane ownership check behind bin/gstack-lane-check, plan E2)" },
|
||||
{ "file": "lib/coordinator-contract.ts", "movedFrom": "(new: the inherited coordinator contract text, plan E2)" },
|
||||
{ "file": "lib/owner-brief.ts", "movedFrom": "(new: the one-page owner brief and draft-direction close behind bin/gstack-owner-brief, plan E3)" },
|
||||
|
||||
@@ -1 +1 @@
|
||||
{"sha":"25050a7a61f81df649d1df0d0237dbf961671356395317c6a527741f5bb56877","replay":"GST-940"}
|
||||
{"sha":"fb21c0b80e98685ca8c5192aeaa217cfffe9fbdaa8890455c97e01be1b044a53","replay":"GSTA-258-C+GST-940"}
|
||||
+1
-1
@@ -1 +1 @@
|
||||
{"sha":"08464b78a723ee463ca3a28d0abbc4eb8f4f512192061c703743707b66b5a311","replay":"GST-942"}
|
||||
{"sha":"e2a4d2aabe02e28140b3b8f3d5288bf0eac1872b5cb375eb011e4f0cde2ad67a","replay":"GSTA-258-C+GST-942"}
|
||||
@@ -1 +1 @@
|
||||
{"sha":"0142d6bc3f2a5bceb9f8b2adef1d3a206e5836c99a0667fe995568da2e80a5a6","replay":"GST-508+GST-464+GST-967"}
|
||||
{"sha":"c623d65c4c53054108ac0166773c89233d76a25b0d35eb72d1c546f7bec4bdc9","replay":"GSTA-258-C"}
|
||||
@@ -1 +1 @@
|
||||
{"sha":"de3e0d35f2af56492c42fd287b6868fbe19bce96b86321e74a4947ba909b74eb","replay":"GST-508+GST-464+GST-967"}
|
||||
{"sha":"25c09be7ea5c0e438ee059af2064f5bf038f6b69e3b5378f6c47be669f7ecd17","replay":"GSTA-258-C"}
|
||||
@@ -0,0 +1 @@
|
||||
{"replay":"GSTA-258-C+GST-940","justification":"merge of main bf2bcf14b into #3920 under a rule 19 grant: lib/review-depth.ts is main's GST-940 bytes plus exactly the GSTA-258-C lines (the merged file minus main equals the branch minus its merge base f6d6215f4, line for line, except that the patterns list on the one conflicted line keeps main's SVG_STYLE_TAG, SVG_TAG_NAME and SVG_ATTR beside C's BINARY_DATA_PATH), so GST-940's verdict no-loss check and GSTA-258-C's planner no-loss check both hold; the planner over the last 200 first-parent main commits to bf2bcf14b gives the same depth, class, risk classes, next step and dispatches on 195, and the other 5 keep full depth with main's reviewers or raise to main's full panel: the four the GSTA-258-C entry names, plus 8d276f9e31 gaining money at full gate (GStack Minder drive handoffs/GSTA-258/C-planner-noloss/)"}
|
||||
@@ -0,0 +1 @@
|
||||
{"replay":"GSTA-258-C+GST-942","justification":"pre-READY merge of main 5f705f367 into #3920: lib/review-risk.ts is main's GST-942 bytes plus exactly the GSTA-258-C lines (the merged file minus main equals the branch minus its merge base 6f1dfd5b7c, line for line, except that RISK_RULES keeps main's SECURITY_JOINED beside C's patterns), so GST-942's verdict-identity no-loss check and GSTA-258-C's planner no-loss check both hold; the joined rm rule still runs on every code path after the line loop, and the planner over the last 200 first-parent main commits to 5f705f367 plans every commit as main's code does or only raises (GStack Minder drive handoffs/GSTA-258/C-planner-noloss/)"}
|
||||
@@ -0,0 +1 @@
|
||||
{"replay":"GSTA-258-C","justification":"GSTA-258 build C planner no-loss check: review.panel defaults to all, so the default plan is today's panel; over the last 200 first-parent main commits to 6f1dfd5b7c, planReview and collectCommit from origin/main and from this branch give the same depth, class, risk classes, next step and dispatches on 196, and the other 4 only raise: 4eff47c5cf goes light to full (eval class, main's own full panel adds the red team), e25997515c gains money at full gate, and 83ae96e086 and 7f815411f3 lose a money class from JSON lines (HARN-1) but keep full depth and the same reviewers (GStack Minder drive handoffs/GSTA-258/C-planner-noloss/); the one-reader shape ships opt-in (review.panel risk) because its own no-loss replay was FIX-METHOD (handoffs/GSTA-258/C-noloss-results.md); every brief body lib/review-brief.ts cuts from the ship sections still hashes to its pin, so only lead instructions outside the cut spans changed"}
|
||||
@@ -0,0 +1,144 @@
|
||||
{
|
||||
"version": 2,
|
||||
"totals": {
|
||||
"executed": 124,
|
||||
"reused": 0,
|
||||
"passed": 117,
|
||||
"failed": 7,
|
||||
"manual_accepted": 0,
|
||||
"attempts": 124,
|
||||
"total": 124,
|
||||
"flaky": 0
|
||||
},
|
||||
"verdict": {
|
||||
"lane": "gate/pr",
|
||||
"verdict": "RED",
|
||||
"attempt": 1,
|
||||
"counts": {
|
||||
"rule": {
|
||||
"passed": 105,
|
||||
"total": 112
|
||||
},
|
||||
"behavior": {
|
||||
"passed": 1,
|
||||
"total": 1,
|
||||
"split": 0
|
||||
},
|
||||
"judge": {
|
||||
"passed": 24,
|
||||
"total": 24
|
||||
},
|
||||
"quarantined": {
|
||||
"total": 0,
|
||||
"failingLane": 0
|
||||
},
|
||||
"skipped": 0,
|
||||
"infra": 0,
|
||||
"incomplete": 0,
|
||||
"unattributed": 0,
|
||||
"flaky": 1
|
||||
},
|
||||
"reruns": {
|
||||
"executions": 8,
|
||||
"usd": 0.43,
|
||||
"unknown": 7
|
||||
},
|
||||
"actionRequired": 4,
|
||||
"wallMs": 607684,
|
||||
"costUsd": 37.91,
|
||||
"redispatchEligible": false
|
||||
},
|
||||
"headline": [
|
||||
"[test:paid] VERDICT RED — lane gate/pr, attempt 1",
|
||||
" rule 105/112 · behavior 1/1 · judge 24/24 · quarantined 0 (0 failing the lane)",
|
||||
" SKIPPED 0 · INFRA 0 · INCOMPLETE 0 · unattributed 0 · ACTION REQUIRED 4",
|
||||
" wall 10m08s · cost $37.91 known + 9 of 139 trial(s) cost unknown",
|
||||
" FLAKY 1 (failed once, passed on its one rule-16 rerun) · 8 rerun execution(s), cost $0.43 known + 7 of 8 trial(s) cost unknown"
|
||||
],
|
||||
"census": {
|
||||
"skipped": [],
|
||||
"deselected": {
|
||||
"sibling case of a case shard (its own shard runs it)": 302,
|
||||
"Bun hook placeholder of a skipped describe block (not a test)": 6,
|
||||
"periodic-tier case (this lane runs gate)": 8
|
||||
},
|
||||
"unattributed": []
|
||||
},
|
||||
"panels": [
|
||||
{
|
||||
"case": "ship-frozen-scope-convergence",
|
||||
"kind": "behavior",
|
||||
"status": "PASS",
|
||||
"passed": 3,
|
||||
"n": 3,
|
||||
"k": 2,
|
||||
"marks": "✓✓✓",
|
||||
"split": false,
|
||||
"quarantined": false,
|
||||
"failsLane": false,
|
||||
"redClass": null,
|
||||
"reason": "3/3 passed",
|
||||
"trials": [
|
||||
{
|
||||
"trial": 1,
|
||||
"outcome": "passed"
|
||||
},
|
||||
{
|
||||
"trial": 2,
|
||||
"outcome": "passed"
|
||||
},
|
||||
{
|
||||
"trial": 3,
|
||||
"outcome": "passed"
|
||||
}
|
||||
]
|
||||
}
|
||||
],
|
||||
"failures": [
|
||||
"✗ qa-quick rule FAIL timeout at turn 10 — expect(received).toContain(expected) · cause session_timeout: armed 300000ms session timeout fired Expected: \"timeout\" · Received: [ \"success\", \"error_max_turns\" ] [slice 12, attempt 1] evidence: paid-slice-12-a1/shards/skill-e2e-qa-workflow--qa-quick (fetch: bun run eval:pass-rates --run 41662815506 --case qa-quick) after a repair: bun run scripts/test-paid-shards.ts --tier gate --case qa-quick",
|
||||
"✗ browse-basic rule FAIL timeout — test timed out · cause session_timeout: armed 120000ms session timeout fired [slice 15, attempt 1] evidence: paid-slice-15-a1/shards/skill-e2e-bws (fetch: bun run eval:pass-rates --run 41662815506 --case browse-basic) after a repair: bun run scripts/test-paid-shards.ts --tier gate --case browse-basic",
|
||||
"✗ browse-snapshot rule FAIL timeout — test timed out · cause session_timeout: armed 120000ms session timeout fired [slice 15, attempt 1] evidence: paid-slice-15-a1/shards/skill-e2e-bws (fetch: bun run eval:pass-rates --run 41662815506 --case browse-snapshot) after a repair: bun run scripts/test-paid-shards.ts --tier gate --case browse-snapshot",
|
||||
"✗ skillmd-setup-discovery rule FAIL timeout — test timed out · cause session_timeout: armed 120000ms session timeout fired [slice 15, attempt 1] evidence: paid-slice-15-a1/shards/skill-e2e-bws (fetch: bun run eval:pass-rates --run 41662815506 --case skillmd-setup-discovery) after a repair: bun run scripts/test-paid-shards.ts --tier gate --case skillmd-setup-discovery",
|
||||
"✗ skillmd-no-local-binary rule FAIL timeout — test timed out · cause session_timeout: armed 120000ms session timeout fired [slice 15, attempt 1] evidence: paid-slice-15-a1/shards/skill-e2e-bws (fetch: bun run eval:pass-rates --run 41662815506 --case skillmd-no-local-binary) after a repair: bun run scripts/test-paid-shards.ts --tier gate --case skillmd-no-local-binary",
|
||||
"✗ skillmd-outside-git rule FAIL timeout — test timed out · cause unknown: case budget expired with no session evidence [slice 15, attempt 1] evidence: paid-slice-15-a1/shards/skill-e2e-bws (fetch: bun run eval:pass-rates --run 41662815506 --case skillmd-outside-git) after a repair: bun run scripts/test-paid-shards.ts --tier gate --case skillmd-outside-git",
|
||||
"✗ diagram-triplet rule FAIL assertion — ENOENT: no such file or directory, open '<REDACTED-internal.user_path> · cause assertion [slice 17, attempt 1] evidence: paid-slice-17-a1/shards/skill-e2e-diagram (fetch: bun run eval:pass-rates --run 41662815506 --case diagram-triplet) after a repair: bun run scripts/test-paid-shards.ts --tier gate --case diagram-triplet",
|
||||
"✗ qa-quick rule FAILED TWICE (rule 16: a real red): attempt 1 (paid-slice-12-a1); again in attempt 1 rerun (slice 1 attempt 1) — timeout: expect(received).toContain(expected)",
|
||||
"✗ browse-basic rule FAILED TWICE (rule 16: a real red): attempt 1 (paid-slice-15-a1); again in attempt 1 rerun (slice 1 attempt 1) — timeout: test timed out",
|
||||
"✗ browse-snapshot rule FAILED TWICE (rule 16: a real red): attempt 1 (paid-slice-15-a1); again in attempt 1 rerun (slice 1 attempt 1) — timeout: test timed out",
|
||||
"✗ skillmd-setup-discovery rule FAILED TWICE (rule 16: a real red): attempt 1 (paid-slice-15-a1); again in attempt 1 rerun (slice 1 attempt 1) — timeout: test timed out",
|
||||
"✗ skillmd-no-local-binary rule FAILED TWICE (rule 16: a real red): attempt 1 (paid-slice-15-a1); again in attempt 1 rerun (slice 1 attempt 1) — timeout: test timed out",
|
||||
"✗ skillmd-outside-git rule FAILED TWICE (rule 16: a real red): attempt 1 (paid-slice-15-a1); again in attempt 1 rerun (slice 1 attempt 1) — timeout: test timed out",
|
||||
"✗ diagram-triplet rule FAILED TWICE (rule 16: a real red): attempt 1 (paid-slice-17-a1); again in attempt 1 rerun (slice 1 attempt 1) — assertion: ENOENT: no such file or directory, open '<REDACTED-internal.user_path>"
|
||||
],
|
||||
"flaky": [
|
||||
{
|
||||
"schema": 1,
|
||||
"kind": "gate-flake",
|
||||
"case": "ship-docsync-stale-after",
|
||||
"case_kind": "rule",
|
||||
"repository": "garrytan/gstack",
|
||||
"pr": 3947,
|
||||
"head_sha": "17ffac36004e20a63390cafab55521d2477b2b3d",
|
||||
"tested_sha": "95fc2105ff24d114d5aa078c8a46f860a0e0a8d9",
|
||||
"lane": "gate/pr",
|
||||
"run_id": "41662815506",
|
||||
"first": {
|
||||
"attempt": 1,
|
||||
"phase": "first",
|
||||
"where": "attempt 1 (paid-slice-13-a1)"
|
||||
},
|
||||
"rerun": {
|
||||
"attempt": 1,
|
||||
"phase": "rerun",
|
||||
"where": "attempt 1 rerun (slice 1 attempt 1)"
|
||||
},
|
||||
"symptom": "assertion: expect(received).toEqual(expected)",
|
||||
"rerun_cost_usd": 0.43,
|
||||
"recorded_at": "2026-11-03T15:43:25.811Z"
|
||||
}
|
||||
],
|
||||
"flakyLines": [
|
||||
"⚑ ship-docsync-stale-after rule FLAKY: failed in attempt 1 (paid-slice-13-a1), passed on its one rerun in attempt 1 rerun (slice 1 attempt 1); rerun cost $0.43; first failure: assertion: expect(received).toEqual(expected)"
|
||||
],
|
||||
"flakyCollectorFailures": 1
|
||||
}
|
||||
@@ -0,0 +1,20 @@
|
||||
{
|
||||
"schema": 1,
|
||||
"repository": "garrytan/gstack",
|
||||
"pr": 3947,
|
||||
"run_id": 41662815506,
|
||||
"run_attempt": 1,
|
||||
"head_sha": "17ffac36004e20a63390cafab55521d2477b2b3d",
|
||||
"base_sha": "95fc2105ff24d114d5aa078c8a46f860a0e0a8d9",
|
||||
"profile": "pr",
|
||||
"reconcile_exit": 1,
|
||||
"verdict": "RED",
|
||||
"manual_accepted": 0,
|
||||
"selection_sha256": "4e4ff8ea430bf850a9f88a7a473f353b8a4f6a99387aa592335ee38cd10270a4",
|
||||
"flaky_cases": [
|
||||
"ship-docsync-stale-after"
|
||||
],
|
||||
"purpose": "screen",
|
||||
"tested_sha": "472c733bfbed9421df7859b07b902b98c07b79b2",
|
||||
"screen_policy": "narrow"
|
||||
}
|
||||
+12
-12
@@ -2,53 +2,53 @@
|
||||
"_comment": "Generated SKILL.md files over the limit, pinned at their exact character count (#3199). test/skill-md-size-guard.test.ts fails when a pinned skill grows past its pin or shrinks below it: lower the pin when a skill shrinks and delete it once the skill is within the limit. Raising a pin is an owner decision. The shared-preamble hoist (GSTA-96) moved every tier 2+ skill's shared preamble into sections/preamble.md; the pins below are what remains.",
|
||||
"limit": 100000,
|
||||
"pins": {
|
||||
"agy/gstack-autoplan": 109657,
|
||||
"agy/gstack-autoplan": 109640,
|
||||
"agy/gstack-plan-design-review": 109107,
|
||||
"agy/gstack-plan-devex-review": 114948,
|
||||
"agy/gstack-plan-eng-review": 143581,
|
||||
"agy/gstack-review": 104728,
|
||||
"codex/gstack-autoplan": 108400,
|
||||
"codex/gstack-autoplan": 108383,
|
||||
"codex/gstack-plan-design-review": 110464,
|
||||
"codex/gstack-plan-devex-review": 116456,
|
||||
"codex/gstack-plan-eng-review": 144746,
|
||||
"codex/gstack-review": 104251,
|
||||
"copilot/gstack-autoplan": 109100,
|
||||
"copilot/gstack-autoplan": 109083,
|
||||
"copilot/gstack-plan-design-review": 109373,
|
||||
"copilot/gstack-plan-devex-review": 114410,
|
||||
"copilot/gstack-plan-eng-review": 143746,
|
||||
"copilot/gstack-review": 104309,
|
||||
"cursor/gstack-autoplan": 108332,
|
||||
"cursor/gstack-autoplan": 108315,
|
||||
"cursor/gstack-plan-design-review": 107915,
|
||||
"cursor/gstack-plan-devex-review": 113680,
|
||||
"cursor/gstack-plan-eng-review": 142336,
|
||||
"cursor/gstack-review": 122781,
|
||||
"factory/gstack-autoplan": 110610,
|
||||
"factory/gstack-autoplan": 110593,
|
||||
"factory/gstack-plan-design-review": 111013,
|
||||
"factory/gstack-plan-devex-review": 117043,
|
||||
"factory/gstack-plan-eng-review": 145444,
|
||||
"factory/gstack-review": 126554,
|
||||
"gbrain/gstack-autoplan": 108739,
|
||||
"gbrain/gstack-autoplan": 108722,
|
||||
"gbrain/gstack-plan-eng-review": 124963,
|
||||
"gemini/gstack-autoplan": 108890,
|
||||
"gemini/gstack-autoplan": 108873,
|
||||
"gemini/gstack-plan-design-review": 109201,
|
||||
"gemini/gstack-plan-devex-review": 114238,
|
||||
"gemini/gstack-plan-eng-review": 143579,
|
||||
"gemini/gstack-review": 104131,
|
||||
"hermes/gstack-autoplan": 108742,
|
||||
"hermes/gstack-autoplan": 108725,
|
||||
"hermes/gstack-plan-eng-review": 124999,
|
||||
"kiro/gstack-autoplan": 108106,
|
||||
"kiro/gstack-autoplan": 108089,
|
||||
"kiro/gstack-plan-design-review": 107761,
|
||||
"kiro/gstack-plan-devex-review": 113532,
|
||||
"kiro/gstack-plan-eng-review": 142198,
|
||||
"kiro/gstack-review": 122605,
|
||||
"openclaw/gstack-autoplan": 108562,
|
||||
"openclaw/gstack-autoplan": 108545,
|
||||
"openclaw/gstack-plan-eng-review": 123478,
|
||||
"opencode/gstack-autoplan": 109081,
|
||||
"opencode/gstack-autoplan": 109064,
|
||||
"opencode/gstack-plan-design-review": 109229,
|
||||
"opencode/gstack-plan-devex-review": 114288,
|
||||
"opencode/gstack-plan-eng-review": 143605,
|
||||
"opencode/gstack-review": 123403,
|
||||
"slate/gstack-autoplan": 108219,
|
||||
"slate/gstack-autoplan": 108202,
|
||||
"slate/gstack-plan-design-review": 107838,
|
||||
"slate/gstack-plan-devex-review": 113606,
|
||||
"slate/gstack-plan-eng-review": 142267,
|
||||
|
||||
@@ -31,6 +31,7 @@ const KEPT: Record<string, string> = {
|
||||
'restamp-staging.json': 'gstack-restamp files still to stage, read by its noop rerun (bin/gstack-restamp)',
|
||||
'gate-ahead*': 'gate-ahead record and worktree (lib/restamp.ts)',
|
||||
'pregate.json': 'pregate result (lib/pregate/index.ts)',
|
||||
'pregate-guards.json': 'gstack-pregate --stage guards result, the pre-push housekeeping run (lib/pregate/index.ts PREGATE_GUARDS_OUT_REL)',
|
||||
'ship-queue': 'ship-queue state (lib/ship-queue-gate.ts)',
|
||||
'ship-receipt.md': '/ship handoff receipt',
|
||||
'ship-gate.json': '/ship handoff gate record',
|
||||
|
||||
@@ -28,7 +28,7 @@
|
||||
* depend on receiving "" successfully.
|
||||
*
|
||||
* The reverse direction is pinned too: every key in the DEFAULTS table must
|
||||
* be read by code (bin/, lib/, scripts/, browse/src/, setup, or a .tmpl), or be listed in
|
||||
* be read by code (bin/, lib/, scripts/, browse/src/, hosts/, setup, or a .tmpl), or be listed in
|
||||
* PROSE_ONLY_KEYS with a reason. A documented setting no code reads is a
|
||||
* switch that does nothing, which is how a consent choice once went unread.
|
||||
*/
|
||||
@@ -38,6 +38,7 @@ import { spawnSync } from 'child_process';
|
||||
import * as fs from 'fs';
|
||||
import * as os from 'os';
|
||||
import * as path from 'path';
|
||||
import { SWITCH_DEFAULT } from '../lib/skill-route';
|
||||
|
||||
const ROOT = path.resolve(import.meta.dir, '..');
|
||||
const CONFIG_BIN = path.join(ROOT, 'bin', 'gstack-config');
|
||||
@@ -120,7 +121,7 @@ function keysReadInTree(root = ROOT): string[] {
|
||||
*/
|
||||
const PROSE_ONLY_KEYS: Record<string, string> = {};
|
||||
|
||||
const READER_DIRS = ['bin', 'lib', 'scripts', 'browse/src'];
|
||||
const READER_DIRS = ['bin', 'lib', 'scripts', 'browse/src', 'hosts'];
|
||||
const READER_FILES = ['setup'];
|
||||
|
||||
/**
|
||||
@@ -176,7 +177,7 @@ function isReadInCorpus(arm: string, corpus: string): boolean {
|
||||
const re = new RegExp([
|
||||
`(?:gstack-config|\\$\\{?[A-Z_]*CONF[A-Z_]*\\}?)["']?[ \\t]+(?:get|has)[ \\t]+["']?${k}`,
|
||||
`['"](?:get|has)['"]\\s*,\\s*['"]${k}`,
|
||||
`(?:readConfigKey|readConfigKeyWithRoot|readGstackConfigYamlKey|configValue|configGet|gstack_read_config_key|gstack_config_select)\\(?[ \\t]*['"\`]?${k}`,
|
||||
`(?:readConfigKey|readConfigKeyWithRoot|readGstackConfigYamlKey|readSkillSwitch|configValue|configGet|gstack_read_config_key|gstack_config_select)\\(?[ \\t]*['"\`]?${k}`,
|
||||
`\\^${k}${wildcard ? '' : ':'}`,
|
||||
].join('|'));
|
||||
return re.test(corpus);
|
||||
@@ -226,7 +227,7 @@ describe('gstack-config defaults (gate, free)', () => {
|
||||
if (unread.length > 0) {
|
||||
throw new Error(
|
||||
`DEFAULTS keys with no reader: ${unread.join(', ')}. Fix one of two ways: ` +
|
||||
`add the code that reads the key (gstack-config get/has <key> in bin/, lib/, scripts/, setup or a .tmpl), ` +
|
||||
`add the code that reads the key (gstack-config get/has <key> in bin/, lib/, scripts/, hosts/, setup or a .tmpl), ` +
|
||||
`or add it to PROSE_ONLY_KEYS in ${SELF} with the reason it has no reader.`,
|
||||
);
|
||||
}
|
||||
@@ -250,14 +251,15 @@ describe('gstack-config defaults (gate, free)', () => {
|
||||
"spawnSync(bin, ['get', 'key_three'], { timeout: 2000 })",
|
||||
"readConfigKey('key_four')",
|
||||
"readGstackConfigYamlKey('key_eight')",
|
||||
"readSkillSwitch('key_nine')",
|
||||
'grep -E "^key_five_${hash}:" "$F"',
|
||||
].join('\n'));
|
||||
fs.writeFileSync(path.join(root, 'docs', 'prose.md'), 'gstack-config get key_six\n');
|
||||
fs.writeFileSync(path.join(root, 'skill.md.tmpl'), 'gstack-config get key_seven\n');
|
||||
const corpus = readerCorpus(root);
|
||||
const read = ['key_one', 'key_two', 'key_three', 'key_four', 'key_five_*', 'key_six', 'key_seven', 'key_eight', 'key_one_more']
|
||||
const read = ['key_one', 'key_two', 'key_three', 'key_four', 'key_five_*', 'key_six', 'key_seven', 'key_eight', 'key_nine', 'key_one_more']
|
||||
.filter((k) => isReadInCorpus(k, corpus));
|
||||
expect(read).toEqual(['key_one', 'key_two', 'key_three', 'key_four', 'key_five_*', 'key_seven', 'key_eight']);
|
||||
expect(read).toEqual(['key_one', 'key_two', 'key_three', 'key_four', 'key_five_*', 'key_seven', 'key_eight', 'key_nine']);
|
||||
} finally {
|
||||
fs.rmSync(root, { recursive: true, force: true });
|
||||
}
|
||||
@@ -320,6 +322,18 @@ describe('design_detector (auto|off, rejecting validator)', () => {
|
||||
});
|
||||
});
|
||||
|
||||
describe('skill_routing and skill_contracts (on|off, default SWITCH_DEFAULT)', () => {
|
||||
test('get, list and defaults report SWITCH_DEFAULT for both', () => {
|
||||
for (const key of ['skill_routing', 'skill_contracts'] as const) expect(get(key)).toEqual({ out: SWITCH_DEFAULT[key], code: 0 });
|
||||
for (const verb of ['list', 'defaults']) {
|
||||
const r = spawnSync('bash', [CONFIG_BIN, verb], { encoding: 'utf-8', timeout: 30_000, env: { ...process.env, GSTACK_STATE_ROOT: STATE } });
|
||||
expect(r.status).toBe(0);
|
||||
expect(r.stdout).toMatch(new RegExp(`skill_routing:\\s+${SWITCH_DEFAULT.skill_routing}`));
|
||||
expect(r.stdout).toMatch(new RegExp(`skill_contracts:\\s+${SWITCH_DEFAULT.skill_contracts}`));
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
describe('design_detector_install_prompted (true|false, rejecting validator)', () => {
|
||||
const env = { ...process.env, GSTACK_STATE_ROOT: STATE };
|
||||
test('defaults to false, rejects a typo with the file unchanged, round-trips true/false, and is enumerated', () => {
|
||||
|
||||
@@ -5,7 +5,8 @@
|
||||
* carries /autoplan's own PreToolUse hook block and allowed tools. Its body is
|
||||
* a minimal scripted phase boundary: init, Phase 1 entry, a prepared close
|
||||
* packet, the report published with the `true autoplan-published` no-op, then the
|
||||
* Phase 2 entry in a later message and the Phase 2 reviewer. The restore point
|
||||
* Phase 2 entry in a later message and the Phase 2 reviewer. The actor sends each reviewer dispatch as a declared
|
||||
* token that startDispatchRelay replaces with the dispatch's exact bytes, so no actor retypes it. The restore point
|
||||
* and phase artifacts live in the project's `.gstack/tmp/autoplan/`, as /autoplan
|
||||
* Step 1 places them. Outcomes come from the guard decision log and the journals.
|
||||
*/
|
||||
@@ -32,7 +33,7 @@ export interface GuardSession {
|
||||
readLog(): Array<Record<string, any>>;
|
||||
/** Content-bounded journal summary for a failure message: tool names, inputs' first keys, result heads. */
|
||||
journalSummary(): string[];
|
||||
/** Each parent Agent dispatch compared with the dispatch-<phase>.txt the actor had to copy, read before cleanup. */
|
||||
/** Each parent Agent dispatch compared with the dispatch-<phase>.txt the relay substitutes, read before cleanup. */
|
||||
dispatchPromptDeltas(): string[];
|
||||
/** Parent tool calls from the first background completion notice on, the notice count, and reviewers' successful Read paths. */
|
||||
noticeEvidence(): { notices: number; afterNotice: string[]; reviewerReads: string[] };
|
||||
@@ -50,7 +51,7 @@ export function stoppedAtPhaseBoundary(screen: string): boolean {
|
||||
if (!marks.length) return false;
|
||||
const last = marks[marks.length - 1]!;
|
||||
const after = screen.slice(last.index! + last[0].length);
|
||||
return /✻[^\n]*\bdone\b/.test(after) && !/Step\s*8|design-phase|Design\s*review/i.test(after);
|
||||
return /✻[^\n]*\bdone\b/.test(after) && !/Step\s*7|design-phase|Design\s*review/i.test(after);
|
||||
}
|
||||
|
||||
/** The close packet Read (step 6) came after the CEO reviewer's completion notice, so it ran in the notice's new turn. */
|
||||
@@ -68,6 +69,122 @@ export function dispatchPromptDelta(expected: string, actual: string): string {
|
||||
return `expected ${expected.length} chars, sent ${actual.length}; first difference at offset ${at}\n expected: ${around(expected)}\n sent: ${around(actual)}`;
|
||||
}
|
||||
|
||||
type DispatchPhase = 'ceo' | 'design';
|
||||
/** The actor's declared dispatch interface: its Agent prompt is exactly this token, never a retyped dispatch. */
|
||||
export const dispatchToken = (phase: DispatchPhase): string => `DISPATCH ${phase}`;
|
||||
const TOKEN_NOTE = "the test driver puts this phase's exact reviewer dispatch in place of that text before the tool runs; " +
|
||||
'send it as written and never read, copy or retype the dispatch yourself';
|
||||
|
||||
/**
|
||||
* PR gate 41247097050 (#3646's delta): the Haiku actor retyped the Design dispatch's 64-hex contentHash and the guard
|
||||
* rightly denied it. The actor is a fixture, so it never copies the dispatch: an Agent tool call whose prompt is exactly
|
||||
* a declared token gets dispatch-<phase>.txt's bytes in its place. Anything else passes unchanged for the guard to judge.
|
||||
*/
|
||||
export function substituteDispatch(block: any, artifacts: string): DispatchPhase | undefined {
|
||||
if (block?.type !== 'tool_use' || block.name !== 'Agent') return;
|
||||
const phase = (['ceo', 'design'] as const).find(p => block.input?.prompt === dispatchToken(p));
|
||||
if (!phase || !fs.existsSync(path.join(artifacts, `dispatch-${phase}.txt`))) return;
|
||||
block.input.prompt = fs.readFileSync(path.join(artifacts, `dispatch-${phase}.txt`), 'utf8');
|
||||
return phase;
|
||||
}
|
||||
|
||||
/**
|
||||
* The request direction (trial 1 of GSTA-262's first panel): with the substituted CEO dispatch in its own history, the
|
||||
* actor imitated it for Design instead of sending its token. Each request shows the actor its own declared call instead.
|
||||
* A reviewer's dispatch arrives as user text, not an Agent tool call, so it is untouched.
|
||||
*/
|
||||
export function restoreDispatchTokens(request: any, artifacts: string): void {
|
||||
const sent = new Map((['ceo', 'design'] as const).flatMap(phase => {
|
||||
const file = path.join(artifacts, `dispatch-${phase}.txt`);
|
||||
return fs.existsSync(file) ? [[fs.readFileSync(file, 'utf8'), dispatchToken(phase)] as const] : [];
|
||||
}));
|
||||
for (const message of Array.isArray(request?.messages) ? request.messages : [])
|
||||
for (const block of Array.isArray(message?.content) ? message.content : [])
|
||||
if (block?.type === 'tool_use' && block.name === 'Agent' && sent.has(block.input?.prompt)) block.input.prompt = sent.get(block.input.prompt);
|
||||
}
|
||||
|
||||
/**
|
||||
* Rewrites one Messages API event stream: an Agent tool_use block is held until its stop, then sent as one input delta
|
||||
* carrying the substituted input, or as its original events when no token matched.
|
||||
*/
|
||||
export function dispatchEventRewriter(artifacts: string, substituted: (phase: DispatchPhase) => void) {
|
||||
let pending = '';
|
||||
let held: { index: number; block: any; json: string; raw: string[] } | null = null;
|
||||
const frame = (data: Record<string, unknown>) => `event: ${data.type}\ndata: ${JSON.stringify(data)}\n\n`;
|
||||
const take = (raw: string): string => {
|
||||
const data = /^data: (.*)$/m.exec(raw)?.[1];
|
||||
const event = data ? JSON.parse(data) : undefined;
|
||||
if (event?.type === 'content_block_start' && event.content_block?.type === 'tool_use' && event.content_block.name === 'Agent') {
|
||||
held = { index: event.index, block: event.content_block, json: '', raw: [raw] };
|
||||
return '';
|
||||
}
|
||||
if (!held || event?.index !== held.index) return raw;
|
||||
held.raw.push(raw);
|
||||
if (event.type === 'content_block_delta' && event.delta?.type === 'input_json_delta') held.json += event.delta.partial_json;
|
||||
if (event.type !== 'content_block_stop') return '';
|
||||
const { index, block, json, raw: original } = held;
|
||||
held = null;
|
||||
let input: unknown;
|
||||
try { input = JSON.parse(json || '{}'); } catch { return original.join(''); }
|
||||
const rewritten = { ...block, input };
|
||||
const phase = substituteDispatch(rewritten, artifacts);
|
||||
if (!phase) return original.join('');
|
||||
substituted(phase);
|
||||
return frame({ type: 'content_block_start', index, content_block: { ...block, input: {} } }) +
|
||||
frame({ type: 'content_block_delta', index, delta: { type: 'input_json_delta', partial_json: JSON.stringify(rewritten.input) } }) + raw;
|
||||
};
|
||||
return {
|
||||
push(chunk: string): string {
|
||||
pending = (pending + chunk).replace(/\r\n/g, '\n');
|
||||
const events = pending.split('\n\n');
|
||||
pending = events.pop()!;
|
||||
return events.map(e => take(e + '\n\n')).join('');
|
||||
},
|
||||
end: (): string => { const rest = (held ? held.raw.join('') : '') + pending; held = null; pending = ''; return rest; },
|
||||
};
|
||||
}
|
||||
|
||||
export interface DispatchRelay { url: string; substituted: DispatchPhase[]; stop(): void }
|
||||
|
||||
/**
|
||||
* A loopback relay to the Messages API for the live case (point ANTHROPIC_BASE_URL at url). Headers pass through and are
|
||||
* never logged; only an Agent tool call carrying a dispatch token, and that call in later requests' history, is rewritten.
|
||||
*/
|
||||
export function startDispatchRelay(artifacts: string, upstream = process.env.ANTHROPIC_BASE_URL || 'https://api.anthropic.com'): DispatchRelay {
|
||||
const substituted: DispatchPhase[] = [];
|
||||
const server = Bun.serve({
|
||||
hostname: '127.0.0.1', port: 0, idleTimeout: 0,
|
||||
async fetch(req) {
|
||||
const url = new URL(req.url), headers = new Headers(req.headers);
|
||||
headers.delete('host');
|
||||
headers.set('accept-encoding', 'identity');
|
||||
let body = req.method === 'GET' || req.method === 'HEAD' ? undefined : await req.text();
|
||||
if (body && url.pathname.endsWith('/messages')) {
|
||||
try { const request = JSON.parse(body); restoreDispatchTokens(request, artifacts); body = JSON.stringify(request); } catch { /* not JSON: forward as sent */ }
|
||||
}
|
||||
const res = await fetch(upstream.replace(/\/+$/, '') + url.pathname + url.search, { method: req.method, headers, body });
|
||||
const out = new Headers(res.headers);
|
||||
out.delete('content-encoding');
|
||||
out.delete('content-length');
|
||||
const type = res.headers.get('content-type') ?? '';
|
||||
if (!url.pathname.endsWith('/messages') || !res.body) return new Response(res.body, { status: res.status, headers: out });
|
||||
if (type.includes('application/json')) {
|
||||
const message = await res.json() as { content?: unknown[] };
|
||||
for (const block of message.content ?? []) { const phase = substituteDispatch(block, artifacts); if (phase) substituted.push(phase); }
|
||||
return new Response(JSON.stringify(message), { status: res.status, headers: out });
|
||||
}
|
||||
if (!type.includes('text/event-stream')) return new Response(res.body, { status: res.status, headers: out });
|
||||
const rewrite = dispatchEventRewriter(artifacts, phase => substituted.push(phase));
|
||||
const decoder = new TextDecoder(), encoder = new TextEncoder();
|
||||
return new Response(res.body.pipeThrough(new TransformStream<Uint8Array, Uint8Array>({
|
||||
transform(chunk, sink) { const text = rewrite.push(decoder.decode(chunk, { stream: true })); if (text) sink.enqueue(encoder.encode(text)); },
|
||||
flush(sink) { const text = rewrite.push(decoder.decode()) + rewrite.end(); if (text) sink.enqueue(encoder.encode(text)); },
|
||||
})), { status: res.status, headers: out });
|
||||
},
|
||||
});
|
||||
return { url: `http://127.0.0.1:${server.port}`, substituted, stop: () => { server.stop(true); } };
|
||||
}
|
||||
|
||||
export function prepareGuardSession(apiKey: string | undefined): GuardSession {
|
||||
const base = fs.realpathSync(fs.mkdtempSync(path.join(os.tmpdir(), 'ap-guard-pty-')));
|
||||
const home = path.join(base, 'home'), config = path.join(home, '.claude'), project = path.join(base, 'project');
|
||||
@@ -102,28 +219,26 @@ console.log(close);
|
||||
const tool = path.join(ROOT, 'bin', 'gstack-autoplan-snapshot.ts');
|
||||
const sections = path.join(config, 'skills', 'gstack', 'autoplan', 'sections');
|
||||
const body = [
|
||||
'Run this scripted phase boundary exactly, from step 1 through step 11 in one continuous run. Each numbered step is its own ' +
|
||||
'assistant message; do nothing else. End your turn before step 11 only where step 5 or step 10 says to wait for a ' +
|
||||
'Run this scripted phase boundary exactly, from step 1 through step 9 in one continuous run. Each numbered step is its own ' +
|
||||
'assistant message; do nothing else. End your turn before step 9 only where step 4 or step 8 says to wait for a ' +
|
||||
'background agent. **Phase 1 complete.** is a phase boundary, not the end of the task.',
|
||||
`1. Run this Bash command alone: bun "${tool}" init "${source}" "${active}" "${restore}"`,
|
||||
`2. Read the file ${path.join(sections, 'ceo-phase.md')} with offset 1 and limit 20.`,
|
||||
`3. Run this Bash command alone: bun "${review}"`,
|
||||
`4. Run this Bash command alone: cat "${path.join(artifacts, 'dispatch-ceo.txt')}"`,
|
||||
'5. Call the Agent tool once with these inputs: prompt = the exact, complete output of step 4 (every line, unchanged), ' +
|
||||
`4. Call the Agent tool once with these inputs: prompt = exactly the text \`${dispatchToken('ceo')}\` (${TOKEN_NOTE}), ` +
|
||||
'description "CEO review", subagent_type "general-purpose", and run_in_background false. Include run_in_background even ' +
|
||||
'if the tool schema lacks it. If its result says the agent is running in the background, end your turn at once with no ' +
|
||||
'further tool call; the completion notification starts your next turn.',
|
||||
'6. Only after the CEO review agent has finished (in the turn its completion notification starts, when it ran in the ' +
|
||||
'5. Only after the CEO review agent has finished (in the turn its completion notification starts, when it ran in the ' +
|
||||
'background), Read the file whose path step 3 printed, completely (no offset or limit).',
|
||||
'7. In one message, write exactly this line as your own text: **Phase 1 complete.** and make exactly one tool call in that ' +
|
||||
'6. In one message, write exactly this line as your own text: **Phase 1 complete.** and make exactly one tool call in that ' +
|
||||
'same message: the Bash command `true autoplan-published ceo`. Do not end your turn after it: when its result arrives, go ' +
|
||||
'straight on to step 8.',
|
||||
`8. As your next assistant message, in the same turn as step 7's result (no user input is coming), Read the file ${path.join(sections, 'design-phase.md')} with offset 1 and limit 20.`,
|
||||
`9. Run this Bash command alone: cat "${path.join(artifacts, 'dispatch-design.txt')}"`,
|
||||
'10. Call the Agent tool once with these inputs: prompt = the exact, complete output of step 9 (every line, unchanged), ' +
|
||||
'straight on to step 7.',
|
||||
`7. As your next assistant message, in the same turn as step 6's result (no user input is coming), Read the file ${path.join(sections, 'design-phase.md')} with offset 1 and limit 20.`,
|
||||
`8. Call the Agent tool once with these inputs: prompt = exactly the text \`${dispatchToken('design')}\` (${TOKEN_NOTE}), ` +
|
||||
'description "Design review", subagent_type "general-purpose". If the agent runs in the background, end your turn and wait ' +
|
||||
'for its completion notification.',
|
||||
'11. Reply with the single word DONE. If any tool call is denied, quote the denial text and stop.',
|
||||
'9. Reply with the single word DONE. If any tool call is denied, quote the denial text and stop.',
|
||||
].join('\n');
|
||||
const skillDir = path.join(project, '.claude', 'skills', GUARD_SKILL);
|
||||
fs.mkdirSync(skillDir, { recursive: true });
|
||||
|
||||
@@ -158,6 +158,7 @@ export const FREE_FIXTURES: Readonly<Record<string, readonly string[]>> = {
|
||||
'test/fixtures/impeccable-captures.meta.json': ['test/design-detect-contract.test.ts', 'test/impeccable-fixtures.test.ts'],
|
||||
'test/fixtures/impeccable-detect-dom-sample.json': ['test/impeccable-fixtures.test.ts'],
|
||||
'test/fixtures/impeccable-detect-help.txt': ['test/impeccable-fixtures.test.ts'],
|
||||
'test/fixtures/inventory-grandfathered.json': ['test/inventory.test.ts'],
|
||||
'test/fixtures/investigate-owned-final-texts-opus.json': ['test/workflow-boundaries-fixture.test.ts'],
|
||||
'test/fixtures/investigate-owned-question-39195396438.json': ['test/workflow-boundaries-fixture.test.ts'],
|
||||
'test/fixtures/judge-calibration/**': ['test/judge-calibration.test.ts'],
|
||||
@@ -214,6 +215,7 @@ export const FREE_FIXTURES: Readonly<Record<string, readonly string[]>> = {
|
||||
'test/fixtures/review-design-lite-reports-ci-36633323521.json': ['test/review-finalization-budget.test.ts'],
|
||||
'test/fixtures/review-gate-integrity-opus-reports.json': ['test/gate-integrity-templates.test.ts'],
|
||||
'test/fixtures/review-wrong-tree/**': ['test/review-dispatch.test.ts'],
|
||||
'test/fixtures/screen-verdict-41662815506/**': ['test/ship-queue-screen-verdict.test.ts'],
|
||||
'test/fixtures/sealed-guard-off-golden.json': ['test/sealed-guard.test.ts'],
|
||||
'test/fixtures/session-update-heartbeat.sh': ['test/session-update-heartbeat.test.ts'],
|
||||
'test/fixtures/shadow-fake-model.ts': ['test/autoplan-shadow-driver.test.ts', 'test/autoplan-shadow.test.ts'],
|
||||
|
||||
@@ -35,7 +35,8 @@ export const DESIGN_BOARD_ACTOR_PROTOCOL = [
|
||||
'If offering that optional step, ask one single-select question with header "Voices",',
|
||||
'beginning "Want outside design voices before the detailed review?" and exactly:',
|
||||
...DESIGN_OUTSIDE_VOICE_OPTIONS.map(option => `- ${option.label}: ${option.description}`),
|
||||
'Option order, a letter prefix, and a trailing (recommended) marker may vary.',
|
||||
'Option order, a letter prefix, a trailing (recommended) marker, and',
|
||||
'keeping or dropping every ✅/❌ bullet in a description may vary.',
|
||||
'Keep this choice separate from other approvals; no previews or extra actions.',
|
||||
'The fixture user chooses No, proceed without. Observe the actual native',
|
||||
'answer before continuing when this question is offered.',
|
||||
@@ -119,7 +120,8 @@ export function createDesignReviewPicker({ cwd, deadlineAt }: { cwd: string; dea
|
||||
|| labels.includes(DESIGN_OUTSIDE_VOICE_OPTIONS[0].label);
|
||||
if (claimsVoices) {
|
||||
const voiceActions = question.options.map((option, index) => DESIGN_OUTSIDE_VOICE_OPTIONS.findIndex(expected =>
|
||||
labels[index] === expected.label && option.description === expected.description && option.preview === undefined));
|
||||
labels[index] === expected.label && option.preview === undefined
|
||||
&& [expected.description, expected.description.replace(/(^| )[✅❌] /g, '$1')].includes(option.description)));
|
||||
if (question.header.trim() !== 'Voices'
|
||||
|| !/^(?:Want outside design voices before the detailed review|Run outside design voices before the 7 passes)\?(?:\s|$)/i.test(lead)
|
||||
|| question.multiSelect || voiceActions.length !== 2 || voiceActions.includes(-1)
|
||||
|
||||
@@ -0,0 +1,220 @@
|
||||
/**
|
||||
* R56 (ETHOS.md: 9. Earn Its Place): every part of gstack has a row in
|
||||
* docs/inventory/evidence.json, validated by its status's own fields, and
|
||||
* bin/gstack-inventory ranks the parts by load per run into docs/inventory.md.
|
||||
*/
|
||||
import { afterAll, describe, expect, test } from 'bun:test';
|
||||
import { spawnSync } from 'node:child_process';
|
||||
import * as fs from 'node:fs';
|
||||
import * as os from 'node:os';
|
||||
import * as path from 'node:path';
|
||||
import { discoverTemplates } from '../scripts/discover-skills';
|
||||
import { DISPATCHES, LEDGER_PATH, ROOT_KEY, SHARED_KEYS, SHARED_PARTS, installReason, inventoryKeys, keyOf, validateEntry, validateLedger } from '../lib/inventory';
|
||||
|
||||
const ROOT = path.resolve(import.meta.dir, '..');
|
||||
const BIN = path.join(ROOT, 'bin', 'gstack-inventory');
|
||||
const tmp = fs.mkdtempSync(path.join(os.tmpdir(), 'gstack-inventory-'));
|
||||
afterAll(() => fs.rmSync(tmp, { recursive: true, force: true }));
|
||||
|
||||
const codes = (key: string, entry: unknown) => validateEntry(key, entry).map(p => p.code);
|
||||
const run = (args: string[]) => spawnSync(BIN, args, { cwd: ROOT, encoding: 'utf8', timeout: 120_000 });
|
||||
|
||||
const VERDICT = { sources: ['gstack-evals run r9 results.json'], models: ['claude-opus-5-5', 'gpt-6.1-sol'], prompt: 'preset', scope: 'the full procedure, told', masked: true };
|
||||
const NEW_UNTESTED = { status: 'untested', reason: 'catches missed migrations', plan: 'test/fixtures/skill-evals/demo/tasks.json', owner: 'GSTA-168', measure_by: '2026-12-01' };
|
||||
|
||||
describe('the key list', () => {
|
||||
test('keys are the skill directories discoverTemplates finds, the root skill as gstack, then the fixed shared keys', () => {
|
||||
const keys = inventoryKeys(ROOT);
|
||||
const skills = discoverTemplates(ROOT).map(t => path.dirname(t.tmpl)).map(d => (d === '.' ? ROOT_KEY : d));
|
||||
expect(keys).toEqual([...skills.sort(), ...SHARED_KEYS]);
|
||||
expect(keys).toContain('gstack');
|
||||
expect(keys).toContain('gstack-upgrade');
|
||||
expect(keys).not.toContain('connect-chrome');
|
||||
expect(new Set(keys).size).toBe(keys.length);
|
||||
});
|
||||
|
||||
test('installed names map to keys: gstack-<name> to <name>, gstack-upgrade stays literal', () => {
|
||||
expect(keyOf('gstack-ship')).toBe('ship');
|
||||
expect(keyOf('ship')).toBe('ship');
|
||||
expect(keyOf('gstack-upgrade')).toBe('gstack-upgrade');
|
||||
expect(keyOf('gstack')).toBe('gstack');
|
||||
});
|
||||
});
|
||||
|
||||
describe('the ledger covers every key and validates', () => {
|
||||
const ledger = JSON.parse(fs.readFileSync(path.join(ROOT, LEDGER_PATH), 'utf8'));
|
||||
|
||||
test('no missing and no extra keys, and every row passes its status schema', () => {
|
||||
const keys = inventoryKeys(ROOT);
|
||||
expect(Object.keys(ledger.parts).filter(k => !keys.includes(k))).toEqual([]);
|
||||
expect(keys.filter(k => !(k in ledger.parts))).toEqual([]);
|
||||
expect(validateLedger(ledger, keys)).toEqual([]);
|
||||
});
|
||||
|
||||
test('the initial backlog is grandfathered, nothing else is, and the guards are safety rows', () => {
|
||||
const backlog: string[] = JSON.parse(fs.readFileSync(path.join(ROOT, 'test', 'fixtures', 'inventory-grandfathered.json'), 'utf8')).keys;
|
||||
for (const key of backlog) expect([key, ledger.parts[key]?.grandfathered]).toEqual([key, true]);
|
||||
for (const [key, row] of Object.entries<any>(ledger.parts)) if (row.grandfathered) expect(backlog).toContain(key);
|
||||
for (const key of ['careful', 'freeze', 'guard', 'unfreeze', 'cso']) expect(ledger.parts[key].status).toBe('safety');
|
||||
for (const key of ['capy-skill', 'capy-factory']) expect(ledger.parts[key]).toMatchObject({ status: 'untested', plan: 'none' });
|
||||
});
|
||||
|
||||
test('a missing and an extra key are each named', () => {
|
||||
const keys = inventoryKeys(ROOT);
|
||||
const { ship, ...rest } = ledger.parts;
|
||||
const problems = validateLedger({ ...ledger, parts: { ...rest, 'no-such-part': ship } }, keys);
|
||||
expect(problems.map(p => `${p.key} ${p.code}`)).toEqual(['ship LEDGER_KEY_MISSING', 'no-such-part LEDGER_KEY_EXTRA']);
|
||||
});
|
||||
});
|
||||
|
||||
describe('the per-status schema', () => {
|
||||
test('passing rows: a grandfathered untested part with plan none, a new part with a dated plan, a fair verdict, a safety guard', () => {
|
||||
expect(codes('a', { status: 'untested', grandfathered: true, plan: 'none' })).toEqual([]);
|
||||
expect(codes('a', NEW_UNTESTED)).toEqual([]);
|
||||
expect(codes('a', { ...NEW_UNTESTED, plan: 'docs/evals/demo/PREREG.md' })).toEqual([]);
|
||||
expect(codes('a', { status: 'helps', ...VERDICT })).toEqual([]);
|
||||
expect(codes('a', { status: 'helps', ...VERDICT, models: ['claude-opus-5-5', 'particle-1'] })).toEqual([]);
|
||||
expect(codes('a', { status: 'no-effect', ...VERDICT, r48: { verdict: 'CUT-CANDIDATE', ref: 'why-null.md' } })).toEqual([]);
|
||||
expect(codes('a', { status: 'safety', reason: 'blocks rm -rf' })).toEqual([]);
|
||||
expect(codes('a', { status: 'untested', grandfathered: true, plan: 'none', prior: [{ ref: 'run 7', why: ['stub', 'unmasked'] }] })).toEqual([]);
|
||||
});
|
||||
|
||||
test('a new untested part needs reason, a concrete plan, owner and measure_by; plan none is refused', () => {
|
||||
const { measure_by, ...noDate } = NEW_UNTESTED;
|
||||
expect(codes('a', noDate)).toEqual(['LEDGER_MEASURE_BY_MISSING']);
|
||||
expect(codes('a', { ...NEW_UNTESTED, measure_by: 'soon' })).toEqual(['LEDGER_MEASURE_BY_MISSING']);
|
||||
const { owner, ...noOwner } = NEW_UNTESTED;
|
||||
expect(codes('a', noOwner)).toEqual(['LEDGER_OWNER_MISSING']);
|
||||
const { reason, ...noReason } = NEW_UNTESTED;
|
||||
expect(codes('a', noReason)).toEqual(['LEDGER_REASON_MISSING']);
|
||||
expect(codes('a', { ...NEW_UNTESTED, plan: 'none' })).toEqual(['LEDGER_PLAN_NONE_NEW']);
|
||||
expect(codes('a', { ...NEW_UNTESTED, plan: 'we will test it later' })).toEqual(['LEDGER_PLAN_NOT_CONCRETE']);
|
||||
expect(codes('a', { ...NEW_UNTESTED, plan: 'GEVA-2 registry row R56-11 (proposed, not yet preregistered)' })).toEqual(['LEDGER_PLAN_NOT_CONCRETE']);
|
||||
expect(codes('a', { ...NEW_UNTESTED, plan: 'prereg later' })).toEqual(['LEDGER_PLAN_NOT_CONCRETE']);
|
||||
expect(codes('a', { ...NEW_UNTESTED, plan: 'wiki/plans/2026-11-01-demo-prereg.md' })).toEqual([]);
|
||||
expect(codes('a', { status: 'untested', grandfathered: true })).toEqual(['LEDGER_PLAN_MISSING']);
|
||||
});
|
||||
|
||||
test('helps, no-effect and hurts need sources, frontier models, the preset prompt, a scope and a masked no-gstack arm', () => {
|
||||
const { sources, ...noSources } = VERDICT;
|
||||
expect(codes('a', { status: 'helps', ...noSources })).toEqual(['LEDGER_SOURCES_MISSING']);
|
||||
expect(codes('a', { status: 'helps', ...VERDICT, sources: [] })).toEqual(['LEDGER_SOURCES_MISSING']);
|
||||
expect(codes('a', { status: 'helps', ...VERDICT, models: ['claude-haiku-5-5'] })).toEqual(['LEDGER_MODELS_NOT_FRONTIER']);
|
||||
expect(codes('a', { status: 'helps', ...VERDICT, models: ['claude-opus-5-5', 'claude-sonnet-5-5'] })).toEqual(['LEDGER_MODELS_NOT_FRONTIER']);
|
||||
expect(codes('a', { status: 'helps', ...VERDICT, models: ['claude-opus-5-5', 'particle-1', 'deepseek-v4.1-flash'] })).toEqual(['LEDGER_MODELS_NOT_FRONTIER']);
|
||||
expect(codes('a', { status: 'helps', ...VERDICT, prompt: 'sdk-stub' })).toEqual(['LEDGER_PROMPT_NOT_PRESET']);
|
||||
expect(codes('a', { status: 'hurts', ...VERDICT, scope: '', r48: { verdict: 'KEEP', ref: 'x' } })).toEqual(['LEDGER_SCOPE_MISSING']);
|
||||
expect(codes('a', { status: 'helps', ...VERDICT, masked: false })).toEqual(['LEDGER_UNMASKED']);
|
||||
});
|
||||
|
||||
test('no-effect and hurts without an R48 verdict and its reference are refused', () => {
|
||||
expect(codes('a', { status: 'no-effect', ...VERDICT })).toEqual(['LEDGER_R48_MISSING']);
|
||||
expect(codes('a', { status: 'hurts', ...VERDICT, r48: { verdict: 'DROP', ref: 'x' } })).toEqual(['LEDGER_R48_MISSING']);
|
||||
expect(codes('a', { status: 'hurts', ...VERDICT, r48: { verdict: 'KEEP' } })).toEqual(['LEDGER_R48_MISSING']);
|
||||
});
|
||||
|
||||
test('safety needs a reason; unknown statuses, unknown fields and bad prior entries are refused', () => {
|
||||
expect(codes('a', { status: 'safety' })).toEqual(['LEDGER_REASON_MISSING']);
|
||||
expect(codes('a', { status: 'exempt', reason: 'x' })).toEqual(['LEDGER_STATUS_INVALID']);
|
||||
expect(codes('a', { status: 'safety', reason: 'x', source: 'typo' })).toEqual(['LEDGER_FIELD_UNKNOWN']);
|
||||
expect(codes('a', { status: 'untested', grandfathered: true, plan: 'none', prior: [{ ref: 'run 7', why: ['vibes'] }] })).toEqual(['LEDGER_PRIOR_INVALID']);
|
||||
expect(codes('a', { status: 'untested', grandfathered: true, plan: 'none', prior: [{ why: ['stub'] }] })).toEqual(['LEDGER_PRIOR_INVALID']);
|
||||
expect(codes('a', { status: 'untested', grandfathered: true, plan: 'none', prior: [{ ref: 'run 7', why: ['toString'] }] })).toEqual(['LEDGER_PRIOR_INVALID']);
|
||||
});
|
||||
|
||||
test('a present r48 is checked on every status, and aliases is a list of names', () => {
|
||||
expect(codes('a', { status: 'helps', ...VERDICT, r48: { verdict: 'DROP', ref: 'x' } })).toEqual(['LEDGER_R48_MISSING']);
|
||||
expect(codes('a', { status: 'helps', ...VERDICT, r48: {} })).toEqual(['LEDGER_R48_MISSING']);
|
||||
expect(codes('a', { status: 'safety', reason: 'x', aliases: 'other-name' })).toEqual(['LEDGER_FIELD_INVALID']);
|
||||
expect(codes('a', { status: 'safety', reason: 'x', aliases: [] })).toEqual(['LEDGER_FIELD_INVALID']);
|
||||
expect(codes('a', { status: 'safety', reason: 'x', aliases: ['other-name'] })).toEqual([]);
|
||||
});
|
||||
});
|
||||
|
||||
/** A small gstack-shaped tree: a root skill, two skills, the shared-part files and the cited dispatch sources. */
|
||||
function fixtureTree(dir: string, parts: (keys: string[]) => Record<string, unknown>): string {
|
||||
const write = (rel: string, text: string) => { fs.mkdirSync(path.dirname(path.join(dir, rel)), { recursive: true }); fs.writeFileSync(path.join(dir, rel), text); };
|
||||
const skill = (rel: string, name: string, tier: number, body: string) => {
|
||||
const fm = `---\nname: ${name}\npreamble-tier: ${tier}\ndescription: |\n Does ${name} things.\n---\n`;
|
||||
write(path.join(rel, 'SKILL.md.tmpl'), fm + body);
|
||||
write(path.join(rel, 'SKILL.md'), fm + '<!-- AUTO-GENERATED from SKILL.md.tmpl -->\n' + body);
|
||||
};
|
||||
skill('', 'gstack', 1, 'router\n');
|
||||
skill('alpha', 'alpha', 2, 'a'.repeat(4000) + '\n');
|
||||
skill('beta', 'beta', 2, 'b'.repeat(1000) + '\n');
|
||||
write('alpha/sections/preamble.md', 'p'.repeat(800));
|
||||
write('beta/sections/preamble.md', 'p'.repeat(800));
|
||||
for (const part of SHARED_PARTS) for (const f of part.files) write(f, 'x'.repeat(500) + '\n');
|
||||
for (const d of DISPATCHES) if (!fs.existsSync(path.join(dir, d.file))) write(d.file, 'filler\n');
|
||||
for (const d of DISPATCHES) fs.appendFileSync(path.join(dir, d.file), `${d.quote}\n`);
|
||||
const keys = inventoryKeys(dir);
|
||||
write(LEDGER_PATH, JSON.stringify({ schema_version: 1, parts: parts(keys), unmapped: [{ row: 'gstack as a whole', why: 'not a single text part' }] }, null, 2));
|
||||
return dir;
|
||||
}
|
||||
|
||||
const grandfathered = (keys: string[]) => Object.fromEntries(keys.map(k => [k, k === 'beta' ? { status: 'safety', grandfathered: true, reason: 'a guard' } : { status: 'untested', grandfathered: true, plan: 'none' }]));
|
||||
|
||||
describe('bin/gstack-inventory', () => {
|
||||
test('determinism: two runs on a fixed tree write the same docs/inventory.md and the same --json', () => {
|
||||
const dir = fixtureTree(path.join(tmp, 'fixed'), grandfathered);
|
||||
const first = run(['--root', dir]);
|
||||
expect(first.status, first.stderr).toBe(0);
|
||||
const a = fs.readFileSync(path.join(dir, 'docs/inventory.md'), 'utf8');
|
||||
const second = run(['--root', dir]);
|
||||
expect(second.status, second.stderr).toBe(0);
|
||||
expect(fs.readFileSync(path.join(dir, 'docs/inventory.md'), 'utf8')).toBe(a);
|
||||
expect(run(['--root', dir, '--json']).stdout).toBe(run(['--root', dir, '--json']).stdout);
|
||||
expect(a).toContain('Measured at commit none');
|
||||
expect(a.indexOf('| alpha |')).toBeLessThan(a.indexOf('| beta |'));
|
||||
const start = a.indexOf('## Measure next');
|
||||
const next = a.slice(start, a.indexOf('\n## ', start + 1));
|
||||
expect(next).toContain('alpha');
|
||||
expect(next).not.toContain('| beta |');
|
||||
expect(a).toContain('gstack as a whole');
|
||||
expect(a).toContain('gstack-<name>');
|
||||
for (const d of DISPATCHES) expect(a).toContain(`${d.file}:`);
|
||||
});
|
||||
|
||||
test('--check: ok on a complete ledger; a missing key exits 1 with its code', () => {
|
||||
const dir = fixtureTree(path.join(tmp, 'check'), grandfathered);
|
||||
const ok = run(['--root', dir, '--check']);
|
||||
expect(ok.status, ok.stderr).toBe(0);
|
||||
expect(ok.stdout).toMatch(/^INVENTORY: ok keys=\d+/m);
|
||||
const bad = fixtureTree(path.join(tmp, 'check-bad'), keys => { const p = grandfathered(keys); delete p.alpha; return p; });
|
||||
const r = run(['--root', bad, '--check']);
|
||||
expect(r.status).toBe(1);
|
||||
expect(r.stderr).toContain('alpha: no row in docs/inventory/evidence.json (LEDGER_KEY_MISSING)');
|
||||
expect(run(['--root', bad]).status).toBe(1);
|
||||
});
|
||||
|
||||
test('--check: a cited dispatch source that lost its quote is named', () => {
|
||||
const dir = fixtureTree(path.join(tmp, 'drift'), grandfathered);
|
||||
fs.writeFileSync(path.join(dir, DISPATCHES[0]!.file), 'rewritten\n');
|
||||
const r = run(['--root', dir, '--check']);
|
||||
expect(r.status).toBe(1);
|
||||
expect(r.stderr).toContain('(INVENTORY_SOURCE_MISSING)');
|
||||
});
|
||||
|
||||
test('a bad --root, a root outside a gstack checkout and a gstack install are refused, and nothing is written', () => {
|
||||
const dir = fixtureTree(path.join(tmp, 'args'), grandfathered);
|
||||
for (const args of [['--root', '--json'], ['--root', '--check'], ['--root']]) expect(run(args).status).toBe(2);
|
||||
expect(fs.existsSync(path.join(dir, 'docs/inventory.md'))).toBe(false);
|
||||
const outside = spawnSync(BIN, [], { cwd: tmp, encoding: 'utf8', timeout: 120_000 });
|
||||
expect(outside.status).toBe(2);
|
||||
expect(outside.stderr).toContain('is not a gstack checkout');
|
||||
const installed = fixtureTree(path.join(tmp, 'home', '.claude', 'skills', 'gstack'), grandfathered);
|
||||
const w = run(['--root', installed]);
|
||||
expect(w.status).toBe(2);
|
||||
expect(w.stderr).toContain('refusing to write docs/inventory.md');
|
||||
expect(fs.existsSync(path.join(installed, 'docs/inventory.md'))).toBe(false);
|
||||
expect(run(['--root', installed, '--check']).status).toBe(0);
|
||||
expect(installReason(dir, { GSTACK_CAPY_ROOT: dir })).toContain('the Capy install');
|
||||
expect(installReason(dir, {}, path.join(tmp, 'home'))).toBeNull();
|
||||
expect(installReason(path.join(tmp, 'capy-root'), {}, path.join(tmp, 'h'))).toBeNull();
|
||||
});
|
||||
|
||||
test('the real ledger passes --check', () => {
|
||||
const r = run(['--check']);
|
||||
expect(r.status, r.stderr).toBe(0);
|
||||
});
|
||||
});
|
||||
@@ -115,6 +115,33 @@ test('the native-only actor declares its scope and declines the complete capture
|
||||
await expectUnsubmitted(published);
|
||||
});
|
||||
|
||||
// Train gate 41470212877 (86bb621f, paid-slice-6-a1, passed on the in-run rerun): the native question put the Pros / cons
|
||||
// bullets in its text and dropped every ✅/❌ bullet from the option descriptions; the 2.1.251 and #77 captures kept them.
|
||||
// Both are readings of the format's "descriptions use literal ✅/❌ bullets", so the protocol declares both spellings.
|
||||
test('the native-only actor declines the declared menu with every ✅/❌ bullet or none, and refuses anything in between', async () => {
|
||||
const published = await board();
|
||||
const bulletFree = structuredClone(outsideVoices.bulletFreeDescriptions41470212877.question) as NativeQuestion;
|
||||
expect(bulletFree.options.map(option => option.description).join(' ')).not.toMatch(/[✅❌]/);
|
||||
expect(picker()(bulletFree)).toBe(2);
|
||||
expect(DESIGN_BOARD_ACTOR_PROTOCOL).toContain('keeping or dropping every ✅/❌ bullet in a description may vary');
|
||||
const changes: ((menu: NativeQuestion) => void)[] = [
|
||||
menu => { menu.options[0]!.description = '✅ ' + menu.options[0]!.description; },
|
||||
menu => { menu.options[1]!.description = menu.options[1]!.description.replace('Findings rest', '❌ Findings rest'); },
|
||||
menu => { [menu.options[0]!.description, menu.options[1]!.description] = [menu.options[1]!.description, menu.options[0]!.description]; },
|
||||
menu => { menu.options[1]!.description = menu.options[1]!.description.replace('Fewest steps', 'Fewer steps'); },
|
||||
menu => { menu.options[1]!.description += ' Also approve the plan.'; },
|
||||
];
|
||||
for (const change of changes) {
|
||||
const menu = structuredClone(bulletFree);
|
||||
change(menu);
|
||||
expect(() => picker()(menu)).toThrow('no unambiguous declared native-only action');
|
||||
}
|
||||
const flipped = structuredClone(outsideVoices.question) as NativeQuestion;
|
||||
flipped.options[1]!.description = flipped.options[1]!.description.replace('❌', '✅');
|
||||
expect(() => picker()(flipped)).toThrow('no unambiguous declared native-only action');
|
||||
await expectUnsubmitted(published);
|
||||
});
|
||||
|
||||
test('the native-only actor accepts the declared question with a same-line explanation', async () => {
|
||||
const published = await board();
|
||||
const retained = structuredClone(outsideVoices.sameLineExplanation77.question) as NativeQuestion;
|
||||
|
||||
@@ -304,6 +304,20 @@ describe('deleted files (paths absent from the head tree)', () => {
|
||||
expect(real.referencers('bin/gstack-next-version')).toContain('review/SKILL.md.tmpl');
|
||||
});
|
||||
|
||||
test('a file the PR adds is read from the checkout under review, not from the selector\'s own tree (an older install)', () => {
|
||||
const added = 'hosts/claude/hooks/zz-added-hook.ts';
|
||||
expect(fs.existsSync(path.join(ROOT, added))).toBe(false);
|
||||
expect(real.referencers(added)).toContain('setup');
|
||||
const checkout = fixtureRepo({ setup: 'SETUP_HOOK=hosts/claude/hooks/zz-added-hook\n', [added]: 'export {};\n' });
|
||||
try {
|
||||
const result = computePaidCaseSelection({ profile: 'pr', rootDir: checkout, env: { EVALS_BASE: 'HEAD' }, changedFiles: [added], packageVersionOnly: true });
|
||||
expect(result.coverage!.unknownFileLabels.filter(l => l.file === added)).toEqual([]);
|
||||
expect(result.reason).not.toContain('deleted but still referenced');
|
||||
} finally {
|
||||
fs.rmSync(checkout, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
test('a deleted file a full-gate input still references, or a deleted full-gate entry, restores the full gate', () => {
|
||||
expect(real.referencers('bin/gstack-install-registry.sh')).toContain('setup');
|
||||
const referenced = pick('bin/gstack-install-registry.sh');
|
||||
|
||||
@@ -240,3 +240,59 @@ describe('lanes runtime and serial files (CD1)', () => {
|
||||
expect(r.out).toMatch(/^tests {6}lanes {5}pass .*2 tests ran in 2 file\(s\) \(1 serial\), exit 0/m);
|
||||
});
|
||||
});
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// GST-1138 (ship-90-90 L6): the pre-push housekeeping run and gstack's own guard list
|
||||
// ---------------------------------------------------------------------------
|
||||
import { matchesAny } from '../lib/ship-policy';
|
||||
|
||||
describe('--stage guards: the pre-push housekeeping run (GST-1138)', () => {
|
||||
// Value: protects=--stage guards runs only regen and guards for the touched files, writes its own record and never replaces the full run's
|
||||
// pregate.json; fails_when=it runs the secrets, strays or test checks, overwrites .gstack/tmp/pregate.json, or accepts --only/--from-run;
|
||||
// why_new=gbrain: 64 of 93 red heads were housekeeping (FOUN-64), #3908 went red on the context-budget ratchet; seam=gstack-pregate run; no_claim=CI
|
||||
test('runs only regen and guards, writes pregate-guards.json, leaves pregate.json alone and refuses --only and --from-run', () => {
|
||||
const repo = guardedFixture({ ...BASE_PREGATE, guards: [SERIAL_GUARD] });
|
||||
commit(repo, { 'test/widget-env.serial.test.js': SERIAL_TEST });
|
||||
const r = pregate(repo, ['--allow-repo-commands', '--stage', 'guards']);
|
||||
expect(r.status).toBe(EXIT.fail);
|
||||
expect(r.out).toMatch(/^preflight {2}guards {4}fail .*serial-manifest.*\(PREGATE_GUARD_FAILED\)/m);
|
||||
expect(fs.existsSync(path.join(repo.work, '.gstack/tmp/pregate.json'))).toBe(false);
|
||||
const rec = JSON.parse(fs.readFileSync(path.join(repo.work, '.gstack/tmp/pregate-guards.json'), 'utf8'));
|
||||
expect(rec.checks.map((c: any) => c.id)).toEqual(['regen', 'guards']);
|
||||
expect(rec.stage2_ms).toBeUndefined();
|
||||
commit(repo, { 'scripts/serial-files.tsv': 'path\treason\ntest/widget-env.serial.test.js\tglobal-state\n' });
|
||||
const ok = pregate(repo, ['--allow-repo-commands', '--stage', 'guards']);
|
||||
expect(ok.status, ok.out + ok.err).toBe(EXIT.ok);
|
||||
expect(ok.out).toContain('1 guard(s) passed: serial-manifest');
|
||||
expect(pregate(repo, ['--stage', 'guards', '--only', 'secrets']).status).toBe(EXIT.usage);
|
||||
expect(pregate(repo, ['--stage', 'guards', '--from-run', '123']).status).toBe(EXIT.usage);
|
||||
});
|
||||
});
|
||||
|
||||
describe("gstack's own pre-push guards (GST-1138)", () => {
|
||||
// Value: protects=gstack declares the housekeeping guards that went red after push (typecheck, skill size, parity, context budget,
|
||||
// module size, goldens, doc caps), each runnable and selected only by its inputs; fails_when=a guard is dropped, names a missing test file,
|
||||
// fails validation, or fires on a docs-only change; why_new=GST-1138; seam=.gstack/pregate.json; no_claim=the guards' own verdicts
|
||||
const GSTACK = path.resolve(import.meta.dir, '..');
|
||||
const policy = JSON.parse(fs.readFileSync(path.join(GSTACK, '.gstack/pregate.json'), 'utf8'));
|
||||
const guards: { id: string; when: string[]; command: string; fix?: string }[] = policy.guards ?? [];
|
||||
const selected = (file: string) => guards.filter(g => matchesAny(g.when, file)).map(g => g.id).sort();
|
||||
|
||||
test('the list validates, every guard has a fix, and every test file it names exists', () => {
|
||||
expect(codes(policy)).toEqual([]);
|
||||
expect(guards.map(g => g.id).sort()).toEqual(['context-budget', 'doc-caps', 'goldens', 'module-size', 'parity-suite', 'skill-size', 'typecheck']);
|
||||
for (const g of guards) {
|
||||
expect(g.fix, g.id).toBeTruthy();
|
||||
for (const f of g.command.match(/test\/[\w./-]+\.test\.ts/g) ?? []) expect(fs.existsSync(path.join(GSTACK, f)), `${g.id}: ${f}`).toBe(true);
|
||||
}
|
||||
});
|
||||
|
||||
test('each guard fires on its own inputs and none fires on a docs-only change', () => {
|
||||
expect(selected('ship/SKILL.md.tmpl')).toEqual(['context-budget', 'goldens', 'parity-suite', 'skill-size']);
|
||||
expect(selected('lib/pregate/index.ts')).toEqual(['module-size', 'typecheck']);
|
||||
expect(selected('test/fixtures/golden/claude-ship-SKILL.md')).toEqual(['goldens']);
|
||||
expect(selected('CLAUDE.md')).toEqual(['doc-caps']);
|
||||
expect(selected('docs/pregate.md')).toEqual([]);
|
||||
expect(selected('changelog.d/x.md')).toEqual([]);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,100 @@
|
||||
import { describe, test, expect } from 'bun:test';
|
||||
import { spawnSync } from 'node:child_process';
|
||||
import * as fs from 'node:fs';
|
||||
import * as os from 'node:os';
|
||||
import * as path from 'node:path';
|
||||
import { classifyDiff, depthOfCommitInput } from '../lib/review-depth';
|
||||
import { matchRisk } from '../lib/review-risk';
|
||||
import { receiptReviewAtHead } from '../lib/ship-receipt-depth';
|
||||
import { ARTIFACT_SCHEMAS, checkSchema, type ValidationError } from '../lib/headless-artifacts';
|
||||
|
||||
const BIN = path.join(import.meta.dir, '..', 'bin');
|
||||
const money = (p: string, lines: string[]) => matchRisk({ codePaths: [p], added: { [p]: lines }, scopeAuth: false }).get('money');
|
||||
|
||||
// GST-1129: gstack-evals#170 (GEVA-221) reserves and binds paid spend in quick/laneb/r3/screen.ts. These are its run() hunk's added
|
||||
// lines without the UBICLOUD_API_KEY line, which was the only thing that took the review to full depth.
|
||||
const SCREEN = 'quick/laneb/r3/screen.ts';
|
||||
const GEVA221 = [
|
||||
' const bound = ledger + ".bound";',
|
||||
' if (existsSync(bound)) throw new Error(boundTo());',
|
||||
' const runner = process.env.UBI_RUNNER ?? "";',
|
||||
' const ttl = process.env.UBI_TTL_HOURS || "3";',
|
||||
' bindCheck(ledger, m, configs);',
|
||||
' throw new Error(boundTo());',
|
||||
];
|
||||
|
||||
describe('GST-1129 item 1: code that reserves or binds spend is money (GEVA-221)', () => {
|
||||
test("GEVA-221's run() hunk, without the UBICLOUD_API_KEY line, plans full depth with money in the risk list", () => {
|
||||
expect(money(SCREEN, GEVA221)).toBe('money line: ' + SCREEN);
|
||||
const d = depthOfCommitInput({ paths: [SCREEN], gateTagged: 0, added: { [SCREEN]: GEVA221 }, perPath: [{ path: SCREEN, lines: GEVA221.length, binary: false }] });
|
||||
expect([d.depth, d.risk]).toEqual(['full', ['money']]);
|
||||
});
|
||||
test.each([['reserveSession(ledger, item)'], ['const ok = bindCap(item, cap);'], ['lease.settle(budget)'], ['await reserveSpend(id)'], ['if (spendCap === null) return;'], ['const leased = leaseCredits(n);']])('money: %s', line => {
|
||||
expect(money('src/run.ts', [line])).toBeDefined();
|
||||
});
|
||||
test.each([['appendEvent(m.ledger, { event: "rendered" });'], ['const bound = Math.min(a, b);'], ['reserve(slots);'], ['handler.bind(this);'], ['const cap = Math.max(0, n);']])('not money: %s', line => {
|
||||
expect(money('src/run.ts', [line])).toBeUndefined();
|
||||
});
|
||||
test('prose never names money by these pairs: skill text says binds and ledger in passing (#3698, #3798)', () => {
|
||||
expect(money('capy/skill/gstack/SKILL.md', ['Each pass binds its reviewers to the review ledger before Step 11.'])).toBeUndefined();
|
||||
expect(money('src/run.ts', ['Each pass binds its reviewers to the review ledger before Step 11.'])).toBeDefined();
|
||||
});
|
||||
test('code paths only (HARN-1): a data file and a stats script with the same words never name money', () => {
|
||||
expect(money('runs/item.json', [' "bindCheck": "ledger", "reserve": "budget"'])).toBeUndefined();
|
||||
expect(money('docs/results/bind-check.py', ['bindCheck(ledger, m)'])).toBeUndefined();
|
||||
});
|
||||
});
|
||||
|
||||
describe('GST-1129 item 2: a declared risk class only raises the depth', () => {
|
||||
const light = { paths: ['src/app.ts'], gateTagged: 0, perPath: [{ path: 'src/app.ts', lines: 10, binary: false }] };
|
||||
test('a class the matcher missed raises light to full, and the result keeps detected and declared apart', () => {
|
||||
expect(depthOfCommitInput(light).depth).toBe('light');
|
||||
const d = depthOfCommitInput({ ...light, declared: ['money'] });
|
||||
expect([d.depth, d.risk, d.detected, d.declared]).toEqual(['full', ['money'], [], ['money']]);
|
||||
});
|
||||
test('declaring nothing or another class never drops a detected one; a docs diff with a declared class plans full', () => {
|
||||
const c = classifyDiff({ paths: ['lib/billing.ts'], gateTagged: 0, declared: [] });
|
||||
expect([c.risk, c.detected, c.declared]).toEqual([['money'], ['money'], []]);
|
||||
const both = classifyDiff({ paths: ['lib/billing.ts'], gateTagged: 0, declared: ['auth'] });
|
||||
expect([both.risk, both.detected, both.declared]).toEqual([['money', 'auth'], ['money'], ['auth']]);
|
||||
expect(depthOfCommitInput({ paths: ['docs/a.md'], gateTagged: 0, declared: ['security'] }).depth).toBe('full');
|
||||
});
|
||||
|
||||
function repo(): { dir: string; head: string; state: string } {
|
||||
const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'declared-'));
|
||||
const state = fs.mkdtempSync(path.join(os.tmpdir(), 'declared-state-'));
|
||||
const git = (...a: string[]) => { const r = spawnSync('git', a, { cwd: dir, encoding: 'utf8', timeout: 30_000 }); if (r.status !== 0) throw new Error(r.stderr); return r.stdout.trim(); };
|
||||
git('init', '-q', '-b', 'main'); git('config', 'user.email', 't@example.com'); git('config', 'user.name', 't'); git('config', 'commit.gpgsign', 'false');
|
||||
fs.writeFileSync(path.join(dir, 'README.md'), '# x\n'); git('add', '.'); git('commit', '-qm', 'base');
|
||||
git('update-ref', 'refs/remotes/origin/main', 'HEAD');
|
||||
git('checkout', '-qb', 'feat');
|
||||
fs.mkdirSync(path.join(dir, 'src')); fs.writeFileSync(path.join(dir, 'src/app.ts'), 'export const a = 1;\n');
|
||||
git('add', '.'); git('commit', '-qm', 'change');
|
||||
return { dir, head: git('rev-parse', 'HEAD'), state };
|
||||
}
|
||||
test('gstack-review-plan --declare-risk prints the declared and detected classes and plans full; an unknown class is a usage error', () => {
|
||||
const r = repo();
|
||||
const plan = (...a: string[]) => spawnSync(path.join(BIN, 'gstack-review-plan'), ['--base', 'main', '--gate-tagged', '0', ...a], { cwd: r.dir, encoding: 'utf8', timeout: 60_000, env: { ...process.env, GSTACK_STATE_ROOT: r.state } });
|
||||
expect(plan().stdout).toMatch(/^REVIEW_DEPTH: light /m);
|
||||
const out = plan('--declare-risk', 'money').stdout;
|
||||
expect(out).toMatch(/^REVIEW_DEPTH: full .*reasons=risk:money,/m);
|
||||
expect(out).toContain('RISK_DECLARED: classes=money detected=none');
|
||||
const bad = plan('--declare-risk', 'cash');
|
||||
expect([bad.status, bad.stdout]).toEqual([2, '']);
|
||||
expect(bad.stderr).toContain('--declare-risk cash');
|
||||
}, 60_000);
|
||||
test('the receipt records declared and detected classes beside the raised depth, and the schema accepts them', () => {
|
||||
const r = repo();
|
||||
const prior = process.env.GSTACK_STATE_ROOT;
|
||||
process.env.GSTACK_STATE_ROOT = r.state;
|
||||
let d: ReturnType<typeof receiptReviewAtHead>['depth'];
|
||||
try {
|
||||
d = receiptReviewAtHead(r.dir, 'main', BIN, r.head, ['money']).depth;
|
||||
expect(receiptReviewAtHead(r.dir, 'main', BIN, r.head).depth).not.toHaveProperty('declared');
|
||||
} finally { if (prior === undefined) delete process.env.GSTACK_STATE_ROOT; else process.env.GSTACK_STATE_ROOT = prior; }
|
||||
expect([d.depth, d.risk, d.declared, d.detected]).toEqual(['full', ['money'], ['money'], []]);
|
||||
const errors: ValidationError[] = [];
|
||||
checkSchema({ schema_version: 1, head: r.head, base: 'main', review_depth: d }, ARTIFACT_SCHEMAS['ship-receipt'], '', errors);
|
||||
expect(errors.filter(e => JSON.stringify(e).includes('review_depth'))).toEqual([]);
|
||||
}, 60_000);
|
||||
});
|
||||
@@ -0,0 +1,246 @@
|
||||
import { describe, test, expect } from 'bun:test';
|
||||
import * as fs from 'node:fs';
|
||||
import * as path from 'node:path';
|
||||
import { classifyDiff, nextStep, planReview, platformFits, scanNumbers, type PlanInput, type ReviewShape } from '../lib/review-depth';
|
||||
import { appendEvent, verify } from '../lib/review-ledger';
|
||||
import * as os from 'node:os';
|
||||
import { createHash } from 'node:crypto';
|
||||
import { matchRisk } from '../lib/review-risk';
|
||||
import { renderBrief, sectionTexts, sliceDiff, sliceNote } from '../lib/review-brief';
|
||||
import { validatePolicy, DEFAULT_POLICY, REVIEW_POLICY_KEY_VERSION } from '../lib/ship-policy';
|
||||
|
||||
const ROOT = path.join(import.meta.dir, '..');
|
||||
const none = scanNumbers({});
|
||||
const HEAD = 'a'.repeat(40);
|
||||
const lean = (o: Partial<ReviewShape> = {}): ReviewShape => ({ panel: 'risk', readerModel: 'lane', platform: 'off', head: HEAD, platformReview: null, ...o });
|
||||
const base = (o: Partial<PlanInput>): PlanInput => ({ paths: ['src/app.ts'], scope: { SCOPE_BACKEND: 'true', SCOPE_STATE_MACHINE: 'false' }, gateTagged: 0, diffLines: 120, history: { fullPasses: 0, last: 'none' }, numbers: none, shape: lean(), ...o });
|
||||
const roles = (p: ReturnType<typeof planReview>) => p.dispatches.map(d => `${d.role}:${d.name}:${d.effort}:${d.model ?? '-'}`);
|
||||
|
||||
describe('HARN-1 false triggers (harness-evals#231 on 1.93.7.0): data, config and stats scripts never raise the depth', () => {
|
||||
const sqlite = 'docs/benchmarks/2026-11-01-installer-cc-codex/results/cc-none.smoke.ledger.sqlite';
|
||||
test('committed sqlite ledgers git cannot count are data, not code, and never name money', () => {
|
||||
const c = classifyDiff({ paths: [sqlite, 'README.md'], gateTagged: 0, perPath: [{ path: sqlite, lines: 0, binary: true }, { path: 'README.md', lines: 3, binary: false }], consumed: [] });
|
||||
expect([c.class, c.errors.map(e => e.code), c.risk]).toEqual(['docs', [], []]);
|
||||
});
|
||||
test('run logs are data too: *.log, *.out and extensionless files under logs/ never name money or security (subagent replay at f6a7eba)', () => {
|
||||
const logs = ['docs/b/results/cc-installer.plan.log', 'docs/b/results/trials/t1/r1/agent.out', 'docs/b/results/trials/t1/r1/logs/ready'];
|
||||
const c = classifyDiff({ paths: logs, gateTagged: 0, added: Object.fromEntries(logs.map(p => [p, ['spend_cap_usd=5 fetch(url) rm -rf /tmp/x']])) });
|
||||
expect([c.class, c.risk]).toEqual(['docs', []]);
|
||||
expect(classifyDiff({ paths: ['src/logs/rotate.ts'], gateTagged: 0 }).class).toBe('code');
|
||||
});
|
||||
test('a sqlite file is data even when a script names it', () => {
|
||||
const c = classifyDiff({ paths: [sqlite, 'scripts/load.py'], gateTagged: 0, perPath: [{ path: sqlite, lines: 0, binary: true }, { path: 'scripts/load.py', lines: 4, binary: false }], consumed: [sqlite] });
|
||||
expect([c.class, c.errors.map(e => e.code), c.runtimeLines]).toEqual(['code', [], 4]);
|
||||
});
|
||||
test('money on lease_usd lines of a JSON config a script reads is not a risk class; the config is judged by its path', () => {
|
||||
const p = 'docs/benchmarks/x/results/cc-none.shard-vm.json';
|
||||
expect(matchRisk({ codePaths: [p], added: { [p]: [' "lease_usd": 32.9,', ' "list_price_usd": 1.2485,'] }, scopeAuth: false }).size).toBe(0);
|
||||
expect(matchRisk({ codePaths: ['config/billing.json'], added: { 'config/billing.json': ['{}'] }, scopeAuth: false }).has('money')).toBe(true);
|
||||
expect(matchRisk({ codePaths: ['lib/lease.ts'], added: { 'lib/lease.ts': ['const lease_usd = 3;'] }, scopeAuth: false }).has('money')).toBe(true);
|
||||
});
|
||||
test('a stats script under a results directory is judged by its path: no security or money from its name or lines', () => {
|
||||
const p = 'docs/benchmarks/2026-11-01-installer-cc-codex/results/cost-intervals.py';
|
||||
expect(matchRisk({ codePaths: [p], added: { [p]: ["rows = [json.loads(l) for l in open(os.path.join(sys.argv[1], 'trials.jsonl'))]"] }, scopeAuth: false }).size).toBe(0);
|
||||
expect(matchRisk({ codePaths: ['tools/fetch.py'], added: { 'tools/fetch.py': ['p = os.path.join(sys.argv[1], name)'] }, scopeAuth: false }).has('security')).toBe(true);
|
||||
});
|
||||
test('a paid-provider call, a spend cap or a key in a results script or config file still counts', () => {
|
||||
const run = 'benchmarks/run_models.py', sweep = 'analysis/sweep.py', cfg = 'evals/config.yaml';
|
||||
expect(matchRisk({ codePaths: [run], added: { [run]: ['resp = client.messages.create(model=m)'] }, scopeAuth: false }).has('money')).toBe(true);
|
||||
expect(matchRisk({ codePaths: [sweep], added: { [sweep]: ['MAX = spend_cap(5)'] }, scopeAuth: false }).has('money')).toBe(true);
|
||||
expect(matchRisk({ codePaths: [run], added: { [run]: ['key = os.environ["OPENAI_API_KEY"]'] }, scopeAuth: false }).has('secret')).toBe(true);
|
||||
expect(matchRisk({ codePaths: [cfg], added: { [cfg]: ['spend_cap: 9999'] }, scopeAuth: false }).has('money')).toBe(true);
|
||||
expect(matchRisk({ codePaths: [cfg], added: { [cfg]: ['lease_usd: 3'] }, scopeAuth: false }).has('money')).toBe(false);
|
||||
expect(matchRisk({ codePaths: ['lib/analysis/billing.py'], added: { 'lib/analysis/billing.py': ['charge(customer, amount_usd)'] }, scopeAuth: false }).has('money')).toBe(true);
|
||||
expect(matchRisk({ codePaths: ['docs/conf.py'], added: { 'docs/conf.py': ['p = os.path.join(sys.argv[1], name)'] }, scopeAuth: false }).has('security')).toBe(true);
|
||||
});
|
||||
test('a secret in a config file still counts', () => {
|
||||
expect(matchRisk({ codePaths: ['deploy/app.yaml'], added: { 'deploy/app.yaml': [' key: ${OPENAI_API_KEY}'] }, scopeAuth: false }).has('secret')).toBe(true);
|
||||
});
|
||||
});
|
||||
|
||||
describe('FOUN-64 item 4: docs that can print a credential are security risk (gbrain#8417)', () => {
|
||||
const docsWith = (line: string) => classifyDiff({ paths: ['docs/mcp.md'], gateTagged: 0, added: { 'docs/mcp.md': [line] } });
|
||||
test.each([['$ claude mcp get gbrain'], ['Authorization: Bearer abcdefgh12345678'], ['export OPENAI_API_KEY=sk-live-0123456789abcdefghij'], ['gh auth token']])('%s', l => expect(docsWith(l).risk).toEqual(['security']));
|
||||
test('ordinary docs stay risk-free', () => expect(docsWith('Run gstack-review-plan --base main to see the plan.').risk).toEqual([]));
|
||||
});
|
||||
|
||||
describe('FOUN-32 §4: eval scorers, judges and graders are risk class eval by path', () => {
|
||||
test.each([['lib/eval-scorer.ts'], ['evals/judges/rubric.py'], ['scripts/grade-trials.ts']])('%s', p => expect(classifyDiff({ paths: [p], gateTagged: 0 }).risk).toContain('eval'));
|
||||
test('a test helper named judge stays tests class, and an ordinary path is not eval', () => {
|
||||
expect(classifyDiff({ paths: ['test/helpers/llm-judge.ts'], gateTagged: 0 }).risk).toEqual([]);
|
||||
expect(classifyDiff({ paths: ['lib/review-pass.ts'], gateTagged: 0 }).risk).not.toContain('eval');
|
||||
});
|
||||
});
|
||||
|
||||
describe('risk_paths: a repo extends the risk classes from its base policy (FOUN-64)', () => {
|
||||
const rp = ['src/balance/**', 'src/outcome*.ts'];
|
||||
test('a matching code path is risk class policy; data and docs under the glob are not', () => {
|
||||
expect(classifyDiff({ paths: ['src/balance/view.tsx'], gateTagged: 0, riskPaths: rp }).risk).toEqual(['policy']);
|
||||
expect(classifyDiff({ paths: ['src/outcome-score.ts'], gateTagged: 0, riskPaths: rp }).risk).toEqual(['policy']);
|
||||
expect(classifyDiff({ paths: ['src/balance/snap.sqlite', 'src/balance/notes.md'], gateTagged: 0, riskPaths: rp, consumed: [] }).risk).toEqual([]);
|
||||
expect(classifyDiff({ paths: ['src/app.ts'], gateTagged: 0, riskPaths: rp }).risk).toEqual([]);
|
||||
});
|
||||
test('a policy risk path plans the merged reader at full depth', () => {
|
||||
const p = planReview(base({ paths: ['src/balance/x.ts'], riskPaths: rp, shape: lean({ panel: 'off' }) }));
|
||||
expect([p.depth, roles(p)]).toEqual(['full', ['adversarial-native:merged:full:lane']]);
|
||||
expect(planReview(base({ paths: ['src/balance/x.ts'], riskPaths: rp })).mode).toBe('panel');
|
||||
});
|
||||
});
|
||||
|
||||
describe('R65 lean dispatch (GSTA-258 build C): one reviewer, once', () => {
|
||||
test('non-risk code: one native reader at the lane\'s reasoning carrying the adversarial brief, the checklists and the Red Team', () => {
|
||||
const p = planReview(base({}));
|
||||
expect([p.mode, roles(p)]).toEqual(['lean', ['adversarial-native:single:full:lane']]);
|
||||
expect(p.dispatches[0]!.checklists).toEqual(['testing', 'security', 'red-team']);
|
||||
expect(p.skipped.join('\n')).toMatch(/outside-voice/);
|
||||
});
|
||||
test('non-risk code that publishes numbers keeps the native reader by default (FOUN-32 §3, ggstack#52); cross-family is a setting', () => {
|
||||
const p = planReview(base({ paths: ['src/app.ts', 'docs/results.md'], numbers: scanNumbers({ 'docs/results.md': ['recall 87% (26/30)'] }) }));
|
||||
expect(roles(p)).toEqual(['adversarial-native:single:full:lane']);
|
||||
expect(roles(planReview(base({ shape: lean({ readerModel: 'cross-family' }) })))).toEqual(['adversarial-native:single:full:cross-family']);
|
||||
});
|
||||
test('panel risk (opt-in, FOUN-64 item 1): a risk-class diff gets the printed full-depth panel; a fixed blocker re-runs the printed plan as round 2 (FOUN-32 §5)', () => {
|
||||
const today = roles(planReview(base({ paths: ['lib/billing.ts'], targeted: { classes: ['money'] }, shape: undefined })));
|
||||
const p = planReview(base({ paths: ['lib/billing.ts'], targeted: { classes: ['money'] } }));
|
||||
expect([p.depth, p.mode, roles(p)]).toEqual(['full', 'panel', today]);
|
||||
expect(today.length).toBeGreaterThan(1);
|
||||
expect(planReview(base({ paths: ['lib/billing.ts'], history: { fullPasses: 1, last: 'full', fixedBlocking: true } })).next.step).toBe('full');
|
||||
expect(planReview(base({ paths: ['lib/billing.ts'], history: { fullPasses: 2, last: 'full', fixedBlocking: true } })).next.step).toBe('stop');
|
||||
expect(planReview(base({ history: { fullPasses: 1, last: 'full', fixedBlocking: true } })).next.step).toBe('fix-check');
|
||||
});
|
||||
test('panel off: risk class gets one reader on the configured model with the merged panel brief; it carries the targeted classes', () => {
|
||||
const p = planReview(base({ paths: ['lib/billing.ts'], targeted: { classes: ['money'] }, shape: lean({ panel: 'off' }) }));
|
||||
expect([p.depth, roles(p)]).toEqual(['full', ['adversarial-native:merged:full:lane']]);
|
||||
expect(p.dispatches[0]!.classes).toEqual(['money']);
|
||||
expect(p.dispatches[0]!.checklists).toContain('red-team');
|
||||
const opus = planReview(base({ paths: ['.github/workflows/ci.yml'], shape: lean({ panel: 'off', readerModel: 'anthropic/claude-opus-5-5' }) }));
|
||||
expect(roles(opus)).toEqual(['adversarial-native:merged:full:anthropic/claude-opus-5-5']);
|
||||
});
|
||||
test('docs and tests stay core only; docs with numbers keep the outside voice', () => {
|
||||
expect(planReview(base({ paths: ['README.md'], gateTagged: null })).dispatches).toEqual([]);
|
||||
expect(roles(planReview(base({ paths: ['docs/r.md'], numbers: scanNumbers({ 'docs/r.md': ['cost $4.10'] }) })))).toEqual(['outside-voice:outside-voice:host:-']);
|
||||
});
|
||||
test('panel all is today\'s panel; panel risk keeps it on risk class only', () => {
|
||||
const today = (o: Partial<PlanInput>) => roles(planReview(base({ ...o, shape: undefined })));
|
||||
expect(roles(planReview(base({ paths: ['lib/billing.ts'], shape: lean({ panel: 'all' }) })))).toEqual(today({ paths: ['lib/billing.ts'] }));
|
||||
expect(roles(planReview(base({ paths: ['lib/billing.ts'], shape: lean({ panel: 'risk' }) })))).toEqual(today({ paths: ['lib/billing.ts'] }));
|
||||
expect(planReview(base({ shape: lean({ panel: 'risk' }) })).mode).toBe('lean');
|
||||
expect(planReview(base({ shape: lean({ panel: 'all' }) })).mode).toBe('panel');
|
||||
});
|
||||
});
|
||||
|
||||
describe('C6 platform depth (FOUN-32, founderstar#496): a planned skip needs the platform review at the exact head', () => {
|
||||
test('runtime code never fits the platform depth, even with no runtime lines (a registered generated output, a mode-only change)', () => {
|
||||
const none = scanNumbers({});
|
||||
expect(platformFits('code', ['dist/app.js'], 0, none)).toBe(false);
|
||||
expect(platformFits('gate', ['scripts/start.sh'], 0, none)).toBe(false);
|
||||
expect(platformFits('docs', ['docs/a.md'], 0, none)).toBe(true);
|
||||
expect(platformFits('tests', ['test/a.test.ts'], 0, none)).toBe(true);
|
||||
});
|
||||
test('verify marks an outside-voice diff file changed after render as a mismatch', () => {
|
||||
const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'payload-'));
|
||||
const ledger = path.join(dir, 'ledger.jsonl'), brief = path.join(dir, 'brief.md'), diff = path.join(dir, 'outside-voice.diff');
|
||||
fs.writeFileSync(brief, 'brief'); fs.writeFileSync(diff, '+code\n');
|
||||
const sha = (s: string) => createHash('sha256').update(s).digest('hex');
|
||||
appendEvent(ledger, { event: 'rendered', reader: 'outside-voice', role: 'outside-voice', file: brief, sha256: sha('brief'), payload: { file: diff, sha256: sha('+code\n') } } as any);
|
||||
expect(verify(ledger).rows.find(r => r.kind === 'BRIEF')!.detail).not.toContain('payload_changed_since_render');
|
||||
fs.writeFileSync(diff, '+docs only\n');
|
||||
const row = verify(ledger).rows.find(r => r.kind === 'BRIEF')!;
|
||||
expect([row.status, row.detail.includes('payload_changed_since_render')]).toEqual(['MISMATCH', true]);
|
||||
});
|
||||
const on = (review: ReviewShape['platformReview']) => lean({ platform: 'on', platformReview: review });
|
||||
const docs = { paths: ['docs/howto-install.md', 'test/a.test.ts'] };
|
||||
test('platform on with the review at HEAD on a docs or tests diff: no reader, a planned SKIP for every reader, the core read included', () => {
|
||||
const p = planReview(base({ ...docs, shape: on({ id: '5744388738', head: HEAD, reviewer: 'capy-ai[bot]' }) }));
|
||||
expect([p.mode, p.next.step, p.dispatches]).toEqual(['platform', 'platform', []]);
|
||||
expect(p.skipped.filter(s => /planned skip/.test(s)).length).toBeGreaterThanOrEqual(2);
|
||||
expect(p.skipped.join('\n')).toMatch(/core.*planned skip.*5744388738/);
|
||||
});
|
||||
test('without the evidence, or with it at another head, the core read stands', () => {
|
||||
expect(planReview(base({ ...docs, shape: on(null) })).mode).toBe('lean');
|
||||
expect(planReview(base({ ...docs, shape: on({ id: '1', head: 'b'.repeat(40), reviewer: 'capy-ai[bot]' }) })).next.step).toBe('full');
|
||||
});
|
||||
test('FOUN-64 item 2: runtime code, results or prereg paths never take the platform depth', () => {
|
||||
const ev = { id: '1', head: HEAD, reviewer: 'capy-ai[bot]' };
|
||||
expect(roles(planReview(base({ shape: on(ev) })))).toEqual(['adversarial-native:single:full:lane']);
|
||||
expect(planReview(base({ paths: ['docs/benchmarks/run/results/summary.md'], shape: on(ev) })).mode).toBe('lean');
|
||||
expect(planReview(base({ paths: ['evals/prereg.md'], shape: on(ev) })).mode).toBe('lean');
|
||||
});
|
||||
test('risk class and published numbers never take the platform depth', () => {
|
||||
const ev = { id: '1', head: HEAD, reviewer: 'capy-ai[bot]' };
|
||||
expect(planReview(base({ paths: ['lib/billing.ts'], shape: on(ev) })).mode).toBe('panel');
|
||||
expect(planReview(base({ paths: ['src/app.ts', 'docs/r.md'], numbers: scanNumbers({ 'docs/r.md': ['p50 4.7 h'] }), shape: on(ev) })).mode).toBe('lean');
|
||||
});
|
||||
test('under the default panel all, platform on still takes the platform depth at HEAD; everything else keeps today\'s panel', () => {
|
||||
const ev = { id: '1', head: HEAD, reviewer: 'capy-ai[bot]' };
|
||||
const all = (review: ReviewShape['platformReview']) => lean({ panel: 'all', platform: 'on', platformReview: review });
|
||||
expect(planReview(base({ ...docs, shape: all(ev) })).mode).toBe('platform');
|
||||
expect(planReview(base({ ...docs, shape: all(null) })).mode).toBe('panel');
|
||||
expect(planReview(base({ shape: all(ev) })).mode).toBe('panel');
|
||||
});
|
||||
test('the platform skip is pass 1 only: a later step plans as lean', () => {
|
||||
const p = planReview(base({ shape: on({ id: '1', head: HEAD, reviewer: 'capy-ai[bot]' }), history: { fullPasses: 1, last: 'full', fixedBlocking: true } }));
|
||||
expect(p.next.step).toBe('fix-check');
|
||||
});
|
||||
});
|
||||
|
||||
describe('R65 lean loop: one fix round, never more than 2 rounds', () => {
|
||||
const h = (o: object) => ({ fullPasses: 1, last: 'full' as const, lean: true, ...o });
|
||||
test('non-risk: a fixed blocker earns one fix-check; below-floor fixes ship as notes', () => {
|
||||
expect(nextStep(h({ fixedBlocking: true }), 'light').step).toBe('fix-check');
|
||||
expect(nextStep(h({ fixedBelow: true }), 'light')).toMatchObject({ step: 'done', reason: expect.stringContaining('notes') });
|
||||
});
|
||||
test('risk: any fix earns one scoped fix-check, a gate-class edit included, never a second full pass', () => {
|
||||
for (const o of [{ fixedBlocking: true }, { fixedBelow: true }, { fixedBelow: true, touchedGate: true }]) expect(nextStep(h(o), 'full').step).toBe('fix-check');
|
||||
});
|
||||
test('a fix-check blocker stops for the owner; a recorded grant adds one fix-check, never two', () => {
|
||||
expect(nextStep(h({ last: 'fix-check', fixedBlocking: true, fixChecks: 1 }), 'full').step).toBe('stop');
|
||||
expect(nextStep(h({ last: 'fix-check', fixedBlocking: true, fixChecks: 1, granted: 'apr_1' }), 'full').step).toBe('fix-check');
|
||||
expect(nextStep(h({ last: 'fix-check', fixedBlocking: true, fixChecks: 2, granted: 'apr_1' }), 'full').step).toBe('stop');
|
||||
expect(nextStep(h({ last: 'fix-check', fixedBlocking: true, granted: 'apr_1' }), 'full').step).toBe('stop');
|
||||
});
|
||||
test('planReview marks the loop lean and a fix-check dispatches one full-effort reader', () => {
|
||||
expect(roles(planReview(base({ paths: ['lib/billing.ts'], history: { fullPasses: 1, last: 'full', fixedBlocking: true }, shape: lean({ panel: 'off' }) })))).toEqual(['fix-check:fix-check:full:-']);
|
||||
});
|
||||
});
|
||||
|
||||
describe('C7 merged reader brief and the outside voice reading its diff from disk', () => {
|
||||
const t = sectionTexts(ROOT);
|
||||
const sha = 'c'.repeat(40);
|
||||
test('the merged reader renders the targeted brief and every checklist in one native adversarial context', () => {
|
||||
const brief = fs.readFileSync(path.join(ROOT, 'review/specialists/targeted.md'), 'utf8');
|
||||
const text = renderBrief('adversarial-native', { diff_base: sha, checklists: '/x/testing.md,/x/red-team.md', classes: 'money', brief, checklist: '/x/targeted.md' }, t);
|
||||
expect(text).toContain(t.native.slice(0, 60));
|
||||
expect(text).toContain('/x/testing.md, /x/red-team.md');
|
||||
expect(text).toMatch(/money/);
|
||||
});
|
||||
test('past the cap the outside voice reads a path-scoped slice that names what it left out (FOUN-32 §3)', () => {
|
||||
const sec = (p: string, n: number) => `diff --git a/${p} b/${p}\n+${'x'.repeat(n)}\n`;
|
||||
const s = sliceDiff(sec('a.ts', 10) + sec('big.json', 500) + sec('b.ts', 10), 100);
|
||||
expect([s.sliced, s.kept, s.files, s.left]).toEqual([true, 2, 3, ['big.json']]);
|
||||
expect(s.text).toContain('b/b.ts');
|
||||
expect(sliceNote(s, 100)).toMatch(/2 of 3 files.*big\.json/);
|
||||
expect(sliceDiff(sec('a.ts', 10), 100).sliced).toBe(false);
|
||||
});
|
||||
test('the outside voice gets a path and a hash, not the pasted diff', () => {
|
||||
const text = renderBrief('outside-voice', { diff_base: sha, diff_file: '/tmp/o/outside-voice.diff', diff_sha256: 'd'.repeat(64), diff_bytes: '1234' }, t);
|
||||
expect(text).toContain('/tmp/o/outside-voice.diff');
|
||||
expect(text).toContain('d'.repeat(64));
|
||||
expect(text).not.toContain('```diff');
|
||||
});
|
||||
});
|
||||
|
||||
describe('ship policy: the base-read review key', () => {
|
||||
const codes = (raw: object) => validatePolicy(raw).map(e => `${e.code}:${e.path}`);
|
||||
test('defaults: panel all (today\'s panel; risk is opt-in until its no-loss replay passes), the lane\'s native reader, platform off, no extra risk paths', () => expect(DEFAULT_POLICY.review).toEqual({ panel: 'all', reader_model: 'lane', platform: 'off', risk_paths: [] }));
|
||||
test('valid values pass with min_gstack_version; bad ones name the field', () => {
|
||||
const v = REVIEW_POLICY_KEY_VERSION;
|
||||
expect(codes({ review: { platform: 'on', reader_model: 'anthropic/claude-opus-5-5', panel: 'risk', risk_paths: ['src/balance/**'] }, min_gstack_version: v })).toEqual([]);
|
||||
expect(codes({ review: { platform: 'yes' }, min_gstack_version: v })).toEqual(['POLICY_INVALID:review.platform']);
|
||||
expect(codes({ review: { panel: 'some' }, min_gstack_version: v })).toEqual(['POLICY_INVALID:review.panel']);
|
||||
expect(codes({ review: { reader_model: 'a b' }, min_gstack_version: v })).toEqual(['POLICY_INVALID:review.reader_model']);
|
||||
expect(codes({ review: { risk_paths: ['../x'] }, min_gstack_version: v })).toEqual(['POLICY_CONTAINMENT:review.risk_paths[0]']);
|
||||
expect(codes({ review: { other: 1 }, min_gstack_version: v })).toEqual(['POLICY_INVALID:review.other']);
|
||||
expect(codes({ review: { platform: 'on' } })).toContain('POLICY_INVALID:review');
|
||||
});
|
||||
});
|
||||
@@ -118,8 +118,11 @@ describe('what review_exclude_paths excludes and keeps', () => {
|
||||
for (const f of KEPT) expect(read.has(f)).toBe(true);
|
||||
const outside = briefs.find(b => b.includes('## Branch diff'))!;
|
||||
expect(outside).toContain(`git diff "$DIFF_BASE" -- . ${m.readers[0].policy_exclude} and git diff "$DIFF_BASE" -- '${reader.keep[0]}'`);
|
||||
for (const f of EXCLUDED) expect(outside).not.toContain(`diff --git a/${f}`);
|
||||
expect(outside).toContain('diff --git a/receipts/b/result.json');
|
||||
const diffFile = path.join(out, 'briefs', 'outside-voice.diff');
|
||||
expect(outside).toContain(`The supplied branch diff is the file ${diffFile} `);
|
||||
const supplied = fs.readFileSync(diffFile, 'utf8');
|
||||
for (const f of EXCLUDED) expect(supplied).not.toContain(`diff --git a/${f}`);
|
||||
expect(supplied).toContain('diff --git a/receipts/b/result.json');
|
||||
for (const b of briefs) expect(b).toContain(`(selection ${m.review_exclude.digest})`);
|
||||
}, 180_000);
|
||||
|
||||
@@ -150,7 +153,8 @@ describe('what review_exclude_paths excludes and keeps', () => {
|
||||
const c = JSON.parse(plan(control, ['--json']).out);
|
||||
const w = JSON.parse(plan(with_, ['--json']).out);
|
||||
expect(c.numbers.publishes).toBe(true);
|
||||
expect(c.targeted.classes).toEqual(['money', 'secret']);
|
||||
// HARN-1 (harness-evals#231): a JSON data file is judged by its path, so cost_usd names no money even unexcluded; its secret line still counts
|
||||
expect(c.targeted.classes).toEqual(['secret']);
|
||||
expect(w.review_exclude.kept_files).toEqual([{ path: 'receipts/s.json', reason: 'secret-line' }]);
|
||||
expect(w.numbers.publishes).toBe(false);
|
||||
expect(w.targeted.classes).toEqual(['secret']);
|
||||
|
||||
@@ -94,20 +94,23 @@ describe('pass manifest', () => {
|
||||
expect(() => renderStage(d, 'specialists', texts, () => '')).toThrow(/cannot read audit\/plan's prompt file/);
|
||||
});
|
||||
|
||||
test('a brief over 1 MiB with invalid UTF-8 in its diff travels in bounded chunks that rejoin to the rendered sha', () => {
|
||||
const m = manifest([{ role: 'outside-voice', name: 'outside-voice', effort: 'host', exclude: [] }]);
|
||||
const diff = Buffer.concat([Buffer.from('+line \u00e9\n'.repeat(90_000)), Buffer.from([0xff, 0xfe, 0x0a]), Buffer.from('+"quoted" \\ back\n'.repeat(20_000))]).toString('utf8');
|
||||
renderStage(m, 'voices', texts, () => diff);
|
||||
// The outside voice now reads its diff from a file (test/review-depth-lean.test.ts), so a large brief here is an audit
|
||||
// reader's prompt file carrying the same bytes.
|
||||
test('a brief over 1 MiB with invalid UTF-8 in it travels in bounded chunks that rejoin to the rendered sha', () => {
|
||||
const big = path.join(tmp('rp-big-'), 'plan.md');
|
||||
fs.writeFileSync(big, Buffer.concat([Buffer.from('+line \u00e9\n'.repeat(90_000)), Buffer.from([0xff, 0xfe, 0x0a]), Buffer.from('+"quoted" \\ back\n'.repeat(20_000))]));
|
||||
const m = manifest([], { extras: [{ name: 'plan', prompt_file: big }] });
|
||||
renderStage(m, 'specialists', texts, () => '');
|
||||
const r = m.readers[0]!;
|
||||
const text = fs.readFileSync(r.file!, 'utf8');
|
||||
expect(Buffer.byteLength(text)).toBeGreaterThan(1024 * 1024);
|
||||
expect(r.chunks).toBe(chunkCount(text));
|
||||
let joined = '';
|
||||
for (let i = 0; i < r.chunks!; i++) {
|
||||
const c = spawnSync(path.join(BIN, 'gstack-review-dispatch'), ['chunk', '--manifest', writeManifestFile(m), '--reader', 'outside-voice', '--index', String(i)], { encoding: 'utf8', timeout: 30_000, maxBuffer: 8 * 1024 * 1024 });
|
||||
const c = spawnSync(path.join(BIN, 'gstack-review-dispatch'), ['chunk', '--manifest', writeManifestFile(m), '--reader', 'audit/plan', '--index', String(i)], { encoding: 'utf8', timeout: 30_000, maxBuffer: 8 * 1024 * 1024 });
|
||||
expect(c.status, c.stderr).toBe(0);
|
||||
const [head, body] = c.stdout.split('\n');
|
||||
expect(head).toBe('BRIEF_CHUNK: reader=outside-voice index=' + i + ' of=' + r.chunks + ' sha256=' + r.sha256);
|
||||
expect(head).toBe('BRIEF_CHUNK: reader=audit/plan index=' + i + ' of=' + r.chunks + ' sha256=' + r.sha256);
|
||||
expect(Buffer.byteLength(body!)).toBeLessThan(1024 * 1024);
|
||||
joined += JSON.parse(body!);
|
||||
}
|
||||
@@ -1024,7 +1027,9 @@ describe('a scoped pass after a blocker fix (GST-619)', () => {
|
||||
expect(redTeam).not.toContain(s.mergeBase);
|
||||
expect(bin(s.dir, s.env, ['stage', '--manifest', s.mf, '--stage', 'voices']).status).toBe(0);
|
||||
const voices = JSON.parse(fs.readFileSync(s.mf, 'utf8')) as Manifest;
|
||||
expect(fs.readFileSync(voices.readers.find(r => r.reader === 'outside-voice')!.file!, 'utf8')).toContain('+export const v1 = 1;');
|
||||
const outsideBrief = voices.readers.find(r => r.reader === 'outside-voice')!.file!;
|
||||
expect(fs.readFileSync(outsideBrief, 'utf8')).toContain(`The supplied branch diff is the file ${path.join(path.dirname(outsideBrief), 'outside-voice.diff')} `);
|
||||
expect(fs.readFileSync(path.join(path.dirname(outsideBrief), 'outside-voice.diff'), 'utf8')).toContain('+export const v1 = 1;');
|
||||
const again = bin(s.dir, s.env, s.args);
|
||||
expect(again.status, again.stdout + again.stderr).toBe(0);
|
||||
expect(again.stdout).toMatch(/^FIX_CHECK: planned effort=full /m);
|
||||
|
||||
@@ -121,7 +121,7 @@ describe('gstack-review-plan', () => {
|
||||
}
|
||||
expect(plan([...capStop.slice(0, -2), '--grant', id]).status).toBe(2);
|
||||
expect(plan([...capStop, '--grant', '--json']).status).toBe(2);
|
||||
});
|
||||
}, 30_000);
|
||||
test('--fix-check-prompt renders the fix-check reviewer prompt from the parent\'s files and refuses a missing one', () => {
|
||||
const r = repo({ 'docs/howto.md': '# x\n' });
|
||||
const w = (n: string, t: string) => { const f = path.join(r.state, n); fs.writeFileSync(f, t); return f; };
|
||||
@@ -271,7 +271,7 @@ describe('gstack-review-plan: the scoped second pass (GSTA-152 D6)', () => {
|
||||
expect(dirty).toMatch(/^FIX_RANGE: unread \(uncommitted or untracked edits.*\)/m);
|
||||
expect(dirty).toMatch(/^NEXT: full /m);
|
||||
expect(run(r, [...pass2, '--range-base']).status).toBe(2);
|
||||
});
|
||||
}, 30_000);
|
||||
test('a fix range that touches a gate path keeps the full pass 2 without --touched-gate, and an empty range is unread', () => {
|
||||
const r = repo({ 'src/a.ts': code(80), '.github/workflows/ci.yml': 'on: push\n' });
|
||||
const git = (...a: string[]) => spawnSync('git', a, { cwd: r.dir, encoding: 'utf8', timeout: 10_000 });
|
||||
|
||||
@@ -293,6 +293,8 @@ describe('setup: registers the hooks that parse, refuses the rest, exits non-zer
|
||||
GSTACK_STATE_ROOT: path.join(fx.home, '.gstack'),
|
||||
GSTACK_SETTINGS_FILE: fx.settings,
|
||||
GSTACK_PLAN_TUNE_HOOKS: 'yes',
|
||||
GSTACK_SKILL_ROUTING: 'off',
|
||||
GSTACK_SKILL_CONTRACTS: 'off',
|
||||
TMPDIR: fx.base,
|
||||
};
|
||||
// A file, not `bash -c`: the script is longer than Windows' 32,767-character command line.
|
||||
|
||||
@@ -48,8 +48,8 @@ describe('Ubuntu fallback retains CPU architecture', () => {
|
||||
cwd: ROOT, env: { PATH: process.env.PATH, HOME: temp(), PLAYWRIGHT_HOST_PLATFORM_OVERRIDE: override }, encoding: 'utf8', timeout: 20_000,
|
||||
});
|
||||
expect(dryRun.status, dryRun.stderr).toBe(0);
|
||||
expect(dryRun.stdout).toContain(expected === 'arm64' ? 'chromium-linux-arm64.zip' : 'chrome-linux64.zip');
|
||||
expect(dryRun.stdout).not.toContain(expected === 'arm64' ? 'chrome-linux64.zip' : 'chromium-linux-arm64.zip');
|
||||
expect(dryRun.stdout).toContain(expected === 'arm64' ? 'chrome-linux-arm64.zip' : 'chrome-linux64.zip');
|
||||
expect(dryRun.stdout).not.toContain(expected === 'arm64' ? 'chrome-linux64.zip' : 'chrome-linux-arm64.zip');
|
||||
}, 30_000);
|
||||
}
|
||||
for (const cpu of ['riscv64', 'i686', 'unknown']) test(`${cpu} reports unsupported instead of installing x64`, () => {
|
||||
|
||||
@@ -5,12 +5,18 @@
|
||||
* settings.json: default off registers nothing, a flag persists and registers
|
||||
* once with a one-time line, a typo warns and is not persisted, and off
|
||||
* removes the registration. Static pins cover the flags and the parse gate.
|
||||
* Each switch's default lives only in SWITCH_DEFAULT (lib/skill-route.ts): a
|
||||
* copy with one table line flipped changes what setup registers, what
|
||||
* gstack-config reports and what the hook does.
|
||||
*/
|
||||
import { afterEach, beforeEach, describe, expect, test } from 'bun:test';
|
||||
import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, test } from 'bun:test';
|
||||
import { spawnSync } from 'node:child_process';
|
||||
import * as fs from 'node:fs';
|
||||
import * as path from 'node:path';
|
||||
import { ROOT, tmpDir } from './helpers/skill-contract-fixture';
|
||||
import { SWITCH_DEFAULT } from '../lib/skill-route';
|
||||
import {
|
||||
J1_TRANSCRIPT, ROOT, hookEnv, runHook, skillsDir, tmpDir,
|
||||
} from './helpers/skill-contract-fixture';
|
||||
|
||||
const setupSrc = fs.readFileSync(path.join(ROOT, 'setup'), 'utf8');
|
||||
const settingsHookSrc = fs.readFileSync(path.join(ROOT, 'bin', 'gstack-settings-hook'), 'utf8');
|
||||
@@ -23,26 +29,31 @@ afterEach(() => { for (const d of [state, home]) fs.rmSync(d, { recursive: true,
|
||||
const settingsFile = () => path.join(home, 'settings.json');
|
||||
const settings = () => (fs.existsSync(settingsFile()) ? JSON.parse(fs.readFileSync(settingsFile(), 'utf8')) : {});
|
||||
|
||||
/** Run the routing gate once: mode is the flag value ("" = none). */
|
||||
function gate(mode: string, env: Record<string, string> = {}, noTeam = 0): { code: number | null; out: string } {
|
||||
const hook = path.join(ROOT, 'hosts', 'claude', 'hooks', 'skill-route-hook');
|
||||
const GATES = {
|
||||
skill_routing: 'GSTACK_SKILL_ROUTING UserPromptSubmit "$2/hosts/claude/hooks/skill-route-hook" gstack-skill-route 5 "routing hint"',
|
||||
skill_contracts: 'GSTACK_SKILL_CONTRACTS Stop "$2/hosts/claude/hooks/skill-contract-stop-hook" gstack-skill-contract 10 "contract check"',
|
||||
};
|
||||
|
||||
/** Run one gate once from the checkout at root: mode is the flag value ("" = none). */
|
||||
function gate(mode: string, env: Record<string, string> = {}, noTeam = 0, root = ROOT, key: keyof typeof GATES = 'skill_routing'): { code: number | null; out: string } {
|
||||
const script = [
|
||||
'set -e',
|
||||
'log() { printf "%s\\n" "$*"; }',
|
||||
`GSTACK_CONFIG="${path.join(ROOT, 'bin', 'gstack-config')}"`,
|
||||
'GSTACK_CONFIG="$2/bin/gstack-config"',
|
||||
`SETTINGS_HOOK="${path.join(ROOT, 'bin', 'gstack-settings-hook')}"`,
|
||||
'SKILL_ROUTE_BIN="$2/bin/gstack-skill-route"',
|
||||
`IS_WINDOWS=0; NO_TEAM_MODE=${noTeam}`,
|
||||
gateFn,
|
||||
`_skill_hook_gate skill_routing "$1" GSTACK_SKILL_ROUTING UserPromptSubmit "${hook}" gstack-skill-route 5 "routing hint"`,
|
||||
`_skill_hook_gate ${key} "$1" ${GATES[key]}`,
|
||||
].join('\n');
|
||||
const r = spawnSync('bash', ['-c', script, 'gate', mode], {
|
||||
const r = spawnSync('bash', ['-c', script, 'gate', mode, root], {
|
||||
env: { PATH: process.env.PATH ?? '', HOME: home, GSTACK_STATE_ROOT: state, GSTACK_SETTINGS_FILE: settingsFile(), ...env },
|
||||
encoding: 'utf8', timeout: 30_000,
|
||||
});
|
||||
return { code: r.status, out: (r.stdout ?? '') + (r.stderr ?? '') };
|
||||
}
|
||||
const config = () => (fs.existsSync(path.join(state, 'config.yaml')) ? fs.readFileSync(path.join(state, 'config.yaml'), 'utf8') : '');
|
||||
const registered = () => (settings().hooks?.UserPromptSubmit ?? []) as Array<{ hooks: Array<{ type: string; command: string; timeout?: number }> }>;
|
||||
const registered = (event = 'UserPromptSubmit') => (settings().hooks?.[event] ?? []) as Array<{ hooks: Array<{ type: string; command: string; timeout?: number }> }>;
|
||||
|
||||
describe('setup: skill hook gates', () => {
|
||||
test('flags parse for both switches, and both appear in --help', () => {
|
||||
@@ -62,12 +73,28 @@ describe('setup: skill hook gates', () => {
|
||||
expect(setupSrc).toMatch(/_skill_hook_gate skill_contracts "\$SKILL_CONTRACTS_MODE" GSTACK_SKILL_CONTRACTS Stop \\\n "\$SKILL_CONTRACT_HOOK" gstack-skill-contract 10/);
|
||||
});
|
||||
|
||||
test('default off: nothing registered, nothing persisted', () => {
|
||||
expect(gate('')).toEqual({ code: 0, out: '' });
|
||||
expect(registered()).toEqual([]);
|
||||
test('no flag, env or config: SWITCH_DEFAULT decides, and nothing is persisted', () => {
|
||||
const g = gate('');
|
||||
expect(g.code).toBe(0);
|
||||
if (SWITCH_DEFAULT.skill_routing === 'on') {
|
||||
expect(g.out).toContain('registered UserPromptSubmit hook: routing hint (on via default;');
|
||||
expect(registered()).toHaveLength(1);
|
||||
} else {
|
||||
expect(g.out).toBe('');
|
||||
expect(registered()).toEqual([]);
|
||||
}
|
||||
expect(config()).not.toContain('skill_routing');
|
||||
});
|
||||
|
||||
test('an unreadable config: warns, registers nothing and removes a live registration, whatever the default', () => {
|
||||
gate('on');
|
||||
fs.rmSync(path.join(state, 'config.yaml'));
|
||||
fs.mkdirSync(path.join(state, 'config.yaml'));
|
||||
const g = gate('');
|
||||
expect(g.out).toContain("WARNING: cannot read skill_routing from config — using 'off' for this run whatever the default");
|
||||
expect(registered()).toEqual([]);
|
||||
});
|
||||
|
||||
test('flag on: persists, registers a schema-valid entry with a one-time line; a re-run is quiet', () => {
|
||||
const first = gate('on');
|
||||
expect(first.code).toBe(0);
|
||||
@@ -82,13 +109,13 @@ describe('setup: skill hook gates', () => {
|
||||
expect(registered()).toHaveLength(1);
|
||||
});
|
||||
|
||||
test('env on registers without persisting; a typo warns, applies off and is not persisted', () => {
|
||||
test('env on registers without persisting; a typo warns, takes SWITCH_DEFAULT and is not persisted', () => {
|
||||
expect(gate('', { GSTACK_SKILL_ROUTING: 'yes' }).out).toContain('on via env GSTACK_SKILL_ROUTING');
|
||||
expect(config()).not.toContain('skill_routing');
|
||||
const typo = gate('onn');
|
||||
expect(typo.out).toContain("WARNING: unrecognized skill_routing value 'onn' (from flag) — using default 'off' for this run, not persisting");
|
||||
expect(typo.out).toContain(`WARNING: unrecognized skill_routing value 'onn' (from flag) — using default '${SWITCH_DEFAULT.skill_routing}' for this run, not persisting`);
|
||||
expect(config()).not.toContain('skill_routing');
|
||||
expect(registered()).toEqual([]);
|
||||
expect(registered()).toHaveLength(SWITCH_DEFAULT.skill_routing === 'on' ? 1 : 0);
|
||||
});
|
||||
|
||||
test('off removes a live registration, and so does --no-team', () => {
|
||||
@@ -102,3 +129,105 @@ describe('setup: skill hook gates', () => {
|
||||
expect(registered()).toEqual([]);
|
||||
});
|
||||
});
|
||||
|
||||
describe('SWITCH_DEFAULT is the one default (a flip PR changes one line)', () => {
|
||||
const TABLE = path.join('lib', 'skill-route.ts');
|
||||
const original = fs.readFileSync(path.join(ROOT, TABLE), 'utf8');
|
||||
const other = (v: 'on' | 'off') => (v === 'on' ? 'off' : 'on');
|
||||
let copy = '', skills = '', work = '';
|
||||
beforeAll(() => {
|
||||
copy = tmpDir('gstack-switch-default-');
|
||||
fs.cpSync(path.join(ROOT, 'lib'), path.join(copy, 'lib'), { recursive: true });
|
||||
fs.cpSync(path.join(ROOT, 'hosts', 'claude', 'hooks'), path.join(copy, 'hosts', 'claude', 'hooks'), { recursive: true });
|
||||
for (const f of ['gstack-config', 'gstack-state-root.sh', 'gstack-skill-route']) fs.cpSync(path.join(ROOT, 'bin', f), path.join(copy, 'bin', f));
|
||||
fs.symlinkSync(path.join(ROOT, 'node_modules'), path.join(copy, 'node_modules'));
|
||||
skills = skillsDir(['investigate']);
|
||||
work = tmpDir('gstack-switch-default-work-');
|
||||
});
|
||||
afterAll(() => { for (const d of [copy, skills, work]) fs.rmSync(d, { recursive: true, force: true }); });
|
||||
|
||||
/** Write the copy's table with only this switch's line flipped, prove the diff is that one line, and return the new default. */
|
||||
function flip(key: keyof typeof GATES): 'on' | 'off' {
|
||||
const line = ` ${key}: '${SWITCH_DEFAULT[key]}',`;
|
||||
const next = other(SWITCH_DEFAULT[key]);
|
||||
expect(original.split('\n').filter(l => l === line)).toHaveLength(1);
|
||||
const flipped = original.replace(line, ` ${key}: '${next}',`);
|
||||
expect(flipped.split('\n').filter((l, i) => l !== original.split('\n')[i])).toEqual([` ${key}: '${next}',`]);
|
||||
fs.writeFileSync(path.join(copy, TABLE), flipped);
|
||||
return next;
|
||||
}
|
||||
const configAt = (root: string, ...args: string[]) => spawnSync(path.join(root, 'bin', 'gstack-config'), args, {
|
||||
env: { PATH: process.env.PATH ?? '', HOME: home, GSTACK_STATE_ROOT: state }, encoding: 'utf8', timeout: 30_000,
|
||||
});
|
||||
const routeAt = (root: string, env: Record<string, string> = {}) => runHook(path.join(root, 'hosts', 'claude', 'hooks', 'skill-route-hook.ts'),
|
||||
JSON.stringify({ session_id: 'sess-d', transcript_path: '/tmp/t.jsonl', cwd: work, hook_event_name: 'UserPromptSubmit', prompt: 'please fix this bug in login' }),
|
||||
hookEnv(state, skills, env));
|
||||
const stopAt = (root: string) => {
|
||||
const file = path.join(work, 'transcript.jsonl');
|
||||
fs.writeFileSync(file, `${J1_TRANSCRIPT.join('\n')}\n`);
|
||||
fs.rmSync(path.join(state, 'skill-contracts'), { recursive: true, force: true });
|
||||
return runHook(path.join(root, 'hosts', 'claude', 'hooks', 'skill-contract-stop-hook.ts'),
|
||||
JSON.stringify({ session_id: 'sess-d', transcript_path: file, cwd: work, hook_event_name: 'Stop', stop_hook_active: false }),
|
||||
hookEnv(state, skills));
|
||||
};
|
||||
const routes = (r: { stdout: string }) => r.stdout !== '' && JSON.parse(r.stdout).hookSpecificOutput.additionalContext.includes('/investigate');
|
||||
const blocks = (r: { stdout: string }) => r.stdout !== '' && JSON.parse(r.stdout).decision === 'block';
|
||||
/** Every reader at root agrees that the switches default to want. */
|
||||
function expectReaders(root: string, want: Record<keyof typeof GATES, 'on' | 'off'>): void {
|
||||
for (const key of ['skill_routing', 'skill_contracts'] as const) {
|
||||
const get = configAt(root, 'get', key);
|
||||
expect({ key, code: get.status, out: get.stdout }).toEqual({ key, code: 0, out: want[key] });
|
||||
expect(configAt(root, 'get', '--stored', key).stdout).toBe('unset');
|
||||
expect(configAt(root, 'defaults').stdout).toMatch(new RegExp(`^ {2}${key}: +${want[key]}$`, 'm'));
|
||||
const g = gate('', {}, 0, root, key);
|
||||
expect(registered(key === 'skill_routing' ? 'UserPromptSubmit' : 'Stop')).toHaveLength(want[key] === 'on' ? 1 : 0);
|
||||
if (want[key] === 'on') expect(g.out).toContain('(on via default;');
|
||||
else expect(g.out).toBe('');
|
||||
}
|
||||
expect(routes(routeAt(root))).toBe(want.skill_routing === 'on');
|
||||
expect(blocks(stopAt(root))).toBe(want.skill_contracts === 'on');
|
||||
expect(config()).toBe('');
|
||||
}
|
||||
|
||||
test('pin: skill_routing defaults to off today (a flip PR changes lib/skill-route.ts line 39 and this pin)', () => {
|
||||
expect(original.split('\n')[38]).toBe(" skill_routing: 'off',");
|
||||
expect(SWITCH_DEFAULT.skill_routing).toBe('off');
|
||||
});
|
||||
|
||||
test('pin: skill_contracts defaults to off today (a flip PR changes lib/skill-route.ts line 40 and this pin)', () => {
|
||||
expect(original.split('\n')[39]).toBe(" skill_contracts: 'off',");
|
||||
expect(SWITCH_DEFAULT.skill_contracts).toBe('off');
|
||||
});
|
||||
|
||||
test('every reader agrees with SWITCH_DEFAULT: gstack-config, setup, the routing hook and the Stop hook', () => {
|
||||
expectReaders(ROOT, SWITCH_DEFAULT);
|
||||
});
|
||||
|
||||
test('an absent key reads empty from gstack-config get, never "false"', () => {
|
||||
const r = configAt(ROOT, 'get', 'no_such_key_gst1088');
|
||||
expect({ code: r.status, out: r.stdout }).toEqual({ code: 1, out: '' });
|
||||
});
|
||||
|
||||
test('flipping skill_routing alone moves every reader; an explicit config value still wins', () => {
|
||||
const next = flip('skill_routing');
|
||||
expectReaders(copy, { ...SWITCH_DEFAULT, skill_routing: next });
|
||||
expect(configAt(copy, 'set', 'skill_routing', SWITCH_DEFAULT.skill_routing).status).toBe(0);
|
||||
gate('', {}, 0, copy);
|
||||
expect(registered()).toHaveLength(SWITCH_DEFAULT.skill_routing === 'on' ? 1 : 0);
|
||||
expect(routes(routeAt(copy))).toBe(SWITCH_DEFAULT.skill_routing === 'on');
|
||||
});
|
||||
|
||||
test('flipping skill_contracts alone moves every reader', () => {
|
||||
const next = flip('skill_contracts');
|
||||
expectReaders(copy, { ...SWITCH_DEFAULT, skill_contracts: next });
|
||||
});
|
||||
|
||||
test('a typo takes the flipped default in setup and in the hook', () => {
|
||||
const next = flip('skill_routing');
|
||||
const typo = gate('onn', {}, 0, copy);
|
||||
expect(typo.out).toContain(`WARNING: unrecognized skill_routing value 'onn' (from flag) — using default '${next}' for this run, not persisting`);
|
||||
expect(registered()).toHaveLength(next === 'on' ? 1 : 0);
|
||||
expect(routes(routeAt(copy, { GSTACK_SKILL_ROUTING: 'onn' }))).toBe(next === 'on');
|
||||
expect(routes(routeAt(ROOT, { GSTACK_SKILL_ROUTING: 'onn' }))).toBe(SWITCH_DEFAULT.skill_routing === 'on');
|
||||
});
|
||||
});
|
||||
@@ -122,6 +122,32 @@ describe('finish, push --early and accept (A1)', () => {
|
||||
// Value: protects=finish never runs a mapped lane locally on the stamped tree: it defers it to accept and runs only pregate's preflight stage; accept carries
|
||||
// it from the green run at the pushed head and reruns the pregate there; fails_when=the stamped tree reruns the suite, or accept trusts a run at another tree;
|
||||
// why_new=design fix A1; seam=fake gh, gstack-evidence, gstack-pregate
|
||||
test('accept checks a carried lane under the label it was recorded with: spaces and colons (TRUT-7)', () => {
|
||||
const f = fx();
|
||||
world(f);
|
||||
expect(f.run(RELEASE, ['push', '--branch', 'feat', '--early', '--base', 'main'], unattended).status).toBe(0);
|
||||
for (const label of ['pnpm lint', 'db:test']) {
|
||||
const ok = f.run(RELEASE, ['accept', '--base', 'main', '--lane', `${label}=${FREE}`, '--wait', '0']);
|
||||
expect([ok.status, ok.stdout], label).toEqual([0, expect.not.stringContaining('not FRESH')]);
|
||||
expect(ok.stdout).toMatch(/^EVIDENCE: FRESH /m);
|
||||
}
|
||||
for (const label of ['pnpm lint', 'db:test']) expect(f.run(path.join(BIN, 'gstack-evidence'), ['check', '--label', label]).stdout).toMatch(/^EVIDENCE: FRESH /m);
|
||||
});
|
||||
|
||||
test('a carried row that no longer grades FRESH leaves no ci_carried entry (TRUT-7: pnpm lint)', () => {
|
||||
const f = fx();
|
||||
world(f);
|
||||
expect(f.run(RELEASE, ['push', '--branch', 'feat', '--early', '--base', 'main'], unattended).status).toBe(0);
|
||||
expect(f.run(RELEASE, ['accept', '--base', 'main', '--lane', `pnpm lint=${FREE}`, '--wait', '0']).status).toBe(0);
|
||||
const carried = () => JSON.parse(f.run(path.join(BIN, 'gstack-evidence'), ['carried', '--base', 'main', '--json']).stdout);
|
||||
expect(carried()).toEqual([expect.objectContaining({ lane: 'pnpm_lint', run: 7, attempt: 1 })]);
|
||||
const [ledger] = findFilesBySuffix(path.join(f.state, 'projects'), '-evidence.jsonl');
|
||||
const rows = fs.readFileSync(ledger!, 'utf8').trim().split('\n').map(l => JSON.parse(l));
|
||||
fs.writeFileSync(ledger!, rows.map(r => JSON.stringify(r.source === 'ci' ? { ...r, ts: at(30 * 60) } : r)).join('\n') + '\n');
|
||||
expect(f.run(path.join(BIN, 'gstack-evidence'), ['check', '--label', 'pnpm lint', '--max-age', '24']).stdout).toMatch(/^EVIDENCE: STALE .* reason=older than 24h/m);
|
||||
expect(carried()).toEqual([]);
|
||||
});
|
||||
|
||||
test('finish defers the mapped lane; accept carries it from the green run at the pushed head', () => {
|
||||
const f = fx();
|
||||
world(f);
|
||||
@@ -285,6 +311,14 @@ describe('GST-1189: a lane carries by named CI jobs instead of an inventory', ()
|
||||
expect(find(f).stdout).toContain('CI_CARRY: none (no_verified_run: ci/free: run 7: tree_mismatch)');
|
||||
});
|
||||
|
||||
test('a pull_request run whose base moved names the one-command fix (TRUT-7)', () => {
|
||||
const f = gbrain();
|
||||
const mergeTree = f.git('rev-parse', 'origin/main^{tree}').stdout.trim();
|
||||
const merge = f.git('commit-tree', mergeTree, '-p', 'HEAD', '-p', 'origin/main', '-m', 'Merge origin/main into feat').stdout.trim();
|
||||
world(f, { jobs: ALL, run: { event: 'pull_request', head_branch: 'feat' }, tree: { ...NO_FILES, tree: mergeTree, sha: merge } });
|
||||
expect(find(f).stdout).toContain("CI_CARRY: none (no_verified_run: ci/free: run 7: tree_mismatch (the pull request's base moved after the run; fix: gh workflow run free.yml --ref feat, then gstack-ci-carry wait))");
|
||||
});
|
||||
|
||||
test('an entry declares exactly one of inventory or jobs', () => {
|
||||
const both = shipFixture({ ...BASE, '.gstack/pregate.json': pregateJson({ jobs: ['verify'], inventory: 'printf x' }) }); fixtures.push(both);
|
||||
world(both, { jobs: ALL, tree: NO_FILES });
|
||||
|
||||
@@ -43,7 +43,7 @@ describe('ship source controller', () => {
|
||||
test('the review plan drives the repeat, the fix-check binds under its own skill, and Step 11.5 accepts a fix-check or a planned adversarial skip', () => {
|
||||
const decide = review.slice(review.indexOf('### Decide whether to repeat Step 9'), review.indexOf('**Required-probe parent gate:**'));
|
||||
for (const flag of ['--fixed-blocking', '--fixed-below', '--blocking-open', '--touched-gate', '--critical-found', '--adversarial-ran', '--reviewed-numbers <fp>']) expect(decide).toContain(flag);
|
||||
expect(decide).toContain('its `NEXT:` alone picks the next decision: `stop` 2, `full` 3, `fix-check` 3.5, `done` 4.');
|
||||
expect(decide).toContain('its `NEXT:` alone picks the next decision: `stop` 2, `full` 3, `fix-check` 3.5, `done` 4, `platform` 4.');
|
||||
expect(decide).toContain('`gstack-review-log --start review-fix-check` token');
|
||||
expect(decide).toContain('`"skill":"review-fix-check"` with `--finish <token>`');
|
||||
expect(compact(read('ship/sections/adversarial.md.tmpl'))).toContain("Step 9's last full-pass plan governs this step");
|
||||
|
||||
@@ -380,4 +380,20 @@ describe('gstack-ship-preflight Step 5 pregate: head policy and timeout_s (GST-4
|
||||
expect(lineOf(r.stdout, 'PREGATE')).toMatch(/^PREGATE: (tests |all )?incomplete/);
|
||||
expect(lineOf(r.stdout, 'PREGATE_DETAIL')).toContain('timed out after 2000 ms (timeout_s from origin/main');
|
||||
});
|
||||
|
||||
// Value: protects=--timeout on preflight reaches Step 5's pregate, so a repo with no timeout_s can run lanes past 600 s without a standalone pregate; fails_when=preflight drops the flag and pregate runs at timeout_s or the 600 s default;
|
||||
// why_new=GST-1196 (GEVA-2, gstack-evals#176/#179): a 128-file group stopped at 600 s; seam=gstack-pregate --timeout; no_claim=the finish fallback
|
||||
test('--timeout reaches Step 5 pregate over the policy and default, and a bad value is usage', () => {
|
||||
const slow = "const test = require('node:test');\ntest('slow', async () => { await new Promise(r => setTimeout(r, 4000)); });\n";
|
||||
const f = fx(base({ timeout_s: 600 }), { 'test/slow.serial.test.js': slow });
|
||||
const r = f.run(PREFLIGHT, ['--tests-only', '--base', 'main', '--timeout', '2', '--lane', LANE]);
|
||||
expect(r.status).toBe(3);
|
||||
expect(lineOf(r.stdout, 'PREGATE')).toMatch(/^PREGATE: (tests |all )?incomplete/);
|
||||
expect(lineOf(r.stdout, 'PREGATE_DETAIL')).toContain('timed out after 2000 ms (--timeout applies');
|
||||
for (const bad of ['0', '3601', '1.5', 'x']) {
|
||||
const u = f.run(PREFLIGHT, ['--tests-only', '--base', 'main', '--timeout', bad, '--lane', LANE]);
|
||||
expect(u.status).toBe(2);
|
||||
expect(u.stdout + u.stderr).toContain('--timeout needs whole seconds from 1 to 3600');
|
||||
}
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,94 @@
|
||||
/**
|
||||
* The screen verdict names every red case, rule cases included, from the
|
||||
* run's real artifacts. Run 41662815506 (#3947's screen) is a correct RED:
|
||||
* seven rule cases failed twice under rule 16, and ship-docsync-stale-after
|
||||
* failed once and passed on its rerun. Before this fix the verdict line named
|
||||
* only the flaky case, so the RED read as if the rerun-pass case caused it.
|
||||
* Fixtures: test/fixtures/screen-verdict-41662815506 holds the run's
|
||||
* gate-binding.json and its collector-outcomes.json minus the per-file list.
|
||||
*/
|
||||
import { describe, expect, test } from 'bun:test';
|
||||
import * as fs from 'node:fs';
|
||||
import * as path from 'node:path';
|
||||
import { screenVerdict, verdictLines } from '../lib/ship-queue-screen';
|
||||
import type { QueueContext } from '../lib/ship-queue';
|
||||
|
||||
const RUN = 41662815506;
|
||||
const SLUG = 'garrytan/gstack';
|
||||
const FIXTURE = path.join(import.meta.dir, 'fixtures', 'screen-verdict-41662815506');
|
||||
const realBinding = () => JSON.parse(fs.readFileSync(path.join(FIXTURE, 'gate-binding.json'), 'utf8'));
|
||||
const realOutcomes = () => JSON.parse(fs.readFileSync(path.join(FIXTURE, 'collector-outcomes.json'), 'utf8'));
|
||||
const TWICE = ['browse-basic', 'browse-snapshot', 'diagram-triplet', 'qa-quick', 'skillmd-no-local-binary', 'skillmd-outside-git', 'skillmd-setup-discovery'];
|
||||
|
||||
function ctxFor(conclusion: string, binding: unknown, outcomes: unknown): QueueContext {
|
||||
const files: Record<string, Record<string, unknown>> = {
|
||||
'paid-plan': { 'manifest.json': { entries: [{ status: 'planned' }] } },
|
||||
'gate-binding-a1': { 'gate-binding.json': binding },
|
||||
'report-verdict-a1': { 'paid-report/collector-outcomes.json': outcomes },
|
||||
};
|
||||
const artifacts = Object.keys(files).map(name => ({ name, expired: false }));
|
||||
return {
|
||||
repoRoot: '/nonexistent', base: 'main',
|
||||
gh: (args) => {
|
||||
if (args[0] === 'run' && args[1] === 'view') return { ok: true, out: JSON.stringify({ databaseId: RUN, status: 'completed', conclusion }) };
|
||||
if (args[0] === 'api') return { ok: true, out: JSON.stringify({ artifacts: args[1]!.includes('name=paid-plan') ? artifacts.filter(a => a.name === 'paid-plan') : artifacts }) };
|
||||
if (args[0] === 'run' && args[1] === 'download') {
|
||||
const name = args[args.indexOf('-n') + 1]!;
|
||||
const dir = args[args.indexOf('-D') + 1]!;
|
||||
if (!files[name]) return { ok: false, out: '', err: 'no artifact' };
|
||||
for (const [rel, body] of Object.entries(files[name]!)) {
|
||||
fs.mkdirSync(path.dirname(path.join(dir, rel)), { recursive: true });
|
||||
fs.writeFileSync(path.join(dir, rel), JSON.stringify(body));
|
||||
}
|
||||
return { ok: true, out: '' };
|
||||
}
|
||||
return { ok: false, out: '', err: `unexpected gh ${args.join(' ')}` };
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
describe('screen verdict from run 41662815506', () => {
|
||||
test('a rule case that failed twice is named red; the rerun-pass case is only flaky', () => {
|
||||
// Value: protects=SCREEN_VERDICT and QUEUE_SCREEN_RED name the rule reds; fails_when=red names come from behavior/judge panels only; why_new=the train fixture models reds as judge panels; seam=none; no_claim=gh is a stub serving the run's downloaded artifacts
|
||||
const read = screenVerdict(ctxFor('failure', realBinding(), realOutcomes()), SLUG, RUN);
|
||||
expect(read.kind).toBe('red');
|
||||
if (read.kind !== 'red') return;
|
||||
expect(read.red).toEqual(TWICE);
|
||||
expect(read.flaky).toEqual(['ship-docsync-stale-after']);
|
||||
const lines = verdictLines('3947', RUN, read);
|
||||
expect(lines[0]).toBe(`SCREEN_VERDICT: #3947 run ${RUN} RED red=${TWICE.join(',')} flaky=ship-docsync-stale-after (verdict RED, reconcile exit 1, run concluded failure, flaky 1)`);
|
||||
expect(lines.slice(1)).toEqual([expect.stringMatching(/^FLAKY: ⚑ ship-docsync-stale-after {2}rule {2}FLAKY: failed in attempt 1/)]);
|
||||
});
|
||||
|
||||
test('negative control: with only the rerun-pass case left, the run reads GREEN with its FLAKY line', () => {
|
||||
const outcomes = { ...realOutcomes(), failures: [] };
|
||||
const read = screenVerdict(ctxFor('success', { ...realBinding(), verdict: 'GREEN', reconcile_exit: 0 }, outcomes), SLUG, RUN);
|
||||
expect(read.kind).toBe('green');
|
||||
if (read.kind !== 'green') return;
|
||||
expect(read.flaky).toEqual(['ship-docsync-stale-after']);
|
||||
expect(verdictLines('3947', RUN, read)).toEqual([
|
||||
`SCREEN_VERDICT: #3947 run ${RUN} GREEN flaky=ship-docsync-stale-after (verdict GREEN, reconcile exit 0, run concluded success, flaky 1)`,
|
||||
`FLAKY: ${realOutcomes().flakyLines[0]}`,
|
||||
]);
|
||||
});
|
||||
|
||||
test('a red that never reran (INFRA, harness, unattributed shard) is still named red', () => {
|
||||
const outcomes = { ...realOutcomes(), failures: [
|
||||
'✗ infra-case rule FAIL infra — runner lost [slice 3, attempt 1]',
|
||||
'✗ test/skill-e2e-x.test.ts rule shard NOT REPORTED [slice 4, attempt 1]',
|
||||
] };
|
||||
const read = screenVerdict(ctxFor('failure', realBinding(), outcomes), SLUG, RUN);
|
||||
expect(read.kind).toBe('red');
|
||||
if (read.kind === 'red') expect(read.red).toEqual(['infra-case', 'test/skill-e2e-x.test.ts']);
|
||||
});
|
||||
|
||||
test('a quarantined or passing split panel line is not named red', () => {
|
||||
const outcomes = { ...realOutcomes(), failures: [
|
||||
'◌ quiet-case behavior (quarantined) FAIL 1/3 (✓✗✗) [t1@slice 2, attempt 1]',
|
||||
'⚠ split-case behavior PASS 2/3 (✓✓✗) [t1@slice 2, attempt 1]',
|
||||
] };
|
||||
const read = screenVerdict(ctxFor('failure', realBinding(), outcomes), SLUG, RUN);
|
||||
expect(read.kind).toBe('red');
|
||||
if (read.kind === 'red') expect(read.red).toEqual([]);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,84 @@
|
||||
import { describe, test, expect, beforeAll, afterAll } from 'bun:test';
|
||||
import { spawnSync } from 'node:child_process';
|
||||
import * as fs from 'node:fs';
|
||||
import * as os from 'node:os';
|
||||
import * as path from 'node:path';
|
||||
import { platformReceiptDepth, receiptDepthVerdict, receiptReviewAtHead } from '../lib/ship-receipt-depth';
|
||||
import { REVIEW_POLICY_KEY_VERSION } from '../lib/ship-policy';
|
||||
import { platformReviewAt } from '../lib/review-platform';
|
||||
import type { ShipReceipt } from '../lib/headless-artifacts';
|
||||
|
||||
const BIN = path.join(import.meta.dir, '..', 'bin');
|
||||
const git = (cwd: string, ...a: string[]) => { const r = spawnSync('git', a, { cwd, encoding: 'utf8', timeout: 30_000 }); if (r.status !== 0) throw new Error(r.stderr); return r.stdout.trim(); };
|
||||
|
||||
function repo(policy: object, files: Record<string, string>): { dir: string; head: string } {
|
||||
const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'plat-'));
|
||||
git(dir, 'init', '-q', '-b', 'main');
|
||||
git(dir, 'config', 'user.email', 't@example.com'); git(dir, 'config', 'user.name', 't'); git(dir, 'config', 'commit.gpgsign', 'false');
|
||||
fs.mkdirSync(path.join(dir, '.gstack'));
|
||||
fs.writeFileSync(path.join(dir, '.gstack/ship-policy.json'), JSON.stringify({ min_gstack_version: REVIEW_POLICY_KEY_VERSION, ...policy }));
|
||||
fs.writeFileSync(path.join(dir, 'README.md'), '# x\n');
|
||||
fs.mkdirSync(path.join(dir, 'src')); fs.writeFileSync(path.join(dir, 'src/a.ts'), 'export const a = 1;\n');
|
||||
git(dir, 'add', '.'); git(dir, 'commit', '-qm', 'base');
|
||||
git(dir, 'update-ref', 'refs/remotes/origin/main', 'HEAD');
|
||||
git(dir, 'checkout', '-qb', 'feat');
|
||||
for (const [f, c] of Object.entries(files)) { fs.mkdirSync(path.dirname(path.join(dir, f)), { recursive: true }); fs.writeFileSync(path.join(dir, f), c); }
|
||||
git(dir, 'add', '.'); git(dir, 'commit', '-qm', 'change');
|
||||
return { dir, head: git(dir, 'rev-parse', 'HEAD') };
|
||||
}
|
||||
|
||||
let fakeBin = '';
|
||||
const withGh = <T,>(head: string, fn: () => T, state = 'COMMENTED'): T => {
|
||||
const prev = process.env.PATH, prevHead = process.env.FAKE_REVIEW_HEAD, prevState = process.env.FAKE_REVIEW_STATE;
|
||||
process.env.PATH = fakeBin + path.delimiter + prev; process.env.FAKE_REVIEW_HEAD = head; process.env.FAKE_REVIEW_STATE = state;
|
||||
try { return fn(); } finally {
|
||||
process.env.PATH = prev;
|
||||
if (prevHead === undefined) delete process.env.FAKE_REVIEW_HEAD; else process.env.FAKE_REVIEW_HEAD = prevHead;
|
||||
if (prevState === undefined) delete process.env.FAKE_REVIEW_STATE; else process.env.FAKE_REVIEW_STATE = prevState;
|
||||
}
|
||||
};
|
||||
let priorState: string | undefined;
|
||||
beforeAll(() => {
|
||||
fakeBin = fs.mkdtempSync(path.join(os.tmpdir(), 'fakegh-'));
|
||||
priorState = process.env.GSTACK_STATE_ROOT;
|
||||
process.env.GSTACK_STATE_ROOT = path.join(fakeBin, 'state');
|
||||
fs.writeFileSync(path.join(fakeBin, 'gh'), '#!/bin/sh\ncase "$1" in\n pr) echo \'{"number":7}\' ;;\n api) echo "[[{\\"id\\":98,\\"user\\":{\\"login\\":\\"capy-ai[bot]\\"},\\"commit_id\\":\\"$FAKE_REVIEW_HEAD\\",\\"state\\":\\"COMMENTED\\"},{\\"id\\":99,\\"user\\":{\\"login\\":\\"capy-ai[bot]\\"},\\"commit_id\\":\\"$FAKE_REVIEW_HEAD\\",\\"state\\":\\"$FAKE_REVIEW_STATE\\"}]]" ;;\nesac\n', { mode: 0o755 });
|
||||
});
|
||||
afterAll(() => { if (priorState === undefined) delete process.env.GSTACK_STATE_ROOT; else process.env.GSTACK_STATE_ROOT = priorState; });
|
||||
|
||||
describe('C6 receipt: review_depth platform, honest and bound to the exact head (FOUN-63)', () => {
|
||||
test('a docs diff under review.platform on with capy-ai[bot]\'s review at HEAD records platform, and the queue accepts it at that head only', () => {
|
||||
const { dir, head } = repo({ review: { platform: 'on' } }, { 'docs/howto.md': 'Run the installer.\n' });
|
||||
const d = receiptReviewAtHead(dir, 'main', BIN, head).depth;
|
||||
expect([d.depth, d.mode]).toEqual(['light', 'panel']);
|
||||
const p = withGh(head, () => platformReceiptDepth(dir, 'main', d, head, '99'));
|
||||
expect(p.ok && [p.depth.depth, p.depth.mode, p.depth.platform_review]).toEqual(['platform', 'platform', { id: '99', head, reviewer: 'capy-ai[bot]' }]);
|
||||
const r = { gstack_version: REVIEW_POLICY_KEY_VERSION, review_depth: (p as any).depth } as ShipReceipt;
|
||||
expect(receiptDepthVerdict(dir, 'origin/main', r, head)).toMatchObject({ ok: true, line: expect.stringContaining('platform_review=99@') });
|
||||
const moved = { ...r, review_depth: { ...r.review_depth!, platform_review: { id: '99', head: 'b'.repeat(40), reviewer: 'capy-ai[bot]' } } } as ShipReceipt;
|
||||
expect(receiptDepthVerdict(dir, 'origin/main', moved, head).ok).toBe(false);
|
||||
});
|
||||
test('refused: platform review off, runtime code, a results path, a review at another head', () => {
|
||||
const off = repo({}, { 'docs/howto.md': 'x\n' });
|
||||
const dOff = receiptReviewAtHead(off.dir, 'main', BIN, off.head).depth;
|
||||
expect(withGh(off.head, () => platformReceiptDepth(off.dir, 'main', dOff, off.head, '99'))).toMatchObject({ ok: false, message: expect.stringContaining('review.platform on') });
|
||||
for (const files of [{ 'src/b.ts': 'export const b = 2;\n' }, { 'docs/results/summary.md': 'Ran it.\n' }] as Record<string, string>[]) {
|
||||
const { dir, head } = repo({ review: { platform: 'on' } }, files);
|
||||
const d = receiptReviewAtHead(dir, 'main', BIN, head).depth;
|
||||
expect(withGh(head, () => platformReceiptDepth(dir, 'main', d, head, '99')).ok).toBe(false);
|
||||
}
|
||||
const { dir, head } = repo({ review: { platform: 'on' } }, { 'docs/howto.md': 'x\n' });
|
||||
const d = receiptReviewAtHead(dir, 'main', BIN, head).depth;
|
||||
expect(withGh('c'.repeat(40), () => platformReceiptDepth(dir, 'main', d, head, '99'))).toMatchObject({ ok: false, message: expect.stringContaining('capy-ai[bot]') });
|
||||
for (const state of ['CHANGES_REQUESTED', 'DISMISSED']) expect(withGh(head, () => platformReceiptDepth(dir, 'main', d, head, '99'), state).ok).toBe(false);
|
||||
expect(withGh(head, () => platformReceiptDepth(dir, 'main', d, head, '99'), 'APPROVED').ok).toBe(true);
|
||||
expect(withGh(head, () => platformReviewAt(dir, head), 'CHANGES_REQUESTED').ok).toBe(false);
|
||||
expect(withGh(head, () => platformReviewAt(dir, head))).toMatchObject({ ok: true, review: { id: '99' } });
|
||||
expect(withGh(head, () => platformReviewAt(dir, head, { id: '98' })).ok).toBe(false);
|
||||
});
|
||||
test('the receipt records the risk_paths class and the panel mode the base policy gives', () => {
|
||||
const { dir, head } = repo({ review: { risk_paths: ['src/balance/**'] } }, { 'src/balance/x.ts': 'export const x = 1;\n' });
|
||||
const d = receiptReviewAtHead(dir, 'main', BIN, head).depth;
|
||||
expect([d.depth, d.risk, d.mode]).toEqual(['full', ['policy'], 'panel']);
|
||||
});
|
||||
});
|
||||
@@ -352,3 +352,99 @@ describe('GSTA-167 part 4a: the post-review route stays path-only', () => {
|
||||
expect(r.stdout).toContain('STOP: behavior, tests or build inputs changed since review (wiki/post/data.json)');
|
||||
});
|
||||
});
|
||||
|
||||
describe('push: the pre-push housekeeping guards (GST-1138)', () => {
|
||||
// Value: protects=push runs gstack-pregate --stage guards before every push and stops on a failing guard with its fix, pushing nothing;
|
||||
// a pass recorded at HEAD's exact tree is reused instead of rerun; fails_when=a housekeeping red reaches CI, a failed guard still pushes,
|
||||
// or a guard reruns on a tree it already passed; why_new=gbrain 64 of 93 red heads were housekeeping (FOUN-64), #3908's context-budget red;
|
||||
// seam=gstack-pregate --stage guards; no_claim=CI's own verdict
|
||||
const policy = (command: string) => JSON.stringify({ guards: [{ id: 'housekeeping', when: ['src/**'], command, fix: 'regenerate the golden' }] }) + '\n';
|
||||
const ready = (f: ShipFx) => { fs.mkdirSync(f.state, { recursive: true }); fs.writeFileSync(path.join(f.state, '.redact-prepush-prompted'), ''); };
|
||||
const pushedHead = (f: ShipFx) => f.git('ls-remote', '--heads', 'origin', 'refs/heads/feat').stdout.trim();
|
||||
|
||||
test('a failing guard stops the push with its fix and nothing is pushed', () => {
|
||||
const f = fx({ '.gstack/pregate.json': policy('echo golden is stale; exit 4') });
|
||||
ready(f);
|
||||
const r = f.run(RELEASE, ['push', '--branch', 'feat']);
|
||||
expect(r.status).toBe(3);
|
||||
expect(lineOf(r.stdout, 'GUARDS')).toMatch(/^GUARDS: fail .*housekeeping/);
|
||||
expect(r.stdout).toContain('STOP: the pre-push guards did not pass');
|
||||
expect(r.stdout).toContain('regenerate the golden');
|
||||
expect(pushedHead(f)).toBe('');
|
||||
});
|
||||
|
||||
test('a passing guard pushes; the next push at the same tree reuses the pass instead of rerunning it', () => {
|
||||
const f = fx({ '.gstack/pregate.json': policy('echo ran >> ../guard-runs') });
|
||||
ready(f);
|
||||
const first = f.run(RELEASE, ['push', '--branch', 'feat']);
|
||||
expect(first.status, first.stdout + first.stderr).toBe(0);
|
||||
expect(lineOf(first.stdout, 'GUARDS')).toMatch(/^GUARDS: pass .*1 guard\(s\) passed: housekeeping/);
|
||||
expect(lineOf(first.stdout, 'PUSH')).toMatch(/^PUSH: pushed /);
|
||||
const again = f.run(RELEASE, ['push', '--branch', 'feat']);
|
||||
expect(lineOf(again.stdout, 'GUARDS')).toMatch(/^GUARDS: pass \(reused from pregate-guards\.json at tree [0-9a-f]{12}: /);
|
||||
expect(fs.readFileSync(path.join(f.root, 'guard-runs'), 'utf8')).toBe('ran\n');
|
||||
f.write({ 'src/c.ts': 'export const c = 3;\n' }); f.commit('more');
|
||||
expect(lineOf(f.run(RELEASE, ['push', '--branch', 'feat']).stdout, 'GUARDS')).not.toContain('reused');
|
||||
expect(fs.readFileSync(path.join(f.root, 'guard-runs'), 'utf8')).toBe('ran\nran\n');
|
||||
});
|
||||
|
||||
test('an uncommitted or untracked file the guards saw stops the push: the verdict must describe the pushed tree', () => {
|
||||
const f = fx({ '.gstack/pregate.json': policy('test -f src/needed.ts') });
|
||||
ready(f);
|
||||
f.write({ 'src/needed.ts': 'export const n = 1;\n' });
|
||||
const r = f.run(RELEASE, ['push', '--branch', 'feat']);
|
||||
expect(r.status).toBe(3);
|
||||
expect(lineOf(r.stdout, 'GUARDS')).toContain('not certified');
|
||||
expect(r.stdout).toContain('STOP: the working tree differs from HEAD');
|
||||
expect(pushedHead(f)).toBe('');
|
||||
});
|
||||
|
||||
test('a base that exists on origin but is not fetched, a mistyped --base, a dangling origin/HEAD and an unreadable policy stop the push', () => {
|
||||
const f = fx({ '.gstack/pregate.json': policy('exit 4') });
|
||||
ready(f);
|
||||
f.git('update-ref', '-d', 'refs/remotes/origin/main');
|
||||
const unfetched = f.run(RELEASE, ['push', '--branch', 'feat']);
|
||||
expect(unfetched.status).toBe(3);
|
||||
expect(lineOf(unfetched.stdout, 'GUARDS')).toMatch(/^GUARDS: refused POLICY_SOURCE_UNAVAILABLE/);
|
||||
expect(unfetched.stdout).toContain('STOP: the pre-push guards could not run');
|
||||
f.git('fetch', '-q', 'origin', 'main:refs/remotes/origin/main');
|
||||
const typo = f.run(RELEASE, ['push', '--branch', 'feat', '--base', 'mian']);
|
||||
expect(typo.status).toBe(3);
|
||||
expect(lineOf(typo.stdout, 'GUARDS')).toMatch(/^GUARDS: refused /);
|
||||
f.git('symbolic-ref', 'refs/remotes/origin/HEAD', 'refs/remotes/origin/retired-default');
|
||||
const dangling = f.run(RELEASE, ['push', '--branch', 'feat']);
|
||||
expect(dangling.status).toBe(3);
|
||||
expect(lineOf(dangling.stdout, 'GUARDS')).toMatch(/^GUARDS: refused /);
|
||||
expect(pushedHead(f)).toBe('');
|
||||
const broken = fx({ '.gstack/pregate.json': '{not json\n' });
|
||||
ready(broken);
|
||||
const refused = broken.run(RELEASE, ['push', '--branch', 'feat']);
|
||||
expect(refused.status).toBe(3);
|
||||
expect(lineOf(refused.stdout, 'GUARDS')).toMatch(/^GUARDS: refused /);
|
||||
});
|
||||
|
||||
test("finish's preflight record is reused only at HEAD's tree with passing regen and guards rows", () => {
|
||||
const f = fx({ '.gstack/pregate.json': policy('echo ran >> ../guard-runs') });
|
||||
ready(f);
|
||||
const tree = f.git('rev-parse', 'HEAD^{tree}').stdout.trim();
|
||||
const record = (at: string, guards: string) => JSON.stringify({ tree: at, checks: [{ id: 'regen', status: 'pass' }, { id: 'guards', status: guards }] });
|
||||
const finishRecord = path.join(f.work, '.git', 'gstack', 'pregate-finish-preflight.json');
|
||||
fs.mkdirSync(path.dirname(finishRecord), { recursive: true });
|
||||
const runs = () => (fs.existsSync(path.join(f.root, 'guard-runs')) ? fs.readFileSync(path.join(f.root, 'guard-runs'), 'utf8') : '');
|
||||
for (const [at, guards] of [[tree, 'fail'], ['0'.repeat(40), 'pass']]) {
|
||||
fs.rmSync(path.join(f.work, '.git', 'gstack', 'pregate-guards.json'), { force: true });
|
||||
fs.writeFileSync(finishRecord, record(at, guards));
|
||||
fs.rmSync(path.join(f.root, 'guard-runs'), { force: true });
|
||||
expect(lineOf(f.run(RELEASE, ['push', '--branch', 'feat']).stdout, 'GUARDS')).not.toContain('reused');
|
||||
expect(runs()).toBe('ran\n');
|
||||
f.git('push', '-q', 'origin', '--delete', 'feat');
|
||||
}
|
||||
fs.rmSync(path.join(f.work, '.git', 'gstack', 'pregate-guards.json'), { force: true });
|
||||
fs.writeFileSync(finishRecord, record(tree, 'pass'));
|
||||
fs.rmSync(path.join(f.root, 'guard-runs'), { force: true });
|
||||
const reused = f.run(RELEASE, ['push', '--branch', 'feat']);
|
||||
expect(lineOf(reused.stdout, 'GUARDS')).toMatch(/^GUARDS: pass \(reused from pregate-finish-preflight\.json at tree /);
|
||||
expect(runs()).toBe('');
|
||||
expect(lineOf(reused.stdout, 'PUSH')).toMatch(/^PUSH: pushed /);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,4 @@
|
||||
{
|
||||
"bytes": 580,
|
||||
"why": "GSTA-258 C: risk-scaled review depth (single/merged reader lines, NEXT: platform planned skip with its PLATFORM_REVIEW id, round 2 on risk class) and GST-1129 --declare-risk"
|
||||
}
|
||||
+37
-2
@@ -47,7 +47,7 @@ describe('gstack-skill-bill', () => {
|
||||
const root = gstackRepo();
|
||||
|
||||
test('a gstack checkout skill reports every level and its last diet', () => {
|
||||
const r = bill(root, ['demo']);
|
||||
const r = bill(root, ['./demo']);
|
||||
expect(r.status).toBe(0);
|
||||
expect(r.stdout).toContain('SKILL: demo (gstack-checkout)');
|
||||
expect(r.stdout).toMatch(/template\s+SKILL\.md\.tmpl/);
|
||||
@@ -58,7 +58,7 @@ describe('gstack-skill-bill', () => {
|
||||
|
||||
test('growth over 10% since the Skill-Diet commit prints DUE, and an unrendered template is RENDER_STALE', () => {
|
||||
fs.appendFileSync(path.join(root, 'demo', 'SKILL.md.tmpl'), 'y'.repeat(200));
|
||||
const r = bill(root, ['demo', '--json']);
|
||||
const r = bill(root, ['./demo', '--json']);
|
||||
const report = JSON.parse(r.stdout);
|
||||
expect(report.baseline.status).toBe('found');
|
||||
expect(report.baseline.due).toBe(true);
|
||||
@@ -114,6 +114,41 @@ describe('gstack-skill-bill', () => {
|
||||
});
|
||||
});
|
||||
|
||||
describe('gstack-skill-bill EVIDENCE line (docs/inventory/evidence.json)', () => {
|
||||
const root = path.join(tmp, 'evidence-repo');
|
||||
const ledger = (parts: Record<string, unknown>) => {
|
||||
fs.mkdirSync(path.join(root, 'docs', 'inventory'), { recursive: true });
|
||||
fs.writeFileSync(path.join(root, 'docs', 'inventory', 'evidence.json'), JSON.stringify({ schema_version: 1, parts }));
|
||||
};
|
||||
fs.mkdirSync(path.join(root, 'bin'), { recursive: true });
|
||||
fs.writeFileSync(path.join(root, 'SKILL.md.tmpl'), 'router');
|
||||
fs.writeFileSync(path.join(root, 'VERSION'), '1.0.0');
|
||||
skill(path.join(root, 'demo'), 'demo', 'body\n', 'SKILL.md.tmpl');
|
||||
|
||||
test('each status prints its source, plan or reason; a skill with no row says so', () => {
|
||||
ledger({ demo: { status: 'untested', grandfathered: true, plan: 'none' } });
|
||||
expect(bill(root, ['./demo']).stdout).toContain('EVIDENCE: untested plan: none (grandfathered)');
|
||||
ledger({ demo: { status: 'untested', reason: 'r', plan: 'test/fixtures/skill-evals/demo/tasks.json', owner: 'GSTA-1', measure_by: '2026-12-01' } });
|
||||
expect(bill(root, ['./demo']).stdout).toContain('EVIDENCE: untested plan: test/fixtures/skill-evals/demo/tasks.json (owner GSTA-1, measure by 2026-12-01)');
|
||||
ledger({ demo: { status: 'helps', sources: ['run-9/results.json'], models: ['claude-opus-5-5'], prompt: 'preset', scope: 's', masked: true } });
|
||||
expect(bill(root, ['./demo']).stdout).toContain('EVIDENCE: helps sources: run-9/results.json');
|
||||
ledger({ demo: { status: 'safety', reason: 'blocks rm -rf' } });
|
||||
expect(bill(root, ['./demo']).stdout).toContain('EVIDENCE: safety reason: blocks rm -rf');
|
||||
ledger({});
|
||||
expect(bill(root, ['./demo']).stdout).toContain('EVIDENCE: missing: no demo row in docs/inventory/evidence.json');
|
||||
expect(JSON.parse(bill(root, ['./demo', '--json']).stdout).evidence).toMatchObject({ key: 'demo', status: 'missing' });
|
||||
fs.writeFileSync(path.join(root, 'docs', 'inventory', 'evidence.json'), '{bad');
|
||||
const broken = bill(root, ['./demo']).stdout;
|
||||
expect(broken).toContain('EVIDENCE: unreadable: cannot read docs/inventory/evidence.json');
|
||||
expect(broken).not.toContain('add one');
|
||||
});
|
||||
|
||||
test('the real checkout: a guard prints its safety reason', () => {
|
||||
const r = spawnSync(BIN, ['careful'], { cwd: path.resolve(import.meta.dir, '..'), encoding: 'utf-8', timeout: 30_000 });
|
||||
expect(r.stdout).toMatch(/^EVIDENCE: safety reason: .+/m);
|
||||
});
|
||||
});
|
||||
|
||||
describe('gstack-skill-bill baselines and the root skill', () => {
|
||||
test('the gstack root skill is a checkout skill, and Skill-Diet: gstack-upgrade is not its diet', () => {
|
||||
const root = path.join(tmp, 'gstack-root');
|
||||
|
||||
@@ -7,9 +7,10 @@
|
||||
import { afterEach, beforeEach, describe, expect, test } from 'bun:test';
|
||||
import * as fs from 'node:fs';
|
||||
import * as path from 'node:path';
|
||||
import { STATE_ERROR_DOC, stateErrorLine } from '../lib/skill-contract';
|
||||
import { AGENT_STALL_MS, STATE_ERROR_DOC, stateErrorLine } from '../lib/skill-contract';
|
||||
import { SWITCH_DEFAULT } from '../lib/skill-route';
|
||||
import {
|
||||
CONTRACT_HOOK, DEBUG_REPORT, INTERACTIVE_TURNS, J1_TRANSCRIPT, ROOT, hookEnv, result, runHook, say, skillsDir, slash, tmpDir, tool, user,
|
||||
CONTRACT_HOOK, DEBUG_REPORT, INTERACTIVE_TURNS, J1_TRANSCRIPT, ROOT, hookEnv, meta, result, runHook, say, skillsDir, slash, tmpDir, tool, user,
|
||||
} from './helpers/skill-contract-fixture';
|
||||
|
||||
const PINNED = fs.readFileSync(path.join(ROOT, 'test', 'fixtures', 'skill-contract', 'block-message.txt'), 'utf8').trimEnd();
|
||||
@@ -135,16 +136,33 @@ describe('never blocks the wrong turn', () => {
|
||||
expect(JSON.parse(stop(file, { GSTACK_HEADLESS: '1' }).stdout).decision).toBe('block');
|
||||
});
|
||||
|
||||
test('Agent readers still running when Stop fires: allowed, nothing recorded', () => {
|
||||
test('Agent readers still running when Stop fires: allowed and recorded deferred until the same set has run AGENT_STALL_MS, then checked', () => {
|
||||
const base = [slash('review'), say('Dispatching 2 specialists: Testing, Security. Skipped: Performance (scope not detected).')];
|
||||
const background = [...base, tool('a1', 'Agent', { description: 'Testing specialist', prompt: 'review', run_in_background: true }),
|
||||
result('a1', 'Async agent launched in the background.'), say('Waiting for the specialists to finish.')];
|
||||
const unanswered = [...base, tool('a2', 'Agent', { description: 'Security specialist', prompt: 'review' })];
|
||||
for (const lines of [background, unanswered]) {
|
||||
expect(stop(transcript(lines))).toEqual({ code: 0, stdout: '', stderr: '' });
|
||||
}
|
||||
expect(fs.existsSync(path.join(state, 'skill-contracts'))).toBe(false);
|
||||
const file = transcript(background);
|
||||
expect(stop(file)).toEqual({ code: 0, stdout: '', stderr: '' });
|
||||
expect(Object.values(record().runs)[0]).toMatchObject({ outcome: 'deferred', deferred_agents: ['a1'] });
|
||||
expect(fs.readFileSync(path.join(state, 'hooks', 'skill-contract.log'), 'utf8')).toContain('event=agents-running');
|
||||
fs.appendFileSync(file, `${user('how is it going?')}\n${meta('<task-notification>bash b9 completed</task-notification>')}\n${say('Still waiting on the specialists.')}\n`);
|
||||
expect(stop(file)).toEqual({ code: 0, stdout: '', stderr: '' });
|
||||
expect(Object.values(record().runs)[0]).toMatchObject({ outcome: 'deferred', deferred_agents: ['a1'] });
|
||||
const rec = record();
|
||||
const key = Object.keys(rec.runs)[0];
|
||||
rec.runs[key].deferred_at = new Date(Date.now() - AGENT_STALL_MS - 1000).toISOString();
|
||||
fs.writeFileSync(path.join(state, 'skill-contracts', 'sess-1.json'), JSON.stringify(rec));
|
||||
expect(JSON.parse(stop(file).stdout).decision).toBe('block');
|
||||
expect(Object.values(record().runs)[0]).toMatchObject({ outcome: 'blocked' });
|
||||
expect(fs.readFileSync(path.join(state, 'hooks', 'skill-contract.log'), 'utf8')).toContain('event=agents-stalled');
|
||||
const unanswered = [...base, tool('a2', 'Agent', { description: 'Security specialist', prompt: 'review' })];
|
||||
expect(stop(transcript(unanswered, 'unanswered.jsonl'), {}, { session_id: 'sess-2' })).toEqual({ code: 0, stdout: '', stderr: '' });
|
||||
});
|
||||
|
||||
test('a stop on a turn that ends on a question never starts the agent clock', () => {
|
||||
const asking = [slash('review'), tool('a1', 'Agent', { description: 'Testing specialist', prompt: 'review', run_in_background: true }),
|
||||
result('a1', 'Async agent launched in the background.'), say('Should I also review the migrations?')];
|
||||
expect(stop(transcript(asking))).toEqual({ code: 0, stdout: '', stderr: '' });
|
||||
expect(fs.existsSync(path.join(state, 'skill-contracts', 'sess-1.json'))).toBe(false);
|
||||
});
|
||||
|
||||
test('no contract skill loaded: allowed', () => {
|
||||
@@ -201,9 +219,15 @@ describe('fails open on state errors with one stderr line', () => {
|
||||
});
|
||||
|
||||
describe('off switch', () => {
|
||||
test('default off, config off, and env off over config on: no output at all', () => {
|
||||
test('with nothing set SWITCH_DEFAULT decides; config off, env off over config on, and an unreadable config: no output at all', () => {
|
||||
const file = transcript(J1_TRANSCRIPT);
|
||||
fs.mkdirSync(path.join(state, 'config.yaml'));
|
||||
expect(runHook(CONTRACT_HOOK, payload(file), hookEnv(state, skills))).toEqual({ code: 0, stdout: '', stderr: '' });
|
||||
fs.rmSync(path.join(state, 'config.yaml'), { recursive: true });
|
||||
const unset = runHook(CONTRACT_HOOK, payload(file), hookEnv(state, skills));
|
||||
if (SWITCH_DEFAULT.skill_contracts === 'on') expect(JSON.parse(unset.stdout).decision).toBe('block');
|
||||
else expect(unset).toEqual({ code: 0, stdout: '', stderr: '' });
|
||||
fs.rmSync(path.join(state, 'skill-contracts'), { recursive: true, force: true });
|
||||
fs.writeFileSync(path.join(state, 'config.yaml'), 'skill_contracts: off\n');
|
||||
expect(runHook(CONTRACT_HOOK, payload(file), hookEnv(state, skills))).toEqual({ code: 0, stdout: '', stderr: '' });
|
||||
fs.writeFileSync(path.join(state, 'config.yaml'), 'skill_contracts: on\n');
|
||||
|
||||
@@ -8,7 +8,8 @@ import { spawnSync } from 'node:child_process';
|
||||
import * as fs from 'node:fs';
|
||||
import * as path from 'node:path';
|
||||
import {
|
||||
DEFAULT_MATCHERS, EXIT, STATUS_RE, blockMessage, checkContract, isCheckCommand, isPlaceholder, parseContract, parseTranscript, skillLoads, UnknownShape,
|
||||
DEFAULT_MATCHERS, EXIT, STATUS_RE, blockMessage, checkContract, isCheckCommand, isMiss, isPlaceholder, parseContract, parseTranscript, reportRows, skillLoads, UnknownShape,
|
||||
type CheckResult,
|
||||
type Contract,
|
||||
} from '../lib/skill-contract';
|
||||
import { readFrontmatter } from '../lib/skill-route';
|
||||
@@ -176,10 +177,33 @@ describe('checkContract is a pure function of the transcript', () => {
|
||||
for (const cmd of ['bun test', 'cd repo && npm test', 'FOO=1 timeout 60 pytest -x', './scripts/run-tests.sh', 'node repro.js | tail -5', 'bin/gstack-verify',
|
||||
'bundle exec rspec', 'uv run pytest', 'poetry run pytest', 'docker compose run app pytest', 'xvfb-run -a npm test', 'bash scripts/test.sh', 'mix test', 'bazel test //...'])
|
||||
expect([cmd, isCheckCommand(cmd)]).toEqual([cmd, true]);
|
||||
for (const cmd of ['git commit -m wip --no-verify', 'git add test/a.test.ts', 'cat test/a.test.ts', 'git diff --check', 'ls go', 'test -f README.md', './scripts/fetch-latest.sh', 'bash scripts/deploy.sh'])
|
||||
for (const cmd of ['npm test', 'npm run test:unit', 'yarn jest', 'go test ./...', 'cargo test', 'make check', 'python -m pytest', 'bun repro.ts', 'npx vitest run', 'curl -s http://localhost:3000/api'])
|
||||
expect([cmd, isCheckCommand(cmd)]).toEqual([cmd, true]);
|
||||
for (const cmd of ['git commit -m wip --no-verify', 'git add test/a.test.ts', 'cat test/a.test.ts', 'git diff --check', 'ls go', 'test -f README.md', './scripts/fetch-latest.sh', 'bash scripts/deploy.sh',
|
||||
'bun install', 'npm install lodash', 'bun add zod', 'go build ./...', 'go mod tidy', 'node --version', 'make clean', 'make', 'npm run build', 'curl --version'])
|
||||
expect([cmd, isCheckCommand(cmd)]).toEqual([cmd, false]);
|
||||
});
|
||||
|
||||
test('check_after_last_edit: an option value is never the verb, and a long chain of wrapper words stays fast', () => {
|
||||
for (const cmd of ['make -C web test', 'npm --prefix web test', 'pnpm --filter app test', 'bun --cwd app test', 'yarn workspace app test',
|
||||
'xcodebuild -scheme App test', 'cargo +nightly test', 'mvn clean test', 'bun run typecheck', 'node --test'])
|
||||
expect([cmd, isCheckCommand(cmd)]).toEqual([cmd, true]);
|
||||
for (const cmd of ['make -C web build', 'npm --prefix web install', 'pnpm --filter app build', 'xcodebuild -scheme App build'])
|
||||
expect([cmd, isCheckCommand(cmd)]).toEqual([cmd, false]);
|
||||
const started = performance.now();
|
||||
expect(isCheckCommand(`${Array(40).fill('sh').join(' ')} echo`)).toBe(false);
|
||||
expect(performance.now() - started).toBeLessThan(500);
|
||||
});
|
||||
|
||||
test('report re-checks a deferred run: unmet is a miss, met is met', () => {
|
||||
const run = { skill: 'review', load_line: 1, outcome: 'deferred' as const, missing: [], waived: [], deferred_agents: ['a1'], updated_at: '' };
|
||||
const rec = { version: 1 as const, session_id: 's', transcript_path: 't', runs: { k: run } };
|
||||
const unmet = reportRows([rec], () => ({ met: false, missing: ['specialists'], waived: [] }) as unknown as CheckResult);
|
||||
expect(unmet.map(r => [r.outcome, isMiss(r)])).toEqual([['miss', true]]);
|
||||
const met = reportRows([rec], () => ({ met: true, missing: [], waived: [] }) as unknown as CheckResult);
|
||||
expect(met.map(r => [r.outcome, isMiss(r)])).toEqual([['met', false]]);
|
||||
});
|
||||
|
||||
test('selected_specialists_dispatched: a selection line only counts from assistant text or a Bash result, with plain names', () => {
|
||||
const red = [tool('c', 'Agent', { description: 'Red team', prompt: 'Think like an attacker and a chaos engineer.' }), result('c', '[]')];
|
||||
const report = say('Pre-Landing Review: 0 blocking\nGate edits: none\nSTATUS: DONE');
|
||||
|
||||
@@ -3,7 +3,8 @@
|
||||
*
|
||||
* On the pinned Claude Code, in an interactive PTY session with default
|
||||
* settings, a project skill carrying /autoplan's own hook block enters Phase 1,
|
||||
* dispatches the CEO reviewer, reads a prepared close packet, publishes the Phase 1 report in a message
|
||||
* dispatches the CEO reviewer (each dispatch is a declared token the driver's relay replaces with the exact
|
||||
* dispatch, so the actor never retypes it), reads a prepared close packet, publishes the Phase 1 report in a message
|
||||
* whose only tool call is `true autoplan-published ceo`, enters Phase 2 in a
|
||||
* later message and dispatches the Phase 2 reviewer. The reviewer dispatch sends run_in_background, which the
|
||||
* 2.1.29x default fork-subagent schema strips. Claude Code 2.1.29x can write a tool call to the journal only
|
||||
@@ -18,9 +19,10 @@
|
||||
* must still raise no permission card.
|
||||
*/
|
||||
import { expect } from 'bun:test';
|
||||
import * as path from 'node:path';
|
||||
import { describeIfSelected, testIfSelected } from './helpers/e2e-helpers';
|
||||
import { launchClaudePty, selectPtyNumberedOption } from './helpers/claude-pty-runner';
|
||||
import { closePacketReadAfterNotice, prepareGuardSession, stoppedAtPhaseBoundary } from './helpers/autoplan-guard-pty';
|
||||
import { closePacketReadAfterNotice, prepareGuardSession, startDispatchRelay, stoppedAtPhaseBoundary } from './helpers/autoplan-guard-pty';
|
||||
|
||||
const MODEL = 'claude-haiku-4-5-20251001';
|
||||
|
||||
@@ -28,7 +30,9 @@ describeIfSelected('autoplan publication guard live PTY', ['autoplan-guard-pty']
|
||||
testIfSelected('autoplan-guard-pty', async () => {
|
||||
if (!process.env.ANTHROPIC_API_KEY) throw new Error('autoplan-guard-pty requires ANTHROPIC_API_KEY; refusing to skip');
|
||||
const s = prepareGuardSession(process.env.ANTHROPIC_API_KEY);
|
||||
const session = await launchClaudePty({ permissionMode: null, model: MODEL, cwd: s.project, env: s.env, timeoutMs: 540_000, observeScreen: true });
|
||||
const relay = startDispatchRelay(path.join(s.project, '.gstack', 'tmp', 'autoplan'));
|
||||
const session = await launchClaudePty({ permissionMode: null, model: MODEL, cwd: s.project, env: { ...s.env, ANTHROPIC_BASE_URL: relay.url },
|
||||
timeoutMs: 540_000, observeScreen: true }).catch(error => { relay.stop(); throw error; });
|
||||
const started = performance.now();
|
||||
const prompts: string[] = [];
|
||||
let phase1Ms: number | null = null;
|
||||
@@ -54,10 +58,10 @@ describeIfSelected('autoplan publication guard live PTY', ['autoplan-guard-pty']
|
||||
}
|
||||
await Bun.sleep(3_000);
|
||||
const log = s.readLog(), evidence = s.noticeEvidence();
|
||||
const summary = { prompts: prompts.length, phase1Ms, totalMs: Math.round(performance.now() - started), evidence,
|
||||
const summary = { prompts: prompts.length, substituted: relay.substituted, phase1Ms, totalMs: Math.round(performance.now() - started), evidence,
|
||||
decisions: log.map(e => ({ decision: e.decision, code: e.code, path: e.path, version: e.claude_code_version })) };
|
||||
console.log(`[autoplan-guard-pty] ${JSON.stringify(summary)}`);
|
||||
if (log.length < 4 && stoppedAtPhaseBoundary(session.visibleText())) console.log('[autoplan-guard-pty] PHASE_BOUNDARY_STOP: the actor ended its turn after "Phase 1 complete." instead of going on to step 8');
|
||||
if (log.length < 4 && stoppedAtPhaseBoundary(session.visibleText())) console.log('[autoplan-guard-pty] PHASE_BOUNDARY_STOP: the actor ended its turn after "Phase 1 complete." instead of going on to step 7');
|
||||
if (log.length !== 4 || log.some(e => e.decision === 'deny') || prompts.length)
|
||||
console.log(`[autoplan-guard-pty] journal:\n${s.journalSummary().join('\n')}\n[autoplan-guard-pty] screen:\n${session.visibleText().slice(-2_000)}\n[autoplan-guard-pty] last non-blank screen:\n${session.lastScreen?.() ?? ''}`);
|
||||
if (log.some(e => e.code === 'dispatch_prompt')) console.log(`[autoplan-guard-pty] dispatch prompt deltas:\n${s.dispatchPromptDeltas().join('\n')}`);
|
||||
@@ -74,6 +78,7 @@ describeIfSelected('autoplan publication guard live PTY', ['autoplan-guard-pty']
|
||||
expect(prompts).toEqual([]);
|
||||
} finally {
|
||||
await session.close();
|
||||
relay.stop();
|
||||
s.cleanup();
|
||||
}
|
||||
}, 600_000);
|
||||
|
||||
+19
-10
@@ -2,13 +2,15 @@
|
||||
* lib/skill-route.ts, the skill-route hook and gstack-skill-route --explain
|
||||
* (GST-1088, lane A): deterministic scoring against installed triggers, one
|
||||
* additionalContext line when it fires, nothing at all (exit 0, no stderr) on
|
||||
* every error, and off by default.
|
||||
* every error. Nothing here assumes a default: tests set the switch, or read
|
||||
* the expected result from SWITCH_DEFAULT (the pins live in
|
||||
* test/setup-skill-hooks-gate.test.ts).
|
||||
*/
|
||||
import { afterEach, beforeEach, describe, expect, test } from 'bun:test';
|
||||
import { spawnSync } from 'node:child_process';
|
||||
import * as fs from 'node:fs';
|
||||
import * as path from 'node:path';
|
||||
import { PROMPT_MAX_CHARS, ROUTED_SKILLS, ROUTE_THRESHOLD, firstStep, hintLine, readSkillSwitch, routePrompt, scanInstalledSkills, scorePrompt, stem } from '../lib/skill-route';
|
||||
import { PROMPT_MAX_CHARS, ROUTED_SKILLS, ROUTE_THRESHOLD, SWITCH_DEFAULT, firstStep, hintLine, readSkillSwitch, routePrompt, scanInstalledSkills, scorePrompt, stem } from '../lib/skill-route';
|
||||
import { ROOT, ROUTE_HOOK, SKILLS, hookEnv, runHook, skillsDir, tmpDir } from './helpers/skill-contract-fixture';
|
||||
|
||||
let state = '', skills = '';
|
||||
@@ -75,12 +77,17 @@ describe('scorer', () => {
|
||||
expect(firstStep(SKILLS.investigate)).toBe('Reproduce the failure before you change code, then fix it and prove the fix.');
|
||||
});
|
||||
|
||||
test('switch precedence: env over config over default off; typos read as off', () => {
|
||||
expect(readSkillSwitch('skill_routing', { GSTACK_STATE_ROOT: state })).toEqual({ on: false, source: 'default' });
|
||||
test('switch precedence: env over config over SWITCH_DEFAULT; a typo takes the default; an unreadable config is off', () => {
|
||||
expect(readSkillSwitch('skill_routing', { GSTACK_STATE_ROOT: state })).toEqual({ on: SWITCH_DEFAULT.skill_routing === 'on', source: 'default' });
|
||||
fs.writeFileSync(path.join(state, 'config.yaml'), 'skill_routing: on\nskill_contracts: maybe\n');
|
||||
expect(readSkillSwitch('skill_routing', { GSTACK_STATE_ROOT: state })).toEqual({ on: true, source: 'config skill_routing' });
|
||||
expect(readSkillSwitch('skill_routing', { GSTACK_STATE_ROOT: state, GSTACK_SKILL_ROUTING: 'off' })).toEqual({ on: false, source: 'env GSTACK_SKILL_ROUTING' });
|
||||
expect(readSkillSwitch('skill_contracts', { GSTACK_STATE_ROOT: state })).toEqual({ on: false, source: 'config skill_contracts', invalid: 'maybe' });
|
||||
expect(readSkillSwitch('skill_contracts', { GSTACK_STATE_ROOT: state })).toEqual({ on: SWITCH_DEFAULT.skill_contracts === 'on', source: 'config skill_contracts', invalid: 'maybe' });
|
||||
fs.rmSync(path.join(state, 'config.yaml'));
|
||||
fs.mkdirSync(path.join(state, 'config.yaml'));
|
||||
for (const name of ['skill_routing', 'skill_contracts'] as const) {
|
||||
expect(readSkillSwitch(name, { GSTACK_STATE_ROOT: state })).toEqual({ on: false, source: 'config unreadable' });
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
@@ -101,8 +108,10 @@ describe('skill-route hook', () => {
|
||||
.toContain('Done means: a DEBUG REPORT with Symptom, Root cause, Fix, Evidence; a test or repro run after your last edit.');
|
||||
});
|
||||
|
||||
test('off by default, off in config, and off in env over config on', () => {
|
||||
expect(route(prompt('fix this bug'), {})).toEqual(silent);
|
||||
test('with nothing set SWITCH_DEFAULT decides; off in config, and off in env over config on', () => {
|
||||
const unset = route(prompt('fix this bug'), {});
|
||||
if (SWITCH_DEFAULT.skill_routing === 'on') expect(expectHookSchema(unset.stdout)).toContain('/investigate');
|
||||
else expect(unset).toEqual(silent);
|
||||
fs.writeFileSync(path.join(state, 'config.yaml'), 'skill_routing: off\n');
|
||||
expect(route(prompt('fix this bug'), {})).toEqual(silent);
|
||||
fs.writeFileSync(path.join(state, 'config.yaml'), 'skill_routing: on\n');
|
||||
@@ -136,7 +145,7 @@ describe('skill-route hook', () => {
|
||||
expect(traceLog()).toContain('event=skill-problems problems="broken: malformed triggers" failures=1');
|
||||
});
|
||||
|
||||
test('unreadable config and an unwritable state root: still exit 0 and silent', () => {
|
||||
test('an unreadable config is off whatever the default; an unwritable state root still routes when on', () => {
|
||||
fs.mkdirSync(path.join(state, 'config.yaml'));
|
||||
expect(route(prompt('fix this bug'), {})).toEqual(silent);
|
||||
const file = path.join(state, 'root-is-a-file');
|
||||
@@ -158,10 +167,10 @@ describe('skill-route hook', () => {
|
||||
describe('gstack-skill-route --explain', () => {
|
||||
const explain = (...args: string[]) => spawnSync(path.join(ROOT, 'bin', 'gstack-skill-route'), args, { env: hookEnv(state, skills), encoding: 'utf8', timeout: 20_000 });
|
||||
|
||||
test('prints the switch, threshold, candidates with scores, the route and the hint (a dry run with the switch off)', () => {
|
||||
test('prints the switch, threshold, candidates with scores, the route and the hint (a dry run whatever the switch)', () => {
|
||||
const r = explain('--explain', 'this test fails, fix it');
|
||||
expect(r.status).toBe(0);
|
||||
expect(r.stdout).toContain('switch: skill_routing off (default)');
|
||||
expect(r.stdout).toContain(`switch: skill_routing ${SWITCH_DEFAULT.skill_routing} (default)`);
|
||||
expect(r.stdout).toContain('threshold: 0.66');
|
||||
expect(r.stdout).toContain('1.00 /investigate "this test fails"');
|
||||
expect(r.stdout).toContain('route: /investigate');
|
||||
|
||||
Reference in new issue
Block a user