mirror of
https://github.com/garrytan/gstack.git
synced 2026-11-06 08:54:04 +01:00
test: office-hours tier order, failed-read fallback, inline doc, design-doc check and offer skip
Rendered-block tests run the Phase 4.5 read and log on a temp state root (first session reads introduction, failed read is marked) and the pre-telemetry check against this run's doc, an empty doc, another session's doc and a path outside the state root. Parity caps take the measured growth. Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>
This commit is contained in:
garrytanandcapy-ai[bot] committed
1 parent
e94f545f06
commit
de8c61686e
2 files changed
+138
-5
No files matched your search
+134
-1
@@ -9,7 +9,7 @@ import * as path from 'path';
|
||||
import * as os from 'os';
|
||||
import { spawnSync } from 'child_process';
|
||||
import { runCapturedCommand } from './helpers/sync-command-capture';
|
||||
import { expectMentions, expectTokens } from './helpers/prompt-structure';
|
||||
import { between, expectAbsent, expectMentions, expectOrdered, expectTokens } from './helpers/prompt-structure';
|
||||
|
||||
const ROOT = path.resolve(import.meta.dir, '..');
|
||||
const MAX_SKILL_DESCRIPTION_LENGTH = 1024;
|
||||
@@ -1951,6 +1951,139 @@ describe('BENEFITS_FROM resolver', () => {
|
||||
// Should contain the INVOKE_SKILL-style loading prose (not the old manual skip list)
|
||||
expect(engContent).toContain('skipping these sections');
|
||||
});
|
||||
// #1958: spawned/headless sessions have no human to answer the offer.
|
||||
test('every prerequisite offer is skipped in spawned and headless sessions, before any option', () => {
|
||||
const renders: Array<[string, string]> = [
|
||||
['plan-ceo-review', ceoContent],
|
||||
['plan-eng-review', engContent],
|
||||
['plan-devex-review', readSkillUnion('plan-devex-review')],
|
||||
['autoplan', readSkillUnion('autoplan')],
|
||||
];
|
||||
for (const [skill, content] of renders) {
|
||||
const offer = extractMarkdownSection(content, '## Prerequisite Skill Offer');
|
||||
expectTokens(offer, ['`SESSION_KIND`', '`spawned`', '`headless`'], `${skill} offer`);
|
||||
expectMentions(offer, [['skip', 'offer', 'session_kind']], `${skill} offer`);
|
||||
expectOrdered(offer, ['`SESSION_KIND`', 'A) Run /office-hours now'], `${skill} offer`);
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
describe('office-hours closing reads the builder profile before logging the session (#2801)', () => {
|
||||
const skeleton = fs.readFileSync(path.join(ROOT, 'office-hours', 'SKILL.md'), 'utf-8');
|
||||
const handoff = fs.readFileSync(path.join(ROOT, 'office-hours', 'sections', 'design-and-handoff.md'), 'utf-8');
|
||||
const phase45 = extractMarkdownSection(skeleton, '## Phase 4.5: Founder Signal Synthesis');
|
||||
const step1 = between(handoff, '### Step 1: Builder Profile', '### Step 2:');
|
||||
const fence = (text: string, after: string) => between(text, after).match(/```bash\n([\s\S]*?)\n```/)![1]!;
|
||||
const readBlock = fence(phase45, '### Builder Profile Read');
|
||||
const logBlock = fence(phase45, '### Builder Profile Append');
|
||||
const CARRY = 'Builder profile before this session:';
|
||||
|
||||
test('Phase 4.5 reads the profile before --log-session and records the prior tier for Phase 6', () => {
|
||||
expect(readBlock).toContain('bin/gstack-builder-profile');
|
||||
expect(logBlock).toContain('bin/gstack-developer-profile --log-session');
|
||||
expectOrdered(phase45, ['bin/gstack-builder-profile', 'bin/gstack-developer-profile --log-session'], 'Phase 4.5');
|
||||
expectTokens(phase45, [CARRY, 'PROFILE_READ=', 'SESSION_TIER=', 'PRIOR_SESSION_COUNT=', 'LAST_ASSIGNMENT=', 'CROSS_PROJECT='], 'Phase 4.5');
|
||||
});
|
||||
|
||||
test('a failed profile read proceeds as a first session and says so', () => {
|
||||
expect(readBlock).toContain('PROFILE_READ: failed');
|
||||
expectMentions(phase45, [['failed', 'introduction']], 'Phase 4.5');
|
||||
expectMentions(step1, [['failed', 'introduction'], ["couldn't read", 'builder profile', 'first']], 'Phase 6 Step 1');
|
||||
});
|
||||
|
||||
test('Phase 6 takes tier and last-session fields from the Phase 4.5 line, never from a post-log read', () => {
|
||||
expectTokens(step1, [CARRY, 'PRIOR_SESSION_COUNT + 1'], 'Phase 6 Step 1');
|
||||
expectMentions(step1, [['never', 'tier', 'session_count']], 'Phase 6 Step 1');
|
||||
expectAbsent(step1, ['SESSION_TIER=$(', 'SESSION_COUNT=$('], 'Phase 6 Step 1');
|
||||
const tiers = between(handoff, '### Step 2: Follow the Tier Path', '## Founder Resources');
|
||||
expectAbsent(tiers, ['[SESSION_COUNT]', 'from profile]'], 'tier paths');
|
||||
});
|
||||
|
||||
test('executing the rendered blocks in order: a first session reads introduction, a failed read is marked', () => {
|
||||
const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'oh-tier-'));
|
||||
try {
|
||||
const home = path.join(dir, 'home'), state = path.join(dir, 'state');
|
||||
const bin = path.join(home, '.claude/skills/gstack/bin');
|
||||
fs.mkdirSync(bin, { recursive: true });
|
||||
for (const file of ['gstack-builder-profile', 'gstack-developer-profile', 'gstack-state-root.sh', 'gstack-slug', 'gstack-remote-identity.sh']) {
|
||||
fs.copyFileSync(path.join(ROOT, 'bin', file), path.join(bin, file));
|
||||
fs.chmodSync(path.join(bin, file), 0o755);
|
||||
}
|
||||
const env = { ...process.env, HOME: home, GSTACK_HOME: state, GSTACK_STATE_ROOT: state, GIT_CEILING_DIRECTORIES: dir };
|
||||
const run = (command: string) => runCapturedCommand('bash', ['-c', command], { cwd: dir, env, timeout: 15000, captureStdout: true });
|
||||
const log = logBlock
|
||||
.replace('TIMESTAMP', '2026-10-07T00:00:00Z').replace('MODE', 'builder').replace('"SLUG"', '"proj"')
|
||||
.replace(':N,', ':2,').replace('SIGNALS_ARRAY', '["taste"]').replace('DOC_PATH', 'd.md')
|
||||
.replace('ASSIGNMENT_TEXT', 'Talk to five users').replace('TOPICS_ARRAY', '[]');
|
||||
const first = run(readBlock);
|
||||
expect(first.status, first.stderr).toBe(0);
|
||||
expect(first.stdout).toMatch(/^SESSION_COUNT: 0$/m);
|
||||
expect(first.stdout).toMatch(/^TIER: introduction$/m);
|
||||
expect(run(log).status).toBe(0);
|
||||
const afterLog = run(readBlock);
|
||||
expect(afterLog.stdout).toMatch(/^TIER: welcome_back$/m);
|
||||
expect(afterLog.stdout).toContain('LAST_ASSIGNMENT: Talk to five users');
|
||||
fs.writeFileSync(path.join(bin, 'gstack-builder-profile'), '#!/usr/bin/env bash\nexit 3\n', { mode: 0o755 });
|
||||
const failed = run(readBlock);
|
||||
expect(failed.status).toBe(0);
|
||||
expect(failed.stdout.trim()).toBe('PROFILE_READ: failed');
|
||||
} finally { fs.rmSync(dir, { recursive: true, force: true }); }
|
||||
});
|
||||
});
|
||||
|
||||
describe('office-hours reports success only when this run wrote its design doc (#1049)', () => {
|
||||
const skeleton = fs.readFileSync(path.join(ROOT, 'office-hours', 'SKILL.md'), 'utf-8');
|
||||
const check = between(skeleton, "## Before telemetry: confirm this run's design doc", '\n## ');
|
||||
const block = check.match(/```bash\n([\s\S]*?)\n```/)![1]!;
|
||||
|
||||
test('success requires DESIGN_DOC: ok; a missing doc maps to abort or error, not success', () => {
|
||||
expect(block).toContain('bin/gstack-paths --get GSTACK_STATE_ROOT');
|
||||
expect(block).not.toMatch(/\bls -t\b|find /);
|
||||
expectMentions(check, [['success', 'design_doc: ok'], ['abort', 'error', 'design_doc']], 'design doc check');
|
||||
expectOrdered(skeleton, ['## Phase 4.5', "## Before telemetry: confirm this run's design doc", '## Important Rules'], 'office-hours');
|
||||
});
|
||||
|
||||
test('the rendered check accepts only this run\'s non-empty doc under the state root', () => {
|
||||
const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'oh-doc-'));
|
||||
try {
|
||||
const home = path.join(dir, 'home'), state = path.join(dir, 'state');
|
||||
const bin = path.join(home, '.claude/skills/gstack/bin');
|
||||
fs.mkdirSync(bin, { recursive: true });
|
||||
for (const file of ['gstack-paths', 'gstack-state-root.sh']) {
|
||||
fs.copyFileSync(path.join(ROOT, 'bin', file), path.join(bin, file));
|
||||
fs.chmodSync(path.join(bin, file), 0o755);
|
||||
}
|
||||
const doc = path.join(state, 'projects', 'proj', 'me-main-design-20261007-120000.md');
|
||||
fs.mkdirSync(path.dirname(doc), { recursive: true });
|
||||
fs.writeFileSync(path.join(path.dirname(doc), 'other-main-design-20261007-130000.md'), '# Another session\n');
|
||||
const outside = path.join(dir, 'me-main-design-20261007-120000.md');
|
||||
fs.writeFileSync(outside, '# Outside the state root\n');
|
||||
const env = { ...process.env, HOME: home, GSTACK_HOME: state, GSTACK_STATE_ROOT: state };
|
||||
const run = (docPath: string) => {
|
||||
const result = runCapturedCommand('bash', ['-c', block.replace('DESIGN_DOC_PATH', docPath)], { cwd: dir, env, timeout: 15000, captureStdout: true });
|
||||
expect(result.status, result.stderr).toBe(0);
|
||||
return result.stdout.trim();
|
||||
};
|
||||
expect(run(doc)).toBe('DESIGN_DOC: missing');
|
||||
expect(run('')).toBe('DESIGN_DOC: missing');
|
||||
expect(run(outside)).toBe('DESIGN_DOC: missing');
|
||||
fs.writeFileSync(doc, '');
|
||||
expect(run(doc)).toBe('DESIGN_DOC: missing');
|
||||
fs.writeFileSync(doc, '# Design: fixture\n');
|
||||
expect(run(doc)).toBe('DESIGN_DOC: ok');
|
||||
} finally { fs.rmSync(dir, { recursive: true, force: true }); }
|
||||
});
|
||||
});
|
||||
|
||||
describe('office-hours shows the design doc before asking for approval (#879)', () => {
|
||||
const handoff = fs.readFileSync(path.join(ROOT, 'office-hours', 'sections', 'design-and-handoff.md'), 'utf-8');
|
||||
|
||||
test('the full doc is printed inline as assistant text, after spec review and before the approval question', () => {
|
||||
const approval = 'Present the reviewed design doc to the user via AskUserQuestion';
|
||||
const beforeApproval = between(handoff, 'spec-review.jsonl', approval);
|
||||
expectMentions(beforeApproval, [['full', 'design doc', 'inline'], ['tool', 'collapsed']], 'pre-approval');
|
||||
expectOrdered(handoff, ['spec-review.jsonl', 'output the full design doc inline', approval, 'A) Approve'], 'design-and-handoff');
|
||||
});
|
||||
});
|
||||
|
||||
// --- {{INVOKE_SKILL}} resolver tests ---
|
||||
|
||||
@@ -223,7 +223,7 @@ export const CARVE_GUARDS: Record<string, CarveGuard> = {
|
||||
// 1.08 → 1.10: the scope-gate exceptions block (+ its adversarial-review
|
||||
// hardening: host-anchored mode signal, precedence, passing-mention
|
||||
// guards) and the plan-mode preamble reword land the union at 1.092.
|
||||
maxSizeRatio: 1.1859, // + clarity rules for saved decisions/setup gates + the Aside probe's failure reason; measured 1.1504. + test value bar and Tests to Retire in the lazy Test review section (~2.6KB); measured 1.168 + W1 guarded state-root resolution (`eval gstack-paths; : "${GSTACK_STATE_ROOT:?…}"`) in the Context Recovery preamble, the eureka log and each state-writing bash block; measured 1.173 (2026-09-30). + v1.91.12.0 merge of #2999 (headless rule: a disallowed question tool never qualifies) with #3002; measured 1.1741 (2026-10-01). + Oct 6 fix wave A3/CEO-12: free text reaches commands only through agent-written mktemp files (mktemp block, write rule, path rebuild and refusal at each posting, prompt and brief site); measured 146,905 / 124,597 = 1.1790 (2026-10-06). Combined Oct 6 wave head (A and B): measured 147,524 / 124,597 = 1.1840 (2026-10-06). + Oct 7 wave B3: the Codex probe runs as a command (select-model capture lines, check-sandbox/probe-model/run-with-timeout subcommands) instead of being sourced; measured 147,693 / 124,597 = 1.1854 (2026-10-07). + Oct 7 wave CEO-28/DX-11: review prompts ask for a severity per finding and design proposals use the proposal gate; measured 147,752 / 124,597 = 1.1858 (2026-10-07).
|
||||
maxSizeRatio: 1.1870, // + clarity rules for saved decisions/setup gates + the Aside probe's failure reason; measured 1.1504. + test value bar and Tests to Retire in the lazy Test review section (~2.6KB); measured 1.168 + W1 guarded state-root resolution (`eval gstack-paths; : "${GSTACK_STATE_ROOT:?…}"`) in the Context Recovery preamble, the eureka log and each state-writing bash block; measured 1.173 (2026-09-30). + v1.91.12.0 merge of #2999 (headless rule: a disallowed question tool never qualifies) with #3002; measured 1.1741 (2026-10-01). + Oct 6 fix wave A3/CEO-12: free text reaches commands only through agent-written mktemp files (mktemp block, write rule, path rebuild and refusal at each posting, prompt and brief site); measured 146,905 / 124,597 = 1.1790 (2026-10-06). Combined Oct 6 wave head (A and B): measured 147,524 / 124,597 = 1.1840 (2026-10-06). + Oct 7 wave B3: the Codex probe runs as a command (select-model capture lines, check-sandbox/probe-model/run-with-timeout subcommands) instead of being sourced; measured 147,693 / 124,597 = 1.1854 (2026-10-07). + Oct 7 wave CEO-28/DX-11: review prompts ask for a severity per finding and design proposals use the proposal gate; measured 147,752 / 124,597 = 1.1858 (2026-10-07). + fix wave PR D: the prerequisite offer is skipped in spawned/headless sessions (#1958); measured 147,864 / 124,597 = 1.1867 (2026-10-07).
|
||||
},
|
||||
'plan-design-review': {
|
||||
skill: 'plan-design-review',
|
||||
@@ -350,10 +350,10 @@ do not launch the downstream skill or open a browser.`,
|
||||
// the #538 opt-out + D1 evidence directive — ratio 1.104 measured.
|
||||
// #2499 project-scope MCP jq in the brain-sync block grew every tier-2+
|
||||
// skeleton ~1.5KB (entry resolution emitted once per SKILL.md).
|
||||
maxSkeletonBytes: 93_150, // Office-hours + sketch outside voices include host guards and completion checks. + v1.91.19.0 wave: Q<N> open-question prose form (#2729, approved Q3), DESIGN_READY --version probe (B4), Aside ~/.local/bin fallback (E7), verdict-form outside voice (INV-1), plus v1.91.17.0's round accounting (#3022) merged in; measured 89,684 (2026-10-03). + Oct 6 fix wave A3/CEO-12: free text reaches commands only through agent-written mktemp files (mktemp block, write rule, path rebuild and refusal at each posting, prompt and brief site); skeleton measured 92,590 (2026-10-06). + Oct 7 wave B3: the Codex probe runs as a command (select-model capture lines, check-sandbox/probe-model/run-with-timeout subcommands) instead of being sourced; skeleton measured 92,869 (2026-10-07). + Oct 7 wave CEO-28/DX-11: review prompts ask for a severity per finding and design proposals use the proposal gate; skeleton measured 92,930 (2026-10-07). + Oct 7 wave guard (CEO-20): conditional `run_in_background` request plus the background-run recovery at each foreground dispatch site; skeleton measured 92,756 (2026-10-07). Merged with the guard area: measured skeleton 93,099 (2026-10-07).
|
||||
maxSkeletonBytes: 94_900, // Office-hours + sketch outside voices include host guards and completion checks. + v1.91.19.0 wave: Q<N> open-question prose form (#2729, approved Q3), DESIGN_READY --version probe (B4), Aside ~/.local/bin fallback (E7), verdict-form outside voice (INV-1), plus v1.91.17.0's round accounting (#3022) merged in; measured 89,684 (2026-10-03). + Oct 6 fix wave A3/CEO-12: free text reaches commands only through agent-written mktemp files (mktemp block, write rule, path rebuild and refusal at each posting, prompt and brief site); skeleton measured 92,590 (2026-10-06). + Oct 7 wave B3: the Codex probe runs as a command (select-model capture lines, check-sandbox/probe-model/run-with-timeout subcommands) instead of being sourced; skeleton measured 92,869 (2026-10-07). + Oct 7 wave CEO-28/DX-11: review prompts ask for a severity per finding and design proposals use the proposal gate; skeleton measured 92,930 (2026-10-07). + Oct 7 wave guard (CEO-20): conditional `run_in_background` request plus the background-run recovery at each foreground dispatch site; skeleton measured 92,756 (2026-10-07). Merged with the guard area: measured skeleton 93,099 (2026-10-07). + fix wave PR D: the Phase 4.5 builder-profile read before the session log (#2801) and the pre-telemetry design-doc check (#1049); measured skeleton 94,877 (2026-10-07).
|
||||
minUnionBytes: 115_800, // Phase 4 wave 4; measured union 118,175
|
||||
mustContain: ['design doc', 'problem statement'],
|
||||
maxSizeRatio: 1.1470, // + Oct 6 fix wave A3/CEO-12: free text reaches commands only through agent-written mktemp files (mktemp block, write rule, path rebuild and refusal at each posting, prompt and brief site); measured 149,758 / 131,110 = 1.1422 (2026-10-06). + Oct 7 wave B3: the Codex probe runs as a command (select-model capture lines, check-sandbox/probe-model/run-with-timeout subcommands) instead of being sourced; measured 150,037 / 131,110 = 1.1444 (2026-10-07). + Oct 7 wave CEO-28/DX-11: review prompts ask for a severity per finding and design proposals use the proposal gate; measured 150,098 / 131,110 = 1.1448 (2026-10-07). + Oct 7 wave guard (CEO-20): conditional `run_in_background` request plus the background-run recovery at each foreground dispatch site; measured 150,011 / 131,110 = 1.1442 (2026-10-07). Merged with the guard area: measured 150,354 / 131,110 = 1.1468 (2026-10-07).
|
||||
maxSizeRatio: 1.1690, // + Oct 6 fix wave A3/CEO-12: free text reaches commands only through agent-written mktemp files (mktemp block, write rule, path rebuild and refusal at each posting, prompt and brief site); measured 149,758 / 131,110 = 1.1422 (2026-10-06). + Oct 7 wave B3: the Codex probe runs as a command (select-model capture lines, check-sandbox/probe-model/run-with-timeout subcommands) instead of being sourced; measured 150,037 / 131,110 = 1.1444 (2026-10-07). + Oct 7 wave CEO-28/DX-11: review prompts ask for a severity per finding and design proposals use the proposal gate; measured 150,098 / 131,110 = 1.1448 (2026-10-07). + Oct 7 wave guard (CEO-20): conditional `run_in_background` request plus the background-run recovery at each foreground dispatch site; measured 150,011 / 131,110 = 1.1442 (2026-10-07). Merged with the guard area: measured 150,354 / 131,110 = 1.1468 (2026-10-07). + fix wave PR D: builder-profile read before the session log with Phase 6 using the carried values (#2801), the inline design doc before approval (#879) and the pre-telemetry design-doc check (#1049); measured 153,206 / 131,110 = 1.1685 (2026-10-07).
|
||||
},
|
||||
'document-release': {
|
||||
skill: 'document-release',
|
||||
@@ -577,7 +577,7 @@ do not launch the downstream skill or open a browser.`,
|
||||
maxSkeletonBytes: 70_000, // Phase-specific outside coverage, native fallback, and harness guard.
|
||||
minUnionBytes: 85_000, // measured union 86,926
|
||||
mustContain: ['6 Decision Principles', 'TASTE DECISION', 'USER CHALLENGE', 'consensus', 'Restore Point'],
|
||||
maxSizeRatio: 1.1295, // Four validated outside invocations replace raw CLI calls; phases keep independent coverage. + Oct 7 wave B3: the Codex probe runs as a command (select-model capture lines, check-sandbox/probe-model/run-with-timeout subcommands) instead of being sourced; measured 114,297 / 101,979 = 1.1208 (2026-10-07). + Oct 7 wave CEO-28/DX-11: review prompts ask for a severity per finding and design proposals use the proposal gate; measured 114,533 / 101,979 = 1.1231 (2026-10-07). + Oct 7 wave guard (CEO-20): conditional `run_in_background` request plus the background-run recovery at each foreground dispatch site, and phase-close publishing each report with the `true autoplan-published <phase>` no-op before the next phase entry in a later message (CEO-1); measured 114,320 / 101,979 = 1.1210 (2026-10-07). Merged with the guard area: measured 115,149 / 101,979 = 1.1291 (2026-10-07).
|
||||
maxSizeRatio: 1.1305, // Four validated outside invocations replace raw CLI calls; phases keep independent coverage. + Oct 7 wave B3: the Codex probe runs as a command (select-model capture lines, check-sandbox/probe-model/run-with-timeout subcommands) instead of being sourced; measured 114,297 / 101,979 = 1.1208 (2026-10-07). + Oct 7 wave CEO-28/DX-11: review prompts ask for a severity per finding and design proposals use the proposal gate; measured 114,533 / 101,979 = 1.1231 (2026-10-07). + Oct 7 wave guard (CEO-20): conditional `run_in_background` request plus the background-run recovery at each foreground dispatch site, and phase-close publishing each report with the `true autoplan-published <phase>` no-op before the next phase entry in a later message (CEO-1); measured 114,320 / 101,979 = 1.1210 (2026-10-07). Merged with the guard area: measured 115,149 / 101,979 = 1.1291 (2026-10-07). + fix wave PR D: the prerequisite offer is skipped in spawned/headless sessions (#1958); measured 115,267 / 101,979 = 1.1303 (2026-10-07).
|
||||
},
|
||||
spec: {
|
||||
skill: 'spec',
|
||||
|
||||
Reference in new issue
Block a user