diff --git a/test/gen-skill-docs.test.ts b/test/gen-skill-docs.test.ts index 5f03336f0..60b3a0855 100644 --- a/test/gen-skill-docs.test.ts +++ b/test/gen-skill-docs.test.ts @@ -9,7 +9,7 @@ import * as path from 'path'; import * as os from 'os'; import { spawnSync } from 'child_process'; import { runCapturedCommand } from './helpers/sync-command-capture'; -import { expectMentions, expectTokens } from './helpers/prompt-structure'; +import { between, expectAbsent, expectMentions, expectOrdered, expectTokens } from './helpers/prompt-structure'; const ROOT = path.resolve(import.meta.dir, '..'); const MAX_SKILL_DESCRIPTION_LENGTH = 1024; @@ -1951,6 +1951,139 @@ describe('BENEFITS_FROM resolver', () => { // Should contain the INVOKE_SKILL-style loading prose (not the old manual skip list) expect(engContent).toContain('skipping these sections'); }); + // #1958: spawned/headless sessions have no human to answer the offer. + test('every prerequisite offer is skipped in spawned and headless sessions, before any option', () => { + const renders: Array<[string, string]> = [ + ['plan-ceo-review', ceoContent], + ['plan-eng-review', engContent], + ['plan-devex-review', readSkillUnion('plan-devex-review')], + ['autoplan', readSkillUnion('autoplan')], + ]; + for (const [skill, content] of renders) { + const offer = extractMarkdownSection(content, '## Prerequisite Skill Offer'); + expectTokens(offer, ['`SESSION_KIND`', '`spawned`', '`headless`'], `${skill} offer`); + expectMentions(offer, [['skip', 'offer', 'session_kind']], `${skill} offer`); + expectOrdered(offer, ['`SESSION_KIND`', 'A) Run /office-hours now'], `${skill} offer`); + } + }); +}); + +describe('office-hours closing reads the builder profile before logging the session (#2801)', () => { + const skeleton = fs.readFileSync(path.join(ROOT, 'office-hours', 'SKILL.md'), 'utf-8'); + const handoff = fs.readFileSync(path.join(ROOT, 'office-hours', 'sections', 'design-and-handoff.md'), 'utf-8'); + const phase45 = extractMarkdownSection(skeleton, '## Phase 4.5: Founder Signal Synthesis'); + const step1 = between(handoff, '### Step 1: Builder Profile', '### Step 2:'); + const fence = (text: string, after: string) => between(text, after).match(/```bash\n([\s\S]*?)\n```/)![1]!; + const readBlock = fence(phase45, '### Builder Profile Read'); + const logBlock = fence(phase45, '### Builder Profile Append'); + const CARRY = 'Builder profile before this session:'; + + test('Phase 4.5 reads the profile before --log-session and records the prior tier for Phase 6', () => { + expect(readBlock).toContain('bin/gstack-builder-profile'); + expect(logBlock).toContain('bin/gstack-developer-profile --log-session'); + expectOrdered(phase45, ['bin/gstack-builder-profile', 'bin/gstack-developer-profile --log-session'], 'Phase 4.5'); + expectTokens(phase45, [CARRY, 'PROFILE_READ=', 'SESSION_TIER=', 'PRIOR_SESSION_COUNT=', 'LAST_ASSIGNMENT=', 'CROSS_PROJECT='], 'Phase 4.5'); + }); + + test('a failed profile read proceeds as a first session and says so', () => { + expect(readBlock).toContain('PROFILE_READ: failed'); + expectMentions(phase45, [['failed', 'introduction']], 'Phase 4.5'); + expectMentions(step1, [['failed', 'introduction'], ["couldn't read", 'builder profile', 'first']], 'Phase 6 Step 1'); + }); + + test('Phase 6 takes tier and last-session fields from the Phase 4.5 line, never from a post-log read', () => { + expectTokens(step1, [CARRY, 'PRIOR_SESSION_COUNT + 1'], 'Phase 6 Step 1'); + expectMentions(step1, [['never', 'tier', 'session_count']], 'Phase 6 Step 1'); + expectAbsent(step1, ['SESSION_TIER=$(', 'SESSION_COUNT=$('], 'Phase 6 Step 1'); + const tiers = between(handoff, '### Step 2: Follow the Tier Path', '## Founder Resources'); + expectAbsent(tiers, ['[SESSION_COUNT]', 'from profile]'], 'tier paths'); + }); + + test('executing the rendered blocks in order: a first session reads introduction, a failed read is marked', () => { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'oh-tier-')); + try { + const home = path.join(dir, 'home'), state = path.join(dir, 'state'); + const bin = path.join(home, '.claude/skills/gstack/bin'); + fs.mkdirSync(bin, { recursive: true }); + for (const file of ['gstack-builder-profile', 'gstack-developer-profile', 'gstack-state-root.sh', 'gstack-slug', 'gstack-remote-identity.sh']) { + fs.copyFileSync(path.join(ROOT, 'bin', file), path.join(bin, file)); + fs.chmodSync(path.join(bin, file), 0o755); + } + const env = { ...process.env, HOME: home, GSTACK_HOME: state, GSTACK_STATE_ROOT: state, GIT_CEILING_DIRECTORIES: dir }; + const run = (command: string) => runCapturedCommand('bash', ['-c', command], { cwd: dir, env, timeout: 15000, captureStdout: true }); + const log = logBlock + .replace('TIMESTAMP', '2026-10-07T00:00:00Z').replace('MODE', 'builder').replace('"SLUG"', '"proj"') + .replace(':N,', ':2,').replace('SIGNALS_ARRAY', '["taste"]').replace('DOC_PATH', 'd.md') + .replace('ASSIGNMENT_TEXT', 'Talk to five users').replace('TOPICS_ARRAY', '[]'); + const first = run(readBlock); + expect(first.status, first.stderr).toBe(0); + expect(first.stdout).toMatch(/^SESSION_COUNT: 0$/m); + expect(first.stdout).toMatch(/^TIER: introduction$/m); + expect(run(log).status).toBe(0); + const afterLog = run(readBlock); + expect(afterLog.stdout).toMatch(/^TIER: welcome_back$/m); + expect(afterLog.stdout).toContain('LAST_ASSIGNMENT: Talk to five users'); + fs.writeFileSync(path.join(bin, 'gstack-builder-profile'), '#!/usr/bin/env bash\nexit 3\n', { mode: 0o755 }); + const failed = run(readBlock); + expect(failed.status).toBe(0); + expect(failed.stdout.trim()).toBe('PROFILE_READ: failed'); + } finally { fs.rmSync(dir, { recursive: true, force: true }); } + }); +}); + +describe('office-hours reports success only when this run wrote its design doc (#1049)', () => { + const skeleton = fs.readFileSync(path.join(ROOT, 'office-hours', 'SKILL.md'), 'utf-8'); + const check = between(skeleton, "## Before telemetry: confirm this run's design doc", '\n## '); + const block = check.match(/```bash\n([\s\S]*?)\n```/)![1]!; + + test('success requires DESIGN_DOC: ok; a missing doc maps to abort or error, not success', () => { + expect(block).toContain('bin/gstack-paths --get GSTACK_STATE_ROOT'); + expect(block).not.toMatch(/\bls -t\b|find /); + expectMentions(check, [['success', 'design_doc: ok'], ['abort', 'error', 'design_doc']], 'design doc check'); + expectOrdered(skeleton, ['## Phase 4.5', "## Before telemetry: confirm this run's design doc", '## Important Rules'], 'office-hours'); + }); + + test('the rendered check accepts only this run\'s non-empty doc under the state root', () => { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'oh-doc-')); + try { + const home = path.join(dir, 'home'), state = path.join(dir, 'state'); + const bin = path.join(home, '.claude/skills/gstack/bin'); + fs.mkdirSync(bin, { recursive: true }); + for (const file of ['gstack-paths', 'gstack-state-root.sh']) { + fs.copyFileSync(path.join(ROOT, 'bin', file), path.join(bin, file)); + fs.chmodSync(path.join(bin, file), 0o755); + } + const doc = path.join(state, 'projects', 'proj', 'me-main-design-20261007-120000.md'); + fs.mkdirSync(path.dirname(doc), { recursive: true }); + fs.writeFileSync(path.join(path.dirname(doc), 'other-main-design-20261007-130000.md'), '# Another session\n'); + const outside = path.join(dir, 'me-main-design-20261007-120000.md'); + fs.writeFileSync(outside, '# Outside the state root\n'); + const env = { ...process.env, HOME: home, GSTACK_HOME: state, GSTACK_STATE_ROOT: state }; + const run = (docPath: string) => { + const result = runCapturedCommand('bash', ['-c', block.replace('DESIGN_DOC_PATH', docPath)], { cwd: dir, env, timeout: 15000, captureStdout: true }); + expect(result.status, result.stderr).toBe(0); + return result.stdout.trim(); + }; + expect(run(doc)).toBe('DESIGN_DOC: missing'); + expect(run('')).toBe('DESIGN_DOC: missing'); + expect(run(outside)).toBe('DESIGN_DOC: missing'); + fs.writeFileSync(doc, ''); + expect(run(doc)).toBe('DESIGN_DOC: missing'); + fs.writeFileSync(doc, '# Design: fixture\n'); + expect(run(doc)).toBe('DESIGN_DOC: ok'); + } finally { fs.rmSync(dir, { recursive: true, force: true }); } + }); +}); + +describe('office-hours shows the design doc before asking for approval (#879)', () => { + const handoff = fs.readFileSync(path.join(ROOT, 'office-hours', 'sections', 'design-and-handoff.md'), 'utf-8'); + + test('the full doc is printed inline as assistant text, after spec review and before the approval question', () => { + const approval = 'Present the reviewed design doc to the user via AskUserQuestion'; + const beforeApproval = between(handoff, 'spec-review.jsonl', approval); + expectMentions(beforeApproval, [['full', 'design doc', 'inline'], ['tool', 'collapsed']], 'pre-approval'); + expectOrdered(handoff, ['spec-review.jsonl', 'output the full design doc inline', approval, 'A) Approve'], 'design-and-handoff'); + }); }); // --- {{INVOKE_SKILL}} resolver tests --- diff --git a/test/helpers/carve-guards.ts b/test/helpers/carve-guards.ts index 5a9b86f84..4c34681ab 100644 --- a/test/helpers/carve-guards.ts +++ b/test/helpers/carve-guards.ts @@ -223,7 +223,7 @@ export const CARVE_GUARDS: Record = { // 1.08 → 1.10: the scope-gate exceptions block (+ its adversarial-review // hardening: host-anchored mode signal, precedence, passing-mention // guards) and the plan-mode preamble reword land the union at 1.092. - maxSizeRatio: 1.1859, // + clarity rules for saved decisions/setup gates + the Aside probe's failure reason; measured 1.1504. + test value bar and Tests to Retire in the lazy Test review section (~2.6KB); measured 1.168 + W1 guarded state-root resolution (`eval gstack-paths; : "${GSTACK_STATE_ROOT:?…}"`) in the Context Recovery preamble, the eureka log and each state-writing bash block; measured 1.173 (2026-09-30). + v1.91.12.0 merge of #2999 (headless rule: a disallowed question tool never qualifies) with #3002; measured 1.1741 (2026-10-01). + Oct 6 fix wave A3/CEO-12: free text reaches commands only through agent-written mktemp files (mktemp block, write rule, path rebuild and refusal at each posting, prompt and brief site); measured 146,905 / 124,597 = 1.1790 (2026-10-06). Combined Oct 6 wave head (A and B): measured 147,524 / 124,597 = 1.1840 (2026-10-06). + Oct 7 wave B3: the Codex probe runs as a command (select-model capture lines, check-sandbox/probe-model/run-with-timeout subcommands) instead of being sourced; measured 147,693 / 124,597 = 1.1854 (2026-10-07). + Oct 7 wave CEO-28/DX-11: review prompts ask for a severity per finding and design proposals use the proposal gate; measured 147,752 / 124,597 = 1.1858 (2026-10-07). + maxSizeRatio: 1.1870, // + clarity rules for saved decisions/setup gates + the Aside probe's failure reason; measured 1.1504. + test value bar and Tests to Retire in the lazy Test review section (~2.6KB); measured 1.168 + W1 guarded state-root resolution (`eval gstack-paths; : "${GSTACK_STATE_ROOT:?…}"`) in the Context Recovery preamble, the eureka log and each state-writing bash block; measured 1.173 (2026-09-30). + v1.91.12.0 merge of #2999 (headless rule: a disallowed question tool never qualifies) with #3002; measured 1.1741 (2026-10-01). + Oct 6 fix wave A3/CEO-12: free text reaches commands only through agent-written mktemp files (mktemp block, write rule, path rebuild and refusal at each posting, prompt and brief site); measured 146,905 / 124,597 = 1.1790 (2026-10-06). Combined Oct 6 wave head (A and B): measured 147,524 / 124,597 = 1.1840 (2026-10-06). + Oct 7 wave B3: the Codex probe runs as a command (select-model capture lines, check-sandbox/probe-model/run-with-timeout subcommands) instead of being sourced; measured 147,693 / 124,597 = 1.1854 (2026-10-07). + Oct 7 wave CEO-28/DX-11: review prompts ask for a severity per finding and design proposals use the proposal gate; measured 147,752 / 124,597 = 1.1858 (2026-10-07). + fix wave PR D: the prerequisite offer is skipped in spawned/headless sessions (#1958); measured 147,864 / 124,597 = 1.1867 (2026-10-07). }, 'plan-design-review': { skill: 'plan-design-review', @@ -350,10 +350,10 @@ do not launch the downstream skill or open a browser.`, // the #538 opt-out + D1 evidence directive — ratio 1.104 measured. // #2499 project-scope MCP jq in the brain-sync block grew every tier-2+ // skeleton ~1.5KB (entry resolution emitted once per SKILL.md). - maxSkeletonBytes: 93_150, // Office-hours + sketch outside voices include host guards and completion checks. + v1.91.19.0 wave: Q open-question prose form (#2729, approved Q3), DESIGN_READY --version probe (B4), Aside ~/.local/bin fallback (E7), verdict-form outside voice (INV-1), plus v1.91.17.0's round accounting (#3022) merged in; measured 89,684 (2026-10-03). + Oct 6 fix wave A3/CEO-12: free text reaches commands only through agent-written mktemp files (mktemp block, write rule, path rebuild and refusal at each posting, prompt and brief site); skeleton measured 92,590 (2026-10-06). + Oct 7 wave B3: the Codex probe runs as a command (select-model capture lines, check-sandbox/probe-model/run-with-timeout subcommands) instead of being sourced; skeleton measured 92,869 (2026-10-07). + Oct 7 wave CEO-28/DX-11: review prompts ask for a severity per finding and design proposals use the proposal gate; skeleton measured 92,930 (2026-10-07). + Oct 7 wave guard (CEO-20): conditional `run_in_background` request plus the background-run recovery at each foreground dispatch site; skeleton measured 92,756 (2026-10-07). Merged with the guard area: measured skeleton 93,099 (2026-10-07). + maxSkeletonBytes: 94_900, // Office-hours + sketch outside voices include host guards and completion checks. + v1.91.19.0 wave: Q open-question prose form (#2729, approved Q3), DESIGN_READY --version probe (B4), Aside ~/.local/bin fallback (E7), verdict-form outside voice (INV-1), plus v1.91.17.0's round accounting (#3022) merged in; measured 89,684 (2026-10-03). + Oct 6 fix wave A3/CEO-12: free text reaches commands only through agent-written mktemp files (mktemp block, write rule, path rebuild and refusal at each posting, prompt and brief site); skeleton measured 92,590 (2026-10-06). + Oct 7 wave B3: the Codex probe runs as a command (select-model capture lines, check-sandbox/probe-model/run-with-timeout subcommands) instead of being sourced; skeleton measured 92,869 (2026-10-07). + Oct 7 wave CEO-28/DX-11: review prompts ask for a severity per finding and design proposals use the proposal gate; skeleton measured 92,930 (2026-10-07). + Oct 7 wave guard (CEO-20): conditional `run_in_background` request plus the background-run recovery at each foreground dispatch site; skeleton measured 92,756 (2026-10-07). Merged with the guard area: measured skeleton 93,099 (2026-10-07). + fix wave PR D: the Phase 4.5 builder-profile read before the session log (#2801) and the pre-telemetry design-doc check (#1049); measured skeleton 94,877 (2026-10-07). minUnionBytes: 115_800, // Phase 4 wave 4; measured union 118,175 mustContain: ['design doc', 'problem statement'], - maxSizeRatio: 1.1470, // + Oct 6 fix wave A3/CEO-12: free text reaches commands only through agent-written mktemp files (mktemp block, write rule, path rebuild and refusal at each posting, prompt and brief site); measured 149,758 / 131,110 = 1.1422 (2026-10-06). + Oct 7 wave B3: the Codex probe runs as a command (select-model capture lines, check-sandbox/probe-model/run-with-timeout subcommands) instead of being sourced; measured 150,037 / 131,110 = 1.1444 (2026-10-07). + Oct 7 wave CEO-28/DX-11: review prompts ask for a severity per finding and design proposals use the proposal gate; measured 150,098 / 131,110 = 1.1448 (2026-10-07). + Oct 7 wave guard (CEO-20): conditional `run_in_background` request plus the background-run recovery at each foreground dispatch site; measured 150,011 / 131,110 = 1.1442 (2026-10-07). Merged with the guard area: measured 150,354 / 131,110 = 1.1468 (2026-10-07). + maxSizeRatio: 1.1690, // + Oct 6 fix wave A3/CEO-12: free text reaches commands only through agent-written mktemp files (mktemp block, write rule, path rebuild and refusal at each posting, prompt and brief site); measured 149,758 / 131,110 = 1.1422 (2026-10-06). + Oct 7 wave B3: the Codex probe runs as a command (select-model capture lines, check-sandbox/probe-model/run-with-timeout subcommands) instead of being sourced; measured 150,037 / 131,110 = 1.1444 (2026-10-07). + Oct 7 wave CEO-28/DX-11: review prompts ask for a severity per finding and design proposals use the proposal gate; measured 150,098 / 131,110 = 1.1448 (2026-10-07). + Oct 7 wave guard (CEO-20): conditional `run_in_background` request plus the background-run recovery at each foreground dispatch site; measured 150,011 / 131,110 = 1.1442 (2026-10-07). Merged with the guard area: measured 150,354 / 131,110 = 1.1468 (2026-10-07). + fix wave PR D: builder-profile read before the session log with Phase 6 using the carried values (#2801), the inline design doc before approval (#879) and the pre-telemetry design-doc check (#1049); measured 153,206 / 131,110 = 1.1685 (2026-10-07). }, 'document-release': { skill: 'document-release', @@ -577,7 +577,7 @@ do not launch the downstream skill or open a browser.`, maxSkeletonBytes: 70_000, // Phase-specific outside coverage, native fallback, and harness guard. minUnionBytes: 85_000, // measured union 86,926 mustContain: ['6 Decision Principles', 'TASTE DECISION', 'USER CHALLENGE', 'consensus', 'Restore Point'], - maxSizeRatio: 1.1295, // Four validated outside invocations replace raw CLI calls; phases keep independent coverage. + Oct 7 wave B3: the Codex probe runs as a command (select-model capture lines, check-sandbox/probe-model/run-with-timeout subcommands) instead of being sourced; measured 114,297 / 101,979 = 1.1208 (2026-10-07). + Oct 7 wave CEO-28/DX-11: review prompts ask for a severity per finding and design proposals use the proposal gate; measured 114,533 / 101,979 = 1.1231 (2026-10-07). + Oct 7 wave guard (CEO-20): conditional `run_in_background` request plus the background-run recovery at each foreground dispatch site, and phase-close publishing each report with the `true autoplan-published ` no-op before the next phase entry in a later message (CEO-1); measured 114,320 / 101,979 = 1.1210 (2026-10-07). Merged with the guard area: measured 115,149 / 101,979 = 1.1291 (2026-10-07). + maxSizeRatio: 1.1305, // Four validated outside invocations replace raw CLI calls; phases keep independent coverage. + Oct 7 wave B3: the Codex probe runs as a command (select-model capture lines, check-sandbox/probe-model/run-with-timeout subcommands) instead of being sourced; measured 114,297 / 101,979 = 1.1208 (2026-10-07). + Oct 7 wave CEO-28/DX-11: review prompts ask for a severity per finding and design proposals use the proposal gate; measured 114,533 / 101,979 = 1.1231 (2026-10-07). + Oct 7 wave guard (CEO-20): conditional `run_in_background` request plus the background-run recovery at each foreground dispatch site, and phase-close publishing each report with the `true autoplan-published ` no-op before the next phase entry in a later message (CEO-1); measured 114,320 / 101,979 = 1.1210 (2026-10-07). Merged with the guard area: measured 115,149 / 101,979 = 1.1291 (2026-10-07). + fix wave PR D: the prerequisite offer is skipped in spawned/headless sessions (#1958); measured 115,267 / 101,979 = 1.1303 (2026-10-07). }, spec: { skill: 'spec',