diff --git a/autoplan/SKILL.md b/autoplan/SKILL.md index 1614c60ec..9dcdf7751 100644 --- a/autoplan/SKILL.md +++ b/autoplan/SKILL.md @@ -269,6 +269,7 @@ sections. Read a section in full before doing its step; do not work from memory. | closing a review phase, after its reviews finish and before announcing completion or loading the next phase (read afresh at each exit) | `sections/phase-close.md` | | presenting the Final Approval Gate (Phase 4) — the aggregator computes $AGGREGATED_TASKS that the gate message substitutes | `sections/tasks-aggregator.md` | | the preamble echoed `SESSION_KIND: unattended` (read before Phase 1 and at the Phase 4 gate) | `sections/unattended.md` | +| the invocation names `--ceo-only` (read before any preamble instruction block) | `sections/ceo-only.md` | --- diff --git a/autoplan/sections/ceo-only.md b/autoplan/sections/ceo-only.md new file mode 100644 index 000000000..b5df3e232 --- /dev/null +++ b/autoplan/sections/ceo-only.md @@ -0,0 +1,41 @@ + + +## CEO-only run (`--ceo-only`) + +This section governs the whole run and wins over the skeleton. The run reviews +the plan; it never approves it. + +**Zero questions.** Never call AskUserQuestion (native or `mcp__*`) or +ExitPlanMode, never ask the user in prose, never wait for a reply, whatever +`SESSION_KIND` says. A preamble instruction block that would ask takes its +non-consenting default (off, not now, skip). Each decision takes its +`(recommended)` option, else the 6 principles; consent, destructive or +irreversible options take the conservative choice. Log each as a Decision Audit +Trail row. + +**Scope.** Run Phases 0, 0.5 and 1 only; send "Phases 2, 2.5, 3 and 4 skipped +(CEO-only run)" and record it in ACTIVE_PLAN. With `--no-outside`, skip every +outside dispatch: `outside_status: disabled (--no-outside)`. + +**Mode.** Instead of SELECTIVE EXPANSION, use plan-ceo-review 0E step 2's rule +(an explicit mode in the request wins), skip step 3, run +`~/.claude/skills/gstack/bin/gstack-ceo-mode-handoff "" --ceo-only --decisions ""` +and send its line verbatim. + +**Never applied:** a user challenge (keep the direction; list it below). +**Skipped and logged `skipped (ceo-only)`:** office-hours and BENEFITS_FROM +offers, 0E's question, plan archive cleanup, Next Steps chaining and CEO plan +promotion; the row satisfies its checklist entry. + +**State.** Write only in `$_AP`, ACTIVE_PLAN and `GSTACK_STATE_ROOT` +(the override; docs/state-root.md). + +**Finish.** After Phase 1 closes, write `$_AP/ceo-review.md`: `# CEO-only +review: `; a line with the mode, outside-voice status and "Not +approved"; then `## User challenges (direction kept)` (plan says, proposal, +why, cost if wrong), `## Scope added`, `## Conservative choices`, +`## Findings` (severity, section, fix, disposition) and `## Decision Audit +Trail` (copied), each "None." when empty. Send the CEO Completion Summary and +both paths, skip Completion's logs, approval and `/ship`, and end with: + +`GSTACK_RESULT: skill=autoplan status=reviewed run=` diff --git a/autoplan/sections/ceo-only.md.tmpl b/autoplan/sections/ceo-only.md.tmpl new file mode 100644 index 000000000..636943ed9 --- /dev/null +++ b/autoplan/sections/ceo-only.md.tmpl @@ -0,0 +1,39 @@ +## CEO-only run (`--ceo-only`) + +This section governs the whole run and wins over the skeleton. The run reviews +the plan; it never approves it. + +**Zero questions.** Never call AskUserQuestion (native or `mcp__*`) or +ExitPlanMode, never ask the user in prose, never wait for a reply, whatever +`SESSION_KIND` says. A preamble instruction block that would ask takes its +non-consenting default (off, not now, skip). Each decision takes its +`(recommended)` option, else the 6 principles; consent, destructive or +irreversible options take the conservative choice. Log each as a Decision Audit +Trail row. + +**Scope.** Run Phases 0, 0.5 and 1 only; send "Phases 2, 2.5, 3 and 4 skipped +(CEO-only run)" and record it in ACTIVE_PLAN. With `--no-outside`, skip every +outside dispatch: `outside_status: disabled (--no-outside)`. + +**Mode.** Instead of SELECTIVE EXPANSION, use plan-ceo-review 0E step 2's rule +(an explicit mode in the request wins), skip step 3, run +`~/.claude/skills/gstack/bin/gstack-ceo-mode-handoff "" --ceo-only --decisions ""` +and send its line verbatim. + +**Never applied:** a user challenge (keep the direction; list it below). +**Skipped and logged `skipped (ceo-only)`:** office-hours and BENEFITS_FROM +offers, 0E's question, plan archive cleanup, Next Steps chaining and CEO plan +promotion; the row satisfies its checklist entry. + +**State.** Write only in `$_AP`, ACTIVE_PLAN and `GSTACK_STATE_ROOT` +(the override; docs/state-root.md). + +**Finish.** After Phase 1 closes, write `$_AP/ceo-review.md`: `# CEO-only +review: `; a line with the mode, outside-voice status and "Not +approved"; then `## User challenges (direction kept)` (plan says, proposal, +why, cost if wrong), `## Scope added`, `## Conservative choices`, +`## Findings` (severity, section, fix, disposition) and `## Decision Audit +Trail` (copied), each "None." when empty. Send the CEO Completion Summary and +both paths, skip Completion's logs, approval and `/ship`, and end with: + +`GSTACK_RESULT: skill=autoplan status=reviewed run=` diff --git a/autoplan/sections/manifest.json b/autoplan/sections/manifest.json index 0afec4240..74071ed66 100644 --- a/autoplan/sections/manifest.json +++ b/autoplan/sections/manifest.json @@ -45,6 +45,12 @@ "file": "unattended.md", "title": "Unattended profile: parent-driven next/submit loop, gate answers, export, terminal line", "trigger": "the preamble echoed `SESSION_KIND: unattended` (read before Phase 1 and at the Phase 4 gate)" + }, + { + "id": "ceo-only", + "file": "ceo-only.md", + "title": "CEO-only run: zero questions, CEO phase only, review report, no gate", + "trigger": "the invocation names `--ceo-only` (read before any preamble instruction block)" } ] } diff --git a/bin/gstack-ceo-mode-handoff b/bin/gstack-ceo-mode-handoff index 393f4886a..ccb4c0a52 100755 --- a/bin/gstack-ceo-mode-handoff +++ b/bin/gstack-ceo-mode-handoff @@ -1,10 +1,11 @@ #!/usr/bin/env bash # gstack-ceo-mode-handoff — print /plan-ceo-review's Step 0E mode handoff line. # -# Usage: gstack-ceo-mode-handoff [--auto] [--decisions ""] +# Usage: gstack-ceo-mode-handoff [--auto | --ceo-only] [--decisions ""] # MODE: HOLD SCOPE | SCOPE EXPANSION | SELECTIVE EXPANSION | SCOPE REDUCTION # (underscores accepted for spaces) # --auto the mode came from a saved AUTO_DECIDE preference +# --ceo-only the mode came from 0E's rule in an /autoplan --ceo-only run # --decisions governing approved rows; defaults to "none" # # The handoff used to be chat-only, and the model often composed it in its @@ -15,10 +16,12 @@ set -euo pipefail mode="${1:-}" [ "$#" -gt 0 ] && shift auto=0 +ceo_only=0 decisions="none" while [ "$#" -gt 0 ]; do case "$1" in --auto) auto=1; shift ;; + --ceo-only) ceo_only=1; shift ;; --decisions) [ "$#" -ge 2 ] || { echo "gstack-ceo-mode-handoff: --decisions needs a value" >&2; exit 2; } decisions="$2"; shift 2 ;; @@ -31,11 +34,14 @@ case "$mode" in "HOLD SCOPE"|"SCOPE EXPANSION"|"SELECTIVE EXPANSION"|"SCOPE REDUCTION") ;; *) echo "gstack-ceo-mode-handoff: unknown mode: ${mode:-}" >&2; exit 2 ;; esac +[ "$auto$ceo_only" != 11 ] || { echo "gstack-ceo-mode-handoff: --auto and --ceo-only conflict" >&2; exit 2; } decisions="${decisions//$'\n'/ }" decisions="${decisions%.}" [ -n "${decisions// /}" ] || decisions="none" -if [ "$auto" = 1 ]; then +if [ "$ceo_only" = 1 ]; then + echo "Auto-selected review mode → $mode (CEO-only run: Step 0E's rule). Approved decisions: $decisions." +elif [ "$auto" = 1 ]; then echo "Auto-decided review mode → $mode (your preference). Change with /plan-tune. Approved decisions: $decisions." else echo "Mode: $mode; approved decisions: $decisions." diff --git a/changelog.d/gsta215-ceo-only-skill.md b/changelog.d/gsta215-ceo-only-skill.md new file mode 100644 index 000000000..f111cd8fc --- /dev/null +++ b/changelog.d/gsta215-ceo-only-skill.md @@ -0,0 +1,10 @@ +**`/autoplan --ceo-only` runs the CEO review by itself and asks nothing from start to finish (GST-537).** Garry asked for a CEO review that "just automatically does the right thing", and the planning eval needed a gstack CEO arm that a harness can run without a simulated user. Before, /autoplan always ran the Eng review too and stopped at a final approval question, and standalone /plan-ceo-review asks for the mode, offers to chain to the next skill and adds a Codex voice. + +### Added +- **`/autoplan --ceo-only`** (`autoplan/sections/ceo-only.md`): runs Phase 0, the outside-reviewer preflight and the CEO phase only. Every question takes its recommendation and gets an audit-trail row; consent and destructive options take the conservative choice. The review mode comes from plan-ceo-review's 0E rule. User challenges are never applied; each one is listed at the top of `/ceo-review.md`, with scope added, conservative choices, findings and the audit trail. The run ends with `GSTACK_RESULT: skill=autoplan status=reviewed run=` and approves nothing. +- **`--no-outside`** with `--ceo-only` skips the outside voice, so one model does the whole review. +- **`gstack-ceo-mode-handoff --ceo-only`** prints `Auto-selected review mode → (CEO-only run: Step 0E's rule)` instead of crediting a saved preference. +- **`status=reviewed`** joins the `GSTACK_RESULT` statuses (`docs/unattended.md`, "CEO-only runs"). + +### For contributors +- The paid case `autoplan-ceo-only` fails if the mode asks anything, skips a later phase's file, or leaves new files in HOME outside its folders. It takes 6 to 16 minutes, so it runs in the weekly marathon lane, not on autoplan PRs. The mode needs Claude Code for now. diff --git a/docs/unattended.md b/docs/unattended.md index 360589bad..bc300cb68 100644 --- a/docs/unattended.md +++ b/docs/unattended.md @@ -65,10 +65,12 @@ A run that writes a record ends with one line the parent greps: GSTACK_RESULT: skill= status= run= ``` -`status` is one of `complete`, `gate_pending`, `incomplete`, `interrupted` or -`refused`; `run` is where the run's record is. `/autoplan` +`status` is one of `complete`, `reviewed`, `gate_pending`, `incomplete`, +`interrupted` or `refused`; `run` is where the run's record is. `/autoplan` always ends `gate_pending` when it reaches the final gate, because the gate is -written and never approved. Pure queries (`gstack-artifact schema`, +written and never approved. `/autoplan --ceo-only` ends `reviewed`: it runs the +CEO phase alone, asks nothing, writes no gate and approves nothing (see +[CEO-only runs](#ceo-only-runs)). Pure queries (`gstack-artifact schema`, `gstack-gate render`) print no result line. Not every workflow writes this line or a run directory today: @@ -583,6 +585,40 @@ and `gstack-owner-brief render --captain` opens with "Nothing pending. Carried forward from gate_rev ", followed by any items taken under "do it" without being shown to the owner. +## CEO-only runs + +`/autoplan --ceo-only` runs the CEO review by itself and asks nothing. It runs +in the session that invokes the skill; `gstack-autoplan`, the driver a parent +agent uses, has no CEO-only mode yet, and `run.json` does not accept the +`reviewed` status. It needs Claude Code: other hosts inline autoplan's sections +and do not carry this one yet. It is for a harness or a crew that wants the strategy pass +without the full gauntlet, such as an eval comparing gstack's CEO review with a +plain prompt. + +- **What runs:** Phase 0, the outside-reviewer preflight and the CEO phase. + Design, DX, Eng and the final gate do not run. +- **How it decides:** every question takes its recommendation and is logged in + the Decision Audit Trail. The review mode comes from plan-ceo-review's 0E + rule (file count, then greenfield, added capability or fix), not autoplan's + fixed SELECTIVE EXPANSION, and the mode line says `CEO-only run`. Consent, + destructive and irreversible options take the conservative choice. +- **What it never does:** apply a user challenge (the plan's direction stays; + each challenge is listed), chain to another skill, promote the CEO plan, or + approve anything. +- **One model:** add `--no-outside` to skip the outside voice, so every + reviewer runs on the host model. +- **Where it writes:** the run directory (`.gstack/tmp/autoplan/` in the + repository), the reviewed plan, and CEO plans and task files under + `GSTACK_STATE_ROOT`. Point that variable inside a trial folder to keep every + write there. +- **What it leaves:** the reviewed plan (its original is kept at the restore + point), `/ceo-review.md` (user challenges, scope added, + conservative choices, findings and the audit trail), and the last line + `GSTACK_RESULT: skill=autoplan status=reviewed run=`. + +`test/skill-e2e-autoplan-ceo-only.test.ts` runs it on a plan that makes the +interactive CEO review ask a question and fails if this mode asks anything. + ## Owner brief `gstack-owner-brief render --out ` is what the owner sees at the final diff --git a/lib/headless-artifacts.ts b/lib/headless-artifacts.ts index 0162d36f2..c1c6e1e5b 100644 --- a/lib/headless-artifacts.ts +++ b/lib/headless-artifacts.ts @@ -23,7 +23,7 @@ export type DecisionKind = 'auto' | 'approval' | 'user_challenge'; export type DecisionStatus = 'pending' | 'approved' | 'overridden' | 'rejected'; export type TaskStatus = 'pending' | 'ready' | 'blocked_on_dependency' | 'blocked_on_decision' | 'in_progress' | 'done'; export type VoiceStatus = 'completed' | 'unavailable' | 'disabled' | 'skipped' | 'failed' | 'pending' | 'running'; -export type ResultStatus = 'complete' | 'gate_pending' | 'amend_pending' | 'incomplete' | 'interrupted' | 'refused'; +export type ResultStatus = 'complete' | 'reviewed' | 'gate_pending' | 'amend_pending' | 'incomplete' | 'interrupted' | 'refused'; export type RunStatus = ResultStatus | 'awaiting_result' | 'running'; export interface FindingRow { @@ -119,7 +119,7 @@ export interface ShipReceipt { base_red?: { cases: string[]; files: string[]; base: string; lane: string; expires_at: string }; } -export const RESULT_STATUSES: readonly ResultStatus[] = ['complete', 'gate_pending', 'amend_pending', 'incomplete', 'interrupted', 'refused']; +export const RESULT_STATUSES: readonly ResultStatus[] = ['complete', 'reviewed', 'gate_pending', 'amend_pending', 'incomplete', 'interrupted', 'refused']; const TERMINAL_RUN: readonly string[] = ['complete', 'gate_pending', 'amend_pending', 'refused', 'incomplete']; const TERMINAL_VOICE: readonly string[] = ['completed', 'unavailable', 'disabled', 'skipped', 'failed']; const RESOLVED: readonly string[] = ['accepted', 'partially_accepted', 'rejected', 'deferred', 'fixed']; diff --git a/test/autoplan-ceo-only.test.ts b/test/autoplan-ceo-only.test.ts new file mode 100644 index 000000000..c301089a1 --- /dev/null +++ b/test/autoplan-ceo-only.test.ts @@ -0,0 +1,123 @@ +import { afterAll, describe, expect, test } from 'bun:test'; +import { spawnSync } from 'node:child_process'; +import * as fs from 'node:fs'; +import * as os from 'node:os'; +import * as path from 'node:path'; +import { parseResult } from '../lib/headless-artifacts'; +import { CEO_REVIEW_HEADINGS, ceoOnlyEvidence, listFiles } from './helpers/autoplan-ceo-only-evidence'; + +const ROOT = path.resolve(import.meta.dir, '..'); +const read = (rel: string) => fs.readFileSync(path.join(ROOT, rel), 'utf8'); +const handoff = (...args: string[]) => spawnSync('bash', [path.join(ROOT, 'bin/gstack-ceo-mode-handoff'), ...args], { encoding: 'utf8', timeout: 10_000 }); + +describe('/autoplan --ceo-only skill text', () => { + const skill = read('autoplan/SKILL.md'); + const section = read('autoplan/sections/ceo-only.md'); + + test('the section index routes --ceo-only to its section before any preamble block', () => { + expect(skill).toContain('| the invocation names `--ceo-only` (read before any preamble instruction block) | `sections/ceo-only.md` |'); + }); + + test('the section forbids every question channel and names the skipped offers', () => { + for (const s of ['Never call AskUserQuestion (native or `mcp__*`) or\nExitPlanMode', 'never ask the user in prose', 'never wait for a reply', + 'office-hours and BENEFITS_FROM', "0E's question", 'Next Steps chaining and CEO plan\npromotion', '**Never applied:** a user challenge']) + expect(section).toContain(s); + }); + + test('the section runs CEO only, honors --no-outside and the state root, and ends status=reviewed', () => { + expect(section).toContain('Run Phases 0, 0.5 and 1 only'); + expect(section).toContain("`outside_status: disabled (--no-outside)`"); + expect(section).toContain('`GSTACK_STATE_ROOT`\n(the override'); + expect(section).toContain('gstack-ceo-mode-handoff "" --ceo-only'); + expect(section).toContain('`GSTACK_RESULT: skill=autoplan status=reviewed run=`'); + for (const h of CEO_REVIEW_HEADINGS) expect(section.replace(/\`\n\`/g, '` `').replace(/\n/g, ' ')).toContain(h.replace(/\n/g, ' ')); + }); +}); + +test('a normal /autoplan run never reads sections/ceo-only.md (the parity raise in carve-guards.ts rests on it)', () => { + const skill = read('autoplan/SKILL.md'); + expect(skill.split('ceo-only.md').length - 1).toBe(1); + expect(skill).toContain('| the invocation names `--ceo-only` (read before any preamble instruction block) | `sections/ceo-only.md` |'); + const others = fs.readdirSync(path.join(ROOT, 'autoplan/sections')).filter(f => f.endsWith('.md') && f !== 'ceo-only.md') + .map(f => `autoplan/sections/${f}`); + const runner = ['bin/gstack-autoplan', 'autoplan/DRIVER.md', ...fs.readdirSync(path.join(ROOT, 'lib')).filter(f => f.startsWith('autoplan-')).map(f => `lib/${f}`)]; + for (const rel of [...others, ...runner]) expect(read(rel), rel).not.toContain('ceo-only.md'); +}); + +describe('gstack-ceo-mode-handoff --ceo-only', () => { + test('names the CEO-only rule, not a saved preference', () => { + const r = handoff('SELECTIVE_EXPANSION', '--ceo-only'); + expect(r.status).toBe(0); + expect(r.stdout.trim()).toBe("Auto-selected review mode → SELECTIVE EXPANSION (CEO-only run: Step 0E's rule). Approved decisions: none."); + }); + test('refuses --auto with --ceo-only', () => { + const r = handoff('HOLD SCOPE', '--auto', '--ceo-only'); + expect(r.status).toBe(2); + expect(r.stderr).toContain('--auto and --ceo-only conflict'); + }); +}); + +test('GSTACK_RESULT parses status=reviewed', () => { + expect(parseResult('x\nGSTACK_RESULT: skill=autoplan status=reviewed run=/r')).toEqual({ skill: 'autoplan', status: 'reviewed', run: '/r' }); +}); + +describe('ceo-only evidence: a passing run and each negative control', () => { + const root = fs.mkdtempSync(path.join(os.tmpdir(), 'ceo-only-evidence-')); + afterAll(() => fs.rmSync(root, { recursive: true, force: true })); + const runDir = path.join(root, 'run'); + fs.mkdirSync(runDir); + const review = ['# CEO-only review: Pricing tier', '', 'Mode: HOLD SCOPE (ceo-only). Not approved.', '', + '## User challenges (direction kept)', 'None.', '## Scope added', 'None.', '## Conservative choices', 'None.', + '## Findings', '- High: the premise was never checked with developers; interview them first.', + '## Decision Audit Trail', '| # | Phase | Decision |', '|---|---|---|', '| 1 | CEO | Mode HOLD SCOPE |', ''].join('\n'); + fs.writeFileSync(path.join(runDir, 'ceo-review.md'), review); + const assistant = (...content: any[]) => ({ type: 'assistant', message: { content } }); + const user = (...content: any[]) => ({ type: 'user', message: { content } }); + const good = [ + assistant({ type: 'tool_use', id: 't1', name: 'Bash', input: { command: '~/.claude/skills/gstack/bin/gstack-ceo-mode-handoff "HOLD SCOPE" --ceo-only --decisions "none"' } }), + user({ type: 'tool_result', tool_use_id: 't1', content: "Auto-selected review mode → HOLD SCOPE (CEO-only run: Step 0E's rule). Approved decisions: none." }), + assistant({ type: 'tool_use', id: 't2', name: 'Agent', input: { prompt: 'native CEO review' } }), + assistant({ type: 'text', text: 'Phases 2, 2.5, 3 and 4 skipped (CEO-only run).' }), + ]; + const output = `Review done.\nGSTACK_RESULT: skill=autoplan status=reviewed run=${runDir}`; + const base = { transcript: good, output, strayWrites: [] as string[], mustMention: [/developer/i, /interview|talk|validat/i] }; + + test('a complete run passes', () => { + const e = ceoOnlyEvidence(base); + expect(e.reasons).toEqual([]); + expect(e.runDir).toBe(runDir); + }); + const fails = (name: string, patch: Partial, reason: RegExp) => test(name, () => { + const e = ceoOnlyEvidence({ ...base, ...patch }); + expect(e.passed).toBe(false); + expect(e.reasons.join('\n')).toMatch(reason); + }); + fails('an AskUserQuestion call fails', { transcript: [...good, assistant({ type: 'tool_use', id: 'q', name: 'AskUserQuestion', input: {} })] }, /asked 1 question/); + fails('an MCP AskUserQuestion variant fails', { transcript: [...good, assistant({ type: 'tool_use', id: 'q', name: 'mcp__conductor__AskUserQuestion', input: {} })] }, /asked 1 question/); + fails('ExitPlanMode fails', { transcript: [...good, assistant({ type: 'tool_use', id: 'q', name: 'ExitPlanMode', input: {} })] }, /tool ExitPlanMode/); + fails('a prose question fails', { output: `Which mode would you like me to use?\n${output}` }, /prose: Which mode would you like/); + fails('a wrong terminal status fails', { output: output.replace('status=reviewed', 'status=gate_pending') }, /status=gate_pending, expected reviewed/); + fails('a missing terminal line fails', { output: 'Review done.' }, /no GSTACK_RESULT line/); + fails('headings without a review fail', { transcript: [assistant({ type: 'text', text: 'Done.' })] }, /no gstack-ceo-mode-handoff[\s\S]*no CEO reviewer subagent/); + fails('loading a skipped phase fails', { transcript: [...good, assistant({ type: 'tool_use', id: 'r', name: 'Read', input: { file_path: '/h/.claude/skills/gstack/autoplan/sections/eng-phase.md' } })] }, /loaded a skipped phase/); + fails('loading a skipped phase through Bash fails', { transcript: [...good, assistant({ type: 'tool_use', id: 'b', name: 'Bash', input: { command: 'cat /h/.claude/skills/gstack/autoplan/sections/dx-phase.md' } })] }, /loaded a skipped phase: cat /); + fails('a write outside its roots fails', { strayWrites: ['.config/x.json'] }, /wrote 1 file\(s\) outside its roots, under \.config\/x\.json/); + fails('a review that misses the seeded flaw fails', { mustMention: [/churn/i] }, /never names the seeded flaw/); + test('a missing ceo-review.md fails', () => { + const empty = path.join(root, 'empty'); + fs.mkdirSync(empty); + const e = ceoOnlyEvidence({ ...base, output: `GSTACK_RESULT: skill=autoplan status=reviewed run=${empty}` }); + expect(e.reasons.join('\n')).toContain('ceo-review.md is missing'); + }); + test('listFiles skips the declared roots', () => { + const home = path.join(root, 'home'); + fs.mkdirSync(path.join(home, 'state'), { recursive: true }); + fs.writeFileSync(path.join(home, 'state', 'a'), ''); + fs.writeFileSync(path.join(home, 'b'), ''); + fs.mkdirSync(path.join(home, '.bun', 'install', 'cache'), { recursive: true }); + fs.writeFileSync(path.join(home, '.bun', 'install', 'cache', 'c'), ''); + fs.mkdirSync(path.join(home, '.bun', 'bin'), { recursive: true }); + fs.writeFileSync(path.join(home, '.bun', 'bin', 'd'), ''); + expect(listFiles(home, ['state', '.bun/install/cache'])).toEqual(['.bun/bin/d', 'b']); + }); +}); diff --git a/test/eng-finding-retry-budget.test.ts b/test/eng-finding-retry-budget.test.ts index d547a0b8e..1b0039bf4 100644 --- a/test/eng-finding-retry-budget.test.ts +++ b/test/eng-finding-retry-budget.test.ts @@ -141,7 +141,7 @@ test('live periodic census fits the declared CI wall including setup', () => { test('registered allocation is deterministic and preserves every discovered file', () => { const files = collectPaidTestFiles(); - expect(files).toHaveLength(114); // + skill-e2e-planning-wave (GSTA-228); + skill-e2e-model-panel (GSTA-173); + skill-e2e-write-skill (GSTA-133); + skill-e2e-ship-frozen-scope (frozen review scope, GSTA-113 PR 2); + skill-e2e-failure-routing (#3157/#3158); + skill-e2e-dx-error-contract (#3226); + skill-e2e-gate-integrity (HONEST_WORK_GATE_INTEGRITY W8); + skill-e2e-autoplan-schema-canary, -guard-pty and -long-session (Oct 7 wave, CEO-13, ENG-18, CEO-15); + skill-e2e-ship-measure-seeded-flake (v1.91.32.0); W5a: 15 carve wrappers became one case-sharded file; + codex-e2e-multiblock-live and skill-e2e-design-model-smoke (v1.91.30.0); + expect(files).toHaveLength(115); // + skill-e2e-autoplan-ceo-only (GST-537); + skill-e2e-planning-wave (GSTA-228); + skill-e2e-model-panel (GSTA-173); + skill-e2e-write-skill (GSTA-133); + skill-e2e-ship-frozen-scope (frozen review scope, GSTA-113 PR 2); + skill-e2e-failure-routing (#3157/#3158); + skill-e2e-dx-error-contract (#3226); + skill-e2e-gate-integrity (HONEST_WORK_GATE_INTEGRITY W8); + skill-e2e-autoplan-schema-canary, -guard-pty and -long-session (Oct 7 wave, CEO-13, ENG-18, CEO-15); + skill-e2e-ship-measure-seeded-flake (v1.91.32.0); W5a: 15 carve wrappers became one case-sharded file; + codex-e2e-multiblock-live and skill-e2e-design-model-smoke (v1.91.30.0); expect(files).toContain('test/skill-e2e-ship-skip.test.ts'); const m = livePlan(files); expect(livePlan([...files].reverse())).toEqual(m); diff --git a/test/helpers/autoplan-ceo-only-evidence.ts b/test/helpers/autoplan-ceo-only-evidence.ts new file mode 100644 index 000000000..224cbac22 --- /dev/null +++ b/test/helpers/autoplan-ceo-only-evidence.ts @@ -0,0 +1,96 @@ +/** + * Evidence for the /autoplan --ceo-only paid case (test/skill-e2e-autoplan-ceo-only.test.ts): + * the run asked nothing, reviewed the plan through the CEO phase, skipped every + * later phase, wrote ceo-review.md, ended status=reviewed, and wrote nothing in + * HOME outside its declared roots. Pure over the transcript and files the test + * collects, so test/autoplan-ceo-only.test.ts proves each check fails on a + * synthetic negative control before any paid run. + */ +import * as fs from 'node:fs'; +import * as path from 'node:path'; + +export const CEO_REVIEW_HEADINGS = [ + '## User challenges (direction kept)', '## Scope added', '## Conservative choices', '## Findings', '## Decision Audit Trail', +] as const; + +const QUESTION_TOOL = /(^|__)(AskUserQuestion|ExitPlanMode)$/; +const PROSE_QUESTION = /\b(would you like|do you want|should i\b|want me to|shall i\b|which (?:option|mode|one) (?:do|would|should) you|reply with|let me know (?:if|which|whether))/i; +const LATER_PHASE = /autoplan\/sections\/(?:design-phase|dx-phase|eng-phase|tasks-aggregator)\.md/; +const DISPATCH_TOOL = /^(Agent|Task)$/; + +export interface CeoOnlyEvidenceInput { + transcript: any[]; + /** The session's final result text. */ + output: string; + /** New files under HOME outside the declared roots (the test's before/after diff). */ + strayWrites: string[]; + /** Patterns the reviewed plan or ceo-review.md must contain for the seeded flaw. */ + mustMention: RegExp[]; +} +export interface CeoOnlyEvidence { passed: boolean; reasons: string[]; runDir?: string; questions: string[] } + +function blocks(transcript: any[]): Array<{ role: string; block: any }> { + return transcript.flatMap(event => { + const role = event?.type === 'assistant' ? 'assistant' : event?.type === 'user' ? 'user' : ''; + const content = event?.message?.content; + return role && Array.isArray(content) ? content.map((block: any) => ({ role, block })) : []; + }); +} + +export function ceoOnlyEvidence(input: CeoOnlyEvidenceInput): CeoOnlyEvidence { + const reasons: string[] = []; + const all = blocks(input.transcript); + const uses = all.filter(b => b.role === 'assistant' && b.block?.type === 'tool_use').map(b => b.block); + const texts = all.filter(b => b.role === 'assistant' && b.block?.type === 'text').map(b => String(b.block.text ?? '')); + const results = all.filter(b => b.role === 'user' && b.block?.type === 'tool_result') + .map(b => (typeof b.block.content === 'string' ? b.block.content : JSON.stringify(b.block.content ?? ''))); + + const questions = [ + ...uses.filter(u => QUESTION_TOOL.test(String(u.name))).map(u => `tool ${u.name}`), + ...[...texts, input.output].flatMap(t => t.split('\n').filter(line => line.trim().endsWith('?') && PROSE_QUESTION.test(line)).map(line => `prose: ${line.trim().slice(0, 160)}`)), + ]; + if (questions.length) reasons.push(`asked ${questions.length} question(s): ${questions.join(' | ')}`); + + const handoff = uses.some(u => u.name === 'Bash' && /gstack-ceo-mode-handoff\b/.test(String(u.input?.command)) && /--ceo-only\b/.test(String(u.input?.command))) + && results.some(r => r.includes('(CEO-only run: Step 0E')); + if (!handoff) reasons.push('no gstack-ceo-mode-handoff --ceo-only call with its CEO-only line'); + if (!uses.some(u => DISPATCH_TOOL.test(String(u.name)))) reasons.push('no CEO reviewer subagent was dispatched'); + const later = uses.map(u => String(u.name === 'Bash' ? u.input?.command : (u.input?.file_path ?? u.input?.path ?? u.input?.pattern ?? ''))) + .filter(arg => LATER_PHASE.test(arg)); + if (later.length) reasons.push(`loaded a skipped phase: ${later.map(arg => arg.slice(0, 160)).join(', ')}`); + + const result = /^GSTACK_RESULT: skill=autoplan status=(\S+) run=(.+)$/m.exec(input.output.split('\n').filter(l => l.startsWith('GSTACK_RESULT: ')).pop() ?? ''); + if (!result) reasons.push('no GSTACK_RESULT line for autoplan in the final output'); + else if (result[1] !== 'reviewed') reasons.push(`GSTACK_RESULT status=${result[1]}, expected reviewed`); + const runDir = result?.[2]?.trim(); + const reviewFile = runDir ? path.join(runDir, 'ceo-review.md') : ''; + const review = reviewFile && fs.existsSync(reviewFile) ? fs.readFileSync(reviewFile, 'utf8') : undefined; + if (runDir && review === undefined) reasons.push(`${reviewFile} is missing`); + if (review !== undefined) { + const missing = CEO_REVIEW_HEADINGS.filter(h => !review.includes(h)); + if (missing.length) reasons.push(`ceo-review.md lacks ${missing.join(', ')}`); + if (!/\|\s*\d+\s*\|/.test(review.slice(review.indexOf('## Decision Audit Trail')))) reasons.push('ceo-review.md has no Decision Audit Trail rows'); + const unmet = input.mustMention.filter(re => !re.test(review)); + if (unmet.length) reasons.push(`ceo-review.md never names the seeded flaw (${unmet.map(String).join(', ')})`); + } + if (input.strayWrites.length) { + const dirs = [...new Set(input.strayWrites.map(f => f.split('/').slice(0, 2).join('/')))]; + reasons.push(`wrote ${input.strayWrites.length} file(s) outside its roots, under ${dirs.join(', ')}: ${input.strayWrites.slice(0, 5).join(', ')}`); + } + return { passed: reasons.length === 0, reasons, runDir, questions }; +} + +/** Every file under root, relative, skipping the given relative prefixes. */ +export function listFiles(root: string, skip: string[] = []): string[] { + const out: string[] = []; + const walk = (dir: string) => { + for (const entry of fs.readdirSync(dir, { withFileTypes: true })) { + const abs = path.join(dir, entry.name); + const rel = path.relative(root, abs); + if (skip.some(s => rel === s || rel.startsWith(s + path.sep))) continue; + if (entry.isDirectory()) walk(abs); else out.push(rel); + } + }; + if (fs.existsSync(root)) walk(root); + return out.sort(); +} diff --git a/test/helpers/carve-guards.ts b/test/helpers/carve-guards.ts index cc4ee6140..bd0b464b2 100644 --- a/test/helpers/carve-guards.ts +++ b/test/helpers/carve-guards.ts @@ -583,7 +583,7 @@ do not launch the downstream skill or open a browser.`, maxSkeletonBytes: 48_175, // GSTA-96 hoist (2026-10-26): lowered from 70_000 by the shared preamble that moved to sections/preamble.md, keeping the pre-hoist headroom of 4,888 bytes; measured 43,287. Before: Phase-specific outside coverage, native fallback, and harness guard. minUnionBytes: 85_000, // measured union 86,926 mustContain: ['6 Decision Principles', 'TASTE DECISION', 'USER CHALLENGE', 'consensus', 'Restore Point'], - maxSizeRatio: 1.1511, // Four validated outside invocations replace raw CLI calls; phases keep independent coverage. + Oct 7 wave B3: the Codex probe runs as a command (select-model capture lines, check-sandbox/probe-model/run-with-timeout subcommands) instead of being sourced; measured 114,297 / 101,979 = 1.1208 (2026-10-07). + Oct 7 wave CEO-28/DX-11: review prompts ask for a severity per finding and design proposals use the proposal gate; measured 114,533 / 101,979 = 1.1231 (2026-10-07). + Oct 7 wave guard (CEO-20): conditional `run_in_background` request plus the background-run recovery at each foreground dispatch site, and phase-close publishing each report with the `true autoplan-published ` no-op before the next phase entry in a later message (CEO-1); measured 114,320 / 101,979 = 1.1210 (2026-10-07). Merged with the guard area: measured 115,149 / 101,979 = 1.1291 (2026-10-07). + Oct 8 wave (permission cards after a background reviewer): Step 1 puts RESTORE_PATH, and so every phase artifact, in the project's git-excluded `.gstack/tmp/autoplan/`, which Claude Code reads without a card after the skill turn ends; measured 115,270 / 101,979 = 1.1303 (2026-10-08). + fix wave PR D: the prerequisite offer is skipped in spawned/headless sessions (#1958); measured 115,267 / 101,979 = 1.1303 (2026-10-07). Both merged: measured 115,126 / 101,979 = 1.1289 (2026-10-08). + Honest Work wave merged (W2/W3 gate integrity in review/ship, W4 kernel + DONE line in every tier-2+ skill, W5/W7 in spec/ship): re-measured on the integrated tree (2026-10-10). + Oct 10 multi-agent wave B9/B10/B12: Phase 0 ESTIMATE and the phase-close timing record, the final gate as a gate list (`gstack-gate render/parse`, stable ids, gate_rev) and the URGENT block above the summary, offset by tightening the Phase 4 prose and the tasks-section; measured 114,980 / 101,979 = 1.1275 (2026-10-10). + Oct 10 multi-agent wave PR D1 (reality rows): the reality-rows check line in the CEO and Eng phase files, the Pre-Gate rule (unattended miss is `incomplete`, never a warning) and the two Pre-Gate cells; measured 117,042 / 101,979 = 1.1477 (2026-10-10). + GSTA-96 shared-preamble hoist (owner-approved raise, Garry 2026-10-25: "do it"): the union now holds sections/preamble.md, so a fresh invocation reads the pointer plus the shared file; measured 117,388 / 101,979 = 1.1511 (2026-10-25). Before: 1.149. + maxSizeRatio: 1.1723, // GST-537 /autoplan --ceo-only (GStack Minder approved A1, 2026-10-31): sections/ceo-only.md (2,168 B) plus its Section index row, read only when the invocation names --ceo-only (test/autoplan-ceo-only.test.ts pins that a normal run never reads it); measured 119,545 / 101,979 = 1.1723 (2026-10-31), +2,276 B over main's 117,269 = 1.1499. Before: 1.1511. Four validated outside invocations replace raw CLI calls; phases keep independent coverage. + Oct 7 wave B3: the Codex probe runs as a command (select-model capture lines, check-sandbox/probe-model/run-with-timeout subcommands) instead of being sourced; measured 114,297 / 101,979 = 1.1208 (2026-10-07). + Oct 7 wave CEO-28/DX-11: review prompts ask for a severity per finding and design proposals use the proposal gate; measured 114,533 / 101,979 = 1.1231 (2026-10-07). + Oct 7 wave guard (CEO-20): conditional `run_in_background` request plus the background-run recovery at each foreground dispatch site, and phase-close publishing each report with the `true autoplan-published ` no-op before the next phase entry in a later message (CEO-1); measured 114,320 / 101,979 = 1.1210 (2026-10-07). Merged with the guard area: measured 115,149 / 101,979 = 1.1291 (2026-10-07). + Oct 8 wave (permission cards after a background reviewer): Step 1 puts RESTORE_PATH, and so every phase artifact, in the project's git-excluded `.gstack/tmp/autoplan/`, which Claude Code reads without a card after the skill turn ends; measured 115,270 / 101,979 = 1.1303 (2026-10-08). + fix wave PR D: the prerequisite offer is skipped in spawned/headless sessions (#1958); measured 115,267 / 101,979 = 1.1303 (2026-10-07). Both merged: measured 115,126 / 101,979 = 1.1289 (2026-10-08). + Honest Work wave merged (W2/W3 gate integrity in review/ship, W4 kernel + DONE line in every tier-2+ skill, W5/W7 in spec/ship): re-measured on the integrated tree (2026-10-10). + Oct 10 multi-agent wave B9/B10/B12: Phase 0 ESTIMATE and the phase-close timing record, the final gate as a gate list (`gstack-gate render/parse`, stable ids, gate_rev) and the URGENT block above the summary, offset by tightening the Phase 4 prose and the tasks-section; measured 114,980 / 101,979 = 1.1275 (2026-10-10). + Oct 10 multi-agent wave PR D1 (reality rows): the reality-rows check line in the CEO and Eng phase files, the Pre-Gate rule (unattended miss is `incomplete`, never a warning) and the two Pre-Gate cells; measured 117,042 / 101,979 = 1.1477 (2026-10-10). + GSTA-96 shared-preamble hoist (owner-approved raise, Garry 2026-10-25: "do it"): the union now holds sections/preamble.md, so a fresh invocation reads the pointer plus the shared file; measured 117,388 / 101,979 = 1.1511 (2026-10-25). Before: 1.149. }, spec: { skill: 'spec', diff --git a/test/helpers/touchfiles-data.ts b/test/helpers/touchfiles-data.ts index 7e3982353..ca9275607 100644 --- a/test/helpers/touchfiles-data.ts +++ b/test/helpers/touchfiles-data.ts @@ -918,6 +918,12 @@ export const E2E_TOUCHFILES: Record = { 'test/fixtures/autoplan-phase-handoff-6714.json','scripts/resolvers/learnings.ts', 'scripts/resolvers/preamble/generate-completion-status.ts', 'scripts/resolvers/preamble/generate-preamble-bash.ts', 'test/fixtures/plan-scope-recovery-av.json', 'test/fixtures/design-scope-checkpoint-at.json', 'scripts/resolvers/composition.ts', 'autoplan/**', 'lib/claude-journal-records.ts', 'lib/claude-owned-journal.ts', 'codex/**', 'bin/gstack-codex-probe', 'scripts/resolvers/review-dashboard.ts', 'scripts/resolvers/plan-gates.ts', 'scripts/resolvers/spec-review.ts', 'scripts/resolvers/outside-voice-steps.ts', 'scripts/resolvers/review-scope.ts', 'scripts/resolvers/outside-voice.ts', 'scripts/resolvers/design.ts', 'test/skill-e2e-autoplan-dual-voice.test.ts', 'bin/gstack-autoplan-snapshot.ts', 'test/fixtures/autoplan/t-ceo-omitted-obligations.json', 'test/fixtures/autoplan/u-ceo-original-loss.json', 'test/fixtures/autoplan/v-ceo-dangling-references.json', 'scripts/resolvers/design-doc-discovery.ts', 'bin/gstack-design-doc-find', 'bin/gstack-ceo-plan-find', 'plan-ceo-review/**', 'ETHOS.md', 'plan-eng-review/**', 'plan-design-review/**', 'plan-devex-review/**', 'scripts/resolvers/testing.ts', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/tasks-section.ts', 'test/helpers/autoplan-phase-observer.ts', 'scripts/resolve-codex-generation-model.ts', 'scripts/resolvers/constants.ts'], + // /autoplan --ceo-only: zero questions through the CEO phase (GST-537) + 'autoplan-ceo-only': ['autoplan/**', 'plan-ceo-review/**', 'ETHOS.md', 'scripts/resolvers/preamble.ts', 'scripts/resolvers/preamble-hoist.ts', + 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'bin/gstack-state-root.sh', 'lib/state-root.ts', + 'lib/headless-artifacts.ts', 'bin/gstack-ceo-mode-handoff', 'test/helpers/autoplan-ceo-only-evidence.ts', 'test/helpers/hermetic-env.ts', + 'test/skill-e2e-autoplan-ceo-only.test.ts', 'test/fixtures/forcing-finding-seeds.ts'], + // Multi-provider benchmark adapters — live API smoke against real claude/codex/gemini CLIs 'benchmark-providers-live': ['bin/gstack-model-benchmark', 'test/helpers/providers/**', 'test/helpers/benchmark-runner.ts', 'test/helpers/pricing.ts', 'test/skill-e2e-benchmark-providers.test.ts'], @@ -1581,6 +1587,7 @@ export const E2E_TIERS: Record = { // Autoplan — periodic (not yet implemented) 'autoplan-dual-voice': 'periodic', + 'autoplan-ceo-only': 'marathon', // Full /autoplan --ceo-only CEO review (349–984 s over 6 runs, GST-537); the gate (I5) and periodic lanes are at their caps // Multi-provider benchmark — periodic (requires external CLIs + auth, paid) 'benchmark-providers-live': 'periodic', @@ -1995,6 +2002,7 @@ export const E2E_KINDS: Record = { 'benchmark-workflow': 'rule', 'setup-deploy-workflow': 'rule', 'autoplan-dual-voice': 'rule', + 'autoplan-ceo-only': 'rule', 'benchmark-providers-live': 'rule', 'scrape-match-path': 'behavior', 'scrape-prototype-path': 'behavior', diff --git a/test/skill-e2e-autoplan-ceo-only.test.ts b/test/skill-e2e-autoplan-ceo-only.test.ts new file mode 100644 index 000000000..f0d251529 --- /dev/null +++ b/test/skill-e2e-autoplan-ceo-only.test.ts @@ -0,0 +1,108 @@ +import { describe, test, expect, beforeEach, afterEach, afterAll } from 'bun:test'; +import { spawnSync } from 'child_process'; +import * as fs from 'fs'; +import * as os from 'os'; +import * as path from 'path'; +import { ceoOnlyEvidence, listFiles } from './helpers/autoplan-ceo-only-evidence'; +import { PTY_LONG_MS } from './helpers/eval-budgets'; +import { runSkillTest } from './helpers/session-runner'; +import { buildSeedConfig, seedHermeticGstackHome, seedHermeticRuntimeView } from './helpers/hermetic-env'; +import { FORCING_FLOOR_CEO } from './fixtures/forcing-finding-seeds'; +import { + ROOT, runId, evalsEnabled, describeIfSelected, logCost, recordE2E, copyDirSync, createEvalCollector, finalizeEvalCollector, +} from './helpers/e2e-helpers'; + +// /autoplan --ceo-only must finish a CEO review without asking anything (GST-537). +// The seed is the plan that makes interactive /plan-ceo-review ask a question +// (test/skill-e2e-plan-ceo-finding-floor.test.ts is the control on the same seed). +// Every check is a pure function over the transcript and files, +// proven against negative controls in test/autoplan-ceo-only.test.ts. + +const evalCollector = createEvalCollector('e2e-autoplan-ceo-only'); +// Declared roots in the isolated HOME: the state root, the host's own config and temp, and +// Bun's package cache (the runtime writes it under HOME on first use; run gsta215-e2e-2). +const OWNED_HOME_ROOTS = ['gstack-home', '.claude', 'tmp', '.bun/install/cache', '.capy/git-shim.log']; + +describeIfSelected('Autoplan CEO-only E2E', ['autoplan-ceo-only'], () => { + let workDir = ''; + let stateDir = ''; + let env: Record = {}; + let homeBefore: string[] = []; + + const cleanup = () => { + for (const dir of [workDir, stateDir]) if (dir) fs.rmSync(dir, { recursive: true, force: true }); + }; + + beforeEach(() => { + try { + workDir = fs.mkdtempSync(path.join(os.tmpdir(), 'skill-e2e-autoplan-ceo-only-')); + stateDir = fs.mkdtempSync(path.join(os.tmpdir(), 'gstack-autoplan-ceo-only-state-')); + const configDir = path.join(stateDir, '.claude'); + const gstackHome = path.join(stateDir, 'gstack-home'); + const tempDir = path.join(stateDir, 'tmp'); + fs.mkdirSync(configDir); + fs.mkdirSync(gstackHome); + fs.mkdirSync(tempDir, { mode: 0o700 }); + fs.writeFileSync(path.join(configDir, '.claude.json'), JSON.stringify(buildSeedConfig({ + apiKey: process.env.ANTHROPIC_API_KEY ?? process.env.GSTACK_ANTHROPIC_API_KEY, trustedDirs: [workDir], + })), { mode: 0o600 }); + seedHermeticGstackHome(gstackHome); + fs.mkdirSync(path.join(configDir, 'skills')); + seedHermeticRuntimeView(ROOT, path.join(configDir, 'skills', 'gstack')); + env = { HOME: stateDir, CLAUDE_CONFIG_DIR: configDir, GSTACK_HOME: gstackHome, GSTACK_STATE_ROOT: gstackHome, + TMPDIR: tempDir, TEMP: tempDir, TMP: tempDir }; + + const git = (...args: string[]) => { + const r = spawnSync('git', args, { cwd: workDir, encoding: 'utf8', timeout: 10_000 }); + if (r.error || r.status !== 0) throw new Error(`CEO-only fixture setup failed: ${r.error?.message ?? r.stderr}`); + }; + git('init', '-b', 'main'); + git('config', 'user.email', 'test@test.com'); + git('config', 'user.name', 'Test'); + // The seed's first line asks for a /tmp plan copy; the plan itself is the rest. + fs.writeFileSync(path.join(workDir, 'PLAN.md'), FORCING_FLOOR_CEO.split('\n').slice(2).join('\n') + '\n'); + fs.writeFileSync(path.join(workDir, 'README.md'), '# Acme API\n\nA paid HTTP API with one pricing tier.\n'); + git('add', '.'); + git('-c', 'commit.gpgsign=false', 'commit', '--no-verify', '-m', 'Plan'); + for (const skill of ['autoplan', 'plan-ceo-review']) { + const dest = path.join(workDir, '.claude', 'skills', skill); + fs.mkdirSync(dest, { recursive: true }); + fs.copyFileSync(path.join(ROOT, skill, 'SKILL.md'), path.join(dest, 'SKILL.md')); + copyDirSync(path.join(ROOT, skill, 'sections'), path.join(dest, 'sections')); + } + homeBefore = listFiles(stateDir, OWNED_HOME_ROOTS); + } catch (error) { + cleanup(); + throw error; + } + }); + + afterEach(cleanup); + afterAll(() => finalizeEvalCollector(evalCollector)); + + // Titled by its E2E id so scripts/lib/paid-cases.ts needs no CASE_TEST_NAMES entry. + test.skipIf(!evalsEnabled)('autoplan-ceo-only', async () => { + const result = await runSkillTest({ + testName: 'autoplan-ceo-only', + workingDirectory: workDir, + env, + prompt: '/autoplan --ceo-only --no-outside\n\nReview the plan in PLAN.md.', + timeout: PTY_LONG_MS, + allowedTools: ['Bash', 'Read', 'Write', 'Edit', 'Grep', 'Glob', 'Agent', 'Skill'], + maxTurns: 200, + runId, + }); + const before = new Set(homeBefore); + const evidence = ceoOnlyEvidence({ + transcript: Array.isArray(result.transcript) ? result.transcript : [], + output: result.output, + strayWrites: listFiles(stateDir, OWNED_HOME_ROOTS).filter(f => !before.has(f)), + mustMention: [/developer/i, /talk|interview|validat|evidence|research/i], + }); + logCost('autoplan-ceo-only', result); + recordE2E(evalCollector, 'autoplan-ceo-only', 'Autoplan CEO-only E2E', result, { + passed: evidence.passed, ...(evidence.reasons.length ? { error: evidence.reasons.join('; ') } : {}), + }); + expect(evidence.reasons).toEqual([]); + }, PTY_LONG_MS + 60_000); +}); diff --git a/test/touchfiles.test.ts b/test/touchfiles.test.ts index 2b4751774..7bd042652 100644 --- a/test/touchfiles.test.ts +++ b/test/touchfiles.test.ts @@ -310,8 +310,10 @@ describe('selectTests', () => { // The formerly keyless periodic AUQ probes drive the CEO skill too. expect(result.selected).toContain('auq-consistency'); expect(result.selected).toContain('auq-verbose-vs-carved-ab'); - expect(result.selected.length).toBe(21); - expect(result.skipped.length).toBe(Object.keys(E2E_TOUCHFILES).length - 21); + // /autoplan --ceo-only runs plan-ceo-review's Phase 1 and its 0E mode rule. + expect(result.selected).toContain('autoplan-ceo-only'); + expect(result.selected.length).toBe(22); + expect(result.skipped.length).toBe(Object.keys(E2E_TOUCHFILES).length - 22); }); test('global touchfile triggers ALL tests', () => {