diff --git a/.github/workflows/evals.yml b/.github/workflows/evals.yml index 897e90ca4..2d7314d2f 100644 --- a/.github/workflows/evals.yml +++ b/.github/workflows/evals.yml @@ -159,6 +159,12 @@ jobs: - name: e2e-ship-docsync file: test/skill-e2e-ship-docsync.test.ts tier: gate + # #2733 behavioral proof: the document-release JSON contract survives + # a firing AUQ gate inside a spawned-marked subagent. Whole-file + # tier-gated like ship-docsync, so the row carries tier: gate. + - name: e2e-docsync-spawned + file: test/skill-e2e-docsync-spawned.test.ts + tier: gate # Consent-gate guardrail for the Third-Party Web Actions contract # (Aside recommended-driver rewrite): hermetic shims, deterministic # grep assertions, gate tier in E2E_TIERS. diff --git a/test/helpers/touchfiles-data.ts b/test/helpers/touchfiles-data.ts index b3f714f19..2a87ed8e5 100644 --- a/test/helpers/touchfiles-data.ts +++ b/test/helpers/touchfiles-data.ts @@ -293,6 +293,21 @@ export const E2E_TOUCHFILES: Record = { 'plan-eng-coverage-audit': ['plan-eng-review/**', 'test/fixtures/coverage-audit-fixture.ts', 'test/skill-e2e-coverage-audit.test.ts'], 'ship-triage': ['ship/**', 'bin/gstack-repo-mode', 'test/skill-e2e-triage.test.ts'], 'ship-docsync': ['ship/**', 'document-release/**', 'scripts/gen-skill-docs.ts', 'scripts/resolvers/sections.ts', 'test/skill-e2e-ship-docsync.test.ts'], + // #2733 behavioral proof: the JSON contract survives a firing gate inside a + // spawned-marked subagent. Deps name every behavior under test — the + // session-kind override, the skill-start gates, both hooks + the shared + // directive, and the AUQ prose rule — so changing any of them selects it. + 'docsync-spawned': [ + 'document-release/**', + 'ship/sections/pr-body.md', + 'bin/gstack-session-kind', + 'bin/gstack-skill-start', + 'hosts/claude/hooks/question-preference-hook.ts', + 'hosts/claude/hooks/auq-error-fallback-hook.ts', + 'hosts/claude/hooks/spawned-directive.ts', + 'scripts/resolvers/preamble/generate-ask-user-format.ts', + 'test/skill-e2e-docsync-spawned.test.ts', + ], // Plan completion audit + verification 'ship-plan-completion': ['ship/**', 'scripts/gen-skill-docs.ts'], @@ -702,6 +717,7 @@ export const E2E_TIERS: Record = { 'ship-coverage-audit': 'gate', 'ship-triage': 'gate', 'ship-docsync': 'gate', + 'docsync-spawned': 'gate', // #2733 JSON-contract-through-a-firing-gate proof (deterministic safety) 'ship-plan-completion': 'gate', 'ship-plan-verification': 'gate', diff --git a/test/skill-e2e-docsync-spawned.test.ts b/test/skill-e2e-docsync-spawned.test.ts new file mode 100644 index 000000000..b8371fbcb --- /dev/null +++ b/test/skill-e2e-docsync-spawned.test.ts @@ -0,0 +1,283 @@ +/** + * Spawned document-release subagent E2E — the behavioral proof of #2733's + * JSON contract THROUGH A FIRING GATE. The existing coverage misses exactly + * this: skill-e2e-ship-docsync.test.ts stubs document-release (no preamble, + * no gates) and asserts only the dispatch; skill-e2e-workflow.test.ts runs + * the real skill but suppresses the gates by prompt ("do NOT use + * AskUserQuestion"). #2733 shipped through that hole — the subagent + * prose-STOPped at the VERSION gate in every Conductor-hosted ship and the + * parent's LAST-line JSON parse failed. + * + * This test plays the PARENT: it drives a claude -p run with the verbatim + * Step 18 dispatch prompt extracted from the live ship/sections/pr-body.md + * (drift-proof — a reworded prompt is exercised, not a copy), against a REAL + * preamble-bearing document-release slice, in a Conductor-ambient env, with + * the AUQ hooks seeded live. The VERSION-bump gate fires (VERSION is NOT + * bumped on the fixture branch); the marked-spawned machinery must resolve + * it to the recommended option (C — Skip) and the run must end with the + * parseable JSON contract, `decisions` non-empty, VERSION untouched. + * + * Fixture layout (fake HOME, same pattern as ship-docsync): + * + * / (passed as env HOME) + * ├── repo/ git fixture: feature branch, committed change, + * │ VERSION deliberately NOT bumped → Step 8 fires + * ├── gstack-home/ hermetic GSTACK_HOME (update_check: false) + * ├── claude-config/ CLAUDE_CONFIG_DIR: seeded .claude.json + + * │ settings.json registering BOTH live AUQ hooks + * │ (question-preference PreToolUse deny, + * │ auq-error-fallback PostToolUse) — production + * │ topology: a slipped AUQ call gets denied without + * │ derailing the run. hermetic-env has no built-in + * │ hook seeding, so this test writes its own. + * └── .claude/skills/gstack/ + * ├── document-release/SKILL.md sliced from the LIVE generated skill + * │ (frontmatter + Preamble + AskUserQuestion + * │ Format + Step 8 VERSION gate) — extract, + * │ never copy the full 1900-line skill + * └── bin/ full live bin/ copy: the preamble's + * `$HOME/.claude/skills/gstack/bin/gstack-skill-start` + * resolves here; sibling bins degrade gracefully + * + * The dispatch prompt instructs the GSTACK_SESSION_KIND=spawned prefix; the + * hooks run with the CHILD env (Conductor vars set, no per-command marker) — + * exactly the production topology where hook env-blindness is permanent. + * + * Gating: gate-tier self-gate (deterministic safety/functional) composed + * with diff selection. Run locally: + * EVALS=1 EVALS_TIER=gate EVALS_ALL=1 bun test test/skill-e2e-docsync-spawned.test.ts + */ +import { expect, beforeAll, afterAll } from 'bun:test'; +import { CAPTURE_LONG_MS } from './helpers/eval-budgets'; +import * as fs from 'fs'; +import * as os from 'os'; +import * as path from 'path'; +import { spawnSync } from 'child_process'; +import { runSkillTest } from './helpers/session-runner'; +import { + ROOT, runId, + describeIfSelected, testConcurrentIfSelected, + createEvalCollector, recordE2E, finalizeEvalCollector, logCost, +} from './helpers/e2e-helpers'; +import { describeE2ETier } from './helpers/e2e-gate'; +import { buildSeedConfig } from './helpers/hermetic-env'; + +const describeE2E = describeE2ETier('gate'); +const evalCollector = createEvalCollector('e2e-docsync-spawned'); + +/** Slice [startMarker, next `\n## ` heading) out of content; throw on drift. */ +function sliceSection(content: string, startMarker: string, what: string): string { + const start = content.indexOf(startMarker); + if (start === -1) { + throw new Error(`docsync-spawned fixture: marker "${startMarker}" moved in ${what} — update the slice`); + } + const end = content.indexOf('\n## ', start + startMarker.length); + return content.slice(start, end === -1 ? undefined : end + 1); +} + +/** Last line of the final message that parses as a JSON object (the model may + * close a code fence after the contract line — scan upward past that). */ +function lastJsonLine(output: string): Record | null { + const lines = output.trim().split('\n').map((l) => l.trim()).filter(Boolean); + for (let i = lines.length - 1; i >= Math.max(0, lines.length - 3); i--) { + const l = lines[i].replace(/^`+|`+$/g, ''); + if (!l.startsWith('{')) continue; + try { return JSON.parse(l); } catch { return null; } + } + return null; +} + +describeE2E('Spawned docsync JSON contract E2E (gate)', () => { + describeIfSelected('Spawned docsync JSON contract', ['docsync-spawned'], () => { + let workDir: string; + let repoDir: string; + let dispatchPrompt: string; + + beforeAll(() => { + workDir = fs.mkdtempSync(path.join(os.tmpdir(), 'gstack-docsync-spawned-')); + repoDir = path.join(workDir, 'repo'); + fs.mkdirSync(repoDir, { recursive: true }); + + const assertOk = (r: ReturnType, what: string) => { + if (r.status !== 0) { + throw new Error( + `docsync-spawned fixture setup failed: ${what} → exit ${r.status}\n${r.stderr?.toString() ?? ''}` + ); + } + }; + const run = (cmd: string, args: string[]) => + assertOk(spawnSync(cmd, args, { cwd: repoDir, stdio: 'pipe', timeout: 15000 }), `${cmd} ${args.join(' ')}`); + run('git', ['init', '-b', 'main']); + run('git', ['config', 'user.email', 'test@test.com']); + run('git', ['config', 'user.name', 'Test']); + run('git', ['config', 'commit.gpgsign', 'false']); + fs.writeFileSync(path.join(repoDir, 'app.ts'), 'export const v = 1;\n'); + fs.writeFileSync(path.join(repoDir, 'README.md'), '# Fixture\n\nA tiny app.\n'); + fs.writeFileSync(path.join(repoDir, 'VERSION'), '0.1.0.0\n'); + fs.writeFileSync( + path.join(repoDir, 'CHANGELOG.md'), + '# Changelog\n\n## [0.1.0.0] - 2026-01-01\n\n- Initial release\n' + ); + run('git', ['add', 'app.ts', 'README.md', 'VERSION', 'CHANGELOG.md']); + run('git', ['commit', '-m', 'initial']); + // Feature branch with a committed code change and VERSION deliberately + // NOT bumped — Step 8's "If VERSION was NOT bumped" AUQ gate fires + // (RECOMMENDATION: C — Skip). The spawned machinery must auto-choose it. + run('git', ['checkout', '-b', 'feature/spawned-docsync']); + fs.writeFileSync(path.join(repoDir, 'app.ts'), 'export const v = 2;\n'); + run('git', ['add', 'app.ts']); + run('git', ['commit', '-m', 'feat: bump v']); + + // --- Sliced document-release skill (extract, never copy the full skill) --- + const skill = fs.readFileSync(path.join(ROOT, 'document-release', 'SKILL.md'), 'utf-8'); + const releaseBody = fs.readFileSync( + path.join(ROOT, 'document-release', 'sections', 'release-body.md'), 'utf-8' + ); + const fmEnd = skill.indexOf('\n---', 3); + if (!skill.startsWith('---') || fmEnd === -1) { + throw new Error('docsync-spawned fixture: document-release frontmatter moved — update the slice'); + } + const frontmatter = skill.slice(0, fmEnd + 5); + const fixtureSkill = [ + frontmatter, + '# Document Release (E2E slice: preamble + AUQ format + VERSION gate)\n', + sliceSection(skill, '## Preamble (run first)', 'document-release/SKILL.md'), + sliceSection(skill, '## AskUserQuestion Format', 'document-release/SKILL.md'), + sliceSection(releaseBody, '## Step 8: VERSION Bump Question', 'document-release/sections/release-body.md'), + '## Workflow end\n\nAfter Step 8 the workflow is complete for this environment — produce your final response exactly as your dispatch instructions specify.\n', + ].join('\n'); + const plantedSkills = path.join(workDir, '.claude', 'skills', 'gstack'); + fs.mkdirSync(path.join(plantedSkills, 'document-release'), { recursive: true }); + fs.writeFileSync(path.join(plantedSkills, 'document-release', 'SKILL.md'), fixtureSkill); + + // Full live bin/ copy: the preamble fence resolves + // $HOME/.claude/skills/gstack/bin/gstack-skill-start here ($0-relative + // siblings like gstack-session-kind resolve too; the rest are + // `|| true`-guarded and degrade silently). + fs.cpSync(path.join(ROOT, 'bin'), path.join(plantedSkills, 'bin'), { recursive: true }); + + // Hermetic GSTACK_HOME — update_check: false keeps the preamble off the + // network (same gate the unit tests use). + fs.mkdirSync(path.join(workDir, 'gstack-home'), { recursive: true }); + fs.writeFileSync( + path.join(workDir, 'gstack-home', 'config.yaml'), 'update_check: false\n' + ); + + // --- CLAUDE_CONFIG_DIR with the LIVE AUQ hooks registered --- + // hermetic-env has no hook-seeding support; write the registration + // ourselves. Commands point at the live worktree hook sources (the + // exact units under test) via `bun` — sibling/lib relative imports + // resolve in place; state writes follow the child's GSTACK_HOME. + const cfgDir = path.join(workDir, 'claude-config'); + fs.mkdirSync(cfgDir, { recursive: true }); + fs.writeFileSync( + path.join(cfgDir, '.claude.json'), + JSON.stringify(buildSeedConfig({ + apiKey: process.env.ANTHROPIC_API_KEY, + trustedDirs: [repoDir], + })) + ); + const hook = (f: string) => ({ + type: 'command', + command: `bun ${path.join(ROOT, 'hosts', 'claude', 'hooks', f)}`, + timeout: 5, + }); + const AUQ_MATCHER = '(AskUserQuestion|mcp__.*__AskUserQuestion)'; + fs.writeFileSync( + path.join(cfgDir, 'settings.json'), + JSON.stringify({ + hooks: { + PreToolUse: [{ matcher: AUQ_MATCHER, hooks: [hook('question-preference-hook.ts')] }], + PostToolUse: [{ matcher: AUQ_MATCHER, hooks: [hook('auq-error-fallback-hook.ts')] }], + }, + }, null, 2) + ); + + // --- The dispatch prompt: verbatim from the LIVE regenerated section --- + const prBody = fs.readFileSync(path.join(ROOT, 'ship', 'sections', 'pr-body.md'), 'utf-8'); + const pStart = prBody.indexOf('**Subagent prompt:**'); + const pEnd = prBody.indexOf('**Parent processing:**'); + if (pStart === -1 || pEnd === -1 || pEnd <= pStart) { + throw new Error('docsync-spawned fixture: Step 18 prompt markers moved in pr-body.md — update the slice'); + } + dispatchPrompt = prBody + .slice(pStart + '**Subagent prompt:**'.length, pEnd) + .split('\n') + .map((l) => l.replace(/^> ?/, '')) + .join('\n') + .replace(//g, 'feature/spawned-docsync') + .replace(//g, 'main') + .trim(); + // Parent-plausible environment note only (a real parent knows $HOME). + // Deliberately NO "do not ask questions" priming — resolving the gate + // without stopping IS the behavior under test. + dispatchPrompt += `\n\n(Environment note: HOME is ${workDir}; the git repo is your working directory.)`; + }); + + afterAll(() => { + try { fs.rmSync(workDir, { recursive: true, force: true }); } catch {} + }); + + testConcurrentIfSelected('docsync-spawned', async () => { + const result = await runSkillTest({ + prompt: dispatchPrompt, + workingDirectory: repoDir, + maxTurns: 24, + allowedTools: ['Bash', 'Read', 'Grep', 'Glob', 'Write', 'Edit'], + timeout: CAPTURE_LONG_MS, + env: { + HOME: workDir, + GSTACK_HOME: path.join(workDir, 'gstack-home'), + CLAUDE_CONFIG_DIR: path.join(workDir, 'claude-config'), + // Production topology: the parent is a Conductor-hosted session and + // the subagent inherits its env. The hermetic default GSTACK_HEADLESS + // is cleared — a real parent session doesn't carry it (and empty + // means unset per the -n guards). + CONDUCTOR_WORKSPACE_PATH: '/tmp/conductor-ws-e2e', + GSTACK_HEADLESS: '', + }, + testName: 'docsync-spawned', + runId, + }); + + logCost('spawned docsync JSON contract', result); + + const contract = lastJsonLine(result.output); + const version = fs.readFileSync(path.join(repoDir, 'VERSION'), 'utf-8'); + + recordE2E(evalCollector, 'spawned docsync JSON contract', 'Spawned docsync JSON contract', result, { + passed: + result.exitReason === 'success' && + contract !== null && + Array.isArray((contract as any)?.decisions) && + ((contract as any).decisions as unknown[]).length >= 1 && + version === '0.1.0.0\n', + }); + + // THE #2733 regression asserts: the run ended in the machine-parseable + // contract (not a prose decision brief waiting for an answer)... + expect(result.exitReason).toBe('success'); + expect(contract, `final message did not end with the JSON contract:\n${result.output.slice(-800)}`).not.toBeNull(); + for (const key of ['files_updated', 'commit_sha', 'pushed', 'documentation_section', 'decisions']) { + expect(Object.keys(contract!), `contract missing key ${key}`).toContain(key); + } + // ...the fired VERSION gate was auto-chosen and RECORDED (transparency + // mechanism — the parent prints these to the ship console)... + const decisions = (contract as any).decisions; + expect(Array.isArray(decisions)).toBe(true); + expect(decisions.length, 'the fired VERSION gate must be recorded in decisions').toBeGreaterThanOrEqual(1); + // ...and the gate resolved to its recommended option (C — Skip): the + // subagent must NOT have bumped VERSION on its own. + expect(version).toBe('0.1.0.0\n'); + + console.log( + `contractKeys=${contract ? Object.keys(contract).join(',') : 'none'} decisions=${JSON.stringify(decisions)} exit=${result.exitReason}` + ); + }, CAPTURE_LONG_MS); + }); +}); + +// Module-level afterAll — finalize eval collector after all tests complete +afterAll(async () => { + await finalizeEvalCollector(evalCollector); +});