import * as fs from 'node:fs'; import * as os from 'node:os'; import * as path from 'node:path'; import { createHash } from 'node:crypto'; import { spawnSync } from 'node:child_process'; import { buildSeedConfig } from './hermetic-env'; import { getProjectEvalDir } from './eval-store'; import type { SkillTestResult } from './session-runner'; export const DOCSYNC_ROOT = path.resolve(import.meta.dir, '../..'); export const DOC_PATH = 'handbook/reference/commands/widget.md.tmpl'; export type DocsScenario = 'updated' | 'current' | 'risky' | 'store' | 'legacy'; export function gitAt(repo: string, ...args: string[]): string { const result = spawnSync('git', args, { cwd: repo, encoding: 'utf8', timeout: 15000, env: { ...process.env, GIT_OPTIONAL_LOCKS: '0' } }); if (result.status !== 0) throw new Error(`fixture git ${args.join(' ')}: ${result.stderr}`); return result.stdout.trimEnd(); } export function repoSnapshot(repo: string) { const files = gitAt(repo, 'ls-files', '-z', '--cached', '--others', '--exclude-standard').split('\0').filter(Boolean); const contents: Record = {}; for (const name of new Set(files)) { const file = path.join(repo, name); if (!fs.existsSync(file)) continue; const resolved = fs.realpathSync(file); if (!resolved.startsWith(fs.realpathSync(repo) + path.sep)) throw new Error(`fixture path escaped: ${name}`); if (fs.statSync(file).isFile()) contents[name] = fs.readFileSync(file).toString('base64'); } return { head: gitAt(repo, 'rev-parse', 'HEAD'), index: gitAt(repo, 'ls-files', '--stage'), contents }; } export function changedFiles(before: ReturnType, after: ReturnType): string[] { return [...new Set([...Object.keys(before.contents), ...Object.keys(after.contents)])] .filter(p => before.contents[p] !== after.contents[p]).sort(); } export function docsCandidate(repo: string, auditId: string, mode: 'edit' | 'read-only', base: string) { const snapshot = repoSnapshot(repo); return { audit_id: auditId, mode, base_sha: base, head: snapshot.head, branch: gitAt(repo, 'branch', '--show-current'), selected_paths: Object.keys(snapshot.contents).filter(p => p !== 'personal-note.txt'), docs_roots: ['handbook'], generated_outputs: [], index: snapshot.index, content_hashes: Object.fromEntries(Object.entries(snapshot.contents).map(([p, bytes]) => [p, createHash('sha256').update(Buffer.from(bytes, 'base64')).digest('hex')])), pre_existing_dirty: gitAt(repo, 'status', '--porcelain', '-z'), }; } export function fixtureDocs(scenario: DocsScenario, generatedRoot = process.env.DOCSYNC_GENERATED_ROOT || DOCSYNC_ROOT) { const home = fs.mkdtempSync(path.join(os.tmpdir(), 'ds-')); const repo = path.join(home, 'repo'); const skills = path.join(home, '.claude/skills/gstack'); fs.mkdirSync(repo); gitAt(repo, 'init', '-b', 'main'); gitAt(repo, 'config', 'user.email', 'test@test.com'); gitAt(repo, 'config', 'user.name', 'Test'); gitAt(repo, 'config', 'commit.gpgsign', 'false'); fs.mkdirSync(path.join(repo, '.qa-state')); fs.appendFileSync(path.join(repo, '.git/info/exclude'), '\n.qa-state/\n'); const write = (file: string, text: string) => { const dest = path.join(repo, file); fs.mkdirSync(path.dirname(dest), { recursive: true }); fs.writeFileSync(dest, text); }; write('README.md', '# Widget CLI\n\nCommand reference: [widget](handbook/reference/commands/widget.md.tmpl).\n'); write('AGENTS.md', '# Documentation\n\nAuthored docs live under handbook/. Edit .md.tmpl sources, not generated pages.\n'); write(DOC_PATH, '# Widget reference\n\nDefault format: text.\n\nUser-maintained note: KEEP THIS EXACTLY.\n'); write('app.ts', 'export const format = "text";\n'); write('VERSION', '0.1.0.0\n'); write('CHANGELOG.md', '# Changelog\n\n## 0.1.0.0\n\n- Original entry: KEEP THIS EXACTLY.\n'); write('TODOS.md', '# TODOs\n\n- Confirm launch readiness.\n'); write('package.json', '{"name":"widget-fixture","version":"0.1.0"}\n'); gitAt(repo, 'add', 'README.md', 'AGENTS.md', DOC_PATH, 'app.ts', 'VERSION', 'CHANGELOG.md', 'TODOS.md', 'package.json'); gitAt(repo, 'commit', '-m', 'fixture baseline'); const base = gitAt(repo, 'rev-parse', 'HEAD'); if (scenario !== 'store') gitAt(repo, 'checkout', '-b', 'feature/docs'); if (scenario === 'current') { write(DOC_PATH, '# Widget reference\n\nDefault format: text.\n\nUser-maintained note: KEEP THIS EXACTLY.\n\nSupports plain text output.\n'); gitAt(repo, 'add', DOC_PATH); gitAt(repo, 'commit', '-m', 'docs: clarify format'); const remote = path.join(home, 'remote.git'); fs.mkdirSync(remote); gitAt(remote, 'init', '--bare', '-b', 'main'); gitAt(repo, 'remote', 'add', 'origin', remote); gitAt(repo, 'push', '-u', 'origin', 'feature/docs'); } else { write('app.ts', 'export const format = "json";\n'); gitAt(repo, 'add', 'app.ts'); write('options.ts', 'export const pretty = true;\n'); write('README.md', '# Widget CLI\n\nCommand reference: [widget](handbook/reference/commands/widget.md.tmpl).\n\nThe default output format is JSON.\n'); } if (scenario === 'risky') { write('SECURITY.md', '# Security Model\n\nAll command output is guaranteed to contain no sensitive data.\n'); write('options.ts', 'export const pretty = true;\nexport const outputIncludesSensitiveData = true;\n'); } write('personal-note.txt', 'Unrelated user content: KEEP THIS EXACTLY.\n'); for (const skill of ['document-release', 'ship']) { fs.mkdirSync(path.join(skills, skill, 'sections'), { recursive: true }); for (const relative of skill === 'ship' ? ['SKILL.md', 'sections/documentation.md', 'sections/pr-body.md'] : ['SKILL.md', 'sections/audit-scope.md', 'sections/release-body.md']) { const source = path.join(generatedRoot, skill, relative); if (!fs.existsSync(source)) throw new Error(`Generate the changed skill before live evaluation: ${source}`); fs.copyFileSync(source, path.join(skills, skill, relative)); } } fs.cpSync(path.join(DOCSYNC_ROOT, 'bin'), path.join(skills, 'bin'), { recursive: true, filter: source => !fs.statSync(source).isFile() || fs.statSync(source).size < 5_000_000, }); const state = path.join(home, 'state'); fs.mkdirSync(state); fs.writeFileSync(path.join(state, 'config.yaml'), 'update_check: false\n'); const config = path.join(home, 'cc'); fs.mkdirSync(config); fs.writeFileSync(path.join(config, '.claude.json'), JSON.stringify(buildSeedConfig({ apiKey: process.env.ANTHROPIC_API_KEY, trustedDirs: [repo], })), { mode: 0o600 }); const hook = (name: string) => ({ type: 'command', command: `bun ${path.join(DOCSYNC_ROOT, 'hosts/claude/hooks', name)}`, timeout: 5 }); fs.writeFileSync(path.join(config, 'settings.json'), JSON.stringify({ hooks: { PreToolUse: [{ matcher: '(AskUserQuestion|mcp__.*__AskUserQuestion)', hooks: [hook('question-preference-hook.ts')] }], PostToolUse: [{ matcher: '(AskUserQuestion|mcp__.*__AskUserQuestion)', hooks: [hook('auq-error-fallback-hook.ts')] }], } })); const before = repoSnapshot(repo); const auditId = `fixture-${scenario}`; const candidate = path.join(home, 'candidate.json'); fs.writeFileSync(candidate, JSON.stringify({ audit_id: auditId, mode: scenario === 'store' ? 'read-only' : 'edit', base_sha: base, head: before.head, branch: gitAt(repo, 'branch', '--show-current'), selected_paths: Object.keys(before.contents).filter(p => p !== 'personal-note.txt'), docs_roots: ['handbook'], index: before.index, content_hashes: Object.fromEntries(Object.entries(before.contents).map(([p, bytes]) => [p, createHash('sha256').update(Buffer.from(bytes, 'base64')).digest('hex')])), pre_existing_dirty: gitAt(repo, 'status', '--porcelain'), }), { mode: 0o600 }); const invocation = path.join(home, 'ship-invocation.md'); return { home, repo, skills, before, candidate, auditId, invocation, env: { HOME: home, GSTACK_HOME: state, CLAUDE_CONFIG_DIR: config, GIT_OPTIONAL_LOCKS: '0', CONDUCTOR_WORKSPACE_PATH: home, GSTACK_HEADLESS: '' }, clean: () => fs.rmSync(home, { recursive: true, force: true }), }; } export function preserveDocsEvidence(fixture: ReturnType, result: Pick, runId: string, name: string, extra: Record = {}): string { if (!runId) throw new Error('EVALS_RUN_ID is required to retain docs evidence'); const dir = path.join(path.dirname(getProjectEvalDir()), 'e2e-runs', runId, `${name}-${path.basename(fixture.home)}-fixture`); fs.mkdirSync(dir, { recursive: true, mode: 0o700 }); fs.chmodSync(dir, 0o700); const file = path.join(dir, 'state.json'); fs.writeFileSync(file, JSON.stringify({ before: fixture.before, after: repoSnapshot(fixture.repo), output: result.output, calls: result.toolCalls, ...extra }, null, 2), { mode: 0o600 }); if (!fs.statSync(file).size) throw new Error('docs evidence was not retained'); return file; } export function sawSpawnedMarker(result: SkillTestResult): boolean { return result.toolCalls.some(call => call.tool === 'Bash' && /gstack-skill-start|"\$_SS"/.test(call.input?.command ?? '') && /^SESSION_KIND: spawned\r?$/m.test(call.output)); }