import * as fs from 'node:fs'; import * as os from 'node:os'; import * as path from 'node:path'; import { createHash } from 'node:crypto'; import { buildSeedConfig } from './hermetic-env'; import { getProjectEvalDir } from './eval-store'; import type { SkillTestResult } from './session-runner'; import { DOC_PATH, docsConfigDir, gitAt, repoSnapshot } from './docsync-repo'; export { DOC_PATH, docsConfigDir, gitAt, repoSnapshot, changedFiles, docsSnapshotArgv, gitBlobId, type DocsCandidateRecord } from './docsync-repo'; export const DOCSYNC_ROOT = path.resolve(import.meta.dir, '../..'); export type DocsScenario = 'updated' | 'current' | 'risky' | 'store' | 'legacy'; export function fixtureDocs(scenario: DocsScenario, generatedRoot = process.env.DOCSYNC_GENERATED_ROOT || DOCSYNC_ROOT) { const home = fs.mkdtempSync(path.join(os.tmpdir(), 'ds-')); const repo = path.join(home, 'repo'); const skills = path.join(home, '.claude/skills/gstack'); fs.mkdirSync(repo); gitAt(repo, 'init', '-b', 'main'); fs.appendFileSync(path.join(repo, '.git/config'), '[user]\n\temail = test@test.com\n\tname = Test\n[commit]\n\tgpgsign = false\n'); fs.mkdirSync(path.join(repo, '.qa-state')); fs.appendFileSync(path.join(repo, '.git/info/exclude'), '\n.qa-state/\n'); const write = (file: string, text: string) => { const dest = path.join(repo, file); fs.mkdirSync(path.dirname(dest), { recursive: true }); fs.writeFileSync(dest, text); }; write('README.md', '# Widget CLI\n\nCommand reference: [widget](handbook/reference/commands/widget.md.tmpl).\n'); write('AGENTS.md', '# Documentation\n\nAuthored docs live under handbook/. Edit .md.tmpl sources, not generated pages.\n'); write(DOC_PATH, '# Widget reference\n\nDefault format: text.\n\nUser-maintained note: KEEP THIS EXACTLY.\n'); write('app.ts', 'export const format = "text";\n'); write('VERSION', '0.1.0.0\n'); write('CHANGELOG.md', '# Changelog\n\n## 0.1.0.0\n\n- Original entry: KEEP THIS EXACTLY.\n'); write('TODOS.md', '# TODOs\n\n- Confirm launch readiness.\n'); write('package.json', '{"name":"widget-fixture","version":"0.1.0"}\n'); gitAt(repo, 'add', 'README.md', 'AGENTS.md', DOC_PATH, 'app.ts', 'VERSION', 'CHANGELOG.md', 'TODOS.md', 'package.json'); gitAt(repo, 'commit', '-m', 'fixture baseline'); const base = gitAt(repo, 'rev-parse', 'HEAD'); if (scenario !== 'store') gitAt(repo, 'checkout', '-b', 'feature/docs'); if (scenario === 'current') { write(DOC_PATH, '# Widget reference\n\nDefault format: text.\n\nUser-maintained note: KEEP THIS EXACTLY.\n\nSupports plain text output.\n'); gitAt(repo, 'add', DOC_PATH); gitAt(repo, 'commit', '-m', 'docs: clarify format'); const remote = path.join(home, 'remote.git'); fs.mkdirSync(remote); gitAt(remote, 'init', '--bare', '-b', 'main'); gitAt(repo, 'remote', 'add', 'origin', remote); gitAt(repo, 'push', '-u', 'origin', 'feature/docs'); } else { write('app.ts', 'export const format = "json";\n'); gitAt(repo, 'add', 'app.ts'); write('options.ts', 'export const pretty = true;\n'); write('README.md', '# Widget CLI\n\nCommand reference: [widget](handbook/reference/commands/widget.md.tmpl).\n\nThe default output format is JSON.\n'); } if (scenario === 'risky') { write('SECURITY.md', '# Security Model\n\nAll command output is guaranteed to contain no sensitive data.\n'); write('options.ts', 'export const pretty = true;\nexport const outputIncludesSensitiveData = true;\n'); } write('personal-note.txt', 'Unrelated user content: KEEP THIS EXACTLY.\n'); for (const skill of ['document-release', 'ship']) { fs.mkdirSync(path.join(skills, skill, 'sections'), { recursive: true }); for (const relative of skill === 'ship' ? ['SKILL.md', 'sections/documentation.md', 'sections/pr-body.md'] : ['SKILL.md', 'sections/audit-scope.md', 'sections/release-body.md']) { const source = path.join(generatedRoot, skill, relative); if (!fs.existsSync(source)) throw new Error(`Generate the changed skill before live evaluation: ${source}`); fs.copyFileSync(source, path.join(skills, skill, relative)); } } fs.cpSync(path.join(DOCSYNC_ROOT, 'bin'), path.join(skills, 'bin'), { recursive: true, filter: source => !fs.statSync(source).isFile() || fs.statSync(source).size < 5_000_000, }); const state = path.join(home, 'state'); fs.mkdirSync(state); fs.writeFileSync(path.join(state, 'config.yaml'), 'update_check: false\n'); const config = docsConfigDir(home); fs.mkdirSync(config); fs.writeFileSync(path.join(config, '.claude.json'), JSON.stringify(buildSeedConfig({ apiKey: process.env.ANTHROPIC_API_KEY, trustedDirs: [repo], })), { mode: 0o600 }); const hook = (name: string) => ({ type: 'command', command: `bun ${path.join(DOCSYNC_ROOT, 'hosts/claude/hooks', name)}`, timeout: 5 }); fs.writeFileSync(path.join(config, 'settings.json'), JSON.stringify({ hooks: { PreToolUse: [{ matcher: '(AskUserQuestion|mcp__.*__AskUserQuestion)', hooks: [hook('question-preference-hook.ts')] }], PostToolUse: [{ matcher: '(AskUserQuestion|mcp__.*__AskUserQuestion)', hooks: [hook('auq-error-fallback-hook.ts')] }], } })); const before = repoSnapshot(repo); const auditId = `fixture-${scenario}`; const candidate = path.join(home, 'candidate.json'); fs.writeFileSync(candidate, JSON.stringify({ audit_id: auditId, mode: scenario === 'store' ? 'read-only' : 'edit', base_sha: base, head: before.head, branch: gitAt(repo, 'branch', '--show-current'), selected_paths: Object.keys(before.contents).filter(p => p !== 'personal-note.txt'), docs_roots: ['handbook'], index: before.index, content_hashes: Object.fromEntries(Object.entries(before.contents).map(([p, bytes]) => [p, createHash('sha256').update(Buffer.from(bytes, 'base64')).digest('hex')])), pre_existing_dirty: gitAt(repo, 'status', '--porcelain'), }), { mode: 0o600 }); const invocation = path.join(home, 'ship-invocation.md'); return { home, repo, skills, before, candidate, auditId, invocation, env: { HOME: home, GSTACK_HOME: state, CLAUDE_CONFIG_DIR: config, GIT_OPTIONAL_LOCKS: '0', CONDUCTOR_WORKSPACE_PATH: home, GSTACK_HEADLESS: '' }, clean: () => fs.rmSync(home, { recursive: true, force: true }), }; } export function preserveDocsEvidence(fixture: ReturnType, result: Pick, runId: string, name: string, extra: Record = {}, projectDir?: string): string { if (!runId) throw new Error('EVALS_RUN_ID is required to retain docs evidence'); const dir = path.join(projectDir ?? path.dirname(getProjectEvalDir()), 'e2e-runs', runId, `${name}-${path.basename(fixture.home)}-fixture`); fs.mkdirSync(dir, { recursive: true, mode: 0o700 }); fs.chmodSync(dir, 0o700); const file = path.join(dir, 'state.json'); fs.writeFileSync(file, JSON.stringify({ before: fixture.before, after: repoSnapshot(fixture.repo), output: result.output, calls: result.toolCalls, ...extra }, null, 2), { mode: 0o600 }); if (!fs.statSync(file).size) throw new Error('docs evidence was not retained'); return file; } export function sawSpawnedMarker(result: SkillTestResult): boolean { return result.toolCalls.some(call => call.tool === 'Bash' && /gstack-skill-start|"\$_SS"/.test(call.input?.command ?? '') && /^SESSION_KIND: spawned\r?$/m.test(call.output)); } /** The dispatched /document-release child's own report of its skill-start echo: * its hand-back, or its completed reply to a parent SendMessage addressed to that * same child (ship's "as echoed in the child output"). Other agents' text and the * parent's own claims never count. */ export function childReportedSpawnedMarker(result: Pick, dispatch: { input: unknown; output: string }): boolean { if (dispatch.output.includes('SESSION_KIND: spawned')) return true; const events = result.transcript as any[]; const uses = events.flatMap(event => event?.type === 'assistant' && Array.isArray(event.message?.content) ? event.message.content : []) .filter((block: any) => block?.type === 'tool_use'); const dispatched = uses.filter((block: any) => ['Agent', 'Task'].includes(block.name) && JSON.stringify(block.input) === JSON.stringify(dispatch.input)); if (dispatched.length !== 1) return false; const started = events.filter(event => event?.type === 'system' && event.subtype === 'task_started' && event.tool_use_id === dispatched[0].id); if (started.length !== 1 || typeof started[0].task_id !== 'string') return false; const child = started[0].task_id; const asks = new Set(uses.filter((block: any) => block.name === 'SendMessage' && block.input?.to === child).map((block: any) => block.id)); return events.some(event => event?.type === 'system' && event.subtype === 'task_notification' && event.task_id === child && asks.has(event.tool_use_id) && event.status === 'completed' && String(event.summary ?? '').includes('SESSION_KIND: spawned')); }