From 1dab2e4ef40f4db8bbde290077865927e5e66cd5 Mon Sep 17 00:00:00 2001 From: garrytan Date: Sun, 18 Oct 2026 19:11:29 +0000 Subject: [PATCH] fix(evals): docsync observer accepts read-only git describe Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> --- test/docsync-command-grammar.test.ts | 100 +++++++++++++++++++++++++++ test/helpers/docsync-observer.ts | 7 +- 2 files changed, 106 insertions(+), 1 deletion(-) diff --git a/test/docsync-command-grammar.test.ts b/test/docsync-command-grammar.test.ts index 627a1062d..75c64dbb6 100644 --- a/test/docsync-command-grammar.test.ts +++ b/test/docsync-command-grammar.test.ts @@ -306,3 +306,103 @@ test('subcommand --help is admitted alone and runs without effect; with any othe expect(fs.existsSync(record)).toBe(false); expect(repoSnapshot(fixture.repo)).toEqual(before); }); + +// PR gate run 39153452209 (ship-docsync-store, head a844821583d664e3dea279fff12b935865e92f52) went red only on the +// document-release child's `git describe --tags --always` (bash#24). Every Bash command of the parent and its child, +// in order; is the fixture home. +const STORE_39153452209 = [ + 'git status', + 'git rev-parse HEAD', + 'git ls-files', + '/.claude/skills/gstack/bin/gstack-docs-candidate --help', + 'git diff --cached', + 'git diff', + 'git branch --show-current', + 'git log --oneline', + 'git check-ignore -v .qa-state', + 'ls -la ', + '/.claude/skills/gstack/bin/gstack-skill-start --help', + 'git ls-files -z --cached --others --exclude-standard', + 'git show HEAD:handbook/reference/commands/widget.md.tmpl', + 'ls -la /state', + 'git hash-object README.md handbook/reference/commands/widget.md.tmpl options.ts app.ts personal-note.txt', + 'git log --oneline --decorate --all', + '/.claude/skills/gstack/bin/gstack-docs-candidate snapshot --out /ship-docs-20261018-c0048c5-a1.json --audit-id ship-docs-20261018-c0048c5-a1 --mode read-only --base c0048c54c542cf0ba46b7587cf7fea1b519c75e4 --select app.ts --select README.md --select options.ts --docs handbook --docs README.md --docs AGENTS.md --docs CHANGELOG.md --docs TODOS.md', + 'GSTACK_SESSION_KIND=spawned /.claude/skills/gstack/bin/gstack-skill-start --skill document-release --model claude', + 'git status', + 'git log --oneline', + 'git diff c0048c54c542cf0ba46b7587cf7fea1b519c75e4 HEAD --stat', + 'git diff --cached', + 'git diff', + 'git ls-files --cached --others --exclude-standard', + 'git hash-object README.md options.ts app.ts personal-note.txt', + 'git describe --tags --always', + '/.claude/skills/gstack/bin/gstack-skill-end --skill document-release --outcome success --session-id 878-1792348356-d8b277ad --tel-start 1792348356 --used-browse no', + '/.claude/skills/gstack/bin/gstack-docs-candidate compare /ship-docs-20261018-c0048c5-a1.json', + 'git status', + 'git rev-parse HEAD', + 'git hash-object README.md handbook/reference/commands/widget.md.tmpl options.ts app.ts personal-note.txt AGENTS.md CHANGELOG.md TODOS.md VERSION package.json', + 'ls -la ', + 'ls -la /repo', + 'cat /ship-docs-20261018-c0048c5-a1-documentation.md >> /ship-report.md', + 'git status', + 'ls -la ', +]; + +test('every Bash command of the captured run 39153452209 ship-docsync-store is inside the interface', () => { + const commands = STORE_39153452209.map(command => command.replaceAll('', fixture.home)); + for (const file of ['ship-docs-20261018-c0048c5-a1-documentation.md', 'ship-report.md']) fs.writeFileSync(path.join(fixture.home, file), ''); + expect(commands).toHaveLength(36); + expect(commands).toContain('git describe --tags --always'); + expect(commands.filter(command => !docsCommandAllowed(command, fixture))).toEqual([]); + expect(docsToolFailures(nativeResult('git describe --tags --always', 'c0048c5', 'docs-dispatch'), fixture, [], true)).toEqual([]); +}); + +const DESCRIBE_READS = ['git describe', 'git describe --tags', 'git describe --always', 'git describe --abbrev=7', + 'git describe --exact-match', 'git describe HEAD', "git describe --tags --always --abbrev=12 'HEAD^{}'", + 'git describe --exact-match --tags main', 'git describe --always main', 'git describe --tags --always 2>/dev/null']; +const DESCRIBE_REFUSED = ['git describe --dirty', 'git describe --dirty=-modified', 'git describe --always --dirty', + 'git describe --broken', 'git describe --broken=-broken', 'git describe --contains HEAD', 'git describe --all', + 'git describe --abbrev', 'git describe --abbrev=x', 'git describe HEAD main', 'git describe -- HEAD', + 'git describe --tags > tag.md', 'git describe --tags; git tag v9', 'git describe --tags && git tag v9']; + +test('read-only git describe forms are declared and leave the index and repository unchanged', () => { + expect(docsNativeInterface(fixture)).toContain('describe with only --tags, --always, --abbrev= or --exact-match and at most one revision'); + const own = fixtureDocs('store'); + try { + gitAt(own.repo, 'tag', '-a', 'v0.1.0', '-m', 'fixture release', 'HEAD'); + const stale = new Date(Date.now() - 3600_000); + fs.utimesSync(path.join(own.repo, 'AGENTS.md'), stale, stale); + const index = () => fs.readFileSync(path.join(own.repo, '.git/index')).toString('base64'); + const before = { repo: repoSnapshot(own.repo), index: index() }; + for (const command of DESCRIBE_READS) { + expect(docsCommandAllowed(command, own), command).toBe(true); + expect(docsToolFailures(nativeResult(command), own, [], true), command).toEqual([]); + const run = spawnSync('bash', ['-c', command], { cwd: own.repo, encoding: 'utf8', timeout: 10000 }); + expect(run.status, `${command}: ${run.stderr}`).toBe(0); + expect({ repo: repoSnapshot(own.repo), index: index() }, command).toEqual(before); + } + } finally { + own.clean(); + } +}); + +test('describe forms that refresh the index or fall outside the declared set, and Git writes, stay outside the interface', () => { + for (const command of [...DESCRIBE_REFUSED, 'git tag v9 HEAD', 'git update-index --refresh', 'git notes add -m x HEAD']) { + expect(docsCommandAllowed(command, fixture), command).toBe(false); + expect(docsToolFailures(nativeResult(command, 'successful tool acknowledgment', 'docs-dispatch'), fixture, [], true), command) + .toEqual(['command outside declared docs observation interface']); + } + const own = fixtureDocs('store'); + try { + const stale = new Date(Date.now() - 3600_000); + fs.utimesSync(path.join(own.repo, 'AGENTS.md'), stale, stale); + const index = () => fs.readFileSync(path.join(own.repo, '.git/index')).toString('base64'); + const before = index(); + const run = spawnSync('git', ['describe', '--always', '--dirty'], { cwd: own.repo, encoding: 'utf8', timeout: 10000 }); + expect(run.status, run.stderr).toBe(0); + expect(index()).not.toBe(before); + } finally { + own.clean(); + } +}); diff --git a/test/helpers/docsync-observer.ts b/test/helpers/docsync-observer.ts index 8a6914e68..ced112fc0 100644 --- a/test/helpers/docsync-observer.ts +++ b/test/helpers/docsync-observer.ts @@ -197,6 +197,7 @@ Keep artifacts concise: update the invocation record in place with ids, counts, const DOCS_READ_COMMANDS = ['pwd', 'ls', 'cat', 'sha256sum', 'stat']; const DOCS_GIT_READS = ['status', 'diff', 'show', 'log', 'ls-files', 'ls-tree', 'check-ignore', 'rev-parse', 'merge-base', 'hash-object']; +const DOCS_DESCRIBE_FLAGS = /^(?:--tags|--always|--exact-match|--abbrev=\d+)$/; const DOCS_SAFE_GIT_CONFIG = /^(?:core\.pager=cat|core\.quotepath=(?:true|false|on|off)|color\.[a-z.]+=(?:true|false|never|always|auto))$/i; /** @@ -261,6 +262,10 @@ function literalDocsCommandAllowed(text: string, fixture: ReturnType /^(?:--output|--ext-diff|--textconv|-w)(?:=|$)/.test(arg))) return false; if (rest[0] === 'hash-object' && rest.some(arg => /^-[^-]*w/.test(arg))) return false; if (rest[0] === 'branch') return rest.length === 2 && rest[1] === '--show-current'; + if (rest[0] === 'describe') { + const operands = rest.slice(1).filter(arg => !DOCS_DESCRIBE_FLAGS.test(arg)); + return operands.length <= 1 && operands.every(arg => !arg.startsWith('-')); + } return DOCS_GIT_READS.includes(rest[0]); } if (commandName === 'bun' || commandName === process.execPath) { @@ -288,7 +293,7 @@ function literalDocsCommandAllowed(text: string, fixture: ReturnType, 'home' | 'repo' | 'skills'>, scripts: string[] = [], transport = false, insert = false, file?: string): string { const skills = fixture.skills.split(path.sep).join('/'); - return `Fixture observation interface (applies to parent and every child; ${file ? `saved at ${file}: each child prompt tells the child to Read that file first instead of copying this text` : 'include this interface in child prompts'}): Bash may execute only separate literal pwd, ls, cat, stat, sha256sum, Git read commands (status, diff, show, log, ls-files, ls-tree, check-ignore, rev-parse, merge-base, hash-object without -w, branch --show-current), the exact generated Preamble block with its spawned prefix, literal installed gstack-skill-start/gstack-skill-end commands for document-release (start requires GSTACK_SESSION_KIND=spawned) or their own --help, or the literal installed ${skills}/bin/gstack-docs-candidate snapshot (its --out a private .json under ${fixture.home}), compare and --help commands. No shell composition, custom interpreters, arbitrary scripts, inline eval or memory-mapped writes. The only additional scripts are ${scripts.length ? scripts.join(', ') : 'none'}.${insert ? ` To insert one saved private Markdown artifact into another (a saved section file into a report), Bash may also run the single literal command cat SOURCE.md >> TARGET.md with absolute paths under ${fixture.home}.` : ''} Read/Glob/Grep remain available; Read skill and section files with Read (offset/limit for ranges), because Bash output over 30KB becomes a preview that no permitted Bash command can page. Use Write/Edit for permitted docs and private JSON/Markdown artifacts under ${fixture.home}; do not rewrite installed skills, config, actor state or scripts. No effects outside the owned fixture. The owner preserves evidence and cleans up. Missing observer coverage blocks acceptance; the Linux kernel monitor covers syscall writes in the product tree, not hostile processes or arbitrary external destinations. + return `Fixture observation interface (applies to parent and every child; ${file ? `saved at ${file}: each child prompt tells the child to Read that file first instead of copying this text` : 'include this interface in child prompts'}): Bash may execute only separate literal pwd, ls, cat, stat, sha256sum, Git read commands (status, diff, show, log, ls-files, ls-tree, check-ignore, rev-parse, merge-base, hash-object without -w, branch --show-current, describe with only --tags, --always, --abbrev= or --exact-match and at most one revision), the exact generated Preamble block with its spawned prefix, literal installed gstack-skill-start/gstack-skill-end commands for document-release (start requires GSTACK_SESSION_KIND=spawned) or their own --help, or the literal installed ${skills}/bin/gstack-docs-candidate snapshot (its --out a private .json under ${fixture.home}), compare and --help commands. No shell composition, custom interpreters, arbitrary scripts, inline eval or memory-mapped writes. The only additional scripts are ${scripts.length ? scripts.join(', ') : 'none'}.${insert ? ` To insert one saved private Markdown artifact into another (a saved section file into a report), Bash may also run the single literal command cat SOURCE.md >> TARGET.md with absolute paths under ${fixture.home}.` : ''} Read/Glob/Grep remain available; Read skill and section files with Read (offset/limit for ranges), because Bash output over 30KB becomes a preview that no permitted Bash command can page. Use Write/Edit for permitted docs and private JSON/Markdown artifacts under ${fixture.home}; do not rewrite installed skills, config, actor state or scripts. No effects outside the owned fixture. The owner preserves evidence and cleans up. Missing observer coverage blocks acceptance; the Linux kernel monitor covers syscall writes in the product tree, not hostile processes or arbitrary external destinations. The working directory for parent and child Bash calls is already ${fixture.repo}. Run Git reads directly, for example: git status, git diff --cached, git merge-base main HEAD, git rev-parse HEAD. Do not use Git global options such as -C, -c, --git-dir or --work-tree, and do not prepend cd or another shell wrapper. The literal git subcommand must immediately follow git; an absolute owned repository path does not make git -C an allowed command.