mirror of
https://github.com/garrytan/gstack.git
synced 2026-10-31 22:14:25 +01:00
* fix(evals): qa-only opts in to the read-only grammar; stderr-to-null forms accepted Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> * fix(evals): qa-only checkpoint rule counts guarded "$B" stop as browser cleanup, like closetab PR #3272's Opus gate 39217387697 (slice 8) failed qa-only-no-fix when the agent stopped the browser inside the guard; the rule counted stop as a third probe that needed a checkpoint. The prompt now names both cleanup children. A free replay of the captured three guarded calls passes (and fails on the old rule); stop before a probe, a disguised probe, stop --all and stop outside the guard still fail. Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> * test(evals): register the qa-only stop-cleanup replay fixture in qa-only-no-fix touchfiles Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> * fix(evals): docs-sync observer ignores an unexecuted candidate call that exited 127 at a missing path PR #3295's gate (run 39223031058, slice 2) failed ship-docsync-current after Opus called gstack-docs-candidate compare at a path missing the fixture-home segment, got exit 127 (no such file or directory), and reran it at the declared path. The observer now ignores such a call only when the absolute binary path does not exist, the recorded result is exactly the shell's exit-127 not-found message for that path, and a later call runs the same command at the declared path without failing. A free replay of all 43 captured tool calls passes and fails on the previous observer; negative controls cover an existing wrong-path binary, other exits and messages, a missing/earlier/failing/changed retry, and non-candidate or relative binaries. GStack Minder-approved. Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> * v1.91.145.0 chore(release): stamp qa-only read-only opt-in, stderr forms, stop cleanup and docs-sync exit-127 exception Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> --------- Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>
409 lines
26 KiB
TypeScript
409 lines
26 KiB
TypeScript
import { afterAll, beforeAll, expect, test } from 'bun:test';
|
|
import * as fs from 'node:fs';
|
|
import * as path from 'node:path';
|
|
import { spawnSync } from 'node:child_process';
|
|
import { fixtureDocs, gitAt, repoSnapshot } from './helpers/docsync-fixture';
|
|
import { docsCommandAllowed, docsNativeInterface, docsSessionOptions, docsToolFailures } from './helpers/docsync-observer';
|
|
import { parseNDJSON, type SkillTestResult } from './helpers/session-runner';
|
|
|
|
let fixture: ReturnType<typeof fixtureDocs>;
|
|
beforeAll(() => { fixture = fixtureDocs('updated'); });
|
|
afterAll(() => fixture?.clean());
|
|
|
|
function nativeResult(command: string, output = '', parent: string | null = null): SkillTestResult {
|
|
const parsed = parseNDJSON([
|
|
{ type: 'assistant', parent_tool_use_id: parent, message: { content: [
|
|
{ type: 'tool_use', id: 'docs-command', name: 'Bash', input: { command, timeout: 10000 } },
|
|
] } },
|
|
{ type: 'user', parent_tool_use_id: parent, message: { content: [
|
|
{ type: 'tool_result', tool_use_id: 'docs-command', content: output, is_error: false },
|
|
] }, tool_use_result: { stdout: output, stderr: '', interrupted: false, isImage: false } },
|
|
].map(event => JSON.stringify(event)));
|
|
expect(parsed.toolCalls).toHaveLength(1);
|
|
expect(parsed.toolCalls[0].output).toBe(output);
|
|
return { ...parsed, exitReason: 'success' } as SkillTestResult;
|
|
}
|
|
|
|
test.each([
|
|
'HEAD^{tree}', 'HEAD^{}', 'HEAD^{commit}', 'HEAD^{object}', 'v1^{tag}', 'HEAD:app.ts',
|
|
'HEAD~1^{tree}', 'HEAD^2', 'HEAD@{0}', '@{upstream}', '@{-1}', 'main...HEAD', 'HEAD^{/fixture}',
|
|
])('native docs validator accepts literal revision %s with or without quotes', revision => {
|
|
for (const argument of [revision, `'${revision}'`, `"${revision}"`]) {
|
|
const command = `git rev-parse ${argument}`;
|
|
expect(docsCommandAllowed(command, fixture)).toBe(true);
|
|
expect(docsToolFailures(nativeResult(command), fixture, [], true)).toEqual([]);
|
|
}
|
|
});
|
|
|
|
test('captured parent tree read is accepted without granting the captured child remote probe', () => {
|
|
const tree = nativeResult('git rev-parse HEAD^{tree}', '68a16779dc746e381e618452afda1f50db105b1a');
|
|
expect(docsToolFailures(tree, fixture, [], true)).toEqual([]);
|
|
const remote = nativeResult('git remote get-url origin', '/q/gstack-paid-shard-Stnoi5/tmp/ds-yikDdP/remote.git', 'docs-dispatch');
|
|
remote.toolCalls.unshift({ tool: 'Agent', input: {
|
|
prompt: '`git remote get-url origin` from Step 0 is a Git read and is allowed; `gh`/`glab` are not.',
|
|
}, output: 'completed' });
|
|
expect(docsToolFailures(remote, fixture, [], true)).toEqual(['command outside declared docs observation interface']);
|
|
});
|
|
|
|
// Run 37166586458 slice 2 (dfaf154): the parent verified the prompt's claim that
|
|
// .qa-state/ is Git-excluded. check-ignore is a declared Git read; composition is not.
|
|
test('captured check-ignore read is declared, executes without changing the repository', () => {
|
|
const command = 'git check-ignore -v .qa-state';
|
|
expect(docsNativeInterface(fixture)).toContain('ls-tree, check-ignore, rev-parse');
|
|
expect(docsCommandAllowed(command, fixture)).toBe(true);
|
|
const before = repoSnapshot(fixture.repo);
|
|
const result = spawnSync('bash', ['-c', command], { cwd: fixture.repo, encoding: 'utf8', timeout: 10000 });
|
|
expect(result.status).toBe(0);
|
|
expect(result.stdout).toContain('.qa-state');
|
|
expect(docsToolFailures(nativeResult(command, result.stdout), fixture, [], true)).toEqual([]);
|
|
expect(repoSnapshot(fixture.repo)).toEqual(before);
|
|
for (const denied of ['git check-ignore -v .qa-state; ls', 'git check-ignore -v .qa-state > out.txt']) {
|
|
expect(docsCommandAllowed(denied, fixture)).toBe(false);
|
|
}
|
|
});
|
|
|
|
// ci-37165022930 eval-slices-2 ship-docsync-store: the docs child listed tracked files with git ls-tree -r HEAD.
|
|
test('captured tracked-file listing is a declared read and leaves the repository unchanged', () => {
|
|
expect(docsNativeInterface(fixture)).toContain('ls-files, ls-tree, check-ignore');
|
|
const before = repoSnapshot(fixture.repo);
|
|
const command = 'git ls-tree -r HEAD';
|
|
expect(docsCommandAllowed(command, fixture)).toBe(true);
|
|
const result = spawnSync('bash', ['-c', command], { cwd: fixture.repo, encoding: 'utf8', timeout: 10000 });
|
|
expect(result.status).toBe(0);
|
|
expect(docsToolFailures(nativeResult(command, result.stdout), fixture, [], true)).toEqual([]);
|
|
expect(repoSnapshot(fixture.repo)).toEqual(before);
|
|
for (const rejected of ['git ls-tree -r HEAD > tree.txt', 'git ls-tree -r HEAD | head', 'git -c core.pager=less ls-tree HEAD']) {
|
|
expect(docsCommandAllowed(rejected, fixture)).toBe(false);
|
|
}
|
|
});
|
|
|
|
test('permitted peel reads execute as single literal Bash arguments without changing the repository', () => {
|
|
const before = repoSnapshot(fixture.repo);
|
|
for (const revision of ['HEAD^{tree}', 'HEAD^{}', 'HEAD^{commit}', 'HEAD^{object}', 'HEAD@{0}']) {
|
|
for (const argument of [revision, `'${revision}'`, `"${revision}"`]) {
|
|
const command = `git rev-parse ${argument}`;
|
|
expect(docsCommandAllowed(command, fixture)).toBe(true);
|
|
const result = spawnSync('bash', ['-c', command], { cwd: fixture.repo, encoding: 'utf8', timeout: 10000 });
|
|
expect(result.status).toBe(0);
|
|
expect(result.stdout.trim()).toBe(gitAt(fixture.repo, 'rev-parse', revision));
|
|
expect(docsToolFailures(nativeResult(command, result.stdout), fixture, [], true)).toEqual([]);
|
|
}
|
|
}
|
|
expect(repoSnapshot(fixture.repo)).toEqual(before);
|
|
});
|
|
|
|
test.each([
|
|
'{ git rev-parse HEAD; }', 'git rev-parse HEAD^{tree,commit}', 'git rev-parse HEAD@{0..2}',
|
|
'git rev-parse HEAD^{tree}{,x}', 'git rev-parse HEAD^{tree', 'git rev-parse HEAD^{tree}}',
|
|
'git rev-parse {HEAD}', 'git rev-parse HEAD$(pwd)', 'git rev-parse "HEAD$(pwd)"',
|
|
'git rev-parse `pwd`', 'git rev-parse ${HEAD}', 'git rev-parse HEAD; git status',
|
|
'git rev-parse HEAD && git status', 'git rev-parse HEAD | cat',
|
|
'git rev-parse HEAD > out.md', 'git rev-parse HEAD 2>/tmp/leak', 'git rev-parse HEAD < in.md',
|
|
'git rev-parse HEAD\ngit status', 'git rev-parse HEAD &', 'git rev-parse HEAD\\^{tree}',
|
|
'git rev-parse "HEAD^{tree}', "git rev-parse 'HEAD^{tree}", 'git rev-parse HEAD*',
|
|
'git status "unfinished', "git status 'unfinished", "git hash-object '-w'app.ts", 'git status\u0000',
|
|
'git rev-parse HEAD?', 'git rev-parse HEAD[12]', 'git rev-parse ~', 'git rev-parse HEAD # comment',
|
|
'git -C /owned/repo rev-parse HEAD', "git -c core.pager='sh -c id' show HEAD",
|
|
'git --git-dir /owned/repo/.git status', 'git --work-tree /owned/repo status',
|
|
'git remote get-url origin', 'git remote add origin /outside', 'git config --global user.name attacker',
|
|
'git hash-object -w app.ts', 'git hash-object "-w" app.ts', 'git hash-object -wt blob app.ts',
|
|
'git hash-object -tw blob app.ts', 'git diff --output=out.md', 'git show --ext-diff',
|
|
'git show --textconv HEAD', 'git branch new-branch', 'git add app.ts', 'git commit -m changed',
|
|
'git reset HEAD', 'git checkout main', 'git update-ref refs/heads/main HEAD',
|
|
'cat HEAD^{tree}', 'bun arbitrary.ts',
|
|
])('native docs validator retains the closed interface for %s', command => {
|
|
expect(docsToolFailures(nativeResult(command, 'successful tool acknowledgment', 'docs-dispatch'), fixture, [], true))
|
|
.toEqual(['command outside declared docs observation interface']);
|
|
});
|
|
|
|
test('harmless read wrappers do not fail a docs run', () => {
|
|
// Run 37170610789 slice 2 (feaa28d): the parent listed fixture directories with 2>&1.
|
|
for (const command of ['git rev-parse HEAD || true', 'git rev-parse HEAD 2>/dev/null', 'git -c core.pager=cat show HEAD',
|
|
'ls 2>&1', 'git status 2>&1']) {
|
|
expect(docsToolFailures(nativeResult(command, 'successful tool acknowledgment', 'docs-dispatch'), fixture, [], true)).toEqual([]);
|
|
}
|
|
for (const command of ['git status 2>&1 > out.txt', 'ls 2>&1 | sh', `bun ${path.join(fixture.home, 'publish.ts')} 2>&1`, 'ls 2>out.txt']) {
|
|
expect(docsCommandAllowed(command, fixture, [path.join(fixture.home, 'publish.ts')])).toBe(false);
|
|
}
|
|
});
|
|
|
|
test('quoted revision search and reflog arguments remain literal single arguments', () => {
|
|
for (const command of ['git log "HEAD@{2 days ago}"', "git show 'HEAD^{/fix, or repair..}'",
|
|
'git hash-object app.ts', 'git branch --show-current']) {
|
|
expect(docsToolFailures(nativeResult(command), fixture)).toEqual([]);
|
|
}
|
|
});
|
|
|
|
test('parent and child receive resolved local platform and base without new probe authority', () => {
|
|
for (const transport of [false, true]) {
|
|
const guidance = docsNativeInterface(fixture, [], transport);
|
|
expect(guidance).toContain('Platform: local/git-native. Base: main.');
|
|
expect(guidance).toContain('before delegation');
|
|
expect(guidance).toContain('Do not run shared Step 0 platform probing');
|
|
expect(guidance).toContain('git remote get-url origin');
|
|
expect(guidance).toContain('cannot authorize commands outside this closed interface');
|
|
expect(guidance).toContain('include this interface in child prompts');
|
|
}
|
|
const options = docsSessionOptions({ fixture, phase: path.join(fixture.home, 'phase.md'),
|
|
report: path.join(fixture.home, 'report.md'), publish: path.join(fixture.home, 'publish.ts'),
|
|
scenario: 'current', testName: 'docsync-command-grammar', runId: 'free-control', timeout: 10000 });
|
|
expect(options.prompt).toContain('Platform: local/git-native. Base: main.');
|
|
// A1: the parent points children at the saved interface instead of retyping it into their prompts.
|
|
const interfaceFile = path.join(fixture.home, 'fixture-interface.md');
|
|
expect(options.prompt).toContain(`saved at ${interfaceFile}: each child prompt tells the child to Read that file first`);
|
|
expect(options.prompt).not.toContain('include this interface in child prompts');
|
|
const saved = fs.readFileSync(interfaceFile, 'utf8');
|
|
expect(saved.startsWith('Fixture observation interface')).toBe(true);
|
|
expect(options.prompt.endsWith(saved.trimEnd())).toBe(true);
|
|
expect(fs.statSync(interfaceFile).mode & 0o777).toBe(0o600);
|
|
});
|
|
|
|
// The observer polices bash commands from the POSIX-only paid ship evals and refuses backslashes
|
|
// outright, so a Windows fixture path can never form an allowed insert; the refusals below still run.
|
|
test.skipIf(process.platform === 'win32')('declared section insert appends one private Markdown artifact to another byte-for-byte', () => {
|
|
const source = path.join(fixture.home, 'audit-1-documentation.md');
|
|
const target = path.join(fixture.home, 'ship-report.md');
|
|
const section = '**Status:** current — no edits.\n\n- Diagram drift: none.';
|
|
fs.writeFileSync(source, section);
|
|
fs.writeFileSync(target, '# Report\n\n## Documentation\n\n');
|
|
const before = repoSnapshot(fixture.repo);
|
|
for (const command of [`cat ${source} >> ${target}`, `cat '${source}' >> '${target}'`, `cat "${source}" >> "${target}"`]) {
|
|
expect(docsCommandAllowed(command, fixture)).toBe(true);
|
|
expect(docsToolFailures(nativeResult(command), fixture, [], true)).toEqual([]);
|
|
}
|
|
const result = spawnSync('bash', ['-c', `cat ${source} >> ${target}`], { cwd: fixture.repo, encoding: 'utf8', timeout: 10000 });
|
|
expect(result.status).toBe(0);
|
|
expect(fs.readFileSync(target, 'utf8')).toBe(`# Report\n\n## Documentation\n\n${section}`);
|
|
expect(repoSnapshot(fixture.repo)).toEqual(before);
|
|
});
|
|
|
|
test('section insert grants no write or source outside private fixture Markdown', () => {
|
|
const home = fixture.home;
|
|
const source = path.join(home, 'audit-1-documentation.md');
|
|
const target = path.join(home, 'ship-report.md');
|
|
fs.writeFileSync(source, 'section');
|
|
const link = path.join(home, 'linked-report.md');
|
|
fs.rmSync(link, { force: true });
|
|
fs.symlinkSync(path.join(fixture.repo, 'README.md'), link);
|
|
for (const command of [
|
|
`cat ${source} > ${target}`, `cat ${source} >> ${path.join(fixture.repo, 'README.md')}`,
|
|
`cat ${path.join(fixture.repo, 'README.md')} >> ${target}`, `cat ${source} >> ${link}`,
|
|
`cat ${source} >> ${path.join(home, 'report.txt')}`, `cat ${source} >> ${path.join(home, 'publication.json')}`,
|
|
`cat ${source} >> ${path.join(fixture.skills, 'ship/SKILL.md')}`, `cat ${source} >> ship-report.md`,
|
|
`cat ${source} ${source} >> ${target}`, `cat ${source} >> ${target}; ls`, `cat ${source} 2>> ${target}`,
|
|
`cat ${source} >> ${target} >> ${target}`, `cat ${path.join(home, '*.md')} >> ${target}`,
|
|
`cat ${path.join(home, '$x.md')} >> ${target}`, `cat ${source} >> ${path.join(home, '..', 'outside.md')}`, `cat ${source} >> ${home}/../outside.md`,
|
|
]) {
|
|
expect(docsCommandAllowed(command, fixture)).toBe(false);
|
|
expect(docsToolFailures(nativeResult(command), fixture, [], true)).toEqual(['command outside declared docs observation interface']);
|
|
}
|
|
});
|
|
|
|
test('only the native ship parent interface declares the section insert', () => {
|
|
const options = docsSessionOptions({ fixture, phase: path.join(fixture.home, 'phase.md'),
|
|
report: path.join(fixture.home, 'report.md'), publish: path.join(fixture.home, 'publish.ts'),
|
|
scenario: 'current', testName: 'docsync-command-grammar', runId: 'free-control', timeout: 10000 });
|
|
expect(options.prompt).toContain('cat SOURCE.md >> TARGET.md');
|
|
expect(options.prompt).toContain('.qa-state/ directory is the fixture owner');
|
|
for (const transport of [false, true]) expect(docsNativeInterface(fixture, [], transport)).not.toContain('>> TARGET.md');
|
|
});
|
|
|
|
test('the installed docs-candidate helper is admitted only with a private .json record, or for its own --help', () => {
|
|
const helper = path.join(fixture.skills, 'bin/gstack-docs-candidate');
|
|
const record = path.join(fixture.home, 'audit-1-candidate.json');
|
|
for (const command of [
|
|
`${helper} snapshot --out ${record} --audit-id audit-1 --mode edit --base main --docs handbook`,
|
|
`${helper} snapshot --out ${record} --audit-id audit-1 --mode read-only --base main --select app.ts --docs handbook`,
|
|
`${helper} compare ${record}`,
|
|
`${helper} --help`,
|
|
]) expect(docsCommandAllowed(command, fixture), command).toBe(true);
|
|
for (const command of [
|
|
`${helper} snapshot --out ${path.join(fixture.repo, 'candidate.json')} --audit-id a --mode edit --base main`,
|
|
`${helper} snapshot --out ${path.join(fixture.home, 'candidate.txt')} --audit-id a --mode edit --base main`,
|
|
`${helper} snapshot --out ${record} --out ${record} --audit-id a --mode edit --base main`,
|
|
`${helper} snapshot --audit-id a --mode edit --base main`,
|
|
`${helper} snapshot --out ${record} --audit-id a --mode edit --base main --exec rm`,
|
|
`${helper} compare ${path.join(fixture.skills, 'bin/x.json')}`,
|
|
`${helper} compare ${record} extra`,
|
|
`${helper} install`,
|
|
`${helper} --help snapshot`,
|
|
`${helper} --version`,
|
|
`~/.claude/skills/gstack/bin/gstack-docs-candidate compare ${record}`,
|
|
]) expect(docsCommandAllowed(command, fixture), command).toBe(false);
|
|
expect(docsNativeInterface(fixture)).toContain(`${fixture.skills.split(path.sep).join('/')}/bin/gstack-docs-candidate snapshot`);
|
|
expect(docsNativeInterface(fixture)).toContain('compare <that .json> and --help commands');
|
|
});
|
|
|
|
test('the installed lifecycle helpers are admitted for their own --help, which starts nothing', () => {
|
|
const skills = fixture.skills.split(path.sep).join('/');
|
|
for (const helper of ['gstack-skill-start', 'gstack-skill-end']) {
|
|
expect(docsCommandAllowed(`${skills}/bin/${helper} --help`, fixture)).toBe(true);
|
|
for (const command of [
|
|
`${skills}/bin/${helper} --help --skill review`,
|
|
`${skills}/bin/${helper} --skill review`,
|
|
`${skills}/bin/${helper} -h`,
|
|
`${skills}/bin/${helper} --help; ${skills}/bin/${helper} --skill document-release`,
|
|
`~/.claude/skills/gstack/bin/${helper} --help`,
|
|
]) expect(docsCommandAllowed(command, fixture), command).toBe(false);
|
|
}
|
|
expect(docsNativeInterface(fixture)).toContain('or their own --help');
|
|
});
|
|
|
|
// PR gate run 38969924945 (ship-docsync-store) went red only on `gstack-skill-start --help`.
|
|
test('every Bash command of the captured ship-docsync-store run is inside the interface', () => {
|
|
const capture = JSON.parse(fs.readFileSync(path.join(import.meta.dir, 'fixtures/docsync-store-38969924945-bash.json'), 'utf8'));
|
|
const commands: string[] = capture.commands.map((command: string) => command.replaceAll('<HOME>', fixture.home));
|
|
for (const file of ['docs-audit-20261016T2140Z-1-documentation.md', 'ship-report.md']) fs.writeFileSync(path.join(fixture.home, file), '');
|
|
const publish = path.join(fixture.home, 'fixture-publish.ts');
|
|
expect(commands).toHaveLength(54);
|
|
expect(commands.filter(command => !docsCommandAllowed(command, fixture, [publish]))).toEqual([]);
|
|
});
|
|
|
|
// PR gate run 38990111230 (ship-docsync-stale-after, head 6c07d85c82d17bda81860cd0d78616de3dc14cc2) went red only on
|
|
// `gstack-docs-candidate snapshot --help` (bash#8), which the helper rejected as an unknown argument. Every Bash
|
|
// command of the parent, in order; <HOME> is the fixture home and <ACTOR> the fault actor.
|
|
const STALE_AFTER_38990111230 = [
|
|
'<HOME>/.claude/skills/gstack/bin/gstack-docs-candidate compare <HOME>/candidate-ship-docs-a1.json',
|
|
'bun <ACTOR> inspect <HOME>/actor-state.json',
|
|
'sha256sum <HOME>/.claude/skills/gstack/document-release/SKILL.md <HOME>/.claude/skills/gstack/document-release/sections/audit-scope.md <HOME>/.claude/skills/gstack/document-release/sections/release-body.md',
|
|
'bun <ACTOR> prepare <HOME>/actor-state.json audit_id=ship-docs-a2',
|
|
'bun <ACTOR> dispatch <HOME>/actor-state.json audit_id=ship-docs-a2 candidate=<HOME>/candidate-ship-docs-a2.json prompt=<HOME>/prompt-ship-docs-a2.md run_in_background=false',
|
|
'bun <ACTOR> inspect <HOME>/actor-state.json',
|
|
'<HOME>/.claude/skills/gstack/bin/gstack-docs-candidate compare <HOME>/candidate-ship-docs-a2.json',
|
|
'<HOME>/.claude/skills/gstack/bin/gstack-docs-candidate snapshot --help',
|
|
'<HOME>/.claude/skills/gstack/bin/gstack-docs-candidate snapshot --out <HOME>/ship-docs-a2-post.json --audit-id ship-docs-a2 --mode edit --base 21cebf26714abdfc6b80267507b88a4df6e933a8 --docs handbook --select AGENTS.md --select CHANGELOG.md --select README.md --select TODOS.md --select VERSION --select app.ts --select handbook/reference/commands/widget.md.tmpl --select package.json',
|
|
'bun <ACTOR> inspect <HOME>/actor-state.json',
|
|
'bun <ACTOR> publish <HOME>/actor-state.json audit_id=ship-docs-a2 report=<HOME>/ship-report.md',
|
|
];
|
|
|
|
test('every Bash command of the captured ship-docsync-stale-after run is inside the fault interface', () => {
|
|
const actor = path.join(import.meta.dir, 'helpers/docsync-fault-actor.ts');
|
|
const commands = STALE_AFTER_38990111230.map(command => command.replaceAll('<HOME>', fixture.home).replaceAll('<ACTOR>', actor));
|
|
expect(commands).toHaveLength(11);
|
|
expect(commands.filter(command => !docsCommandAllowed(command, fixture, [actor]))).toEqual([]);
|
|
});
|
|
|
|
test('subcommand --help is admitted alone and runs without effect; with any other argument it stays outside the interface', () => {
|
|
const helper = path.join(fixture.skills, 'bin/gstack-docs-candidate');
|
|
const record = path.join(fixture.home, 'help-probe.json');
|
|
const before = repoSnapshot(fixture.repo);
|
|
for (const sub of ['snapshot', 'compare']) {
|
|
expect(docsCommandAllowed(`${helper} ${sub} --help`, fixture)).toBe(true);
|
|
const run = spawnSync(helper, [sub, '--help'], { cwd: fixture.repo, encoding: 'utf8', timeout: 20_000 });
|
|
expect(run.status, run.stderr).toBe(0);
|
|
expect(run.stdout).toContain('usage: gstack-docs-candidate snapshot --out <candidate.json>');
|
|
}
|
|
for (const command of [
|
|
`${helper} snapshot --help --out ${record}`,
|
|
`${helper} compare --help ${record}`,
|
|
`${helper} snapshot --out ${record} --help`,
|
|
`${helper} snapshot -h`,
|
|
`${helper} compare -h`,
|
|
`${helper} snapshot --help --out ${path.join(fixture.repo, 'inside.json')}`,
|
|
`${helper} snapshot --out ${path.join(fixture.repo, 'inside.json')} --audit-id a --mode edit --base main`,
|
|
`${helper} snapshot --help; ${helper} compare ${record}`,
|
|
]) expect(docsCommandAllowed(command, fixture), command).toBe(false);
|
|
expect(fs.existsSync(record)).toBe(false);
|
|
expect(repoSnapshot(fixture.repo)).toEqual(before);
|
|
});
|
|
|
|
// PR gate run 39153452209 (ship-docsync-store, head a844821583d664e3dea279fff12b935865e92f52) went red only on the
|
|
// document-release child's `git describe --tags --always` (bash#24). Every Bash command of the parent and its child,
|
|
// in order; <HOME> is the fixture home.
|
|
const STORE_39153452209 = [
|
|
'git status',
|
|
'git rev-parse HEAD',
|
|
'git ls-files',
|
|
'<HOME>/.claude/skills/gstack/bin/gstack-docs-candidate --help',
|
|
'git diff --cached',
|
|
'git diff',
|
|
'git branch --show-current',
|
|
'git log --oneline',
|
|
'git check-ignore -v .qa-state',
|
|
'ls -la <HOME>',
|
|
'<HOME>/.claude/skills/gstack/bin/gstack-skill-start --help',
|
|
'git ls-files -z --cached --others --exclude-standard',
|
|
'git show HEAD:handbook/reference/commands/widget.md.tmpl',
|
|
'ls -la <HOME>/state',
|
|
'git hash-object README.md handbook/reference/commands/widget.md.tmpl options.ts app.ts personal-note.txt',
|
|
'git log --oneline --decorate --all',
|
|
'<HOME>/.claude/skills/gstack/bin/gstack-docs-candidate snapshot --out <HOME>/ship-docs-20261018-c0048c5-a1.json --audit-id ship-docs-20261018-c0048c5-a1 --mode read-only --base c0048c54c542cf0ba46b7587cf7fea1b519c75e4 --select app.ts --select README.md --select options.ts --docs handbook --docs README.md --docs AGENTS.md --docs CHANGELOG.md --docs TODOS.md',
|
|
'GSTACK_SESSION_KIND=spawned <HOME>/.claude/skills/gstack/bin/gstack-skill-start --skill document-release --model claude',
|
|
'git status',
|
|
'git log --oneline',
|
|
'git diff c0048c54c542cf0ba46b7587cf7fea1b519c75e4 HEAD --stat',
|
|
'git diff --cached',
|
|
'git diff',
|
|
'git ls-files --cached --others --exclude-standard',
|
|
'git hash-object README.md options.ts app.ts personal-note.txt',
|
|
'git describe --tags --always',
|
|
'<HOME>/.claude/skills/gstack/bin/gstack-skill-end --skill document-release --outcome success --session-id 878-1792348356-d8b277ad --tel-start 1792348356 --used-browse no',
|
|
'<HOME>/.claude/skills/gstack/bin/gstack-docs-candidate compare <HOME>/ship-docs-20261018-c0048c5-a1.json',
|
|
'git status',
|
|
'git rev-parse HEAD',
|
|
'git hash-object README.md handbook/reference/commands/widget.md.tmpl options.ts app.ts personal-note.txt AGENTS.md CHANGELOG.md TODOS.md VERSION package.json',
|
|
'ls -la <HOME>',
|
|
'ls -la <HOME>/repo',
|
|
'cat <HOME>/ship-docs-20261018-c0048c5-a1-documentation.md >> <HOME>/ship-report.md',
|
|
'git status',
|
|
'ls -la <HOME>',
|
|
];
|
|
|
|
test('every Bash command of the captured run 39153452209 ship-docsync-store is inside the interface', () => {
|
|
const commands = STORE_39153452209.map(command => command.replaceAll('<HOME>', fixture.home));
|
|
for (const file of ['ship-docs-20261018-c0048c5-a1-documentation.md', 'ship-report.md']) fs.writeFileSync(path.join(fixture.home, file), '');
|
|
expect(commands).toHaveLength(36);
|
|
expect(commands).toContain('git describe --tags --always');
|
|
expect(commands.filter(command => !docsCommandAllowed(command, fixture))).toEqual([]);
|
|
expect(docsToolFailures(nativeResult('git describe --tags --always', 'c0048c5', 'docs-dispatch'), fixture, [], true)).toEqual([]);
|
|
});
|
|
|
|
const DESCRIBE_READS = ['git describe', 'git describe --tags', 'git describe --always', 'git describe --abbrev=7',
|
|
'git describe --exact-match', 'git describe HEAD', "git describe --tags --always --abbrev=12 'HEAD^{}'",
|
|
'git describe --exact-match --tags main', 'git describe --always main', 'git describe --tags --always 2>/dev/null'];
|
|
const DESCRIBE_REFUSED = ['git describe --dirty', 'git describe --dirty=-modified', 'git describe --always --dirty',
|
|
'git describe --broken', 'git describe --broken=-broken', 'git describe --contains HEAD', 'git describe --all',
|
|
'git describe --abbrev', 'git describe --abbrev=x', 'git describe HEAD main', 'git describe -- HEAD',
|
|
'git describe --tags > tag.md', 'git describe --tags; git tag v9', 'git describe --tags && git tag v9'];
|
|
|
|
test('read-only git describe forms are declared and leave the index and repository unchanged', () => {
|
|
expect(docsNativeInterface(fixture)).toContain('describe with only --tags, --always, --abbrev=<n> or --exact-match and at most one revision');
|
|
const own = fixtureDocs('store');
|
|
try {
|
|
gitAt(own.repo, 'tag', '-a', 'v0.1.0', '-m', 'fixture release', 'HEAD');
|
|
const stale = new Date(Date.now() - 3600_000);
|
|
fs.utimesSync(path.join(own.repo, 'AGENTS.md'), stale, stale);
|
|
const index = () => fs.readFileSync(path.join(own.repo, '.git/index')).toString('base64');
|
|
const before = { repo: repoSnapshot(own.repo), index: index() };
|
|
for (const command of DESCRIBE_READS) {
|
|
expect(docsCommandAllowed(command, own), command).toBe(true);
|
|
expect(docsToolFailures(nativeResult(command), own, [], true), command).toEqual([]);
|
|
const run = spawnSync('bash', ['-c', command], { cwd: own.repo, encoding: 'utf8', timeout: 10000 });
|
|
expect(run.status, `${command}: ${run.stderr}`).toBe(0);
|
|
expect({ repo: repoSnapshot(own.repo), index: index() }, command).toEqual(before);
|
|
}
|
|
} finally {
|
|
own.clean();
|
|
}
|
|
});
|
|
|
|
test('describe forms that refresh the index or fall outside the declared set, and Git writes, stay outside the interface', () => {
|
|
for (const command of [...DESCRIBE_REFUSED, 'git tag v9 HEAD', 'git update-index --refresh', 'git notes add -m x HEAD']) {
|
|
expect(docsCommandAllowed(command, fixture), command).toBe(false);
|
|
expect(docsToolFailures(nativeResult(command, 'successful tool acknowledgment', 'docs-dispatch'), fixture, [], true), command)
|
|
.toEqual(['command outside declared docs observation interface']);
|
|
}
|
|
const own = fixtureDocs('store');
|
|
try {
|
|
const stale = new Date(Date.now() - 3600_000);
|
|
fs.utimesSync(path.join(own.repo, 'AGENTS.md'), stale, stale);
|
|
const index = () => fs.readFileSync(path.join(own.repo, '.git/index')).toString('base64');
|
|
const before = index();
|
|
const run = spawnSync('git', ['describe', '--always', '--dirty'], { cwd: own.repo, encoding: 'utf8', timeout: 10000 });
|
|
expect(run.status, run.stderr).toBe(0);
|
|
expect(index()).not.toBe(before);
|
|
} finally {
|
|
own.clean();
|
|
}
|
|
});
|