mirror of
https://github.com/garrytan/gstack.git
synced 2026-10-31 14:06:50 +01:00
* fix(evals): local-hook detector credits 'set it up by hand'; read-only grammar steers globbed options to the quoted form Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> * chore: changelog and fixture touchfile for the 2026-10-21 paid flake repairs Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> * v1.91.174.0 stamp Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> --------- Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>
449 lines
29 KiB
TypeScript
449 lines
29 KiB
TypeScript
/**
|
|
* The shared read-only grammar (GSTA-49): accepted pure reads, a negative
|
|
* control for every rejected class, and free replays of the captured Opus
|
|
* slips through the opted-in docsync observer.
|
|
*/
|
|
import { afterAll, beforeAll, describe, expect, test } from 'bun:test';
|
|
import * as fs from 'node:fs';
|
|
import * as os from 'node:os';
|
|
import * as path from 'node:path';
|
|
import { spawnSync } from 'node:child_process';
|
|
import { fixtureDocs } from './helpers/docsync-fixture';
|
|
import { docsCommandAllowed, docsCompletedRead, docsNativeInterface, docsToolFailures } from './helpers/docsync-observer';
|
|
import { readOnlyCommandAllowed, readOnlyCommandRejection, readOnlyInterfaceText, READ_TOOLS, STDIN_TOOLS } from './helpers/read-only-commands';
|
|
import { assertQaBrowserDeadline, qaDeadlineShellPolicy } from './helpers/qa-browser-deadline-evidence';
|
|
import type { SkillTestResult } from './helpers/session-runner';
|
|
|
|
let base: string;
|
|
let root: string;
|
|
let outside: string;
|
|
const scope = () => ({ root, deny: (file: string) => path.basename(file) === 'credentials.json' });
|
|
|
|
beforeAll(() => {
|
|
if (process.platform === 'win32') return;
|
|
base = fs.mkdtempSync(path.join(fs.realpathSync(os.tmpdir()), 'ro-grammar-'));
|
|
root = path.join(base, 'root');
|
|
outside = path.join(base, 'outside');
|
|
for (const [file, text] of Object.entries({
|
|
'qa-only/SKILL.md': '# QA only\nstep one\nstep two\n',
|
|
'qa-only/sections/report.md': 'report\n',
|
|
'qa/sections/browser-setup.md': 'setup\n',
|
|
'qa/templates/report.md': 'template\n',
|
|
'docs/guide.md': 'b\na\nb\n',
|
|
'private/credentials.json': '{"token":"secret"}\n',
|
|
})) {
|
|
fs.mkdirSync(path.join(root, path.dirname(file)), { recursive: true });
|
|
fs.writeFileSync(path.join(root, file), text);
|
|
}
|
|
fs.mkdirSync(outside);
|
|
fs.writeFileSync(path.join(outside, 'secret.txt'), 'outside\n');
|
|
fs.symlinkSync(outside, path.join(root, 'escape'));
|
|
fs.symlinkSync(path.join(root, 'docs/guide.md'), path.join(root, 'guide-link.md'));
|
|
});
|
|
afterAll(() => { if (base) fs.rmSync(base, { recursive: true, force: true }); });
|
|
|
|
// Paid fixtures run on Linux; the grammar resolves POSIX paths only, so on Windows it accepts nothing.
|
|
test.if(process.platform === 'win32')('the grammar fails closed on Windows', () => {
|
|
const root = fs.realpathSync(os.tmpdir());
|
|
for (const command of ['pwd', 'ls', 'wc -l x']) expect(readOnlyCommandAllowed(command, { root })).toBe(false);
|
|
});
|
|
|
|
function tree(dir: string): Record<string, string> {
|
|
const out: Record<string, string> = {};
|
|
for (const name of fs.readdirSync(dir, { recursive: true }).map(String).sort()) {
|
|
const file = path.join(dir, name);
|
|
const entry = fs.lstatSync(file);
|
|
out[name] = entry.isFile() ? fs.readFileSync(file, 'base64') : entry.isSymbolicLink() ? 'link:' + fs.readlinkSync(file) : 'dir';
|
|
}
|
|
return out;
|
|
}
|
|
|
|
const ACCEPTED = [
|
|
'pwd', 'ls', 'ls -la docs', 'ls -la qa-only/*', 'ls qa-only qa-only/sections qa/sections qa/templates', 'ls -R qa',
|
|
'wc -l docs/guide.md', 'wc -l qa-only/SKILL.md qa-only/sections/*.md', 'head -n 2 docs/guide.md', 'head -2 qa-only/SKILL.md',
|
|
'tail -n +2 docs/guide.md', 'cat -n docs/guide.md', 'cat guide-link.md', 'stat docs/guide.md', 'stat -c %s docs/guide.md',
|
|
'file docs/guide.md', 'du -sh qa', 'du -d 1 .', 'find . -name "*.md" -type f', "find qa -maxdepth 2 -name '*.md' -print",
|
|
'find . -path ./private -prune -o -type f -print', 'grep -n step qa-only/SKILL.md', 'rg -n step qa-only', 'grep -rn template qa',
|
|
'grep -n -i "step\\|report" qa-only/SKILL.md', 'grep -c -e step -e report qa-only/SKILL.md', 'sort docs/guide.md',
|
|
'sort -u docs/guide.md', 'uniq -c docs/guide.md', 'cut -c1-3 docs/guide.md', 'nl -ba docs/guide.md', 'sha256sum docs/guide.md',
|
|
'md5sum docs/guide.md qa-only/SKILL.md', 'sha1sum docs/guide.md', 'sha512sum --tag docs/guide.md', 'ls qa | sha256sum', 'readlink -f guide-link.md', 'realpath docs/guide.md',
|
|
'cat docs/guide.md | sort | uniq -c', 'ls qa | wc -l', 'grep -rn step qa-only | head -5', 'find qa -type f | sort',
|
|
'wc -l qa-only/SKILL.md; ls qa-only', 'ls docs && wc -l docs/guide.md', 'ls missing || ls docs', 'cd qa && ls sections',
|
|
'cd qa-only; wc -l SKILL.md', 'ls docs;', 'echo ----', 'echo -n done', "printf '%s\\n' done", 'printf -- -x',
|
|
'cat docs/guide.md; echo ----; cat qa-only/SKILL.md', 'echo done | wc -c', 'sed -n 1,2p docs/guide.md', 'grep -n step qa-only/SKILL.md | sed -n 1,5p',
|
|
'ls docs 2>/dev/null', 'ls missing 2>/dev/null || ls docs', 'cat docs/guide.md 2>&1 | head -1', 'ls missing 2>&1 | wc -l',
|
|
"ls qa | tr '\\n' ' '", "cat docs/guide.md | tr -d '\\r'", "cat docs/guide.md | tr '[:upper:]' '[:lower:]' | sort", 'ls | tr -cs a-z x',
|
|
];
|
|
|
|
describe.skipIf(process.platform === 'win32')('accepted pure local reads', () => {
|
|
test('every listed tool has at least one accepted form', () => {
|
|
for (const tool of Object.keys(READ_TOOLS)) expect(ACCEPTED.some(command => command.split(/[;&|]+\s*/).some(part => part.trim().split(' ')[0] === tool)), tool).toBe(true);
|
|
});
|
|
|
|
test.each(ACCEPTED)('accepts and executes without changing either tree: %s', command => {
|
|
expect(readOnlyCommandRejection(command, scope())).toBeUndefined();
|
|
const before = [tree(root), tree(outside)];
|
|
const run = spawnSync('bash', ['-c', command], { cwd: root, encoding: 'utf8', timeout: 10000 });
|
|
expect(run.error).toBeUndefined();
|
|
expect([tree(root), tree(outside)]).toEqual(before);
|
|
});
|
|
|
|
test('the declared interface text names every tool and the pipe targets', () => {
|
|
const text = readOnlyInterfaceText(root);
|
|
for (const tool of [...Object.keys(READ_TOOLS), ...STDIN_TOOLS]) expect(text).toContain(tool);
|
|
expect(text).toContain(root);
|
|
});
|
|
});
|
|
|
|
// One negative control per rejected class; the reason pins which rule fired.
|
|
const REJECTED: Array<[string, string, RegExp]> = [
|
|
['write', 'rm docs/guide.md', /unlisted tool: rm/],
|
|
['write', 'touch docs/new.md', /unlisted tool: touch/],
|
|
['write', 'sort -o docs/sorted.md docs/guide.md', /unsupported sort option: -o/],
|
|
['write', 'uniq docs/guide.md docs/out.md', /at most 1 operand/],
|
|
['write', 'find . -name "*.md" -delete', /unsupported find primary: -delete/],
|
|
['write', "find . -exec rm '{}' ';'", /unsupported find primary: -exec/],
|
|
['write', "find . -ok cat '{}' ';'", /unsupported find primary: -ok/],
|
|
['write', 'find . -fprint list.txt', /unsupported find primary: -fprint/],
|
|
['write', 'find . -fprintf list.txt %p', /unsupported find primary: -fprintf/],
|
|
['redirect', 'cat docs/guide.md > copy.md', /redirect/],
|
|
['redirect', 'cat docs/guide.md >> copy.md', /redirect/],
|
|
['redirect', 'ls docs 2>docs/err.txt', /redirect/],
|
|
['redirect', 'ls docs 2> /dev/null', /redirect/],
|
|
['redirect', 'ls docs 1>/dev/null', /redirect/],
|
|
['redirect', 'ls docs 3>/dev/null', /redirect/],
|
|
['redirect', 'ls docs >/dev/null', /redirect/],
|
|
['redirect', 'ls docs >>/dev/null', /redirect/],
|
|
['redirect', 'ls docs &>/dev/null', /background job or unsupported operator/],
|
|
['redirect', 'ls docs >&docs/out.txt', /redirect/],
|
|
['redirect', 'ls docs 2>&docs/out.txt', /redirect/],
|
|
['redirect', 'ls docs 2>&2', /redirect/],
|
|
['redirect', 'ls docs 2>/dev/tcp/example.com/80', /redirect/],
|
|
['redirect', 'ls docs 2>/dev/nullx', /redirect/],
|
|
['redirect', '2>/dev/null ls docs', /redirect/],
|
|
['redirect', "ls docs '2'>/dev/null", /redirect/],
|
|
['redirect', 'ls docs 2>/dev/null > docs/out.txt', /redirect/],
|
|
['writing pipe', 'ls docs 2>&1 | tee docs/out.txt', /unlisted tool: tee/],
|
|
['redirect', 'wc -l < docs/guide.md', /redirect/],
|
|
['writing pipe', 'cat docs/guide.md | tee copy.md', /unlisted tool: tee/],
|
|
['writing pipe', 'ls | xargs rm', /unlisted tool: xargs/],
|
|
['writing pipe', 'cat docs/guide.md | sh', /unlisted tool: sh/],
|
|
['writing pipe', 'cat docs/guide.md | sort -o copy.md', /unsupported sort option: -o/],
|
|
['writing pipe', 'ls | find . -delete', /find cannot read a pipe/],
|
|
['substitution', 'cat $(ls docs)', /substitution or variable/],
|
|
['substitution', 'cat "$(touch x)"', /substitution or variable/],
|
|
['substitution', 'cat ' + '\x60' + 'ls' + '\x60', /substitution or variable/],
|
|
['substitution', 'cat <(ls)', /redirect/],
|
|
['variable', 'ls $HOME', /substitution or variable/],
|
|
['network', 'curl https://example.com', /unlisted tool: curl/],
|
|
['network', 'wget -q https://example.com', /unlisted tool: wget/],
|
|
['network', 'nc example.com 80', /unlisted tool: nc/],
|
|
['network', 'git fetch origin', /unlisted tool: git/],
|
|
['cd out of root', 'cd /tmp && ls', /cd outside root/],
|
|
['cd out of root', 'cd .. && ls', /cd outside root/],
|
|
['cd out of root', 'cd escape && cat secret.txt', /cd outside root/],
|
|
['cd out of root', 'cd ~ && ls', /tilde/],
|
|
['cd out of root', 'cd - && ls', /cd needs exactly one literal directory/],
|
|
['path escape: ..', 'cat ../outside/secret.txt', /path outside root/],
|
|
['path escape: ..', 'cat docs/../../outside/secret.txt', /path outside root/],
|
|
['path escape: ..', 'grep -rn outside ..', /path outside root/],
|
|
['path escape: symlink', 'cat escape/secret.txt', /path outside root/],
|
|
['path escape: symlink', 'ls escape/*', /glob outside root/],
|
|
['path escape: symlink', 'wc -l escape', /path outside root/],
|
|
['path escape: absolute', 'cat /etc/passwd', /path outside root/],
|
|
['path escape: absolute', 'ls /*', /glob outside root/],
|
|
['path escape: absolute', 'find / -name passwd', /path outside root/],
|
|
['path escape: option value', 'grep -f /etc/passwd docs/guide.md', /path outside root/],
|
|
['path escape: option value', 'find docs -newer /etc/passwd', /path outside root/],
|
|
['denied path', 'cat private/credentials.json', /denied path/],
|
|
['denied path', 'grep -rn token .', /recursive search reaches a denied path/],
|
|
['denied path', 'ls private/*', /denied path/],
|
|
['symlink following', 'grep -R step .', /unsupported grep option: -R/],
|
|
['symlink following', 'find -L . -name x', /unsupported find primary: -L/],
|
|
['symlink following', 'du -L .', /unsupported du option: -L/],
|
|
['external program', 'rg --pre cat step', /unsupported rg option: --pre/],
|
|
['external program', 'rg -z step', /unsupported rg option: -z/],
|
|
['sudo', 'sudo cat docs/guide.md', /unlisted tool: sudo/],
|
|
['unlisted tool', 'python3 -c "print(1)"', /unlisted tool: python3/],
|
|
['unlisted tool', 'od -c docs/guide.md', /unlisted tool: od/],
|
|
['sed', 'sed -i s/a/b/ docs/guide.md', /sed takes only -n and a line-range print script/],
|
|
['sed', 'sed -n 1w docs/out.txt docs/guide.md', /sed takes only -n and a line-range print script/],
|
|
['sed', 'sed -n -e 1p docs/guide.md', /sed takes only -n and a line-range print script/],
|
|
['sed', "sed -n '1p;w docs/out.txt' docs/guide.md", /sed takes only -n and a line-range print script/],
|
|
['path escape', 'sed -n 1p escape/secret.txt', /outside root/],
|
|
['unlisted tool', 'awk 1 docs/guide.md', /unlisted tool: awk/],
|
|
['tr', 'tr a-z A-Z', /tr reads only a pipe/],
|
|
['tr', 'tr a-z A-Z docs/guide.md', /tr reads only a pipe/],
|
|
['tr', 'cat docs/guide.md | tr a-z A-Z docs/guide.md', /tr takes one or two literal sets/],
|
|
['tr', 'cat docs/guide.md | tr a-z A-Z > docs/out.md', /redirect/],
|
|
['tr', 'cat docs/guide.md | tr a-z A-Z | sh', /unlisted tool: sh/],
|
|
['tr', "cat docs/guide.md | tr '$X' y", /tr sets must be literal/],
|
|
['tr', 'cat docs/guide.md | tr "$X" y', /substitution or variable/],
|
|
['tr', "cat docs/guide.md | tr '" + '\x60' + "id" + '\x60' + "' y", /tr sets must be literal/],
|
|
['tr', 'cat docs/guide.md | tr [a] b', /tr sets must be literal/],
|
|
['tr', "cat docs/guide.md | tr '{a,b}' c", /tr sets must be literal/],
|
|
['tr', 'cat docs/guide.md | tr -x a b', /unsupported tr option: -x/],
|
|
['tr', 'cat docs/guide.md | tr', /tr takes one or two literal sets/],
|
|
['echo/printf', 'echo hi > docs/new.md', /redirect/],
|
|
['echo/printf', 'echo hi >> docs/guide.md', /redirect/],
|
|
['echo/printf', 'echo $(touch x)', /substitution or variable/],
|
|
['echo/printf', 'echo ' + '\x60' + 'touch x' + '\x60', /substitution or variable/],
|
|
['echo/printf', 'echo $X', /substitution or variable/],
|
|
['echo/printf', 'echo ${X}', /substitution or variable/],
|
|
['echo/printf', 'echo "$X"', /substitution or variable/],
|
|
['echo/printf', 'printf -v X hi', /unsupported printf option: -v/],
|
|
['echo/printf', 'echo *', /unquoted glob in literal arguments/],
|
|
['echo/printf', 'echo hi | tee docs/new.md', /unlisted tool: tee/],
|
|
['unlisted tool', 'env cat docs/guide.md', /unlisted tool: env/],
|
|
['unlisted tool', '/bin/cat docs/guide.md', /unlisted tool/],
|
|
['shell syntax', 'ls &', /background job/],
|
|
['shell syntax', 'ls\nrm docs/guide.md', /multi-line/],
|
|
['shell syntax', 'cat docs/{guide,other}.md', /brace expansion/],
|
|
['shell syntax', '(ls)', /subshell/],
|
|
['shell syntax', 'tail -f docs/guide.md', /unsupported tail option: -f/],
|
|
];
|
|
|
|
describe.skipIf(process.platform === 'win32')('rejected classes stay rejected', () => {
|
|
test.each(REJECTED)('%s: %s', (_, command, reason) => {
|
|
expect(readOnlyCommandRejection(command, scope())).toMatch(reason);
|
|
expect(readOnlyCommandAllowed(command, scope())).toBe(false);
|
|
});
|
|
|
|
test('every class the brief names has a control', () => {
|
|
const classes = new Set(REJECTED.map(([kind]) => kind.replace(/:.*/, '')));
|
|
for (const kind of ['write', 'redirect', 'writing pipe', 'substitution', 'network', 'cd out of root', 'path escape', 'sudo', 'unlisted tool', 'echo/printf'])
|
|
expect(classes.has(kind), kind).toBe(true);
|
|
});
|
|
});
|
|
|
|
function nativeResult(commands: string[]): SkillTestResult {
|
|
return { toolCalls: commands.map(command => ({ tool: 'Bash', input: { command }, output: '' })) } as unknown as SkillTestResult;
|
|
}
|
|
|
|
// Captured Opus slips. Fixture paths are templated; {home} is the docsync fixture home.
|
|
const DOCSYNC_SLIPS = [
|
|
// PR #3275 gate 39173752116, slice 13, ship-docsync-missing-asset: its whole Bash stream.
|
|
['wc -l {home}/.claude/skills/gstack/ship/SKILL.md',
|
|
'ls -la {home}/.claude/skills/gstack/document-release/sections {home}'],
|
|
// TM1 effort study at e616a75, ship-docsync-missing-asset t10 (high effort): its Bash stream before the actor call.
|
|
['ls -la {home} {home}/.claude/skills/gstack {home}/.claude/skills/gstack/ship',
|
|
'ls -la {home}/.claude/skills/gstack/ship/sections {home}/.claude/skills/gstack/document-release {home}/.claude/skills/gstack/document-release/*'],
|
|
];
|
|
// PR #3275 E2E run 39181697029, slice 3, ship-docsync-current: its whole Bash stream; the composed
|
|
// cat; echo ----; cat was the only command outside the docsync interface.
|
|
const DOCSYNC_CURRENT_STREAM = [
|
|
"ls {home}/.claude/skills/gstack/document-release {home}/.claude/skills/gstack/document-release/sections",
|
|
"git status",
|
|
"git merge-base main HEAD",
|
|
"git ls-files",
|
|
"{home}/.claude/skills/gstack/bin/gstack-docs-candidate --help",
|
|
"git diff 40d5e4c2069e23a19017c25494636fb8d0bd4c00 HEAD",
|
|
"cat personal-note.txt",
|
|
"git check-ignore -v .qa-state",
|
|
"{home}/.claude/skills/gstack/bin/gstack-docs-candidate snapshot --out {home}/docs-audit-20261019-r2a1.json --audit-id docs-audit-20261019-r2a1 --mode edit --base 40d5e4c2069e23a19017c25494636fb8d0bd4c00 --docs handbook --docs README.md --docs AGENTS.md --docs CHANGELOG.md --docs TODOS.md",
|
|
"cat {home}/fixture-interface.md; echo ----; cat {home}/docs-audit-20261019-r2a1.json",
|
|
"GSTACK_SESSION_KIND=spawned {home}/.claude/skills/gstack/bin/gstack-skill-start --skill document-release --model claude",
|
|
"git status",
|
|
"git rev-parse HEAD",
|
|
"git diff 40d5e4c2069e23a19017c25494636fb8d0bd4c00 HEAD",
|
|
"git diff --cached",
|
|
"git diff",
|
|
"git ls-files -z --cached --others --exclude-standard",
|
|
"git log 40d5e4c2069e23a19017c25494636fb8d0bd4c00..HEAD --oneline",
|
|
"git ls-files -s",
|
|
"git hash-object personal-note.txt",
|
|
"git check-ignore -v .qa-state",
|
|
"{home}/.claude/skills/gstack/bin/gstack-skill-end --skill document-release --outcome success --session-id 825-1792371119-d40648a8 --tel-start 1792371119 --used-browse no",
|
|
"{home}/.claude/skills/gstack/bin/gstack-docs-candidate compare {home}/docs-audit-20261019-r2a1.json",
|
|
"git status",
|
|
"git rev-parse HEAD",
|
|
"cat app.ts",
|
|
"cat handbook/reference/commands/widget.md.tmpl",
|
|
"cat README.md",
|
|
"cat personal-note.txt",
|
|
"{home}/.claude/skills/gstack/bin/gstack-docs-candidate snapshot --out {home}/docs-audit-20261019-r2a1-post.json --audit-id docs-audit-20261019-r2a1 --mode edit --base 40d5e4c2069e23a19017c25494636fb8d0bd4c00 --docs handbook --docs README.md --docs AGENTS.md --docs CHANGELOG.md --docs TODOS.md",
|
|
"cat {home}/docs-audit-20261019-r2a1-documentation.md >> {home}/ship-report.md",
|
|
"bun {home}/fixture-publish.ts",
|
|
];
|
|
|
|
// TM1 focused qa-only-no-fix at e03bfeb: the first Bash call; {dir} is the qa-only fixture root.
|
|
const QA_ONLY_SLIP = 'wc -l {dir}/qa-only/SKILL.md; ls {dir}/qa-only {dir}/qa-only/sections {dir}/qa/sections {dir}/qa/templates';
|
|
|
|
describe.skipIf(process.platform === 'win32')('free replays of the captured slips', () => {
|
|
let fixture: ReturnType<typeof fixtureDocs>;
|
|
beforeAll(() => { fixture = fixtureDocs('current'); });
|
|
afterAll(() => fixture?.clean());
|
|
|
|
test.each(DOCSYNC_SLIPS.map((stream, index) => [index, stream] as const))('docsync stream %i now passes the observer with no command failure', (_, stream) => {
|
|
const commands = stream.map(command => command.replaceAll('{home}', fixture.home));
|
|
expect(docsToolFailures(nativeResult(commands), fixture, [], true)).toEqual([]);
|
|
for (const command of commands) expect(docsCommandAllowed(command, fixture), command).toBe(true);
|
|
});
|
|
|
|
test('the ship-docsync-current stream of 39181697029 passes the observer with no command failure', () => {
|
|
const current = fixtureDocs('current');
|
|
try {
|
|
const publish = path.join(current.home, 'fixture-publish.ts');
|
|
const commands = DOCSYNC_CURRENT_STREAM.map(command => command.replaceAll('{home}', current.home));
|
|
expect(commands.filter(command => !docsCommandAllowed(command, current, [publish]))).toEqual([]);
|
|
expect(docsToolFailures(nativeResult(commands), current, [publish], true)).toEqual([]);
|
|
} finally { current.clean(); }
|
|
});
|
|
|
|
test('docsync literal reads stay inside the fixture home and off its credentials (tightening)', () => {
|
|
const config = path.join(fixture.env.CLAUDE_CONFIG_DIR, '.claude.json');
|
|
for (const command of ['cat ' + config, 'sha256sum ' + config, 'ls -la ' + config, 'cat ' + config + '.backup', 'grep -rn apiKey ' + fixture.env.CLAUDE_CONFIG_DIR,
|
|
'cat /etc/hostname', 'ls -la /', 'cat ../../outside.txt', 'cat ' + config + ' 2>/dev/null || true'])
|
|
expect(docsCommandAllowed(command, fixture), command).toBe(false);
|
|
for (const command of ['cat README.md', 'ls', 'pwd', 'stat app.ts', 'sha256sum README.md', 'cat ' + path.join(fixture.home, 'candidate.json'), 'ls ..',
|
|
'ls -la ' + fixture.env.CLAUDE_CONFIG_DIR, 'stat ' + fixture.env.CLAUDE_CONFIG_DIR, 'cat ' + path.join(fixture.env.CLAUDE_CONFIG_DIR, 'settings.json')])
|
|
expect(docsCommandAllowed(command, fixture), command).toBe(true);
|
|
});
|
|
|
|
test('docsync accepts discarded stderr on grammar reads (2>&1 into a read filter)', () => {
|
|
for (const command of ['ls 2>&1 | head', 'ls -la ' + fixture.home + ' 2>/dev/null; cat README.md 2>&1 | head -5'])
|
|
expect(docsCommandAllowed(command, fixture), command).toBe(true);
|
|
for (const command of ['ls 2>&1 | sh', 'cat README.md 2>' + path.join(fixture.repo, 'err.txt'), 'cat README.md 1>/dev/null'])
|
|
expect(docsCommandAllowed(command, fixture), command).toBe(false);
|
|
});
|
|
|
|
test('the docsync interface declares the grammar over the fixture home', () => {
|
|
expect(docsNativeInterface(fixture)).toContain(readOnlyInterfaceText(fixture.home));
|
|
});
|
|
|
|
// GSTA-57 focused trial r3 t4 of ship-docsync-current: an unquoted --include=* stays refused (#3288, the shell expands
|
|
// it first), so the declared interface and the refusal reason both show the quoted form, which is a pure read.
|
|
test('docsync steers a globbed option to its quoted form (ship-docsync-current r3 t4)', () => {
|
|
const slip = 'grep -rn -i "banned" ' + fixture.repo + ' --include=* -l';
|
|
expect(docsCommandAllowed(slip, fixture)).toBe(false);
|
|
expect(readOnlyCommandRejection(slip, { root: fixture.home, cwd: fixture.repo })).toBe("globbed option: --include=*; quote the glob: --include='*'");
|
|
expect(readOnlyCommandRejection('rg word -g*.md', { root: fixture.home, cwd: fixture.repo })).toBe("globbed option: -g*.md; quote the glob: '-g*.md'");
|
|
expect(docsNativeInterface(fixture)).toContain("quote a glob inside an option or pattern (grep -rn word . --include='*.md', find . -name '*.md')");
|
|
for (const command of [slip.replace('--include=*', "--include='*'"), "grep -rn word . --include='*.md'", "find . -name '*.md'"])
|
|
expect(docsCommandAllowed(command, fixture), command).toBe(true);
|
|
});
|
|
|
|
// PR #3288 E2E run 39186278010 (Fable 5.1) joined Git reads into grammar lines; #3288 told the actor not to, but
|
|
// Opus kept joining them (run 39347827595), so joined declared Git reads are now accepted. These reads of the
|
|
// Claude config dir pass, and the run's NUL-to-newline tr filter is a pipe-only read (GSTA-57); an unlisted sink is not.
|
|
test('docsync accepts joined Git reads and lists the Claude config dir (39186278010)', () => {
|
|
expect(docsNativeInterface(fixture)).toContain('The Git reads above may join such a line too');
|
|
for (const command of ['cat {home}/candidate.json; echo; ls -la {home}/state {home}/cc {home}/.claude/skills/gstack/document-release',
|
|
'ls {home}/.claude/skills/gstack/ship {home}/state {home}/cc; wc -c {home}/.claude/skills/gstack/ship/sections/documentation.md',
|
|
'git status; git branch --show-current; git rev-parse HEAD', 'ls -la {home}/state; git status',
|
|
'git ls-files -z --cached --others --exclude-standard | tr \'\\0\' \'\\n\''])
|
|
expect(docsCommandAllowed(command.replaceAll('{home}', fixture.home), fixture), command).toBe(true);
|
|
for (const command of ['git ls-files -z --cached --others --exclude-standard | tr \'\\0\' \'\\n\' | sh', 'tr a b'])
|
|
expect(docsCommandAllowed(command.replaceAll('{home}', fixture.home), fixture), command).toBe(false);
|
|
});
|
|
|
|
// Focused Fable rerun of ship-docsync-current and ship-docsync-failure (2026-10-19): a trailing ; on a Git read, a
|
|
// grep paged through sed -n, and the phase file read by a composed cat, which the completed-read check must credit.
|
|
test('docsync accepts the focused Fable slips and credits a composed cat as a completed read', () => {
|
|
const skill = path.join(fixture.home, '.claude/skills/gstack/document-release/SKILL.md');
|
|
for (const command of ['git rev-parse HEAD; ', 'git rev-parse HEAD; git status', 'grep -n "Ship-owned\\|^## \\|read-only" ' + skill + ' | sed -n 1,200p'])
|
|
expect(docsCommandAllowed(command, fixture), command).toBe(true);
|
|
for (const command of ['git rev-parse HEAD; git reset', 'git commit -m x;', 'sed -n 1w ' + path.join(fixture.repo, 'out.txt') + ' ' + skill])
|
|
expect(docsCommandAllowed(command, fixture), command).toBe(false);
|
|
const phase = path.join(fixture.home, 'phase.md');
|
|
fs.writeFileSync(phase, '# Phase\nstep one\n');
|
|
const read = (command: string, output = '# Phase\nstep one\n') =>
|
|
docsCompletedRead({ toolCalls: [{ tool: 'Bash', input: { command }, output }] } as unknown as SkillTestResult, phase, fixture);
|
|
const invocation = path.join(fixture.home, 'ship-invocation.md');
|
|
expect(read('cat ' + invocation + '; echo ----; cat ' + phase + '; echo ----; ls -la ' + fixture.home)).toBe(true);
|
|
expect(read('cat ' + phase + ' | head -1')).toBe(false);
|
|
expect(read('cd ' + fixture.repo + ' && cat ../phase.md')).toBe(false);
|
|
expect(read('cat ' + phase + '; echo ----', '# Phase\n')).toBe(false);
|
|
expect(read('cat ' + phase + ' > ' + path.join(fixture.home, 'copy.md'))).toBe(false);
|
|
});
|
|
|
|
test('docsync keeps its git interface and denies credentials and actor state', () => {
|
|
for (const command of [
|
|
'git status; rm app.ts', 'cd repo && git fetch', 'git log | sh',
|
|
'wc -l ' + path.join(fixture.home, 'cc/.claude.json'),
|
|
'grep -rn apiKey ' + fixture.home,
|
|
'head ' + path.join(fixture.home, 'actor-state.json'),
|
|
'wc -l /etc/passwd',
|
|
'wc -l ' + path.join(fixture.home, '.claude/skills/gstack/ship/SKILL.md') + ' > ' + path.join(fixture.repo, 'count.txt'),
|
|
]) expect(docsCommandAllowed(command, fixture), command).toBe(false);
|
|
});
|
|
|
|
test('the qa-only slip is a pure read inside its fixture root', () => {
|
|
expect(readOnlyCommandRejection(QA_ONLY_SLIP.replaceAll('{dir}', root), scope())).toBeUndefined();
|
|
expect(readOnlyCommandRejection(QA_ONLY_SLIP.replaceAll('{dir}', root) + ' > ' + path.join(root, 'out.txt'), scope())).toMatch(/redirect/);
|
|
});
|
|
});
|
|
|
|
// qa-only-no-fix opts in through its deadline fixture: replay the captured slip inside a real guarded attempt.
|
|
describe.skipIf(process.platform === 'win32')('qa-only deadline fixture opt-in', () => {
|
|
const guard = path.join(path.resolve(import.meta.dir, '..'), 'bin/gstack-qa-deadline');
|
|
const browse = process.execPath;
|
|
const quote = (value: string) => "'" + value.replaceAll("'", "'\"'\"'") + "'";
|
|
function attempt(slip: string, slipOutput?: string) {
|
|
const started = Date.now();
|
|
const directory = fs.mkdtempSync(path.join(fs.realpathSync(os.tmpdir()), 'qa-only-ro-'));
|
|
for (const dir of ['qa/sections', 'qa/templates', 'qa-only/sections', 'qa-reports/screenshots']) fs.mkdirSync(path.join(directory, dir), { recursive: true });
|
|
fs.copyFileSync(path.join(path.resolve(import.meta.dir, '..'), 'qa/sections/browser-setup.md'), path.join(directory, 'qa/sections/browser-setup.md'));
|
|
fs.writeFileSync(path.join(directory, 'qa-only/SKILL.md'), '# qa-only\n');
|
|
const policy = qaDeadlineShellPolicy(directory, guard, browse);
|
|
const calls: Array<{ tool: string; input: any; output: string }> = [];
|
|
const bash = (command: string, output?: string) => {
|
|
const result = spawnSync('bash', ['-c', command], { cwd: directory, encoding: 'utf8', timeout: 10000 });
|
|
expect(result.error).toBeUndefined();
|
|
calls.push({ tool: 'Bash', input: { command }, output: output ?? result.stdout + result.stderr });
|
|
};
|
|
bash(['bun', guard, 'start', policy.file, '30'].map(quote).join(' '));
|
|
bash(slip.replaceAll('{dir}', directory), slipOutput);
|
|
bash(['bun', guard, 'run', policy.file, '--', browse, '--version'].map(quote).join(' '));
|
|
const check = () => assertQaBrowserDeadline(calls, { directory, guard, browse, started, ended: Date.now() });
|
|
return { directory, policy, check, cleanup: () => fs.rmSync(directory, { recursive: true, force: true }) };
|
|
}
|
|
|
|
// Focused Opus 5.5 qa-only-no-fix at 3766eda + this opt-in: the only rejected call discarded stderr.
|
|
test('the captured ls; ls; cat 2>/dev/null | head slip passes the deadline assertion', () => {
|
|
const run = attempt('ls; ls qa/templates; cat TODOS.md 2>/dev/null | head -20');
|
|
try { expect(run.check()).toMatchObject({ launchedRuns: 1 }); } finally { run.cleanup(); }
|
|
});
|
|
|
|
test('the captured wc -l; ls slip passes the deadline assertion and the prompt declares the grammar', () => {
|
|
const run = attempt(QA_ONLY_SLIP);
|
|
try {
|
|
expect(run.check()).toMatchObject({ launchedRuns: 1 });
|
|
expect(run.policy.prompt).toContain(readOnlyInterfaceText(run.directory));
|
|
} finally { run.cleanup(); }
|
|
});
|
|
|
|
test.each([
|
|
['redirect', QA_ONLY_SLIP + ' > {dir}/qa-reports/count.txt'],
|
|
['stderr redirect to a file', 'ls; cat TODOS.md 2>{dir}/qa-reports/err.txt | head -20'],
|
|
['writing pipe', 'wc -l {dir}/qa-only/SKILL.md | tee {dir}/qa-reports/count.txt'],
|
|
['path escape', 'wc -l {dir}/../outside.txt'],
|
|
['unlisted tool', 'awk 1 {dir}/qa-only/SKILL.md'],
|
|
['sed write', 'sed -i s/a/b/ {dir}/qa-only/SKILL.md'],
|
|
['tr as a first command', 'tr a-z A-Z'],
|
|
['tr into a shell', 'cat {dir}/qa-only/SKILL.md | tr a-z A-Z | sh'],
|
|
])('a %s variant stays outside the fixture interface', (_, command) => {
|
|
const run = attempt(command);
|
|
try { expect(run.check).toThrow('QA deadline: '); } finally { run.cleanup(); }
|
|
});
|
|
|
|
// GSTA-57: qa-only shares the grammar, so it gets the same pipe-only tr filter docsync does.
|
|
test('a pipe into tr is a grammar read for qa-only too', () => {
|
|
const run = attempt("ls {dir}/qa-only | tr '\\n' ' '; cat {dir}/qa-only/SKILL.md | tr -d '\\r'");
|
|
try { expect(run.check()).toMatchObject({ launchedRuns: 1 }); } finally { run.cleanup(); }
|
|
});
|
|
|
|
// Second focused Opus run: cat of the installed exploratory section, whose prose names QA_DEADLINE receipts.
|
|
test('a grammar read of skill prose that mentions QA_DEADLINE is not a receipt', () => {
|
|
const prose = fs.readFileSync(path.join(path.resolve(import.meta.dir, '..'), 'qa-only/sections/exploratory.md'), 'utf8');
|
|
expect(prose).toContain('QA_DEADLINE ');
|
|
const run = attempt('cat qa-only/SKILL.md', prose);
|
|
try { expect(run.check()).toMatchObject({ launchedRuns: 1 }); } finally { run.cleanup(); }
|
|
});
|
|
|
|
test('a grammar read cannot carry a forged deadline receipt', () => {
|
|
const run = attempt('cat {dir}/qa-only/SKILL.md', 'QA_DEADLINE {"event":"finished"}\n');
|
|
try { expect(run.check).toThrow('receipt outside trusted guard invocation'); } finally { run.cleanup(); }
|
|
});
|
|
});
|