/** * CEO-12 / DX-5 / ENG-8 / ENG-14: free text never enters a shell command. * * 1. Every allowlisted identifier grammar rejects the characters that would * let a value escape its quoting, and every entry is used somewhere. * 2. Hostile payloads run through the real generated posting blocks * (Greptile replies, test-failure issues, /spec filing and archive) with * stub gh/glab: the text arrives byte for byte and nothing in it runs. * 3. A block whose file was never written refuses with the path and the * manual command, and sends nothing. */ import { afterAll, describe, expect, test } from 'bun:test'; import { spawnSync } from 'child_process'; import * as fs from 'fs'; import * as os from 'os'; import * as path from 'path'; import { IDENTIFIER_PLACEHOLDERS } from './helpers/placeholder-allowlist'; import { generateTestFailureTriage } from '../scripts/resolvers/preamble'; import { freeTextFileBash } from '../scripts/resolvers/free-text-file'; const ROOT = path.resolve(import.meta.dir, '..'); const tmp = fs.mkdtempSync(path.join(os.tmpdir(), 'gstack-free-text-')); afterAll(() => fs.rmSync(tmp, { recursive: true, force: true })); const QUOTE_BREAKERS = ["it's", 'say "hi"', 'a`id`b', 'a$(id)b', '$HOME', 'a\\b', 'line\nline']; const UNQUOTED_BREAKERS = ['a;b', 'a|b', 'a&b', 'a(b)', 'ab', 'a?b', 'a[b]', 'a{b}', 'a!b', 'a#b']; function bashFences(text: string): string[] { const out: string[] = []; const lines = text.split('\n'); for (let i = 0; i < lines.length; i++) { const open = lines[i].match(/^(\s*)(`{3,})(.*)$/); if (!open) continue; let end = i + 1; while (end < lines.length && !new RegExp(`^\\s*\`{${open[2].length},}\\s*$`).test(lines[end])) end++; if (open[3].trim() === 'bash') out.push(lines.slice(i + 1, end).map(l => l.slice(open[1].length)).join('\n')); i = end; } return out; } const fence = (text: string, marker: string) => { const found = bashFences(text).filter(f => f.includes(marker)); if (found.length !== 1) throw new Error(`expected one fence containing ${marker}, found ${found.length}`); return found[0]; }; /** A git repo with stub gh/glab/gstack binaries that record what they were given. */ function sandbox(name: string) { const dir = fs.mkdtempSync(path.join(tmp, `${name}-`)); const repo = path.join(dir, 'repo'); const bin = path.join(dir, 'bin'); const log = path.join(dir, 'log'); const runtime = path.join(dir, 'runtime'); for (const d of [repo, bin, log, path.join(runtime, 'bin'), path.join(dir, 'state')]) fs.mkdirSync(d, { recursive: true }); // The fixture ignores the machine's git config (a url.insteadOf rewrite // would change the origin gstack-post resolves). fs.writeFileSync(path.join(dir, 'gitconfig'), ''); const gitEnv = { GIT_CONFIG_GLOBAL: path.join(dir, 'gitconfig'), GIT_CONFIG_NOSYSTEM: '1' }; expect(spawnSync('git', ['init', '-q'], { cwd: repo, timeout: 10_000, env: { ...process.env, ...gitEnv } }).status).toBe(0); expect(spawnSync('git', ['remote', 'add', 'origin', 'https://github.com/o/r.git'], { cwd: repo, timeout: 10_000, env: { ...process.env, ...gitEnv } }).status).toBe(0); const stub = (file: string, body: string) => fs.writeFileSync(file, `#!/usr/bin/env bash\n${body}\n`, { mode: 0o755 }); stub(path.join(bin, 'gh'), ` LOG="${log}" n=$(ls "$LOG" | wc -l | tr -d ' ') case "$1 $2" in "repo view") case "$*" in *visibility*) echo PRIVATE ;; *) echo o/r ;; esac; exit 0 ;; "pr view") echo 7; exit 0 ;; "issue view") echo "https://github.com/o/r/issues/$3"; exit 0 ;; esac printf '%s\\n' "$@" > "$LOG/$n.args" prev="" for a in "$@"; do case "$a" in body=@-|--field=body=@-|--body-file=-) cat > "$LOG/$n.body" ;; body=@*) cp "\${a#body=@}" "$LOG/$n.body" ;; --title=*) printf '%s' "\${a#--title=}" > "$LOG/$n.title" ;; esac [ "$prev" = "--body-file" ] && cp "$a" "$LOG/$n.body" [ "$prev" = "--title" ] && printf '%s' "$a" > "$LOG/$n.title" prev="$a" done [ "$1 $2" = "issue create" ] && echo "https://github.com/o/r/issues/42" exit 0`); stub(path.join(bin, 'glab'), ` LOG="${log}"; n=$(ls "$LOG" | wc -l | tr -d ' '); prev="" for a in "$@"; do [ "$prev" = "-t" ] && printf '%s' "$a" > "$LOG/$n.title" [ "$prev" = "-d" ] && printf '%s' "$a" > "$LOG/$n.body" prev="$a" done printf '%s\\n' "$@" > "$LOG/$n.args"`); stub(path.join(runtime, 'bin', 'gstack-decision-log'), `printf '%s' "$1" > "${log}/decision.json"`); stub(path.join(runtime, 'bin', 'gstack-post'), `exec ${JSON.stringify(process.execPath)} ${JSON.stringify(path.join(ROOT, 'bin', 'gstack-post'))} "$@"`); stub(path.join(runtime, 'bin', 'gstack-paths'), `echo "${path.join(dir, 'state')}"`); stub(path.join(runtime, 'bin', 'gstack-slug'), 'echo proj'); const run = (script: string) => spawnSync('bash', ['-c', script.replaceAll('~/.claude/skills/gstack', runtime)], { cwd: repo, encoding: 'utf8', timeout: 15_000, env: { PATH: `${bin}:${process.env.PATH}`, HOME: path.join(dir, 'home'), TMPDIR: dir, ...gitEnv }, }); /** Run a create block, then do what the agent's file-write tool does: write each printed file. */ const create = (script: string, texts: Record) => { const r = run(script); expect(r.status, r.stderr).toBe(0); const names: Record = {}; for (const m of r.stdout.matchAll(/^([A-Z_]+_FILE): (\S+) \(name: (\S+)\)$/gm)) { expect(path.basename(m[2])).toBe(m[3]); expect(fs.statSync(m[2]).mode & 0o777).toBe(0o600); if (texts[m[1]] !== undefined) fs.writeFileSync(m[2], texts[m[1]]); names[m[1]] = m[3]; } return names; }; const calls = () => fs.readdirSync(log).filter(f => f.endsWith('.args')).sort().map(f => { const n = f.replace('.args', ''); const read = (ext: string) => fs.existsSync(path.join(log, `${n}.${ext}`)) ? fs.readFileSync(path.join(log, `${n}.${ext}`), 'utf8') : undefined; return { args: fs.readFileSync(path.join(log, f), 'utf8').trim().split('\n'), body: read('body'), title: read('title') }; }); return { dir, repo, log, run, create, calls }; } /** Backticks, $(...), an apostrophe, double quotes, and lines equal to common heredoc delimiters. */ function hostile(canary: string): string { return [ `**Fixed** in \`touch ${canary}-backtick\`.`, `$(touch ${canary}-subst) and \${HOME} stay literal; it's "quoted"`, 'EOF', 'GSTACK_REPLY', 'GSTACK_ISSUE', 'REDACT_BODY_EOF', `'; touch ${canary}-quote; '`, '```diff', '- old', '+ new', '```', '', ].join('\n'); } const ranAnything = (dir: string, canary: string) => fs.readdirSync(dir).concat(fs.readdirSync(path.join(dir, 'repo'))).filter(f => f.startsWith(path.basename(canary))); describe('identifier grammars (CEO-12, ENG-8)', () => { for (const [placeholder, entry] of Object.entries(IDENTIFIER_PLACEHOLDERS)) { test(`${placeholder} rejects values that could escape its quoting`, () => { for (const bad of QUOTE_BREAKERS) expect(entry.grammar.test(bad), `${placeholder} accepted ${JSON.stringify(bad)}`).toBe(false); if (entry.quoted) return; for (const bad of UNQUOTED_BREAKERS) expect(entry.grammar.test(bad), `${placeholder} accepted ${JSON.stringify(bad)}`).toBe(false); if (!entry.list) for (const bad of ['a b', 'a\tb', 'a*b']) expect(entry.grammar.test(bad), `${placeholder} accepted ${JSON.stringify(bad)}`).toBe(false); }); } test('git accepts branch names with shell metacharacters; the ref grammar does not, and keeps valid refs', () => { const ref = IDENTIFIER_PLACEHOLDERS[''].grammar; for (const name of ['fix;id', 'fix`id`', 'x;curl-evil', 'a$(id)']) { expect(spawnSync('git', ['check-ref-format', '--branch', name], { timeout: 10_000 }).status).toBe(0); expect(ref.test(name)).toBe(false); } for (const name of ['feature/foo+bar@2', 'fix/ünïcode', 'release-1.2_rc']) expect(ref.test(name)).toBe(true); }); test('numeric ids are digits only', () => { for (const key of ['', '', '', '']) { expect(IDENTIFIER_PLACEHOLDERS[key].grammar.test('12345')).toBe(true); expect(IDENTIFIER_PLACEHOLDERS[key].grammar.test('123 4')).toBe(false); expect(IDENTIFIER_PLACEHOLDERS[key].grammar.test('0x1f')).toBe(false); } }); }); describe('Greptile replies through the real review/greptile-triage.md blocks', () => { const doc = fs.readFileSync(path.join(ROOT, 'review/greptile-triage.md'), 'utf8'); const createBlock = fence(doc, 'REPLY_FILE=$(mktemp'); const lineBlock = fence(doc, 'gstack-post reply "$PR_NUMBER"'); const topBlock = fence(doc, 'gstack-post pr-comment "$PR_NUMBER"'); test('the create block is the shared free-text block', () => { expect(createBlock).toBe(freeTextFileBash([{ variable: 'REPLY_FILE', stem: 'reply' }])); }); test('the fetched JSON goes to a mktemp directory, not fixed /tmp paths', () => { expect(doc).not.toMatch(/\/tmp\/greptile_/); expect(fence(doc, 'GREPTILE_DIR=$(mktemp -d')).toContain('"$GREPTILE_DIR/line.json"'); }); for (const [kind, block, args] of [['line-level', lineBlock, ['api', '--method=POST', 'repos/{owner}/{repo}/pulls/7/comments/123/replies', '--field=body=@-']], ['top-level', topBlock, ['pr', 'comment', '7', '--body-file=-']]] as const) { test(`${kind}: a hostile reply posts byte for byte and nothing in it runs`, () => { const s = sandbox(`greptile-${kind}`); const canary = path.join(s.dir, 'PWNED'); const body = hostile(canary); const names = s.create(createBlock, { REPLY_FILE: body }); const r = s.run(block.replaceAll('', names.REPLY_FILE).replaceAll('', '123')); expect(r.status, r.stderr).toBe(0); const [call] = s.calls(); expect(call.args).toEqual([...args]); expect(call.body).toBe(body); expect(ranAnything(s.dir, canary)).toEqual([]); expect(fs.readdirSync(path.join(s.repo, '.gstack/tmp'))).toEqual([]); expect(fs.readFileSync(path.join(s.repo, '.git/info/exclude'), 'utf8')).toContain('/.gstack/tmp/'); expect(s.run('git status --porcelain').stdout).toBe(''); }); } test('an unwritten reply is refused with the path and the manual command; nothing is posted', () => { const s = sandbox('greptile-refuse'); const names = s.create(createBlock, {}); const r = s.run(lineBlock.replaceAll('', names.REPLY_FILE).replaceAll('', '123')); expect(r.status).toBe(1); const file = path.join(fs.realpathSync(s.repo), '.gstack/tmp', names.REPLY_FILE); expect(r.stderr).toContain(`Not sent: ${file} is missing or empty`); expect(r.stderr).toContain(`gstack-post reply 7 --to 123 --body-file ${file}`); expect(s.calls()).toEqual([]); expect(fs.existsSync(file)).toBe(true); }); }); describe('test-failure issues through the real generated triage text', () => { const text = generateTestFailureTriage(); const createBlock = fence(text, 'TITLE_FILE=$(mktemp'); const postBlock = fence(text, 'case "" in'); test('the generated ship section carries the same text, with no literal \\n or nested triple backticks', () => { const section = fs.readFileSync(path.join(ROOT, 'ship/sections/tests.md'), 'utf8'); expect(section).toContain(postBlock); expect(text).not.toContain('\\n\\n**Error:**'); expect(postBlock).not.toContain('```'); expect(text).toMatch(/\*\*Error:\*\*\n~~~\n\n~~~/); }); for (const platform of ['github', 'gitlab'] as const) { test(`${platform}: hostile title, error output and branch name post as data`, () => { const s = sandbox(`issue-${platform}`); const canary = path.join(s.dir, 'PWNED'); const title = `Pre-existing test failure: \`touch ${canary}-title\` $(touch ${canary}-t2) it's`; const body = `Failing on x;touch ${canary}-branch; pre-existing.\n\n**Error:**\n~~~\n${hostile(canary)}~~~\n`; const names = s.create(createBlock, { TITLE_FILE: `${title}\n`, BODY_FILE: body }); const r = s.run(postBlock.replaceAll('', names.TITLE_FILE).replaceAll('', names.BODY_FILE) .replaceAll('', platform).replaceAll('', 'octocat').replaceAll('', 'octo')); expect(r.status, r.stderr).toBe(0); const [call] = s.calls(); expect(call.title).toBe(title); expect(call.body).toBe(platform === 'github' ? body : body.replace(/\n+$/, '')); expect(call.args).toContain(platform === 'github' ? 'octocat' : 'octo'); expect(ranAnything(s.dir, canary)).toEqual([]); expect(fs.readdirSync(path.join(s.repo, '.gstack/tmp'))).toEqual([]); }); } test('an unknown platform sends nothing and keeps the files', () => { const s = sandbox('issue-unknown'); const names = s.create(createBlock, { TITLE_FILE: 't\n', BODY_FILE: 'b\n' }); const r = s.run(postBlock.replaceAll('', names.TITLE_FILE).replaceAll('', names.BODY_FILE).replaceAll('', 'unknown')); expect(r.status).not.toBe(0); expect(r.stderr).toContain('Not sent: no GitHub or GitLab remote'); expect(s.calls()).toEqual([]); expect(fs.readdirSync(path.join(s.repo, '.gstack/tmp')).length).toBe(2); }); }); describe('/spec filing and archive through the real generated gate-and-file text', () => { const doc = fs.readFileSync(path.join(ROOT, 'spec/sections/gate-and-file.md'), 'utf8'); const draftBlock = fence(doc, 'REDACT_FILE=$(mktemp'); const titleBlock = fence(doc, 'TITLE_FILE=$(mktemp'); const fileBlock = fence(doc, 'gstack-post issue-create --title-file "$TITLE_FILE"'); const archiveBlock = fence(doc, 'ARCHIVE_PATH.tmp'); test('hostile title, body and approach are filed and archived as literal text', () => { const s = sandbox('spec'); const canary = path.join(s.dir, 'PWNED'); const title = `Spec: \`touch ${canary}-title\` $(touch ${canary}-t2) it's`; const body = hostile(canary); const approach = `Use $(touch ${canary}-approach) "carefully"`; const draft = s.create(draftBlock, { REDACT_FILE: body }); const files = s.create(titleBlock, { TITLE_FILE: `${title}\n`, APPROACH_FILE: `${approach}\n` }); const sub = (block: string) => block.replaceAll('', draft.REDACT_FILE) .replaceAll('', files.TITLE_FILE).replaceAll('', files.APPROACH_FILE).replaceAll('', '42'); let r = s.run(sub(fileBlock)); expect(r.status, r.stderr).toBe(0); expect(r.stdout).toContain('ISSUE_NUMBER: 42'); const [call] = s.calls(); expect(call.title).toBe(title); expect(call.body).toBe(body); expect(JSON.parse(fs.readFileSync(path.join(s.log, 'decision.json'), 'utf8'))).toMatchObject({ decision: `Spec filed #42: ${title}`, rationale: approach, issue: '42' }); r = s.run(sub(archiveBlock)); expect(r.status, r.stderr).toBe(0); const archived = /^Archived: (\S+)/m.exec(r.stdout)![1]; const text = fs.readFileSync(archived, 'utf8'); expect(text).toContain('spec_issue_number: 42\nspec_issue_url: https://github.com/o/r/issues/42\n'); expect(text.endsWith(`---\n\n# ${title}\n\n${body}`)).toBe(true); expect(ranAnything(s.dir, canary)).toEqual([]); }); test('a missing title file refuses filing with the manual command', () => { const s = sandbox('spec-refuse'); const draft = s.create(draftBlock, { REDACT_FILE: 'body\n' }); const files = s.create(titleBlock, {}); const r = s.run(fileBlock.replaceAll('', draft.REDACT_FILE).replaceAll('', files.TITLE_FILE).replaceAll('', files.APPROACH_FILE)); expect(r.status).toBe(1); expect(r.stderr).toContain('Not filed:'); expect(r.stderr).toContain('gstack-post issue-create --title-file '); expect(s.calls()).toEqual([]); }); });