/** * Provider CLIs baked into the CI image must be pinned to EXACT versions. * * The PTY harness (test/helpers/claude-pty-runner.ts) screen-scrapes the * claude CLI's TUI — trust dialog, input-prompt ready marker, spinner glyphs. * The image used to install `npm i -g @anthropic-ai/claude-code` UNPINNED and * rebuild weekly "to pick up CLI updates", while bun sat carefully pinned at * 1.3.13 two RUN lines above — the exact drift class the bun pin exists for. * Receipts: TUI drift broke the harness three separate times (welcome-screen * wedge vs CLI 2.1.233, skillify HOME discovery on 2.1.237, guard/freeze * hooks on 2.1.162), each debugged as a "flake" before being traced to an * unpinned weekly-latest CLI. * * This tripwire fails the free suite when any globally-installed npm package * in Dockerfile.ci lacks an exact `@X.Y.Z` pin. Bumps are deliberate: edit * the pin in a PR and run the PTY gate against the new TUI before merging. * * Bun comes from its release archive, verified by SHA-256 for every * architecture the image builds (#1706): nothing in the image is piped from * curl into a shell. */ import { describe, expect, test } from 'bun:test'; import * as fs from 'node:fs'; import * as path from 'node:path'; const ROOT = path.resolve(__dirname, '..'); const DOCKERFILE = path.join(__dirname, '..', '.github', 'docker', 'Dockerfile.ci'); /** Package specs from every `npm i -g` / `npm install -g` in the Dockerfile. */ export function globalNpmInstallSpecs(source: string): string[] { const specs: string[] = []; for (const match of source.matchAll(/npm\s+(?:i|install)\s+(?:-g|--global)\s+([^\n\\&|;]+)/g)) { for (const spec of match[1].trim().split(/\s+/)) { if (spec.startsWith('-')) continue; // flags like --no-fund specs.push(spec); } } return specs; } /** Exact pin = a trailing @ with no range operator (no ^ ~ x *). */ export function isExactlyPinned(spec: string): boolean { // Scoped (@scope/name@1.2.3) or bare (name@1.2.3); version must be exact. const at = spec.lastIndexOf('@'); if (at <= 0) return false; // no version at all (or a bare scope) const version = spec.slice(at + 1); return /^\d+\.\d+\.\d+(?:-[\w.]+)?$/.test(version); } describe('ci image provider-CLI pins', () => { const source = fs.readFileSync(DOCKERFILE, 'utf-8'); const specs = globalNpmInstallSpecs(source); test('the image installs at least the claude CLI globally (scan must not rot)', () => { expect( specs.some((s) => s.startsWith('@anthropic-ai/claude-code@')), `expected a pinned @anthropic-ai/claude-code install in ${path.relative(ROOT, DOCKERFILE)}; found: ${specs.join(', ') || '(none)'}`, ).toBe(true); }); test('every global npm install carries an exact @X.Y.Z pin', () => { const unpinned = specs.filter((s) => !isExactlyPinned(s)); expect( unpinned, `unpinned global npm installs in Dockerfile.ci: ${unpinned.join(', ')}\n` + 'Pin the exact version (name@X.Y.Z) and bump via a PR that runs the ' + 'PTY gate against the new TUI — weekly-latest CLI drift broke the ' + 'harness three times before this tripwire existed.', ).toHaveLength(0); }); test('the image installs the codex CLI globally so the Codex evals execute in CI', () => { expect( specs.some((s) => s.startsWith('@openai/codex@')), `expected a pinned @openai/codex install in ${path.relative(ROOT, DOCKERFILE)}; found: ${specs.join(', ')}`, ).toBe(true); }); }); /** Dockerfile source with `\`-continued lines joined into logical lines. */ function logicalLines(source: string): string[] { return source.replace(/\\\n/g, ' ').split('\n'); } /** Logical lines that pipe a download straight into a shell. */ export function curlPipedToShell(source: string): string[] { return logicalLines(source).filter((line) => !/^\s*#/.test(line) && /\b(?:curl|wget)\b/.test(line) && /\|\s*(?:sudo\s+)?(?:ba|z|da)?sh\b/.test(line)); } /** `) bun_target=...; bun_sha256=` arms of the Bun install `case`. */ export function bunArchiveArms(source: string): Map { const arms = new Map(); for (const m of source.matchAll(/^\s*([\w|]+)\)\s*bun_target=[\w-]+;\s*bun_sha256=([0-9a-f]+)\s*;;/gm)) { for (const arch of m[1].split('|')) arms.set(arch, m[2]); } return arms; } /** Docker architectures each build of Dockerfile.ci produces (no `platforms:` = the amd64 runner's). */ export function builtArchitectures(workflowsDir: string): Set { const archs = new Set(); for (const name of fs.readdirSync(workflowsDir)) { if (!/\.ya?ml$/.test(name)) continue; const workflow = Bun.YAML.parse(fs.readFileSync(path.join(workflowsDir, name), 'utf-8')) as any; for (const job of Object.values(workflow?.jobs ?? {})) { for (const step of job?.steps ?? []) { if (!String(step?.uses ?? '').startsWith('docker/build-push-action@')) continue; if (step.with?.file !== '.github/docker/Dockerfile.ci') continue; const platforms = String(step.with?.platforms ?? 'linux/amd64').split(/[\s,]+/).filter(Boolean); for (const platform of platforms) archs.add(platform.replace(/^linux\//, '')); } } } return archs; } describe('ci image Bun install (#1706)', () => { const source = fs.readFileSync(DOCKERFILE, 'utf-8'); const bunRun = logicalLines(source).find((line) => line.includes('oven-sh/bun/releases/download/bun-v${BUN_VERSION}/')) ?? ''; test('nothing is piped from a download into a shell', () => { expect(curlPipedToShell(source), 'download the release archive and verify its SHA-256 instead').toEqual([]); }); test('the tripwire recognizes the old installer pipe and leaves non-shell pipes alone', () => { expect(curlPipedToShell('RUN curl -fsSL https://bun.sh/install \\\n | bash -s "bun-v1.4.0"\n')).toHaveLength(1); expect(curlPipedToShell('RUN curl -fsSL https://example.test/x.sh | sh\n')).toHaveLength(1); expect(curlPipedToShell('RUN curl -fsSL https://example.test/key.gpg | gpg --dearmor -o /k.gpg\n')).toEqual([]); }); test('every architecture the image builds has a SHA-256-verified Bun archive', () => { const built = builtArchitectures(path.join(ROOT, '.github', 'workflows')); expect([...built], 'scan found no Dockerfile.ci build').toContain('amd64'); const arms = bunArchiveArms(source); for (const arch of built) { expect(arms.get(arch), `no verified Bun archive for ${arch}`).toMatch(/^[0-9a-f]{64}$/); } }); test('the archive is checked before it is unpacked, and unknown architectures fail closed', () => { expect(bunRun, 'Bun must come from the GitHub release archive').not.toBe(''); const download = bunRun.indexOf('releases/download/'); const verify = bunRun.indexOf('sha256sum -c'); const unpack = bunRun.indexOf('unzip'); expect(verify).toBeGreaterThan(download); expect(unpack).toBeGreaterThan(verify); expect(bunRun).toMatch(/\*\)(?:(?!;;).)*exit 1\s*;;/); expect(bunRun).toContain('test "$(bun --version)" = "${BUN_VERSION}"'); }); }); describe('ci image codex authentication', () => { const workflowsDir = path.join(ROOT, '.github', 'workflows'); const lanes: Array<{ lane: string; steps: any[] }> = []; for (const name of fs.readdirSync(workflowsDir).sort()) { if (!/\.ya?ml$/.test(name)) continue; const workflow = Bun.YAML.parse(fs.readFileSync(path.join(workflowsDir, name), 'utf-8')) as any; for (const [id, job] of Object.entries(workflow?.jobs ?? {})) { const steps: any[] = job?.steps ?? []; if (job?.container && steps.some((step) => /test-paid-shards\.ts .*--slice/.test(String(step?.run ?? '')))) { lanes.push({ lane: `${name}:${id}`, steps }); } } } test('every paid lane in the image logs the codex CLI in from the secret before it runs', () => { expect(lanes.length, 'scan found no paid lanes').toBeGreaterThanOrEqual(4); for (const { lane, steps } of lanes) { const login = steps.findIndex((step) => /codex"? login --with-api-key/.test(String(step?.run ?? ''))); const run = steps.findIndex((step) => /test-paid-shards\.ts .*--slice/.test(String(step?.run ?? ''))); expect(login, `${lane}: no codex login step`).toBeGreaterThanOrEqual(0); expect(login, `${lane}: codex login must precede the paid run`).toBeLessThan(run); const step = steps[login]; expect(step.env?.OPENAI_API_KEY, `${lane}: key must arrive through env`).toBe('${{ secrets.OPENAI_API_KEY }}'); expect(String(step.run), `${lane}: never inline a secret into run`).not.toContain('secrets.'); expect(String(step.run)).toMatch(/codex"? login status/); expect(String(step.if ?? ''), `${lane}: same-repo runs only`) .toBe("github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository"); } }); }); describe('other CI Bun installs match the image (#1713, #1706)', () => { const docker = fs.readFileSync(DOCKERFILE, 'utf-8'); const imageSha = bunArchiveArms(docker).get('amd64'); const imageVersion = docker.match(/^ARG BUN_VERSION=(\S+)$/m)?.[1]; const engines = JSON.parse(fs.readFileSync(path.join(ROOT, 'package.json'), 'utf-8')).engines.bun; for (const rel of ['.gitlab-ci.yml', 'scripts/ubicloud/setup-free-suite.sh']) { const source = fs.readFileSync(path.join(ROOT, rel), 'utf-8'); test(`${rel} downloads the verified linux-x64 archive instead of piping an installer into a shell`, () => { expect(curlPipedToShell(source)).toEqual([]); const download = source.indexOf('/releases/download/bun-v$BUN_VERSION/bun-linux-x64.zip'); const verify = source.indexOf('sha256sum -c', download); const unpack = source.indexOf('unzip -q'); expect(download).toBeGreaterThan(-1); expect(verify).toBeGreaterThan(download); expect(unpack).toBeGreaterThan(verify); expect(source).toContain('test "$(bun --version)" = "$BUN_VERSION"'); }); test(`${rel} links bunx next to the verified bun, as the bun.sh installer did`, () => { expect(source).toContain('ln -sf bun "$HOME/.bun/bin/bunx"'); expect(source.indexOf('ln -sf bun "$HOME/.bun/bin/bunx"')).toBeGreaterThan(source.indexOf('unzip -q')); }); test(`${rel} pins the same Bun version and SHA-256 as Dockerfile.ci`, () => { expect(imageSha).toMatch(/^[0-9a-f]{64}$/); const hashes = [...source.matchAll(/echo "([0-9a-f]{64}) /g)].map((m) => m[1]); expect(hashes).toEqual([imageSha!]); const version = source.match(/^\s*BUN_VERSION[=:]\s*"?([\d.]+)"?\s*$/m)?.[1]; expect(version).toBe(imageVersion!); expect(engines).toBe(`>=${imageVersion}`); }); } }); describe('CI Node.js installs match the image and are verified (#3354)', () => { const docker = fs.readFileSync(DOCKERFILE, 'utf-8'); const imageVersion = docker.match(/^ENV NODE_VERSION=(\S+)$/m)?.[1]; const imageSha = docker.match(/^ARG NODE_SHA256=(\S+)$/m)?.[1]; const nodeRun = logicalLines(docker).find((line) => line.includes('nodejs.org/dist/v${NODE_VERSION}/')) ?? ''; test('Dockerfile.ci pins an exact Node.js version and the SHA-256 of its linux-x64 archive', () => { expect(imageVersion).toMatch(/^\d+\.\d+\.\d+$/); expect(imageSha).toMatch(/^[0-9a-f]{64}$/); }); test('Dockerfile.ci checks the Node.js archive before it is unpacked', () => { expect(nodeRun, 'Node.js must come from the nodejs.org release archive').not.toBe(''); const download = nodeRun.indexOf('-o /tmp/node.tar.xz'); const verify = nodeRun.indexOf('echo "${NODE_SHA256} /tmp/node.tar.xz" | sha256sum -c -'); const unpack = nodeRun.indexOf('tar -xJ'); expect(download).toBeGreaterThan(-1); expect(verify).toBeGreaterThan(download); expect(unpack).toBeGreaterThan(verify); }); const rel = 'scripts/ubicloud/setup-free-suite.sh'; const source = fs.readFileSync(path.join(ROOT, rel), 'utf-8'); test(`${rel} pins the same Node.js version and SHA-256 as Dockerfile.ci`, () => { expect(imageSha).toMatch(/^[0-9a-f]{64}$/); expect(source.match(/^NODE_VERSION=(\S+)$/m)?.[1]).toBe(imageVersion!); expect(source.match(/^NODE_SHA256=(\S+)$/m)?.[1]).toBe(imageSha!); }); test(`${rel} checks the Node.js archive before it is unpacked`, () => { const download = source.indexOf('/node-v$NODE_VERSION-linux-x64.tar.xz" -o "$node_tmp/node.tar.xz"'); const verify = source.indexOf('echo "$NODE_SHA256 $node_tmp/node.tar.xz" | sha256sum -c -'); const unpack = source.indexOf('tar -xJ'); expect(download).toBeGreaterThan(-1); expect(verify).toBeGreaterThan(download); expect(unpack).toBeGreaterThan(verify); }); });