From be3e9f0bfe16a3756e0fac391592cb3360271478 Mon Sep 17 00:00:00 2001 From: Garry Tan Date: Mon, 31 Aug 2026 04:05:59 +0000 Subject: [PATCH] fix: pin the claude CLI to an exact version in the CI image + tripwire MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The image installed @anthropic-ai/claude-code UNPINNED and rebuilt weekly 'to pick up CLI updates' — while bun sat carefully pinned at 1.3.13 two RUN lines above. The PTY harness screen-scrapes this CLI's TUI, and that drift broke it three separate times (welcome-screen wedge on 2.1.233, skillify HOME discovery on 2.1.237, guard/freeze hooks on 2.1.162), each debugged as a flake first. Pin 2.1.251 (current latest), bump deliberately via a PR that runs the PTY gate, and enforce with test/ci-image-cli-pin.test.ts: any global npm install in Dockerfile.ci without an exact @X.Y.Z pin fails the free suite. The weekly ci-image cron stays as a cheap tag self-heal. Co-Authored-By: Claude Fable 5 --- .github/docker/Dockerfile.ci | 11 +++++- .github/workflows/ci-image.yml | 10 +++-- test/ci-image-cli-pin.test.ts | 67 ++++++++++++++++++++++++++++++++++ 3 files changed, 83 insertions(+), 5 deletions(-) create mode 100644 test/ci-image-cli-pin.test.ts diff --git a/.github/docker/Dockerfile.ci b/.github/docker/Dockerfile.ci index 550c90d3a..39247f4a1 100644 --- a/.github/docker/Dockerfile.ci +++ b/.github/docker/Dockerfile.ci @@ -74,8 +74,15 @@ ENV BUN_INSTALL="/usr/local" RUN curl --retry 5 --retry-delay 5 --retry-connrefused -fsSL https://bun.sh/install \ | bash -s "bun-v1.3.13" -# Claude CLI -RUN npm i -g @anthropic-ai/claude-code +# Claude CLI — pinned to an EXACT version, same discipline as the bun pin +# above. The PTY harness (test/helpers/claude-pty-runner.ts) screen-scrapes +# this CLI's TUI (trust dialog, input prompt, spinner glyphs); an unpinned +# install rebuilt weekly rode the TUI wherever it drifted, and that drift +# broke the harness three separate times (welcome-screen wedge on 2.1.233, +# skillify HOME discovery on 2.1.237, guard/freeze hooks on 2.1.162). +# Bump deliberately, via a PR that runs the PTY gate against the new TUI. +# test/ci-image-cli-pin.test.ts fails the free suite if this pin is removed. +RUN npm i -g @anthropic-ai/claude-code@2.1.251 # Playwright system deps (Chromium) — needed for browse E2E tests RUN npx playwright install-deps chromium diff --git a/.github/workflows/ci-image.yml b/.github/workflows/ci-image.yml index 19eceb94b..cfd896ab4 100644 --- a/.github/workflows/ci-image.yml +++ b/.github/workflows/ci-image.yml @@ -1,8 +1,12 @@ name: Build CI Image on: - # Rebuild weekly (Monday 4am UTC) to pick up CLI updates — deliberately 2h - # BEFORE evals-periodic's 6am cron so the weekly eval run finds a fresh - # image instead of racing a half-pushed tag or duplicating the build. + # Weekly self-heal (Monday 4am UTC) — deliberately 2h BEFORE + # evals-periodic's 6am cron so the weekly eval run finds the image instead + # of racing a half-pushed tag. With the claude CLI pinned in Dockerfile.ci + # (v1.76+), this cron no longer pulls CLI updates: when the content-hash + # tag already exists it's a ~30s no-op, and it only rebuilds if the tag + # was somehow lost. CLI bumps happen by editing the Dockerfile pin in a PR + # that runs the PTY gate against the new TUI. schedule: - cron: '0 4 * * 1' # Rebuild on Dockerfile or lockfile changes. package.json is deliberately diff --git a/test/ci-image-cli-pin.test.ts b/test/ci-image-cli-pin.test.ts new file mode 100644 index 000000000..e07a19cba --- /dev/null +++ b/test/ci-image-cli-pin.test.ts @@ -0,0 +1,67 @@ +/** + * Provider CLIs baked into the CI image must be pinned to EXACT versions. + * + * The PTY harness (test/helpers/claude-pty-runner.ts) screen-scrapes the + * claude CLI's TUI — trust dialog, input-prompt ready marker, spinner glyphs. + * The image used to install `npm i -g @anthropic-ai/claude-code` UNPINNED and + * rebuild weekly "to pick up CLI updates", while bun sat carefully pinned at + * 1.3.13 two RUN lines above — the exact drift class the bun pin exists for. + * Receipts: TUI drift broke the harness three separate times (welcome-screen + * wedge vs CLI 2.1.233, skillify HOME discovery on 2.1.237, guard/freeze + * hooks on 2.1.162), each debugged as a "flake" before being traced to an + * unpinned weekly-latest CLI. + * + * This tripwire fails the free suite when any globally-installed npm package + * in Dockerfile.ci lacks an exact `@X.Y.Z` pin. Bumps are deliberate: edit + * the pin in a PR and run the PTY gate against the new TUI before merging. + */ +import { describe, expect, test } from 'bun:test'; +import * as fs from 'node:fs'; +import * as path from 'node:path'; + +const ROOT = path.resolve(__dirname, '..'); +const DOCKERFILE = path.join(ROOT, '.github', 'docker', 'Dockerfile.ci'); + +/** Package specs from every `npm i -g` / `npm install -g` in the Dockerfile. */ +export function globalNpmInstallSpecs(source: string): string[] { + const specs: string[] = []; + for (const match of source.matchAll(/npm\s+(?:i|install)\s+(?:-g|--global)\s+([^\n\\&|;]+)/g)) { + for (const spec of match[1].trim().split(/\s+/)) { + if (spec.startsWith('-')) continue; // flags like --no-fund + specs.push(spec); + } + } + return specs; +} + +/** Exact pin = a trailing @ with no range operator (no ^ ~ x *). */ +export function isExactlyPinned(spec: string): boolean { + // Scoped (@scope/name@1.2.3) or bare (name@1.2.3); version must be exact. + const at = spec.lastIndexOf('@'); + if (at <= 0) return false; // no version at all (or a bare scope) + const version = spec.slice(at + 1); + return /^\d+\.\d+\.\d+(?:-[\w.]+)?$/.test(version); +} + +describe('ci image provider-CLI pins', () => { + const source = fs.readFileSync(DOCKERFILE, 'utf-8'); + const specs = globalNpmInstallSpecs(source); + + test('the image installs at least the claude CLI globally (scan must not rot)', () => { + expect( + specs.some((s) => s.startsWith('@anthropic-ai/claude-code@')), + `expected a pinned @anthropic-ai/claude-code install in ${path.relative(ROOT, DOCKERFILE)}; found: ${specs.join(', ') || '(none)'}`, + ).toBe(true); + }); + + test('every global npm install carries an exact @X.Y.Z pin', () => { + const unpinned = specs.filter((s) => !isExactlyPinned(s)); + expect( + unpinned, + `unpinned global npm installs in Dockerfile.ci: ${unpinned.join(', ')}\n` + + 'Pin the exact version (name@X.Y.Z) and bump via a PR that runs the ' + + 'PTY gate against the new TUI — weekly-latest CLI drift broke the ' + + 'harness three times before this tripwire existed.', + ).toHaveLength(0); + }); +});