diff --git a/.github/docker/Dockerfile.ci b/.github/docker/Dockerfile.ci index efe5aca83..841291cca 100644 --- a/.github/docker/Dockerfile.ci +++ b/.github/docker/Dockerfile.ci @@ -60,6 +60,21 @@ RUN curl --retry 5 --retry-delay 5 --retry-connrefused -fsSL \ && rm -rf /var/lib/apt/lists/* \ && git --no-lazy-fetch --version +# Partial clone: the eval jobs check out with `filter: blob:none` and read +# older files through on-demand blob fetches (.github/scripts/history-read.sh). +# Prove this Git makes a blobless clone and fetches a missing blob, so the image +# fails here instead of in a slice's history read. +RUN set -e; d=$(mktemp -d); \ + git init -q -b main "$d/src"; echo one > "$d/src/f"; git -C "$d/src" add f; \ + git -C "$d/src" -c user.name=ci -c user.email=ci@localhost commit -qm one; \ + echo two > "$d/src/f"; git -C "$d/src" -c user.name=ci -c user.email=ci@localhost commit -qam two; \ + git -C "$d/src" config uploadpack.allowFilter true; \ + git clone -q --filter=blob:none --no-local "file://$d/src" "$d/clone"; \ + test "$(git -C "$d/clone" config remote.origin.promisor)" = true; \ + if GIT_NO_LAZY_FETCH=1 git -C "$d/clone" cat-file -e HEAD~1:f 2>/dev/null; then echo "blob present: not a partial clone" >&2; exit 1; fi; \ + test "$(git -C "$d/clone" show HEAD~1:f)" = one; \ + rm -rf "$d"; echo "partial clone: ok ($(git --version))" + RUN python3 -m venv /tmp/gstack-ci-venv \ && /tmp/gstack-ci-venv/bin/python -m pip --version \ && rm -rf /tmp/gstack-ci-venv diff --git a/.github/scripts/history-read.sh b/.github/scripts/history-read.sh new file mode 100755 index 000000000..1b711924d --- /dev/null +++ b/.github/scripts/history-read.sh @@ -0,0 +1,47 @@ +#!/usr/bin/env bash +# Read one file at a commit in a CI checkout that may be a blobless clone +# (actions/checkout `filter: blob:none`), where git fetches an old file's +# contents on demand and that fetch can fail. lib/history-read.ts is the +# TypeScript twin. +# +# Usage: history-read.sh [--absent ] +# +# Existence comes from the ref's tree (`git ls-tree`, local data only). A path +# the tree does not list is absent: prints (default: nothing), exits 0. +# A listed path is read with `git show`, tried 3 times in a partial clone +# (only a fetch can recover) and once otherwise, waiting +# HISTORY_READ_BACKOFF_SECONDS (default 2) times the attempt between tries. +# Anything else (a ref that does not resolve, a fetch that keeps failing) exits +# 1 with HISTORY_READ_FAILED naming the ref, the path and the git version, and +# never reads as absent. +# docs/troubleshooting.md#history-read-failed +set -euo pipefail + +if [ "$#" -ne 2 ] && { [ "$#" -ne 4 ] || [ "$3" != --absent ]; }; then + echo "usage: history-read.sh [--absent ]" >&2 + exit 2 +fi +ref=$1 file=$2 absent=${4:-} +out=$(mktemp) err=$(mktemp) +trap 'rm -f "$out" "$err"' EXIT + +fail() { + echo "::error::cannot read ${file} at ${ref}. $1 ($(git --version)). Fix: see docs/troubleshooting.md#history-read-failed (HISTORY_READ_FAILED)" >&2 + exit 1 +} + +listed=$(git ls-tree "$ref" -- "$file" 2>"$err") || fail "git ls-tree failed: $(head -n 1 "$err")" +if [ -z "$listed" ]; then + printf '%s' "$absent" + exit 0 +fi + +tries=1 +if git config --get-regexp '^(remote\..+\.promisor|extensions\.partialclone)$' | grep -qvi ' false$'; then tries=3; fi +try=1 +until git show "${ref}:${file}" >"$out" 2>"$err"; do + [ "$try" -lt "$tries" ] || fail "git show failed ${tries} time(s): $(head -n 1 "$err")" + sleep "$(( try * ${HISTORY_READ_BACKOFF_SECONDS:-2} ))" + try=$(( try + 1 )) +done +cat "$out" diff --git a/.github/workflows/evals-main-backstop.yml b/.github/workflows/evals-main-backstop.yml index 2826bf353..9a37c2bfb 100644 --- a/.github/workflows/evals-main-backstop.yml +++ b/.github/workflows/evals-main-backstop.yml @@ -45,11 +45,12 @@ jobs: with: fetch-depth: 0 - # Read with git and jq only: without a train block nothing below runs. + # Read with git and jq only (history-read.sh is git alone): without a train block nothing below runs. - name: Read the train policy (main) id: policy run: | - train=$(git show "${GITHUB_SHA}:.gstack/ship-policy.json" 2>/dev/null | jq -c 'if (.train | type) == "object" then .train else null end' 2>/dev/null || true) + policy=$(bash .github/scripts/history-read.sh "$GITHUB_SHA" .gstack/ship-policy.json --absent null) + train=$(jq -c 'if (.train | type) == "object" then .train else null end' <<<"$policy") || { echo "::error::${GITHUB_SHA}:.gstack/ship-policy.json is not valid JSON; fix the policy file" >&2; exit 1; } if [ -z "$train" ] || [ "$train" = null ]; then echo "train=false" >> "$GITHUB_OUTPUT" echo "- Backstop inert: main's .gstack/ship-policy.json declares no train block; per-PR gates cover main" >> "$GITHUB_STEP_SUMMARY" diff --git a/.github/workflows/evals.yml b/.github/workflows/evals.yml index e7b81e55b..de2b56c46 100644 --- a/.github/workflows/evals.yml +++ b/.github/workflows/evals.yml @@ -179,13 +179,14 @@ jobs: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 with: fetch-depth: 0 + filter: blob:none persist-credentials: false - uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2 with: bun-version: 1.4.2 - # Read with git and jq only: no repository code decides whether PR pushes start paid work. + # Read with git and jq only (history-read.sh is git alone): no repository code decides whether PR pushes start paid work. - name: Read the train policy (base revision) id: policy env: @@ -193,9 +194,10 @@ jobs: DEFAULT_BRANCH: ${{ github.event.repository.default_branch }} run: | base="${BASE_SHA:-origin/${DEFAULT_BRANCH}}" - train=$(git show "${base}:.gstack/ship-policy.json" 2>/dev/null | jq -c 'if (.train | type) == "object" then .train else null end' 2>/dev/null || true) - artifact=$(jq -r '.record_artifact // ""' <<<"${train:-null}" 2>/dev/null || true) - check=$(jq -r '.full_gate_check // ""' <<<"${train:-null}" 2>/dev/null || true) + policy=$(bash .github/scripts/history-read.sh "$base" .gstack/ship-policy.json --absent null) + train=$(jq -c 'if (.train | type) == "object" then .train else null end' <<<"$policy") || { echo "::error::${base}:.gstack/ship-policy.json is not valid JSON; fix the policy file" >&2; exit 1; } + artifact=$(jq -r '.record_artifact // ""' <<<"${train:-null}") + check=$(jq -r '.full_gate_check // ""' <<<"${train:-null}") if [ -z "$train" ] || [ "$train" = null ]; then echo "train=false" >> "$GITHUB_OUTPUT" echo "- Train policy: none at ${base}; this run routes as before" >> "$GITHUB_STEP_SUMMARY" @@ -470,6 +472,7 @@ jobs: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 with: fetch-depth: 0 + filter: blob:none persist-credentials: false - uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2 @@ -528,6 +531,7 @@ jobs: # Preserve full history for merge-base diff selection. Moving the # planner off the eval image must not change its selection inputs. fetch-depth: 0 + filter: blob:none persist-credentials: false - uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2 @@ -670,6 +674,7 @@ jobs: # shards on the lane's first live run ("ambiguous argument # 'main...HEAD'"). The manifest still governs WHICH shards run. fetch-depth: 0 + filter: blob:none persist-credentials: false - name: Fix bun temp @@ -903,6 +908,7 @@ jobs: with: ref: ${{ inputs.screen_merge_sha }} fetch-depth: 0 + filter: blob:none persist-credentials: false - name: Fix bun temp @@ -1029,6 +1035,7 @@ jobs: with: ref: ${{ fromJSON(needs.route.outputs.bisect_refs)[matrix.slice] }} fetch-depth: 0 + filter: blob:none persist-credentials: false - name: Verify the prefix checkout @@ -1173,6 +1180,7 @@ jobs: # history of its prefix commits T1..Tk for the train record. ref: ${{ inputs.screen_merge_sha }} fetch-depth: ${{ needs.plan-slices.outputs.mode == 'train' && '0' || '1' }} + filter: blob:none persist-credentials: false - uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2 diff --git a/.github/workflows/free-tests.yml b/.github/workflows/free-tests.yml index 443888a40..69903754b 100644 --- a/.github/workflows/free-tests.yml +++ b/.github/workflows/free-tests.yml @@ -65,8 +65,10 @@ jobs: with: persist-credentials: false # Full history: the seed ratchet compares the allowlist with its - # merge-base copy, which a depth-1 checkout cannot read. + # merge-base copy, which a depth-1 checkout cannot read. Blobless: + # that copy is fetched on demand, and a failed read fails the ratchet. fetch-depth: 0 + filter: blob:none - uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2 with: bun-version: 1.4.2 diff --git a/.github/workflows/version-gate.yml b/.github/workflows/version-gate.yml index 1be24d4e6..b64ec092a 100644 --- a/.github/workflows/version-gate.yml +++ b/.github/workflows/version-gate.yml @@ -23,6 +23,9 @@ jobs: uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 with: fetch-depth: 0 + # Blobless: full history with file contents fetched on demand; the + # reads below fail loudly when a fetch fails (.github/scripts/history-read.sh). + filter: blob:none ref: ${{ github.event.pull_request.head.sha }} # The head's release_tool runs inside the gate (lib/version-gate-train.ts); # no token may sit in the checkout it stages. The base fetch below is public. @@ -49,8 +52,12 @@ jobs: set -euo pipefail PR_VERSION=$(cat VERSION | tr -d '[:space:]') BASE_REF="${{ github.event.pull_request.base.ref }}" - git fetch origin "$BASE_REF" --quiet || true - BASE_VERSION=$(git show "origin/$BASE_REF:VERSION" 2>/dev/null | tr -d '[:space:]' || echo "0.0.0.0") + for try in 1 2 3; do + git fetch origin "$BASE_REF" --quiet && break + if [ "$try" = 3 ]; then echo "::error::git fetch origin $BASE_REF failed 3 times ($(git --version)). Fix: see docs/troubleshooting.md#history-read-failed (HISTORY_READ_FAILED)"; exit 1; fi + sleep $((try * 2)) + done + BASE_VERSION=$(bash .version-gate/.github/scripts/history-read.sh "origin/$BASE_REF" VERSION --absent 0.0.0.0 | tr -d '[:space:]') { echo "pr_version=$PR_VERSION" echo "base_version=$BASE_VERSION" diff --git a/docs/troubleshooting.md b/docs/troubleshooting.md index a8c51edff..59bb4e2a9 100644 --- a/docs/troubleshooting.md +++ b/docs/troubleshooting.md @@ -2818,6 +2818,28 @@ fetching it cannot create that ref, so pass a branch that exists on `origin` (`git branch -r --list 'origin/*'` lists them). `gstack-pregate` says so in its fix instead of suggesting the fetch. + +### `cannot read at . (git version ). Fix: see docs/troubleshooting.md#history-read-failed (HISTORY_READ_FAILED)` + +**Meaning.** A CI job read a file at an older commit and git could not +produce it. The PR and train path jobs (Free Tests' plan, the Version Gate, +and E2E Evals' route, screen summary, planner, slices, reruns, bisect and +report) check out blobless clones (`filter: blob:none`): every commit and +tree is there, but an older file's contents are fetched from GitHub the first +time something reads them. When that fetch fails three times (a GitHub outage, +a network drop), or the ref does not resolve at all, the read fails here +instead of counting as "the file is absent". A file the commit really does not +have still reads as absent: no ship policy at the base means no train, no +VERSION means `0.0.0.0`, as before. `.github/scripts/history-read.sh` (the +workflow steps) and `lib/history-read.ts` (the Version Gate's train +classification) print this line. The Version Gate prints the same code when +`git fetch origin ` fails three times. + +**Fix.** Rerun the failed job once GitHub is reachable. If it fails the same +way on a healthy network, check that the ref exists on `origin` and that the +runner's git supports partial clone (`git --version` 2.22 or newer; the eval +image asserts it at build time in `.github/docker/Dockerfile.ci`). + ### `POLICY_INVALID` diff --git a/lib/history-read.ts b/lib/history-read.ts new file mode 100644 index 000000000..87cf0652c --- /dev/null +++ b/lib/history-read.ts @@ -0,0 +1,39 @@ +/** + * Read one file at a commit when the checkout may be a blobless clone + * (`filter: blob:none`), where git fetches an old file's contents on demand + * and that fetch can fail. The TypeScript twin of + * .github/scripts/history-read.sh: existence comes from the ref's tree + * (`git ls-tree`, local data only); a listed path is read with `git show`, + * tried 3 times in a partial clone (only a fetch can recover) and once + * otherwise, waiting HISTORY_READ_BACKOFF_SECONDS (default 2) times the attempt + * between tries; every other failure throws HISTORY_READ_FAILED instead of + * reading as absent (docs/troubleshooting.md#history-read-failed). + */ +import { spawnSync } from 'node:child_process'; + +export const HISTORY_READ_TRIES = 3; + +export class HistoryReadError extends Error { + readonly code = 'HISTORY_READ_FAILED'; +} + +/** The file's text at `ref`, or null when the ref's tree does not list it. */ +export function readHistoryFile(repoRoot: string, ref: string, file: string): string | null { + const git = (args: string[]) => spawnSync('git', args, { cwd: repoRoot, encoding: 'utf8', timeout: 60_000, maxBuffer: 64 * 1024 * 1024 }); + const fail = (cause: string) => { + const version = git(['--version']).stdout?.trim() || 'git version unknown'; + return new HistoryReadError(`cannot read ${file} at ${ref}. ${cause} (${version}). Fix: see docs/troubleshooting.md#history-read-failed (HISTORY_READ_FAILED)`); + }; + const firstLine = (r: ReturnType) => (r.stderr ?? '').split('\n')[0] || r.error?.message || `exit ${r.status}`; + const listed = git(['ls-tree', ref, '--', file]); + if (listed.status !== 0) throw fail(`git ls-tree failed: ${firstLine(listed)}`); + if (!listed.stdout.trim()) return null; + const promisors = (git(['config', '--get-regexp', '^(remote\\..+\\.promisor|extensions\\.partialclone)$']).stdout ?? '').split('\n'); + const tries = promisors.some(line => line && !/ false$/i.test(line)) ? HISTORY_READ_TRIES : 1; + for (let attempt = 1; ; attempt++) { + const shown = git(['show', `${ref}:${file}`]); + if (shown.status === 0) return shown.stdout; + if (attempt === tries) throw fail(`git show failed ${tries === 1 ? 'once (not a partial clone, so no retry)' : `${tries} times`}: ${firstLine(shown)}`); + Bun.sleepSync(attempt * 1000 * Number(process.env.HISTORY_READ_BACKOFF_SECONDS ?? 2)); + } +} diff --git a/lib/version-gate-train.ts b/lib/version-gate-train.ts index 0a95371dc..2a18867ec 100644 --- a/lib/version-gate-train.ts +++ b/lib/version-gate-train.ts @@ -13,6 +13,7 @@ import * as os from 'node:os'; import * as path from 'node:path'; import { parseChangelog } from './changelog-check'; import { effectiveBump, readFragments } from './changelog-fragments'; +import { HistoryReadError, readHistoryFile } from './history-read'; import { matchesAny } from './ship-policy'; export type TrainHead = @@ -90,17 +91,23 @@ export function regeneratedAtBase(o: { repoRoot: string; head: string; baseVersi } } +/** A history read that fails (not an absent file) classifies the head invalid with HISTORY_READ_FAILED. */ export function classifyTrainHead(o: { repoRoot: string; baseRef: string; head: string; author?: string }): TrainHead { + try { return classify(o); } catch (e) { if (e instanceof HistoryReadError) return { kind: 'invalid', reason: e.message }; throw e; } +} + +function classify(o: { repoRoot: string; baseRef: string; head: string; author?: string }): TrainHead { const base = `origin/${o.baseRef.replace(/^origin\//, '')}`; - const show = (rev: string, file: string) => git(o.repoRoot, ['show', `${rev}:${file}`]); + const show = (rev: string, file: string) => readHistoryFile(o.repoRoot, rev, file); + const policyText = show(base, '.gstack/ship-policy.json'); let policy: any = null; - try { policy = JSON.parse(show(base, '.gstack/ship-policy.json') ?? 'null'); } catch { policy = null; } + try { policy = JSON.parse(policyText ?? 'null'); } catch { policy = null; } const fragDir: string | null = typeof policy?.changelog_fragments?.dir === 'string' ? policy.changelog_fragments.dir : null; if (!policy || ((typeof policy.train !== 'object' || policy.train === null) && !fragDir && policy.bump !== 'never')) return { kind: 'off' }; const mergeBase = git(o.repoRoot, ['merge-base', base, o.head])?.trim(); if (!mergeBase) return { kind: 'invalid', reason: `no merge base between ${base} and ${o.head}` }; - const npmVersion = (rev: string) => { try { return JSON.parse(show(rev, 'package.json') ?? '{}').version ?? null; } catch { return null; } }; + const npmVersion = (rev: string) => { const text = show(rev, 'package.json'); try { return JSON.parse(text ?? '{}').version ?? null; } catch { return null; } }; if (policy.bump === 'never') return show(mergeBase, 'VERSION') === show(o.head, 'VERSION') && npmVersion(mergeBase) === npmVersion(o.head) ? { kind: 'unstamped', entry: 'kept' } : { kind: 'off' }; const changed = (git(o.repoRoot, ['diff', '--name-only', mergeBase, o.head]) ?? '').split('\n').filter(Boolean); const releaseOutputs: string[] = Array.isArray(policy.release_outputs) ? policy.release_outputs : []; @@ -128,10 +135,7 @@ export function classifyTrainHead(o: { repoRoot: string; baseRef: string; head: } const file = typeof policy.changelog === 'string' ? policy.changelog : 'CHANGELOG.md'; - const exists = (rev: string) => git(o.repoRoot, ['cat-file', '-e', `${rev}:${file}`]) !== null; - const read = (rev: string) => (exists(rev) ? show(rev, file) : ''); - const [beforeText, afterText] = [read(mergeBase), read(o.head)]; - if (beforeText === null || afterText === null) return { kind: 'invalid', reason: `cannot read ${file} at ${beforeText === null ? `the merge base ${mergeBase.slice(0, 12)}` : o.head}` }; + const [beforeText, afterText] = [show(mergeBase, file) ?? '', show(o.head, file) ?? '']; const before = parseChangelog(beforeText); const after = parseChangelog(afterText); const unreleased = after.sections.filter(s => s.unreleased); diff --git a/test/ci-history-read.test.ts b/test/ci-history-read.test.ts new file mode 100644 index 000000000..3e4c4e2c2 --- /dev/null +++ b/test/ci-history-read.test.ts @@ -0,0 +1,272 @@ +/** + * Blobless checkouts on the PR and train path (GSTA-203 PR 5). + * + * A `filter: blob:none` checkout keeps every commit and tree but fetches an + * older file's contents on demand, and that fetch can fail. Every history read + * those jobs gate on must tell "absent" (the ref's tree does not list the path) + * from "could not read" (a failed fetch, an unknown ref): the first keeps its + * old meaning, the second fails with HISTORY_READ_FAILED. The fixtures here + * are real blobless clones whose promisor remote is then made unreachable. + * + * The checkout-mode pin keeps every full-history checkout blobless except in + * the workflows FULL_CLONE_WORKFLOWS names, each with its reason. + */ +import { afterAll, describe, expect, test } from 'bun:test'; +import { spawnSync } from 'node:child_process'; +import * as fs from 'node:fs'; +import * as os from 'node:os'; +import * as path from 'node:path'; +import { HistoryReadError, readHistoryFile } from '../lib/history-read'; +import { classifyTrainHead } from '../lib/version-gate-train'; + +const ROOT = path.resolve(import.meta.dir, '..'); +const HELPER = path.join(ROOT, '.github', 'scripts', 'history-read.sh'); +const WORKFLOWS = path.join(ROOT, '.github', 'workflows'); +const SCRATCH = fs.mkdtempSync(path.join(os.tmpdir(), 'ci-history-read-')); +const savedBackoff = process.env.HISTORY_READ_BACKOFF_SECONDS; +process.env.HISTORY_READ_BACKOFF_SECONDS = '0'; +afterAll(() => { + if (savedBackoff === undefined) delete process.env.HISTORY_READ_BACKOFF_SECONDS; + else process.env.HISTORY_READ_BACKOFF_SECONDS = savedBackoff; + fs.rmSync(SCRATCH, { recursive: true, force: true }); +}); + +/** Workflows that keep full clones, by file name, each with its reason. */ +const FULL_CLONE_WORKFLOWS: Record = { + 'weekly-release.yml': 'release: cuts and tags from main; off the PR and train path and not yet dispatched once blobless', + 'cso-runtime-promote.yml': 'CSO image promotion; off the PR and train path and not yet dispatched once blobless', + 'cso-runtime-qualification.yml': 'CSO runtime qualification; off the PR and train path and not yet dispatched once blobless', + 'evals-periodic.yml': 'periodic evals; off the PR and train path and not yet dispatched once blobless', + 'evals-marathon.yml': 'marathon evals; off the PR and train path and not yet dispatched once blobless', + 'eval-sweep.yml': 'eval sweep; off the PR and train path and not yet dispatched once blobless', + 'free-durations-record.yml': 'duration recorder; off the PR and train path and not yet dispatched once blobless', + 'evals-main-backstop.yml': 'reads every train note blob (554 notes on refs/notes/gstack-trains at 5ba425a); a blobless clone fetches each on demand at about 0.58 s apiece, minutes more than the 10 s a blobless checkout saves on a job off the PR path', +}; + +/** The PR and train path jobs that check out blobless full history, by workflow. */ +const BLOBLESS_JOBS: Record = { + 'free-tests.yml': ['free-plan'], + 'version-gate.yml': ['check'], + 'evals.yml': ['route', 'screen-summary', 'plan-slices', 'eval-slices', 'eval-rerun', 'train-bisect', 'slices-report'], +}; + +const GIT_ID = ['-c', 'user.name=Fixture', '-c', 'user.email=fixture@example.com', '-c', 'commit.gpgsign=false']; +function git(cwd: string, ...args: string[]) { + return spawnSync('git', [...GIT_ID, ...args], { cwd, encoding: 'utf8', timeout: 30_000 }); +} +function ok(cwd: string, ...args: string[]): string { + const r = git(cwd, ...args); + if (r.status !== 0) throw new Error(`git ${args.join(' ')}: ${r.stderr}`); + return r.stdout.trim(); +} + +interface Fixture { clone: string; one: string; two: string; cut: () => void } +/** + * A source repository with commits one and two, and a blobless clone of it + * checked out at one, so two's changed files are missing until fetched. `cut` + * points the clone's promisor remote at a path that does not exist. + */ +function fixture(name: string, one: Record, two: Record): Fixture { + const dir = path.join(SCRATCH, name); + const src = path.join(dir, 'src'); + fs.mkdirSync(src, { recursive: true }); + ok(src, 'init', '-q', '-b', 'main'); + const write = (files: Record) => { + for (const [rel, text] of Object.entries(files)) { + const target = path.join(src, rel); + if (text === null) { fs.rmSync(target, { force: true }); continue; } + fs.mkdirSync(path.dirname(target), { recursive: true }); + fs.writeFileSync(target, text); + } + ok(src, 'add', '-A'); + }; + write(one); + ok(src, 'commit', '-q', '-m', 'one'); + write(two); + ok(src, 'commit', '-q', '-m', 'two'); + ok(src, 'config', 'uploadpack.allowFilter', 'true'); + const clone = path.join(dir, 'clone'); + ok(dir, 'clone', '-q', '--filter=blob:none', '--no-local', '--no-checkout', `file://${src}`, clone); + ok(clone, 'checkout', '-q', 'origin/main~1'); + return { clone, one: ok(clone, 'rev-parse', 'HEAD'), two: ok(clone, 'rev-parse', 'origin/main'), cut: () => { ok(clone, 'remote', 'set-url', 'origin', `file://${dir}/gone`); } }; +} + +const TRAIN = { gate_workflow: 'evals.yml', free_workflow: 'free-tests.yml', full_gate_check: 'gstack/full-gate', record_artifact: 'gstack-train-record', max_size: 5, fill_hours: 3, screen_selection: 'none', trial_reservation_usd: 1.5 }; +const POLICY = '.gstack/ship-policy.json'; +const ONE = { VERSION: '1.0.0.0\n', [POLICY]: JSON.stringify({ bump: 'patch' }), 'README.md': 'one\n' }; + +function helper(cwd: string, ...args: string[]) { + return spawnSync('bash', [HELPER, ...args], { cwd, encoding: 'utf8', timeout: 60_000, env: { PATH: process.env.PATH ?? '', HOME: cwd, HISTORY_READ_BACKOFF_SECONDS: '0' } }); +} +const missing = (fx: Fixture, rev: string, rel: string) => spawnSync('git', ['cat-file', '-e', `${rev}:${rel}`], { cwd: fx.clone, timeout: 30_000, env: { ...process.env, GIT_NO_LAZY_FETCH: '1' } }).status !== 0; + +describe('history-read.sh and lib/history-read.ts on a blobless clone', () => { + test('a listed file is read, its blob fetched on demand', () => { + const fx = fixture('read', ONE, { VERSION: '1.0.0.1\n' }); + expect(missing(fx, fx.two, 'VERSION')).toBe(true); + const r = helper(fx.clone, fx.two, 'VERSION'); + expect(r.stderr).toBe(''); + expect([r.status, r.stdout]).toEqual([0, '1.0.0.1\n']); + expect(readHistoryFile(fx.clone, fx.one, 'VERSION')).toBe('1.0.0.0\n'); + }); + + test('a path the tree does not list is absent even with the remote gone', () => { + const fx = fixture('absent', ONE, { [POLICY]: null }); + fx.cut(); + expect(helper(fx.clone, fx.two, POLICY, '--absent', 'null')).toMatchObject({ status: 0, stdout: 'null' }); + expect(helper(fx.clone, fx.two, POLICY)).toMatchObject({ status: 0, stdout: '' }); + expect(readHistoryFile(fx.clone, fx.two, POLICY)).toBeNull(); + }); + + test('a blob that cannot be fetched fails after 3 tries with HISTORY_READ_FAILED, never as absent', () => { + const fx = fixture('unreachable', ONE, { [POLICY]: JSON.stringify({ bump: 'patch', train: TRAIN }) }); + fx.cut(); + const r = helper(fx.clone, fx.two, POLICY, '--absent', 'null'); + expect(r.status).toBe(1); + expect(r.stdout).toBe(''); + for (const part of [`::error::cannot read ${POLICY} at ${fx.two}.`, 'git show failed 3 time(s)', '(git version ', 'docs/troubleshooting.md#history-read-failed', '(HISTORY_READ_FAILED)']) expect(r.stderr).toContain(part); + let thrown: unknown; + try { readHistoryFile(fx.clone, fx.two, POLICY); } catch (e) { thrown = e; } + expect(thrown).toBeInstanceOf(HistoryReadError); + expect((thrown as HistoryReadError).code).toBe('HISTORY_READ_FAILED'); + for (const part of [`cannot read ${POLICY} at ${fx.two}.`, 'git show failed 3 times', '(git version ', '#history-read-failed', '(HISTORY_READ_FAILED)']) expect((thrown as Error).message).toContain(part); + expect(fs.readFileSync(path.join(ROOT, 'docs', 'troubleshooting.md'), 'utf8')).toContain(''); + }); + + test('a ref that does not resolve fails instead of reading as absent', () => { + const fx = fixture('bad-ref', ONE, { VERSION: '1.0.0.1\n' }); + const r = helper(fx.clone, 'no-such-ref', 'VERSION', '--absent', '0.0.0.0'); + expect(r.status).toBe(1); + expect(r.stderr).toContain('cannot read VERSION at no-such-ref. git ls-tree failed'); + expect(() => readHistoryFile(fx.clone, 'no-such-ref', 'VERSION')).toThrow('(HISTORY_READ_FAILED)'); + expect(helper(fx.clone, 'only-one-arg').status).toBe(2); + }); + + test('the Version Gate classifies a head whose base policy cannot be fetched as invalid; an absent base policy stays off', () => { + const unreadable = fixture('gate-unreadable', ONE, { [POLICY]: JSON.stringify({ bump: 'patch', train: TRAIN }) }); + unreadable.cut(); + const kind = classifyTrainHead({ repoRoot: unreadable.clone, baseRef: 'main', head: unreadable.one }); + expect(kind.kind).toBe('invalid'); + expect(kind.kind === 'invalid' && kind.reason).toContain(`cannot read ${POLICY} at origin/main.`); + expect(kind.kind === 'invalid' && kind.reason).toContain('(HISTORY_READ_FAILED)'); + const absent = fixture('gate-absent', ONE, { [POLICY]: null }); + absent.cut(); + expect(classifyTrainHead({ repoRoot: absent.clone, baseRef: 'main', head: absent.one })).toEqual({ kind: 'off' }); + }); +}); + +type Step = { name?: string; uses?: string; run?: string; with?: Record }; +type Workflow = { jobs: Record }; +const workflow = (file: string) => Bun.YAML.parse(fs.readFileSync(path.join(WORKFLOWS, file), 'utf8')) as Workflow; +const stepOf = (file: string, job: string, name: string) => workflow(file).jobs[job]!.steps!.find(st => st.name === name)!; + +describe.skipIf(!Bun.which('jq') || !Bun.which('bash'))('the workflow steps that read history fail loudly', () => { + function runStep(script: string, fx: Fixture, env: Record, helperAt = '.github/scripts') { + fs.mkdirSync(path.join(fx.clone, helperAt), { recursive: true }); + fs.copyFileSync(HELPER, path.join(fx.clone, helperAt, 'history-read.sh')); + const bin = path.join(fx.clone, '.stub-bin'); + fs.mkdirSync(bin, { recursive: true }); + fs.writeFileSync(path.join(bin, 'sleep'), '#!/bin/sh\nexit 0\n', { mode: 0o755 }); + const out = path.join(fx.clone, '.github-output'); + fs.writeFileSync(out, ''); + const r = spawnSync('bash', ['-e', '-c', script], { cwd: fx.clone, encoding: 'utf8', timeout: 60_000, + env: { PATH: `${bin}:${process.env.PATH ?? ''}`, HOME: fx.clone, HISTORY_READ_BACKOFF_SECONDS: '0', GITHUB_OUTPUT: out, GITHUB_STEP_SUMMARY: path.join(fx.clone, '.summary'), ...env } }); + const outputs = Object.fromEntries(fs.readFileSync(out, 'utf8').split('\n').filter(Boolean).map(line => [line.slice(0, line.indexOf('=')), line.slice(line.indexOf('=') + 1)])); + return { status: r.status, log: r.stdout + r.stderr, outputs }; + } + const policySteps = [ + { file: 'evals.yml', job: 'route', name: 'Read the train policy (base revision)', env: (fx: Fixture) => ({ BASE_SHA: fx.two, DEFAULT_BRANCH: 'main' }) }, + { file: 'evals-main-backstop.yml', job: 'backstop', name: 'Read the train policy (main)', env: (fx: Fixture) => ({ GITHUB_SHA: fx.two }) }, + ]; + + for (const { file, job, name, env } of policySteps) { + test(`${file} ${job}: an unfetchable policy fails, a malformed one fails, an absent one routes as before`, () => { + const script = stepOf(file, job, name).run!; + expect(script).not.toContain('|| true'); + expect(script).not.toContain('2>/dev/null'); + + const trained = { [POLICY]: JSON.stringify({ bump: 'patch', train: TRAIN }) }; + const reachable = fixture(`${job}-reachable`, ONE, trained); + expect(runStep(script, reachable, env(reachable))).toMatchObject({ status: 0, outputs: { train: 'true' } }); + const unreadable = fixture(`${job}-unreadable`, ONE, trained); + unreadable.cut(); + const failed = runStep(script, unreadable, env(unreadable)); + expect(failed.status).not.toBe(0); + expect(failed.log).toContain('(HISTORY_READ_FAILED)'); + expect(failed.outputs).toEqual({}); + + const malformed = fixture(`${job}-malformed`, ONE, { [POLICY]: '{not json' }); + const bad = runStep(script, malformed, env(malformed)); + expect(bad.status).not.toBe(0); + expect(bad.log).toContain(`${malformed.two}:${POLICY} is not valid JSON`); + + const absent = fixture(`${job}-absent`, ONE, { [POLICY]: null }); + absent.cut(); + expect(runStep(script, absent, env(absent))).toMatchObject({ status: 0, outputs: { train: 'false' } }); + }); + } + + test('version-gate.yml Read versions: the base VERSION is read, an absent one is 0.0.0.0, a failed fetch fails', () => { + const script = stepOf('version-gate.yml', 'check', 'Read versions').run! + .replace('${{ github.event.pull_request.base.ref }}', 'main'); + expect(script).not.toContain('|| true'); + expect(script).not.toContain('2>/dev/null'); + + const bumped = fixture('gate-bumped', ONE, { VERSION: '1.0.0.1\n' }); + expect(runStep(script, bumped, {}, '.version-gate/.github/scripts')).toMatchObject({ status: 0, outputs: { pr_version: '1.0.0.0', base_version: '1.0.0.1', base_ref: 'main' } }); + + const unversioned = fixture('gate-unversioned', ONE, { VERSION: null }); + expect(runStep(script, unversioned, {}, '.version-gate/.github/scripts').outputs.base_version).toBe('0.0.0.0'); + + bumped.cut(); + const failed = runStep(script, bumped, {}, '.version-gate/.github/scripts'); + expect(failed.status).not.toBe(0); + expect(failed.log).toContain('git fetch origin main failed 3 times'); + expect(failed.log).toContain('(HISTORY_READ_FAILED)'); + }); +}); + +describe('checkout mode on the PR and train path', () => { + const files = fs.readdirSync(WORKFLOWS).filter(f => /\.ya?ml$/.test(f)).sort(); + + test('every full-history checkout is blobless unless its workflow keeps full clones for a named reason', () => { + const problems: string[] = []; + const fullHistory = new Set(); + for (const file of files) { + const text = fs.readFileSync(path.join(WORKFLOWS, file), 'utf8'); + const checkoutLines = text.split('\n').flatMap((line, i) => (/^\s*(-\s+)?uses:\s*actions\/checkout@/.test(line) ? [i + 1] : [])); + const steps = Object.entries((Bun.YAML.parse(text) as Workflow).jobs ?? {}).flatMap(([job, j]) => (j.steps ?? []).filter(st => st.uses?.startsWith('actions/checkout@')).map(st => ({ job, st }))); + expect(steps.length, file).toBe(checkoutLines.length); + steps.forEach(({ job, st }, i) => { + const depth = st.with?.['fetch-depth']; + if (!(depth === 0 || depth === '0' || (typeof depth === 'string' && depth.includes("'0'")))) return; + fullHistory.add(file); + const at = `.github/workflows/${file}:${checkoutLines[i]}`; + if (FULL_CLONE_WORKFLOWS[file]) { + if (st.with?.filter !== undefined) problems.push(`${at}: job ${job} sets filter but ${file} is in FULL_CLONE_WORKFLOWS; remove one or the other`); + return; + } + if (st.with?.filter !== 'blob:none') problems.push(`${at}: job ${job} checks out full history without a blobless filter. Add \`filter: blob:none\` under its \`with:\`, or name ${file} in FULL_CLONE_WORKFLOWS (test/ci-history-read.test.ts) with the reason it needs every blob.`); + }); + } + expect(problems).toEqual([]); + for (const [file, reason] of Object.entries(FULL_CLONE_WORKFLOWS)) { + expect(reason.length, file).toBeGreaterThan(20); + expect(fullHistory.has(file), `${file} has no full-history checkout left; drop it from FULL_CLONE_WORKFLOWS`).toBe(true); + } + }); + + test('the PR and train path jobs check out blobless full history', () => { + for (const [file, jobs] of Object.entries(BLOBLESS_JOBS)) { + for (const job of jobs) { + const checkout = workflow(file).jobs[job]!.steps!.find(st => st.uses?.startsWith('actions/checkout@'))!; + expect(checkout.with?.filter, `${file} ${job}`).toBe('blob:none'); + } + } + }); + + test('the eval image asserts its Git makes a blobless clone and fetches a missing blob', () => { + const dockerfile = fs.readFileSync(path.join(ROOT, '.github', 'docker', 'Dockerfile.ci'), 'utf8'); + for (const part of ['git clone -q --filter=blob:none --no-local', 'config remote.origin.promisor)" = true', 'GIT_NO_LAZY_FETCH=1 git -C "$d/clone" cat-file -e HEAD~1:f', 'show HEAD~1:f)" = one']) expect(dockerfile).toContain(part); + }); +}); diff --git a/test/evals-train-workflow.test.ts b/test/evals-train-workflow.test.ts index cf3c8b71e..8d7cd4afc 100644 --- a/test/evals-train-workflow.test.ts +++ b/test/evals-train-workflow.test.ts @@ -379,6 +379,8 @@ describe.skipIf(!Bun.which('jq') || !Bun.which('bash'))('the route job reads the fs.writeFileSync(path.join(dir, 'README.md'), 'x\n'); git('add', '-A'); git('commit', '-q', '-m', 'base'); + fs.mkdirSync(path.join(dir, '.github/scripts'), { recursive: true }); + fs.copyFileSync(path.join(ROOT, '.github/scripts/history-read.sh'), path.join(dir, '.github/scripts/history-read.sh')); return { dir, base: git('rev-parse', 'HEAD').stdout.trim() }; } function run(script: string, dir: string, env: Record): Record { @@ -392,7 +394,7 @@ describe.skipIf(!Bun.which('jq') || !Bun.which('bash'))('the route job reads the const block = { gate_workflow: 'evals.yml', free_workflow: 'free-tests.yml', full_gate_check: 'gstack/full-gate', record_artifact: 'gstack-train-record', max_size: 5, fill_hours: 3, screen_selection: 'none', trial_reservation_usd: 1.5 }; test('no policy, no train block, or an invalid one: train=false, and a PR push routes pr', () => { - for (const policy of [undefined, { bump: 'patch' }, { bump: 'patch', train: null }, { train: { ...block, record_artifact: 'bad name!' } }, '{not json']) { + for (const policy of [undefined, { bump: 'patch' }, { bump: 'patch', train: null }, { train: { ...block, record_artifact: 'bad name!' } }]) { const fx = fixture(policy); const out = run(policyStep.run!, fx.dir, { BASE_SHA: fx.base, DEFAULT_BRANCH: 'main' }); expect(out, JSON.stringify(policy)).toEqual({ train: 'false' }); @@ -401,6 +403,12 @@ describe.skipIf(!Bun.which('jq') || !Bun.which('bash'))('the route job reads the } }); + test('a policy that is not JSON fails the route instead of routing as before (GSTA-203 H1)', () => { + const fx = fixture('{not json'); + expect(() => run(policyStep.run!, fx.dir, { BASE_SHA: fx.base, DEFAULT_BRANCH: 'main' })).toThrow('.gstack/ship-policy.json is not valid JSON'); + fs.rmSync(fx.dir, { recursive: true, force: true }); + }); + test('a train block names the record artifact and check, and a PR push routes pr-summary', () => { const fx = fixture({ bump: 'patch', train: block }); const out = run(policyStep.run!, fx.dir, { BASE_SHA: fx.base, DEFAULT_BRANCH: 'main' });