/** * Verified first-attempt reuse for PR-lane E2E shards, on the same receipts as * the workflow-judge reuse (scripts/eval-input-cache.ts). * * A PR-profile shard (a file, or one case of a case-sharded file) is reused * only when every consumed input is byte-identical to a fresh pass recorded in * this PR within the receipt age: * - files: the test file's literal import closure (helpers, fixtures loaded * as modules, installed packages), every tracked file matched by the * touchfile patterns of every case the file registers, the global * touchfiles, the paid runner and this module, the workflow and its setup * actions, bun.lock and the CI Dockerfile; * - prompts: the test source that builds each selected case's prompt; * - parameters: case ids, name pattern, expected count, retries, wall, * within-shard concurrency, tier/profile, the root package without its * release label, and every EVALS_/GSTACK_/CLAUDE_/ANTHROPIC_/... variable * the child receives (secrets contribute presence only); * - runtime: the immutable CI image manifest, Bun, Node, OS/arch and the * Claude CLI version. * Anything unknown fails closed: a computed case registration, a touchfile * pattern matching no tracked file, retries other than zero (a retried pass * cannot prove its first attempt), custom preload/endpoints, missing scope, * or a lane other than the PR gate. Failures are never stored; the weekly * census, marathon and release lanes always execute fresh. */ import * as fs from 'node:fs'; import * as path from 'node:path'; import { spawnSync } from 'node:child_process'; import { buildEvalInputIdentity, lookupEvalInputCache, sourceDependencyClosure, storeEvalInputCache, type EvalCacheValue, type EvalInputIdentity, type EvalPassingProof } from './eval-input-cache'; import { matchGlob } from '../test/helpers/test-selection'; import { E2E_TOUCHFILES, GLOBAL_TOUCHFILES } from '../test/helpers/touchfiles-data'; export interface E2EShardReuseRequest { root: string; /** Shard key: `` or `#`. */ key: string; file: string; /** Selected case ids this shard executes (the PR profile's exact expectation). */ caseIds: string[]; /** Every E2E id the file registers; all of their touchfiles are consumed inputs. */ registeredIds: string[]; registrationKnown: boolean; casePattern: string; expectedCases: number; retries: number; timeoutMs: number; withinShardConcurrency: number; tier: string; profile: string; /** The exact environment the child receives. */ env: NodeJS.ProcessEnv; } export interface E2EShardReuseHit { key: string; source: EvalPassingProof['source'] } const HARNESS_FILES = ['scripts/test-paid-shards.ts', 'scripts/e2e-shard-reuse.ts', 'scripts/eval-input-cache.ts', 'lib/eval-model.ts', 'bun.lock', '.github/docker/Dockerfile.ci', '.github/workflows/evals.yml', '.github/actions/fix-bun-temp/action.yml', '.github/actions/restore-deps/action.yml', '.github/actions/seed-claude-config/action.yml', '.github/actions/register-gstack-skills/action.yml']; const ENV_PREFIXES = ['EVALS_', 'GSTACK_', 'CLAUDE_', 'ANTHROPIC_', 'OPENAI_', 'GEMINI_', 'BUN_', 'NODE_', 'PLAYWRIGHT_']; /** Run-scoped values: provenance or transport, never behavior. Selection is bound as case ids. */ const RUN_SCOPED_ENV = new Set(['EVALS_RUN_ID', 'GSTACK_EVAL_DIR', 'EVALS_CACHE_DIR', 'EVALS_CACHE_PR', 'EVALS_CACHE_REPOSITORY', 'EVALS_CACHE_RUNTIME_ID', 'EVALS_CACHE_PURPOSE', 'EVALS_SELECTION_JSON', 'EVALS_JUDGE_SELECTION_JSON']); const SECRET_ENV = /KEY|TOKEN|SECRET|PASSWORD|CREDENTIAL/; /** The reuse-relevant environment the child sees; secrets contribute presence only. */ export function e2eReuseEnvironment(env: NodeJS.ProcessEnv): Record { return Object.fromEntries(Object.keys(env).sort() .filter(name => (ENV_PREFIXES.some(prefix => name.startsWith(prefix)) || ['PATH', 'HOME'].includes(name)) && !RUN_SCOPED_ENV.has(name)) .map(name => [name, SECRET_ENV.test(name) ? 'set' : env[name] ?? ''])); } /** Why reuse cannot apply to this lane or environment, else null. */ export function e2eReuseLaneProblem(env: NodeJS.ProcessEnv, profileMode: string | undefined): string | null { const pr = Number(env.EVALS_CACHE_PR); if (profileMode !== 'pr') return 'Only the fast PR profile reuses results'; if (!env.EVALS_CACHE_DIR || !env.EVALS_CACHE_REPOSITORY || !Number.isSafeInteger(pr) || pr <= 0) return 'No trusted same-PR cache scope'; if (!/^(?:sha256:)?[a-f0-9]{64}$/.test(env.EVALS_CACHE_RUNTIME_ID ?? '')) return 'No immutable runtime identity'; if (env.EVALS_TIER !== 'gate' || env.EVALS_FRESH === '1') return 'Fresh validation requested'; if (['release', 'periodic', 'marathon'].includes(env.EVALS_CACHE_PURPOSE ?? '')) return 'Scheduled and release lanes execute fresh'; if (env.NODE_OPTIONS || env.BUN_OPTIONS) return 'Preload options change execution outside the consumed source'; if (env.ANTHROPIC_BASE_URL && env.ANTHROPIC_BASE_URL !== 'https://api.anthropic.com') return 'Custom model endpoint'; return null; } function trackedFiles(root: string): string[] { const listed = spawnSync('git', ['ls-files', '-z'], { cwd: root, encoding: 'utf8', timeout: 10_000, maxBuffer: 64 * 1024 * 1024 }); if (listed.status !== 0) throw new Error('Cannot list tracked files'); return listed.stdout.split('\0').filter(Boolean); } /** * Every repository file one shard consumes: the test's import closure plus the * harness closure, and every tracked file the registered cases' touchfiles and * the global touchfiles match. Throws when a pattern matches nothing (unknown). */ export function e2eShardInputFiles(request: Pick): string[] { const tracked = trackedFiles(request.root); const declared = new Set(); for (const pattern of new Set([...request.registeredIds.flatMap(id => E2E_TOUCHFILES[id] ?? []), ...GLOBAL_TOUCHFILES])) { const matches = tracked.filter(file => matchGlob(file, pattern)); if (!matches.length) throw new Error(`Touchfile pattern matches no tracked file: ${pattern}`); for (const file of matches) declared.add(file); } const closure = sourceDependencyClosure(request.root, [request.file, ...HARNESS_FILES]); return [...new Set([...closure, ...declared])].filter(file => file !== 'package.json').sort(); } /** The consumed input identity of one PR shard, or why it is ineligible. */ export function e2eShardIdentity(request: E2EShardReuseRequest): { status: 'eligible'; identity: EvalInputIdentity } | { status: 'ineligible'; reason: string } { try { if (!/^test\/skill-e2e-.+\.test\.ts$/.test(request.file) || /overlay-harness/.test(request.file)) return { status: 'ineligible', reason: 'Not an audited E2E file' }; if (!request.registrationKnown || !request.registeredIds.length) return { status: 'ineligible', reason: 'Case registration is not statically complete' }; if (!request.caseIds.length || request.caseIds.length !== request.expectedCases || request.caseIds.some(id => !request.registeredIds.includes(id))) { return { status: 'ineligible', reason: 'Selected cases are not exactly known' }; } if (request.retries !== 0) return { status: 'ineligible', reason: 'A retried pass cannot prove its first attempt' }; const files = e2eShardInputFiles(request); const { version: _releaseLabel, ...rootPackage } = JSON.parse(fs.readFileSync(path.join(request.root, 'package.json'), 'utf8')); const source = fs.readFileSync(path.join(request.root, request.file), 'utf8'); const env = request.env; const result = buildEvalInputIdentity({ root: request.root, scope: { repository: env.EVALS_CACHE_REPOSITORY!, pullRequest: Number(env.EVALS_CACHE_PR) }, coverage: { dependencies: 'complete', prompts: 'complete', environment: 'complete' }, unknownDependencies: [], files, prompts: Object.fromEntries(request.caseIds.map(id => [id, source])), parameters: { rootPackage, key: request.key, caseIds: [...request.caseIds].sort(), casePattern: request.casePattern, expectedCases: request.expectedCases, retries: request.retries, timeoutMs: request.timeoutMs, withinShardConcurrency: request.withinShardConcurrency, tier: request.tier, profile: request.profile, environment: e2eReuseEnvironment(env) }, runtime: { image: env.EVALS_CACHE_RUNTIME_ID!, bun: Bun.version, node: process.versions.node, platform: process.platform, arch: process.arch, claudeCli: claudeCliVersion(env) }, }); return result; } catch (error) { return { status: 'ineligible', reason: error instanceof Error ? error.message : 'Cannot identify consumed inputs' }; } } function claudeCliVersion(env: NodeJS.ProcessEnv): string { const version = spawnSync('claude', ['--version'], { encoding: 'utf8', timeout: 5_000, env }); const line = version.status === 0 ? version.stdout.split('\n')[0]!.trim() : ''; if (!line) throw new Error('Claude CLI version is unknown'); return line; } const validResult = (identity: EvalInputIdentity, key: string) => (value: EvalCacheValue) => !!value && typeof value === 'object' && !Array.isArray(value) && value.key === key && JSON.stringify(value.cases) === JSON.stringify(identity.caseIds); /** * Prepare reuse for one shard: `lookup` returns a verified receipt for these * exact inputs; `publish` stores a receipt after a fresh first-attempt pass * whose inputs did not change during execution. */ export function prepareE2EShardReuse(request: E2EShardReuseRequest): { lookup(): E2EShardReuseHit | null; publish(): void; } | null { if (e2eReuseLaneProblem(request.env, 'pr') !== null) return null; const before = e2eShardIdentity(request); if (before.status !== 'eligible') return null; const common = { cacheDir: request.env.EVALS_CACHE_DIR!, purpose: 'gate' as const }; return { lookup() { const found = lookupEvalInputCache({ ...common, identity: before.identity, validateResult: validResult(before.identity, request.key) }); return found.status === 'reused' ? { key: found.key, source: found.source } : null; }, publish() { const after = e2eShardIdentity(request); const env = request.env; const runId = env.GITHUB_RUN_ID ? `${env.GITHUB_RUN_ID}/${env.GITHUB_RUN_ATTEMPT ?? '1'}` : env.EVALS_RUN_ID; const revision = spawnSync('git', ['rev-parse', 'HEAD'], { cwd: request.root, encoding: 'utf8', timeout: 3_000 }); if (after.status !== 'eligible' || !runId || revision.status !== 0) return; storeEvalInputCache({ ...common, before: before.identity, after: after.identity, proof: { execution: 'new', finalized: true, completeAttemptHistory: true, exitCode: 0, timedOut: false, cancelled: false, skipped: 0, failed: 0, passed: before.identity.caseIds.length, cases: before.identity.caseIds.map(id => ({ id, outcome: 'passed' as const, attempt: 1 as const })), source: { runId, revision: revision.stdout.trim(), completedAt: Date.now() }, result: { key: request.key, cases: before.identity.caseIds }, } }); }, }; }