/** * The merge-train side of evals.yml (plan GSTA-49 lane T1b; * docs/ship-policy.md#run-a-merge-train). gstack-ship-queue (lib/ship-queue-train*.ts) * dispatches the runs and reads what they publish; this file is the workflow's half. * * route (route job, dispatches) names the run's mode and validates * its inputs: a screen needs a PR, its pinned merge SHA and a * policy; a bisect needs case ids from the full gate's own plan * and prefix SHAs that are ancestors of the train head on * train/. Anything else is refused before any spend. * bisect-plan (plan-slices) one executor per prefix, each expecting its SHA. * bisect-prefix (eval-slices) one prefix: every red case once, one trial, * written as the per-prefix record `train bisect` reads. * train-record (slices-report, train runs) the green-train record: the census * from the run's own plan manifest, one panel per planned case, * and the prefix trees T1..Tk modulo stamps. GREEN only for a * complete census: every planned case executed and passed, no * unbound result, partial or cancelled shard, or slice killed * before its upload step. * screen-summary (plan-slices, PR runs under a train policy) the screening * selection and its estimated cost; no paid case runs. */ import { spawnSync } from 'node:child_process'; import { createHash } from 'node:crypto'; import * as fs from 'node:fs'; import * as path from 'node:path'; import { normalizeChangelog } from '../lib/changelog-check'; import { normalizeLockfile } from '../lib/evidence-bundle'; import { stampMentions, rewriteStamps, versionAtRef } from '../lib/restamp'; import { loadPolicyFromHead, matchesAny, type ShipPolicy } from '../lib/ship-policy'; import type { BisectPrefixRecord, TrainGateRecord } from '../lib/ship-queue-train-verdict'; import type { CasePanel } from '../lib/ship-queue-train-record'; import { E2E_TIERS, LLM_JUDGE_TOUCHFILES } from '../test/helpers/touchfiles-data'; import { panelVerdict } from '../test/helpers/eval-store'; import { UNRECORDED_SHARD_MS, nonceScope, plannedSerialMs } from './eval-bg'; import { caseTrialPlan, fileCaseRegistration, shardCaseId, shardFile } from './lib/paid-cases'; import { buildRunManifest, loadPaidTestDurations, parseRunManifest, recordedShardMs, type PaidRunManifest } from './lib/paid-plan'; import { loadSliceArtifacts } from './lib/paid-report'; import type { FlakeRecord } from './lib/paid-rerun'; import { paidSelectionEnv } from './lib/paid-select'; import { ROOT } from './lib/paid-types'; const SHA = /^[0-9a-f]{40}$/; const CASE_ID = /^[A-Za-z0-9 _.:/-]{1,128}$/; export const JUDGE_FILE = 'test/skill-llm-eval.test.ts'; /** T1 caps a train at 6 PRs, so a bisect has at most 7 prefixes (T0..T6). */ export const MAX_PREFIXES = 7; export const MAX_BISECT_CASES = 40; /** A full gate's recorded cost (MEASUREMENT.md: $90.93 to $94.70; lib/gate-cost.ts uses the same scale). */ export const FULL_GATE_USD = 93.14; /** T1's prefix commits: `train T: merge # at ` (lib/ship-queue-train.ts buildChain). */ const PREFIX_SUBJECT = /^train T(\d+): merge #(\d+) at [0-9a-f]{12}$/; export type Git = (args: string[]) => { ok: boolean; out: string }; export const gitIn = (cwd: string): Git => (args) => { const r = spawnSync('git', args, { cwd, encoding: 'utf8', timeout: 60_000, maxBuffer: 256 * 1024 * 1024 }); return r.status === 0 ? { ok: true, out: (r.stdout ?? '').replace(/\n$/, '') } : { ok: false, out: (r.stderr || String(r.error ?? '')).trim() }; }; export type RunMode = 'screen' | 'train' | 'bisect' | 'dispatch'; export interface RouteInput { nonce: string; refName: string; head: string; expectedSha: string; evalsAll: boolean; trainPolicy: boolean; cases: string; prefixShas: string; screenPr: string; screenMergeSha: string; screenPolicy: string; } export interface Route { mode: RunMode; cases: string[]; prefixes: string[] } const list = (text: string) => text.split(',').map(item => item.trim()).filter(Boolean); /** * The run's mode from a dispatch's inputs, or why it is refused. Screens, * trains and bisects exist only under a train policy; a bisect's case ids must * be in the full gate's plan and each prefix an ancestor of the train head. */ export function routeDispatch(i: RouteInput, deps: { isAncestor(commit: string, head: string): boolean; gateCases(): Map }): Route | { error: string } { const scope = i.nonce ? nonceScope(i.nonce) : null; const trainOnly = (mode: string) => (i.trainPolicy ? null : { error: `a ${mode} dispatch needs a train block in the default branch's .gstack/ship-policy.json` }); if (i.screenPr || i.screenMergeSha || i.screenPolicy) { if (!/^[1-9]\d{0,8}$/.test(i.screenPr)) return { error: `screen_pr ${JSON.stringify(i.screenPr)} is not a PR number` }; if (!SHA.test(i.screenMergeSha)) return { error: `screen_merge_sha ${JSON.stringify(i.screenMergeSha)} is not a full commit SHA` }; if (i.screenPolicy !== 'narrow' && i.screenPolicy !== 'broad') return { error: `screen_policy ${JSON.stringify(i.screenPolicy)} is not narrow or broad` }; if (i.cases || i.prefixShas) return { error: 'a screen takes no cases or prefix_shas' }; if (i.evalsAll) return { error: 'a screen runs the PR selection: dispatch it with evals_all=false' }; if (i.expectedSha && i.expectedSha !== i.screenMergeSha) return { error: `expected_sha ${i.expectedSha.slice(0, 12)} is not the screen's merge SHA ${i.screenMergeSha.slice(0, 12)}` }; if (scope && (scope.purpose !== 'screen' || scope.policy !== i.screenPolicy)) return { error: `nonce ${i.nonce} is not a ${i.screenPolicy} screen nonce` }; return trainOnly('screen') ?? { mode: 'screen', cases: [], prefixes: [] }; } if (i.cases || i.prefixShas || scope?.purpose === 'bisect') { const cases = list(i.cases); const prefixes = list(i.prefixShas); if (scope?.purpose !== 'bisect') return { error: `a bisect needs a bisect nonce (got ${JSON.stringify(i.nonce)})` }; if (!i.refName.startsWith('train/')) return { error: `a bisect runs on train/, not ${i.refName}` }; if (i.expectedSha !== i.head) return { error: `expected_sha ${i.expectedSha.slice(0, 12) || '(empty)'} is not the train head ${i.head.slice(0, 12)} checked out from ${i.refName}` }; if (!cases.length || cases.length > MAX_BISECT_CASES || new Set(cases).size !== cases.length) return { error: `cases must name 1 to ${MAX_BISECT_CASES} distinct case ids` }; if (!prefixes.length || prefixes.length > MAX_PREFIXES || new Set(prefixes).size !== prefixes.length) return { error: `prefix_shas must name 1 to ${MAX_PREFIXES} distinct commits` }; const bad = prefixes.find(sha => !SHA.test(sha)); if (bad) return { error: `prefix ${JSON.stringify(bad)} is not a full commit SHA` }; const outside = prefixes.find(sha => !deps.isAncestor(sha, i.head)); if (outside) return { error: `prefix ${outside.slice(0, 12)} is not an ancestor of the train head ${i.head.slice(0, 12)} on ${i.refName}` }; const plan = deps.gateCases(); const unknown = cases.filter(id => !CASE_ID.test(id) || !plan.has(id)); if (unknown.length) return { error: `case(s) not in the full gate's plan: ${unknown.slice(0, 5).join(', ')}` }; return trainOnly('bisect') ?? { mode: 'bisect', cases, prefixes }; } if (scope?.purpose === 'train') { if (!i.refName.startsWith('train/')) return { error: `a train gate runs on train/, not ${i.refName}` }; if (!i.evalsAll) return { error: 'a train gate is the full gate: dispatch it with evals_all=true' }; if (i.expectedSha !== i.head) return { error: `expected_sha ${i.expectedSha.slice(0, 12) || '(empty)'} is not the train head ${i.head.slice(0, 12)}` }; return trainOnly('train') ?? { mode: 'train', cases: [], prefixes: [] }; } return { mode: 'dispatch', cases: [], prefixes: [] }; } /** Every case a plan manifest plans, with its kind: the census a complete gate must execute. */ export function plannedCases(manifest: PaidRunManifest, rootDir = ROOT): Map { const cases = new Map(); const e2e = manifest.selection?.e2e ?? null; for (const entry of manifest.entries.filter(item => item.status === 'planned')) { const file = shardFile(entry.file); if (entry.trial) { cases.set(shardCaseId(entry.file)!, entry.trial.kind); continue; } if (file === JUDGE_FILE) { for (const id of manifest.selection?.judges ?? Object.keys(LLM_JUDGE_TOUCHFILES)) cases.set(id, 'judge'); continue; } const only = shardCaseId(entry.file); const source = fs.readFileSync(path.join(rootDir, file), 'utf8'); for (const id of fileCaseRegistration(file, source).registered) { if (E2E_TIERS[id] === manifest.tier && (only === null || id === only) && (e2e === null || e2e.includes(id)) && !(entry.excludeCases ?? []).includes(id)) cases.set(id, 'rule'); } } return cases; } /** The full gate's planned cases on this checkout (what a bisect may name). */ export function fullGateCases(rootDir = ROOT): Map { return plannedCases(buildRunManifest({ tier: 'gate', profile: 'full', sliceCount: 1, evalsAll: true, env: { EVALS_ALL: '1' }, rootDir }), rootDir); } /** * A release output or stamp path without its stamps: every version mentioned * in a stamp form (the built-in forms and the policy's `stamp_patterns` * templates) collapses to one placeholder. Normalising only the commit's own * version left a tentative `*_KEY_VERSION = ''` as written on the * unstamped prefix and normalised once the slot stamp moved it to the released * version, so the two digests differed and train-backstop found no covering * record for main after that train (#3512), and a base-version mention on a * line the base already carried was normalised on the prefix and kept on the * stamped main. Every stamp-form mention collapsing makes the digest blind to * which version a stamp names; the receipt (`stampOnlyLine`) is what judges a * stamp commit's content. */ export function stampFreeText(text: string, templates: readonly string[]): string { let out = text; for (const v of [...stampMentions(text, templates)].sort((a, b) => b.length - a.length)) out = rewriteStamps(out, v, '\0VERSION', templates).text; return out; } /** * A commit's tree modulo stamps: every path with its mode and blob, except the * files a restamp rewrites, which contribute their content without the stamp * (the version file nothing, package.json without its version, lockfiles * without their root version, the CHANGELOG with its headings normalized, * release outputs and stamp paths with every stamp-form version mention * replaced). Two commits that differ only by a restamp have the same digest. */ export function stampFreeTree(git: Git, commit: string, repoRoot: string, policy: ShipPolicy): string | { error: string } { const listed = git(['ls-tree', '-r', '-z', '--full-tree', commit]); if (!listed.ok) return { error: `cannot list ${commit.slice(0, 12)} (${listed.out.split('\n')[0]})` }; const at = versionAtRef(repoRoot, commit); const versionRel = 'rel' in at ? at.rel : 'VERSION'; const lines: string[] = []; for (const item of listed.out.split('\0').filter(Boolean)) { const [meta, rel] = [item.slice(0, item.indexOf('\t')), item.slice(item.indexOf('\t') + 1)]; const [mode, , blob] = meta.split(' ') as [string, string, string]; const base = path.basename(rel); const stamp = rel === versionRel ? 'version' : base === 'package.json' ? 'manifest' : ['bun.lock', 'package-lock.json', 'npm-shrinkwrap.json'].includes(base) ? 'lockfile' : rel === policy.changelog ? 'changelog' : matchesAny([...policy.stamp_paths, ...policy.release_outputs, ...policy.stamp_files], rel) ? 'release' : null; if (!stamp) { lines.push(`${rel}\0${mode}\0${blob}`); continue; } if (stamp === 'version') { lines.push(`${rel}\0${mode}\0(version)`); continue; } const shown = git(['cat-file', 'blob', blob]); if (!shown.ok) return { error: `cannot read ${rel} at ${commit.slice(0, 12)}` }; let text = shown.out; if (stamp === 'manifest') { try { const parsed = JSON.parse(text); delete parsed.version; text = JSON.stringify(parsed); } catch { /* not JSON: kept verbatim */ } } else if (stamp === 'lockfile') text = normalizeLockfile(base, text); else if (stamp === 'changelog') text = normalizeChangelog(text); else text = stampFreeText(text, policy.stamp_patterns); lines.push(`${rel}\0${mode}\0${createHash('sha256').update(text).digest('hex')}`); } return createHash('sha256').update(lines.sort().join('\n')).digest('hex'); } export interface TrainPrefixTree { index: number; commit: string; tree: string; stamp_free_tree: string } /** T1..Tk from the train head, walking first parents over T1's prefix commits. */ export function trainPrefixes(git: Git, head: string, repoRoot: string, policy: ShipPolicy): TrainPrefixTree[] | { error: string } { const log = git(['log', '--first-parent', `--max-count=${MAX_PREFIXES}`, '--format=%H%x00%P%x00%s', head]); if (!log.ok) return { error: `cannot walk ${head.slice(0, 12)}'s first parents (${log.out.split('\n')[0]})` }; const found: TrainPrefixTree[] = []; for (const line of log.out.split('\n')) { const [commit, parents, subject] = line.split('\0') as [string, string, string]; const m = PREFIX_SUBJECT.exec(subject ?? ''); if (!m || parents.split(' ').length !== 2) break; const index = Number(m[1]); if (found.length && index !== found.at(-1)!.index - 1) return { error: `prefix T${index} follows T${found.at(-1)!.index} at ${commit.slice(0, 12)}` }; const digest = stampFreeTree(git, commit, repoRoot, policy); if (typeof digest !== 'string') return digest; found.push({ index, commit, tree: git(['rev-parse', `${commit}^{tree}`]).out, stamp_free_tree: digest }); } if (!found.length || found.at(-1)!.index !== 1) return { error: `${head.slice(0, 12)} is not a train head (no T1..Tk prefix chain)` }; return found.reverse(); } interface OutcomeLine { case?: string; kind?: string; attempt?: number; outcome?: string; failure_class?: string; cost_usd?: number; cost_known?: boolean; execution?: string } interface CollectorPanel { case: string; status: string; redClass: string | null } function readJson(file: string): T | null { try { return JSON.parse(fs.readFileSync(file, 'utf8')) as T; } catch { return null; } } /** One panel per planned case from the run's trial outcomes (trial panels from the collector's panelVerdict). */ function casePanels(planned: Map, outcomes: OutcomeLine[], collector: CollectorPanel[], flaky: FlakeRecord[] = []): CasePanel[] { return [...planned].sort(([a], [b]) => (a < b ? -1 : 1)).map(([id, kind]) => { const mine = outcomes.filter(o => o.case === id && o.execution !== 'reused'); const cost = mine.length && mine.every(o => o.cost_known !== false && typeof o.cost_usd === 'number') ? Math.round(mine.reduce((sum, o) => sum + o.cost_usd!, 0) * 10_000) / 10_000 : null; // Rule 16: the report resolved this case FLAKY (failed once, passed its one rerun): a PASS that names both runs. const flake = flaky.find(f => f.case === id); if (flake) return { case: id, kind, status: 'PASS', red_class: null, cost_usd: cost, flaky: { first: flake.first.where, rerun: flake.rerun.where } }; const panel = collector.find(p => p.case === id); if (panel) return { case: id, kind, status: panel.status, red_class: panel.redClass, cost_usd: cost }; const failed = mine.filter(o => o.outcome === 'failed'); if (failed.length) return { case: id, kind, status: 'FAIL', red_class: failed.every(o => o.failure_class === 'infra') ? 'INFRA' : 'VERDICT', cost_usd: cost }; if (mine.some(o => o.outcome === 'passed')) return { case: id, kind, status: 'PASS', red_class: null, cost_usd: cost }; return { case: id, kind, status: mine.length ? 'SKIPPED' : 'MISSING', red_class: 'INCOMPLETE', cost_usd: cost }; }); } export type TrainRecord = TrainGateRecord & { run: { id: number; attempt: number; url: string }; prefixes: TrainPrefixTree[] }; /** The green-train record a train's own run publishes (T1's TrainGateRecord plus the run and the prefix trees). */ export function buildTrainRecord(o: { reportDir: string; nonce: string; head: string; tree: string; reconcileExit: number; rootDir?: string; prefixes: TrainPrefixTree[]; run: TrainRecord['run'] }): TrainRecord { const manifest = parseRunManifest(fs.readFileSync(path.join(o.reportDir, 'manifest.json'), 'utf8')); const planned = plannedCases(manifest, o.rootDir); let lines: OutcomeLine[] = []; try { lines = fs.readFileSync(path.join(o.reportDir, 'trial-outcomes.jsonl'), 'utf8').split('\n').filter(Boolean).map(line => JSON.parse(line) as OutcomeLine); } catch { lines = []; } const primary = Math.min(...lines.map(l => l.attempt ?? 1), Infinity); const collector = readJson<{ verdict?: { verdict?: string }; census?: { unattributed?: unknown[] }; panels?: CollectorPanel[]; flaky?: FlakeRecord[] }>(path.join(o.reportDir, 'collector-outcomes.json')); const panels = casePanels(planned, lines.filter(l => (l.attempt ?? 1) === primary), collector?.panels ?? [], collector?.flaky ?? []); const artifacts = loadSliceArtifacts(o.reportDir); const first = Math.min(...artifacts.map(a => a.result.attempt ?? 1), Infinity); const results = artifacts.filter(a => (a.result.attempt ?? 1) === first).map(a => a.result); const reportedSlices = new Set(results.map(r => r.sliceIndex)); const reportedFiles = new Set(results.flatMap(r => r.outcomes.flatMap(outcome => outcome.files))); const count = (judge: boolean, from: Iterable<[string, string]>) => [...from].filter(([, kind]) => (kind === 'judge') === judge).length; const executed = panels.filter(p => p.status === 'PASS' || p.status === 'FAIL').map(p => [p.case, p.kind] as [string, string]); const census = { complete: false, planned: { rule: count(false, planned), judge: count(true, planned) }, executed: { rule: count(false, executed), judge: count(true, executed) }, unbound: collector?.census?.unattributed?.length ?? 0, partial_shards: results.filter(r => r.checkpoint).length, cancelled_shards: manifest.entries.filter(e => e.status === 'planned' && reportedSlices.has(e.slice) && !reportedFiles.has(e.file)).length, killed_before_upload: Array.from({ length: manifest.sliceCount }, (_, i) => i + 1).filter(index => !reportedSlices.has(index)).length, }; census.complete = o.reconcileExit === 0 && !!collector && planned.size > 0 && census.executed.rule === census.planned.rule && census.executed.judge === census.planned.judge && !census.unbound && !census.partial_shards && !census.cancelled_shards && !census.killed_before_upload && panels.every(p => p.status === 'PASS'); const verdict = census.complete && collector?.verdict?.verdict === 'GREEN' ? 'GREEN' : 'RED'; return { schema: 1, kind: 'train-gate', nonce: o.nonce, head_sha: o.head, tree: o.tree, verdict, census, panels, run: o.run, prefixes: o.prefixes }; } export interface BisectPlan { schema: 1; nonce: string; cases: Array<{ id: string; kind: string }>; prefixes: string[] } /** One executor per prefix; each case runs one trial (and at most one rule-16 rerun inside its 30 minutes), so a slice's ceiling grows with the case count. */ export function bisectMatrix(plan: BisectPlan): { slices: number[]; timeouts: number[]; refs: string[] } { const minutes = Math.min(350, 20 + 30 * plan.cases.length); return { slices: plan.prefixes.map((_, i) => i + 1), timeouts: [0, ...plan.prefixes.map(() => minutes)], refs: ['', ...plan.prefixes] }; } export type CaseRunner = (id: string, kind: string, evalDir: string) => Promise<{ status: string; redClass: string | null }>; /** * Run every red case once on the checked-out prefix; a case that leaves no * verdict is INCOMPLETE (T1 relaunches it). Rule 16 holds here as in the * gate: a case verdict that fails (red class VERDICT) reruns once on the same * prefix; passing makes it a flaky PASS naming both runs, failing again keeps * it red. INFRA and INCOMPLETE are never rerun. */ export async function runBisectPrefix(plan: BisectPlan, index: number, o: { rootDir: string; evalDir: string; checkedOut: string; runCase?: CaseRunner }): Promise { const sha = plan.prefixes[index - 1]; if (!sha) throw new Error(`bisect slice ${index} names no prefix (${plan.prefixes.length} planned)`); if (o.checkedOut !== sha) throw new Error(`bisect slice ${index} checked out ${o.checkedOut.slice(0, 12)}, not its prefix ${sha.slice(0, 12)}`); const runCase = o.runCase ?? ((id, kind, evalDir) => runOneTrial(id, kind, evalDir, o.rootDir)); const panels: CasePanel[] = []; for (const c of plan.cases) { const dir = path.join(o.evalDir, c.id.replace(/[^A-Za-z0-9._-]/g, '_')); const run = async (evalDir: string): Promise<{ status: string; redClass: string | null }> => { try { return await runCase(c.id, c.kind, evalDir); } catch (error) { console.error(`[train-gate] ${c.id}: ${error instanceof Error ? error.message : String(error)}`); return { status: 'INCOMPLETE', redClass: 'INCOMPLETE' }; } }; const verdict = await run(dir); const rerun = verdict.status === 'FAIL' && verdict.redClass === 'VERDICT' ? await run(`${dir}.rerun`) : null; const flaky = rerun?.status === 'PASS'; const final = flaky ? rerun! : verdict; panels.push({ case: c.id, kind: c.kind, status: final.status, red_class: final.redClass, cost_usd: null, ...(flaky ? { flaky: { first: `prefix ${sha.slice(0, 12)} slice ${index}`, rerun: `prefix ${sha.slice(0, 12)} slice ${index} rerun` } } : {}) }); console.log(`[train-gate] prefix ${sha.slice(0, 12)} (slice ${index}): ${c.id} ${verdict.status}${verdict.redClass ? ` (${verdict.redClass})` : ''}` + `${rerun ? `; rule-16 rerun ${rerun.status}${rerun.redClass ? ` (${rerun.redClass})` : ''}${flaky ? ': FLAKY, counted as PASS' : ': failed twice, red'}` : ''}`); } return { schema: 1, kind: 'train-bisect-prefix', nonce: plan.nonce, prefix_sha: sha, checked_out_sha: o.checkedOut, panels }; } /** One trial of one case exactly as CI runs it: E2E cases through the panel runner, a judge through its file with only it selected. */ async function runOneTrial(id: string, kind: string, evalDir: string, rootDir: string): Promise<{ status: string; redClass: string | null }> { const { runPaidShards } = await import('./test-paid-shards'); if (kind !== 'judge') { const { runCaseDiagnosis } = await import('./lib/paid-report'); const v = await runCaseDiagnosis(id, { trials: 1, jobs: 1, rootDir, evalDirBase: evalDir, runShards: runPaidShards }); return { status: v.status, redClass: v.redClass }; } const env = { ...process.env, EVALS: '1', EVALS_TIER: 'gate', EVALS_PREFLIGHT_OK: '1', EVALS_ALL: '1', ...paidSelectionEnv('full', { e2e: [], judges: [id] }, 'train bisect') }; const [outcome] = (await runPaidShards([[JUDGE_FILE]], { jobs: 1, rootDir, env, evalDirBase: evalDir })).outcomes; const trials = outcome && outcome.runnerError === undefined && ['passed', 'failed', 'timed-out'].includes(outcome.status) ? [{ trial: 1, outcome: outcome.status === 'passed' ? 'passed' as const : 'failed' as const, ...(outcome.status === 'passed' ? {} : { failure_class: outcome.status === 'timed-out' ? 'timeout' as const : 'assertion' as const }) }] : []; const v = panelVerdict({ case: id, kind: 'judge', panel: { n: 1, k: 1 }, trials, quarantined: caseTrialPlan(id).quarantined }); return { status: v.status, redClass: v.redClass }; } /** The free summary of a PR push under a train policy: how to ask for the screen, then its selection and estimated cost. */ export function screenSummary(manifest: PaidRunManifest, pr: string, rootDir = ROOT): string[] { const planned = plannedCases(manifest, rootDir); const recorded = loadPaidTestDurations(rootDir, manifest.tier); const serial = manifest.entries.filter(e => e.status === 'planned').reduce((sum, e) => sum + (recordedShardMs(recorded, e.file) ?? UNRECORDED_SHARD_MS), 0); const full = plannedSerialMs('gate', rootDir); const usd = full > 0 ? Math.min(FULL_GATE_USD, FULL_GATE_USD * serial / full) : null; const judges = [...planned.values()].filter(kind => kind === 'judge').length; return [ `To run the paid screen of this head: \`gstack-ship-queue ready --pr ${pr} --screen\` (add \`--broad\` for today's full PR selection). PR pushes start no paid slices under this repository's train policy; the train's full gate covers every merge.`, `- Screening selection: ${planned.size - judges} case(s) + ${judges} judge(s), PR coverage mode \`${manifest.prCoverage?.mode ?? 'broad'}\``, `- Estimated screen cost: ${usd === null ? 'unknown' : `~$${usd.toFixed(2)}`} (recorded shard durations scaled to the $${FULL_GATE_USD} full gate; shards without a recorded duration count ${UNRECORDED_SHARD_MS / 60_000} min). No paid case ran on this push.`, ]; } const USAGE = `usage: bun run scripts/train-gate.ts route --github-output (inputs from the route job's env) bun run scripts/train-gate.ts bisect-plan --cases --prefixes --nonce --out [--github-output ] bun run scripts/train-gate.ts bisect-prefix --plan --index --out bun run scripts/train-gate.ts train-record --nonce --head --reconcile-exit --run-id --run-attempt --repo --out bun run scripts/train-gate.ts screen-summary --pr `; if (import.meta.main) { const [command, ...rest] = process.argv.slice(2); const flag = (name: string) => { const i = rest.indexOf(`--${name}`); return i >= 0 ? rest[i + 1] : undefined; }; const fail = (message: string): never => { console.error(`::error::${message}`); process.exit(1); }; const usage = (): never => { console.error(USAGE); process.exit(2); }; const output = (lines: string[]) => { const file = flag('github-output'); if (file) fs.appendFileSync(file, `${lines.join('\n')}\n`); }; const git = gitIn(process.cwd()); if (command === 'route') { const env = process.env; const route = routeDispatch({ nonce: env.NONCE ?? '', refName: env.GITHUB_REF_NAME ?? '', head: git(['rev-parse', 'HEAD']).out, expectedSha: env.EXPECTED_SHA ?? '', evalsAll: env.EVALS_ALL === 'true', trainPolicy: env.TRAIN_POLICY === 'true', cases: env.CASES ?? '', prefixShas: env.PREFIX_SHAS ?? '', screenPr: env.SCREEN_PR ?? '', screenMergeSha: env.SCREEN_MERGE_SHA ?? '', screenPolicy: env.SCREEN_POLICY ?? '' }, { isAncestor: (commit, head) => git(['merge-base', '--is-ancestor', commit, head]).ok, gateCases: () => fullGateCases(process.cwd()) }); if ('error' in route) fail(`dispatch refused before any paid work: ${route.error}`); const ok = route as Route; output([`mode=${ok.mode}`, `cases=${ok.cases.join(',')}`, `prefixes=${ok.prefixes.join(',')}`]); console.log(`- Run mode: ${ok.mode}${ok.prefixes.length ? ` (${ok.cases.length} case(s) on ${ok.prefixes.length} prefix(es))` : ''}`); } else if (command === 'bisect-plan') { const [cases, prefixes, nonce, out] = [list(flag('cases') ?? ''), list(flag('prefixes') ?? ''), flag('nonce'), flag('out')]; if (!cases.length || !prefixes.length || !nonce || !out) usage(); const plan = fullGateCases(process.cwd()); const bisect: BisectPlan = { schema: 1, nonce: nonce!, cases: cases.map(id => ({ id, kind: plan.get(id) ?? fail(`case ${id} is not in the full gate's plan`) })), prefixes }; fs.mkdirSync(path.dirname(out!), { recursive: true }); fs.writeFileSync(out!, `${JSON.stringify(bisect, null, 2)}\n`); const m = bisectMatrix(bisect); output([`slices=${JSON.stringify(m.slices)}`, `slice_timeouts=${JSON.stringify(m.timeouts)}`, `prefix_refs=${JSON.stringify(m.refs)}`]); console.log(`- Bisect: ${cases.length} case(s) x ${prefixes.length} prefix(es), one trial each`); } else if (command === 'bisect-prefix') { const [planPath, index, out] = [flag('plan'), Number(flag('index')), flag('out')]; if (!planPath || !Number.isSafeInteger(index) || index < 1 || !out) usage(); const plan = readJson(planPath!) ?? fail(`unreadable bisect plan ${planPath}`); const record = await runBisectPrefix(plan, index, { rootDir: process.cwd(), evalDir: path.join(out!, 'eval'), checkedOut: git(['rev-parse', 'HEAD']).out }); fs.mkdirSync(out!, { recursive: true }); fs.writeFileSync(path.join(out!, 'bisect-prefix.json'), `${JSON.stringify(record, null, 2)}\n`); } else if (command === 'train-record') { const [reportDir, nonce, head, repo, out] = [rest[0], flag('nonce'), flag('head'), flag('repo'), flag('out')]; if (!reportDir || !nonce || !head || !SHA.test(head) || !repo || !out) usage(); const prefixes = trainPrefixes(git, head!, process.cwd(), loadPolicyFromHead(process.cwd()).policy); if ('error' in prefixes) fail(`no train record: ${prefixes.error}`); const runId = Number(flag('run-id')); const record = buildTrainRecord({ reportDir: reportDir!, nonce: nonce!, head: head!, tree: git(['rev-parse', `${head}^{tree}`]).out, reconcileExit: /^\d+$/.test(flag('reconcile-exit') ?? '') ? Number(flag('reconcile-exit')) : -1, prefixes: prefixes as TrainPrefixTree[], run: { id: runId, attempt: Number(flag('run-attempt')), url: `https://github.com/${repo}/actions/runs/${runId}` } }); fs.mkdirSync(path.dirname(out!), { recursive: true }); fs.writeFileSync(out!, `${JSON.stringify(record, null, 2)}\n`); const c = record.census; console.log(`- Train record: ${record.verdict} (rule ${c.executed.rule}/${c.planned.rule}, judge ${c.executed.judge}/${c.planned.judge}, unbound ${c.unbound}, partial ${c.partial_shards}, cancelled ${c.cancelled_shards}, killed before upload ${c.killed_before_upload}); prefixes T1..T${record.prefixes.length}`); } else if (command === 'screen-summary') { const [manifestPath, pr] = [rest[0], flag('pr')]; if (!manifestPath || !pr) usage(); for (const line of screenSummary(parseRunManifest(fs.readFileSync(manifestPath!, 'utf8')), pr!)) console.log(line); } else usage(); }