mirror of
https://github.com/garrytan/gstack.git
synced 2026-11-10 18:50:18 +01:00
Merge remote-tracking branch 'origin/main' into garrytan/reliability-followups-oct4
# Conflicts: # .github/workflows/evals-periodic.yml # CHANGELOG.md # CONTRIBUTING.md # TODOS.md # docs/TESTING_INTERNALS.md # scripts/eval-flake-rank.ts # scripts/lib/paid-report.ts # scripts/paid-test-durations.json # scripts/test-paid-shards.ts # test/eng-finding-retry-budget.test.ts # test/fixtures/module-size-ratchet.json # test/helpers/carve-guards.ts # test/helpers/touchfiles-data.ts # test/paid-retry-supervision.test.ts # test/qa-probe-gates.test.ts # test/review-workflow-clarity.test.ts # test/ship-control-flow.test.ts
This commit is contained in:
garrytan committed
commit
0647fd2c32
280 files changed
+13376
-10128
No files matched your search
@@ -0,0 +1,113 @@
|
||||
#!/usr/bin/env bun
|
||||
/**
|
||||
* Keep HARNESS_VERSION honest (CEO-27/ENG-3/DX-6). scripts/harness-version.json
|
||||
* pins the blob of every shared-harness file (GLOBAL_TOUCHFILES plus its
|
||||
* `patterns`). test/harness-version.test.ts fails when a harness file changes
|
||||
* without a decision, and this script records the decision:
|
||||
*
|
||||
* --bump "<reason>" the change alters how cases behave: version + 1,
|
||||
* which starts a new pass-rate series for every case
|
||||
* --non-behavioral "<reason>" the change cannot alter a verdict (refactor,
|
||||
* comment, logging): the new blobs are recorded as
|
||||
* compatible and no series resets
|
||||
* --check print drift and the fix; exit 1 on drift
|
||||
*
|
||||
* Entries are one line per file, so two PRs touching different harness files
|
||||
* merge cleanly. When both bump, or both touch one file, git conflicts on the
|
||||
* manifest: take the higher version, then rerun this script.
|
||||
*/
|
||||
import * as fs from 'node:fs';
|
||||
import * as path from 'node:path';
|
||||
import { spawnSync } from 'node:child_process';
|
||||
import { harnessFiles, treeEntries, type TreeEntry } from './eval-trial-series';
|
||||
|
||||
const ROOT = path.resolve(import.meta.dir, '..');
|
||||
export const MANIFEST = 'scripts/harness-version.json';
|
||||
export const DOC_ANCHOR = 'docs/TESTING_INTERNALS.md#harness-version';
|
||||
|
||||
export interface ManifestEntry { blob: string; since: number; nonBehavioral?: string }
|
||||
export interface HarnessManifest { version: number; patterns: string[]; files: Record<string, ManifestEntry> }
|
||||
export interface Drift { changed: string[]; added: string[]; removed: string[] }
|
||||
|
||||
export function readManifest(root = ROOT): HarnessManifest {
|
||||
return JSON.parse(fs.readFileSync(path.join(root, MANIFEST), 'utf8'));
|
||||
}
|
||||
|
||||
/** The tracked harness files with blobs of their working-tree bytes (an unstaged edit counts; a deleted file drops out). */
|
||||
export function workingHarnessFiles(root: string, patterns: readonly string[]): TreeEntry[] {
|
||||
const tracked = harnessFiles(treeEntries(root), undefined, patterns).filter(entry => fs.existsSync(path.join(root, entry.file)));
|
||||
if (tracked.length === 0) return [];
|
||||
const hashed = spawnSync('git', ['hash-object', '--', ...tracked.map(entry => entry.file)], { cwd: root, encoding: 'utf8', timeout: 30_000 });
|
||||
if (hashed.status !== 0) throw new Error(`git hash-object failed: ${hashed.stderr}`);
|
||||
const blobs = hashed.stdout.trim().split('\n');
|
||||
return tracked.map((entry, index) => ({ file: entry.file, blob: blobs[index]! }));
|
||||
}
|
||||
|
||||
/** Harness files whose blob differs from, is missing from, or no longer matches an entry in the manifest. */
|
||||
export function harnessDrift(manifest: HarnessManifest, files: TreeEntry[]): Drift {
|
||||
const current = new Map(files.map(entry => [entry.file, entry.blob]));
|
||||
return {
|
||||
changed: files.filter(entry => manifest.files[entry.file] && manifest.files[entry.file]!.blob !== entry.blob).map(entry => entry.file).sort(),
|
||||
added: files.filter(entry => !manifest.files[entry.file]).map(entry => entry.file).sort(),
|
||||
removed: Object.keys(manifest.files).filter(file => !current.has(file)).sort(),
|
||||
};
|
||||
}
|
||||
|
||||
export const hasDrift = (drift: Drift) => drift.changed.length + drift.added.length + drift.removed.length > 0;
|
||||
|
||||
/** Problem, offending files, both fix commands and the docs anchor (DX-6). */
|
||||
export function formatDrift(drift: Drift, version: number): string {
|
||||
const list = (label: string, files: string[]) => files.length ? [` ${label}: ${files.join(', ')}`] : [];
|
||||
return [
|
||||
`Shared-harness files changed without a HARNESS_VERSION decision (${MANIFEST}, version ${version}):`,
|
||||
...list('changed', drift.changed), ...list('added', drift.added), ...list('removed', drift.removed),
|
||||
'Fix (pick one):',
|
||||
' bun run scripts/bump-harness-version.ts --non-behavioral "<why no verdict can change>" # refactor/comment/logging: no series reset',
|
||||
' bun run scripts/bump-harness-version.ts --bump "<what behavior changed>" # starts a new pass-rate series for every case',
|
||||
`Merge conflict in ${MANIFEST}: take the higher version, then rerun the script. See ${DOC_ANCHOR}.`,
|
||||
].join('\n');
|
||||
}
|
||||
|
||||
/** Record a decision over the current harness files; returns the new manifest. */
|
||||
export function decide(manifest: HarnessManifest, files: TreeEntry[], decision: 'bump' | 'non-behavioral', reason: string): HarnessManifest {
|
||||
if (reason.trim().length < 10) throw new Error('a reason of at least 10 characters is required');
|
||||
const drift = harnessDrift(manifest, files);
|
||||
if (decision === 'bump') {
|
||||
const version = manifest.version + 1;
|
||||
return { ...manifest, version, files: Object.fromEntries(files.map(entry => [entry.file, { blob: entry.blob, since: version }])) };
|
||||
}
|
||||
const touched = new Set([...drift.changed, ...drift.added]);
|
||||
return { ...manifest, files: Object.fromEntries(files.map(entry => [entry.file, touched.has(entry.file)
|
||||
? { blob: entry.blob, since: manifest.version, nonBehavioral: reason.trim() }
|
||||
: manifest.files[entry.file]!])) };
|
||||
}
|
||||
|
||||
/** Stable, one-entry-per-line JSON so concurrent edits to different files merge cleanly. */
|
||||
export function serializeManifest(manifest: HarnessManifest): string {
|
||||
const files = Object.keys(manifest.files).sort().map(file => ` ${JSON.stringify(file)}: ${JSON.stringify(manifest.files[file])}`);
|
||||
return `{\n "version": ${manifest.version},\n "patterns": [\n${manifest.patterns.map(pattern => ` ${JSON.stringify(pattern)}`).join(',\n')}\n ],\n`
|
||||
+ ` "files": {\n${files.join(',\n')}\n }\n}\n`;
|
||||
}
|
||||
|
||||
if (import.meta.main) {
|
||||
const argv = process.argv.slice(2);
|
||||
const manifest = readManifest();
|
||||
const files = workingHarnessFiles(ROOT, manifest.patterns);
|
||||
const drift = harnessDrift(manifest, files);
|
||||
const mode = argv[0];
|
||||
if (mode === '--check' || mode === undefined) {
|
||||
if (hasDrift(drift)) { console.error(formatDrift(drift, manifest.version)); process.exit(1); }
|
||||
console.log(`harness version ${manifest.version}: ${files.length} pinned file(s), no drift`);
|
||||
process.exit(0);
|
||||
}
|
||||
if (mode !== '--bump' && mode !== '--non-behavioral') {
|
||||
console.error('usage: bun run scripts/bump-harness-version.ts --check | --bump "<reason>" | --non-behavioral "<reason>"');
|
||||
process.exit(2);
|
||||
}
|
||||
if (mode === '--non-behavioral' && !hasDrift(drift)) { console.log('no harness drift: nothing to record'); process.exit(0); }
|
||||
const next = decide(manifest, files, mode === '--bump' ? 'bump' : 'non-behavioral', argv[1] ?? '');
|
||||
fs.writeFileSync(path.join(ROOT, MANIFEST), serializeManifest(next));
|
||||
console.log(mode === '--bump'
|
||||
? `HARNESS_VERSION ${manifest.version} -> ${next.version}: every case starts a new pass-rate series (reason: ${argv[1]})`
|
||||
: `recorded ${drift.changed.length + drift.added.length} file(s) as non-behavioral at HARNESS_VERSION ${next.version}${drift.removed.length ? `, dropped ${drift.removed.length} removed file(s)` : ''}`);
|
||||
}
|
||||
+60
-35
@@ -18,44 +18,69 @@ import { PARITY_INVARIANTS } from '../test/helpers/parity-harness';
|
||||
|
||||
const ROOT = path.resolve(import.meta.dir, '..');
|
||||
|
||||
function arg(name: string): string | undefined {
|
||||
const i = process.argv.indexOf(name);
|
||||
if (i === -1) return undefined;
|
||||
return process.argv[i + 1];
|
||||
const USAGE = `Usage: bun run scripts/capture-baseline.ts [--tag <tag>] [--out <path>]
|
||||
|
||||
Captures a parity baseline snapshot of every SKILL.md and writes it to
|
||||
test/fixtures/parity-baseline-<tag>.json (parity-baseline-current.json
|
||||
without --tag), or to --out.`;
|
||||
|
||||
export type CaptureArgs = { help: true } | { help: false; tag?: string; out?: string };
|
||||
|
||||
export function parseCaptureArgs(argv: string[]): CaptureArgs {
|
||||
const parsed: { tag?: string; out?: string } = {};
|
||||
for (let i = 0; i < argv.length; i++) {
|
||||
const flag = argv[i]!;
|
||||
if (flag === '--help' || flag === '-h') return { help: true };
|
||||
if (flag !== '--tag' && flag !== '--out') throw new Error(`unknown argument "${flag}"`);
|
||||
const value = argv[++i];
|
||||
if (!value || value.startsWith('-')) throw new Error(`${flag} needs a value`);
|
||||
parsed[flag === '--tag' ? 'tag' : 'out'] = value;
|
||||
}
|
||||
return { help: false, ...parsed };
|
||||
}
|
||||
|
||||
const tag = arg('--tag');
|
||||
const outOverride = arg('--out');
|
||||
const defaultOut = path.join(
|
||||
ROOT,
|
||||
'test',
|
||||
'fixtures',
|
||||
`parity-baseline-${tag ?? 'current'}.json`,
|
||||
);
|
||||
const outPath = outOverride ? path.resolve(outOverride) : defaultOut;
|
||||
function main(): void {
|
||||
let args: CaptureArgs;
|
||||
try {
|
||||
args = parseCaptureArgs(process.argv.slice(2));
|
||||
} catch (err) {
|
||||
console.error(`capture-baseline: ${(err as Error).message}\n\n${USAGE}`);
|
||||
process.exit(2);
|
||||
}
|
||||
if (args.help) {
|
||||
console.log(USAGE);
|
||||
return;
|
||||
}
|
||||
const { tag } = args;
|
||||
const outPath = args.out
|
||||
? path.resolve(args.out)
|
||||
: path.join(ROOT, 'test', 'fixtures', `parity-baseline-${tag ?? 'current'}.json`);
|
||||
|
||||
const baseline = captureBaseline({
|
||||
repoRoot: ROOT,
|
||||
tag,
|
||||
// Carved skills record UNION bytes (skeleton + sections/*.md) so the
|
||||
// baseline measures the same thing parity-harness checks against.
|
||||
sectionedSkills: PARITY_INVARIANTS.filter(i => i.sectioned).map(i => i.skill),
|
||||
});
|
||||
const baseline = captureBaseline({
|
||||
repoRoot: ROOT,
|
||||
tag,
|
||||
// Carved skills record UNION bytes (skeleton + sections/*.md) so the
|
||||
// baseline measures the same thing parity-harness checks against.
|
||||
sectionedSkills: PARITY_INVARIANTS.filter(i => i.sectioned).map(i => i.skill),
|
||||
});
|
||||
|
||||
fs.mkdirSync(path.dirname(outPath), { recursive: true });
|
||||
fs.writeFileSync(outPath, JSON.stringify(baseline, null, 2) + '\n');
|
||||
fs.mkdirSync(path.dirname(outPath), { recursive: true });
|
||||
fs.writeFileSync(outPath, JSON.stringify(baseline, null, 2) + '\n');
|
||||
|
||||
const totalKB = Math.round(baseline.totalCorpusBytes / 1024);
|
||||
const top3 = baseline.topHeaviest.slice(0, 3);
|
||||
console.log(`Parity baseline captured: ${outPath}`);
|
||||
console.log(` tag: ${baseline.tag}`);
|
||||
console.log(` commit: ${baseline.capturedFromCommit}`);
|
||||
console.log(` branch: ${baseline.capturedFromBranch}`);
|
||||
console.log(` skills: ${baseline.totalSkills}`);
|
||||
console.log(` total corpus: ${totalKB} KB`);
|
||||
console.log(` catalog tokens: ~${baseline.estTotalCatalogTokens}`);
|
||||
console.log(` top 3 heaviest:`);
|
||||
for (const s of top3) {
|
||||
const kb = Math.round(s.skillMdBytes / 1024);
|
||||
console.log(` ${s.skill.padEnd(28)} ${kb} KB (${s.skillMdLines} lines, ~${s.estTokens} tokens)`);
|
||||
const totalKB = Math.round(baseline.totalCorpusBytes / 1024);
|
||||
const top3 = baseline.topHeaviest.slice(0, 3);
|
||||
console.log(`Parity baseline captured: ${outPath}`);
|
||||
console.log(` tag: ${baseline.tag}`);
|
||||
console.log(` commit: ${baseline.capturedFromCommit}`);
|
||||
console.log(` branch: ${baseline.capturedFromBranch}`);
|
||||
console.log(` skills: ${baseline.totalSkills}`);
|
||||
console.log(` total corpus: ${totalKB} KB`);
|
||||
console.log(` catalog tokens: ~${baseline.estTotalCatalogTokens}`);
|
||||
console.log(` top 3 heaviest:`);
|
||||
for (const s of top3) {
|
||||
const kb = Math.round(s.skillMdBytes / 1024);
|
||||
console.log(` ${s.skill.padEnd(28)} ${kb} KB (${s.skillMdLines} lines, ~${s.estTokens} tokens)`);
|
||||
}
|
||||
}
|
||||
|
||||
if (import.meta.main) main();
|
||||
+57
-12
@@ -169,6 +169,13 @@ export function prepareE2EShardReuse(request: E2EShardReuseRequest): {
|
||||
/** True when the inputs are unchanged since `before` (the outcome may carry inputKey). */
|
||||
unchanged(): boolean;
|
||||
publish(): void;
|
||||
/**
|
||||
* A completed single-trial shard that did not pass: write its negative
|
||||
* receipt at the execution boundary (CEO-25), beside the pass receipts the
|
||||
* slice artifact carries, so a cancelled run's FAIL still blocks an older
|
||||
* PASS for the same inputs when its report never runs.
|
||||
*/
|
||||
publishFailure(): void;
|
||||
} | null {
|
||||
if (e2eReuseLaneProblem(request.env, 'pr') !== null) return null;
|
||||
const before = e2eShardIdentity(request);
|
||||
@@ -194,21 +201,31 @@ export function prepareE2EShardReuse(request: E2EShardReuseRequest): {
|
||||
},
|
||||
publish() {
|
||||
const after = e2eShardIdentity(request);
|
||||
const env = request.env;
|
||||
const runId = env.GITHUB_RUN_ID ? `${env.GITHUB_RUN_ID}/${env.GITHUB_RUN_ATTEMPT ?? '1'}` : env.EVALS_RUN_ID;
|
||||
const revision = spawnSync('git', ['rev-parse', 'HEAD'], { cwd: request.root, encoding: 'utf8', timeout: 3_000 });
|
||||
if (after.status !== 'eligible' || !runId || revision.status !== 0) return;
|
||||
const source = executionSource(request);
|
||||
if (after.status !== 'eligible' || !source) return;
|
||||
storeEvalInputCache({ ...common, before: before.identity, after: after.identity, proof: {
|
||||
execution: 'new', finalized: true, completeAttemptHistory: true, exitCode: 0, timedOut: false,
|
||||
cancelled: false, skipped: 0, failed: 0, passed: before.identity.caseIds.length,
|
||||
cases: before.identity.caseIds.map(id => ({ id, outcome: 'passed' as const, attempt: 1 as const })),
|
||||
source: { runId, revision: revision.stdout.trim(), completedAt: Date.now() },
|
||||
result: { key: request.key, cases: before.identity.caseIds },
|
||||
source, result: { key: request.key, cases: before.identity.caseIds },
|
||||
} });
|
||||
},
|
||||
publishFailure() {
|
||||
const source = executionSource(request);
|
||||
if (source) writeNegativeReceipt(common.cacheDir, { schema: 1, key: before.identity.key, source });
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
/** Provenance of a receipt written by this execution: `<run id>/<attempt>`, the checked-out revision and now. */
|
||||
function executionSource(request: E2EShardReuseRequest): { runId: string; revision: string; completedAt: number } | null {
|
||||
const env = request.env;
|
||||
const runId = env.GITHUB_RUN_ID ? `${env.GITHUB_RUN_ID}/${env.GITHUB_RUN_ATTEMPT ?? '1'}` : env.EVALS_RUN_ID;
|
||||
const revision = spawnSync('git', ['rev-parse', 'HEAD'], { cwd: request.root, encoding: 'utf8', timeout: 3_000 });
|
||||
if (!runId || revision.status !== 0) return null;
|
||||
return { runId, revision: revision.stdout.trim(), completedAt: Date.now() };
|
||||
}
|
||||
|
||||
// ─── Panel receipts, negative receipts and the planner's receipt selection ──
|
||||
//
|
||||
// Reuse is decided by the planner, once per panel: it ships the plan a
|
||||
@@ -231,6 +248,7 @@ export interface PanelReceipt {
|
||||
export interface NegativeReceipt { schema: 1; key: string; source: { runId: string; revision: string; completedAt: number } }
|
||||
|
||||
const RECEIPT_KEY = /^[a-f0-9]{64}$/;
|
||||
const RECEIPT_FILE = /^[a-f0-9]{64}(?:\.panel|\.fail)?\.json$/;
|
||||
const validSource = (source: any) => !!source && typeof source.runId === 'string' && /^[\w./-]{1,160}$/.test(source.runId)
|
||||
&& typeof source.revision === 'string' && /^[a-f0-9]{40}$/.test(source.revision) && Number.isSafeInteger(source.completedAt) && source.completedAt > 0;
|
||||
|
||||
@@ -268,10 +286,24 @@ export function writeNegativeReceipt(dir: string, receipt: NegativeReceipt): voi
|
||||
fs.writeFileSync(path.join(dir, `${receipt.key}.fail.json`), `${JSON.stringify(receipt)}\n`, { mode: 0o600 });
|
||||
}
|
||||
|
||||
const receiptTime = (file: string): number => {
|
||||
const receiptSource = (file: string): { runId: string; completedAt: number } => {
|
||||
const parsed = readJson(file);
|
||||
return Number(parsed?.source?.completedAt ?? parsed?.proof?.source?.completedAt) || 0;
|
||||
const source = parsed?.source ?? parsed?.proof?.source;
|
||||
return { runId: typeof source?.runId === 'string' ? source.runId : '', completedAt: Number(source?.completedAt) || 0 };
|
||||
};
|
||||
const receiptTime = (file: string): number => receiptSource(file).completedAt;
|
||||
|
||||
/**
|
||||
* Receipt order (ENG-6): within one CI run, a later attempt is newer
|
||||
* whatever its clock says; otherwise the later `completedAt` is newer.
|
||||
* Positive when `a` is newer than `b`, zero on a tie.
|
||||
*/
|
||||
export function compareReceiptSources(a: { runId: string; completedAt: number }, b: { runId: string; completedAt: number }): number {
|
||||
const [runA, attemptA] = a.runId.split('/');
|
||||
const [runB, attemptB] = b.runId.split('/');
|
||||
if (runA && runA === runB && Number(attemptA) !== Number(attemptB)) return (Number(attemptA) || 0) - (Number(attemptB) || 0);
|
||||
return a.completedAt - b.completedAt;
|
||||
}
|
||||
|
||||
/**
|
||||
* Planner-side selection: copy `from` into `to`, dropping every pass or panel
|
||||
@@ -284,7 +316,7 @@ export function selectPlanReceipts(from: string, to: string, now = Date.now()):
|
||||
const blocked: string[] = [];
|
||||
let shipped = 0;
|
||||
let names: string[] = [];
|
||||
try { names = fs.readdirSync(from).filter(name => name.endsWith('.json')); } catch { return { shipped, blocked }; }
|
||||
try { names = fs.readdirSync(from).filter(name => RECEIPT_FILE.test(name)); } catch { return { shipped, blocked }; }
|
||||
for (const name of names) {
|
||||
const file = path.join(from, name);
|
||||
const [key, suffix] = [name.slice(0, 64), name.slice(64)];
|
||||
@@ -298,21 +330,34 @@ export function selectPlanReceipts(from: string, to: string, now = Date.now()):
|
||||
return { shipped, blocked };
|
||||
}
|
||||
|
||||
/** Merge receipt directories into one store, keeping the newest file per name. */
|
||||
/**
|
||||
* Merge receipt directories into one store, keeping the newest file per name
|
||||
* by (run attempt, completedAt); a tie keeps the file already there. Negative
|
||||
* receipts merge before passes, so a reader of a partial merge never sees a
|
||||
* PASS without the FAIL that blocks it. The recovery checkpoint carries over.
|
||||
*/
|
||||
export function mergeReceiptDirs(out: string, dirs: string[]): number {
|
||||
fs.mkdirSync(out, { recursive: true });
|
||||
let merged = 0;
|
||||
const negativesFirst = (a: string, b: string) => Number(b.endsWith('.fail.json')) - Number(a.endsWith('.fail.json')) || (a < b ? -1 : a > b ? 1 : 0);
|
||||
for (const dir of dirs) {
|
||||
let names: string[] = [];
|
||||
try { names = fs.readdirSync(dir).filter(name => name.endsWith('.json')); } catch { continue; }
|
||||
try { names = fs.readdirSync(dir).filter(name => RECEIPT_FILE.test(name)).sort(negativesFirst); } catch { continue; }
|
||||
for (const name of names) {
|
||||
const source = path.join(dir, name);
|
||||
const target = path.join(out, name);
|
||||
if (!fs.lstatSync(source).isFile()) continue;
|
||||
if (fs.existsSync(target) && receiptTime(target) >= receiptTime(source)) continue;
|
||||
if (fs.existsSync(target) && compareReceiptSources(receiptSource(source), receiptSource(target)) <= 0) continue;
|
||||
fs.copyFileSync(source, target);
|
||||
merged++;
|
||||
}
|
||||
// The recovery checkpoint (scripts/recover-receipts.ts) travels with the store; the newest wins.
|
||||
const checkpoint = path.join(dir, 'recovery.json');
|
||||
const current = path.join(out, 'recovery.json');
|
||||
if (fs.existsSync(checkpoint) && fs.lstatSync(checkpoint).isFile()
|
||||
&& (Number(readJson(checkpoint)?.updatedAt) || 0) > (fs.existsSync(current) ? Number(readJson(current)?.updatedAt) || 0 : -1)) {
|
||||
fs.copyFileSync(checkpoint, current);
|
||||
}
|
||||
}
|
||||
return merged;
|
||||
}
|
||||
|
||||
@@ -0,0 +1,380 @@
|
||||
#!/usr/bin/env bun
|
||||
/**
|
||||
* eval-bg — the one entry point for background paid evals (`bun run eval:bg:<lane>`).
|
||||
*
|
||||
* Two backends, one completion contract (W5d, CEO-17, DX-1..3/7, ENG-12/19):
|
||||
*
|
||||
* dispatch The pushed, clean ref is dispatched to CI (evals.yml and/or
|
||||
* evals-periodic.yml on garrytan/gstack) with an `expected_sha` the
|
||||
* workflow verifies and a `nonce` its run-name carries; this process
|
||||
* resolves the run by that nonce and follows it to its conclusion.
|
||||
* local The lane's sharded runner on this machine, capped at
|
||||
* ceil(1.5 x planned serial seconds / EVALS_JOBS) + 20 min, at most 4 h.
|
||||
*
|
||||
* Both run under bin/gstack-detach with the `gstack-evals` lock, a run-scoped
|
||||
* log under ~/.gstack-dev/eval-runs/ whose first line names the backend and the
|
||||
* tested revision, and the `### gstack-detach EXIT=<code> ###` sentinel. A
|
||||
* followed CI run maps its conclusion: success 0, failure 1, cancelled 130,
|
||||
* anything else 2. `status <run-id|log>` reconnects later.
|
||||
*
|
||||
* Dispatch needs every one of: gh installed and authenticated, origin is
|
||||
* garrytan/gstack with push rights, a branch whose remote head (read with
|
||||
* `gh api`, never git fetch) equals HEAD, and a clean tree (no staged,
|
||||
* unstaged or untracked files, the rule test/helpers/test-selection.ts uses).
|
||||
* Otherwise it runs locally and prints why; `--dispatch` refuses instead.
|
||||
* Dispatched runs are validation runs: fresh, never writing PR receipts.
|
||||
*/
|
||||
import * as fs from 'node:fs';
|
||||
import * as os from 'node:os';
|
||||
import * as path from 'node:path';
|
||||
import { spawnSync } from 'node:child_process';
|
||||
import { detectBaseBranch } from '../test/helpers/touchfiles';
|
||||
import { DEFAULT_JOBS, ROOT, type PaidTier } from './lib/paid-types';
|
||||
import { buildRunManifest, loadPaidTestDurations, recordedShardMs } from './lib/paid-plan';
|
||||
|
||||
export type EvalBgLane = 'pr' | 'gate' | 'periodic' | 'release';
|
||||
export type Backend = 'dispatch' | 'local';
|
||||
export const EVAL_BG_LANES: readonly EvalBgLane[] = ['pr', 'gate', 'periodic', 'release'];
|
||||
export const CANONICAL_REPO = 'garrytan/gstack';
|
||||
export const EVAL_LOCK = 'gstack-evals';
|
||||
export const LOCAL_CAP_SECONDS = 4 * 60 * 60;
|
||||
export const LOCAL_SETUP_SECONDS = 20 * 60;
|
||||
export const LOCAL_CAP_FACTOR = 1.5;
|
||||
/** Packing weight of a shard with no recorded wall: the PR lane's ~10-minute case target. */
|
||||
export const UNRECORDED_SHARD_MS = 600_000;
|
||||
/** How long the dispatch follower may hold the lock: the longest CI lane plus queueing. */
|
||||
export const DISPATCH_FOLLOW_SECONDS = 4 * 60 * 60;
|
||||
|
||||
export interface WorkflowDispatch { workflow: 'evals.yml' | 'evals-periodic.yml'; inputs: Record<string, string> }
|
||||
|
||||
interface LaneSpec {
|
||||
/** The package script the local backend runs. */
|
||||
script: string;
|
||||
/** Worker count the local command defaults to (EVALS_JOBS overrides). */
|
||||
jobs: number;
|
||||
/** Tiers the local command plans, in order. */
|
||||
tiers: Array<{ tier: PaidTier; profile: 'pr' | 'full' }>;
|
||||
}
|
||||
|
||||
export const LANE_SPECS: Record<EvalBgLane, LaneSpec> = {
|
||||
pr: { script: 'test:pr', jobs: 2, tiers: [{ tier: 'gate', profile: 'pr' }] },
|
||||
gate: { script: 'test:gate:sharded', jobs: DEFAULT_JOBS, tiers: [{ tier: 'gate', profile: 'full' }] },
|
||||
periodic: { script: 'test:periodic:sharded', jobs: DEFAULT_JOBS, tiers: [{ tier: 'periodic', profile: 'full' }] },
|
||||
release: { script: 'test:release', jobs: DEFAULT_JOBS, tiers: [{ tier: 'gate', profile: 'full' }, { tier: 'periodic', profile: 'full' }] },
|
||||
};
|
||||
|
||||
/** The workflows a lane dispatches and their inputs (names agreed with evals.yml's workflow_dispatch). */
|
||||
export function dispatchPlan(lane: EvalBgLane, ids: { sha: string; baseRef: string; baseSha: string; nonce: string; prNumber?: number | null }): WorkflowDispatch[] {
|
||||
const evals = (all: boolean): WorkflowDispatch => ({ workflow: 'evals.yml', inputs: {
|
||||
evals_all: String(all), expected_sha: ids.sha, nonce: ids.nonce,
|
||||
...(all ? {} : { base_ref: ids.baseRef, base_sha: ids.baseSha }),
|
||||
// CEO-29/ENG-8: a PR lane run on a ref with an open same-repo PR reads that PR's receipts read-only.
|
||||
...(!all && ids.prNumber ? { pr_receipts: String(ids.prNumber) } : {}),
|
||||
} });
|
||||
const periodic: WorkflowDispatch = { workflow: 'evals-periodic.yml', inputs: { expected_sha: ids.sha, nonce: ids.nonce } };
|
||||
if (lane === 'pr') return [evals(false)];
|
||||
if (lane === 'gate') return [evals(true)];
|
||||
if (lane === 'periodic') return [periodic];
|
||||
return [evals(true), periodic];
|
||||
}
|
||||
|
||||
/**
|
||||
* Local cap: ceil(1.5 x planned serial seconds / jobs) + 20 min setup, at most
|
||||
* 4 h. Serial seconds sum the recorded wall of every planned shard
|
||||
* (scripts/paid-test-durations.json), unrecorded shards at UNRECORDED_SHARD_MS.
|
||||
*/
|
||||
export function localCapSeconds(serialMs: number, jobs: number): { seconds: number; capped: boolean } {
|
||||
if (!Number.isSafeInteger(jobs) || jobs < 1) throw new Error('EVALS_JOBS must be a positive integer');
|
||||
const computed = Math.ceil(LOCAL_CAP_FACTOR * serialMs / jobs / 1000) + LOCAL_SETUP_SECONDS;
|
||||
return { seconds: Math.min(computed, LOCAL_CAP_SECONDS), capped: computed > LOCAL_CAP_SECONDS };
|
||||
}
|
||||
|
||||
/** Planned serial work of a lane on this checkout, from the same planner CI uses. */
|
||||
export function plannedSerialMs(lane: EvalBgLane, rootDir = ROOT): number {
|
||||
return LANE_SPECS[lane].tiers.reduce((total, { tier, profile }) => {
|
||||
const plan = (all: boolean) => buildRunManifest({ tier, profile: all ? 'full' : profile, sliceCount: 1, evalsAll: all,
|
||||
env: all ? { EVALS_ALL: '1' } : {}, rootDir });
|
||||
let manifest;
|
||||
try { manifest = plan(profile === 'full'); }
|
||||
catch { manifest = plan(true); } // a PR diff that needs full validation runs the full gate locally
|
||||
const recorded = loadPaidTestDurations(rootDir, tier);
|
||||
return total + manifest.entries.filter(entry => entry.status === 'planned')
|
||||
.reduce((sum, entry) => sum + (recordedShardMs(recorded, entry.file) ?? UNRECORDED_SHARD_MS), 0);
|
||||
}, 0);
|
||||
}
|
||||
|
||||
export interface CheckoutFacts {
|
||||
sha: string;
|
||||
branch: string | null;
|
||||
/** `git status --porcelain --untracked-files=all` lines. */
|
||||
dirty: string[];
|
||||
/** owner/name of origin, null when unknown. */
|
||||
repo: string | null;
|
||||
ghReady: boolean;
|
||||
canPush: boolean;
|
||||
/** Remote head of the branch on garrytan/gstack (gh api), null when absent. */
|
||||
remoteSha: string | null;
|
||||
/** Commits HEAD is ahead of remoteSha, null when unknown. */
|
||||
ahead: number | null;
|
||||
}
|
||||
|
||||
export type BackendChoice =
|
||||
| { backend: Backend; reason: string }
|
||||
| { backend: 'refuse'; reason: string };
|
||||
|
||||
/** DX-1 checkout contract, one row per reason dispatch is not possible. */
|
||||
export function dispatchBlockers(facts: CheckoutFacts): string[] {
|
||||
const sha = facts.sha.slice(0, 12);
|
||||
const blockers: string[] = [];
|
||||
if (!facts.ghReady) blockers.push('gh is not installed or not authenticated (run `gh auth status`)');
|
||||
if (facts.repo !== CANONICAL_REPO) blockers.push(`origin is ${facts.repo ?? 'unknown'}, not ${CANONICAL_REPO} (a fork cannot dispatch its workflows)`);
|
||||
else if (facts.ghReady && !facts.canPush) blockers.push(`this gh account cannot dispatch workflows on ${CANONICAL_REPO}`);
|
||||
if (!facts.branch) blockers.push(`HEAD ${sha} is detached; dispatch needs a pushed branch`);
|
||||
else if (facts.ghReady && facts.repo === CANONICAL_REPO && facts.remoteSha === null) blockers.push(`branch ${facts.branch} is not on ${CANONICAL_REPO}; push it`);
|
||||
else if (facts.remoteSha !== null && facts.remoteSha !== facts.sha) {
|
||||
blockers.push(facts.ahead !== null
|
||||
? `HEAD ${sha} is ${facts.ahead} commits ahead of origin/${facts.branch}`
|
||||
: `HEAD ${sha} differs from origin/${facts.branch} (${facts.remoteSha.slice(0, 12)})`);
|
||||
}
|
||||
if (facts.dirty.length) blockers.push(`${facts.dirty.length} files are modified or untracked`);
|
||||
return blockers;
|
||||
}
|
||||
|
||||
/** Choose the backend: an explicit mode wins; otherwise dispatch when nothing blocks it. */
|
||||
export function chooseBackend(facts: CheckoutFacts, mode: Backend | 'auto'): BackendChoice {
|
||||
if (mode === 'local') return { backend: 'local', reason: 'local requested (--local or GSTACK_EVAL_BG_MODE=local)' };
|
||||
const blockers = dispatchBlockers(facts);
|
||||
if (!blockers.length) return { backend: 'dispatch', reason: `HEAD ${facts.sha.slice(0, 12)} is clean and pushed to ${CANONICAL_REPO}/${facts.branch}` };
|
||||
if (mode === 'dispatch') return { backend: 'refuse', reason: `${blockers.join(' and ')}; commit and push, or rerun with --local` };
|
||||
return { backend: 'local', reason: `cannot dispatch: ${blockers.join('; ')}` };
|
||||
}
|
||||
|
||||
/** CI conclusion -> the detach exit code (DX-2). */
|
||||
export function conclusionExitCode(conclusion: string | null | undefined): number {
|
||||
if (conclusion === 'success') return 0;
|
||||
if (conclusion === 'failure') return 1;
|
||||
if (conclusion === 'cancelled') return 130;
|
||||
return 2;
|
||||
}
|
||||
|
||||
/** Status words `status` prints, from a detach EXIT code or a CI run. */
|
||||
export function exitStatusWord(code: string): 'passed' | 'failed' | 'cancelled' {
|
||||
if (code === '0') return 'passed';
|
||||
return code === '130' ? 'cancelled' : 'failed';
|
||||
}
|
||||
|
||||
/** Nonce: lane, tested revision and base, plus a random suffix; the dedupe key is everything but the suffix. */
|
||||
export function makeNonce(lane: EvalBgLane, sha: string, baseSha: string, random = Math.random().toString(36).slice(2, 8)): string {
|
||||
return `${dedupeKey(lane, sha, baseSha)}${random}`;
|
||||
}
|
||||
export function dedupeKey(lane: EvalBgLane, sha: string, baseSha: string): string {
|
||||
return `${lane}-${sha.slice(0, 12)}-${(baseSha || 'none').slice(0, 12)}-`;
|
||||
}
|
||||
|
||||
export interface RunSummary { databaseId: number; displayTitle: string; status: string; conclusion: string | null; url: string; event: string }
|
||||
|
||||
/**
|
||||
* DX-3 / ENG-12: an existing dispatch of the same lane, revision and base that
|
||||
* is queued, running or completed successfully is followed instead of
|
||||
* dispatching a duplicate. Only dispatches match (by the nonce in run-name); a
|
||||
* failed or cancelled run is never reused.
|
||||
*/
|
||||
export function findDuplicateRun(runs: readonly RunSummary[], key: string): RunSummary | null {
|
||||
return runs.find(run => run.event === 'workflow_dispatch' && run.displayTitle.includes(`eval-bg ${key}`)
|
||||
&& (run.status !== 'completed' || run.conclusion === 'success')) ?? null;
|
||||
}
|
||||
|
||||
// ─── Process plumbing (git / gh / detach) ────────────────────────────────────
|
||||
|
||||
type Exec = (command: string, args: string[]) => { status: number | null; stdout: string };
|
||||
|
||||
const exec: Exec = (command, args) => {
|
||||
const result = spawnSync(command, args, { cwd: ROOT, encoding: 'utf8', timeout: 60_000, maxBuffer: 16 * 1024 * 1024 });
|
||||
return { status: result.error ? null : result.status, stdout: result.stdout ?? '' };
|
||||
};
|
||||
|
||||
function originRepo(run: Exec): string | null {
|
||||
const url = run('git', ['config', '--get', 'remote.origin.url']).stdout.trim();
|
||||
return /github\.com[:/]([^/]+\/[^/]+?)(?:\.git)?\/?$/.exec(url)?.[1] ?? null;
|
||||
}
|
||||
|
||||
/** Gather the DX-1 facts. Remote state comes from `gh api` only (ENG-19). */
|
||||
export function gatherCheckoutFacts(run: Exec = exec): CheckoutFacts {
|
||||
const sha = run('git', ['rev-parse', 'HEAD']).stdout.trim();
|
||||
const branch = run('git', ['branch', '--show-current']).stdout.trim() || null;
|
||||
const dirty = run('git', ['status', '--porcelain', '--untracked-files=all']).stdout.split('\n').filter(Boolean);
|
||||
const repo = originRepo(run);
|
||||
const ghReady = run('gh', ['auth', 'status']).status === 0;
|
||||
const canPush = ghReady && repo === CANONICAL_REPO
|
||||
&& run('gh', ['api', `repos/${CANONICAL_REPO}`, '--jq', '.permissions.push']).stdout.trim() === 'true';
|
||||
const remote = ghReady && repo === CANONICAL_REPO && branch
|
||||
? run('gh', ['api', `repos/${CANONICAL_REPO}/branches/${encodeURIComponent(branch)}`, '--jq', '.commit.sha']) : null;
|
||||
const remoteSha = remote && remote.status === 0 && /^[0-9a-f]{40}$/.test(remote.stdout.trim()) ? remote.stdout.trim() : null;
|
||||
const count = remoteSha && remoteSha !== sha ? run('git', ['rev-list', '--count', `${remoteSha}..HEAD`]) : null;
|
||||
return { sha, branch, dirty, repo, ghReady, canPush, remoteSha,
|
||||
ahead: count && count.status === 0 ? Number(count.stdout.trim()) : null };
|
||||
}
|
||||
|
||||
/** Run-scoped log path, the same layout gstack-detach uses for its own default. */
|
||||
export function runScopedLog(label: string, branch: string | null, now = new Date()): string {
|
||||
const stamp = now.toISOString().replace(/[-:]/g, '').replace('T', '-').slice(0, 15);
|
||||
return path.join(os.homedir(), '.gstack-dev', 'eval-runs',
|
||||
`${label}-${path.basename(ROOT)}-${(branch ?? 'nobranch').replace(/\//g, '-')}-${stamp}-${process.pid}.log`);
|
||||
}
|
||||
|
||||
/** Start `command` under gstack-detach with the eval lock; the header becomes the log's first line. */
|
||||
export function launchDetached(opts: { log: string; label: string; timeoutSeconds: number; header: string; command: string[]; env?: NodeJS.ProcessEnv }): string {
|
||||
fs.mkdirSync(path.dirname(opts.log), { recursive: true });
|
||||
fs.writeFileSync(opts.log, `${opts.header}\n`, { flag: 'a' });
|
||||
const result = spawnSync(path.join(ROOT, 'bin', 'gstack-detach'), ['--log', opts.log, '--lock', EVAL_LOCK,
|
||||
'--timeout', String(opts.timeoutSeconds), '--label', opts.label, '--', ...opts.command],
|
||||
{ cwd: ROOT, encoding: 'utf8', timeout: 30_000, env: opts.env ?? process.env });
|
||||
if (result.status !== 0) throw new Error(`gstack-detach failed to start: ${result.stderr || result.error?.message}`);
|
||||
return result.stdout.trim();
|
||||
}
|
||||
|
||||
const sleep = (ms: number) => new Promise(resolve => setTimeout(resolve, ms));
|
||||
const pollMs = () => Number(process.env.GSTACK_EVAL_BG_POLL_MS) || 30_000;
|
||||
|
||||
/** Dispatch one workflow (or adopt a duplicate) and follow it; returns the mapped exit code. */
|
||||
async function followWorkflow(dispatch: WorkflowDispatch, ref: string, key: string, run: Exec): Promise<number> {
|
||||
const list = () => {
|
||||
const out = run('gh', ['run', 'list', '--repo', CANONICAL_REPO, '--workflow', dispatch.workflow, '--event', 'workflow_dispatch',
|
||||
'--limit', '30', '--json', 'databaseId,displayTitle,status,conclusion,url,event']);
|
||||
try { return out.status === 0 ? JSON.parse(out.stdout) as RunSummary[] : []; } catch { return []; }
|
||||
};
|
||||
let target = findDuplicateRun(list(), key);
|
||||
if (target) console.log(`[eval-bg] ${dispatch.workflow}: following existing run ${target.url} (same lane, revision and base; no duplicate dispatch)`);
|
||||
else {
|
||||
const args = ['workflow', 'run', dispatch.workflow, '--repo', CANONICAL_REPO, '--ref', ref,
|
||||
...Object.entries(dispatch.inputs).flatMap(([name, value]) => ['-f', `${name}=${value}`])];
|
||||
if (run('gh', args).status !== 0) { console.log(`[eval-bg] ${dispatch.workflow}: gh workflow run failed`); return 2; }
|
||||
const deadline = Date.now() + 10 * pollMs();
|
||||
while (!target && Date.now() < deadline) {
|
||||
await sleep(Math.min(pollMs(), 10_000));
|
||||
target = list().find(item => item.displayTitle.includes(`eval-bg ${dispatch.inputs.nonce}`)) ?? null;
|
||||
}
|
||||
if (!target) { console.log(`[eval-bg] ${dispatch.workflow}: dispatched, but no run named ${dispatch.inputs.nonce} appeared`); return 2; }
|
||||
}
|
||||
console.log(`[eval-bg] ${dispatch.workflow}: run ${target.databaseId} ${target.url}`);
|
||||
let last = '';
|
||||
while (true) {
|
||||
const view = run('gh', ['run', 'view', String(target.databaseId), '--repo', CANONICAL_REPO, '--json', 'status,conclusion']);
|
||||
let state: { status?: string; conclusion?: string | null } = {};
|
||||
try { state = JSON.parse(view.stdout); } catch { /* transient gh failure: poll again */ }
|
||||
if (state.status && state.status !== last) console.log(`[eval-bg] ${dispatch.workflow}: ${last = state.status}`);
|
||||
if (state.status === 'completed') {
|
||||
const code = conclusionExitCode(state.conclusion);
|
||||
console.log(`[eval-bg] ${dispatch.workflow}: conclusion ${state.conclusion ?? 'unknown'} -> exit ${code}`);
|
||||
return code;
|
||||
}
|
||||
await sleep(pollMs());
|
||||
}
|
||||
}
|
||||
|
||||
/** The detached follower: every planned workflow, then the first non-zero code. */
|
||||
export async function follow(plan: WorkflowDispatch[], ref: string, key: string, run: Exec = exec): Promise<number> {
|
||||
const codes = await Promise.all(plan.map(dispatch => followWorkflow(dispatch, ref, key, run)));
|
||||
return codes.find(code => code !== 0) ?? 0;
|
||||
}
|
||||
|
||||
/** `status <run-id|log>`: running / passed / failed / cancelled / incomplete. */
|
||||
export function statusOf(target: string, run: Exec = exec, alive: (pgid: number) => boolean = pgidAlive): string {
|
||||
if (/^\d+$/.test(target)) {
|
||||
const view = run('gh', ['run', 'view', target, '--repo', CANONICAL_REPO, '--json', 'status,conclusion']);
|
||||
const state = JSON.parse(view.stdout || '{}') as { status?: string; conclusion?: string };
|
||||
return state.status === 'completed' ? exitStatusWord(String(conclusionExitCode(state.conclusion))) : 'running';
|
||||
}
|
||||
const log = fs.readFileSync(target, 'utf8');
|
||||
const exit = /### gstack-detach EXIT=(\S+) ###/.exec(log)?.[1];
|
||||
if (exit) return exitStatusWord(exit);
|
||||
const pgid = Number(/### gstack-detach START label=\S+ pgid=(\d+) ###/.exec(log)?.[1]);
|
||||
return pgid && alive(pgid) ? 'running' : 'incomplete';
|
||||
}
|
||||
|
||||
function pgidAlive(pgid: number): boolean {
|
||||
try { process.kill(-pgid, 0); return true; } catch { return false; }
|
||||
}
|
||||
|
||||
const HELP = `Usage:
|
||||
bun run scripts/eval-bg.ts <pr|gate|periodic|release> [--local | --dispatch] [--timeout SECS] [--base REF]
|
||||
bun run scripts/eval-bg.ts status <run-id|log>
|
||||
|
||||
Runs a paid eval lane in the background and returns immediately. Poll the printed
|
||||
log; it ends with "### gstack-detach EXIT=<code> ###" (0 passed, 1 failed,
|
||||
130 cancelled, 2 other).
|
||||
|
||||
pr diff-selected PR gate (CI: evals.yml, evals_all=false)
|
||||
gate full gate tier (CI: evals.yml, evals_all=true)
|
||||
periodic full periodic tier (CI: evals-periodic.yml, periodic lane only)
|
||||
release gate + periodic (CI: both workflows)
|
||||
|
||||
Backend: a clean HEAD pushed to ${CANONICAL_REPO} dispatches the CI workflow and
|
||||
follows it; anything else runs locally under bin/gstack-detach, capped at
|
||||
ceil(1.5 x planned serial seconds / EVALS_JOBS) + 20 min (at most 4 h).
|
||||
--local / --dispatch force a backend (also GSTACK_EVAL_BG_MODE=local|dispatch);
|
||||
--dispatch refuses instead of falling back
|
||||
--timeout SECS override the computed local cap
|
||||
--base REF base branch for the pr lane (default: detected base)
|
||||
Needs: gh with repo access for dispatch; ANTHROPIC_API_KEY for local runs.`;
|
||||
|
||||
async function main(argv: string[]): Promise<number> {
|
||||
if (argv.length === 0 || argv.includes('--help') || argv.includes('-h')) { console.log(HELP); return argv.length ? 0 : 2; }
|
||||
const [command, ...rest] = argv;
|
||||
if (command === 'status') {
|
||||
if (!rest[0]) throw new Error('status needs a run id or a log path');
|
||||
console.log(statusOf(rest[0]));
|
||||
return 0;
|
||||
}
|
||||
if (command === '__follow') {
|
||||
const [planJson, ref, key] = rest;
|
||||
return follow(JSON.parse(planJson!), ref!, key!);
|
||||
}
|
||||
if (!EVAL_BG_LANES.includes(command as EvalBgLane)) throw new Error(`Unknown lane ${command}; expected ${EVAL_BG_LANES.join(', ')} or status (see --help)`);
|
||||
const lane = command as EvalBgLane;
|
||||
let mode: Backend | 'auto' = process.env.GSTACK_EVAL_BG_MODE === 'local' || process.env.GSTACK_EVAL_BG_MODE === 'dispatch' ? process.env.GSTACK_EVAL_BG_MODE : 'auto';
|
||||
let timeoutOverride: number | null = null;
|
||||
let baseRef = detectBaseBranch(ROOT) || 'main';
|
||||
for (let index = 0; index < rest.length; index++) {
|
||||
const flag = rest[index];
|
||||
if (flag === '--local' || flag === '--dispatch') mode = flag.slice(2) as Backend;
|
||||
else if (flag === '--timeout') {
|
||||
timeoutOverride = Number(rest[++index]);
|
||||
if (!Number.isSafeInteger(timeoutOverride) || timeoutOverride <= 0) throw new Error('--timeout needs positive seconds');
|
||||
} else if (flag === '--base') baseRef = rest[++index] ?? baseRef;
|
||||
else throw new Error(`Unknown argument: ${flag} (see --help)`);
|
||||
}
|
||||
const facts = gatherCheckoutFacts();
|
||||
const choice = chooseBackend(facts, mode);
|
||||
if (choice.backend === 'refuse') { console.error(`[eval-bg] refusing to dispatch: ${choice.reason}`); return 1; }
|
||||
const label = `evals-${lane}`;
|
||||
const log = runScopedLog(label, facts.branch);
|
||||
const tested = `${facts.sha}${facts.dirty.length ? ` + ${facts.dirty.length} uncommitted file(s)` : ''}`;
|
||||
if (choice.backend === 'dispatch') {
|
||||
const baseSha = exec('git', ['merge-base', `origin/${baseRef}`, 'HEAD']).stdout.trim();
|
||||
const pr = lane === 'pr' ? exec('gh', ['api', `repos/${CANONICAL_REPO}/pulls?head=${CANONICAL_REPO.split('/')[0]}:${facts.branch}&state=open`, '--jq', '.[0].number']).stdout.trim() : '';
|
||||
const nonce = makeNonce(lane, facts.sha, baseSha);
|
||||
const plan = dispatchPlan(lane, { sha: facts.sha, baseRef, baseSha, nonce, prNumber: Number(pr) || null });
|
||||
const header = `[eval-bg] backend=dispatch lane=${lane} tested=${facts.sha} ref=${facts.branch} workflows=${plan.map(item => item.workflow).join(',')} nonce=${nonce} (${choice.reason})`;
|
||||
console.log(header);
|
||||
console.log(launchDetached({ log, label, timeoutSeconds: DISPATCH_FOLLOW_SECONDS, header,
|
||||
command: [process.execPath, 'run', path.join(ROOT, 'scripts', 'eval-bg.ts'), '__follow', JSON.stringify(plan), facts.branch!, dedupeKey(lane, facts.sha, baseSha)] }));
|
||||
return 0;
|
||||
}
|
||||
const jobs = Number(process.env.EVALS_JOBS) || LANE_SPECS[lane].jobs;
|
||||
const cap = timeoutOverride !== null ? { seconds: timeoutOverride, capped: false } : localCapSeconds(plannedSerialMs(lane), jobs);
|
||||
const header = `[eval-bg] backend=local lane=${lane} tested=${tested} cap=${cap.seconds}s jobs=${jobs}${timeoutOverride !== null ? ' (--timeout)' : ''} (${choice.reason})`;
|
||||
console.log(header);
|
||||
if (cap.capped) console.log(`[eval-bg] WARNING: planned work exceeds the ${LOCAL_CAP_SECONDS}s local cap; the tail may not run. Dispatch to CI or raise EVALS_JOBS.`);
|
||||
console.log(launchDetached({ log, label, timeoutSeconds: cap.seconds, header, command: ['bun', 'run', LANE_SPECS[lane].script] }));
|
||||
return 0;
|
||||
}
|
||||
|
||||
if (import.meta.main) {
|
||||
try {
|
||||
process.exitCode = await main(process.argv.slice(2));
|
||||
} catch (error) {
|
||||
console.error(`[eval-bg] ${error instanceof Error ? error.message : String(error)}`);
|
||||
process.exitCode = 1;
|
||||
}
|
||||
}
|
||||
@@ -61,7 +61,7 @@ if (args.length === 2) {
|
||||
.sort((a, b) => path.basename(b).localeCompare(path.basename(a)));
|
||||
|
||||
if (files.length === 0) {
|
||||
console.log('No eval runs yet. Run: EVALS=1 bun run test:evals');
|
||||
console.log('No eval runs yet. Run: bun run eval:bg:pr');
|
||||
process.exit(0);
|
||||
}
|
||||
if (files.length < 2) {
|
||||
@@ -78,7 +78,7 @@ if (args.length === 2) {
|
||||
}
|
||||
});
|
||||
if (!latest) {
|
||||
console.log('No completed eval runs yet. Run: EVALS=1 bun run test:evals');
|
||||
console.log('No completed eval runs yet. Run: bun run eval:bg:pr');
|
||||
process.exit(0);
|
||||
}
|
||||
afterFile = latest;
|
||||
|
||||
+72
-136
@@ -4,16 +4,24 @@
|
||||
*
|
||||
* Reads trial records (one JSONL line per trial: case, kind, trial, outcome,
|
||||
* exit_reason, duration, cost, model, CLI version, series identity, run id,
|
||||
* sha, policy_version) from the last N completed `evals-periodic.yml` runs on
|
||||
* the current branch and `main` (downloading only each run's small
|
||||
* `trial-outcomes` artifact through `gh`), plus any local eval dirs, and
|
||||
* sha, policy_version) from the last N completed `evals-periodic.yml` runs of
|
||||
* the weekly history: scheduled runs on `main` plus `main` dispatches
|
||||
* (`--branch <name>` reads one branch's runs for inspection). Trials from
|
||||
* branch census runs in the same window pool into a series main has also
|
||||
* run, never into a new one; weeks for quarantine expiry count main only,
|
||||
* downloading only each run's small `trial-outcomes` artifact through `gh`,
|
||||
* plus any local eval dirs, and
|
||||
* prints per-case per-trial pass rates with 95% Wilson intervals.
|
||||
*
|
||||
* A series is one case under one input identity: the case's own touchfiles
|
||||
* minus GLOBAL_TOUCHFILES (`caseSeriesIdentities`), grouped by model and CLI
|
||||
* version, per policy_version. A new identity starts a new series; earlier
|
||||
* series stay visible. Only post-policy trials of the current series feed the
|
||||
* labels and alarms. Legacy eval-store records (`--backfill`, `--dir`) are
|
||||
* A series is one case under one input identity (EVAL_POLICY v2: the bytes
|
||||
* the case owns plus HARNESS_VERSION, stamped by scripts/eval-trial-series.ts
|
||||
* caseSeriesIdentitiesV2), grouped by model and CLI version, per
|
||||
* policy_version. A new identity starts a new series; earlier
|
||||
* series stay visible. Only trials of the current series under the reader's
|
||||
* own EVAL_POLICY.version feed the labels and alarms: an older policy's
|
||||
* trials are display-only, and a newer policy's trials (written by a later
|
||||
* checkout) are ignored with a printed count, so a reader rolled back past a
|
||||
* policy bump never pools records it cannot interpret. Legacy eval-store records (`--backfill`, `--dir`) are
|
||||
* imported as pre-policy trials (first attempt only; a missing attempt means
|
||||
* 1) and are display-only.
|
||||
*
|
||||
@@ -31,7 +39,7 @@
|
||||
* fetched fails the gate closed.
|
||||
*
|
||||
* Usage:
|
||||
* bun run eval:pass-rates # last 10 weekly runs, this branch + main
|
||||
* bun run eval:pass-rates # last 10 weekly runs (main: scheduled + dispatched)
|
||||
* bun run eval:pass-rates --case <id> --runs 20
|
||||
* bun run eval:pass-rates --dir <path> # local eval dirs / downloaded artifacts (repeatable)
|
||||
* bun run eval:pass-rates --backfill # also import legacy slice artifacts, labeled pre-policy
|
||||
@@ -41,20 +49,17 @@
|
||||
*/
|
||||
|
||||
import * as fs from 'node:fs';
|
||||
import * as os from 'node:os';
|
||||
import * as path from 'node:path';
|
||||
import { spawnSync } from 'node:child_process';
|
||||
import { createHash } from 'node:crypto';
|
||||
import { isPartialEval, isFinalizedEvalResultFile, evalEntryOutcome, failureClassOf, parseTrialOutcomes, sanitizeTrialError,
|
||||
TRIAL_OUTCOME_SCHEMA, type EvalCaseKind, type EvalResult, type TrialOutcomeRecord } from '../test/helpers/eval-store';
|
||||
import { flakeLedgerPath, type FlakeLedgerEntry } from './test-free-shards';
|
||||
import { E2E_KINDS, E2E_TIERS, E2E_TOUCHFILES, GLOBAL_TOUCHFILES, LLM_JUDGE_TOUCHFILES } from '../test/helpers/touchfiles-data';
|
||||
import { CASE_QUARANTINE, EVAL_POLICY } from '../test/helpers/periodic-exclude-data';
|
||||
import { matchGlob } from '../test/helpers/test-selection';
|
||||
import { CASE_TEST_NAMES } from './test-paid-shards';
|
||||
import { resolveStateRoot } from '../lib/state-root';
|
||||
import { GH_JOBS, PASS_RATES_USAGE, criticalPath, formatCriticalPath, formatHeadroom, formatRedLedger, headroom, headroomAlarms, parsePassRatesArgs,
|
||||
redLedger, triageRun, CENSUS_RED_GUIDE, type CriticalPath } from './lib/eval-history';
|
||||
import { downloadRunArtifacts, GH_HISTORY, isPooledTrialRun, isWeeklyHistoryRun, listWeeklyRuns, parseFlakeLedger, repoSlug, TRIAL_OUTCOMES_MAX_BYTES, type WeeklyRun } from './lib/ci-history';
|
||||
|
||||
interface TestSeries {
|
||||
name: string;
|
||||
@@ -127,32 +132,23 @@ export function collectEvalFiles(dir: string, sinceDays = 60): string[] {
|
||||
return out;
|
||||
}
|
||||
|
||||
/** The local free-suite flake ledger (CI ledgers are read by test:health through scripts/lib/ci-history.ts). */
|
||||
function readFreeLedger(): FlakeLedgerEntry[] {
|
||||
// Per-LINE parse: one malformed JSONL line (torn write, manual edit) must
|
||||
// drop that line, never vanish the whole series (codex adversarial finding).
|
||||
let raw: string;
|
||||
try {
|
||||
raw = fs.readFileSync(flakeLedgerPath(), 'utf-8');
|
||||
return parseFlakeLedger(fs.readFileSync(flakeLedgerPath(), 'utf-8'));
|
||||
} catch { return []; }
|
||||
const out: FlakeLedgerEntry[] = [];
|
||||
for (const line of raw.split('\n')) {
|
||||
if (!line.trim()) continue;
|
||||
try { out.push(JSON.parse(line)); } catch { /* torn line — skip */ }
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
// --- Trial records ---
|
||||
|
||||
/**
|
||||
* A trial record as pass-rates reads it: eval-store's trial-outcomes schema
|
||||
* plus the series identity the report job stamps (caseSeriesIdentities).
|
||||
* policy_version 0 marks a pre-policy (backfilled) record.
|
||||
* plus what the report job stamps (scripts/eval-trial-series.ts): the series
|
||||
* identity, the full consumed-input fingerprint and HARNESS_VERSION, both
|
||||
* provenance only. policy_version 0 marks a pre-policy (backfilled) record.
|
||||
*/
|
||||
export type TrialRecord = TrialOutcomeRecord & { series_identity?: string };
|
||||
export type TrialRecord = TrialOutcomeRecord & { series_identity?: string; series_fingerprint?: string; harness_version?: number };
|
||||
|
||||
/** Per-file cap for downloaded artifacts: pass-rates parses data only, never executes it. */
|
||||
export const TRIAL_OUTCOMES_MAX_BYTES = 8 * 1024 * 1024;
|
||||
|
||||
/** Every `trial-outcomes*.jsonl` file under a directory, size-capped, schema-validated by eval-store. */
|
||||
export function readTrialOutcomeDir(dir: string): { records: TrialRecord[]; errors: string[] } {
|
||||
@@ -215,26 +211,6 @@ export function attributeLegacyRecord(name: string, shard: string | undefined, r
|
||||
return null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Series identity per case: a hash of the git blob ids of the files matching
|
||||
* the case's own touchfiles, excluding GLOBAL_TOUCHFILES (harness edits are
|
||||
* markers, not new series). The report job stamps this on every trial record.
|
||||
*/
|
||||
export function caseSeriesIdentities(ids: string[], root: string, registry: Registry = LIVE_REGISTRY): Record<string, string> {
|
||||
const listed = spawnSync('git', ['ls-files', '-s'], { cwd: root, encoding: 'utf8', timeout: 20_000, maxBuffer: 64 * 1024 * 1024 });
|
||||
if (listed.status !== 0) throw new Error(`git ls-files failed: ${listed.stderr}`);
|
||||
const blobs = listed.stdout.split('\n').filter(Boolean).map(line => {
|
||||
const [meta, file] = line.split('\t');
|
||||
return { file: file!, blob: meta!.split(' ')[1]! };
|
||||
}).filter(entry => !registry.globals.some(pattern => matchGlob(entry.file, pattern)));
|
||||
return Object.fromEntries(ids.map(id => {
|
||||
const patterns = registry.touchfiles[id] ?? registry.judgeTouchfiles[id] ?? [];
|
||||
const lines = blobs.filter(entry => patterns.some(pattern => matchGlob(entry.file, pattern)))
|
||||
.map(entry => `${entry.file} ${entry.blob}`).sort();
|
||||
return [id, createHash('sha256').update(`${id}\n${lines.join('\n')}`).digest('hex').slice(0, 16)];
|
||||
}));
|
||||
}
|
||||
|
||||
/**
|
||||
* Import legacy eval-store result files as pre-policy trials (policy_version
|
||||
* 0, source 'backfill'): first attempt only (a missing attempt means 1),
|
||||
@@ -372,8 +348,14 @@ export interface PassRateReport {
|
||||
policyVersion: number;
|
||||
cases: CasePassRate[];
|
||||
alarms: Alarm[];
|
||||
/** Trials under this reader's EVAL_POLICY.version: the only ones scored. */
|
||||
postPolicyTrials: number;
|
||||
/** Backfilled pre-policy trials (policy_version 0): display only. */
|
||||
prePolicyTrials: number;
|
||||
/** Trials under an earlier policy version: display only, never scored. */
|
||||
olderPolicyTrials: number;
|
||||
/** Trials under a later policy version than this reader: ignored entirely. */
|
||||
newerPolicyTrials: number;
|
||||
unattributed: string[];
|
||||
errors: string[];
|
||||
}
|
||||
@@ -384,6 +366,12 @@ export interface AnalyzeOptions {
|
||||
policy?: PassRatePolicy;
|
||||
/** Completed weekly-run timestamps in the window, for quarantine expiry. */
|
||||
weeklyRuns?: string[];
|
||||
/**
|
||||
* Run ids of branch census runs (isPooledTrialRun). Their trials count only
|
||||
* toward a series that a non-pooled record also has, so a branch can extend
|
||||
* main's history but never start, or become, a case's current series.
|
||||
*/
|
||||
pooledRunIds?: ReadonlySet<string>;
|
||||
now?: number;
|
||||
unattributed?: string[];
|
||||
errors?: string[];
|
||||
@@ -457,8 +445,10 @@ export function analyzePassRates(records: TrialRecord[], options: AnalyzeOptions
|
||||
const quarantine = options.quarantine ?? CASE_QUARANTINE;
|
||||
const policy = options.policy ?? EVAL_POLICY;
|
||||
const now = options.now ?? Date.now();
|
||||
const newerPolicyTrials = records.filter(record => record.policy_version > policy.version).length;
|
||||
const byCase = new Map<string, TrialRecord[]>();
|
||||
for (const record of records) {
|
||||
if (record.policy_version > policy.version) continue;
|
||||
const list = byCase.get(record.case) ?? [];
|
||||
list.push(record);
|
||||
byCase.set(record.case, list);
|
||||
@@ -468,16 +458,20 @@ export function analyzePassRates(records: TrialRecord[], options: AnalyzeOptions
|
||||
for (const [id, list] of [...byCase].sort(([a], [b]) => a.localeCompare(b))) {
|
||||
list.sort((a, b) => at(a).localeCompare(at(b)) || runOf(a).localeCompare(runOf(b)) || a.trial - b.trial);
|
||||
const groups = new Map<string, TrialRecord[]>();
|
||||
const keyOf = (record: TrialRecord) => record.policy_version === 0 ? 'pre-policy'
|
||||
: [record.series_identity ?? 'unknown', record.model ?? 'unknown', record.cli_version ?? 'unknown', `v${record.policy_version}`].join('|');
|
||||
const pooled = (record: TrialRecord) => !!options.pooledRunIds?.has(record.run_id ?? '');
|
||||
const mainKeys = new Set(list.filter(record => !pooled(record)).map(keyOf));
|
||||
for (const record of list) {
|
||||
const key = record.policy_version === 0 ? 'pre-policy'
|
||||
: [record.series_identity ?? 'unknown', record.model ?? 'unknown', record.cli_version ?? 'unknown', `v${record.policy_version}`].join('|');
|
||||
const key = keyOf(record);
|
||||
if (pooled(record) && !mainKeys.has(key)) continue;
|
||||
const group = groups.get(key) ?? [];
|
||||
group.push(record);
|
||||
groups.set(key, group);
|
||||
}
|
||||
const series = [...groups].map(([key, group]) => seriesStats(key, group))
|
||||
.sort((a, b) => a.lastSeen.localeCompare(b.lastSeen));
|
||||
const post = series.filter(entry => entry.policyVersion !== 0);
|
||||
const post = series.filter(entry => entry.policyVersion === policy.version);
|
||||
const current = post[post.length - 1] ?? null;
|
||||
const previous = post[post.length - 2] ?? null;
|
||||
const scored = current ? groups.get(current.key)!.filter(record => record.outcome !== 'skipped') : [];
|
||||
@@ -532,9 +526,10 @@ export function analyzePassRates(records: TrialRecord[], options: AnalyzeOptions
|
||||
}
|
||||
alarms.push(...quarantinePolicyProblems(quarantine, registry, policy, now));
|
||||
|
||||
const post = records.filter(record => record.policy_version !== 0).length;
|
||||
return { policyVersion: policy.version, cases, alarms, postPolicyTrials: post, prePolicyTrials: records.length - post,
|
||||
unattributed: options.unattributed ?? [], errors: options.errors ?? [] };
|
||||
const count = (match: (version: number) => boolean) => records.filter(record => match(record.policy_version)).length;
|
||||
return { policyVersion: policy.version, cases, alarms, postPolicyTrials: count(version => version === policy.version),
|
||||
prePolicyTrials: count(version => version === 0), olderPolicyTrials: count(version => version > 0 && version < policy.version),
|
||||
newerPolicyTrials, unattributed: options.unattributed ?? [], errors: options.errors ?? [] };
|
||||
}
|
||||
|
||||
function pct(value: number): string { return `${Math.round(value * 1000) / 10}%`; }
|
||||
@@ -546,7 +541,12 @@ function formatStats(stats: SeriesStats | null): string {
|
||||
|
||||
export function formatPassRates(report: PassRateReport, options: { caseFilter?: string } = {}): string {
|
||||
const lines: string[] = [];
|
||||
lines.push(`pass-rates: policy v${report.policyVersion}, ${report.postPolicyTrials} post-policy trial(s), ${report.prePolicyTrials} pre-policy (display only)`);
|
||||
lines.push(`pass-rates: policy v${report.policyVersion}, ${report.postPolicyTrials} post-policy trial(s), ${report.prePolicyTrials} pre-policy (display only)`
|
||||
+ (report.olderPolicyTrials ? `, ${report.olderPolicyTrials} under an older policy (display only)` : ''));
|
||||
if (report.newerPolicyTrials) {
|
||||
lines.push(` ignored ${report.newerPolicyTrials} trial(s) recorded under a policy newer than v${report.policyVersion}: this checkout predates them. `
|
||||
+ 'Fix: run pass-rates from a checkout at or after the commit that bumped EVAL_POLICY.version (docs/TESTING_INTERNALS.md#pass-rate-policy-versions).');
|
||||
}
|
||||
if (report.postPolicyTrials === 0) lines.push(' no post-policy trials yet: every series starts INCONCLUSIVE');
|
||||
const cases = report.cases.filter(entry => !options.caseFilter || entry.case === options.caseFilter);
|
||||
lines.push(' label kind tier current series pre-policy manual case');
|
||||
@@ -565,82 +565,10 @@ export function formatPassRates(report: PassRateReport, options: { caseFilter?:
|
||||
return lines.join('\n');
|
||||
}
|
||||
|
||||
// --- GitHub history ---
|
||||
// --- GitHub history (shared reader: scripts/lib/ci-history.ts) ---
|
||||
|
||||
export interface WeeklyRun { id: number; attempt: number; sha: string; branch: string; createdAt: string }
|
||||
export interface RunArtifact { id: number; name: string; size: number }
|
||||
|
||||
/** The GitHub calls pass-rates makes; injectable so the free tests never touch the network. */
|
||||
export interface HistoryFetcher {
|
||||
listRuns(repo: string, workflow: string, branch: string, limit: number): WeeklyRun[];
|
||||
listArtifacts(repo: string, runId: number): RunArtifact[];
|
||||
downloadZip(repo: string, artifactId: number, destination: string): void;
|
||||
}
|
||||
|
||||
function gh(args: string[]): Buffer {
|
||||
const result = spawnSync('gh', args, { timeout: 300_000, maxBuffer: 256 * 1024 * 1024 });
|
||||
if (result.status !== 0) throw new Error(`gh ${args.slice(0, 2).join(' ')} failed: ${String(result.stderr || result.error || '').trim()}`);
|
||||
return result.stdout;
|
||||
}
|
||||
|
||||
function jsonLines<T>(buffer: Buffer): T[] {
|
||||
return buffer.toString('utf8').split('\n').filter(Boolean).map(line => JSON.parse(line) as T);
|
||||
}
|
||||
|
||||
export const GH_HISTORY: HistoryFetcher = {
|
||||
listRuns: (repo, workflow, branch, limit): WeeklyRun[] => jsonLines(gh(['api',
|
||||
`repos/${repo}/actions/workflows/${workflow}/runs?branch=${encodeURIComponent(branch)}&status=completed&per_page=${limit}`,
|
||||
'--jq', '.workflow_runs[] | {id, attempt: .run_attempt, sha: .head_sha, branch: .head_branch, createdAt: .created_at}'])),
|
||||
listArtifacts: (repo, runId): RunArtifact[] => jsonLines(gh(['api', `repos/${repo}/actions/runs/${runId}/artifacts?per_page=100`,
|
||||
'--paginate', '--jq', '.artifacts[] | select(.expired | not) | {id, name, size: .size_in_bytes}'])),
|
||||
downloadZip: (repo, artifactId, destination) => fs.writeFileSync(destination, gh(['api', `repos/${repo}/actions/artifacts/${artifactId}/zip`])),
|
||||
};
|
||||
|
||||
/** The last `limit` completed runs of `workflow` on each branch, newest first, deduplicated. */
|
||||
export function listWeeklyRuns(opts: { repo: string; workflow: string; branches: string[]; limit: number; fetcher?: HistoryFetcher }): WeeklyRun[] {
|
||||
const fetcher = opts.fetcher ?? GH_HISTORY;
|
||||
const runs = new Map<number, WeeklyRun>();
|
||||
for (const branch of opts.branches) for (const run of fetcher.listRuns(opts.repo, opts.workflow, branch, opts.limit)) runs.set(run.id, run);
|
||||
return [...runs.values()].sort((a, b) => b.createdAt.localeCompare(a.createdAt));
|
||||
}
|
||||
|
||||
/**
|
||||
* Download the artifacts of one run whose names match into a per-run cache
|
||||
* directory (reused on later calls) and return the extracted directories.
|
||||
* Oversized or oddly named artifacts are skipped: downloads are data only.
|
||||
*/
|
||||
export function downloadRunArtifacts(opts: { repo: string; run: WeeklyRun; match: (name: string) => boolean; cacheDir: string;
|
||||
fetcher?: HistoryFetcher; maxBytes?: number }): string[] {
|
||||
const fetcher = opts.fetcher ?? GH_HISTORY;
|
||||
const dirs: string[] = [];
|
||||
for (const artifact of fetcher.listArtifacts(opts.repo, opts.run.id)) {
|
||||
if (!opts.match(artifact.name) || !/^[A-Za-z0-9._-]+$/.test(artifact.name)) continue;
|
||||
if (artifact.size > (opts.maxBytes ?? TRIAL_OUTCOMES_MAX_BYTES)) continue;
|
||||
const dir = path.join(opts.cacheDir, `${opts.run.id}`, artifact.name);
|
||||
if (!fs.existsSync(path.join(dir, '.complete'))) {
|
||||
fs.rmSync(dir, { recursive: true, force: true });
|
||||
fs.mkdirSync(dir, { recursive: true });
|
||||
const zip = path.join(dir, 'artifact.zip');
|
||||
fetcher.downloadZip(opts.repo, artifact.id, zip);
|
||||
const unzip = spawnSync('unzip', ['-o', '-q', zip, '-d', dir], { timeout: 120_000 });
|
||||
if (unzip.status !== 0) throw new Error(`unzip failed for ${artifact.name}: ${String(unzip.stderr || unzip.error || '')}`);
|
||||
fs.rmSync(zip, { force: true });
|
||||
fs.writeFileSync(path.join(dir, '.complete'), '');
|
||||
}
|
||||
dirs.push(dir);
|
||||
}
|
||||
return dirs;
|
||||
}
|
||||
|
||||
function gitOutput(args: string[]): string | null {
|
||||
const result = spawnSync('git', args, { encoding: 'utf8', timeout: 5_000 });
|
||||
return result.status === 0 ? result.stdout.trim() : null;
|
||||
}
|
||||
|
||||
function repoSlug(): string {
|
||||
const url = gitOutput(['remote', 'get-url', 'origin']) ?? '';
|
||||
return url.match(/[:/]([^/:]+\/[^/]+?)(?:\.git)?$/)?.[1] ?? 'garrytan/gstack';
|
||||
}
|
||||
export { downloadRunArtifacts, GH_HISTORY, listWeeklyRuns, TRIAL_OUTCOMES_MAX_BYTES,
|
||||
type HistoryFetcher, type RunArtifact, type WeeklyRun } from './lib/ci-history';
|
||||
|
||||
if (import.meta.main) {
|
||||
const parsed = parsePassRatesArgs(process.argv.slice(2), id => Object.hasOwn(E2E_TIERS, id) || Object.hasOwn(LLM_JUDGE_TOUCHFILES, id)
|
||||
@@ -650,7 +578,7 @@ if (import.meta.main) {
|
||||
const { dirs, json: asJson, gate, backfill, caseFilter, runs: runsLimit, sinceDays, view } = parsed;
|
||||
const repo = parsed.repo ?? repoSlug();
|
||||
const workflow = parsed.workflow ?? 'evals-periodic.yml';
|
||||
const branch = parsed.branch ?? gitOutput(['rev-parse', '--abbrev-ref', 'HEAD']) ?? 'main';
|
||||
const branch = parsed.branch ?? 'main';
|
||||
|
||||
const records: TrialRecord[] = [];
|
||||
const unattributed = new Set<string>();
|
||||
@@ -659,6 +587,7 @@ if (import.meta.main) {
|
||||
let weeklyRuns: string[] | undefined;
|
||||
let fetched: WeeklyRun[] = [];
|
||||
const cacheDir = path.join(path.resolve(resolveStateRoot()), 'eval-pass-rates-cache', repo.replace('/', '-'));
|
||||
let pooledRunIds: Set<string> | undefined;
|
||||
|
||||
const manualReviews: string[] = [];
|
||||
const importDir = (dir: string, run: { run_id: string; sha?: string; timestamp?: string } | undefined, legacyDays: number) => {
|
||||
@@ -675,9 +604,16 @@ if (import.meta.main) {
|
||||
for (const dir of dirs) importDir(dir, undefined, sinceDays);
|
||||
} else {
|
||||
try {
|
||||
const runs = listWeeklyRuns({ repo, workflow, branches: [...new Set([branch, 'main'])], limit: runsLimit });
|
||||
const weekly = listWeeklyRuns({ repo, workflow, branches: [branch], limit: runsLimit }).filter(run => branch !== 'main' || isWeeklyHistoryRun(run));
|
||||
weeklyRuns = weekly.map(run => run.createdAt);
|
||||
// Branch census trials pool into main's matching series over the same window (isPooledTrialRun).
|
||||
const oldest = weekly[weekly.length - 1]?.createdAt;
|
||||
const pooledRuns = branch === 'main' && oldest
|
||||
? listWeeklyRuns({ repo, workflow, branches: [''], limit: 100 }).filter(run => isPooledTrialRun(run) && run.createdAt >= oldest)
|
||||
: [];
|
||||
pooledRunIds = new Set(pooledRuns.map(run => `${run.id}`));
|
||||
const runs = [...weekly, ...pooledRuns];
|
||||
if (parsed.runId !== undefined && !runs.some(run => run.id === parsed.runId)) runs.push(GH_JOBS.getRun(repo, parsed.runId));
|
||||
weeklyRuns = runs.map(run => run.createdAt);
|
||||
fetched = runs;
|
||||
const match = backfill
|
||||
? (name: string) => name.startsWith('trial-outcomes') || /^(paid-slice-\d+|gate-census-\d+)(-a\d+)?$/.test(name)
|
||||
@@ -691,8 +627,8 @@ if (import.meta.main) {
|
||||
}
|
||||
}
|
||||
|
||||
const scope = dirs.length ? `local dirs ${dirs.join(', ')}` : `${repo} ${workflow} on ${[...new Set([branch, 'main'])].join(' + ')}, `
|
||||
+ `last ${runsLimit} completed run(s) per branch: ${fetched.map(run => run.id).join(', ') || 'none'}`;
|
||||
const scope = dirs.length ? `local dirs ${dirs.join(', ')}` : `${repo} ${workflow} on ${branch}${pooledRunIds?.size ? ` + ${pooledRunIds.size} pooled branch census run(s)` : ''}, `
|
||||
+ `last ${runsLimit} completed run(s): ${fetched.map(run => run.id).join(', ') || 'none'}`;
|
||||
const scoped = caseFilter ? records.filter(record => record.case === caseFilter) : records;
|
||||
const caseHeadroom = headroom(scoped);
|
||||
if (view !== 'rates') {
|
||||
@@ -714,7 +650,7 @@ if (import.meta.main) {
|
||||
}
|
||||
process.exit(historyError ? 1 : 0);
|
||||
}
|
||||
const full = analyzePassRates(records, { weeklyRuns, unattributed: [...unattributed].sort(), errors, manualReviews });
|
||||
const full = analyzePassRates(records, { weeklyRuns, pooledRunIds, unattributed: [...unattributed].sort(), errors, manualReviews });
|
||||
const alarms = [...full.alarms, ...(gate ? headroomAlarms(caseHeadroom) : [])].filter(alarm => !caseFilter || alarm.case === caseFilter);
|
||||
const report = { ...full, cases: full.cases.filter(entry => !caseFilter || entry.case === caseFilter), alarms };
|
||||
const ledger = readFreeLedger();
|
||||
|
||||
@@ -40,7 +40,7 @@ for (let i = 0; i < args.length; i++) {
|
||||
const files = listEvalJsonFiles(EVAL_DIR);
|
||||
|
||||
if (files.length === 0) {
|
||||
console.log('No eval runs yet. Run: EVALS=1 bun run test:evals');
|
||||
console.log('No eval runs yet. Run: bun run eval:bg:pr');
|
||||
process.exit(0);
|
||||
}
|
||||
|
||||
|
||||
@@ -2,10 +2,15 @@
|
||||
/**
|
||||
* Show which E2E and LLM-judge tests would run based on the current git diff.
|
||||
*
|
||||
* The default profile is `pr` (the PR lane's selection, matching `test:pr`
|
||||
* and `eval:bg:pr`); `--profile full` (or EVALS_PROFILE=full) shows the plain
|
||||
* touchfile selection over every E2E and LLM-judge entry.
|
||||
*
|
||||
* Usage:
|
||||
* bun run eval:select # human-readable output
|
||||
* bun run eval:select --json # machine-readable JSON
|
||||
* bun run eval:select --base main # override base branch
|
||||
* bun run eval:select # PR-lane selection, human-readable
|
||||
* bun run eval:select --json # machine-readable JSON
|
||||
* bun run eval:select --base main # override base branch
|
||||
* bun run eval:select --profile full # full touchfile selection
|
||||
*/
|
||||
|
||||
import * as path from 'path';
|
||||
@@ -25,7 +30,7 @@ const jsonMode = args.includes('--json');
|
||||
const baseIdx = args.indexOf('--base');
|
||||
const baseOverride = baseIdx >= 0 ? args[baseIdx + 1] : undefined;
|
||||
const profileIdx = args.indexOf('--profile');
|
||||
const profile = profileIdx >= 0 ? args[profileIdx + 1] : process.env.EVALS_PROFILE ?? 'full';
|
||||
const profile = profileIdx >= 0 ? args[profileIdx + 1] : process.env.EVALS_PROFILE ?? 'pr';
|
||||
if (profile !== 'pr' && profile !== 'full') throw new Error('--profile must be pr or full');
|
||||
|
||||
// Detect base branch
|
||||
|
||||
@@ -16,7 +16,7 @@ const EVAL_DIR = getProjectEvalDir();
|
||||
const files = listEvalJsonFiles(EVAL_DIR);
|
||||
|
||||
if (files.length === 0) {
|
||||
console.log('No eval runs yet. Run: EVALS=1 bun run test:evals');
|
||||
console.log('No eval runs yet. Run: bun run eval:bg:pr');
|
||||
process.exit(0);
|
||||
}
|
||||
|
||||
|
||||
@@ -1,34 +1,239 @@
|
||||
#!/usr/bin/env bun
|
||||
/**
|
||||
* Stamp `series_identity` on a report's trial-outcomes JSONL (the pass-rates
|
||||
* history key: a hash of each case's own touchfiles, GLOBAL_TOUCHFILES
|
||||
* excluded; scripts/eval-flake-rank.ts caseSeriesIdentities). A separate step
|
||||
* Stamp `series_identity`, `series_fingerprint` and `harness_version` on a
|
||||
* report's trial-outcomes JSONL (the pass-rates history key). A separate step
|
||||
* after `test-paid-shards.ts --report`, so the paid runner's closure never
|
||||
* imports the history tool.
|
||||
*
|
||||
* Usage: bun run scripts/eval-trial-series.ts <trial-outcomes.jsonl>
|
||||
* EVAL_POLICY v2 identity (`caseSeriesIdentitiesV2`, CEO-27/ENG-3): only the
|
||||
* bytes a case owns (its paid test file, its fixtures, and the prompt files of
|
||||
* the skills it names: templates, sections and the generated SKILL.md the
|
||||
* model reads) plus HARNESS_VERSION. Shared helpers no longer reset every
|
||||
* case's history; the execution harness every case consumes is pinned by
|
||||
* scripts/harness-version.json and changes only by a deliberate bump
|
||||
* (scripts/bump-harness-version.ts). Every record also carries the full
|
||||
* consumed-input fingerprint (all touchfiles, globals included) as provenance.
|
||||
*
|
||||
* Usage:
|
||||
* bun run scripts/eval-trial-series.ts <trial-outcomes.jsonl>
|
||||
* bun run scripts/eval-trial-series.ts --backtest [--weeks 30] [--records <dir>]
|
||||
*/
|
||||
import * as fs from 'node:fs';
|
||||
import * as path from 'node:path';
|
||||
import { caseSeriesIdentities } from './eval-flake-rank';
|
||||
import { spawnSync } from 'node:child_process';
|
||||
import { createHash } from 'node:crypto';
|
||||
import { LIVE_REGISTRY, readTrialOutcomeDir, type Registry } from './eval-flake-rank';
|
||||
import { formatTrialOutcomes, parseTrialOutcomes } from '../test/helpers/eval-store';
|
||||
import harnessManifest from './harness-version.json';
|
||||
|
||||
const ROOT = path.resolve(import.meta.dir, '..');
|
||||
|
||||
/** Deliberate shared-harness version: part of every v2 series identity. Bumped only by scripts/bump-harness-version.ts. */
|
||||
export const HARNESS_VERSION: number = harnessManifest.version;
|
||||
/** Harness patterns beyond GLOBAL_TOUCHFILES whose bytes HARNESS_VERSION pins. */
|
||||
export const HARNESS_PATTERNS: readonly string[] = harnessManifest.patterns;
|
||||
|
||||
export interface TreeEntry { file: string; blob: string }
|
||||
|
||||
/** test-selection's matchGlob semantics, compiled once per pattern (the backtest matches thousands of files per revision). */
|
||||
const globCache = new Map<string, RegExp>();
|
||||
export function globRegex(pattern: string): RegExp {
|
||||
let regex = globCache.get(pattern);
|
||||
if (!regex) {
|
||||
regex = new RegExp(`^${pattern.replace(/\./g, '\\.').replace(/\*\*/g, '{{GLOBSTAR}}').replace(/\*/g, '[^/]*').replace(/\{\{GLOBSTAR\}\}/g, '.*')}$`);
|
||||
globCache.set(pattern, regex);
|
||||
}
|
||||
return regex;
|
||||
}
|
||||
const matches = (file: string, patterns: readonly string[]) => patterns.some(pattern => globRegex(pattern).test(file));
|
||||
|
||||
/** Tracked files with their blob ids, at the index (`ref` omitted) or at a commit. */
|
||||
export function treeEntries(root: string, ref?: string): TreeEntry[] {
|
||||
const args = ref ? ['ls-tree', '-r', ref] : ['ls-files', '-s'];
|
||||
const listed = spawnSync('git', args, { cwd: root, encoding: 'utf8', timeout: 60_000, maxBuffer: 256 * 1024 * 1024 });
|
||||
if (listed.status !== 0) throw new Error(`git ${args.join(' ')} failed: ${listed.stderr}`);
|
||||
return listed.stdout.split('\n').filter(Boolean).map(line => {
|
||||
const [meta, file] = line.split('\t');
|
||||
const parts = meta!.split(' ');
|
||||
return { file: file!, blob: (ref ? parts[2] : parts[1])! };
|
||||
});
|
||||
}
|
||||
|
||||
/** Top-level directories that hold a skill template. */
|
||||
export function skillDirs(entries: TreeEntry[]): Set<string> {
|
||||
return new Set(entries.filter(entry => /^[^/]+\/SKILL\.md\.tmpl$/.test(entry.file)).map(entry => entry.file.split('/')[0]!));
|
||||
}
|
||||
|
||||
/**
|
||||
* A file a case owns: a paid test file (not a unit test under test/helpers),
|
||||
* a fixture, or a skill prompt file (markdown, template or JSON manifest in a
|
||||
* skill directory, generated SKILL.md included: it is what the model reads).
|
||||
*/
|
||||
export function isCaseOwnedFile(file: string, skills: Set<string>): boolean {
|
||||
if (file.endsWith('.test.ts')) return !file.startsWith('test/helpers/');
|
||||
if (/(^|\/)fixtures\//.test(file)) return true;
|
||||
if (file === 'SKILL.md' || file === 'SKILL.md.tmpl') return true;
|
||||
return skills.has(file.split('/')[0]!) && /\.(md|tmpl|json)$/.test(file);
|
||||
}
|
||||
|
||||
/** Files HARNESS_VERSION pins: GLOBAL_TOUCHFILES plus HARNESS_PATTERNS, unit tests excluded. */
|
||||
export function harnessFiles(entries: TreeEntry[], registry: Registry = LIVE_REGISTRY, patterns: readonly string[] = HARNESS_PATTERNS): TreeEntry[] {
|
||||
const all = [...registry.globals, ...patterns];
|
||||
return entries.filter(entry => !entry.file.endsWith('.test.ts') && matches(entry.file, all));
|
||||
}
|
||||
|
||||
const digest = (text: string) => createHash('sha256').update(text).digest('hex').slice(0, 16);
|
||||
const lines = (entries: TreeEntry[]) => entries.map(entry => `${entry.file} ${entry.blob}`).sort().join('\n');
|
||||
|
||||
/** v2 series identity and full provenance fingerprint per case. */
|
||||
export function caseSeriesIdentitiesV2(ids: string[], entries: TreeEntry[], registry: Registry = LIVE_REGISTRY,
|
||||
harnessVersion = HARNESS_VERSION): Record<string, { identity: string; fingerprint: string }> {
|
||||
const skills = skillDirs(entries);
|
||||
return Object.fromEntries(ids.map(id => {
|
||||
const patterns = registry.touchfiles[id] ?? registry.judgeTouchfiles[id] ?? [];
|
||||
const consumed = entries.filter(entry => matches(entry.file, patterns));
|
||||
const owned = consumed.filter(entry => isCaseOwnedFile(entry.file, skills));
|
||||
const full = entries.filter(entry => matches(entry.file, [...patterns, ...registry.globals]));
|
||||
return [id, { identity: digest(`${id}\nharness ${harnessVersion}\n${lines(owned)}`), fingerprint: digest(`${id}\n${lines(full)}`) }];
|
||||
}));
|
||||
}
|
||||
|
||||
/** Rewrite the file with every record stamped; an invalid line fails the whole stamp. */
|
||||
export function stampTrialSeries(file: string, root = ROOT): number {
|
||||
const { records, errors } = parseTrialOutcomes(fs.readFileSync(file, 'utf8'));
|
||||
if (errors.length) throw new Error(`${file}: ${errors.join('; ')}`);
|
||||
const identities = caseSeriesIdentities([...new Set(records.map(record => record.case))], root);
|
||||
const stamped = records.map(record => ({ ...record, series_identity: identities[record.case] }));
|
||||
const identities = caseSeriesIdentitiesV2([...new Set(records.map(record => record.case))], treeEntries(root));
|
||||
const stamped = records.map(record => ({ ...record, series_identity: identities[record.case]!.identity,
|
||||
series_fingerprint: identities[record.case]!.fingerprint, harness_version: HARNESS_VERSION }));
|
||||
fs.writeFileSync(file, formatTrialOutcomes(stamped));
|
||||
return stamped.length;
|
||||
}
|
||||
|
||||
// --- Backtest (ENG-3/DX-12): would the identity let cases reach the quarantine minimum? ---
|
||||
|
||||
export interface BacktestRow { variant: string; latestQualifying: number; present: number; meanShareLast8: number; medianChanges: number }
|
||||
|
||||
/**
|
||||
* Over weekly main revisions (Monday 06:00 UTC, the census cron), how many
|
||||
* blocking cases would hold one identity long enough to reach `minTrials`
|
||||
* within the last `window` weekly runs (rule and judge cases record one
|
||||
* trial per run, behavior cases a panel of 3). `v1` is today's identity
|
||||
* (own touchfiles minus GLOBAL_TOUCHFILES); `v2` is caseSeriesIdentitiesV2;
|
||||
* `v2+harness` also resets every series whenever the pinned harness bytes
|
||||
* change (the cost of bumping HARNESS_VERSION on every harness edit).
|
||||
*/
|
||||
export function weeklyBacktest(opts: { root: string; weeks: number; now: number; registry?: Registry; minTrials?: number; window?: number }):
|
||||
{ revisions: string[]; harnessChanges: number; rows: BacktestRow[] } {
|
||||
const registry = opts.registry ?? LIVE_REGISTRY;
|
||||
const minTrials = opts.minTrials ?? 10, window = opts.window ?? 10;
|
||||
const ids = Object.keys(registry.tiers).filter(id => registry.tiers[id] === 'gate' || registry.tiers[id] === 'periodic');
|
||||
const monday = new Date(opts.now);
|
||||
monday.setUTCHours(6, 0, 0, 0);
|
||||
monday.setUTCDate(monday.getUTCDate() - ((monday.getUTCDay() + 6) % 7));
|
||||
const revisions: string[] = [];
|
||||
for (let week = opts.weeks - 1; week >= 0; week--) {
|
||||
const at = new Date(monday.getTime() - week * 7 * 86_400_000).toISOString();
|
||||
const sha = spawnSync('git', ['rev-list', '-1', '--first-parent', `--before=${at}`, 'HEAD'], { cwd: opts.root, encoding: 'utf8', timeout: 10_000 }).stdout.trim();
|
||||
if (sha) revisions.push(sha);
|
||||
}
|
||||
const series: Record<string, Record<string, string[]>> = { v1: {}, v2: {} };
|
||||
const harness: string[] = [];
|
||||
for (const sha of revisions) {
|
||||
const entries = treeEntries(opts.root, sha);
|
||||
const present = new Set(ids.filter(id => entries.some(entry => entry.file.endsWith('.test.ts')
|
||||
&& matches(entry.file, registry.touchfiles[id] ?? []))));
|
||||
const v2 = caseSeriesIdentitiesV2(ids, entries, registry, 0);
|
||||
const nonGlobal = entries.filter(entry => !matches(entry.file, registry.globals));
|
||||
for (const id of ids) {
|
||||
const own = nonGlobal.filter(entry => matches(entry.file, registry.touchfiles[id] ?? []));
|
||||
(series.v1![id] ??= []).push(present.has(id) ? digest(`${id}\n${lines(own)}`) : '');
|
||||
(series.v2![id] ??= []).push(present.has(id) ? v2[id]!.identity : '');
|
||||
}
|
||||
harness.push(digest(lines(harnessFiles(entries, registry))));
|
||||
}
|
||||
const perRun = (id: string) => (registry.kinds[id] === 'behavior' ? 3 : 1);
|
||||
const row = (variant: string, values: Record<string, string[]>, resetOnHarness: boolean): BacktestRow => {
|
||||
const shares: number[] = [];
|
||||
let latestQualifying = 0, present = 0;
|
||||
for (let week = 0; week < revisions.length; week++) {
|
||||
let qualifying = 0, live = 0;
|
||||
for (const id of ids) {
|
||||
const list = values[id]!;
|
||||
if (!list[week]) continue;
|
||||
live++;
|
||||
let runs = 1;
|
||||
while (runs < window && week - runs >= 0 && list[week - runs] === list[week] && (!resetOnHarness || harness[week - runs] === harness[week])) runs++;
|
||||
if (runs * perRun(id) >= minTrials) qualifying++;
|
||||
}
|
||||
shares.push(live ? qualifying / live : 0);
|
||||
if (week === revisions.length - 1) { latestQualifying = qualifying; present = live; }
|
||||
}
|
||||
const changes = ids.map(id => values[id]!.filter(Boolean)).map(list => list.slice(1).filter((value, index) => value !== list[index]).length).sort((a, b) => a - b);
|
||||
const last8 = shares.slice(-8);
|
||||
return { variant, latestQualifying, present, meanShareLast8: last8.reduce((a, b) => a + b, 0) / Math.max(1, last8.length), medianChanges: changes[Math.floor(changes.length / 2)] ?? 0 };
|
||||
};
|
||||
return { revisions, harnessChanges: harness.slice(1).filter((value, index) => value !== harness[index]).length,
|
||||
rows: [row('v1', series.v1!, false), row('v2', series.v2!, false), row('v2+harness', series.v2!, true)] };
|
||||
}
|
||||
|
||||
/**
|
||||
* Recorded-trial backtest: cases reaching `minTrials` on one series when the
|
||||
* recorded trials (any lane) are re-keyed by the v2 identity of the revision
|
||||
* that produced them, vs the series_identity they were stamped with.
|
||||
*/
|
||||
export function recordBacktest(opts: { root: string; dir: string; registry?: Registry; minTrials?: number }):
|
||||
{ trials: number; revisions: number; missing: string[]; recorded: number; v2: number; blocking: number } {
|
||||
const registry = opts.registry ?? LIVE_REGISTRY;
|
||||
const minTrials = opts.minTrials ?? 10;
|
||||
const seen = new Set<string>();
|
||||
const records = readTrialOutcomeDir(opts.dir).records.filter(record => {
|
||||
const key = `${record.run_id}|${record.attempt}|${record.case}|${record.trial}`;
|
||||
if (record.outcome === 'skipped' || seen.has(key) || !record.sha) return false;
|
||||
seen.add(key);
|
||||
return true;
|
||||
});
|
||||
const cases = [...new Set(records.map(record => record.case))];
|
||||
const identity = new Map<string, Record<string, { identity: string }>>();
|
||||
const missing: string[] = [];
|
||||
for (const sha of new Set(records.map(record => record.sha!))) {
|
||||
try { identity.set(sha, caseSeriesIdentitiesV2(cases, treeEntries(opts.root, sha), registry, 0)); } catch { missing.push(sha); }
|
||||
}
|
||||
const qualifying = (key: (record: (typeof records)[number]) => string | undefined) => {
|
||||
const counts = new Map<string, number>();
|
||||
for (const record of records) {
|
||||
const series = key(record);
|
||||
if (series) counts.set(`${record.case}|${series}|${record.model}|${record.cli_version}`, (counts.get(`${record.case}|${series}|${record.model}|${record.cli_version}`) ?? 0) + 1);
|
||||
}
|
||||
const best = new Map<string, number>();
|
||||
for (const [series, count] of counts) best.set(series.split('|')[0]!, Math.max(best.get(series.split('|')[0]!) ?? 0, count));
|
||||
return [...best].filter(([id, count]) => (registry.tiers[id] === 'gate' || registry.tiers[id] === 'periodic') && count >= minTrials).length;
|
||||
};
|
||||
return { trials: records.length, revisions: identity.size, missing,
|
||||
recorded: qualifying(record => record.series_identity), v2: qualifying(record => identity.get(record.sha!)?.[record.case]?.identity),
|
||||
blocking: cases.filter(id => registry.tiers[id] === 'gate' || registry.tiers[id] === 'periodic').length };
|
||||
}
|
||||
|
||||
if (import.meta.main) {
|
||||
const file = process.argv[2];
|
||||
const argv = process.argv.slice(2);
|
||||
if (argv[0] === '--backtest') {
|
||||
const flag = (name: string) => { const index = argv.indexOf(name); return index === -1 ? undefined : argv[index + 1]; };
|
||||
const weekly = weeklyBacktest({ root: ROOT, weeks: Number(flag('--weeks') ?? 30), now: Date.now() });
|
||||
console.log(`weekly main revisions: ${weekly.revisions.length}; pinned-harness changes between consecutive weeks: ${weekly.harnessChanges}`);
|
||||
console.log('variant qualifying now mean share (last 8 weeks) median identity changes/case');
|
||||
for (const row of weekly.rows) {
|
||||
console.log(`${row.variant.padEnd(11)} ${`${row.latestQualifying}/${row.present}`.padEnd(15)} ${`${(row.meanShareLast8 * 100).toFixed(1)}%`.padEnd(26)} ${row.medianChanges}`);
|
||||
}
|
||||
const dir = flag('--records');
|
||||
if (dir) {
|
||||
const recorded = recordBacktest({ root: ROOT, dir });
|
||||
console.log(`recorded trials: ${recorded.trials} over ${recorded.revisions} revision(s)${recorded.missing.length ? ` (${recorded.missing.length} revision(s) not in this clone: fetch them)` : ''}`);
|
||||
console.log(`blocking cases with >= 10 trials on one series: recorded identity ${recorded.recorded}/${recorded.blocking}, v2 identity ${recorded.v2}/${recorded.blocking}`);
|
||||
}
|
||||
process.exit(0);
|
||||
}
|
||||
const file = argv[0];
|
||||
if (!file) {
|
||||
console.error('usage: bun run scripts/eval-trial-series.ts <trial-outcomes.jsonl>');
|
||||
console.error('usage: bun run scripts/eval-trial-series.ts <trial-outcomes.jsonl> | --backtest [--weeks N] [--records <dir>]');
|
||||
process.exit(2);
|
||||
}
|
||||
console.log(`[eval-trial-series] stamped ${stampTrialSeries(file)} record(s) in ${file}`);
|
||||
|
||||
@@ -1,194 +0,0 @@
|
||||
/**
|
||||
* Live E2E test watcher dashboard.
|
||||
*
|
||||
* Reads heartbeat (e2e-live.json) for current test status and
|
||||
* partial eval results (_partial-e2e*.json) for completed tests.
|
||||
* Renders a terminal dashboard every 1s.
|
||||
*
|
||||
* Usage: bun run eval:watch [--tail]
|
||||
*/
|
||||
|
||||
import * as fs from 'fs';
|
||||
import * as path from 'path';
|
||||
import * as os from 'os';
|
||||
import { evalEntryOutcome, getProjectEvalDir } from '../test/helpers/eval-store';
|
||||
import type { EvalTestEntry } from '../test/helpers/eval-store';
|
||||
|
||||
const GSTACK_DEV_DIR = path.join(os.homedir(), '.gstack-dev');
|
||||
// Heartbeat + per-run progress logs are GLOBAL by design — session-runner.ts
|
||||
// writes ~/.gstack-dev/e2e-live.json regardless of project ("heartbeat stays
|
||||
// global"). The PARTIAL file is per-project: EvalCollector writes it into
|
||||
// getProjectEvalDir() (or GSTACK_EVAL_DIR), so watching the legacy global
|
||||
// path missed it whenever slug detection succeeded — i.e. the normal case.
|
||||
const HEARTBEAT_PATH = path.join(GSTACK_DEV_DIR, 'e2e-live.json');
|
||||
const STALE_THRESHOLD_SEC = 600; // 10 minutes
|
||||
|
||||
export interface HeartbeatData {
|
||||
runId: string;
|
||||
pid?: number;
|
||||
startedAt: string;
|
||||
currentTest: string;
|
||||
status: string;
|
||||
turn: number;
|
||||
toolCount: number;
|
||||
lastTool: string;
|
||||
lastToolAt: string;
|
||||
elapsedSec: number;
|
||||
}
|
||||
|
||||
export interface PartialData {
|
||||
tests: Array<Partial<EvalTestEntry> & Pick<EvalTestEntry, 'name' | 'passed' | 'cost_usd' | 'duration_ms'>>;
|
||||
total_cost_usd: number;
|
||||
_partial?: boolean;
|
||||
}
|
||||
|
||||
/** Read and parse a JSON file, returning null on any error. */
|
||||
function readJSON<T>(filePath: string): T | null {
|
||||
try {
|
||||
return JSON.parse(fs.readFileSync(filePath, 'utf-8'));
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
/** Read the legacy collector and suite collectors in this shard, never finals. */
|
||||
export function readPartialResults(evalDir = process.env.GSTACK_EVAL_DIR || getProjectEvalDir()): PartialData | null {
|
||||
let names: string[];
|
||||
try { names = fs.readdirSync(evalDir).sort(); } catch { return null; }
|
||||
const partials = names.filter(name => /^_partial-e2e(?:-[a-z0-9]+(?:-[a-z0-9]+)*)?\.json$/.test(name))
|
||||
.map(name => ({ name, data: readJSON<PartialData>(path.join(evalDir, name)) }))
|
||||
.filter((item): item is { name: string; data: PartialData } => Array.isArray(item.data?.tests));
|
||||
if (partials.length === 0) return null;
|
||||
if (partials.length === 1) return partials[0].data;
|
||||
const named = partials.filter(item => item.name !== '_partial-e2e.json').flatMap(item => item.data.tests);
|
||||
const namedSuites = new Set(named.map(test => test.suite ?? null));
|
||||
const legacy = partials.find(item => item.name === '_partial-e2e.json')?.data.tests ?? [];
|
||||
// A legacy accumulator can remain after adopting suite files. Prefer the
|
||||
// entire current suite snapshot, including its own retries. Old retries and
|
||||
// retired tests must not leak back in from the legacy snapshot.
|
||||
const tests = [...legacy.filter(test => !namedSuites.has(test.suite ?? null)), ...named];
|
||||
return { tests, total_cost_usd: tests.reduce((sum, test) => sum + test.cost_usd, 0), _partial: true };
|
||||
}
|
||||
|
||||
/** Check if a process is alive (signal 0 = existence check, doesn't kill). */
|
||||
function isProcessAlive(pid: number): boolean {
|
||||
try {
|
||||
process.kill(pid, 0);
|
||||
return true;
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
/** Format seconds as Xm Ys */
|
||||
function formatDuration(sec: number): string {
|
||||
if (sec < 60) return `${sec}s`;
|
||||
const m = Math.floor(sec / 60);
|
||||
const s = sec % 60;
|
||||
return `${m}m ${s}s`;
|
||||
}
|
||||
|
||||
/** Render dashboard from heartbeat + partial data. Pure function for testability. */
|
||||
export function renderDashboard(heartbeat: HeartbeatData | null, partial: PartialData | null): string {
|
||||
const lines: string[] = [];
|
||||
|
||||
if (!heartbeat && !partial) {
|
||||
lines.push('E2E Watch — No active run detected');
|
||||
lines.push('');
|
||||
lines.push(`Heartbeat: ${HEARTBEAT_PATH} (not found)`);
|
||||
lines.push(`Partial: ${path.join(process.env.GSTACK_EVAL_DIR || getProjectEvalDir(), '_partial-e2e*.json')} (not found)`);
|
||||
lines.push('');
|
||||
lines.push('Start a run with: EVALS=1 bun test test/skill-e2e-*.test.ts');
|
||||
return lines.join('\n');
|
||||
}
|
||||
|
||||
const runId = heartbeat?.runId || 'unknown';
|
||||
const elapsed = heartbeat?.elapsedSec || 0;
|
||||
lines.push(`E2E Watch \u2014 Run ${runId} \u2014 ${formatDuration(elapsed)}`);
|
||||
lines.push('\u2550'.repeat(55));
|
||||
|
||||
// Completed tests from partial
|
||||
if (partial?.tests) {
|
||||
for (const t of partial.tests) {
|
||||
const manual = evalEntryOutcome(t) === 'manual-review';
|
||||
const icon = manual ? 'M' : evalEntryOutcome(t) === 'passed' ? '\u2713' : '\u2717';
|
||||
const cost = `$${t.cost_usd.toFixed(2)}`;
|
||||
const dur = `${Math.round(t.duration_ms / 1000)}s`;
|
||||
const turns = t.turns_used !== undefined ? `${t.turns_used} turns` : '';
|
||||
const name = t.name.length > 30 ? t.name.slice(0, 27) + '...' : t.name.padEnd(30);
|
||||
const approval = manual ? ` MANUAL/unscored; approved by ${t.manual_review!.approval.approved_by} (${t.manual_review!.approval.approval_url})` : '';
|
||||
lines.push(` ${icon} ${name} ${cost.padStart(6)} ${dur.padStart(5)} ${turns}${approval}`);
|
||||
}
|
||||
}
|
||||
|
||||
// Current test from heartbeat
|
||||
if (heartbeat && heartbeat.status === 'running') {
|
||||
const name = heartbeat.currentTest.length > 30
|
||||
? heartbeat.currentTest.slice(0, 27) + '...'
|
||||
: heartbeat.currentTest.padEnd(30);
|
||||
lines.push(` \u29D6 ${name} ${formatDuration(heartbeat.elapsedSec).padStart(6)} turn ${heartbeat.turn} last: ${heartbeat.lastTool}`);
|
||||
|
||||
// Stale detection
|
||||
const lastToolTime = new Date(heartbeat.lastToolAt).getTime();
|
||||
const staleSec = Math.round((Date.now() - lastToolTime) / 1000);
|
||||
if (staleSec > STALE_THRESHOLD_SEC) {
|
||||
lines.push(` \u26A0 STALE: last tool call was ${formatDuration(staleSec)} ago \u2014 run may have crashed`);
|
||||
}
|
||||
}
|
||||
|
||||
lines.push('\u2500'.repeat(55));
|
||||
|
||||
// Summary
|
||||
const completedCount = partial?.tests?.length || 0;
|
||||
const totalCost = partial?.total_cost_usd || 0;
|
||||
const running = heartbeat?.status === 'running' ? 1 : 0;
|
||||
lines.push(` Completed: ${completedCount} Running: ${running} Cost: $${totalCost.toFixed(2)} Elapsed: ${formatDuration(elapsed)}`);
|
||||
const manualAccepted = partial?.tests?.filter(t => evalEntryOutcome(t) === 'manual-review').length ?? 0;
|
||||
if (manualAccepted) lines.push(` Manual accepted: ${manualAccepted} unscored provider refusal(s)`);
|
||||
|
||||
if (heartbeat?.runId) {
|
||||
const logPath = path.join(GSTACK_DEV_DIR, 'e2e-runs', heartbeat.runId, 'progress.log');
|
||||
lines.push(` Logs: ${logPath}`);
|
||||
}
|
||||
|
||||
return lines.join('\n');
|
||||
}
|
||||
|
||||
// --- Main ---
|
||||
|
||||
if (import.meta.main) {
|
||||
const showTail = process.argv.includes('--tail');
|
||||
|
||||
const render = () => {
|
||||
let heartbeat = readJSON<HeartbeatData>(HEARTBEAT_PATH);
|
||||
const partial = readPartialResults();
|
||||
|
||||
// Auto-clear heartbeat if the process is dead
|
||||
if (heartbeat?.pid && !isProcessAlive(heartbeat.pid)) {
|
||||
try { fs.unlinkSync(HEARTBEAT_PATH); } catch { /* already gone */ }
|
||||
process.stdout.write('\x1B[2J\x1B[H');
|
||||
process.stdout.write(`Cleared stale heartbeat — PID ${heartbeat.pid} is no longer running.\n\n`);
|
||||
heartbeat = null;
|
||||
}
|
||||
|
||||
// Clear screen
|
||||
process.stdout.write('\x1B[2J\x1B[H');
|
||||
process.stdout.write(renderDashboard(heartbeat, partial) + '\n');
|
||||
|
||||
// --tail: show last 10 lines of progress.log
|
||||
if (showTail && heartbeat?.runId) {
|
||||
const logPath = path.join(GSTACK_DEV_DIR, 'e2e-runs', heartbeat.runId, 'progress.log');
|
||||
try {
|
||||
const content = fs.readFileSync(logPath, 'utf-8');
|
||||
const tail = content.split('\n').filter(l => l.trim()).slice(-10);
|
||||
process.stdout.write('\nRecent progress:\n');
|
||||
for (const line of tail) {
|
||||
process.stdout.write(line + '\n');
|
||||
}
|
||||
} catch { /* log file may not exist yet */ }
|
||||
}
|
||||
};
|
||||
|
||||
render();
|
||||
setInterval(render, 1000);
|
||||
}
|
||||
@@ -0,0 +1,644 @@
|
||||
{
|
||||
"version": 1,
|
||||
"recordedAt": "2026-10-04T19:14:06.352Z",
|
||||
"durations": {
|
||||
"browse/test/activity.test.ts": 128,
|
||||
"browse/test/adversarial-security.test.ts": 85,
|
||||
"browse/test/batch.test.ts": 6404,
|
||||
"browse/test/bridge-chromium-e2e.test.ts": 3673,
|
||||
"browse/test/browse-client.test.ts": 189,
|
||||
"browse/test/browse-eval-wrapping.test.ts": 89,
|
||||
"browse/test/browser-manager-custom-chromium.test.ts": 1005,
|
||||
"browse/test/browser-skills-storage.test.ts": 179,
|
||||
"browse/test/build-command-response.test.ts": 981,
|
||||
"browse/test/build.test.ts": 160,
|
||||
"browse/test/bun-polyfill.test.ts": 2455,
|
||||
"browse/test/busy-daemon-iron-rule.test.ts": 7811,
|
||||
"browse/test/busy-daemon-recovery.test.ts": 779,
|
||||
"browse/test/cdp-allowlist.test.ts": 40,
|
||||
"browse/test/cdp-e2e.test.ts": 1134,
|
||||
"browse/test/cdp-inspector-history-cap.test.ts": 109,
|
||||
"browse/test/cdp-mutex.test.ts": 841,
|
||||
"browse/test/cdp-session-cleanup.test.ts": 62,
|
||||
"browse/test/claude-bin.test.ts": 51,
|
||||
"browse/test/cli-lock.test.ts": 81,
|
||||
"browse/test/cli-loopback-no-proxy.test.ts": 66,
|
||||
"browse/test/cli-setsid-daemonize.test.ts": 61,
|
||||
"browse/test/cli-start-final-healthcheck.test.ts": 49,
|
||||
"browse/test/cli-supervisor.test.ts": 91,
|
||||
"browse/test/content-security.test.ts": 1659,
|
||||
"browse/test/cookie-auth-verification.test.ts": 19136,
|
||||
"browse/test/cookie-credential-deadline.test.ts": 371,
|
||||
"browse/test/cookie-database.test.ts": 675,
|
||||
"browse/test/cookie-fixture-delete-lease.test.ts": 1305,
|
||||
"browse/test/cookie-import-browser.test.ts": 105,
|
||||
"browse/test/cookie-import-native-job.test.ts": 99288,
|
||||
"browse/test/cookie-import-native.test.ts": 7321,
|
||||
"browse/test/cookie-import-node.test.ts": 4376,
|
||||
"browse/test/cookie-import-operation.test.ts": 1784,
|
||||
"browse/test/cookie-import-reliability.test.ts": 1231,
|
||||
"browse/test/cookie-import-transport.test.ts": 55,
|
||||
"browse/test/cookie-picker-binding-browser.test.ts": 3233,
|
||||
"browse/test/cookie-picker-binding.test.ts": 725,
|
||||
"browse/test/cookie-picker-csrf-browser.test.ts": 1163,
|
||||
"browse/test/cookie-picker-routes.test.ts": 601,
|
||||
"browse/test/cookie-picker-ui.test.ts": 40992,
|
||||
"browse/test/daemon-log-hygiene.test.ts": 113,
|
||||
"browse/test/daemon-mismatch-refuse.test.ts": 325,
|
||||
"browse/test/data-platform.test.ts": 158,
|
||||
"browse/test/domain-skills-e2e.test.ts": 1063,
|
||||
"browse/test/domain-skills-storage.test.ts": 140,
|
||||
"browse/test/dotenv-isolation.test.ts": 498,
|
||||
"browse/test/dual-listener.test.ts": 1057,
|
||||
"browse/test/dx-polish.test.ts": 54,
|
||||
"browse/test/error-handling.test.ts": 46,
|
||||
"browse/test/extension-sender-auth.test.ts": 76,
|
||||
"browse/test/extension-token.test.ts": 1710,
|
||||
"browse/test/file-drop.test.ts": 53,
|
||||
"browse/test/file-permissions.test.ts": 512,
|
||||
"browse/test/fill-change-event.test.ts": 2056,
|
||||
"browse/test/find-browse.test.ts": 126,
|
||||
"browse/test/from-file-path-validation.test.ts": 68,
|
||||
"browse/test/handoff.test.ts": 7769,
|
||||
"browse/test/headless-gpu-and-reap.test.ts": 639,
|
||||
"browse/test/headless-launch-options.test.ts": 1235,
|
||||
"browse/test/launch-signal-flags.test.ts": 57,
|
||||
"browse/test/learnings-injection.test.ts": 2308,
|
||||
"browse/test/media-extract-unit.test.ts": 135,
|
||||
"browse/test/memory-command.test.ts": 1273,
|
||||
"browse/test/memory-leak-reproducer.test.ts": 490,
|
||||
"browse/test/navigation-guard.test.ts": 9998,
|
||||
"browse/test/output-path-symlinks.test.ts": 106,
|
||||
"browse/test/pair-agent-optin-gate.test.ts": 868,
|
||||
"browse/test/pdf-flags.test.ts": 63,
|
||||
"browse/test/playwright-core-patch.test.ts": 53,
|
||||
"browse/test/poisoned-bundle-probe.test.ts": 524,
|
||||
"browse/test/process-liveness-windows.test.ts": 189,
|
||||
"browse/test/proxy-config.test.ts": 63,
|
||||
"browse/test/proxy-redact.test.ts": 39,
|
||||
"browse/test/pty-inject-scan.test.ts": 1455,
|
||||
"browse/test/pty-session-lease.test.ts": 78,
|
||||
"browse/test/rebrand-signed-bundle.test.ts": 59,
|
||||
"browse/test/regression-pr1169-pdf-from-file-invalid-json.test.ts": 79,
|
||||
"browse/test/restart-env.test.ts": 62,
|
||||
"browse/test/sanitize.test.ts": 40,
|
||||
"browse/test/screenshot-size-guard.test.ts": 382,
|
||||
"browse/test/security-adversarial-fixes.test.ts": 78,
|
||||
"browse/test/security-adversarial.test.ts": 62,
|
||||
"browse/test/security-bench.test.ts": 52,
|
||||
"browse/test/security-classifier-download-cleanup.test.ts": 59,
|
||||
"browse/test/security-classifier.test.ts": 42,
|
||||
"browse/test/security-integration.test.ts": 44,
|
||||
"browse/test/security-live-playwright.test.ts": 1037,
|
||||
"browse/test/security-sidecar-client.test.ts": 85,
|
||||
"browse/test/security.test.ts": 73,
|
||||
"browse/test/server-auth.test.ts": 725,
|
||||
"browse/test/server-factory.test.ts": 2959,
|
||||
"browse/test/server-flush-trackers.test.ts": 62,
|
||||
"browse/test/server-launch-resolution.test.ts": 97,
|
||||
"browse/test/server-lock-errors.test.ts": 110,
|
||||
"browse/test/server-no-import-side-effects.test.ts": 679,
|
||||
"browse/test/server-pty-lease-routes.test.ts": 636,
|
||||
"browse/test/server-route-auth-blackbox.test.ts": 2355,
|
||||
"browse/test/server-route-dispatch-ratchet.test.ts": 574,
|
||||
"browse/test/server-sanitize-surrogates.test.ts": 675,
|
||||
"browse/test/server-tmp-state-path.test.ts": 56,
|
||||
"browse/test/session-cookie-store.test.ts": 81,
|
||||
"browse/test/sidebar-tabs.test.ts": 749,
|
||||
"browse/test/sidebar-ux.test.ts": 693,
|
||||
"browse/test/sidepanel-patient-autoconnect.test.ts": 59,
|
||||
"browse/test/sidepanel-reattach.test.ts": 89,
|
||||
"browse/test/sidepanel-restart-dispose.test.ts": 63,
|
||||
"browse/test/skill-token.test.ts": 61,
|
||||
"browse/test/socks-bridge.test.ts": 604,
|
||||
"browse/test/sse-helpers.test.ts": 201,
|
||||
"browse/test/sse-session-cookie.test.ts": 58,
|
||||
"browse/test/state-ttl.test.ts": 49,
|
||||
"browse/test/stealth-extended.test.ts": 47,
|
||||
"browse/test/stealth-layer-c.test.ts": 56,
|
||||
"browse/test/stealth-webdriver.test.ts": 9862,
|
||||
"browse/test/stop-ack-before-shutdown.test.ts": 184,
|
||||
"browse/test/stop-dead-daemon.test.ts": 484,
|
||||
"browse/test/tab-each.test.ts": 111,
|
||||
"browse/test/tab-guardrail.test.ts": 681,
|
||||
"browse/test/tab-isolation.test.ts": 718,
|
||||
"browse/test/tab-session-frame-detach.test.ts": 63,
|
||||
"browse/test/telemetry-optout.test.ts": 198,
|
||||
"browse/test/telemetry.test.ts": 287,
|
||||
"browse/test/temp-dirs.test.ts": 144,
|
||||
"browse/test/terminal-agent-detach-reattach.test.ts": 71,
|
||||
"browse/test/terminal-agent-import.test.ts": 4581,
|
||||
"browse/test/terminal-agent-keepalive.test.ts": 59,
|
||||
"browse/test/terminal-agent-native-observation.test.ts": 82,
|
||||
"browse/test/terminal-agent-owner-watchdog.test.ts": 3255,
|
||||
"browse/test/terminal-agent-pid-identity.test.ts": 167,
|
||||
"browse/test/terminal-agent-port-range.test.ts": 79,
|
||||
"browse/test/terminal-agent-ring-buffer-runtime.test.ts": 77,
|
||||
"browse/test/terminal-agent-session-routing.test.ts": 49,
|
||||
"browse/test/terminal-agent-watchdog.test.ts": 49,
|
||||
"browse/test/terminal-agent.test.ts": 728,
|
||||
"browse/test/terminal-pty-lifecycle.test.ts": 43,
|
||||
"browse/test/token-registry.test.ts": 59,
|
||||
"browse/test/tunnel-gate-unit.test.ts": 650,
|
||||
"browse/test/watch.test.ts": 590,
|
||||
"browse/test/welcome-page.test.ts": 95,
|
||||
"browse/test/windows-spawn-hide.test.ts": 116,
|
||||
"browse/test/xprotect-heal.test.ts": 452,
|
||||
"browser-skills/hackernews-frontpage/script.test.ts": 63,
|
||||
"design/test/brief.test.ts": 46,
|
||||
"design/test/cli-persistence.test.ts": 7156,
|
||||
"design/test/cli-version.test.ts": 90,
|
||||
"design/test/feedback-roundtrip-daemon.test.ts": 447,
|
||||
"design/test/feedback-roundtrip.test.ts": 3442,
|
||||
"design/test/image-args.test.ts": 104,
|
||||
"design/test/image-gen-pairing.test.ts": 45,
|
||||
"design/test/receipted-fetch.test.ts": 62,
|
||||
"ios-qa/daemon/test/allowlist.test.ts": 53,
|
||||
"ios-qa/daemon/test/audit.test.ts": 49,
|
||||
"ios-qa/daemon/test/auth-mint.test.ts": 58,
|
||||
"ios-qa/daemon/test/daemon-integration.test.ts": 445,
|
||||
"ios-qa/daemon/test/proxy-classify.test.ts": 63,
|
||||
"ios-qa/daemon/test/proxy-transport.test.ts": 1876,
|
||||
"ios-qa/daemon/test/session-tokens.test.ts": 52,
|
||||
"ios-qa/daemon/test/single-instance.test.ts": 56,
|
||||
"ios-qa/daemon/test/tailscale-localapi.test.ts": 75,
|
||||
"ios-qa/scripts/gen-accessors.test.ts": 173,
|
||||
"make-pdf/test/asideClient.test.ts": 62,
|
||||
"make-pdf/test/cli-args.test.ts": 48,
|
||||
"make-pdf/test/cli-exit-codes.test.ts": 487,
|
||||
"make-pdf/test/diagram-prepass.test.ts": 162,
|
||||
"make-pdf/test/e2e/ci-prereqs.test.ts": 47,
|
||||
"make-pdf/test/e2e/combined-gate.test.ts": 58,
|
||||
"make-pdf/test/e2e/diagram-gate.test.ts": 59,
|
||||
"make-pdf/test/e2e/emoji-gate.test.ts": 54,
|
||||
"make-pdf/test/e2e/format-gate.test.ts": 53,
|
||||
"make-pdf/test/e2e/landscape-gate.test.ts": 54,
|
||||
"make-pdf/test/e2e/toc-gate.test.ts": 60,
|
||||
"make-pdf/test/image-policy.test.ts": 43,
|
||||
"make-pdf/test/render-offline-sanitize.test.ts": 77,
|
||||
"make-pdf/test/render.test.ts": 101,
|
||||
"make-pdf/test/setup-smoke.test.ts": 86,
|
||||
"make-pdf/test/toc-anchors.test.ts": 71,
|
||||
"make-pdf/test/toc-pages.test.ts": 91,
|
||||
"test/agents-digest.test.ts": 57,
|
||||
"test/analytics.test.ts": 68,
|
||||
"test/anthropic-preflight.test.ts": 43,
|
||||
"test/archaeology-lint.test.ts": 2835,
|
||||
"test/arm-benchmark-selftest.test.ts": 2846,
|
||||
"test/arm-setup-smoke-workflow.test.ts": 189,
|
||||
"test/auq-format-always-loaded.test.ts": 111,
|
||||
"test/auq-native-capture.test.ts": 425,
|
||||
"test/auq-parallel.test.ts": 30247,
|
||||
"test/autoplan-artifact-windows-argv.test.ts": 298,
|
||||
"test/autoplan-init.test.ts": 2142,
|
||||
"test/autoplan-method-read-audit.test.ts": 686,
|
||||
"test/autoplan-owned-state.test.ts": 838,
|
||||
"test/autoplan-pending-artifact.test.ts": 53,
|
||||
"test/autoplan-pending-question.test.ts": 4731,
|
||||
"test/autoplan-phase-handoff.test.ts": 252,
|
||||
"test/autoplan-phase-observer.test.ts": 83,
|
||||
"test/autoplan-phase-order.test.ts": 155,
|
||||
"test/autoplan-public-narration.test.ts": 116,
|
||||
"test/autoplan-publication-generation.test.ts": 198,
|
||||
"test/autoplan-publication-guard.test.ts": 6926,
|
||||
"test/autoplan-publication-hook.test.ts": 7511,
|
||||
"test/benchmark-runner.test.ts": 66,
|
||||
"test/binding-template-drift.test.ts": 286,
|
||||
"test/bootstrap-retention-shard.test.ts": 49,
|
||||
"test/bootstrap-session-lifecycle.test.ts": 45,
|
||||
"test/brain-cache-roundtrip.test.ts": 415,
|
||||
"test/brain-cache-spec.test.ts": 61,
|
||||
"test/brain-preflight.test.ts": 65,
|
||||
"test/build-script-shell-compat.test.ts": 43,
|
||||
"test/bun-subprocess-fd-lifetime.test.ts": 37,
|
||||
"test/bun-version-drift.test.ts": 55,
|
||||
"test/cache-concurrent-refresh.test.ts": 95,
|
||||
"test/carve-guard-completeness.test.ts": 52,
|
||||
"test/carve-guards-negative.test.ts": 57,
|
||||
"test/carve-plan-fixture.test.ts": 528,
|
||||
"test/carve-section-ordering.test.ts": 70,
|
||||
"test/carve-section-sharding.test.ts": 4092,
|
||||
"test/catalog-budget.test.ts": 83,
|
||||
"test/catalog-mode-full.test.ts": 2266,
|
||||
"test/catalog-trim.test.ts": 187,
|
||||
"test/ceo-barless-submit.test.ts": 188,
|
||||
"test/ceo-count-ad-v2.test.ts": 66,
|
||||
"test/ceo-expansion-auq.test.ts": 175,
|
||||
"test/ceo-expansion-pacing-native.test.ts": 160,
|
||||
"test/ceo-hold-posture-review.test.ts": 267,
|
||||
"test/ceo-mode-expansion-disposition.test.ts": 159,
|
||||
"test/ceo-mode-labels-native.test.ts": 98,
|
||||
"test/ceo-mode-option.test.ts": 675,
|
||||
"test/ceo-mode-pending-submit.test.ts": 103,
|
||||
"test/ceo-mode-posture-native.test.ts": 116,
|
||||
"test/ceo-mode-prerequisite.test.ts": 101,
|
||||
"test/ceo-mode-routing-fixture.test.ts": 5434,
|
||||
"test/ceo-plan-mode-fixture.test.ts": 3870,
|
||||
"test/ceo-posture-packet.test.ts": 130,
|
||||
"test/ceo-section-loading-fixture.test.ts": 636,
|
||||
"test/ceo-split-collection.test.ts": 761,
|
||||
"test/ceo-split-question-policy.test.ts": 640,
|
||||
"test/ceo-stale-fill-decision.test.ts": 142,
|
||||
"test/ceo-workflow-clarity.test.ts": 69,
|
||||
"test/changed-files-union.test.ts": 2654,
|
||||
"test/ci-image-cli-pin.test.ts": 86,
|
||||
"test/ci-image-tag-binding.test.ts": 55,
|
||||
"test/ci-paid-coordination.test.ts": 19997,
|
||||
"test/claude-code-runner.test.ts": 5039,
|
||||
"test/claude-code-windows-job.test.ts": 3266,
|
||||
"test/claude-md-size.test.ts": 182,
|
||||
"test/claude-overlay-models.test.ts": 222,
|
||||
"test/claude-provider-keychain.test.ts": 383,
|
||||
"test/codex-carve-fixture.test.ts": 2277,
|
||||
"test/codex-generation-model.test.ts": 310,
|
||||
"test/codex-invocation-sites.test.ts": 629,
|
||||
"test/codex-resume-flag-semantics.test.ts": 79,
|
||||
"test/conductor-env-shim.test.ts": 51,
|
||||
"test/context-budget-ratchet.test.ts": 293,
|
||||
"test/context-remaining-work-provenance.test.ts": 52,
|
||||
"test/cookie-workflow-judge-input.test.ts": 1403,
|
||||
"test/cookie-workflow-manual-review.test.ts": 184,
|
||||
"test/coverage-audit.test.ts": 6141,
|
||||
"test/cso-bounded-file.test.ts": 69,
|
||||
"test/cso-docker-policy.test.ts": 86,
|
||||
"test/cso-git-hardening.test.ts": 87,
|
||||
"test/cso-history.test.ts": 52,
|
||||
"test/cso-image-provisioning.test.ts": 287,
|
||||
"test/cso-lease-identity.test.ts": 369,
|
||||
"test/cso-node-lifecycle-integration.test.ts": 91,
|
||||
"test/cso-ntfs-fixture.test.ts": 586,
|
||||
"test/cso-postgresql-policy.test.ts": 72,
|
||||
"test/cso-preparation.test.ts": 180,
|
||||
"test/cso-preserved.test.ts": 66,
|
||||
"test/cso-public-ghcr.test.ts": 128,
|
||||
"test/cso-runtime-promotion.test.ts": 90,
|
||||
"test/cso-scanner-docker-integration.test.ts": 90,
|
||||
"test/cso-scanners.test.ts": 173,
|
||||
"test/cso-snapshot-disappearance-race.test.ts": 62,
|
||||
"test/cso-spec-taxonomy-alignment.test.ts": 54,
|
||||
"test/cso-stack-cold-integration.test.ts": 90,
|
||||
"test/cso-verifier.test.ts": 69,
|
||||
"test/cso-windows-build-contract.test.ts": 70,
|
||||
"test/declared-annotation.test.ts": 70,
|
||||
"test/dependency-security.test.ts": 167,
|
||||
"test/deps-smoke.test.ts": 150,
|
||||
"test/design-board-reload.test.ts": 328,
|
||||
"test/design-catalog.test.ts": 118,
|
||||
"test/design-checklist-sync.test.ts": 1858,
|
||||
"test/design-completion-handoff-scored.test.ts": 211,
|
||||
"test/design-consultation-contract.test.ts": 115,
|
||||
"test/design-daemon-windows-identity.test.ts": 167,
|
||||
"test/design-detect-contract.test.ts": 280,
|
||||
"test/design-doc-data-guard.test.ts": 80,
|
||||
"test/design-flag-utils.test.ts": 179,
|
||||
"test/design-html-section-completion.test.ts": 8849,
|
||||
"test/design-md.test.ts": 1644,
|
||||
"test/design-printed-paths.test.ts": 249,
|
||||
"test/design-research-fixture.test.ts": 68,
|
||||
"test/devex-finding-fixture.test.ts": 2226,
|
||||
"test/devex-peer-comparison-calibration.test.ts": 974,
|
||||
"test/diagram-render-drift.test.ts": 166,
|
||||
"test/disabled-dated-record-at.test.ts": 200,
|
||||
"test/discover-section-templates.test.ts": 67,
|
||||
"test/docsync-authority.test.ts": 4021,
|
||||
"test/docsync-child-marker.test.ts": 104,
|
||||
"test/docsync-lifecycle-interface.test.ts": 730,
|
||||
"test/document-skills-redaction.test.ts": 108,
|
||||
"test/dom-dump-hygiene.test.ts": 688,
|
||||
"test/dx-selected-navigation-ap.test.ts": 220,
|
||||
"test/e2e-harness-audit.test.ts": 108,
|
||||
"test/e2e-tier-alignment.test.ts": 393,
|
||||
"test/eng-batching-current-ledger.test.ts": 534,
|
||||
"test/eng-batching-native-replay.test.ts": 199,
|
||||
"test/eng-batching-saved-ledger.test.ts": 1316,
|
||||
"test/eng-count-question-policy.test.ts": 108,
|
||||
"test/eng-devex-s-count.test.ts": 134,
|
||||
"test/eng-finding-retry-budget.test.ts": 2220,
|
||||
"test/eng-first-review.test.ts": 550,
|
||||
"test/eng-published-navigation.test.ts": 195,
|
||||
"test/eng-resolution-block-position.test.ts": 578,
|
||||
"test/eng-review-routing.test.ts": 95,
|
||||
"test/eng-seeded-completion-ai.test.ts": 15320,
|
||||
"test/eng-seeded-coverage.test.ts": 4093,
|
||||
"test/eng-semantic-terminal.test.ts": 216,
|
||||
"test/eval-budgets-policy.test.ts": 177,
|
||||
"test/eval-flake-rank.test.ts": 1456,
|
||||
"test/eval-kinds.test.ts": 345,
|
||||
"test/eval-model.test.ts": 74,
|
||||
"test/evals-workflow-wiring.test.ts": 1679,
|
||||
"test/exit-propagation.test.ts": 643,
|
||||
"test/extension-pty-inject-invariant.test.ts": 152,
|
||||
"test/external-host-carve.test.ts": 5073,
|
||||
"test/flake-ledger.test.ts": 156,
|
||||
"test/free-home-guard.test.ts": 107,
|
||||
"test/free-seed-ratchet.test.ts": 459,
|
||||
"test/free-tests-workflow-wiring.test.ts": 170,
|
||||
"test/gbrain-detection-override.test.ts": 1989,
|
||||
"test/gbrain-exec-invariant.test.ts": 58,
|
||||
"test/gbrain-guards.test.ts": 96,
|
||||
"test/gbrain-source-gitignore.test.ts": 83,
|
||||
"test/gbrain-sources-parse.test.ts": 67,
|
||||
"test/gbrain-spawn-windows-shell.test.ts": 61,
|
||||
"test/gbrain-sync-voyage-code-3-integration.test.ts": 86,
|
||||
"test/gen-skill-docs-checks.test.ts": 27558,
|
||||
"test/gen-skill-docs-idempotency.test.ts": 4950,
|
||||
"test/gen-skill-docs-import-purity.test.ts": 257,
|
||||
"test/gen-skill-docs-prune-stale.test.ts": 1203,
|
||||
"test/generated-docs-fences.test.ts": 1349,
|
||||
"test/generator-eexist.test.ts": 667,
|
||||
"test/git-ref-fixture-tripwire.test.ts": 234,
|
||||
"test/global-discover.test.ts": 806,
|
||||
"test/gstack-ci-gate.test.ts": 77,
|
||||
"test/gstack-config-gbrain-refresh.test.ts": 1528,
|
||||
"test/gstack-design-detect-plugins.test.ts": 915,
|
||||
"test/gstack-home-module-scope.test.ts": 194,
|
||||
"test/gstack-render-cli.test.ts": 852,
|
||||
"test/gstack-schema-pack.test.ts": 59,
|
||||
"test/gstack-upgrade-migration-v1_78_0_0.test.ts": 432,
|
||||
"test/harness-version.test.ts": 318,
|
||||
"test/helpers-unit.test.ts": 110,
|
||||
"test/helpers/budget-override.test.ts": 68,
|
||||
"test/helpers/capture-parity-baseline.test.ts": 397,
|
||||
"test/helpers/claude-pty-runner.auq.unit.test.ts": 111,
|
||||
"test/helpers/claude-pty-runner.boundaries.unit.test.ts": 138,
|
||||
"test/helpers/claude-pty-runner.classify.unit.test.ts": 127,
|
||||
"test/helpers/claude-pty-runner.launch.unit.test.ts": 62,
|
||||
"test/helpers/claude-pty-runner.plan-native.unit.test.ts": 101,
|
||||
"test/helpers/claude-pty-runner.runners.unit.test.ts": 1696,
|
||||
"test/helpers/claude-pty-runner.scope-gate-floor.unit.test.ts": 105,
|
||||
"test/helpers/claude-pty-runner.screen.unit.test.ts": 109,
|
||||
"test/helpers/e2e-gate.unit.test.ts": 62,
|
||||
"test/helpers/eval-store.test.ts": 2499,
|
||||
"test/helpers/hermetic-env.test.ts": 15841,
|
||||
"test/helpers/observability.test.ts": 449,
|
||||
"test/helpers/providers/gemini.test.ts": 79,
|
||||
"test/helpers/run-bin.test.ts": 199,
|
||||
"test/heredoc-pipe-deadlock.test.ts": 629,
|
||||
"test/instruction-facts.test.ts": 145,
|
||||
"test/investigate-freeze-path.test.ts": 21734,
|
||||
"test/ios-debug-bridge-release-guard.test.ts": 80,
|
||||
"test/ios-qa-daemon-cli.test.ts": 184,
|
||||
"test/ios-qa-stateserver-hardening.test.ts": 61,
|
||||
"test/ios-qa-swift-build.test.ts": 75,
|
||||
"test/ios-qa.test.ts": 357,
|
||||
"test/is-conductor.test.ts": 57,
|
||||
"test/jargon-list.test.ts": 79,
|
||||
"test/judge-calibration.test.ts": 150,
|
||||
"test/land-and-deploy-flow.test.ts": 15166,
|
||||
"test/land-and-deploy-postfail.test.ts": 85,
|
||||
"test/llm-judge-abort.test.ts": 90,
|
||||
"test/llm-judge-frontier.test.ts": 300,
|
||||
"test/llm-judge-stream.test.ts": 115,
|
||||
"test/llms-txt-shape.test.ts": 115,
|
||||
"test/maintenance-cli-args.test.ts": 538,
|
||||
"test/memory-ingest-timeout.test.ts": 94,
|
||||
"test/migration-checkpoint-ownership.test.ts": 2143,
|
||||
"test/mktemp-portability.test.ts": 806,
|
||||
"test/model-overlays.test.ts": 95,
|
||||
"test/module-size-ratchet.test.ts": 278,
|
||||
"test/native-auto-decide-pty.test.ts": 13309,
|
||||
"test/native-auto-decide.test.ts": 789,
|
||||
"test/no-quoted-tilde-assignments.test.ts": 107,
|
||||
"test/no-suicide-exit.test.ts": 335,
|
||||
"test/office-hours-budget.test.ts": 145,
|
||||
"test/office-hours-open-question-form.test.ts": 69,
|
||||
"test/office-hours-phase4-caller.test.ts": 104,
|
||||
"test/office-hours-review.test.ts": 1134,
|
||||
"test/office-posture-recording.test.ts": 124,
|
||||
"test/one-way-doors.test.ts": 51,
|
||||
"test/open-gstack-browser-liveness.test.ts": 55,
|
||||
"test/openclaw-native-skills.test.ts": 51,
|
||||
"test/osv-config-wiring.test.ts": 52,
|
||||
"test/osv-ignore-expiry.test.ts": 61,
|
||||
"test/outside-review-result.test.ts": 610,
|
||||
"test/outside-voice-evidence.test.ts": 215,
|
||||
"test/outside-voice-failure-policy.test.ts": 328,
|
||||
"test/outside-voice-fixture.test.ts": 1705,
|
||||
"test/outside-voice-routing.test.ts": 1944,
|
||||
"test/overlay-lifecycle.test.ts": 468,
|
||||
"test/overlay-measurement.test.ts": 821,
|
||||
"test/overlay-recording-order.test.ts": 85,
|
||||
"test/overlay-sdk-cancel-eof.test.ts": 177,
|
||||
"test/paid-free-boundary.test.ts": 8707,
|
||||
"test/paid-orphan-tripwire.test.ts": 266,
|
||||
"test/paid-overlay-scheduling.test.ts": 1805,
|
||||
"test/paid-pr-profile.test.ts": 4294,
|
||||
"test/paid-report-census.test.ts": 135,
|
||||
"test/paid-report-fail-open.test.ts": 2256,
|
||||
"test/paid-retry-supervision.test.ts": 3444,
|
||||
"test/paid-run-manifest.test.ts": 5261,
|
||||
"test/paid-selection-propagation.test.ts": 142,
|
||||
"test/paid-shard-settlement.test.ts": 1775,
|
||||
"test/paid-shards.test.ts": 4911,
|
||||
"test/paid-slice-deadline.test.ts": 1645,
|
||||
"test/pair-agent-token-hygiene.test.ts": 57,
|
||||
"test/parity-baseline-integrity.test.ts": 69,
|
||||
"test/parity-sectioned.test.ts": 65,
|
||||
"test/parity-suite.test.ts": 291,
|
||||
"test/pending-question-completion.test.ts": 413,
|
||||
"test/periodic-exclude-policy.test.ts": 365,
|
||||
"test/plan-count-checkbox.test.ts": 122,
|
||||
"test/plan-count-completion.test.ts": 685,
|
||||
"test/plan-count-cropped-wrap.test.ts": 242,
|
||||
"test/plan-count-cross-cwd-ancestry.test.ts": 394,
|
||||
"test/plan-count-design-ui-recovery.test.ts": 191,
|
||||
"test/plan-count-dx-handoff.test.ts": 262,
|
||||
"test/plan-count-empty-review.test.ts": 129,
|
||||
"test/plan-count-history.test.ts": 74,
|
||||
"test/plan-count-long-edit.test.ts": 128,
|
||||
"test/plan-count-native-input.test.ts": 146,
|
||||
"test/plan-count-owned-permission.test.ts": 125,
|
||||
"test/plan-count-pending-exit.test.ts": 299,
|
||||
"test/plan-count-prerequisite.test.ts": 167,
|
||||
"test/plan-count-preview-footer.test.ts": 158,
|
||||
"test/plan-count-session-cwd.test.ts": 257,
|
||||
"test/plan-count-timeout.test.ts": 70,
|
||||
"test/plan-count-transcript.test.ts": 220,
|
||||
"test/plan-count-truncated-border.test.ts": 127,
|
||||
"test/plan-count-truncated-question.test.ts": 122,
|
||||
"test/plan-create-combined-permission.test.ts": 375,
|
||||
"test/plan-create-permission.test.ts": 229,
|
||||
"test/plan-design-floor-fixture.test.ts": 1636,
|
||||
"test/plan-design-sdk-fixture.test.ts": 3179,
|
||||
"test/plan-design-with-ui-fixture.test.ts": 7398,
|
||||
"test/plan-edit-cropped-permission.test.ts": 425,
|
||||
"test/plan-eng-resume.test.ts": 67,
|
||||
"test/plan-floor-dx-actor.test.ts": 267,
|
||||
"test/plan-floor-review.test.ts": 118,
|
||||
"test/plan-floor-target.test.ts": 14872,
|
||||
"test/plan-format-kind-note.test.ts": 77,
|
||||
"test/plan-pending-question-pty.test.ts": 137,
|
||||
"test/plan-review-board-feedback.test.ts": 12699,
|
||||
"test/plan-review-calibration.test.ts": 1199,
|
||||
"test/plan-review-decisions.test.ts": 410,
|
||||
"test/plan-review-report-recording.test.ts": 8160,
|
||||
"test/plan-scope-selection.test.ts": 3556,
|
||||
"test/plan-skill-question-hook-scope.test.ts": 9517,
|
||||
"test/plan-skill-read-permission.test.ts": 134,
|
||||
"test/plan-skill-webfetch-permission.test.ts": 4989,
|
||||
"test/platform-qualification-workflow.test.ts": 52,
|
||||
"test/post-rename-doc-regen.test.ts": 68,
|
||||
"test/pr-direct-cases.test.ts": 5659,
|
||||
"test/pr-shared-input-selection.test.ts": 1716,
|
||||
"test/pr-title-sync-workflow-safety.test.ts": 78,
|
||||
"test/prompt-audit-manifest.test.ts": 695,
|
||||
"test/provider-model-defaults.test.ts": 89,
|
||||
"test/pty-askuserquestion-single-line.test.ts": 105,
|
||||
"test/pty-numbered-option-indent-native.test.ts": 137,
|
||||
"test/pty-option-selection.test.ts": 65,
|
||||
"test/pty-output-wake.test.ts": 3152,
|
||||
"test/pty-screen-session.test.ts": 57,
|
||||
"test/pty-screen-supervision.test.ts": 10111,
|
||||
"test/pty-screen-unicode-ap.test.ts": 192,
|
||||
"test/pty-screen.test.ts": 192,
|
||||
"test/pty-skill-seeding-wiring.test.ts": 426,
|
||||
"test/pty-trust-dialog.test.ts": 54,
|
||||
"test/qa-browser-preservation.test.ts": 11468,
|
||||
"test/qa-bugs-fixture.test.ts": 73,
|
||||
"test/qa-caller-authority.test.ts": 185,
|
||||
"test/qa-caller-freshness-order.test.ts": 114,
|
||||
"test/qa-caller-report-observer.test.ts": 77,
|
||||
"test/qa-checkpoint-evidence.test.ts": 258,
|
||||
"test/qa-deadline-publication-observer.test.ts": 80,
|
||||
"test/qa-deadline-selection.test.ts": 91,
|
||||
"test/qa-deadline.test.ts": 23157,
|
||||
"test/qa-evidence-selection.test.ts": 193,
|
||||
"test/qa-evidence.test.ts": 17401,
|
||||
"test/qa-fix-loop-fixture.test.ts": 5504,
|
||||
"test/qa-health-rubric.test.ts": 42,
|
||||
"test/qa-only-browser-probe.test.ts": 1441,
|
||||
"test/qa-probe-gates.test.ts": 119,
|
||||
"test/qa-supervision-selection.test.ts": 75,
|
||||
"test/question-tuning-registry-path.test.ts": 48,
|
||||
"test/redact-context-exemptions.test.ts": 96,
|
||||
"test/redact-doc-resolver.test.ts": 42,
|
||||
"test/redact-dotenv-filename-false-positive.test.ts": 46,
|
||||
"test/redact-engine-autoredact.test.ts": 44,
|
||||
"test/redact-engine.test.ts": 77,
|
||||
"test/redact-parcel-id-false-positive.test.ts": 46,
|
||||
"test/redact-pattern-lint.test.ts": 56,
|
||||
"test/redact-prepush-target.test.ts": 46225,
|
||||
"test/redact-span-binding.test.ts": 107,
|
||||
"test/redact-unicode-offsets.test.ts": 187,
|
||||
"test/regression-1539-review-self-verify.test.ts": 59,
|
||||
"test/regression-issue2091-bsd-mktemp.test.ts": 603,
|
||||
"test/regression-pr1169-mktemp-fallbacks.test.ts": 166,
|
||||
"test/regression-transcript-frontmatter-fence.test.ts": 103,
|
||||
"test/remote-identity-vectors.test.ts": 223,
|
||||
"test/replay-pr-selection.test.ts": 1498,
|
||||
"test/resolvers-gbrain-put-rewrite.test.ts": 85,
|
||||
"test/resolvers-gbrain-save-results.test.ts": 48,
|
||||
"test/retired-commands.test.ts": 100,
|
||||
"test/review-army-budget.test.ts": 2958,
|
||||
"test/review-consensus-lifecycle.test.ts": 72,
|
||||
"test/review-count-markdown.test.ts": 180,
|
||||
"test/review-entry-and-design-clarity-au.test.ts": 130,
|
||||
"test/review-enum-lifecycle.test.ts": 96,
|
||||
"test/review-finalization-budget.test.ts": 5927,
|
||||
"test/review-n-plus-one-contract.test.ts": 3174,
|
||||
"test/review-quality-provenance.test.ts": 77,
|
||||
"test/review-workflow-clarity.test.ts": 93,
|
||||
"test/router-disabled-skills.test.ts": 1302,
|
||||
"test/run-shard-child.test.ts": 1404,
|
||||
"test/safety-rule-registry.test.ts": 90,
|
||||
"test/salience-allowlist.test.ts": 82,
|
||||
"test/schema-version-migration.test.ts": 346,
|
||||
"test/section-manifest-consistency.test.ts": 74,
|
||||
"test/session-runner-browse-errors.test.ts": 1939,
|
||||
"test/session-runner-stream-lifecycle.test.ts": 378,
|
||||
"test/session-runner-timeout.test.ts": 76,
|
||||
"test/session-runner-tools.test.ts": 243,
|
||||
"test/setup-alias-name-uniqueness.test.ts": 20718,
|
||||
"test/setup-browser-hint.test.ts": 608,
|
||||
"test/setup-bun-cmd-and-pipe-bugs.test.ts": 361,
|
||||
"test/setup-cleanup-orphans.test.ts": 81,
|
||||
"test/setup-codex-design-pdf-runtime.test.ts": 79,
|
||||
"test/setup-codex-model.test.ts": 79,
|
||||
"test/setup-codex-scope-review-boundaries.test.ts": 157,
|
||||
"test/setup-cso-windows-probe.test.ts": 79,
|
||||
"test/setup-deploy-key-presence.test.ts": 139,
|
||||
"test/setup-gbrain-bin-invocation-paths.test.ts": 72,
|
||||
"test/setup-gbrain-path4-caller.test.ts": 5288,
|
||||
"test/setup-gbrain-path4-structure.test.ts": 67,
|
||||
"test/setup-gbrain-remote-caller.test.ts": 10158,
|
||||
"test/setup-help.test.ts": 451,
|
||||
"test/setup-playwright-platform.test.ts": 7531,
|
||||
"test/setup-prune-stale-generated.test.ts": 2139,
|
||||
"test/setup-sections-linking.test.ts": 57,
|
||||
"test/setup-skill-copies.test.ts": 77,
|
||||
"test/setup-staged-copy.test.ts": 377,
|
||||
"test/setup-windows-fallback.test.ts": 69,
|
||||
"test/setup-windows-rerun-refresh.test.ts": 1511,
|
||||
"test/shard-cli-parity.test.ts": 314,
|
||||
"test/shared-libs-cancellation.test.ts": 328,
|
||||
"test/shared-libs-evidence.test.ts": 91,
|
||||
"test/shared-libs-plan-actor.test.ts": 208,
|
||||
"test/shared-libs-rendering.test.ts": 2639,
|
||||
"test/shared-libs-source-reads.test.ts": 12864,
|
||||
"test/ship-apple-gate.test.ts": 76,
|
||||
"test/ship-control-flow.test.ts": 3719,
|
||||
"test/ship-coverage-audit-af.test.ts": 153,
|
||||
"test/ship-hook-windows-paths.test.ts": 2096,
|
||||
"test/ship-pr-liveness-policy.test.ts": 72,
|
||||
"test/ship-publication-gates.test.ts": 1628,
|
||||
"test/ship-reentry-gates.test.ts": 78,
|
||||
"test/ship-review-loop.test.ts": 79,
|
||||
"test/ship-section-fixture.test.ts": 1645,
|
||||
"test/ship-skip-requeue.test.ts": 129,
|
||||
"test/ship-template-redaction.test.ts": 76,
|
||||
"test/ship-test-detection-markers.test.ts": 1628,
|
||||
"test/ship-triage-labels.test.ts": 69,
|
||||
"test/ship-workflow-clarity.test.ts": 126,
|
||||
"test/shortcut-debt-ledger.test.ts": 69,
|
||||
"test/skill-browse-state-extraction.test.ts": 278,
|
||||
"test/skill-budget-regression.test.ts": 329,
|
||||
"test/skill-check-driver.test.ts": 397,
|
||||
"test/skill-collision-sentinel.test.ts": 78,
|
||||
"test/skill-coverage-floor.test.ts": 151,
|
||||
"test/skill-cross-model-recommendation-emit.test.ts": 77,
|
||||
"test/skill-parser.test.ts": 99,
|
||||
"test/skill-positional-literals.test.ts": 7526,
|
||||
"test/skill-preflight-budget.test.ts": 67,
|
||||
"test/skillify-host-paths.test.ts": 2351,
|
||||
"test/sol-skill-fixture.test.ts": 3379,
|
||||
"test/spawnsync-timeout-tripwire.test.ts": 549,
|
||||
"test/spec-template-invariants.test.ts": 112,
|
||||
"test/state-root-parity.test.ts": 2452,
|
||||
"test/strict-output-capture.test.ts": 225,
|
||||
"test/strict-output-formats.test.ts": 2977,
|
||||
"test/strict-output-settlement.test.ts": 4288,
|
||||
"test/strict-output.test.ts": 63,
|
||||
"test/sync-gbrain-readiness-fixture.test.ts": 597,
|
||||
"test/takes-fence-fallback.test.ts": 48,
|
||||
"test/tasks-section-jq.test.ts": 236,
|
||||
"test/template-context-parity.test.ts": 57,
|
||||
"test/terse-build.test.ts": 62,
|
||||
"test/test-free-shards-sandbox-knobs.test.ts": 428,
|
||||
"test/test-free-shards-settlement-lifecycle.test.ts": 80,
|
||||
"test/test-free-shards-settlement-ownership.test.ts": 75,
|
||||
"test/test-health-report.test.ts": 643,
|
||||
"test/test-health-workflow.test.ts": 52,
|
||||
"test/test-of-test-ratchet.test.ts": 405,
|
||||
"test/test-pr-profile.test.ts": 90,
|
||||
"test/test-value-bar.test.ts": 428,
|
||||
"test/third-party-actions-recording.test.ts": 542,
|
||||
"test/third-party-actions.test.ts": 478,
|
||||
"test/touchfiles-facade.test.ts": 99,
|
||||
"test/touchfiles-map-diff.test.ts": 1770,
|
||||
"test/tracker-guard-wiring.test.ts": 142,
|
||||
"test/troubleshooting-anchors.test.ts": 53,
|
||||
"test/typecheck-test-ratchet.test.ts": 98,
|
||||
"test/uninstall-windows-copies.test.ts": 9656,
|
||||
"test/v0-dormancy.test.ts": 85,
|
||||
"test/version-source.test.ts": 75,
|
||||
"test/windows-free-selection-autoplan-guard.test.ts": 80,
|
||||
"test/windows-native-workflows.test.ts": 534,
|
||||
"test/workflow-action-pins.test.ts": 88,
|
||||
"test/workflow-boundaries-fixture.test.ts": 15849,
|
||||
"test/workflow-concurrency.test.ts": 81,
|
||||
"test/workflow-excerpt.test.ts": 307,
|
||||
"test/workflow-judge-cache.test.ts": 13496,
|
||||
"test/worktree-isolated-shapes.test.ts": 122,
|
||||
"test/worktree.test.ts": 5426
|
||||
}
|
||||
}
|
||||
+1025
-986
File diff suppressed because it is too large.
Load diff
@@ -0,0 +1,6 @@
|
||||
{
|
||||
"entries": [
|
||||
{"file": "test/host-conformance.test.ts", "reason": "real ./setup per host and layout; the time is setup's own skill-linking loop, not fixture copies (shared fixture landed in W6c)"},
|
||||
{"file": "test/shared-libs-stage-actor.test.ts", "reason": "re-runs the paid shared-libs runner body against git fixture repos per test; owned by the paid-harness value-bar review"}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,53 @@
|
||||
{
|
||||
"version": 1,
|
||||
"patterns": [
|
||||
"test/helpers/agent-sdk-runner.ts",
|
||||
"test/helpers/codex-session-runner.ts",
|
||||
"test/helpers/claude-pty-runner.ts",
|
||||
"test/helpers/pty/**",
|
||||
"test/helpers/pty-screen.ts",
|
||||
"test/helpers/e2e-gate.ts",
|
||||
"test/helpers/llm-judge.ts"
|
||||
],
|
||||
"files": {
|
||||
"scripts/lib/paid-cases.ts": {"blob":"9bacd52446192a7ecf3dfaae901ea443cf9c9ae2","since":1,"nonBehavioral":"reliability follow-ups integration before the first v2 census: session ledger, trial cause/cost fields and stream liveness are diagnostics; EVAL_POLICY v2 already starts fresh series"},
|
||||
"scripts/lib/paid-plan.ts": {"blob":"45721ce128a112666a0caed6c13c66637b74ca54","since":1,"nonBehavioral":"audit-wave integration before the first v2 census (EVAL_POLICY v2 already starts fresh series)"},
|
||||
"scripts/lib/paid-report.ts": {"blob":"866a9a2bb41e9152989fca23b64e138ba61bbfee","since":1,"nonBehavioral":"reliability follow-ups integration before the first v2 census: session ledger, trial cause/cost fields and stream liveness are diagnostics; EVAL_POLICY v2 already starts fresh series"},
|
||||
"scripts/lib/published-text.ts": {"blob":"3f2e4d616b5dd0204035710c66b64d49055c5714","since":1,"nonBehavioral":"reliability follow-ups integration before the first v2 census: session ledger, trial cause/cost fields and stream liveness are diagnostics; EVAL_POLICY v2 already starts fresh series"},
|
||||
"scripts/lib/shard-engine.ts": {"blob":"3cd0b1aaa84018b00bf9ea0ce48093f3c8492f66","since":1,"nonBehavioral":"audit-wave integration before the first v2 census (EVAL_POLICY v2 already starts fresh series)"},
|
||||
"scripts/test-paid-shards.ts": {"blob":"f335b16112f0de585320d1c7de03ce22b56f0e65","since":1,"nonBehavioral":"reliability follow-ups integration before the first v2 census: session ledger, trial cause/cost fields and stream liveness are diagnostics; EVAL_POLICY v2 already starts fresh series"},
|
||||
"scripts/test-pr-profile.ts": {"blob":"d7860897397de8fd605623570d1e7ebbbd5ec743","since":1,"nonBehavioral":"audit-wave integration before the first v2 census (EVAL_POLICY v2 already starts fresh series)"},
|
||||
"scripts/test-strict-output.ts": {"blob":"01185d20dc9761b04b9b2f751ec98664fe0eaf93","since":1},
|
||||
"test/helpers/agent-sdk-runner.ts": {"blob":"210bba92d241a90df578b2c7fc943ac5755d0025","since":1,"nonBehavioral":"reliability follow-ups integration before the first v2 census: session ledger, trial cause/cost fields and stream liveness are diagnostics; EVAL_POLICY v2 already starts fresh series"},
|
||||
"test/helpers/auq-substance-panel.ts": {"blob":"85d785f25fe6a24277d4009f9ccb7b1e05a270ef","since":1,"nonBehavioral":"reliability follow-ups integration before the first v2 census: session ledger, trial cause/cost fields and stream liveness are diagnostics; EVAL_POLICY v2 already starts fresh series"},
|
||||
"test/helpers/claude-pty-runner.ts": {"blob":"20cc5894532873aa6145fc6617fdeec8cef94a2d","since":1},
|
||||
"test/helpers/codex-session-runner.ts": {"blob":"4eea35931a35a0afb006363402e3d4191e379737","since":1},
|
||||
"test/helpers/e2e-gate.ts": {"blob":"efb7e197dd5f66c3ad0e34074b14e640754f7a45","since":1,"nonBehavioral":"audit-wave integration before the first v2 census (EVAL_POLICY v2 already starts fresh series)"},
|
||||
"test/helpers/e2e-helpers.ts": {"blob":"2d4a2a53063cfc1f003db366d2c861ab1e379e6e","since":1},
|
||||
"test/helpers/eval-budgets.ts": {"blob":"667d0e09b20fe5d7366f57b77b5fb9b04baa37ef","since":1,"nonBehavioral":"reliability follow-ups integration before the first v2 census: session ledger, trial cause/cost fields and stream liveness are diagnostics; EVAL_POLICY v2 already starts fresh series"},
|
||||
"test/helpers/eval-store.ts": {"blob":"bc135ddcf13a45c9803995df2af1aff54a471e2d","since":1,"nonBehavioral":"reliability follow-ups integration before the first v2 census: session ledger, trial cause/cost fields and stream liveness are diagnostics; EVAL_POLICY v2 already starts fresh series"},
|
||||
"test/helpers/hermetic-env.ts": {"blob":"6473f78285384f4420f48cbe5928658919078e0b","since":1},
|
||||
"test/helpers/llm-judge.ts": {"blob":"453a4c1d355f3811deca53f953767109a9ec9292","since":1},
|
||||
"test/helpers/paid-test-set.ts": {"blob":"ff9bfe43c43ec2bdd11114f7b8ec7c5736116f75","since":1,"nonBehavioral":"audit-wave integration before the first v2 census (EVAL_POLICY v2 already starts fresh series)"},
|
||||
"test/helpers/pty-screen.ts": {"blob":"4f77569f9ce11f9cee9f347216e468cd7c956820","since":1},
|
||||
"test/helpers/pty/auq.ts": {"blob":"16e5ea2bb5b9f28f44808a1a09024dbd3e6f28c4","since":1},
|
||||
"test/helpers/pty/binary.ts": {"blob":"404d7515076d5c00804f89e23ea69296312c00e1","since":1},
|
||||
"test/helpers/pty/boundaries.ts": {"blob":"53eb76e57c0558c00254366d6334a1c833126575","since":1},
|
||||
"test/helpers/pty/classify.ts": {"blob":"3f2daa12ff6484f96b8d28d910a50231c4bb3ae8","since":1},
|
||||
"test/helpers/pty/fake-session.ts": {"blob":"554e064ef259b8444280eddfe9b0db93c66a8202","since":1},
|
||||
"test/helpers/pty/judge.ts": {"blob":"81c5bb4c48ab928aca2f186a1ade9df7169d7fbe","since":1},
|
||||
"test/helpers/pty/launch.ts": {"blob":"4ef0bcb7b5e7c35d2aac0cbffd7f8a303440b6fe","since":1,"nonBehavioral":"reliability follow-ups integration before the first v2 census: session ledger, trial cause/cost fields and stream liveness are diagnostics; EVAL_POLICY v2 already starts fresh series"},
|
||||
"test/helpers/pty/plan-native.ts": {"blob":"ddba9cb68c7c0da935cff7c1cea258dee2dff740","since":1},
|
||||
"test/helpers/pty/runners/counting.ts": {"blob":"8c5c05e54abf961259d34d9c3d77db67138d7794","since":1,"nonBehavioral":"reliability follow-ups integration before the first v2 census: session ledger, trial cause/cost fields and stream liveness are diagnostics; EVAL_POLICY v2 already starts fresh series"},
|
||||
"test/helpers/pty/runners/floor.ts": {"blob":"465f01641fb3a18ae1c9d2613a9c2b26f8fb503e","since":1,"nonBehavioral":"reliability follow-ups integration before the first v2 census: session ledger, trial cause/cost fields and stream liveness are diagnostics; EVAL_POLICY v2 already starts fresh series"},
|
||||
"test/helpers/pty/runners/observation.ts": {"blob":"1c2b75fbb984708725abaa8939125f7cab969f9f","since":1,"nonBehavioral":"reliability follow-ups integration before the first v2 census: session ledger, trial cause/cost fields and stream liveness are diagnostics; EVAL_POLICY v2 already starts fresh series"},
|
||||
"test/helpers/pty/screen.ts": {"blob":"ee005f36ffa48617e6832ccbeb35e8e96873df9b","since":1,"nonBehavioral":"reliability follow-ups integration before the first v2 census: session ledger, trial cause/cost fields and stream liveness are diagnostics; EVAL_POLICY v2 already starts fresh series"},
|
||||
"test/helpers/pty/session.ts": {"blob":"58533cea9d81cc87bb3b499bdfe0db8e3a8709c5","since":1,"nonBehavioral":"reliability follow-ups integration before the first v2 census: session ledger, trial cause/cost fields and stream liveness are diagnostics; EVAL_POLICY v2 already starts fresh series"},
|
||||
"test/helpers/session-drain-policy.ts": {"blob":"6211239dd0485e6495a700b824a7e0e61d1bcd97","since":1},
|
||||
"test/helpers/session-ledger.ts": {"blob":"6e3c9e837e46f98addba2d12eaebff802da320bb","since":1,"nonBehavioral":"reliability follow-ups integration before the first v2 census: session ledger, trial cause/cost fields and stream liveness are diagnostics; EVAL_POLICY v2 already starts fresh series"},
|
||||
"test/helpers/session-runner.ts": {"blob":"d40eb29e8b8fdc0675b511693e4a90823b5963fc","since":1,"nonBehavioral":"reliability follow-ups integration before the first v2 census: session ledger, trial cause/cost fields and stream liveness are diagnostics; EVAL_POLICY v2 already starts fresh series"},
|
||||
"test/helpers/skill-fixture.ts": {"blob":"b188064594eb4201a54a80ba1bc6f28e31d7f49e","since":1},
|
||||
"test/helpers/test-selection.ts": {"blob":"51655f89f66be7a5ec487e2abfe4744300ebd5f6","since":1},
|
||||
"test/helpers/touchfiles.ts": {"blob":"15f369056d41b9dc42dcd86f81f60c878074704c","since":1}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,327 @@
|
||||
/**
|
||||
* The one reader of GitHub Actions history: run lists, run jobs, artifact
|
||||
* downloads and the free lane's CI flake ledgers. `scripts/eval-flake-rank.ts`
|
||||
* (pass-rates) and `scripts/test-health-report.ts` (test:health) both read CI
|
||||
* history through this module; every call goes through an injectable client so
|
||||
* the free tests never touch the network.
|
||||
*
|
||||
* Downloads are data only: artifacts are size-capped, name-checked and parsed,
|
||||
* never executed. A failed call throws; callers that report metrics turn the
|
||||
* error into a named "unavailable: <reason>" with `unavailableReason`, never 0.
|
||||
*/
|
||||
import * as fs from 'node:fs';
|
||||
import * as path from 'node:path';
|
||||
import { spawnSync } from 'node:child_process';
|
||||
import type { FlakeLedgerEntry } from '../test-free-shards';
|
||||
|
||||
// --- Clients ---
|
||||
|
||||
export interface WeeklyRun { id: number; attempt: number; sha: string; branch: string; createdAt: string; event?: string }
|
||||
export interface RunArtifact { id: number; name: string; size: number }
|
||||
|
||||
/** The GitHub calls pass-rates makes; injectable so the free tests never touch the network. */
|
||||
export interface HistoryFetcher {
|
||||
listRuns(repo: string, workflow: string, branch: string, limit: number): WeeklyRun[];
|
||||
listArtifacts(repo: string, runId: number): RunArtifact[];
|
||||
downloadZip(repo: string, artifactId: number, destination: string): void;
|
||||
}
|
||||
|
||||
/** One GitHub REST page as parsed JSON, plus artifact zip downloads. */
|
||||
export interface GhClient {
|
||||
getJson(apiPath: string): unknown;
|
||||
downloadZip(repo: string, artifactId: number, destination: string): void;
|
||||
}
|
||||
|
||||
/** Per-file cap for downloaded artifacts: CI history is parsed as data only, never executed. */
|
||||
export const TRIAL_OUTCOMES_MAX_BYTES = 8 * 1024 * 1024;
|
||||
|
||||
function gh(args: string[]): Buffer {
|
||||
const result = spawnSync('gh', args, { timeout: 300_000, maxBuffer: 256 * 1024 * 1024 });
|
||||
if (result.status !== 0) throw new Error(`gh ${args.slice(0, 2).join(' ')} failed: ${String(result.stderr || result.error || '').trim()}`);
|
||||
return result.stdout;
|
||||
}
|
||||
|
||||
function jsonLines<T>(buffer: Buffer): T[] {
|
||||
return buffer.toString('utf8').split('\n').filter(Boolean).map(line => JSON.parse(line) as T);
|
||||
}
|
||||
|
||||
export const GH_CLIENT: GhClient = {
|
||||
getJson: (apiPath) => JSON.parse(gh(['api', apiPath]).toString('utf8')),
|
||||
downloadZip: (repo, artifactId, destination) => fs.writeFileSync(destination, gh(['api', `repos/${repo}/actions/artifacts/${artifactId}/zip`])),
|
||||
};
|
||||
|
||||
export const GH_HISTORY: HistoryFetcher = {
|
||||
listRuns: (repo, workflow, branch, limit): WeeklyRun[] => jsonLines(gh(['api',
|
||||
`repos/${repo}/actions/workflows/${workflow}/runs?${branch ? `branch=${encodeURIComponent(branch)}&` : ''}status=completed&per_page=${limit}`,
|
||||
'--jq', '.workflow_runs[] | {id, attempt: .run_attempt, sha: .head_sha, branch: .head_branch, createdAt: .created_at, event}'])),
|
||||
listArtifacts: (repo, runId): RunArtifact[] => jsonLines(gh(['api', `repos/${repo}/actions/runs/${runId}/artifacts?per_page=100`,
|
||||
'--paginate', '--jq', '.artifacts[] | select(.expired | not) | {id, name, size: .size_in_bytes}'])),
|
||||
downloadZip: GH_CLIENT.downloadZip,
|
||||
};
|
||||
|
||||
/**
|
||||
* A named reason plus the next step for a failed GitHub read (DX-6): rate
|
||||
* limits, server errors, auth and a missing CLI each say what to do next.
|
||||
*/
|
||||
export function unavailableReason(error: unknown): string {
|
||||
const message = (error instanceof Error ? error.message : String(error)).replace(/\s+/g, ' ').trim();
|
||||
if (/rate limit|secondary rate|abuse detection/i.test(message)) {
|
||||
return `GitHub API rate limit (${message.slice(0, 160)}); next: \`gh api rate_limit --jq .resources.core.reset\` gives the reset time, rerun after it`;
|
||||
}
|
||||
if (/HTTP 5\d\d|\b50[0234]\b|bad gateway|service unavailable|gateway time-?out/i.test(message)) {
|
||||
return `GitHub API server error (${message.slice(0, 160)}); next: rerun later, the history is unchanged`;
|
||||
}
|
||||
if (/ENOENT|not found: gh|command not found/i.test(message)) {
|
||||
return 'gh CLI not installed; next: install it (https://cli.github.com) and run `gh auth login`';
|
||||
}
|
||||
if (/HTTP 401|HTTP 403|auth|not logged|credentials/i.test(message)) {
|
||||
return `gh cannot read this repository (${message.slice(0, 160)}); next: \`gh auth status\` and \`gh auth login\` with repo read access`;
|
||||
}
|
||||
if (/HTTP 404|not found/i.test(message)) {
|
||||
return `not found (${message.slice(0, 160)}); next: check --repo and that the workflow or artifact still exists (artifacts expire)`;
|
||||
}
|
||||
return `${message.slice(0, 200) || 'unknown error'}; next: fix the input named above, then rerun`;
|
||||
}
|
||||
|
||||
// --- Runs, jobs and artifacts ---
|
||||
|
||||
export interface CiRun {
|
||||
id: number; attempt: number; sha: string; branch: string; event: string;
|
||||
status: string; conclusion: string | null; createdAt: string; updatedAt: string; startedAt: string;
|
||||
prNumbers: number[];
|
||||
}
|
||||
|
||||
export interface CiJob { id: number; name: string; conclusion: string | null; startedAt: string | null; completedAt: string | null }
|
||||
|
||||
const PAGE = 100;
|
||||
|
||||
/**
|
||||
* Every completed run of `workflow` created at or after `since`, newest first,
|
||||
* paging until a page is short or older than `since`. A failure on any page
|
||||
* throws: a partial history is never reported as a complete one.
|
||||
*/
|
||||
export function listWorkflowRuns(client: GhClient, repo: string, workflow: string,
|
||||
opts: { since: string; branch?: string; event?: string; maxPages?: number }): CiRun[] {
|
||||
const out: CiRun[] = [];
|
||||
const query = [`status=completed`, `per_page=${PAGE}`, `created=${encodeURIComponent(`>=${opts.since.slice(0, 10)}`)}`,
|
||||
...(opts.branch ? [`branch=${encodeURIComponent(opts.branch)}`] : []), ...(opts.event ? [`event=${encodeURIComponent(opts.event)}`] : [])];
|
||||
for (let page = 1; page <= (opts.maxPages ?? 20); page++) {
|
||||
const body = client.getJson(`repos/${repo}/actions/workflows/${workflow}/runs?${query.join('&')}&page=${page}`) as { workflow_runs?: any[] };
|
||||
const runs = Array.isArray(body?.workflow_runs) ? body.workflow_runs : [];
|
||||
for (const run of runs) {
|
||||
if (String(run.created_at) < opts.since) continue;
|
||||
if ((opts.branch && run.head_branch !== opts.branch) || (opts.event && run.event !== opts.event)) continue;
|
||||
out.push({ id: Number(run.id), attempt: Number(run.run_attempt ?? 1), sha: String(run.head_sha ?? ''), branch: String(run.head_branch ?? ''),
|
||||
event: String(run.event ?? ''), status: String(run.status ?? ''), conclusion: run.conclusion ?? null,
|
||||
createdAt: String(run.created_at ?? ''), updatedAt: String(run.updated_at ?? ''), startedAt: String(run.run_started_at ?? run.created_at ?? ''),
|
||||
prNumbers: Array.isArray(run.pull_requests) ? run.pull_requests.map((pr: any) => Number(pr.number)).filter(Number.isFinite) : [] });
|
||||
}
|
||||
if (runs.length < PAGE) break;
|
||||
}
|
||||
return out.sort((a, b) => b.createdAt.localeCompare(a.createdAt));
|
||||
}
|
||||
|
||||
/** Every job of one run attempt (all pages). */
|
||||
export function listRunJobs(client: GhClient, repo: string, runId: number, attempt = 1): CiJob[] {
|
||||
const out: CiJob[] = [];
|
||||
for (let page = 1; page <= 10; page++) {
|
||||
const body = client.getJson(`repos/${repo}/actions/runs/${runId}/attempts/${attempt}/jobs?per_page=${PAGE}&page=${page}`) as { jobs?: any[] };
|
||||
const jobs = Array.isArray(body?.jobs) ? body.jobs : [];
|
||||
out.push(...jobs.map(job => ({ id: Number(job.id), name: String(job.name ?? ''), conclusion: job.conclusion ?? null,
|
||||
startedAt: job.started_at ?? null, completedAt: job.completed_at ?? null })));
|
||||
if (jobs.length < PAGE) break;
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
/** Unexpired artifacts of one run (all pages). */
|
||||
export function listRunArtifacts(client: GhClient, repo: string, runId: number): RunArtifact[] {
|
||||
const out: RunArtifact[] = [];
|
||||
for (let page = 1; page <= 10; page++) {
|
||||
const body = client.getJson(`repos/${repo}/actions/runs/${runId}/artifacts?per_page=${PAGE}&page=${page}`) as { artifacts?: any[] };
|
||||
const artifacts = Array.isArray(body?.artifacts) ? body.artifacts : [];
|
||||
out.push(...artifacts.filter(artifact => !artifact.expired)
|
||||
.map(artifact => ({ id: Number(artifact.id), name: String(artifact.name ?? ''), size: Number(artifact.size_in_bytes ?? 0) })));
|
||||
if (artifacts.length < PAGE) break;
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
/** A HistoryFetcher over a GhClient (artifact listings memoized per run), so downloads share one cache and one parser. */
|
||||
export function historyFetcher(client: GhClient): HistoryFetcher {
|
||||
const artifacts = new Map<string, RunArtifact[]>();
|
||||
return {
|
||||
listRuns: (repo, workflow, branch, limit) => listWorkflowRuns(client, repo, workflow, { since: '1970-01-01', branch, maxPages: 1 })
|
||||
.slice(0, limit).map(run => ({ id: run.id, attempt: run.attempt, sha: run.sha, branch: run.branch, createdAt: run.createdAt, event: run.event })),
|
||||
listArtifacts: (repo, runId) => {
|
||||
const key = `${repo}#${runId}`;
|
||||
if (!artifacts.has(key)) artifacts.set(key, listRunArtifacts(client, repo, runId));
|
||||
return artifacts.get(key)!;
|
||||
},
|
||||
downloadZip: (repo, artifactId, destination) => client.downloadZip(repo, artifactId, destination),
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* EVAL_POLICY v2 weekly history (W4b): scheduled runs on main plus main
|
||||
* dispatches, never a branch dispatch, so back-to-back validation runs on a
|
||||
* branch neither feed pass rates nor count as weeks for quarantine expiry.
|
||||
*/
|
||||
export function isWeeklyHistoryRun(run: Pick<WeeklyRun, 'branch' | 'event'>): boolean {
|
||||
return run.branch === 'main' && (run.event === 'schedule' || run.event === 'workflow_dispatch');
|
||||
}
|
||||
|
||||
/**
|
||||
* Branch census runs whose trials may pool into a case's main series (D1
|
||||
* option b, approved 2026-10-05): any completed non-main run of the census
|
||||
* workflow. A pooled trial counts only toward a series main has also run
|
||||
* (same case-owned bytes, HARNESS_VERSION, model and CLI); weeks for
|
||||
* quarantine expiry still come from isWeeklyHistoryRun alone.
|
||||
*/
|
||||
export function isPooledTrialRun(run: Pick<WeeklyRun, 'branch'>): boolean {
|
||||
return run.branch !== '' && run.branch !== 'main';
|
||||
}
|
||||
|
||||
/** The last `limit` completed runs of `workflow` on each branch, newest first, deduplicated. */
|
||||
export function listWeeklyRuns(opts: { repo: string; workflow: string; branches: string[]; limit: number; fetcher?: HistoryFetcher }): WeeklyRun[] {
|
||||
const fetcher = opts.fetcher ?? GH_HISTORY;
|
||||
const runs = new Map<number, WeeklyRun>();
|
||||
for (const branch of opts.branches) for (const run of fetcher.listRuns(opts.repo, opts.workflow, branch, opts.limit)) runs.set(run.id, run);
|
||||
return [...runs.values()].sort((a, b) => b.createdAt.localeCompare(a.createdAt));
|
||||
}
|
||||
|
||||
/**
|
||||
* Download the artifacts of one run whose names match into a per-run cache
|
||||
* directory (reused on later calls) and return the extracted directories.
|
||||
* Oversized or oddly named artifacts are skipped: downloads are data only.
|
||||
*/
|
||||
export function downloadRunArtifacts(opts: { repo: string; run: Pick<WeeklyRun, 'id'>; match: (name: string) => boolean; cacheDir: string;
|
||||
fetcher?: HistoryFetcher; maxBytes?: number; artifacts?: RunArtifact[] }): string[] {
|
||||
const fetcher = opts.fetcher ?? GH_HISTORY;
|
||||
const dirs: string[] = [];
|
||||
for (const artifact of opts.artifacts ?? fetcher.listArtifacts(opts.repo, opts.run.id)) {
|
||||
if (!opts.match(artifact.name) || !/^[A-Za-z0-9._-]+$/.test(artifact.name)) continue;
|
||||
if (artifact.size > (opts.maxBytes ?? TRIAL_OUTCOMES_MAX_BYTES)) continue;
|
||||
const dir = path.join(opts.cacheDir, `${opts.run.id}`, artifact.name);
|
||||
if (!fs.existsSync(path.join(dir, '.complete'))) {
|
||||
fs.rmSync(dir, { recursive: true, force: true });
|
||||
fs.mkdirSync(dir, { recursive: true });
|
||||
const zip = path.join(dir, 'artifact.zip');
|
||||
fetcher.downloadZip(opts.repo, artifact.id, zip);
|
||||
const unzip = spawnSync('unzip', ['-o', '-q', zip, '-d', dir], { timeout: 120_000 });
|
||||
if (unzip.status !== 0) throw new Error(`unzip failed for ${artifact.name}: ${String(unzip.stderr || unzip.error || '')}`);
|
||||
fs.rmSync(zip, { force: true });
|
||||
fs.writeFileSync(path.join(dir, '.complete'), '');
|
||||
}
|
||||
dirs.push(dir);
|
||||
}
|
||||
return dirs;
|
||||
}
|
||||
|
||||
/** Every regular file under `dir` whose basename matches, recursively. */
|
||||
export function filesUnder(dir: string, match: (base: string) => boolean): string[] {
|
||||
if (!fs.existsSync(dir)) return [];
|
||||
return (fs.readdirSync(dir, { recursive: true }) as string[])
|
||||
.filter(name => match(path.basename(name)))
|
||||
.map(name => path.join(dir, name))
|
||||
.filter(full => fs.statSync(full).isFile());
|
||||
}
|
||||
|
||||
// --- Free-lane flake ledgers (W7a) ---
|
||||
|
||||
/** Per-line parse: one torn or hand-edited JSONL line drops that line, never the file. */
|
||||
export function parseFlakeLedger(text: string): FlakeLedgerEntry[] {
|
||||
const out: FlakeLedgerEntry[] = [];
|
||||
for (const line of text.split('\n')) {
|
||||
if (!line.trim()) continue;
|
||||
try {
|
||||
const entry = JSON.parse(line);
|
||||
if (entry && entry.kind === 'flaky-pass' && typeof entry.file === 'string') out.push(entry);
|
||||
} catch { /* torn line: skip */ }
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
/** Every flaky-pass entry in the `*.jsonl` files under a directory of downloaded `flake-ledger-*` artifacts. */
|
||||
export function readFlakeLedgerDir(dir: string): FlakeLedgerEntry[] {
|
||||
return filesUnder(dir, base => base.endsWith('.jsonl')).flatMap(file => parseFlakeLedger(fs.readFileSync(file, 'utf8')));
|
||||
}
|
||||
|
||||
/** The pre-registered flaky-file rule (CEO-15): > 5% of main free-tests runs over a window of >= 20 runs. */
|
||||
export const FLAKE_POLICY = { maxRate: 0.05, minRuns: 20 } as const;
|
||||
|
||||
export interface FlakyFile { file: string; flakyRuns: number; entries: number; rate: number; enforced: boolean; failing: boolean }
|
||||
export interface FlakeWindow { runs: number; flakyRuns: number; since: string; files: FlakyFile[]; enforceable: boolean }
|
||||
|
||||
/**
|
||||
* Rank files by the share of runs in which they needed the retry pass. A file
|
||||
* fails only when the window holds at least `minRuns` runs and its rate is
|
||||
* strictly above `maxRate`; a thinner window is report-only.
|
||||
*/
|
||||
export function rankFlakyFiles(runEntries: Map<number, FlakeLedgerEntry[]>, runCount: number, since: string,
|
||||
policy: { maxRate: number; minRuns: number } = FLAKE_POLICY): FlakeWindow {
|
||||
const perFile = new Map<string, { runs: Set<number>; entries: number }>();
|
||||
for (const [runId, entries] of runEntries) for (const entry of entries) {
|
||||
const slot = perFile.get(entry.file) ?? { runs: new Set<number>(), entries: 0 };
|
||||
slot.runs.add(runId);
|
||||
slot.entries += 1;
|
||||
perFile.set(entry.file, slot);
|
||||
}
|
||||
const enforceable = runCount >= policy.minRuns;
|
||||
const files = [...perFile].map(([file, slot]) => {
|
||||
const rate = runCount ? slot.runs.size / runCount : 0;
|
||||
return { file, flakyRuns: slot.runs.size, entries: slot.entries, rate, enforced: enforceable, failing: enforceable && rate > policy.maxRate };
|
||||
}).sort((a, b) => b.rate - a.rate || a.file.localeCompare(b.file));
|
||||
return { runs: runCount, flakyRuns: runEntries.size, since, files, enforceable };
|
||||
}
|
||||
|
||||
/**
|
||||
* Read the flake ledgers of main's completed free-tests runs: the runs created
|
||||
* in the last `sinceDays`, widened to the newest `minRuns` runs (within
|
||||
* `maxLookbackDays`) so the 5% rule has a window of at least 20 runs. A run
|
||||
* with no `flake-ledger-*` artifact had no flaky passes (the upload is skipped
|
||||
* when the ledger file does not exist).
|
||||
*/
|
||||
export function readCiFlakeLedgers(opts: { client: GhClient; repo: string; sinceDays: number; now: number; cacheDir: string;
|
||||
minRuns?: number; maxLookbackDays?: number; workflow?: string }): FlakeWindow {
|
||||
const minRuns = opts.minRuns ?? FLAKE_POLICY.minRuns;
|
||||
const iso = (days: number) => new Date(opts.now - days * 86_400_000).toISOString();
|
||||
const all = listWorkflowRuns(opts.client, opts.repo, opts.workflow ?? 'free-tests.yml',
|
||||
{ since: iso(Math.max(opts.sinceDays, opts.maxLookbackDays ?? 60)), branch: 'main' })
|
||||
.filter(run => run.conclusion === 'success' || run.conclusion === 'failure');
|
||||
const recent = all.filter(run => run.createdAt >= iso(opts.sinceDays));
|
||||
const window = recent.length >= minRuns ? recent : all.slice(0, minRuns);
|
||||
const fetcher = historyFetcher(opts.client);
|
||||
const runEntries = new Map<number, FlakeLedgerEntry[]>();
|
||||
for (const run of window) {
|
||||
const dirs = downloadRunArtifacts({ repo: opts.repo, run, cacheDir: opts.cacheDir, fetcher, match: name => name.startsWith('flake-ledger-') });
|
||||
const entries = dirs.flatMap(readFlakeLedgerDir);
|
||||
if (entries.length) runEntries.set(run.id, entries);
|
||||
}
|
||||
return rankFlakyFiles(runEntries, window.length, window[window.length - 1]?.createdAt ?? iso(opts.sinceDays));
|
||||
}
|
||||
|
||||
/** Markdown table of one run's merged flake ledgers, for a job summary. */
|
||||
export function formatFlakeSummary(entries: FlakeLedgerEntry[]): string {
|
||||
if (entries.length === 0) return '### Free-suite flaky passes\n\nNone: every file passed on its first run.\n';
|
||||
const byFile = new Map<string, FlakeLedgerEntry[]>();
|
||||
for (const entry of entries) byFile.set(entry.file, [...(byFile.get(entry.file) ?? []), entry]);
|
||||
const rows = [...byFile].sort((a, b) => b[1].length - a[1].length || a[0].localeCompare(b[0]))
|
||||
.map(([file, list]) => `| \`${file}\` | ${list.length} | ${[...new Set(list.map(entry => entry.shard).filter(shard => shard !== undefined))].join(', ') || '-'} |`);
|
||||
return ['### Free-suite flaky passes', '',
|
||||
`${entries.length} flaky pass(es) in ${byFile.size} file(s): each failed, then passed on the serial retry. They are green, not fixed.`, '',
|
||||
'| File | Flaky passes | Shard(s) |', '|---|---:|---|', ...rows, '',
|
||||
'Weekly enforcement: `bun run test:health --enforce` fails when a file flakes in more than 5% of main runs over at least 20 runs.', ''].join('\n');
|
||||
}
|
||||
|
||||
// --- Repository ---
|
||||
|
||||
export function gitOutput(args: string[], cwd?: string): string | null {
|
||||
const result = spawnSync('git', args, { encoding: 'utf8', timeout: 5_000, cwd });
|
||||
return result.status === 0 ? result.stdout.trim() : null;
|
||||
}
|
||||
|
||||
export function repoSlug(cwd?: string): string {
|
||||
const url = gitOutput(['remote', 'get-url', 'origin'], cwd) ?? '';
|
||||
return url.match(/[:/]([^/:]+\/[^/]+?)(?:\.git)?$/)?.[1] ?? 'garrytan/gstack';
|
||||
}
|
||||
@@ -23,7 +23,7 @@ import { spawnSync } from 'node:child_process';
|
||||
import { panelVerdict, type TrialOutcomeRecord } from '../../test/helpers/eval-store';
|
||||
import { HEADROOM_FAIL, HEADROOM_WARN } from '../../test/helpers/eval-budgets';
|
||||
import { sanitizeFixedFenceLine } from './published-text';
|
||||
import type { HistoryFetcher, WeeklyRun } from '../eval-flake-rank';
|
||||
import type { HistoryFetcher, WeeklyRun } from './ci-history';
|
||||
import { CASE_SHARDED_FILES, trialShardKey } from './paid-cases';
|
||||
import { shardSlug } from '../test-paid-shards';
|
||||
import { CENSUS_RED_GUIDE } from './paid-report';
|
||||
|
||||
@@ -0,0 +1,164 @@
|
||||
/**
|
||||
* Free-lane health signals that keep the duration seed honest: the
|
||||
* unseeded-file warning (--ci-plan), the shard overrun warning (--ci-verify
|
||||
* job summary), and the
|
||||
* seed growth ratchet (60 s ceiling with a shrink-only allowlist).
|
||||
* Every message names the problem, the offending entry, the fix command and
|
||||
* the docs anchor.
|
||||
*/
|
||||
import { spawnSync } from 'node:child_process';
|
||||
import * as fs from 'node:fs';
|
||||
import type { FreeCiPlan, FreeCiResult } from '../test-free-shards';
|
||||
import type { CurationResult } from './windows-curation';
|
||||
|
||||
export const UNSEEDED_WARN_LIMIT = 5;
|
||||
export const SHARD_OVERRUN_FACTOR = 1.5;
|
||||
/** Overruns smaller than this are runner noise (startup, cache restore), not a stale seed. */
|
||||
export const SHARD_OVERRUN_FLOOR_MS = 30_000;
|
||||
export const SEED_CEILING_MS = 60_000;
|
||||
export const SEED_ALLOWLIST_FILE = 'scripts/free-test-seed-allowlist.json';
|
||||
export const SEED_REFRESH_COMMAND = 'bun run test:ubicloud --record-durations';
|
||||
export const SEED_DOCS_ANCHOR = 'docs/TESTING_INTERNALS.md#free-suite-duration-seed';
|
||||
|
||||
const seconds = (ms: number) => `${Math.round(ms / 1000)}s`;
|
||||
|
||||
export function unseededWarning(unseeded: string[]): string | null {
|
||||
if (unseeded.length <= UNSEEDED_WARN_LIMIT) return null;
|
||||
const shown = unseeded.slice(0, 10).join(', ') + (unseeded.length > 10 ? `, +${unseeded.length - 10} more` : '');
|
||||
return `WARNING: ${unseeded.length} free test files have no recorded duration (limit ${UNSEEDED_WARN_LIMIT}); `
|
||||
+ `they are packed at the 99th-percentile estimate, so shard balance degrades until the seed is refreshed: ${shown}. `
|
||||
+ `Fix: run \`${SEED_REFRESH_COMMAND}\` and commit scripts/free-test-durations.json. See ${SEED_DOCS_ANCHOR}.`;
|
||||
}
|
||||
|
||||
export interface ShardTiming { shard: number; predictedMs: number; elapsedMs: number }
|
||||
|
||||
export function shardOverruns(timings: ShardTiming[]): ShardTiming[] {
|
||||
return timings.filter(({ predictedMs, elapsedMs }) =>
|
||||
elapsedMs > predictedMs * SHARD_OVERRUN_FACTOR && elapsedMs - predictedMs >= SHARD_OVERRUN_FLOOR_MS);
|
||||
}
|
||||
|
||||
export function overrunWarning(overruns: ShardTiming[]): string | null {
|
||||
if (overruns.length === 0) return null;
|
||||
const listed = overruns.map(({ shard, predictedMs, elapsedMs }) =>
|
||||
`shard ${shard} took ${seconds(elapsedMs)} vs ${seconds(predictedMs)} predicted`).join('; ');
|
||||
return `WARNING: ${overruns.length} shard(s) exceeded ${SHARD_OVERRUN_FACTOR}x their predicted duration (${listed}). `
|
||||
+ `A file in them is slower than its seed entry or unseeded. Fix: run \`${SEED_REFRESH_COMMAND}\` `
|
||||
+ `and commit scripts/free-test-durations.json; split any file over ${seconds(SEED_CEILING_MS)}. See ${SEED_DOCS_ANCHOR}.`;
|
||||
}
|
||||
|
||||
/** Job-summary signal derived from the strict results: shards that outran their seed. */
|
||||
export function ciHealthSummary(plan: FreeCiPlan, results: FreeCiResult[]): string[] {
|
||||
const ordered = [...results].sort((a, b) => a.outcome.shard - b.outcome.shard);
|
||||
const overruns = shardOverruns(ordered.flatMap(({ outcome }) => {
|
||||
const predictedMs = plan.shards[outcome.shard - 1]?.predictedMs;
|
||||
return predictedMs === undefined ? [] : [{ shard: outcome.shard, predictedMs, elapsedMs: outcome.elapsedMs }];
|
||||
}));
|
||||
const warning = overrunWarning(overruns);
|
||||
return warning ? [warning] : [];
|
||||
}
|
||||
|
||||
/** Printed by every Windows run and its job summary so silent shrinkage of the curated set is visible. */
|
||||
export function windowsCurationLine({ safe, excluded }: CurationResult): string {
|
||||
const byReason = new Map<string, number>();
|
||||
for (const { reason } of excluded) byReason.set(reason, (byReason.get(reason) ?? 0) + 1);
|
||||
const top = [...byReason].sort((a, b) => b[1] - a[1] || a[0].localeCompare(b[0])).slice(0, 4)
|
||||
.map(([reason, count]) => `${count} ${reason}`).join('; ');
|
||||
return `curated ${safe.length} Windows-safe tests (${excluded.length} excluded${top ? `: ${top}` : ''})`;
|
||||
}
|
||||
|
||||
/** Append markdown to the GitHub job summary when running in Actions; no-op locally. */
|
||||
export function appendStepSummary(markdown: string, env: NodeJS.ProcessEnv = process.env): void {
|
||||
if (!env.GITHUB_STEP_SUMMARY) return;
|
||||
fs.appendFileSync(env.GITHUB_STEP_SUMMARY, `${markdown}\n\n`);
|
||||
}
|
||||
|
||||
// ─── Seed growth ratchet ────────────────────────────────────────────────────
|
||||
|
||||
export interface SeedAllowlistEntry {
|
||||
file: string;
|
||||
reason: string;
|
||||
/** Required on an entry the merge-base allowlist lacks: the annotated exception names its follow-up. */
|
||||
todo?: string;
|
||||
}
|
||||
|
||||
export function parseSeedAllowlist(raw: string, source = SEED_ALLOWLIST_FILE): SeedAllowlistEntry[] {
|
||||
const parsed = JSON.parse(raw) as { entries?: unknown };
|
||||
if (!Array.isArray(parsed.entries)) throw new Error(`${source}: expected {"entries": [...]}`);
|
||||
return parsed.entries.map((entry, index) => {
|
||||
const { file, reason, todo } = (entry ?? {}) as Record<string, unknown>;
|
||||
if (typeof file !== 'string' || !file || typeof reason !== 'string' || !reason.trim()) {
|
||||
throw new Error(`${source}: entries[${index}] needs a non-empty "file" and "reason"`);
|
||||
}
|
||||
if (todo !== undefined && (typeof todo !== 'string' || !todo.trim())) {
|
||||
throw new Error(`${source}: entries[${index}] "todo" must be a non-empty string when present`);
|
||||
}
|
||||
return todo === undefined ? { file, reason } : { file, reason, todo };
|
||||
});
|
||||
}
|
||||
|
||||
export interface SeedRatchetInput {
|
||||
durations: Record<string, number>;
|
||||
allowlist: SeedAllowlistEntry[];
|
||||
/** The merge-base allowlist; null when the merge-base has no allowlist file (the first allowlist). */
|
||||
baseAllowlist: SeedAllowlistEntry[] | null;
|
||||
}
|
||||
|
||||
export interface SeedRatchetResult {
|
||||
violations: string[];
|
||||
/** Annotated exceptions added on this branch; printed so reviewers see them. */
|
||||
exceptions: string[];
|
||||
}
|
||||
|
||||
export function seedRatchet({ durations, allowlist, baseAllowlist }: SeedRatchetInput): SeedRatchetResult {
|
||||
const violations: string[] = [];
|
||||
const exceptions: string[] = [];
|
||||
const allowed = new Map(allowlist.map(entry => [entry.file, entry]));
|
||||
const fix = `See ${SEED_DOCS_ANCHOR}.`;
|
||||
|
||||
for (const [file, ms] of Object.entries(durations).sort(([a], [b]) => a.localeCompare(b))) {
|
||||
if (ms > SEED_CEILING_MS && !allowed.has(file)) {
|
||||
violations.push(`${file} is seeded at ${seconds(ms)}, over the ${seconds(SEED_CEILING_MS)} ceiling. `
|
||||
+ `Fix: split it (by describe block or fixture) so the packer can balance it, then rerun \`${SEED_REFRESH_COMMAND}\`. ${fix}`);
|
||||
}
|
||||
}
|
||||
for (const entry of allowlist) {
|
||||
const ms = durations[entry.file];
|
||||
if (ms === undefined || ms <= SEED_CEILING_MS) {
|
||||
violations.push(`${SEED_ALLOWLIST_FILE} lists ${entry.file}, which is ${ms === undefined ? 'not in the seed' : `seeded at ${seconds(ms)}`}. `
|
||||
+ `Fix: delete its entry; the allowlist holds only files still over ${seconds(SEED_CEILING_MS)}. ${fix}`);
|
||||
}
|
||||
}
|
||||
if (baseAllowlist === null) return { violations, exceptions };
|
||||
const baseFiles = new Set(baseAllowlist.map(entry => entry.file));
|
||||
for (const entry of allowlist) {
|
||||
if (baseFiles.has(entry.file)) continue;
|
||||
if (entry.todo) {
|
||||
exceptions.push(`annotated exception: ${entry.file} (reason: ${entry.reason}; todo: ${entry.todo})`);
|
||||
} else {
|
||||
violations.push(`${SEED_ALLOWLIST_FILE} adds ${entry.file}, but the allowlist is shrink-only. `
|
||||
+ `Fix: split the file under ${seconds(SEED_CEILING_MS)}, or add a "todo" naming the follow-up that splits it (an annotated exception). ${fix}`);
|
||||
}
|
||||
}
|
||||
return { violations, exceptions };
|
||||
}
|
||||
|
||||
export type BaseAllowlistRead =
|
||||
| { status: 'ok'; entries: SeedAllowlistEntry[] | null }
|
||||
| { status: 'unavailable'; reason: string };
|
||||
|
||||
/**
|
||||
* The merge-base copy of the allowlist (`git show <base>:<file>`). A base that
|
||||
* predates the allowlist file yields null (the first allowlist); a base git
|
||||
* cannot resolve (shallow checkout, missing ref) is unavailable, never "empty".
|
||||
*/
|
||||
export function readBaseSeedAllowlist(root: string, baseRef: string): BaseAllowlistRead {
|
||||
const commit = spawnSync('git', ['rev-parse', '--verify', '--quiet', `${baseRef}^{commit}`], { cwd: root, encoding: 'utf8', timeout: 10_000 });
|
||||
if (commit.status !== 0) return { status: 'unavailable', reason: `cannot resolve merge-base ${baseRef}` };
|
||||
const sha = commit.stdout.trim();
|
||||
const listed = spawnSync('git', ['ls-tree', '--name-only', sha, '--', SEED_ALLOWLIST_FILE], { cwd: root, encoding: 'utf8', timeout: 10_000 });
|
||||
if (listed.status !== 0) return { status: 'unavailable', reason: `cannot list ${SEED_ALLOWLIST_FILE} at ${sha}` };
|
||||
if (!listed.stdout.trim()) return { status: 'ok', entries: null };
|
||||
const shown = spawnSync('git', ['show', `${sha}:${SEED_ALLOWLIST_FILE}`], { cwd: root, encoding: 'utf8', timeout: 10_000 });
|
||||
if (shown.status !== 0) return { status: 'unavailable', reason: `cannot read ${SEED_ALLOWLIST_FILE} at ${sha}` };
|
||||
return { status: 'ok', entries: parseSeedAllowlist(shown.stdout, `${sha}:${SEED_ALLOWLIST_FILE}`) };
|
||||
}
|
||||
+38
-50
@@ -2,60 +2,44 @@
|
||||
* Case and trial shard keys for the paid lane: which files shard per case, how a case shard and its trials are named, and expansion of files into case/trial shards. Moved from scripts/test-paid-shards.ts.
|
||||
*/
|
||||
import * as fs from 'node:fs';
|
||||
import * as os from 'node:os';
|
||||
import * as path from 'node:path';
|
||||
import { spawnSync } from 'node:child_process';
|
||||
import { createBootstrapRetentionScope } from '../../test/helpers/bootstrap-retention';
|
||||
import {
|
||||
BunTestOutputClassifier,
|
||||
createShardSandbox,
|
||||
exactTestFileSelectors,
|
||||
forwardAndClassify,
|
||||
isTerminationRequested,
|
||||
nextShardLogPath,
|
||||
normalizeRelativePath,
|
||||
openShardLog,
|
||||
parseCliFlags,
|
||||
readDurationSeed,
|
||||
removeShardSandbox,
|
||||
runShardChild,
|
||||
strictShardStatus,
|
||||
writeDurationSeed,
|
||||
zeroExecutionVerdict,
|
||||
type LanePolicy,
|
||||
type ShardChildResult,
|
||||
type ShardLog,
|
||||
} from './shard-engine';
|
||||
import { PAID_TEST_GLOBS, isPaidTestFile } from '../../test/helpers/paid-test-set';
|
||||
import { CASE_CI_EXCLUDE, CASE_QUARANTINE, EVAL_POLICY, PERIODIC_CI_EXCLUDE } from '../../test/helpers/periodic-exclude-data';
|
||||
import { FILE_RETRY_BUDGETS, STRICT_RETRY_CASE_BUDGETS } from '../../test/helpers/eval-budgets';
|
||||
import {
|
||||
getProjectEvalDir, getClaudeCliVersion, isFinalizedEvalResultFile, evalEntryOutcome, failureClassOf, panelVerdict,
|
||||
sanitizeTrialError, formatTrialOutcomes, CONTRACT_VIOLATIONS_FILE, TRIAL_ENV, TRIAL_OUTCOME_SCHEMA, TRIAL_OUTCOMES_FILE,
|
||||
type EvalCaseKind, type PanelShape, type PanelVerdict, type TrialFailureClass, type TrialOutcome, type TrialOutcomeRecord,
|
||||
} from '../../test/helpers/eval-store';
|
||||
import { normalizeRelativePath } from './shard-engine';
|
||||
import { CASE_CI_EXCLUDE, CASE_QUARANTINE, EVAL_POLICY } from '../../test/helpers/periodic-exclude-data';
|
||||
import type { EvalCaseKind, PanelShape } from '../../test/helpers/eval-store';
|
||||
import { E2E_KINDS } from '../../test/helpers/touchfiles-data';
|
||||
import { manualReviewProblem } from '../../test/helpers/cookie-workflow-manual-review';
|
||||
import { preflightAnthropicApi } from '../../test/helpers/anthropic-preflight';
|
||||
import { OVERLAY_MIN_FILE_WALL_MS } from '../../test/helpers/overlay-case-policy';
|
||||
import { PR_PROFILE_CASE_IDS, PR_PROFILE_FILES, packageChangeOnlyVersion, selectPrProfile, type PrProfileSelection } from '../test-pr-profile';
|
||||
import { e2eReuseLaneProblem, prepareE2EShardReuse, selectPlanReceipts, writeNegativeReceipt, writePanelReceipt } from '../e2e-shard-reuse';
|
||||
import { E2E_TOUCHFILES, E2E_TIERS } from '../../test/helpers/touchfiles';
|
||||
import { type PaidTier, ROOT } from './paid-types';
|
||||
|
||||
import {
|
||||
detectBaseBranch,
|
||||
getChangedFiles,
|
||||
selectTests,
|
||||
E2E_TOUCHFILES,
|
||||
E2E_TIERS,
|
||||
LLM_JUDGE_TOUCHFILES,
|
||||
GLOBAL_TOUCHFILES,
|
||||
} from '../../test/helpers/touchfiles';
|
||||
/**
|
||||
* The E2E ids a paid file registers: the touchfile registrations that list the
|
||||
* file. `known` is true only when those ids are complete: no computed
|
||||
* registration (testName, *IfSelected, describeIfSelected with a non-literal
|
||||
* argument) and every literal registration argument is among them
|
||||
* (`unregistered` lists the literal ids that are not). Quoted strings
|
||||
* elsewhere (comments, skill paths) never count.
|
||||
*/
|
||||
export function fileCaseRegistration(
|
||||
file: string, source: string,
|
||||
touchfiles: Record<string, string[]> = E2E_TOUCHFILES,
|
||||
tiers: Record<string, string> = E2E_TIERS,
|
||||
): { registered: string[]; known: boolean; computed: boolean; unregistered: string[] } {
|
||||
const rel = normalizeRelativePath(file);
|
||||
const registered = Object.keys(touchfiles).filter(key => touchfiles[key]!.includes(rel));
|
||||
const computed = /testName\s*:\s*(?!string\b)(?:`[^`]*\$\{|[A-Za-z_$])/.test(source)
|
||||
|| /\btest(?:Concurrent)?IfSelected\s*\(\s*(?:`[^`]*\$\{|[A-Za-z_$])/.test(source)
|
||||
|| /\bdescribeIfSelected\s*\([^,]*,(?!\s*\[)/.test(source)
|
||||
|| [...source.matchAll(/\bdescribeIfSelected\s*\([^,]*,\s*\[([^\]]*)\]/g)].some(m => m[1]!.split(',')
|
||||
.map(item => item.trim()).some(item => item && !/^(['"`])[^'"`$]*\1$/.test(item)));
|
||||
const literal = [
|
||||
...[...source.matchAll(/testName\s*:\s*(['"`])([^'"`]+)\1/g)].map(m => m[2]!),
|
||||
...[...source.matchAll(/\btest(?:Concurrent)?IfSelected\s*\(\s*(['"`])([^'"`]+)\1/g)].map(m => m[2]!),
|
||||
...[...source.matchAll(/\bdescribeIfSelected\s*\([^,]*,\s*\[([^\]]*)\]/g)]
|
||||
.flatMap(m => [...m[1]!.matchAll(/(['"`])([^'"`]+)\1/g)].map(n => n[2]!)),
|
||||
].filter(id => id in tiers);
|
||||
const unregistered = [...new Set(literal.filter(id => !registered.includes(id)))];
|
||||
return { registered, known: registered.length > 0 && !computed && unregistered.length === 0, computed, unregistered };
|
||||
}
|
||||
|
||||
export { PAID_TEST_GLOBS, isPaidTestFile };
|
||||
export { PERIODIC_CI_EXCLUDE };
|
||||
|
||||
type E2EShardReuse = NonNullable<ReturnType<typeof prepareE2EShardReuse>>;
|
||||
import { type PaidTier, ROOT, fileCaseRegistration } from '../test-paid-shards';
|
||||
|
||||
/**
|
||||
* Files whose cases run in separate processes, one shard per registered E2E
|
||||
@@ -67,6 +51,8 @@ import { type PaidTier, ROOT, fileCaseRegistration } from '../test-paid-shards';
|
||||
* id or its CASE_TEST_NAMES label (test/paid-shards.test.ts scans the sources).
|
||||
*/
|
||||
export const CASE_SHARDED_FILES: readonly string[] = [
|
||||
// W5a: one carved skill per case shard (was 15 one-line wrapper files under one case id).
|
||||
'test/carve-section-loading.test.ts',
|
||||
'test/skill-e2e-deploy.test.ts',
|
||||
'test/skill-e2e-design.test.ts',
|
||||
'test/skill-e2e-plan.test.ts',
|
||||
@@ -75,6 +61,8 @@ export const CASE_SHARDED_FILES: readonly string[] = [
|
||||
'test/skill-e2e-shared-libs.test.ts',
|
||||
'test/skill-e2e-ship-docsync.test.ts',
|
||||
'test/skill-e2e-qa-callers.test.ts',
|
||||
// W5c: its gate cases (qa-quick, qa-only-no-fix, qa-bootstrap) no longer share one ~7-minute runner.
|
||||
'test/skill-e2e-qa-workflow.test.ts',
|
||||
];
|
||||
|
||||
/** Bun test names that differ from their E2E id. */
|
||||
|
||||
+65
-84
@@ -2,61 +2,17 @@
|
||||
* Paid-lane planner: duration seeds, slice packing, the run manifest and its verification. Moved from scripts/test-paid-shards.ts.
|
||||
*/
|
||||
import * as fs from 'node:fs';
|
||||
import * as os from 'node:os';
|
||||
import * as path from 'node:path';
|
||||
import { spawnSync } from 'node:child_process';
|
||||
import { createBootstrapRetentionScope } from '../../test/helpers/bootstrap-retention';
|
||||
import {
|
||||
BunTestOutputClassifier,
|
||||
createShardSandbox,
|
||||
exactTestFileSelectors,
|
||||
forwardAndClassify,
|
||||
isTerminationRequested,
|
||||
nextShardLogPath,
|
||||
normalizeRelativePath,
|
||||
openShardLog,
|
||||
parseCliFlags,
|
||||
readDurationSeed,
|
||||
removeShardSandbox,
|
||||
runShardChild,
|
||||
strictShardStatus,
|
||||
writeDurationSeed,
|
||||
zeroExecutionVerdict,
|
||||
type LanePolicy,
|
||||
type ShardChildResult,
|
||||
type ShardLog,
|
||||
} from './shard-engine';
|
||||
import { PAID_TEST_GLOBS, isPaidTestFile } from '../../test/helpers/paid-test-set';
|
||||
import { CASE_CI_EXCLUDE, CASE_QUARANTINE, EVAL_POLICY, PERIODIC_CI_EXCLUDE } from '../../test/helpers/periodic-exclude-data';
|
||||
import { normalizeRelativePath } from './shard-engine';
|
||||
import { CASE_QUARANTINE } from '../../test/helpers/periodic-exclude-data';
|
||||
import { FILE_RETRY_BUDGETS, STRICT_RETRY_CASE_BUDGETS } from '../../test/helpers/eval-budgets';
|
||||
import {
|
||||
getProjectEvalDir, getClaudeCliVersion, isFinalizedEvalResultFile, evalEntryOutcome, failureClassOf, panelVerdict,
|
||||
sanitizeTrialError, formatTrialOutcomes, CONTRACT_VIOLATIONS_FILE, TRIAL_ENV, TRIAL_OUTCOME_SCHEMA, TRIAL_OUTCOMES_FILE,
|
||||
type EvalCaseKind, type PanelShape, type PanelVerdict, type TrialFailureClass, type TrialOutcome, type TrialOutcomeRecord,
|
||||
} from '../../test/helpers/eval-store';
|
||||
import { evalEntryOutcome, type EvalCaseKind } from '../../test/helpers/eval-store';
|
||||
import { E2E_KINDS } from '../../test/helpers/touchfiles-data';
|
||||
import { manualReviewProblem } from '../../test/helpers/cookie-workflow-manual-review';
|
||||
import { preflightAnthropicApi } from '../../test/helpers/anthropic-preflight';
|
||||
import { OVERLAY_MIN_FILE_WALL_MS } from '../../test/helpers/overlay-case-policy';
|
||||
import { PR_PROFILE_CASE_IDS, PR_PROFILE_FILES, packageChangeOnlyVersion, selectPrProfile, type PrProfileSelection } from '../test-pr-profile';
|
||||
import { e2eReuseLaneProblem, prepareE2EShardReuse, selectPlanReceipts, writeNegativeReceipt, writePanelReceipt } from '../e2e-shard-reuse';
|
||||
|
||||
import {
|
||||
detectBaseBranch,
|
||||
getChangedFiles,
|
||||
selectTests,
|
||||
E2E_TOUCHFILES,
|
||||
E2E_TIERS,
|
||||
LLM_JUDGE_TOUCHFILES,
|
||||
GLOBAL_TOUCHFILES,
|
||||
} from '../../test/helpers/touchfiles';
|
||||
|
||||
export { PAID_TEST_GLOBS, isPaidTestFile };
|
||||
export { PERIODIC_CI_EXCLUDE };
|
||||
|
||||
type E2EShardReuse = NonNullable<ReturnType<typeof prepareE2EShardReuse>>;
|
||||
import { prProfileCaseAllowed, prProfileFileMap, type PrProfileSelection } from '../test-pr-profile';
|
||||
import { E2E_TOUCHFILES, E2E_TIERS, LLM_JUDGE_TOUCHFILES } from '../../test/helpers/touchfiles';
|
||||
import { CASE_KEY_SEPARATOR, CASE_SHARDED_FILES, type CaseTrialPlan, caseTrialPlan, codexShardAccess, expandCaseShards, expandTrialShards, isIsolatedCase, partitionCaseExclusions, sameTrialPlan, shardCaseId, shardFile, shardTrial } from './paid-cases';
|
||||
import { DEFAULT_JOBS, OVERLAY_MAX_ACTIVE_SHARDS, PAID_TIERS, type PaidCaseSelection, type PaidProfile, type PaidShardBudget, type PaidTier, ROOT, type ShardOutcome, type ShardStatus, type ShardTrialRecord, collectPaidTestFiles, computePaidCaseSelection, expectedPrCaseCount, isAllSkippedPass, isOverlayTestFile, paidShardWallUpperBoundMs, partitionShardsByDiffSelection, planPaidShards, prProfileFileSelected, resolvePaidShardBudget, resolvePaidShardTimeoutMs, sameBudget, selectPaidTestFiles, shardSlug, validatedProfile } from '../test-paid-shards';
|
||||
import { DEFAULT_JOBS, OVERLAY_MAX_ACTIVE_SHARDS, PAID_TIERS, SLICE_UPLOAD_RESERVE_MS, type PaidCaseSelection, type PaidProfile, type PaidShardBudget, type PaidTier, ROOT, type ShardOutcome, type ShardStatus, type ShardTrialRecord, isOverlayTestFile } from './paid-types';
|
||||
import { collectPaidTestFiles, computePaidCaseSelection, expectedPrCaseCount, isAllSkippedPass, paidShardWallUpperBoundMs, partitionShardsByDiffSelection, planPaidShards, prProfileFileSelected, resolvePaidShardBudget, resolvePaidShardTimeoutMs, sameBudget, selectPaidTestFiles, shardSlug, validatedProfile } from './paid-select';
|
||||
|
||||
// ─── Planner / executor / report (the CI re-platform surface) ──────────────
|
||||
// One PLANNER computes selection and the slice plan ONCE; K executor jobs
|
||||
@@ -88,7 +44,10 @@ export interface PaidSlicePlan {
|
||||
sliceBudgetMs: number;
|
||||
jobs: number;
|
||||
estimatedSliceMs: number[];
|
||||
/** The largest per-slice ceiling: one number for executors that use a single job timeout. */
|
||||
ciTimeoutMinutes: number;
|
||||
/** Per-slice CI job ceilings (sliceCiTimeoutMinutes); absent in plans written before W2c. */
|
||||
sliceCiTimeoutMinutes?: number[];
|
||||
/** Slices that run only Codex shards; CI executes them on the host-run Codex job, where bubblewrap can start. */
|
||||
codexSlices?: number[];
|
||||
}
|
||||
@@ -213,6 +172,32 @@ function sharesPanel(planned: readonly string[], key: string): boolean {
|
||||
/** Setup, image pull and artifact upload allowance on top of a slice's supervised wall. */
|
||||
export const CI_SETUP_ALLOWANCE_MINUTES = 20;
|
||||
|
||||
/**
|
||||
* CI job ceiling of one slice (W2c, ENG-2): its envelope is the larger of twice
|
||||
* the slice budget, the longest single shard's supervised wall, and, for the
|
||||
* overlay slice, the serialized overlay group; plus the setup allowance. A hang
|
||||
* fails within this bound instead of holding a runner for the sum of every
|
||||
* shard's worst case; the executor's in-process deadline (sliceDeadlineMs)
|
||||
* stops starting work 5 minutes earlier so results still upload. Same cases run.
|
||||
*/
|
||||
export function sliceCiTimeoutMinutes(files: string[], budgetMs: number, jobs: number, timeoutMs?: number): number {
|
||||
const longestShardMs = Math.max(0, ...files.map(file => resolvePaidShardTimeoutMs([file], timeoutMs)));
|
||||
const overlayEnvelopeMs = files.some(isOverlayTestFile) ? sliceSupervisedWallMs(files, jobs, timeoutMs) : 0;
|
||||
return Math.ceil(Math.max(2 * budgetMs, longestShardMs, overlayEnvelopeMs) / 60_000) + CI_SETUP_ALLOWANCE_MINUTES;
|
||||
}
|
||||
|
||||
/**
|
||||
* The executor's in-process deadline: job start (the workflow's first step
|
||||
* records it in GSTACK_SLICE_JOB_STARTED_AT, epoch seconds) plus the slice's
|
||||
* job ceiling, minus the upload reserve. Without a recorded job start (local
|
||||
* runs) the executor's own start stands in.
|
||||
*/
|
||||
export function sliceDeadlineMs(ceilingMinutes: number, jobStartedAtSeconds: string | undefined, fallbackStartMs: number): number {
|
||||
const recorded = Number(jobStartedAtSeconds);
|
||||
const start = Number.isSafeInteger(recorded) && recorded > 0 ? recorded * 1000 : fallbackStartMs;
|
||||
return start + ceilingMinutes * 60_000 - SLICE_UPLOAD_RESERVE_MS;
|
||||
}
|
||||
|
||||
/** Estimated wall of one executor running `files` in order on `jobs` FIFO workers. */
|
||||
export function estimatedSliceMs(files: string[], weight: (file: string) => number, jobs: number): number {
|
||||
const workers = Array<number>(Math.max(1, jobs)).fill(0);
|
||||
@@ -241,12 +226,11 @@ export function sliceSupervisedWallMs(files: string[], jobs: number, overrideMs?
|
||||
* wall weighs the whole budget, so unknown cost gets a runner of its own. A
|
||||
* file longer than the budget runs alone. Overlay wrappers keep one shared
|
||||
* final slice (one wrapper at a time). Codex shards never share a slice with
|
||||
* other shards. The CI timeout covers every slice's supervised worst case plus
|
||||
* the setup allowance.
|
||||
* other shards. Each slice gets its own CI job ceiling (sliceCiTimeoutMinutes).
|
||||
*/
|
||||
export function packBySliceBudget(files: string[], budgetMs: number, jobs: number,
|
||||
recorded: Record<string, number>, timeoutMs?: number, codexShard: (file: string) => boolean = () => false): {
|
||||
slices: string[][]; estimates: Record<string, number>; estimatedSliceMs: number[]; ciTimeoutMinutes: number;
|
||||
slices: string[][]; estimates: Record<string, number>; estimatedSliceMs: number[]; ciTimeoutMinutes: number; sliceCiTimeoutMinutes: number[];
|
||||
} {
|
||||
const estimates = Object.fromEntries(files.map(file => [file, recordedShardMs(recorded, file) ?? budgetMs]));
|
||||
const weight = (file: string) => estimates[file]!;
|
||||
@@ -270,9 +254,8 @@ export function packBySliceBudget(files: string[], budgetMs: number, jobs: numbe
|
||||
if (!slices.length) slices.push([]);
|
||||
const estimatedSliceMsList = slices.map(planned => planned.some(isOverlayTestFile)
|
||||
? estimatedSliceMs(planned, weight, Math.min(jobs, OVERLAY_MAX_ACTIVE_SHARDS)) : estimatedSliceMs(planned, weight, jobs));
|
||||
const worst = Math.max(0, ...slices.map(planned => sliceSupervisedWallMs(planned, jobs, timeoutMs)));
|
||||
return { slices, estimates, estimatedSliceMs: estimatedSliceMsList,
|
||||
ciTimeoutMinutes: Math.ceil(worst / 60_000) + CI_SETUP_ALLOWANCE_MINUTES };
|
||||
const ceilings = slices.map(planned => sliceCiTimeoutMinutes(planned, budgetMs, jobs, timeoutMs));
|
||||
return { slices, estimates, estimatedSliceMs: estimatedSliceMsList, ciTimeoutMinutes: Math.max(...ceilings), sliceCiTimeoutMinutes: ceilings };
|
||||
}
|
||||
|
||||
/** Worker counts whose worst-case slice wall duration packing may never worsen. */
|
||||
@@ -324,7 +307,7 @@ export function buildRunManifest(opts: {
|
||||
const profile = opts.profile ?? validatedProfile(env.EVALS_PROFILE, 'EVALS_PROFILE');
|
||||
if (profile === 'pr' && opts.tier !== 'gate') throw new Error('PR profile requires gate tier; use --profile full for periodic coverage');
|
||||
const discovered = opts.discovered ?? collectPaidTestFiles(rootDir);
|
||||
const tierSelection = selectPaidTestFiles(discovered, opts.tier, rootDir, env);
|
||||
const tierSelection = selectPaidTestFiles(discovered, opts.tier, rootDir);
|
||||
const judge = (file: string) => /^test\/skill-llm-eval[^/]*\.test\.ts$/.test(normalizeRelativePath(file));
|
||||
const selected = opts.skipJudges ? tierSelection.selected.filter(file => !judge(file)) : tierSelection.selected;
|
||||
const kinds = opts.kinds ?? E2E_KINDS;
|
||||
@@ -366,7 +349,7 @@ export function buildRunManifest(opts: {
|
||||
selectionReason: cases.reason, profile, selection: cases.selection,
|
||||
...(cases.coverage ? { prCoverage: cases.coverage } : {}),
|
||||
plan: { sliceBudgetMs: opts.sliceBudgetMs!, jobs: opts.jobs!, estimatedSliceMs: plan.estimatedSliceMs, ciTimeoutMinutes: plan.ciTimeoutMinutes,
|
||||
codexSlices: codexSlicesOf(opts.tier, entries) },
|
||||
sliceCiTimeoutMinutes: plan.sliceCiTimeoutMinutes, codexSlices: codexSlicesOf(opts.tier, entries) },
|
||||
entries,
|
||||
} satisfies PaidRunManifest));
|
||||
}
|
||||
@@ -491,21 +474,6 @@ export function buildRunManifest(opts: {
|
||||
return parseRunManifest(JSON.stringify(manifest));
|
||||
}
|
||||
|
||||
/**
|
||||
* Narrow a built manifest to a curated case subset (validation phases): the
|
||||
* selection binds every child, and a case shard outside it can execute
|
||||
* nothing, so it becomes skipped instead of an empty planned shard.
|
||||
*/
|
||||
export function restrictManifestSelection(manifest: PaidRunManifest, selection: PaidCaseSelection, reason: string): PaidRunManifest {
|
||||
const entries = manifest.entries.map(entry => {
|
||||
const caseId = shardCaseId(entry.file);
|
||||
if (entry.status !== 'planned' || caseId === null || selection.e2e === null || selection.e2e.includes(caseId)) return entry;
|
||||
const { estimatedMs: _estimate, budget: _budget, ...rest } = entry;
|
||||
return { ...rest, slice: 0, status: 'skipped-by-diff' as const, reason };
|
||||
});
|
||||
return parseRunManifest(JSON.stringify({ ...manifest, selection, entries }));
|
||||
}
|
||||
|
||||
export function parseRunManifest(raw: string): PaidRunManifest {
|
||||
const parsed = JSON.parse(raw) as PaidRunManifest;
|
||||
if (parsed.version !== 1) throw new Error(`unsupported manifest version: ${(parsed as { version?: unknown }).version}`);
|
||||
@@ -522,7 +490,7 @@ export function parseRunManifest(raw: string): PaidRunManifest {
|
||||
if (parsed.profile === 'pr') {
|
||||
const coverage = parsed.prCoverage;
|
||||
if (parsed.tier !== 'gate' || !parsed.selection || !coverage ||
|
||||
!['pr', 'full-fallback'].includes(coverage.mode) || !Array.isArray(coverage.deferred) ||
|
||||
!['pr', 'dependents', 'full-fallback'].includes(coverage.mode) || !Array.isArray(coverage.deferred) ||
|
||||
!Array.isArray(coverage.unknownFiles) || !Array.isArray(coverage.missingCoverage) ||
|
||||
!Array.isArray(coverage.deferredPromptFiles) || coverage.deferredPromptFiles.some(file => typeof file !== 'string') ||
|
||||
!Array.isArray(coverage.e2e) || !Array.isArray(coverage.judges) ||
|
||||
@@ -532,7 +500,7 @@ export function parseRunManifest(raw: string): PaidRunManifest {
|
||||
JSON.stringify(parsed.selection.judges) !== JSON.stringify(coverage.judges)) {
|
||||
throw new Error('manifest PR coverage/selection invalid or requires full validation');
|
||||
}
|
||||
if (coverage.mode === 'pr' && coverage.e2e.some(id => !(PR_PROFILE_CASE_IDS as readonly string[]).includes(id))) {
|
||||
if (coverage.mode === 'pr' && coverage.e2e.some(id => !prProfileCaseAllowed(id, coverage.directCases ?? []))) {
|
||||
throw new Error('manifest PR selection contains a broad-only case');
|
||||
}
|
||||
if (coverage.deferred.some(item => !Object.hasOwn(E2E_TOUCHFILES, item.id) || E2E_TIERS[item.id] !== item.tier || typeof item.reason !== 'string')) {
|
||||
@@ -605,7 +573,7 @@ export function parseRunManifest(raw: string): PaidRunManifest {
|
||||
}
|
||||
if (parsed.prCoverage?.mode === 'pr') {
|
||||
const planned = parsed.entries.filter(entry => entry.status === 'planned').map(entry => normalizeRelativePath(entry.file));
|
||||
const required: string[][] = Object.entries(PR_PROFILE_FILES).flatMap(([file, ids]) => {
|
||||
const required: string[][] = Object.entries(prProfileFileMap(parsed.selection!.e2e)).flatMap(([file, ids]) => {
|
||||
const selected = ids.filter(id => parsed.selection!.e2e!.includes(id));
|
||||
if (!selected.length) return [];
|
||||
const owners = new Set(selected.flatMap(id => {
|
||||
@@ -629,7 +597,10 @@ export function parseRunManifest(raw: string): PaidRunManifest {
|
||||
if (!plan || typeof plan !== 'object' || !count(plan.sliceBudgetMs) || !count(plan.jobs) || !count(plan.ciTimeoutMinutes)
|
||||
|| !Array.isArray(plan.estimatedSliceMs) || plan.estimatedSliceMs.length !== parsed.sliceCount
|
||||
|| !plan.estimatedSliceMs.every(ms => Number.isSafeInteger(ms) && ms >= 0)
|
||||
|| parsed.entries.some(entry => entry.status === 'planned' && entry.estimatedMs === undefined)) {
|
||||
|| parsed.entries.some(entry => entry.status === 'planned' && entry.estimatedMs === undefined)
|
||||
|| (plan.sliceCiTimeoutMinutes !== undefined && (!Array.isArray(plan.sliceCiTimeoutMinutes)
|
||||
|| plan.sliceCiTimeoutMinutes.length !== parsed.sliceCount || !plan.sliceCiTimeoutMinutes.every(count)
|
||||
|| Math.max(...plan.sliceCiTimeoutMinutes) !== plan.ciTimeoutMinutes))) {
|
||||
throw new Error('manifest slice plan malformed');
|
||||
}
|
||||
const codexSlices = codexSlicesOf(parsed.tier, parsed.entries);
|
||||
@@ -685,7 +656,9 @@ export interface SliceResult {
|
||||
/** Epoch ms bounds of the slice's shard execution (lane wall time). */
|
||||
startedAt?: number;
|
||||
finishedAt?: number;
|
||||
outcomes: Array<Pick<ShardOutcome, 'files' | 'status' | 'exitCode' | 'elapsedMs' | 'executedTests' | 'skippedTests' | 'budget' | 'reused' | 'runnerError' | 'trial' | 'inputKey'>>;
|
||||
/** A progress checkpoint the executor never replaced: the job ended (ceiling or cancellation) before its final result. */
|
||||
checkpoint?: true;
|
||||
outcomes: Array<Pick<ShardOutcome, 'files' | 'status' | 'exitCode' | 'elapsedMs' | 'executedTests' | 'skippedTests' | 'budget' | 'reused' | 'runnerError' | 'trial' | 'inputKey' | 'sliceDeadline'>>;
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -729,13 +702,14 @@ export function verifySliceResults(
|
||||
problems.push(`slice ${result.sliceIndex} did not bind the manifest PR case selection`);
|
||||
}
|
||||
if (byIndex.has(result.sliceIndex)) problems.push(`duplicate result for slice ${result.sliceIndex}`);
|
||||
if (result.checkpoint) problems.push(`slice ${result.sliceIndex}/${result.sliceCount} ended before its final result (job ceiling or cancellation); its outcomes come from the last checkpoint: TIMEOUT`);
|
||||
byIndex.set(result.sliceIndex, result);
|
||||
}
|
||||
for (let index = 1; index <= manifest.sliceCount; index += 1) {
|
||||
if (!byIndex.has(index)) problems.push(`slice ${index}/${manifest.sliceCount} reported NO result — cancelled/crashed executor, not a pass`);
|
||||
}
|
||||
|
||||
const reported = new Map<string, { slice: number; status: ShardStatus; trial?: ShardTrialRecord }>();
|
||||
const reported = new Map<string, { slice: number; status: ShardStatus; trial?: ShardTrialRecord; sliceDeadline?: ShardOutcome['sliceDeadline'] }>();
|
||||
const planned = new Map(manifest.entries.map(entry => [normalizeRelativePath(entry.file), entry]));
|
||||
for (const result of results) {
|
||||
for (const outcome of result.outcomes) {
|
||||
@@ -779,7 +753,8 @@ export function verifySliceResults(
|
||||
}
|
||||
if (outcome.inputKey !== undefined && !/^[a-f0-9]{64}$/.test(outcome.inputKey)) problems.push(`${file}: malformed input identity`);
|
||||
if (reported.has(file)) problems.push(`${file} reported by two slices`);
|
||||
reported.set(file, { slice: result.sliceIndex, status: outcome.status, ...(outcome.trial ? { trial: outcome.trial } : {}) });
|
||||
reported.set(file, { slice: result.sliceIndex, status: outcome.status, ...(outcome.trial ? { trial: outcome.trial } : {}),
|
||||
...(outcome.sliceDeadline ? { sliceDeadline: outcome.sliceDeadline } : {}) });
|
||||
const registered = FILE_RETRY_BUDGETS.find(budget => budget.file === shardFile(file));
|
||||
const finding = STRICT_RETRY_CASE_BUDGETS.find(budget => budget.file === file);
|
||||
if (finding) {
|
||||
@@ -812,7 +787,12 @@ export function verifySliceResults(
|
||||
// Isolated trial shards: harness health only; the panel verdict gates.
|
||||
if (entry.trial) {
|
||||
if (got.trial?.outcome === null) problems.push(`${entry.file}: no trial record (${got.trial.harness ?? 'unknown'})`);
|
||||
} else if (got.status !== 'passed') problems.push(`${entry.file}: ${got.status}`);
|
||||
} else if (got.status !== 'passed') {
|
||||
// The slice deadline distinguishes work that never started (INFRA) from work killed in flight (TIMEOUT).
|
||||
const why = got.sliceDeadline === 'not_run' ? ' (not_run: the slice deadline passed before it started)'
|
||||
: got.sliceDeadline === 'hung' ? ' (hung: killed at the slice deadline)' : '';
|
||||
problems.push(`${entry.file}${why}: ${got.status}`);
|
||||
}
|
||||
}
|
||||
return { ok: problems.length === 0, problems };
|
||||
}
|
||||
@@ -823,11 +803,12 @@ export function formatSlicePlan(manifest: PaidRunManifest): string[] {
|
||||
if (!plan) return [];
|
||||
const minutes = (ms: number) => (ms / 60_000).toFixed(1);
|
||||
const lines = [`[test:paid] slice plan: ${manifest.sliceCount} slice(s) x ${plan.jobs} worker(s), budget ${minutes(plan.sliceBudgetMs)}m per slice, `
|
||||
+ `longest estimate ${minutes(Math.max(0, ...plan.estimatedSliceMs))}m, CI job timeout ${plan.ciTimeoutMinutes}m`];
|
||||
+ `longest estimate ${minutes(Math.max(0, ...plan.estimatedSliceMs))}m, CI job timeout ${plan.ciTimeoutMinutes}m (largest per-slice ceiling)`];
|
||||
for (let slice = 1; slice <= manifest.sliceCount; slice++) {
|
||||
const mine = sliceExecutionOrder(manifest.entries.filter(entry => entry.status === 'planned' && entry.slice === slice));
|
||||
const over = plan.estimatedSliceMs[slice - 1]! > plan.sliceBudgetMs ? ' [over budget: longer than one runner allows]' : '';
|
||||
lines.push(` slice ${slice}: ~${minutes(plan.estimatedSliceMs[slice - 1]!)}m${over}`);
|
||||
const ceiling = plan.sliceCiTimeoutMinutes?.[slice - 1];
|
||||
lines.push(` slice ${slice}: ~${minutes(plan.estimatedSliceMs[slice - 1]!)}m${ceiling !== undefined ? `, job ceiling ${ceiling}m` : ''}${over}`);
|
||||
for (const entry of mine) lines.push(` ${entry.file} ~${minutes(entry.estimatedMs ?? 0)}m retries=${retriesForFiles([entry.file])}`);
|
||||
}
|
||||
return lines;
|
||||
|
||||
+14
-51
@@ -2,65 +2,26 @@
|
||||
* Paid-lane local diagnosis and the report: JUnit parsing, panel verdicts, history records and the human readout. Moved from scripts/test-paid-shards.ts.
|
||||
*/
|
||||
import * as fs from 'node:fs';
|
||||
import * as os from 'node:os';
|
||||
import * as path from 'node:path';
|
||||
import { spawnSync } from 'node:child_process';
|
||||
import { createBootstrapRetentionScope } from '../../test/helpers/bootstrap-retention';
|
||||
import { normalizeRelativePath } from './shard-engine';
|
||||
import { EVAL_POLICY } from '../../test/helpers/periodic-exclude-data';
|
||||
import { STALL_WINDOW_MS } from '../../test/helpers/eval-budgets';
|
||||
import {
|
||||
BunTestOutputClassifier,
|
||||
createShardSandbox,
|
||||
exactTestFileSelectors,
|
||||
forwardAndClassify,
|
||||
isTerminationRequested,
|
||||
nextShardLogPath,
|
||||
normalizeRelativePath,
|
||||
openShardLog,
|
||||
parseCliFlags,
|
||||
readDurationSeed,
|
||||
removeShardSandbox,
|
||||
runShardChild,
|
||||
strictShardStatus,
|
||||
writeDurationSeed,
|
||||
zeroExecutionVerdict,
|
||||
type LanePolicy,
|
||||
type ShardChildResult,
|
||||
type ShardLog,
|
||||
} from './shard-engine';
|
||||
import { PAID_TEST_GLOBS, isPaidTestFile } from '../../test/helpers/paid-test-set';
|
||||
import { CASE_CI_EXCLUDE, CASE_QUARANTINE, EVAL_POLICY, PERIODIC_CI_EXCLUDE } from '../../test/helpers/periodic-exclude-data';
|
||||
import { FILE_RETRY_BUDGETS, STALL_WINDOW_MS, STRICT_RETRY_CASE_BUDGETS } from '../../test/helpers/eval-budgets';
|
||||
import {
|
||||
getProjectEvalDir, getClaudeCliVersion, isFinalizedEvalResultFile, evalEntryOutcome, failureClassOf, panelVerdict,
|
||||
sanitizeTrialError, formatTrialOutcomes, CONTRACT_VIOLATIONS_FILE, TRIAL_ENV, TRIAL_OUTCOME_SCHEMA, TRIAL_OUTCOMES_FILE,
|
||||
type EvalCaseKind, type PanelShape, type PanelVerdict, type TrialFailureClass, type TrialOutcome, type TrialOutcomeRecord,
|
||||
isFinalizedEvalResultFile, failureClassOf, panelVerdict, sanitizeTrialError, formatTrialOutcomes, TRIAL_OUTCOME_SCHEMA, TRIAL_OUTCOMES_FILE,
|
||||
type EvalCaseKind, type PanelVerdict, type TrialFailureClass, type TrialOutcome, type TrialOutcomeRecord,
|
||||
trialCostKnown, trialFailureFields, trialSessions, type TrialFailureDetail,
|
||||
} from '../../test/helpers/eval-store';
|
||||
import { readSessionLedger, type SessionLedgerRow } from '../../test/helpers/session-ledger';
|
||||
import { publishedFence, sanitizeFixedFenceLine } from './published-text';
|
||||
import { E2E_KINDS } from '../../test/helpers/touchfiles-data';
|
||||
import { manualReviewProblem } from '../../test/helpers/cookie-workflow-manual-review';
|
||||
import { preflightAnthropicApi } from '../../test/helpers/anthropic-preflight';
|
||||
import { OVERLAY_MIN_FILE_WALL_MS } from '../../test/helpers/overlay-case-policy';
|
||||
import { PR_PROFILE_CASE_IDS, PR_PROFILE_FILES, packageChangeOnlyVersion, selectPrProfile, type PrProfileSelection } from '../test-pr-profile';
|
||||
import { e2eReuseLaneProblem, prepareE2EShardReuse, selectPlanReceipts, writeNegativeReceipt, writePanelReceipt } from '../e2e-shard-reuse';
|
||||
|
||||
import {
|
||||
detectBaseBranch,
|
||||
getChangedFiles,
|
||||
selectTests,
|
||||
E2E_TOUCHFILES,
|
||||
E2E_TIERS,
|
||||
LLM_JUDGE_TOUCHFILES,
|
||||
GLOBAL_TOUCHFILES,
|
||||
} from '../../test/helpers/touchfiles';
|
||||
|
||||
export { PAID_TEST_GLOBS, isPaidTestFile };
|
||||
export { PERIODIC_CI_EXCLUDE };
|
||||
|
||||
type E2EShardReuse = NonNullable<ReturnType<typeof prepareE2EShardReuse>>;
|
||||
import { CASE_TEST_NAMES, type CaseTrialPlan, caseTestNamePattern, caseTrialPlan, shardCaseId, shardFile, trialShardKey } from './paid-cases';
|
||||
import { writeNegativeReceipt, writePanelReceipt } from '../e2e-shard-reuse';
|
||||
import { E2E_TOUCHFILES, E2E_TIERS, LLM_JUDGE_TOUCHFILES } from '../../test/helpers/touchfiles';
|
||||
import { CASE_TEST_NAMES, type CaseTrialPlan, caseTestNamePattern, caseTrialPlan, fileCaseRegistration, shardCaseId, shardFile, trialShardKey } from './paid-cases';
|
||||
import { type ManifestEntry, PAID_TEST_DURATIONS_FILE, type PaidRunManifest, type SliceResult, collectorOutcomeCounts, formatProfileCoverage, loadPaidTestDurations, mergePaidTestDurations, parseRunManifest, trialPanelKey, verifySliceResults, writePaidTestDurations } from './paid-plan';
|
||||
import { DEFAULT_JOBS, type PaidCaseSelection, type PaidTier, ROOT, type RunShardsOptions, type ShardTrialRecord, collectPaidTestFiles, fileCaseRegistration, isAllSkippedPass, runPaidShards, shardSlug, paidSelectionEnv } from '../test-paid-shards';
|
||||
import { DEFAULT_JOBS, type PaidCaseSelection, type PaidTier, ROOT, type ShardTrialRecord } from './paid-types';
|
||||
import { collectPaidTestFiles, isAllSkippedPass, shardSlug, paidSelectionEnv } from './paid-select';
|
||||
import type { RunShardsOptions } from '../test-paid-shards';
|
||||
|
||||
// ─── Local diagnosis: one case through the CI panel runner (A9) ────────────
|
||||
|
||||
@@ -83,7 +44,7 @@ export function caseSelection(id: string, rootDir = ROOT, discovered = collectPa
|
||||
return { file, source, ...fileCaseRegistration(file, source) };
|
||||
}).filter(f => f.registered.includes(id));
|
||||
const owners = files.filter(f => f.known).length ? files.filter(f => f.known) : files;
|
||||
if (owners.length !== 1) throw new Error(`--case ${id}: ${owners.length ? `registered by ${owners.map(f => f.file).join(', ')}` : 'no paid file registers it'}; it needs exactly one`);
|
||||
if (owners.length !== 1) throw new Error(`--case ${id}: ${owners.length ? `registered by ${owners.map(f => f.file).join(', ')}` : 'no paid file statically registers it'}; it needs exactly one`);
|
||||
const owner = owners[0]!;
|
||||
const name = escapeRe(CASE_TEST_NAMES[id] ?? id);
|
||||
if (new RegExp(`\\b(?:test|it|test\\.\\w+|test(?:Concurrent)?IfSelected)\\s*\\(\\s*(['"\`])${name}\\1|testName\\s*:\\s*(['"\`])${name}\\2`).test(owner.source)) {
|
||||
@@ -120,6 +81,8 @@ export async function runCaseDiagnosis(id: string, options: {
|
||||
const keys = Array.from({ length: n }, (_, i) => trialShardKey(file, id, i + 1));
|
||||
const tier = E2E_TIERS[id] as PaidTier;
|
||||
log(`[test:paid] --case ${id}: ${n} trial(s) of ${file} (kind ${plan.kind}, PASS at ${plan.panel.k}/${n}${plan.quarantined ? ', quarantined' : ''}), tier=${tier}`);
|
||||
// The runner is the CLI module; load it lazily so this library never imports it statically (no cycle).
|
||||
const { runPaidShards } = await import('../test-paid-shards');
|
||||
const shardOptions = { withinShardConcurrency: options.withinShardConcurrency, timeoutMs: options.timeoutMs, rootDir, log, commandFor: options.commandFor,
|
||||
env: { ...(options.env ?? process.env), EVALS: '1', EVALS_TIER: tier, EVALS_PREFLIGHT_OK: '1', EVALS_ALL: '1',
|
||||
...paidSelectionEnv('full', { e2e: [id], judges: [] }, `--case ${id}`) } };
|
||||
|
||||
@@ -0,0 +1,520 @@
|
||||
/**
|
||||
* Paid-lane selection and shard planning: tier classification, diff/PR case selection, shard keys, budgets and
|
||||
* supervision bounds. Moved from scripts/test-paid-shards.ts; imports only leaf modules (no import cycle).
|
||||
*/
|
||||
import * as fs from 'node:fs';
|
||||
import * as path from 'node:path';
|
||||
import { spawnSync } from 'node:child_process';
|
||||
import { normalizeRelativePath } from './shard-engine';
|
||||
import { isPaidTestFile } from '../../test/helpers/paid-test-set';
|
||||
import { PERIODIC_CI_EXCLUDE } from '../../test/helpers/periodic-exclude-data';
|
||||
import { FILE_RETRY_BUDGETS } from '../../test/helpers/eval-budgets';
|
||||
import { OVERLAY_MIN_FILE_WALL_MS } from '../../test/helpers/overlay-case-policy';
|
||||
import { packageChangeOnlyVersion, prProfileFileCases, selectPrProfile, type PrProfileSelection } from '../test-pr-profile';
|
||||
import { detectBaseBranch, getChangedFiles, selectTests, E2E_TOUCHFILES, E2E_TIERS, LLM_JUDGE_TOUCHFILES, GLOBAL_TOUCHFILES } from '../../test/helpers/touchfiles';
|
||||
import { caseTestNamePattern, fileCaseRegistration, shardCaseId, shardFile, shardTrial } from './paid-cases';
|
||||
import { DEFAULT_MAX_FILES_PER_SHARD, DEFAULT_SHARD_TIMEOUT_MS, DEFAULT_WITHIN_SHARD_CONCURRENCY, PAID_TIERS, ROOT, isOverlayTestFile,
|
||||
type PaidCaseSelection, type PaidProfile, type PaidShardBudget, type PaidTier, type ShardOutcome } from './paid-types';
|
||||
|
||||
/** Compatibility helper for callers that only need the effective wall. */
|
||||
export function resolvePaidShardTimeoutMs(files: string[], explicitTimeoutMs?: number): number {
|
||||
return resolvePaidShardBudget(files, explicitTimeoutMs).timeoutMs;
|
||||
}
|
||||
|
||||
export function collectPaidTestFiles(rootDir = ROOT): string[] {
|
||||
const testDir = path.join(rootDir, 'test');
|
||||
if (!fs.existsSync(testDir)) return [];
|
||||
return fs.readdirSync(testDir)
|
||||
.map((name) => `test/${name}`)
|
||||
.filter(isPaidTestFile)
|
||||
.sort();
|
||||
}
|
||||
|
||||
export interface TierClassification {
|
||||
included: boolean;
|
||||
reason: string;
|
||||
}
|
||||
|
||||
/**
|
||||
* Decide whether a paid test file has anything to run in `tier`.
|
||||
*
|
||||
* Per-TEST tier filtering already happens at runtime: test/helpers/e2e-helpers.ts
|
||||
* intersects the selected tests with E2E_TIERS whenever EVALS_TIER is set, and
|
||||
* this runner passes EVALS_TIER down to every shard. So this file-level pass is
|
||||
* only an optimization — skipping a file merely saves one near-instant shard.
|
||||
*
|
||||
* Exclusion is the dangerous direction (a wrongly-skipped gate test is exactly
|
||||
* the invisible-non-execution bug this runner exists to kill), so the only
|
||||
* exclusion evidence accepted is an explicit whole-file tier guard: either the
|
||||
* raw `EVALS_TIER === '<other>'` predicate or the consolidated helper form
|
||||
* `describeE2ETier('<other>')` / `e2eTierEnabled('<other>')` from
|
||||
* test/helpers/e2e-gate.ts (same semantics, read from env at module load).
|
||||
* Inferring a file's tier from which E2E_TIERS names appear in its source
|
||||
* is guesswork that silently drops real work: short keys like 'retro' match
|
||||
* unrelated strings, and LLM-judge tests are keyed off LLM_JUDGE_TOUCHFILES and
|
||||
* carry no E2E_TIERS name at all. Everything without an explicit other-tier
|
||||
* guard runs and self-skips.
|
||||
*/
|
||||
export function classifyPaidTestFile(source: string, tier: PaidTier): TierClassification {
|
||||
const declares = (candidate: PaidTier) =>
|
||||
new RegExp(`EVALS_TIER\\s*===\\s*['"\`]${candidate}['"\`]`).test(source) ||
|
||||
new RegExp(`\\b(?:describeE2ETier|e2eTierEnabled)\\(\\s*['"\`]${candidate}['"\`]`).test(source);
|
||||
|
||||
if (declares(tier)) return { included: true, reason: `declares tier '${tier}'` };
|
||||
const others = PAID_TIERS.filter(candidate => candidate !== tier && declares(candidate));
|
||||
if (others.length) return { included: false, reason: `declares tier ${others.map(other => `'${other}'`).join(' and ')} only` };
|
||||
return { included: true, reason: 'no whole-file tier guard — runtime E2E_TIERS filter decides' };
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* A file is skipped for a tier lane only when it registers E2E ids, none of
|
||||
* them has that tier, and no literal registration names an id outside its
|
||||
* touchfile registration. A computed registration (a name built at runtime)
|
||||
* can only run in this lane by producing an id of this tier (the child's
|
||||
* EVALS_TIER filter drops every other), so it is skipped too when every id of
|
||||
* the tier is registered to a test file (its touchfile list names one), which
|
||||
* is not this file. An id of the tier that no test file registers, or no id
|
||||
* at all, keeps today's scheduling (the child's runtime filter decides).
|
||||
*/
|
||||
export function tierSkipReason(
|
||||
file: string, source: string, tier: PaidTier,
|
||||
touchfiles: Record<string, string[]> = E2E_TOUCHFILES,
|
||||
tiers: Record<string, string> = E2E_TIERS,
|
||||
): string | null {
|
||||
const { registered, computed, unregistered } = fileCaseRegistration(file, source, touchfiles, tiers);
|
||||
if (registered.length === 0 || unregistered.length > 0 || registered.some(id => tiers[id] === tier)) return null;
|
||||
if (!computed) return `skipped: no E2E_TIERS id has tier ${tier}`;
|
||||
const unowned = Object.keys(tiers).some(id => tiers[id] === tier && !(touchfiles[id] ?? []).some(dep => /^test\/[^/]+\.test\.ts$/.test(dep)));
|
||||
return unowned ? null : `skipped: no E2E_TIERS id has tier ${tier} (its computed names can only produce ${tier} ids other test files register)`;
|
||||
}
|
||||
|
||||
/**
|
||||
* The marathon lane selects positively: a file runs there only when it
|
||||
* declares the marathon tier or registers a marathon-tier case. When its source
|
||||
* names some registered ids, only those count: another case's touchfile entry
|
||||
* is a dependency, not a registration (W2f:
|
||||
* plan-decision-classification is a dependency of the marathon
|
||||
* plan-ceo-split-overflow case and planned a hollow marathon shard). Files
|
||||
* without marathon work never cost a marathon runner, and gate/periodic files
|
||||
* never gain a third execution.
|
||||
*/
|
||||
export function marathonSkipReason(
|
||||
file: string, source: string,
|
||||
touchfiles: Record<string, string[]> = E2E_TOUCHFILES,
|
||||
tiers: Record<string, string> = E2E_TIERS,
|
||||
): string | null {
|
||||
if (classifyPaidTestFile(source, 'marathon').reason === "declares tier 'marathon'") return null;
|
||||
const { registered } = fileCaseRegistration(file, source, touchfiles, tiers);
|
||||
// When the source names some of its registered ids, those are its own cases; the rest are cases that merely depend on it.
|
||||
const named = registered.filter(id => knownTestNamesInSource(source, [id]).length > 0);
|
||||
return (named.length ? named : registered).some(id => tiers[id] === 'marathon')
|
||||
? null : 'skipped: declares no marathon tier and registers no marathon case';
|
||||
}
|
||||
|
||||
export interface TierSelection {
|
||||
selected: string[];
|
||||
excluded: Array<{ file: string; reason: string }>;
|
||||
}
|
||||
|
||||
export function selectPaidTestFiles(files: string[], tier: PaidTier, rootDir = ROOT): TierSelection {
|
||||
const selected: string[] = [];
|
||||
const excluded: Array<{ file: string; reason: string }> = [];
|
||||
// Scheduled-lane exclusions (documented-red / manual-hardware files): a
|
||||
// known-red weekly shard is triage waste locally AND in CI, so the list
|
||||
// applies to every periodic and marathon run, with the reason surfaced per file.
|
||||
const ciExcluded = (file: string): { reason: string; tracking: string } | undefined =>
|
||||
tier !== 'gate' ? PERIODIC_CI_EXCLUDE[normalizeRelativePath(file)] : undefined;
|
||||
for (const file of files) {
|
||||
const exclusion = ciExcluded(file);
|
||||
if (exclusion) {
|
||||
excluded.push({ file, reason: `excluded: ${exclusion.reason} [${exclusion.tracking}]` });
|
||||
continue;
|
||||
}
|
||||
const source = fs.readFileSync(path.join(rootDir, file), 'utf8');
|
||||
const classification = classifyPaidTestFile(source, tier);
|
||||
const skip = !classification.included ? null
|
||||
: tier === 'marathon' ? marathonSkipReason(file, source) : tierSkipReason(file, source, tier);
|
||||
if (classification.included && !skip) selected.push(file);
|
||||
else excluded.push({ file, reason: skip ?? classification.reason });
|
||||
}
|
||||
return { selected, excluded };
|
||||
}
|
||||
|
||||
// --- Parent-side diff selection (shard skipping) ---
|
||||
|
||||
/**
|
||||
* The test names the parent mapper recognizes: every E2E map key. LLM-judge
|
||||
* keys are deliberately excluded — skill-llm-eval.test.ts is not a
|
||||
* skill-e2e-* file, so it is always kept (child self-skip authoritative).
|
||||
*/
|
||||
export const PARENT_MAPPER_TEST_NAMES: string[] = [
|
||||
...new Set([...Object.keys(E2E_TOUCHFILES), ...Object.keys(E2E_TIERS)]),
|
||||
];
|
||||
|
||||
/**
|
||||
* Which of `names` appear in `source` as a quoted string ('x', "x", or `x`).
|
||||
* Same class of detection test/e2e-tier-alignment.test.ts uses: exact
|
||||
* quote-delimited match, raw source (comments count — a false hit can only
|
||||
* KEEP a shard, and the registration union below covers constructed names).
|
||||
*/
|
||||
export function knownTestNamesInSource(source: string, names: Iterable<string>): string[] {
|
||||
const hits: string[] = [];
|
||||
for (const name of names) {
|
||||
if (
|
||||
source.includes(`'${name}'`)
|
||||
|| source.includes(`"${name}"`)
|
||||
|| source.includes(`\`${name}\``)
|
||||
) hits.push(name);
|
||||
}
|
||||
return hits;
|
||||
}
|
||||
|
||||
export interface PaidDiffSelection {
|
||||
/** null = run everything (EVALS_ALL, or no changes vs base). */
|
||||
selectedNames: Set<string> | null;
|
||||
reason: string;
|
||||
totalTests: number;
|
||||
}
|
||||
|
||||
/**
|
||||
* Compute diff selection in the PARENT, mirroring the module-scope selection
|
||||
* block in test/helpers/e2e-helpers.ts exactly: EVALS_ALL → run all;
|
||||
* base = EVALS_BASE || detectBaseBranch || 'main'; empty changed-file union →
|
||||
* run all. (e2e-helpers additionally gates on EVALS=1, which this runner sets
|
||||
* for every child unconditionally, so the parent mirror omits it.)
|
||||
*
|
||||
* getChangedFiles THROWS on git errors (fail-closed) — the children would hit
|
||||
* the same throw at module load, so the parent surfaces it before any shard
|
||||
* spawns.
|
||||
*/
|
||||
export function computePaidDiffSelection(
|
||||
env: NodeJS.ProcessEnv = process.env,
|
||||
rootDir = ROOT,
|
||||
): PaidDiffSelection {
|
||||
const totalTests = Object.keys(E2E_TOUCHFILES).length;
|
||||
if (env.EVALS_ALL) {
|
||||
return { selectedNames: null, reason: 'run-all (EVALS_ALL=1)', totalTests };
|
||||
}
|
||||
const baseBranch = env.EVALS_BASE || detectBaseBranch(rootDir) || 'main';
|
||||
const changedFiles = getChangedFiles(baseBranch, rootDir);
|
||||
if (changedFiles.length === 0) {
|
||||
return { selectedNames: null, reason: `run-all (no changes vs ${baseBranch})`, totalTests };
|
||||
}
|
||||
const selection = selectTests(changedFiles, E2E_TOUCHFILES, GLOBAL_TOUCHFILES, {
|
||||
baseRef: baseBranch, cwd: rootDir,
|
||||
});
|
||||
return { selectedNames: new Set(selection.selected), reason: selection.reason, totalTests };
|
||||
}
|
||||
|
||||
/**
|
||||
* Serialize the parent's diff selection for shard children (EVALS_SELECTION_JSON).
|
||||
*
|
||||
* Children's e2e-helpers module-load path adopts this instead of re-deriving
|
||||
* the selection per shard — which, when touchfiles-data.ts is in the diff,
|
||||
* spawned one bun subprocess PER CHILD to evaluate the old data file (the
|
||||
* map-diff path in test/helpers/test-selection.ts, 20s timeout each; 46-68
|
||||
* redundant children per full run). `selected: null` means run-all, mirroring
|
||||
* PaidDiffSelection.selectedNames. The child-side parser lives in
|
||||
* test/helpers/e2e-helpers.ts (parseEvalsSelectionJson); round-trip parity is
|
||||
* pinned by test/paid-selection-propagation.test.ts.
|
||||
*/
|
||||
export function serializePaidDiffSelection(selection: PaidDiffSelection): string {
|
||||
return JSON.stringify({
|
||||
version: 1,
|
||||
selected: selection.selectedNames === null ? null : [...selection.selectedNames].sort(),
|
||||
reason: selection.reason,
|
||||
});
|
||||
}
|
||||
|
||||
/** Both selectors are computed once; execution consumes the exact persisted IDs. */
|
||||
export function computePaidCaseSelection(options: {
|
||||
profile: PaidProfile;
|
||||
env?: NodeJS.ProcessEnv;
|
||||
rootDir?: string;
|
||||
changedFiles?: string[];
|
||||
/** Whether package.json differs from the base only in `version`; computed from git when omitted. */
|
||||
packageVersionOnly?: boolean;
|
||||
}): { selection: PaidCaseSelection; reason: string; coverage?: PrProfileSelection } {
|
||||
const env = options.env ?? process.env;
|
||||
const rootDir = options.rootDir ?? ROOT;
|
||||
const baseRef = env.EVALS_BASE || detectBaseBranch(rootDir) || 'main';
|
||||
const files = options.changedFiles ?? (env.EVALS_ALL ? [] : getChangedFiles(baseRef, rootDir));
|
||||
const all = !!env.EVALS_ALL || files.length === 0;
|
||||
const effectiveFiles = files.filter(file => options.profile !== 'pr' || file !== 'package.json' ||
|
||||
!(options.packageVersionOnly ?? packageVersionOnlySinceBase(rootDir, baseRef)));
|
||||
const sourceAliases = options.profile === 'pr' ? existingPromptSourceAliases(effectiveFiles, rootDir) : {};
|
||||
const selectionFiles = [...new Set([...effectiveFiles, ...Object.values(sourceAliases)])];
|
||||
const select = (table: Record<string, string[]>) => all ? null
|
||||
: selectTests(selectionFiles, table, GLOBAL_TOUCHFILES, { baseRef, cwd: rootDir }).selected;
|
||||
const selection = { e2e: select(E2E_TOUCHFILES), judges: select(LLM_JUDGE_TOUCHFILES) };
|
||||
if (options.profile === 'full') return { selection, reason: all ? 'run-all' : 'diff' };
|
||||
const coverage = selectPrProfile({ selectedE2E: selection.e2e, selectedJudges: selection.judges, changedFiles: effectiveFiles, sourceAliases });
|
||||
if (coverage.needsFullValidation) {
|
||||
throw new Error(`PR profile requires full validation: ${coverage.missingCoverage.join(', ')}. Use --profile full and the relevant periodic cases.`);
|
||||
}
|
||||
return { selection: { e2e: coverage.e2e, judges: coverage.judges }, reason: coverage.reasons.join('; '), coverage };
|
||||
}
|
||||
|
||||
export function existingPromptSourceAliases(files: readonly string[], rootDir = ROOT): Record<string, string> {
|
||||
const aliases: Record<string, string> = {};
|
||||
for (const file of files) {
|
||||
if (!file.endsWith('.md')) continue;
|
||||
const template = `${file}.tmpl`;
|
||||
try { if (fs.statSync(path.join(rootDir, template)).isFile()) aliases[file] = template; }
|
||||
catch { /* Unknown/generated-only content must keep its own dependency identity. */ }
|
||||
}
|
||||
return aliases;
|
||||
}
|
||||
|
||||
function packageVersionOnlySinceBase(rootDir: string, baseRef: string): boolean {
|
||||
try {
|
||||
const options = { cwd: rootDir, encoding: 'utf8' as const, timeout: 10_000, maxBuffer: 1024 * 1024 };
|
||||
const base = spawnSync('git', ['merge-base', baseRef, 'HEAD'], options);
|
||||
const sha = base.stdout?.trim() ?? '';
|
||||
if (base.status !== 0 || !/^[a-f0-9]{40,64}$/.test(sha)) return false;
|
||||
const old = spawnSync('git', ['show', `${sha}:package.json`], options);
|
||||
return old.status === 0 && packageChangeOnlyVersion(old.stdout, fs.readFileSync(path.join(rootDir, 'package.json'), 'utf8'));
|
||||
} catch { return false; }
|
||||
}
|
||||
|
||||
/** Only audited per-case files, plus the separately selected judge, enter the fast profile. */
|
||||
/** The selected PR-profile case ids a shard key owns (a case key owns at most its own case). */
|
||||
/** `exclude`: isolated case ids a file shard leaves to their trial shards. */
|
||||
export function prProfileShardIds(key: string, selection: PaidCaseSelection, exclude: readonly string[] = []): string[] {
|
||||
const caseId = shardCaseId(key);
|
||||
return prProfileFileCases(shardFile(key), selection.e2e)
|
||||
.filter(id => (caseId === null || id === caseId) && (selection.e2e === null || selection.e2e.includes(id)) && !exclude.includes(id));
|
||||
}
|
||||
|
||||
export function prProfileFileSelected(file: string, selection: PaidCaseSelection, exclude: readonly string[] = []): boolean {
|
||||
if (file === 'test/skill-llm-eval.test.ts') return selection.judges === null || selection.judges.length > 0;
|
||||
return prProfileShardIds(file, selection, exclude).length > 0;
|
||||
}
|
||||
|
||||
export function expectedPrCaseCount(file: string, selection: PaidCaseSelection, exclude: readonly string[] = []): number {
|
||||
if (file === 'test/skill-llm-eval.test.ts') return selection.judges?.length ?? Object.keys(LLM_JUDGE_TOUCHFILES).length;
|
||||
return prProfileShardIds(file, selection, exclude).length;
|
||||
}
|
||||
|
||||
export function prProfileTestNamePattern(file: string, selection: PaidCaseSelection, exclude: readonly string[] = []): string {
|
||||
const ids = file === 'test/skill-llm-eval.test.ts'
|
||||
? selection.judges ?? Object.keys(LLM_JUDGE_TOUCHFILES)
|
||||
: prProfileShardIds(file, selection, exclude);
|
||||
if (ids.length === 0) throw new Error(`No selected PR cases for ${file}`);
|
||||
return caseTestNamePattern(ids);
|
||||
}
|
||||
|
||||
export function paidSelectionEnv(profile: PaidProfile, selection: PaidCaseSelection, reason: string): NodeJS.ProcessEnv {
|
||||
const encode = (selected: string[] | null) => JSON.stringify({ version: 1, selected, reason });
|
||||
return { EVALS_PROFILE: profile, EVALS_SELECTION_JSON: encode(selection.e2e), EVALS_JUDGE_SELECTION_JSON: encode(selection.judges) };
|
||||
}
|
||||
|
||||
export interface ShardSkipDecision {
|
||||
file: string;
|
||||
kept: boolean;
|
||||
reason: string;
|
||||
}
|
||||
|
||||
export interface DiffSkipOptions {
|
||||
rootDir?: string;
|
||||
/** Injectable for tests. Throwing reads fail OPEN (shard kept). */
|
||||
readSource?: (file: string) => string;
|
||||
/** Injectable name census (default: PARENT_MAPPER_TEST_NAMES). */
|
||||
allNames?: string[];
|
||||
/** Injectable registration map (default: E2E_TOUCHFILES). */
|
||||
e2eTouchfiles?: Record<string, string[]>;
|
||||
/** File shard -> isolated case ids its trial shards run instead. */
|
||||
excludeCases?: Record<string, string[]>;
|
||||
}
|
||||
|
||||
/**
|
||||
* Decide whether a paid test file can be skipped under the current diff
|
||||
* selection. A file's MAPPED names are the union of:
|
||||
* - E2E map keys quoted in its source, and
|
||||
* - E2E map keys whose dep list registers the file (the tier-alignment
|
||||
* mapping) — this covers files whose testNames are constructed rather
|
||||
* than literal.
|
||||
*
|
||||
* FAIL-OPEN by construction: run-all selection, non-skill-e2e paid files
|
||||
* (llm-judge / codex-e2e / routing, keyed off other maps),
|
||||
* unreadable sources, and files with zero mapped names all KEEP their shard —
|
||||
* the child's self-skip stays authoritative. A parent bug may only run
|
||||
* extra work, never drop it.
|
||||
*/
|
||||
export function diffSkipDecisionForFile(
|
||||
file: string,
|
||||
selectedNames: Set<string> | null,
|
||||
options: DiffSkipOptions = {},
|
||||
): ShardSkipDecision {
|
||||
if (selectedNames === null) return { file, kept: true, reason: 'run-all selection' };
|
||||
const caseId = shardCaseId(file);
|
||||
if (caseId !== null) {
|
||||
return selectedNames.has(caseId) ? { file, kept: true, reason: `selected: ${caseId}` } : { file, kept: false, reason: `case ${caseId} not selected` };
|
||||
}
|
||||
const rel = normalizeRelativePath(file);
|
||||
if (!/^test\/skill-e2e-.*\.test\.ts$/.test(rel)) {
|
||||
return { file, kept: true, reason: 'non-skill-e2e paid file — child self-skip authoritative' };
|
||||
}
|
||||
let source: string;
|
||||
try {
|
||||
const read = options.readSource
|
||||
?? ((f: string) => fs.readFileSync(path.join(options.rootDir ?? ROOT, f), 'utf8'));
|
||||
source = read(file);
|
||||
} catch {
|
||||
return { file, kept: true, reason: 'source unreadable — fail-open' };
|
||||
}
|
||||
const allNames = options.allNames ?? PARENT_MAPPER_TEST_NAMES;
|
||||
const touchfiles = options.e2eTouchfiles ?? E2E_TOUCHFILES;
|
||||
const quoted = knownTestNamesInSource(source, allNames);
|
||||
const registered = Object.keys(touchfiles).filter((k) => touchfiles[k].includes(rel));
|
||||
const isolated = options.excludeCases?.[rel] ?? [];
|
||||
const mapped = [...new Set([...quoted, ...registered])].filter(name => !isolated.includes(name));
|
||||
if (mapped.length === 0) {
|
||||
return { file, kept: true, reason: 'no mappable test names — fail-open, child self-skip authoritative' };
|
||||
}
|
||||
const selectedHere = mapped.filter((n) => selectedNames.has(n));
|
||||
if (selectedHere.length > 0) {
|
||||
const shown = selectedHere.slice(0, 3).join(', ') + (selectedHere.length > 3 ? ', …' : '');
|
||||
return { file, kept: true, reason: `selected: ${shown}` };
|
||||
}
|
||||
return { file, kept: false, reason: `none of its ${mapped.length} mapped test(s) selected` };
|
||||
}
|
||||
|
||||
/**
|
||||
* Partition planned shards into runnable vs skipped-by-diff. A shard is
|
||||
* skipped only when EVERY file in it is skippable.
|
||||
*/
|
||||
export function partitionShardsByDiffSelection(
|
||||
shards: string[][],
|
||||
selectedNames: Set<string> | null,
|
||||
options: DiffSkipOptions = {},
|
||||
): { runnable: string[][]; skipped: Array<{ files: string[]; reason: string }> } {
|
||||
if (selectedNames === null) return { runnable: shards, skipped: [] };
|
||||
const runnable: string[][] = [];
|
||||
const skipped: Array<{ files: string[]; reason: string }> = [];
|
||||
for (const shard of shards) {
|
||||
const decisions = shard.map((file) => diffSkipDecisionForFile(file, selectedNames, options));
|
||||
if (decisions.every((d) => !d.kept)) {
|
||||
skipped.push({ files: shard, reason: [...new Set(decisions.map((d) => d.reason))].join('; ') });
|
||||
} else {
|
||||
runnable.push(shard);
|
||||
}
|
||||
}
|
||||
return { runnable, skipped };
|
||||
}
|
||||
|
||||
export function planPaidShards(
|
||||
files: string[],
|
||||
options: { maxFilesPerShard?: number; ownShard?: ReadonlySet<string> } = {},
|
||||
): string[][] {
|
||||
const size = Math.max(1, options.maxFilesPerShard ?? DEFAULT_MAX_FILES_PER_SHARD);
|
||||
const unique = [...new Set(files.map(normalizeRelativePath))].sort();
|
||||
const shards: string[][] = [];
|
||||
let pending: string[] = [];
|
||||
for (const file of unique) {
|
||||
if (isOverlayTestFile(file) || shardCaseId(file) !== null || FILE_RETRY_BUDGETS.some(budget => budget.file === shardFile(file))
|
||||
|| options.ownShard?.has(file)) {
|
||||
if (pending.length) shards.push(pending);
|
||||
pending = [];
|
||||
shards.push([file]);
|
||||
} else {
|
||||
pending.push(file);
|
||||
if (pending.length === size) { shards.push(pending); pending = []; }
|
||||
}
|
||||
}
|
||||
if (pending.length) shards.push(pending);
|
||||
return shards;
|
||||
}
|
||||
|
||||
|
||||
/** Explicit caller limits win; registered supervision preserves existing attempts. */
|
||||
export function resolvePaidShardBudget(files: string[], overrideMs?: number): PaidShardBudget {
|
||||
const finding = FILE_RETRY_BUDGETS.find(budget => files.map(shardFile).includes(budget.file));
|
||||
if (finding && files.length !== 1) throw new Error('Registered retry budget requires its own shard');
|
||||
if (overrideMs !== undefined && (!Number.isSafeInteger(overrideMs) || overrideMs <= 0 || overrideMs > 2_147_483_647)) {
|
||||
throw new Error('Shard timeout must be a finite positive timer-safe integer');
|
||||
}
|
||||
const overlay = files.some(isOverlayTestFile);
|
||||
if (overlay && files.length !== 1) throw new Error('Overlay budget requires its own shard');
|
||||
if (overlay && overrideMs !== undefined && overrideMs < OVERLAY_MIN_FILE_WALL_MS) {
|
||||
throw new Error(`Overlay shard requires at least ${OVERLAY_MIN_FILE_WALL_MS}ms; explicit wall ${overrideMs}ms cannot preserve its work and finalization budget`);
|
||||
}
|
||||
// A registered file's case shard supervises its one case.
|
||||
const registeredMs = finding && shardCaseId(files[0]!) !== null
|
||||
? finding.caseMs + finding.shardReserveMs : finding?.shardMs;
|
||||
return {
|
||||
timeoutMs: overrideMs ?? (registeredMs ?? (overlay ? OVERLAY_MIN_FILE_WALL_MS : DEFAULT_SHARD_TIMEOUT_MS)),
|
||||
source: overrideMs !== undefined ? 'explicit' : finding ? 'registered' : 'default',
|
||||
policyId: finding?.id ?? null,
|
||||
};
|
||||
}
|
||||
|
||||
export function sameBudget(actual: PaidShardBudget | undefined, expected: PaidShardBudget): boolean {
|
||||
return actual?.timeoutMs === expected.timeoutMs && actual.source === expected.source && actual.policyId === expected.policyId;
|
||||
}
|
||||
|
||||
export function buildPaidShardArgs(
|
||||
files: string[],
|
||||
timeoutMs: number,
|
||||
maxConcurrency: number = DEFAULT_WITHIN_SHARD_CONCURRENCY,
|
||||
retries?: number,
|
||||
): string[] {
|
||||
// Explicit --concurrent/--max-concurrency: the legacy path always set one;
|
||||
// omitting it here made within-shard parallelism differ silently between
|
||||
// the two runners (observed: 1.6x sumdur/wall sharded vs 8x legacy).
|
||||
// Paid evals never retry (retriesForFiles); `--retry 0` is explicit so a
|
||||
// bunfig default can never reintroduce one.
|
||||
return ['test', ...files, '--retry', String(retries ?? 0), '--concurrent', `--max-concurrency=${maxConcurrency}`, `--timeout=${timeoutMs}`];
|
||||
}
|
||||
|
||||
/**
|
||||
* Stable per-shard eval-dir slug: test filename sans extension, sanitized.
|
||||
* Stable across runs so each shard baselines against its own prior run.
|
||||
*/
|
||||
export function shardSlug(files: string[]): string {
|
||||
return files
|
||||
.map((file) => path.basename(shardFile(file)).replace(/\.test\.(?:[cm]?[jt]s|tsx|jsx)$/, '')
|
||||
+ (shardCaseId(file) === null ? '' : `--${shardCaseId(file)}`)
|
||||
+ (shardTrial(file) === null ? '' : `.t${shardTrial(file)}`))
|
||||
.join('+')
|
||||
.replace(/[^a-zA-Z0-9._+-]/g, '-');
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* True when a shard "passed" without verifying anything: every test bun ran
|
||||
* was a skip. Legitimate for external-service files on hosts without the
|
||||
* binary, but it must surface as a census warning, never read as coverage.
|
||||
*/
|
||||
export function isAllSkippedPass(outcome: Pick<ShardOutcome, 'status' | 'executedTests' | 'skippedTests'>): boolean {
|
||||
return outcome.status === 'passed'
|
||||
&& outcome.executedTests !== null
|
||||
&& outcome.executedTests > 0
|
||||
&& outcome.skippedTests === outcome.executedTests;
|
||||
}
|
||||
|
||||
/** Upper bound for one ordered FIFO group with the same admission limit.
|
||||
* At each launch the least-loaded worker has at most total prior work / jobs,
|
||||
* and at most floor(prior files / jobs) files of the largest prior wall.
|
||||
* Both bounds hold when earlier files finish below their ceilings. Overlay
|
||||
* groups must use their separate admission limit, as the runner does.
|
||||
*/
|
||||
export function paidShardWallUpperBoundMs(files: string[], jobs: number, overrideMs?: number): number {
|
||||
if (!Number.isSafeInteger(jobs) || jobs < 1) throw new Error('Worker count must be a positive integer');
|
||||
let priorWork = 0, priorLargest = 0, bound = 0;
|
||||
files.forEach((file, index) => {
|
||||
const wall = resolvePaidShardTimeoutMs([file], overrideMs);
|
||||
const start = Math.min(priorWork / jobs, Math.floor(index / jobs) * priorLargest);
|
||||
bound = Math.max(bound, start + wall);
|
||||
priorWork += wall;
|
||||
priorLargest = Math.max(priorLargest, wall);
|
||||
});
|
||||
return Math.ceil(bound);
|
||||
}
|
||||
|
||||
export function validatedProfile(value: string | undefined, source: string): PaidProfile {
|
||||
if (value === undefined || value === '') return 'full';
|
||||
if (value !== 'pr' && value !== 'full') throw new Error(`${source} must be pr or full. Received: ${value}`);
|
||||
return value;
|
||||
}
|
||||
@@ -0,0 +1,132 @@
|
||||
/**
|
||||
* Paid-lane shared types and constants: the leaf module every paid-lane file imports. Moved from scripts/test-paid-shards.ts
|
||||
* so scripts/lib/paid-*.ts never import the CLI module back (no import cycle).
|
||||
*/
|
||||
import * as path from 'node:path';
|
||||
import { normalizeRelativePath, type LanePolicy } from './shard-engine';
|
||||
import type { EvalCaseKind, PanelShape, TrialFailureCause, TrialFailureClass, TrialFailureDetail, TrialOutcome, TrialSessionSummary } from '../../test/helpers/eval-store';
|
||||
|
||||
export const ROOT = path.resolve(import.meta.dir, '..', '..');
|
||||
|
||||
export type PaidTier = 'gate' | 'periodic' | 'marathon';
|
||||
export const PAID_TIERS: readonly PaidTier[] = ['gate', 'periodic', 'marathon'];
|
||||
export type PaidProfile = 'pr' | 'full';
|
||||
|
||||
export interface PaidCaseSelection {
|
||||
e2e: string[] | null;
|
||||
judges: string[] | null;
|
||||
}
|
||||
|
||||
export const DEFAULT_TIER: PaidTier = 'gate';
|
||||
export const DEFAULT_SHARD_TIMEOUT_MS = 30 * 60_000;
|
||||
export const DEFAULT_MAX_FILES_PER_SHARD = 1;
|
||||
// 8 jobs × 2 within-shard ≈ 10-13 real in-flight sessions (39 of 75
|
||||
// skill-e2e files hold exactly ONE test, so within-shard concurrency is
|
||||
// dead weight for most shards) — under the documented-safe ~15 the legacy
|
||||
// 40-way runner established. The old 4×4 yielded only ~4-6 in-flight and a
|
||||
// 13-wave local gate worst case (~6.5h); 8×2 halves it. Watch the WS1
|
||||
// flake telemetry for sustained 429 storms across 2 PR cycles — that is
|
||||
// the rollback trigger. Prerequisite (landed): per-shard TMPDIR/
|
||||
// CHROMIUM_PROFILE isolation in runPaidShard.
|
||||
export const DEFAULT_JOBS = 8;
|
||||
export const DEFAULT_WITHIN_SHARD_CONCURRENCY = 2;
|
||||
|
||||
/**
|
||||
* Paid-lane classification policy. Seeds keep only positive walls (a zero
|
||||
* is not a real paid-shard measurement). A shard that passed with zero
|
||||
* executed tests is legitimate under selection (in-file diff/tier
|
||||
* self-skips) and only warns; under EVALS_ALL it is hollow: 'passed-empty'.
|
||||
*/
|
||||
export const PAID_LANE_POLICY: LanePolicy = {
|
||||
acceptsSeedDuration: (ms) => ms > 0,
|
||||
zeroExecution: ({ promisedAll }) => (promisedAll ? 'passed-empty' : 'passed-with-warning'),
|
||||
};
|
||||
|
||||
/** One overlay process preserves the original process-wide SDK semaphore. */
|
||||
export const OVERLAY_MAX_ACTIVE_SHARDS = 1;
|
||||
|
||||
export function isOverlayTestFile(file: string): boolean {
|
||||
return /^skill-e2e-overlay-harness-.+\.test\.ts$/.test(path.basename(normalizeRelativePath(file)));
|
||||
}
|
||||
|
||||
export interface PaidShardBudget {
|
||||
timeoutMs: number;
|
||||
source: 'explicit' | 'registered' | 'default';
|
||||
policyId: string | null;
|
||||
}
|
||||
|
||||
export type ShardStatus =
|
||||
| 'passed'
|
||||
| 'failed'
|
||||
| 'timed-out'
|
||||
| 'never-started'
|
||||
| 'skipped-by-diff'
|
||||
// exit 0 with ZERO executed tests on a run that promised everything
|
||||
// (EVALS_ALL): the hollow-file green the census backstop exists to catch.
|
||||
// Under selective runs, 0-executed passed shards stay 'passed' (in-file
|
||||
// diff/tier self-skips are legitimate there) and get a WARNING line only.
|
||||
| 'passed-empty';
|
||||
|
||||
export interface ShardOutcome {
|
||||
shard: number;
|
||||
files: string[];
|
||||
status: ShardStatus;
|
||||
exitCode: number | null;
|
||||
elapsedMs: number;
|
||||
groupPid: number | null;
|
||||
/** Tests bun reported executing ("Ran N tests ..."), null when unknown. */
|
||||
executedTests: number | null;
|
||||
/** Tests bun reported skipping (" N skip" count line), null when unknown.
|
||||
* "Ran N tests" COUNTS skips, so executedTests alone cannot distinguish a
|
||||
* shard that verified work from one whose every test self-skipped —
|
||||
* codex/gemini files green-by-skip on every CI runner (no binary) and the
|
||||
* weekly census read them as covered. */
|
||||
skippedTests: number | null;
|
||||
/** Effective supervised wall; absent only for unstarted or legacy outcomes. */
|
||||
budget?: PaidShardBudget;
|
||||
/** Present when a verified receipt replaced execution (PR lane only). */
|
||||
reused?: { inputKey: string; runId: string; revision: string; completedAt: number };
|
||||
/** The parent could not run the shard at all (a runner error, never a trial verdict). */
|
||||
runnerError?: string;
|
||||
/** PR lane: the reuse input identity of a freshly executed shard whose inputs stayed unchanged. */
|
||||
inputKey?: string;
|
||||
/** Isolated trial shards only: the trial record this shard produced. */
|
||||
trial?: ShardTrialRecord;
|
||||
/** The slice's in-process deadline (job ceiling minus the upload reserve) decided this outcome:
|
||||
* `not_run` = never started (INFRA), `hung` = killed in flight (TIMEOUT). */
|
||||
sliceDeadline?: 'not_run' | 'hung';
|
||||
}
|
||||
|
||||
/**
|
||||
* One isolated trial's record, derived from its shard status and the records
|
||||
* in its own eval dir. `outcome` null means the harness produced no trial
|
||||
* (never started, hollow, isolation broken, runner error): the panel is then
|
||||
* INCOMPLETE and the slice exits non-zero. A failed, timed-out or crashed
|
||||
* trial is a trial verdict; the slice still exits zero and the report decides.
|
||||
*/
|
||||
export interface ShardTrialRecord {
|
||||
case: string;
|
||||
trial: number;
|
||||
kind: EvalCaseKind;
|
||||
panel: PanelShape;
|
||||
quarantined: boolean;
|
||||
outcome: TrialOutcome | null;
|
||||
harness?: string;
|
||||
failure_class?: TrialFailureClass;
|
||||
exit_reason?: string;
|
||||
error?: string;
|
||||
timeout_at_turn?: number;
|
||||
cost_usd: number;
|
||||
duration_ms: number;
|
||||
model?: string;
|
||||
failure_cause?: TrialFailureCause;
|
||||
failure_cause_evidence?: string;
|
||||
failure_detail?: TrialFailureDetail;
|
||||
sessions?: TrialSessionSummary[];
|
||||
cost_known?: false;
|
||||
}
|
||||
|
||||
/** Upload reserve between a slice's in-process deadline and its CI job ceiling (ENG-2). */
|
||||
export const SLICE_UPLOAD_RESERVE_MS = 5 * 60_000;
|
||||
/** Env var the executor workflows set at their first step: the job's start in epoch seconds. */
|
||||
export const SLICE_JOB_STARTED_AT_ENV = 'GSTACK_SLICE_JOB_STARTED_AT';
|
||||
@@ -713,17 +713,30 @@ export function strictShardStatus(input: {
|
||||
? 'passed' : 'failed';
|
||||
}
|
||||
|
||||
/** Usage text for `--help`: the lane's own text, else its declared flags. */
|
||||
export function cliUsage(handlers: Record<string, unknown>, usage?: string, script = process.argv[1] ?? 'runner'): string {
|
||||
return usage ?? [`Usage: bun run ${path.relative(process.cwd(), script) || script} [flags]`, '', 'Flags:',
|
||||
...Object.keys(handlers).sort().map(flag => ` ${flag}`), '', 'See docs/TESTING_INTERNALS.md for what each flag does.'].join('\n');
|
||||
}
|
||||
|
||||
/**
|
||||
* Shared flag loop. Each lane declares its flags; a handler that takes a
|
||||
* value calls `next()` (the following argv entry, or undefined) and owns its
|
||||
* own validation message. Any undeclared flag is `Unknown argument: <flag>`.
|
||||
* `--help` / `-h` (unless a lane declares them) print the usage and exit 0
|
||||
* before any work starts, so asking for help never runs a suite.
|
||||
*/
|
||||
export function parseCliFlags(
|
||||
argv: string[],
|
||||
handlers: Record<string, (next: () => string | undefined) => void>,
|
||||
usage?: string,
|
||||
): void {
|
||||
for (let index = 0; index < argv.length; index += 1) {
|
||||
const arg = argv[index];
|
||||
if ((arg === '--help' || arg === '-h') && !Object.hasOwn(handlers, arg)) {
|
||||
process.stdout.write(`${cliUsage(handlers, usage)}\n`);
|
||||
process.exit(0);
|
||||
}
|
||||
const handler = Object.hasOwn(handlers, arg) ? handlers[arg] : undefined;
|
||||
if (!handler) throw new Error(`Unknown argument: ${arg}`);
|
||||
handler(() => argv[++index]);
|
||||
|
||||
@@ -0,0 +1,416 @@
|
||||
/**
|
||||
* Windows-safe curation for the free suite (`--windows-only`): POSIX-only
|
||||
* source patterns, explicit known-incompatible files, and force-includes for
|
||||
* pattern hits that do not apply. Pinned by test/test-free-shards.test.ts and
|
||||
* test/windows-native-workflows.test.ts.
|
||||
*/
|
||||
import * as fs from 'node:fs';
|
||||
import * as path from 'node:path';
|
||||
|
||||
const ROOT = path.resolve(import.meta.dir, '../..');
|
||||
|
||||
// POSIX-only patterns that indicate a test will fail on windows-latest no
|
||||
// matter how the runner shards. Codex's v1.18.0.0 review flagged the first
|
||||
// three as concrete examples in the existing free suite (test/ship-version-sync.test.ts:72,
|
||||
// test/helpers/providers/claude.ts:22, package.json:12). We scan the test's
|
||||
// own content here so the filter stays automatic as new tests land. The
|
||||
// "Windows-incompatible APIs" patterns at the bottom were added after the
|
||||
// first windows-free-tests CI run surfaced concrete failure modes.
|
||||
const WINDOWS_FRAGILE_PATTERNS: Array<{ pattern: RegExp; reason: string }> = [
|
||||
// Hardcoded POSIX shells / commands.
|
||||
{ pattern: /['"`]\/bin\/(?:ba)?sh/, reason: 'hardcoded /bin/sh or /bin/bash' },
|
||||
{ pattern: /spawnSync\(['"]sh['"],|spawn\(['"]sh['"],|exec\(['"]sh /, reason: 'spawn("sh", ...)' },
|
||||
{ pattern: /['"]bash -c['"]|['"]sh -c['"]/, reason: 'bash -c / sh -c' },
|
||||
{ pattern: /['"`]\/tmp\//, reason: 'raw /tmp/ path (use os.tmpdir())' },
|
||||
{ pattern: /['"]chmod\b/, reason: 'chmod shell command' },
|
||||
{ pattern: /['"]xargs\b/, reason: 'xargs pipeline' },
|
||||
{ pattern: /\bwhich claude\b/, reason: 'which claude (use Bun.which)' },
|
||||
// Windows-incompatible APIs.
|
||||
{ pattern: /\.mode\s*&\s*0o[0-7]+/, reason: 'POSIX file mode bitmask (mode & 0o600 etc — Windows fakes mode bits)' },
|
||||
{ pattern: /\.endsWith\(['"]\//, reason: 'hardcoded forward-slash path assertion (Windows uses \\\\)' },
|
||||
{ pattern: /['"]\.\/[a-zA-Z][^"']*['"]\)\s*\.\s*toBe\(true\)/, reason: 'forward-slash path comparison' },
|
||||
// Tests that spawn a bash shebang script in bin/ via spawnSync. Git Bash on
|
||||
// Windows can run `bash /path/to/script` but spawnSync(scriptPath, ...)
|
||||
// tries to execute the file directly via CreateProcess, which fails on the
|
||||
// shebang. The pattern matches `, 'bin'` as a path-join argument (closing
|
||||
// OR followed by another segment), which catches:
|
||||
// - path.join(ROOT, 'bin', 'script-name') — typical
|
||||
// - join(import.meta.dir, '..', 'bin', 'name') — destructured (diff-scope)
|
||||
// - path.join(ROOT, 'bin') — bare BIN constant (brain-sync)
|
||||
{ pattern: /,\s*['"]bin['"]\s*[,)]|['"]\.?\/?bin\/[a-z][\w-]+['"]/, reason: 'spawns bin/ shebang script (Windows CreateProcess does not parse shebangs)' },
|
||||
// Tests that spawn the browse server as a subprocess via `bun run server.ts`.
|
||||
// The Bun → server.ts → Playwright path is the same one that doesn't work
|
||||
// on Windows (PR #1238 windows-pty-bun-pty-fix). Tests typically set
|
||||
// BROWSE_HEADLESS_SKIP=1 to skip the browser launch but still need a working
|
||||
// server, which they don't get on Windows.
|
||||
{ pattern: /BROWSE_HEADLESS_SKIP|spawn\(\[['"]bun['"],\s*['"]run['"]/, reason: 'spawns the browse server subprocess (Bun-driven path is Windows-broken)' },
|
||||
];
|
||||
|
||||
// Explicit known-Windows-incompatible test files that don't fit a regex
|
||||
// pattern. Listed here with the precise reason. Prefer adding a pattern above
|
||||
// when possible; this list is for environment-/runtime-specific tests where
|
||||
// the failure mode is structural rather than detectable via source-file scan.
|
||||
export const KNOWN_WINDOWS_INCOMPATIBLE: Array<{ file: string; reason: string }> = [
|
||||
{
|
||||
file: 'test/qa-evidence-producer.test.ts',
|
||||
reason: 'executes the registered Linux native actor and its inotify observer; portable capture and Windows job behavior are covered by qa-evidence.test.ts',
|
||||
},
|
||||
{
|
||||
file: 'test/qa-functional-fixture.test.ts',
|
||||
reason: 'executes graceful POSIX signal cancellation; Bun on Windows uses TerminateProcess and cannot run the fixture SIGTERM cleanup handler',
|
||||
},
|
||||
{
|
||||
file: 'test/qa-functional-observer-atomic.test.ts',
|
||||
reason: 'exercises real Linux inotify inode and directory watches through libc.so.6; Windows has no equivalent kernel interface',
|
||||
},
|
||||
{
|
||||
file: 'test/docsync-report-interface.test.ts',
|
||||
reason: 'executes registered native documentation callbacks with their real Linux inotify write observer before the model boundary',
|
||||
},
|
||||
{
|
||||
file: 'test/setup-gbrain-fixture.test.ts',
|
||||
reason: 'the fixture invokes real POSIX detector/verifier helpers through executable shebang wrappers',
|
||||
},
|
||||
{
|
||||
file: 'test/hermetic-skills-seeding.test.ts',
|
||||
reason: 'seeds the POSIX PTY skill runtime, whose embedded shell paths require a POSIX temporary root',
|
||||
},
|
||||
{
|
||||
file: 'test/hermetic-wiring.test.ts',
|
||||
reason: 'its runtime contract check seeds the POSIX PTY skill runtime; the curated Windows lane does not run that harness',
|
||||
},
|
||||
{
|
||||
file: 'test/pty-workspace-trust.test.ts',
|
||||
reason: 'launches the POSIX PTY harness with a fake executable and bound skill runtime',
|
||||
},
|
||||
{
|
||||
file: 'test/host-config.test.ts',
|
||||
reason: 'asserts "claude" binary on PATH (only true when running inside Claude Code, not on bare CI runner)',
|
||||
},
|
||||
{
|
||||
file: 'browse/test/findport.test.ts',
|
||||
reason: 'asserts Bun.serve.stop() is fire-and-forget — Bun behavior differs on Windows for this polyfill',
|
||||
},
|
||||
// First full run of the expanded lane (v1.66, 13 → ~258 files) surfaced
|
||||
// seven POSIX-bound files the content patterns cannot see (their
|
||||
// POSIX-ness is what they TEST, or arrives via a variable). Receipts:
|
||||
// PR #2593 windows-free-tests run 31918591602.
|
||||
{
|
||||
file: 'test/codex-under-codex-detection.test.ts',
|
||||
reason: 'drives the rendered preflight bash under a hardcoded POSIX PATH (/usr/bin:/bin) — bash is unreachable through that PATH on Windows, so every case sees empty output (v1.67 windows lane run 95234224148)',
|
||||
},
|
||||
{
|
||||
file: 'test/regression-pr1169-build-app-sed.test.ts',
|
||||
reason: 'tests sed escape sequences in build-app.sh — sed/bash are the subject under test',
|
||||
},
|
||||
{
|
||||
file: 'test/setup-conductor-worktree.test.ts',
|
||||
reason: 'tests ln -snf symlink semantics in the setup script — POSIX ln is the subject under test',
|
||||
},
|
||||
{
|
||||
file: 'test/artifacts-init-migration.test.ts',
|
||||
reason: 'runs a bash migration script + jq against a scaffolded git state — POSIX toolchain paths break under cmd spawn',
|
||||
},
|
||||
{
|
||||
file: 'test/gstack-decision-semantic.test.ts',
|
||||
reason: 'installs a fake gbrain SHEBANG SHIM on PATH; Windows spawn cannot exec shebang scripts',
|
||||
},
|
||||
{
|
||||
file: 'test/question-log-hook.test.ts',
|
||||
reason: 'spawns the PostToolUse hook script (bash shebang) directly; Windows spawn cannot exec it',
|
||||
},
|
||||
{
|
||||
file: 'browse/test/browser-skills-e2e.test.ts',
|
||||
reason: 'asserts forward-slash tier paths (<repo>/browser-skills/) that resolve with backslashes on Windows',
|
||||
},
|
||||
{
|
||||
file: 'design/test/variants-retry-after.test.ts',
|
||||
reason: 'wall-clock retry-timing assertions — flaky on the slow windows-latest runner even with widened bounds',
|
||||
},
|
||||
// Round-2 census (PR #2593 run 31919227507) after the first seven:
|
||||
{
|
||||
file: 'test/skill-census.test.ts',
|
||||
reason: 'census walk throws at module load on Windows (skill-census.ts:63) — the skills-tree symlink layout needs Developer Mode that CI runners lack',
|
||||
},
|
||||
{
|
||||
file: 'browse/test/browser-manager-unit.test.ts',
|
||||
reason: 'wedges the shard to its wall deadline on windows-latest (in-flight at kill); needs a Windows repro to diagnose — macOS + Linux lanes cover the file',
|
||||
},
|
||||
// Round-3 census (PR #2593 run 31919871680): the round-2 wedge had been
|
||||
// TRUNCATING its shard, so these seven only surfaced once shard 2 completed.
|
||||
// All the same POSIX-environment classes: PID/cmdline identity probing,
|
||||
// bash scripts as the subject under test, env-scrubbed child spawns.
|
||||
{
|
||||
file: 'browse/test/server-embedder-terminal-port.test.ts',
|
||||
reason: 'identity-based terminal-agent kill probes PID/cmdline with POSIX semantics; teardown asserts fail on windows-latest',
|
||||
},
|
||||
{
|
||||
file: 'design/test/daemon-discovery.test.ts',
|
||||
reason: 'verifyIdentity matches a spawned daemon via /proc-style cmdline probing — POSIX identity semantics',
|
||||
},
|
||||
{
|
||||
file: 'test/context-save-hardening.test.ts',
|
||||
reason: 'bash context-save/migration scripts (HOME-unset semantics, random-suffix path) are the subject under test',
|
||||
},
|
||||
{
|
||||
file: 'test/eval-list-cli.test.ts',
|
||||
reason: 'spawns the eval:list CLI via bun with a constructed env — bun resolution fails under Windows spawn',
|
||||
},
|
||||
{
|
||||
file: 'test/memory-cache-injection.test.ts',
|
||||
reason: 'exercises hook/deny-enforcement shell scripts — POSIX toolchain is the subject under test',
|
||||
},
|
||||
{
|
||||
file: 'test/migrations-v1.65.0.0.test.ts',
|
||||
reason: 'bash migration script (bunx re-fetch, .done markers) is the subject under test',
|
||||
},
|
||||
{
|
||||
file: 'test/question-preference-hook.test.ts',
|
||||
reason: 'spawns the PreToolUse preference hook (shebang script) directly; Windows spawn cannot exec it',
|
||||
},
|
||||
// Round-4 census (PR #2593 run 31920052810): unhandled errors with no
|
||||
// (fail) lines — attributed statically (the lane had no log artifact yet).
|
||||
{
|
||||
file: 'browse/test/browser-skill-commands.test.ts',
|
||||
reason: 'spawnSkill spawns bun with a constructed env — bun resolution fails under Windows spawn (unhandled, no (fail) line)',
|
||||
},
|
||||
{
|
||||
file: 'browse/test/security-audit-r2.test.ts',
|
||||
reason: 'symlink-attack fixtures (evil-link) need Developer Mode CI runners lack; expect(toThrow) fires unhandled on Windows',
|
||||
},
|
||||
// CSO comprehensive execution is qualified only for Linux containers behind
|
||||
// the POSIX watchdog and Unix-domain registry broker. Keep the portable
|
||||
// static/parser contracts in the Windows lane while leaving these exact
|
||||
// containment suites to the Linux and macOS gates.
|
||||
{
|
||||
file: 'test/cso-preparation-adversarial.test.ts',
|
||||
reason: 'exercises POSIX prepared-tree and archive-cache containment for qualified Linux Docker execution, which Windows does not admit',
|
||||
},
|
||||
{
|
||||
file: 'test/cso-preparation-container.test.ts',
|
||||
reason: 'asserts POSIX permission and symlink semantics for inert exports consumed by qualified Linux Docker execution',
|
||||
},
|
||||
{
|
||||
file: 'test/cso-preparation-executor.test.ts',
|
||||
reason: 'executes the Linux Docker acquisition path and its Unix-domain registry broker; comprehensive execution is unavailable on Windows',
|
||||
},
|
||||
{
|
||||
file: 'test/cso-verification-cleanup.test.ts',
|
||||
reason: 'spawns the POSIX detached watchdog used by contained repair verification, which Windows intentionally leaves unavailable',
|
||||
},
|
||||
{
|
||||
file: 'test/cso-witness.test.ts',
|
||||
reason: 'tests the contained repair witness with POSIX private-directory and compiled-helper assumptions; comprehensive execution is unavailable on Windows',
|
||||
},
|
||||
{
|
||||
file: 'test/shard-engine-equivalence.test.ts',
|
||||
reason: 'its classification golden was recorded from the POSIX runners (process-group wall kill); the win32 engine path is pinned by the mocked-platform case in shard-engine.test.ts',
|
||||
},
|
||||
{
|
||||
file: 'test/claude-overlay-setup-default.test.ts',
|
||||
reason: 'every case runs the real ./setup under describe.skipIf(win32); its bin/ spawns live in test/helpers/claude-overlay-fixture.ts, so the source scan misses them',
|
||||
},
|
||||
{
|
||||
file: 'test/claude-overlay-setup-installs.test.ts',
|
||||
reason: 'every case runs the real ./setup under describe.skipIf(win32); its bin/ spawns live in test/helpers/claude-overlay-fixture.ts, so the source scan misses them',
|
||||
},
|
||||
{
|
||||
file: 'test/cso-scanner-cli.test.ts',
|
||||
reason: 'drives the prebuilt POSIX CSO launcher with /usr/bin/git and a POSIX-only PATH; native Windows launcher behavior is covered by the dedicated cso-windows-launcher gate',
|
||||
},
|
||||
];
|
||||
|
||||
// Force-include overrides: files a WINDOWS_FRAGILE_PATTERNS regex excludes for
|
||||
// a reason that does not actually apply to them. Each entry documents WHY the
|
||||
// pattern hit is a false positive — the point of these files is Windows
|
||||
// coverage, so auto-excluding them defeats the regression tests they carry.
|
||||
export const KNOWN_WINDOWS_SAFE: Array<{ file: string; reason: string }> = [
|
||||
{ file: 'test/ship-hook-windows-paths.test.ts', reason: 'runs bin/ helpers through explicit bash and Bun argv with forward-slash paths; never executes a shebang; the path-spelling simulation is skipIf win32' },
|
||||
{ file: 'test/state-root-parity.test.ts', reason: 'runs the bash twin and lib/state-root.ts over an env table with PATH empty; no shebang execution, raw-string comparison is platform-neutral' },
|
||||
{ file: 'test/generator-eexist.test.ts',
|
||||
reason: 'E4: runs the generators through Bun argv with the Windows EEXIST emulation preload; no shebang execution' },
|
||||
{ file: 'test/gstack-config-gbrain-refresh.test.ts', reason: 'E6: runs bin/gstack-config through explicit bash; its gbrain/python3 shims are found by bash PATH lookup, never launched by CreateProcess' },
|
||||
{
|
||||
file: 'test/qa-evidence.test.ts',
|
||||
reason: 'invokes the production helper through Bun argv and exercises native Windows job cleanup, private file captures and backpressured receipt output',
|
||||
},
|
||||
{
|
||||
file: 'test/qa-evidence-selection.test.ts',
|
||||
reason: 'bin/ strings are literal dependency and Windows-selection assertions; no native actor or shebang command is launched',
|
||||
},
|
||||
{
|
||||
file: 'test/qa-deadline.test.ts',
|
||||
reason: 'launches the guard through Bun argv; mode assertions and POSIX signal cases are platform-gated, while Windows job cleanup must execute natively',
|
||||
},
|
||||
{
|
||||
file: 'test/qa-deadline-selection.test.ts',
|
||||
reason: 'bin/ strings are dependency-selection inputs; this suite never launches a shebang executable',
|
||||
},
|
||||
{
|
||||
file: 'test/shared-libs-source-reads.test.ts',
|
||||
reason: 'bin/ literal is a mocked launch assertion; actual worktree fingerprinting explicitly invokes Bash on Windows',
|
||||
},
|
||||
{
|
||||
file: 'test/claude-code-windows-job.test.ts',
|
||||
reason: 'invokes Bun directly; verifies Windows job containment at the standalone CLI boundary',
|
||||
},
|
||||
{
|
||||
file: 'test/claude-code-runner.test.ts',
|
||||
// The bin/ path is launched through process.execPath (Bun), never as a shebang; keep taskkill supervision in the Windows lane.
|
||||
reason: 'invokes the runner via Bun argv; fake CLI and timeout descendant assertions cover native Windows taskkill',
|
||||
},
|
||||
{
|
||||
file: 'test/setup-gbrain-remote-caller.test.ts',
|
||||
// bin is an expected PATH component; the adapter injects the SDK boundary
|
||||
// and never launches a shebang. Keep the native delimiter cases in CI.
|
||||
reason: 'replays the registered SDK callback with fixture-only bin paths; covers native Windows PATH composition',
|
||||
},
|
||||
{
|
||||
file: 'test/cso-windows-build-contract.test.ts',
|
||||
// bin is a temporary staging directory. The adapter injects spawnSync;
|
||||
// real PowerShell/native execution remains in cso-windows-launcher.
|
||||
reason: 'replays the native build callbacks with an injected subprocess; bin paths are staging fixtures, not shebang launches',
|
||||
},
|
||||
{
|
||||
file: 'test/setup-windows-rerun-refresh.test.ts',
|
||||
// Trips the "spawns bin/ shebang script" pattern via path.join(..., 'bin',
|
||||
// 'tool.sh') fixture paths, but every spawn goes through test/helpers/bash-script.ts
|
||||
// (bash <tempfile>) — Git Bash executes it fine on windows-latest, with no argv-length ceiling. This file IS
|
||||
// the #2444 Windows regression coverage (IS_WINDOWS=1 copy-refresh path);
|
||||
// excluding it here would keep the bug class unexercised on the one
|
||||
// platform it bites.
|
||||
reason: 'bin/ hits are fixture path segments; spawns bash explicitly — the IS_WINDOWS=1 refresh path must run on windows-latest',
|
||||
},
|
||||
{
|
||||
file: 'test/uninstall-windows-copies.test.ts',
|
||||
// Trips the "spawns bin/ shebang script" pattern via the
|
||||
// path.join(ROOT, 'bin', 'gstack-uninstall') constant, but the script is
|
||||
// always spawned through spawnSync('bash', [UNINSTALL, ...]). This file
|
||||
// carries the #2563 Windows real-dir-copy uninstall coverage — the bug
|
||||
// ONLY reproduces on the copy install shape windows-latest exercises.
|
||||
// The symlink-shape describe block self-skips on win32.
|
||||
reason: 'bin/ hit is a bash-spawned script path; #2563 real-dir uninstall coverage must run on windows-latest',
|
||||
},
|
||||
{
|
||||
file: 'browse/test/file-permissions.test.ts',
|
||||
// Trips the POSIX-mode-bitmask pattern, but every `mode & 0o777` assertion
|
||||
// is platform-guarded: win32-only tests return early, POSIX-only tests
|
||||
// guard the bitmask behind `process.platform !== 'win32'`, and the
|
||||
// symlink-skip regression test both wraps symlinkSync in try/catch
|
||||
// (runners without Developer Mode can't create symlinks) and guards its
|
||||
// bitmask — on win32 it asserts behavior (warns, skips, doesn't throw,
|
||||
// target stays usable), never fake Windows mode bits (dirs stat 0o777
|
||||
// there, so a 0o755 expectation fails on runner semantics, not our code).
|
||||
// This file carries the win32-only icacls-by-SID regression tests, which
|
||||
// can ONLY execute on windows-latest — excluding it here means the
|
||||
// machine-account ACL lockout regression is never exercised on the one
|
||||
// platform it bricks.
|
||||
reason: 'every mode-bitmask assertion is guarded off win32 (behavior asserted instead); win32-only ACL regression tests must run on windows-latest',
|
||||
},
|
||||
{
|
||||
file: 'browse/test/terminal-agent-owner-watchdog.test.ts',
|
||||
// Trips the spawn(['bun','run',...]) pattern, whose reason is the
|
||||
// Playwright-bound browse server. This test spawns terminal-agent.ts,
|
||||
// which imports only fs/path/crypto + local helpers (no Playwright, no
|
||||
// PTY at module scope) and boots under Bun on Windows — the owner-PID
|
||||
// orphan leak it pins was reported on Windows (#2019).
|
||||
reason: 'spawns terminal-agent (no Playwright), not the browse server; owner-orphan leak is a Windows defect',
|
||||
},
|
||||
];
|
||||
|
||||
/**
|
||||
* Pattern hits disproved by running each excluded file alone on windows-latest
|
||||
* (temporary probe run 37221995077, Bun 1.4.0, Chromium installed): every case
|
||||
* passed and at least one case executed. Their /tmp/ literals are fixture
|
||||
* strings or platform-gated, never opened on Windows.
|
||||
*/
|
||||
export const WINDOWS_PROBE_SAFE: string[] = [
|
||||
'browse/test/data-platform.test.ts',
|
||||
'browse/test/dual-listener.test.ts',
|
||||
'browse/test/error-handling.test.ts',
|
||||
'browse/test/learnings-injection.test.ts',
|
||||
'browse/test/pdf-flags.test.ts',
|
||||
'browse/test/security-adversarial.test.ts',
|
||||
'browse/test/tunnel-gate-unit.test.ts',
|
||||
'browse/test/xprotect-heal.test.ts',
|
||||
'design/test/feedback-roundtrip.test.ts',
|
||||
'ios-qa/daemon/test/tailscale-localapi.test.ts',
|
||||
'make-pdf/test/asideClient.test.ts',
|
||||
'make-pdf/test/e2e/diagram-gate.test.ts',
|
||||
'make-pdf/test/e2e/emoji-gate.test.ts',
|
||||
'make-pdf/test/e2e/format-gate.test.ts',
|
||||
'make-pdf/test/e2e/landscape-gate.test.ts',
|
||||
'test/brain-preflight.test.ts',
|
||||
'test/ceo-barless-submit.test.ts',
|
||||
'test/ceo-split-collection.test.ts',
|
||||
'test/ceo-split-question-policy.test.ts',
|
||||
'test/ci-paid-coordination.test.ts',
|
||||
'test/design-md.test.ts',
|
||||
'test/document-skills-redaction.test.ts',
|
||||
'test/evals-workflow-wiring.test.ts',
|
||||
'test/gbrain-guards.test.ts',
|
||||
'test/global-discover.test.ts',
|
||||
'test/helpers-unit.test.ts',
|
||||
'test/helpers/hermetic-env.test.ts',
|
||||
'test/office-hours-review.test.ts',
|
||||
'test/paid-run-manifest.test.ts',
|
||||
'test/paid-shards.test.ts',
|
||||
'test/plan-count-completion.test.ts',
|
||||
'test/plan-design-floor-fixture.test.ts',
|
||||
'test/pty-skill-seeding-wiring.test.ts',
|
||||
'test/regression-issue2091-bsd-mktemp.test.ts',
|
||||
'test/regression-pr1169-mktemp-fallbacks.test.ts',
|
||||
'test/resolvers-gbrain-save-results.test.ts',
|
||||
'test/skill-preflight-budget.test.ts',
|
||||
'test/takes-fence-fallback.test.ts',
|
||||
'test/terse-build.test.ts',
|
||||
'test/typecheck-test-ratchet.test.ts',
|
||||
];
|
||||
|
||||
/**
|
||||
* Returns the first POSIX-only pattern hit in the file, or null if Windows-safe.
|
||||
*/
|
||||
export function detectWindowsFragility(absolutePath: string): { reason: string } | null {
|
||||
let content: string;
|
||||
try {
|
||||
content = fs.readFileSync(absolutePath, 'utf-8');
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
for (const { pattern, reason } of WINDOWS_FRAGILE_PATTERNS) {
|
||||
if (pattern.test(content)) return { reason };
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
export interface CurationResult {
|
||||
safe: string[];
|
||||
excluded: Array<{ file: string; reason: string }>;
|
||||
}
|
||||
|
||||
export function curateWindowsSafe(files: string[], rootDir = ROOT): CurationResult {
|
||||
const safe: string[] = [];
|
||||
const excluded: Array<{ file: string; reason: string }> = [];
|
||||
const knownBad = new Map(KNOWN_WINDOWS_INCOMPATIBLE.map((e) => [e.file, e.reason]));
|
||||
const knownSafe = new Set([...KNOWN_WINDOWS_SAFE.map((e) => e.file), ...WINDOWS_PROBE_SAFE]);
|
||||
for (const relativePath of files) {
|
||||
const knownReason = knownBad.get(relativePath);
|
||||
if (knownReason) {
|
||||
excluded.push({ file: relativePath, reason: knownReason });
|
||||
continue;
|
||||
}
|
||||
if (knownSafe.has(relativePath)) {
|
||||
safe.push(relativePath);
|
||||
continue;
|
||||
}
|
||||
const absolute = path.join(rootDir, relativePath);
|
||||
const fragility = detectWindowsFragility(absolute);
|
||||
if (fragility) {
|
||||
excluded.push({ file: relativePath, reason: fragility.reason });
|
||||
} else {
|
||||
safe.push(relativePath);
|
||||
}
|
||||
}
|
||||
return { safe, excluded };
|
||||
}
|
||||
|
||||
@@ -10,7 +10,6 @@
|
||||
"test/skill-e2e-context-skills.test.ts": 1000,
|
||||
"test/skill-e2e-coverage-audit.test.ts": 65669,
|
||||
"test/skill-e2e-cso.test.ts": 229620,
|
||||
"test/skill-e2e-deploy.test.ts": 423978,
|
||||
"test/skill-e2e-deploy.test.ts#benchmark-workflow": 120646,
|
||||
"test/skill-e2e-deploy.test.ts#canary-workflow": 42804,
|
||||
"test/skill-e2e-deploy.test.ts#land-and-deploy-first-run": 103709,
|
||||
@@ -56,7 +55,9 @@
|
||||
"test/skill-e2e-qa-callers.test.ts#ship-exploratory-unavailable": 191231,
|
||||
"test/skill-e2e-qa-functional-fix.test.ts": 200616,
|
||||
"test/skill-e2e-qa-functional.test.ts": 188766,
|
||||
"test/skill-e2e-qa-workflow.test.ts": 407169,
|
||||
"test/skill-e2e-qa-workflow.test.ts#qa-bootstrap": 35863,
|
||||
"test/skill-e2e-qa-workflow.test.ts#qa-only-no-fix": 201333,
|
||||
"test/skill-e2e-qa-workflow.test.ts#qa-quick": 169852,
|
||||
"test/skill-e2e-retro.test.ts": 200434,
|
||||
"test/skill-e2e-review-army.test.ts": 520000,
|
||||
"test/skill-e2e-review-army.test.ts#review-army-delivery-audit": 39313,
|
||||
@@ -107,21 +108,21 @@
|
||||
"test/skill-routing-e2e.test.ts": 1000
|
||||
},
|
||||
"periodic": {
|
||||
"test/carve-section-loading-browse.test.ts": 114126,
|
||||
"test/carve-section-loading-codex.test.ts": 118734,
|
||||
"test/carve-section-loading-design-consultation.test.ts": 280881,
|
||||
"test/carve-section-loading-design-html.test.ts": 302488,
|
||||
"test/carve-section-loading-design-shotgun.test.ts": 151045,
|
||||
"test/carve-section-loading-document-release.test.ts": 77050,
|
||||
"test/carve-section-loading-land-and-deploy.test.ts": 187570,
|
||||
"test/carve-section-loading-plan-design-review.test.ts": 140567,
|
||||
"test/carve-section-loading-plan-devex-review.test.ts": 372694,
|
||||
"test/carve-section-loading-plan-eng-review.test.ts": 336197,
|
||||
"test/carve-section-loading-qa.test.ts": 112117,
|
||||
"test/carve-section-loading-retro.test.ts": 146908,
|
||||
"test/carve-section-loading-review.test.ts": 132284,
|
||||
"test/carve-section-loading-setup-gbrain.test.ts": 73575,
|
||||
"test/carve-section-loading-spec.test.ts": 137613,
|
||||
"test/carve-section-loading.test.ts#carve-section-loading-browse": 92911,
|
||||
"test/carve-section-loading.test.ts#carve-section-loading-codex": 150172,
|
||||
"test/carve-section-loading.test.ts#carve-section-loading-design-consultation": 318385,
|
||||
"test/carve-section-loading.test.ts#carve-section-loading-design-html": 245660,
|
||||
"test/carve-section-loading.test.ts#carve-section-loading-design-shotgun": 140298,
|
||||
"test/carve-section-loading.test.ts#carve-section-loading-document-release": 97810,
|
||||
"test/carve-section-loading.test.ts#carve-section-loading-land-and-deploy": 156293,
|
||||
"test/carve-section-loading.test.ts#carve-section-loading-plan-design-review": 223606,
|
||||
"test/carve-section-loading.test.ts#carve-section-loading-plan-devex-review": 384295,
|
||||
"test/carve-section-loading.test.ts#carve-section-loading-plan-eng-review": 297627,
|
||||
"test/carve-section-loading.test.ts#carve-section-loading-qa": 149637,
|
||||
"test/carve-section-loading.test.ts#carve-section-loading-retro": 199931,
|
||||
"test/carve-section-loading.test.ts#carve-section-loading-review": 247902,
|
||||
"test/carve-section-loading.test.ts#carve-section-loading-setup-gbrain": 80994,
|
||||
"test/carve-section-loading.test.ts#carve-section-loading-spec": 176639,
|
||||
"test/codex-e2e-multiblock-live.test.ts": 180000,
|
||||
"test/codex-e2e-recommendation-substance.test.ts": 14617,
|
||||
"test/codex-e2e-shared-libs.test.ts": 49563,
|
||||
@@ -135,14 +136,11 @@
|
||||
"test/skill-e2e-auq-verbose-vs-carved-ab.test.ts": 65480,
|
||||
"test/skill-e2e-auto-decide-preserved.test.ts": 234151,
|
||||
"test/skill-e2e-autoplan-dual-voice.test.ts": 154207,
|
||||
"test/skill-e2e-autoplan-journal-drift.test.ts": 19341,
|
||||
"test/skill-e2e-benchmark-providers.test.ts": 8847,
|
||||
"test/skill-e2e-bws.test.ts": 1000,
|
||||
"test/skill-e2e-context-skills.test.ts": 144874,
|
||||
"test/skill-e2e-context-skills.test.ts#context-restore-provenance-order": 37945,
|
||||
"test/skill-e2e-coverage-audit.test.ts": 1000,
|
||||
"test/skill-e2e-cso.test.ts": 303949,
|
||||
"test/skill-e2e-deploy.test.ts": 1000,
|
||||
"test/skill-e2e-design-model-smoke.test.ts": 90000,
|
||||
"test/skill-e2e-design.test.ts": 817000,
|
||||
"test/skill-e2e-design.test.ts#design-consultation-core": 173162,
|
||||
@@ -158,14 +156,10 @@
|
||||
"test/skill-e2e-ios-device.test.ts": 1000,
|
||||
"test/skill-e2e-learnings.test.ts": 1000,
|
||||
"test/skill-e2e-office-hours-brain-writeback.test.ts": 203044,
|
||||
"test/skill-e2e-office-hours-brain-writeback.test.ts#office-hours-brain-writeback": 304346,
|
||||
"test/skill-e2e-office-hours-design-draft.test.ts": 210162,
|
||||
"test/skill-e2e-office-hours-phase4.test.ts": 41421,
|
||||
"test/skill-e2e-office-hours-phase4.test.ts#office-hours-phase4-fork": 43846,
|
||||
"test/skill-e2e-office-hours-section-loading.test.ts": 1200000,
|
||||
"test/skill-e2e-office-hours.test.ts": 136970,
|
||||
"test/skill-e2e-office-hours.test.ts#office-hours-builder-wildness": 101538,
|
||||
"test/skill-e2e-office-hours.test.ts#office-hours-forcing-energy": 54653,
|
||||
"test/skill-e2e-outside-plan-disabled.test.ts": 72070,
|
||||
"test/skill-e2e-outside-voice.test.ts": 195221,
|
||||
"test/skill-e2e-overlay-harness-claude-dedicated-tools-vs-bash-opus-5-5.test.ts": 57374,
|
||||
@@ -186,15 +180,7 @@
|
||||
"test/skill-e2e-plan-eng-multi-finding-batching.test.ts": 409161,
|
||||
"test/skill-e2e-plan-eng-plan-mode.test.ts": 170914,
|
||||
"test/skill-e2e-plan-format.test.ts": 219670,
|
||||
"test/skill-e2e-plan-format.test.ts#plan-ceo-review-format-approach": 103713,
|
||||
"test/skill-e2e-plan-format.test.ts#plan-ceo-review-format-mode": 77296,
|
||||
"test/skill-e2e-plan-format.test.ts#plan-eng-review-format-coverage": 99395,
|
||||
"test/skill-e2e-plan-format.test.ts#plan-eng-review-format-kind": 92320,
|
||||
"test/skill-e2e-plan-prosons.test.ts": 163659,
|
||||
"test/skill-e2e-plan-prosons.test.ts#plan-ceo-review-prosons-cadence": 51625,
|
||||
"test/skill-e2e-plan-prosons.test.ts#plan-review-prosons-format": 32249,
|
||||
"test/skill-e2e-plan-prosons.test.ts#plan-review-prosons-hardstop-neg": 53487,
|
||||
"test/skill-e2e-plan-prosons.test.ts#plan-review-prosons-neutral-neg": 54403,
|
||||
"test/skill-e2e-plan-tune.test.ts": 1000,
|
||||
"test/skill-e2e-plan.test.ts": 824000,
|
||||
"test/skill-e2e-plan.test.ts#plan-ceo-review": 274619,
|
||||
@@ -202,7 +188,7 @@
|
||||
"test/skill-e2e-plan.test.ts#plan-ceo-review-selective": 265163,
|
||||
"test/skill-e2e-plan.test.ts#plan-eng-review": 169400,
|
||||
"test/skill-e2e-qa-bugs.test.ts": 355966,
|
||||
"test/skill-e2e-qa-workflow.test.ts": 411770,
|
||||
"test/skill-e2e-qa-workflow.test.ts#qa-fix-loop": 411728,
|
||||
"test/skill-e2e-retro.test.ts": 150139,
|
||||
"test/skill-e2e-review-army.test.ts": 511000,
|
||||
"test/skill-e2e-review-army.test.ts#review-army-consensus": 107375,
|
||||
@@ -211,7 +197,6 @@
|
||||
"test/skill-e2e-review-army.test.ts#review-army-simplification-precision": 25020,
|
||||
"test/skill-e2e-review-attribution.test.ts": 1000,
|
||||
"test/skill-e2e-review.test.ts": 179362,
|
||||
"test/skill-e2e-review.test.ts#review-design-lite": 146332,
|
||||
"test/skill-e2e-safety-codex-boundary.test.ts": 102851,
|
||||
"test/skill-e2e-safety-codex-consult.test.ts": 1000,
|
||||
"test/skill-e2e-safety-design-risk.test.ts": 54889,
|
||||
@@ -223,11 +208,8 @@
|
||||
"test/skill-e2e-setup-gbrain-path4-local-pglite.test.ts": 128414,
|
||||
"test/skill-e2e-setup-gbrain-remote.test.ts": 89701,
|
||||
"test/skill-e2e-shared-libs-periodic.test.ts": 217221,
|
||||
"test/skill-e2e-shared-libs-periodic.test.ts#shared-libs-opportunity-judgment": 157280,
|
||||
"test/skill-e2e-ship-section-loading.test.ts": 189463,
|
||||
"test/skill-e2e-skillify.test.ts": 58522,
|
||||
"test/skill-e2e-skillify.test.ts#scrape-match-path": 65868,
|
||||
"test/skill-e2e-skillify.test.ts#scrape-prototype-path": 36223,
|
||||
"test/skill-e2e-sync-gbrain-readiness.test.ts": 47502,
|
||||
"test/skill-e2e-test-value.test.ts": 332000,
|
||||
"test/skill-e2e-third-party-actions.test.ts": 1000,
|
||||
|
||||
@@ -0,0 +1,299 @@
|
||||
/**
|
||||
* Derived PR-lane dependencies (W1 follow-up): which paid cases consume a
|
||||
* repository file that no touchfile names, computed from real references on
|
||||
* the checked-out tree instead of hand lists.
|
||||
*
|
||||
* A case consumes the transitive reference closure of its owning paid test
|
||||
* file plus every tracked file its touchfiles declare. References are:
|
||||
* - static and literal dynamic imports of code (resolved relative paths);
|
||||
* - repository paths written in code, shell or skill text (bin/, lib/,
|
||||
* scripts/, browse/, design/, make-pdf/, hosts/, test/helpers/ ...),
|
||||
* including `path.join(..., 'lib', 'x.ts')` segments;
|
||||
* - bin script names (`gstack-*`) a skill, test or script invokes;
|
||||
* - compiled binaries (scripts/build.sh `--outfile`): text naming
|
||||
* `design/dist/design` consumes the whole `design/src/**` tree;
|
||||
* - package scripts invoked as `bun run <name>`;
|
||||
* - template placeholders: a skill whose template uses `{{NAME}}` consumes
|
||||
* the resolver module behind RESOLVERS.NAME and that module's imports;
|
||||
* - host configs: `hosts/<id>.ts` and `hosts/<id>/**` feed cases whose test
|
||||
* names that host id.
|
||||
* Selection data modules are opaque (their literals classify files rather
|
||||
* than consume them), and the walk never enters another paid test file.
|
||||
*
|
||||
* A file the derivation cannot place is not guessed: PR selection restores
|
||||
* the full gate for it (scripts/test-pr-profile.ts), and PR_FULL_GATE_FILES
|
||||
* there lists the inputs every case consumes that must always do so.
|
||||
*/
|
||||
import * as fs from 'node:fs';
|
||||
import * as path from 'node:path';
|
||||
import { spawnSync } from 'node:child_process';
|
||||
import { matchGlob } from '../test/helpers/test-selection';
|
||||
import { isPaidTestFile } from '../test/helpers/paid-test-set';
|
||||
|
||||
export interface DependencyMaps {
|
||||
e2eTouchfiles: Record<string, string[]>;
|
||||
judgeTouchfiles: Record<string, string[]>;
|
||||
globalTouchfiles: readonly string[];
|
||||
}
|
||||
|
||||
export interface DerivedDependencies {
|
||||
/** Repository file -> E2E case ids whose reference closure contains it. */
|
||||
e2e: Map<string, Set<string>>;
|
||||
/** Repository file -> LLM-judge ids whose reference closure contains it. */
|
||||
judges: Map<string, Set<string>>;
|
||||
/** Files every E2E case's closure contains: an edit there is as broad as a global touchfile. */
|
||||
everyCase: Set<string>;
|
||||
/** The global touchfiles and their code-import closure (the paid runner and harness every case runs through). */
|
||||
global: Set<string>;
|
||||
/** Every tracked file at derivation time. */
|
||||
tracked: Set<string>;
|
||||
/** The reference chain from an E2E case's declared inputs to `file` (for fix text and audits), else null. */
|
||||
trace(caseId: string, file: string): string[] | null;
|
||||
/**
|
||||
* Tracked files whose reference text still names `missing`, a path absent
|
||||
* from the tree (deleted in the diff): its path or extensionless import
|
||||
* target, a relative import resolving to it, its bin name, an ancestor
|
||||
* directory below the top two levels, or the outfile of a binary built from
|
||||
* its tree. Whole-line comments and selection data never count.
|
||||
*/
|
||||
referencers(missing: string): string[];
|
||||
}
|
||||
|
||||
const CODE = /\.(?:[cm]?[jt]s|tsx|jsx)$/;
|
||||
const TEXT = /\.(?:[cm]?[jt]s|tsx|jsx|sh|md|tmpl|json|ya?ml|toml|txt|py)$/;
|
||||
const REPO_PATH = /(?:^|[^\w.@-])((?:bin|lib|scripts|browse|design|make-pdf|hosts|extension|model-overlays|agents|test\/helpers|test\/fixtures)\/[\w.@-]+(?:\/[\w.@-]+)*)/g;
|
||||
/** `path.join(ROOT, 'lib', 'x.ts')`: a top-level directory and its entry as separate string literals. */
|
||||
const JOINED_PATH = /['"`](bin|lib|scripts|browse|design|make-pdf|hosts|extension|model-overlays|agents)['"`]\s*,\s*['"`]([\w.@-]+)['"`]/g;
|
||||
const BIN_NAME = /\bgstack-[a-z0-9][a-z0-9-]*[a-z0-9]\b/g;
|
||||
const PLACEHOLDER = /\{\{([A-Z][A-Z0-9_]*)/g;
|
||||
const PACKAGE_RUN = /\b(?:bun|npm) run ([a-z][\w:.-]*)/g;
|
||||
const OWNER = /^test\/[^/]+\.test\.ts$/;
|
||||
/**
|
||||
* Selection and inventory data: their string literals name files they
|
||||
* classify, not files they consume, so references inside them are not
|
||||
* followed (the same rule as test/helpers/touchfile-closure.ts).
|
||||
*/
|
||||
export const SELECTION_DATA_MODULES: readonly string[] = [
|
||||
'test/helpers/touchfiles-data.ts', 'test/helpers/touchfiles.ts', 'test/helpers/test-selection.ts',
|
||||
'test/helpers/touchfile-closure.ts', 'test/helpers/free-fixtures-data.ts', 'test/helpers/paid-test-set.ts',
|
||||
'test/helpers/periodic-exclude-data.ts', 'test/helpers/eval-budgets.ts', 'scripts/test-pr-profile.ts',
|
||||
'scripts/pr-dependencies.ts', 'scripts/lib/paid-cases.ts', 'scripts/lib/paid-select.ts', 'scripts/lib/paid-types.ts',
|
||||
// The build entry list: every compiled binary, which skills reach through their `--outfile` path instead.
|
||||
'scripts/build.sh',
|
||||
];
|
||||
const DEFAULT_JUDGE_OWNER = 'test/skill-llm-eval.test.ts';
|
||||
|
||||
const transpiler = new Bun.Transpiler({ loader: 'tsx' });
|
||||
|
||||
/**
|
||||
* Text a reference scan reads: whole-line comments are prose about other
|
||||
* files, not consumption (block comments that open a line, `//` lines, and
|
||||
* shell `#` lines). Trailing comments stay, so a stripped reference is never
|
||||
* one the code executes.
|
||||
*/
|
||||
export function referenceText(file: string, text: string): string {
|
||||
if (CODE.test(file)) return text.replace(/^[ \t]*\/\*[\s\S]*?\*\//gm, '').replace(/^[ \t]*\/\/.*$/gm, '');
|
||||
if (/\.json$/.test(file)) return '';
|
||||
if (!/\.(?:md|tmpl|txt)$/.test(file)) return text.replace(/^[ \t]*#(?!!).*$/gm, '');
|
||||
return text;
|
||||
}
|
||||
|
||||
function trackedFiles(root: string): string[] {
|
||||
const listed = spawnSync('git', ['ls-files', '-z'], { cwd: root, encoding: 'utf8', timeout: 20_000, maxBuffer: 64 * 1024 * 1024 });
|
||||
if (listed.status !== 0) throw new Error('git ls-files failed; dependency derivation needs a git checkout');
|
||||
return listed.stdout.split('\0').filter(Boolean);
|
||||
}
|
||||
|
||||
/** Resolver module (repo path) -> placeholder names whose RESOLVERS entry calls into it. */
|
||||
function placeholderModules(root: string, read: (file: string) => string): Map<string, string[]> {
|
||||
const index = 'scripts/resolvers/index.ts';
|
||||
const source = read(index);
|
||||
const fnModule = new Map<string, string>();
|
||||
for (const match of source.matchAll(/import\s*\{([^}]*)\}\s*from\s*'\.\/([\w./-]+)'/g)) {
|
||||
const file = resolveSpecifier(root, `./${match[2]}`, index);
|
||||
if (!file) continue;
|
||||
for (const name of match[1]!.split(',').map(part => part.trim().replace(/^type\s+/, '').split(/\s+as\s+/).pop()!).filter(Boolean)) fnModule.set(name, file);
|
||||
}
|
||||
const body = source.slice(source.indexOf('export const RESOLVERS'));
|
||||
const byModule = new Map<string, string[]>();
|
||||
for (const entry of body.matchAll(/^\s+([A-Z][A-Z0-9_]*):\s*(.+?),?\s*$/gm)) {
|
||||
for (const ident of entry[2]!.matchAll(/\b([A-Za-z_$][\w$]*)\b/g)) {
|
||||
const file = fnModule.get(ident[1]!);
|
||||
if (file) byModule.set(file, [...new Set([...(byModule.get(file) ?? []), entry[1]!])]);
|
||||
}
|
||||
}
|
||||
return byModule;
|
||||
}
|
||||
|
||||
function resolveSpecifier(root: string, specifier: string, from: string): string | null {
|
||||
if (!specifier.startsWith('.')) return null;
|
||||
const base = path.posix.normalize(path.posix.join(path.posix.dirname(from), specifier));
|
||||
for (const candidate of [base, `${base}.ts`, `${base}.tsx`, `${base}.js`, `${base}.mjs`, `${base}/index.ts`, `${base}/index.js`]) {
|
||||
try { if (fs.statSync(path.join(root, candidate)).isFile()) return candidate; } catch { /* next */ }
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
/** Binary outfile path -> the source tree it is compiled from (scripts/build.sh). */
|
||||
function compiledBinaries(read: (file: string) => string): Array<{ outfile: string; tree: string }> {
|
||||
return [...read('scripts/build.sh').matchAll(/--compile\b[^\n]*?\s([\w./-]+\.ts)\s+--outfile\s+([\w./-]+)/g)]
|
||||
.map(match => ({ outfile: match[2]!, tree: match[1]!.startsWith('bin/') ? match[1]! : `${path.posix.dirname(match[1]!)}/**` }));
|
||||
}
|
||||
|
||||
export function deriveDependencies(maps: DependencyMaps, root: string): DerivedDependencies {
|
||||
const tracked = trackedFiles(root);
|
||||
const trackedSet = new Set(tracked);
|
||||
const sources = new Map<string, string>();
|
||||
const read = (file: string): string => {
|
||||
if (!sources.has(file)) {
|
||||
let text = '';
|
||||
try { if (trackedSet.has(file) && (TEXT.test(file) || file.startsWith('bin/') || file === 'setup')) text = fs.readFileSync(path.join(root, file), 'utf8'); } catch { /* unreadable */ }
|
||||
sources.set(file, text);
|
||||
}
|
||||
return sources.get(file)!;
|
||||
};
|
||||
const binNames = new Map(tracked.filter(file => file.startsWith('bin/')).map(file => [path.posix.basename(file).replace(/\.ts$/, ''), file]));
|
||||
const binaries = compiledBinaries(read);
|
||||
let scripts: Record<string, string> = {};
|
||||
try { scripts = JSON.parse(read('package.json')).scripts ?? {}; } catch { /* no package scripts */ }
|
||||
const placeholders = placeholderModules(root, read);
|
||||
const placeholderOwners = new Map<string, string>();
|
||||
for (const [file, names] of placeholders) for (const name of names) placeholderOwners.set(name, file);
|
||||
const expanded = new Map<string, string[]>();
|
||||
const expand = (pattern: string): string[] => {
|
||||
if (!expanded.has(pattern)) expanded.set(pattern, pattern.includes('*') ? tracked.filter(file => matchGlob(file, pattern)) : trackedSet.has(pattern) ? [pattern] : []);
|
||||
return expanded.get(pattern)!;
|
||||
};
|
||||
const dirs = new Set(tracked.flatMap(file => file.split('/').slice(0, -1).map((_, i, parts) => parts.slice(0, i + 1).join('/'))));
|
||||
|
||||
const edges = new Map<string, string[]>();
|
||||
const refs = (file: string): string[] => {
|
||||
if (edges.has(file)) return edges.get(file)!;
|
||||
const raw = read(file);
|
||||
const text = (SELECTION_DATA_MODULES as readonly string[]).includes(file) ? '' : referenceText(file, raw);
|
||||
const out = new Set<string>();
|
||||
if (CODE.test(file)) {
|
||||
try { for (const item of transpiler.scanImports(raw)) { const hit = resolveSpecifier(root, item.path, file); if (hit) out.add(hit); } } catch { /* literal scan below still applies */ }
|
||||
}
|
||||
for (const match of text.matchAll(REPO_PATH)) {
|
||||
const ref = match[1]!.replace(/[.]+$/, '');
|
||||
if (trackedSet.has(ref)) out.add(ref);
|
||||
else if (dirs.has(ref)) for (const child of expand(`${ref}/**`)) out.add(child);
|
||||
}
|
||||
for (const match of text.matchAll(JOINED_PATH)) {
|
||||
const ref = `${match[1]}/${match[2]}`;
|
||||
if (trackedSet.has(ref)) out.add(ref);
|
||||
else if (dirs.has(ref)) for (const child of expand(`${ref}/**`)) out.add(child);
|
||||
}
|
||||
for (const match of text.matchAll(BIN_NAME)) { const bin = binNames.get(match[0]); if (bin) out.add(bin); }
|
||||
for (const { outfile, tree } of binaries) if (text.includes(outfile)) for (const child of expand(tree)) out.add(child);
|
||||
for (const match of text.matchAll(PACKAGE_RUN)) {
|
||||
const command = scripts[match[1]!];
|
||||
if (command) for (const ref of command.matchAll(REPO_PATH)) for (const child of expand(ref[1]!)) out.add(child);
|
||||
}
|
||||
if (/\.(?:md|tmpl)$/.test(file)) for (const match of text.matchAll(PLACEHOLDER)) { const owner = placeholderOwners.get(match[1]!); if (owner) out.add(owner); }
|
||||
out.delete(file);
|
||||
const list = [...out];
|
||||
edges.set(file, list);
|
||||
return list;
|
||||
};
|
||||
const closure = (entries: Iterable<string>, parents?: Map<string, string>): Set<string> => {
|
||||
const seen = new Set<string>();
|
||||
const queue = [...entries].filter(file => trackedSet.has(file));
|
||||
while (queue.length) {
|
||||
const file = queue.pop()!;
|
||||
if (seen.has(file)) continue;
|
||||
seen.add(file);
|
||||
for (const next of refs(file)) if (!seen.has(next) && !/\.test\.tsx?$/.test(next)) { queue.push(next); if (parents && !parents.has(next)) parents.set(next, file); }
|
||||
}
|
||||
return seen;
|
||||
};
|
||||
|
||||
const hostIds = tracked.filter(file => /^hosts\/[\w-]+\.ts$/.test(file) && file !== 'hosts/index.ts').map(file => path.posix.basename(file, '.ts'));
|
||||
const consumers = (table: Record<string, string[]>, defaultOwner: string | null): Map<string, Set<string>> => {
|
||||
const byFile = new Map<string, Set<string>>();
|
||||
for (const [id, patterns] of Object.entries(table)) {
|
||||
const owners = patterns.filter(pattern => OWNER.test(pattern) && isPaidTestFile(pattern));
|
||||
const declared = [...(owners.length ? owners : defaultOwner ? [defaultOwner] : []), ...patterns.flatMap(expand)];
|
||||
const files = closure(declared);
|
||||
const ownerText = (owners.length ? owners : defaultOwner ? [defaultOwner] : []).map(read).join('\n');
|
||||
for (const host of hostIds) {
|
||||
if (new RegExp(`['"\`]${host}['"\`]|--host[ =]${host}\\b`).test(ownerText)) for (const file of closure([`hosts/${host}.ts`, ...expand(`hosts/${host}/**`)])) files.add(file);
|
||||
}
|
||||
for (const file of files) {
|
||||
if (!byFile.has(file)) byFile.set(file, new Set());
|
||||
byFile.get(file)!.add(id);
|
||||
}
|
||||
}
|
||||
return byFile;
|
||||
};
|
||||
const e2e = consumers(maps.e2eTouchfiles, null);
|
||||
const total = Object.keys(maps.e2eTouchfiles).length;
|
||||
const everyCase = new Set([...e2e].filter(([, ids]) => ids.size === total).map(([file]) => file));
|
||||
const global = new Set<string>();
|
||||
const pending = maps.globalTouchfiles.flatMap(expand);
|
||||
while (pending.length) {
|
||||
const file = pending.pop()!;
|
||||
if (global.has(file)) continue;
|
||||
global.add(file);
|
||||
if (!CODE.test(file)) continue;
|
||||
try { for (const item of transpiler.scanImports(read(file))) { const hit = resolveSpecifier(root, item.path, file); if (hit && !global.has(hit)) pending.push(hit); } } catch { /* unparseable: its own entry stays global */ }
|
||||
}
|
||||
const importTargets = new Map<string, string[]>();
|
||||
const targetsOf = (file: string): string[] => {
|
||||
if (!importTargets.has(file)) {
|
||||
const targets: string[] = [];
|
||||
if (CODE.test(file) && !(SELECTION_DATA_MODULES as readonly string[]).includes(file)) {
|
||||
try {
|
||||
for (const item of transpiler.scanImports(read(file))) {
|
||||
if (item.path.startsWith('.')) targets.push(path.posix.normalize(path.posix.join(path.posix.dirname(file), item.path)));
|
||||
}
|
||||
} catch { /* literal mentions below still apply */ }
|
||||
}
|
||||
importTargets.set(file, targets);
|
||||
}
|
||||
return importTargets.get(file)!;
|
||||
};
|
||||
const referencerCache = new Map<string, string[]>();
|
||||
const referencers = (missing: string): string[] => {
|
||||
if (!referencerCache.has(missing)) referencerCache.set(missing, findReferencers(missing));
|
||||
return referencerCache.get(missing)!;
|
||||
};
|
||||
const findReferencers = (missing: string): string[] => {
|
||||
const bare = missing.replace(/\.(?:[cm]?[jt]s|tsx|jsx)$/, '');
|
||||
const parts = missing.split('/');
|
||||
const ancestors = parts.slice(0, -1).map((_, i) => parts.slice(0, i + 1).join('/')).filter(dir => dir.split('/').length >= 3);
|
||||
const binName = missing.startsWith('bin/') ? path.posix.basename(bare) : null;
|
||||
const outfiles = binaries.filter(({ tree }) => matchGlob(missing, tree)).map(({ outfile }) => outfile);
|
||||
const needles = [missing, ...(bare !== missing ? [bare] : []), ...ancestors.map(dir => `${dir}/`), ...outfiles];
|
||||
return tracked.filter(file => {
|
||||
if (file === missing || (SELECTION_DATA_MODULES as readonly string[]).includes(file)) return false;
|
||||
const raw = read(file);
|
||||
if (!raw) return false;
|
||||
const text = referenceText(file, raw);
|
||||
if (needles.some(needle => text.includes(needle))) return true;
|
||||
if (binName && new RegExp(`\\b${binName.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')}\\b`).test(text)) return true;
|
||||
return targetsOf(file).some(target => target === missing || target === bare || `${target}/index.ts` === missing);
|
||||
});
|
||||
};
|
||||
const trace = (caseId: string, file: string): string[] | null => {
|
||||
const patterns = maps.e2eTouchfiles[caseId] ?? [];
|
||||
const parents = new Map<string, string>();
|
||||
const reached = closure([...patterns.filter(pattern => OWNER.test(pattern)), ...patterns.flatMap(expand)], parents);
|
||||
if (!reached.has(file)) return null;
|
||||
const chain = [file];
|
||||
while (parents.has(chain[0]!)) chain.unshift(parents.get(chain[0]!)!);
|
||||
return chain;
|
||||
};
|
||||
return { e2e, judges: consumers(maps.judgeTouchfiles, DEFAULT_JUDGE_OWNER), everyCase, global, tracked: trackedSet, trace, referencers };
|
||||
}
|
||||
|
||||
const cache = new Map<string, DerivedDependencies>();
|
||||
|
||||
/** Derived dependencies of the checkout at `root`, computed once per process and map set. */
|
||||
export function derivedDependencies(maps: DependencyMaps, root: string): DerivedDependencies {
|
||||
const key = `${root}\0${Object.keys(maps.e2eTouchfiles).length}\0${Object.keys(maps.judgeTouchfiles).length}`;
|
||||
if (!cache.has(key)) cache.set(key, deriveDependencies(maps, root));
|
||||
return cache.get(key)!;
|
||||
}
|
||||
|
||||
@@ -10,10 +10,11 @@
|
||||
* `opus-4-7.md` with no unresolved inheritance directives.
|
||||
* 5. A local `claude` binary exists at `which claude` so binary pinning is possible.
|
||||
*
|
||||
* Run: bun run scripts/preflight-agent-sdk.ts
|
||||
* Run: bun run scripts/preflight-agent-sdk.ts [--live]
|
||||
*
|
||||
* Exit 0 on success. Exit non-zero with a clear message on any failure. No
|
||||
* side effects beyond stdout and a ~15 token API call.
|
||||
* Checks 1-2 and 4-5 are free. Check 3 (the live SDK query, ~15 tokens of API
|
||||
* spend) runs only with --live. Exit 0 on success, 1 on any failed check,
|
||||
* 2 on a bad argument.
|
||||
*/
|
||||
|
||||
import '../lib/conductor-env-shim';
|
||||
@@ -21,7 +22,35 @@ import { query, type SDKMessage } from '@anthropic-ai/claude-agent-sdk';
|
||||
import { readOverlay } from './resolvers/model-overlay';
|
||||
import { resolveClaudeBinary } from '../lib/claude-bin';
|
||||
|
||||
const USAGE = `Usage: bun run scripts/preflight-agent-sdk.ts [--live]
|
||||
|
||||
Free checks: overlay resolution and local claude binary pinning.
|
||||
--live also runs one end-to-end SDK query against claude-opus-4-7
|
||||
(~15 tokens of API spend; needs ANTHROPIC_API_KEY).`;
|
||||
|
||||
function parsePreflightArgs(argv: string[]): { help: boolean; live: boolean } {
|
||||
const parsed = { help: false, live: false };
|
||||
for (const arg of argv) {
|
||||
if (arg === '--help' || arg === '-h') parsed.help = true;
|
||||
else if (arg === '--live') parsed.live = true;
|
||||
else throw new Error(`unknown argument "${arg}"`);
|
||||
}
|
||||
return parsed;
|
||||
}
|
||||
|
||||
async function main() {
|
||||
let args: { help: boolean; live: boolean };
|
||||
try {
|
||||
args = parsePreflightArgs(process.argv.slice(2));
|
||||
} catch (err) {
|
||||
console.error(`preflight-agent-sdk: ${(err as Error).message}\n\n${USAGE}`);
|
||||
process.exit(2);
|
||||
}
|
||||
if (args.help) {
|
||||
console.log(USAGE);
|
||||
return;
|
||||
}
|
||||
|
||||
const failures: string[] = [];
|
||||
const pass = (msg: string) => console.log(` ok ${msg}`);
|
||||
const fail = (msg: string) => {
|
||||
@@ -54,7 +83,9 @@ async function main() {
|
||||
|
||||
// 3. SDK query end-to-end
|
||||
console.log('\n3. SDK query end-to-end');
|
||||
if (!process.env.ANTHROPIC_API_KEY) {
|
||||
if (!args.live) {
|
||||
console.log(' skip live query not requested — rerun with --live to spend ~15 tokens on it');
|
||||
} else if (!process.env.ANTHROPIC_API_KEY) {
|
||||
console.log(' skip ANTHROPIC_API_KEY not set — cannot test live query');
|
||||
} else {
|
||||
try {
|
||||
@@ -122,7 +153,9 @@ async function main() {
|
||||
console.log('PREFLIGHT OK');
|
||||
}
|
||||
|
||||
main().catch((err) => {
|
||||
console.error(err);
|
||||
process.exit(1);
|
||||
});
|
||||
if (import.meta.main) {
|
||||
main().catch((err) => {
|
||||
console.error(err);
|
||||
process.exit(1);
|
||||
});
|
||||
}
|
||||
@@ -0,0 +1,296 @@
|
||||
/**
|
||||
* Cancellation recovery for the PR lane's receipt store (W1c: CEO-14/25/34,
|
||||
* ENG-4/5/7/8).
|
||||
*
|
||||
* A PR run cancelled by the next push never reaches its report, so its
|
||||
* finished receipts never enter the store the report saves. The
|
||||
* `recover-receipts` job in evals.yml runs this script from the base ref
|
||||
* (never PR code) with `actions: read`, before `plan-slices`:
|
||||
*
|
||||
* collect restores nothing itself: it reads the store the job restored,
|
||||
* lists this PR's evals.yml runs newer than that store (plus runs
|
||||
* the checkpoint left unresolved), downloads the receipts of
|
||||
* completed cancelled runs (at most 5, merged oldest first,
|
||||
* negatives first), reconstructs completed behavior panels from
|
||||
* their manifest and slice results to publish panel negatives
|
||||
* (partial panels get no credit), and writes the checkpoint
|
||||
* `recovery.json` into the store.
|
||||
* decide (plan-slices) reads the checkpoint and turns reuse off when
|
||||
* recovery degraded or a cancelled run is still unresolved, so a
|
||||
* FAIL that may exist is never bypassed by an older PASS.
|
||||
*
|
||||
* Recovery never fails the run: a gh error, rate limit, missing artifact or
|
||||
* an exhausted download budget degrades to "no recovered receipts" with a
|
||||
* named reason, and the run executes unreduced.
|
||||
*/
|
||||
import * as fs from 'node:fs';
|
||||
import * as os from 'node:os';
|
||||
import * as path from 'node:path';
|
||||
import { spawnSync } from 'node:child_process';
|
||||
import { mergeReceiptDirs, writeNegativeReceipt } from './e2e-shard-reuse';
|
||||
import { loadSliceArtifacts, panelReports, parseRunManifest, shardFile } from './test-paid-shards';
|
||||
|
||||
export const RECOVERY_FILE = 'recovery.json';
|
||||
export const MAX_RECOVERED_RUNS = 5;
|
||||
const PROCESSED_KEEP = 200;
|
||||
|
||||
export interface RecoveryCheckpoint {
|
||||
schema: 1;
|
||||
updatedAt: number;
|
||||
mode: 'pr' | 'dispatch';
|
||||
/** Run ids whose evidence is merged (or that needed none). */
|
||||
processed: string[];
|
||||
/** Cancelled runs whose evidence could not be read yet; reuse stays off until a later run reconciles them. */
|
||||
unresolved: Array<{ runId: string; reason: string }>;
|
||||
/** Why recovery could not run at all; reuse stays off. */
|
||||
degraded?: string;
|
||||
recovered: number;
|
||||
}
|
||||
|
||||
export interface WorkflowRun {
|
||||
id: number;
|
||||
status: string;
|
||||
conclusion: string | null;
|
||||
event: string;
|
||||
head_sha: string;
|
||||
run_attempt?: number;
|
||||
updated_at?: string;
|
||||
head_repository?: { full_name?: string } | null;
|
||||
pull_requests?: Array<{ number: number }>;
|
||||
}
|
||||
|
||||
/** One `gh` invocation; injectable so free tests drive recovery from recorded responses. */
|
||||
export type GhRunner = (args: string[], timeoutMs: number) => { status: number | null; stdout: string; stderr: string };
|
||||
|
||||
export const runGh: GhRunner = (args, timeoutMs) => {
|
||||
const result = spawnSync('gh', args, { encoding: 'utf8', timeout: Math.max(1_000, timeoutMs), maxBuffer: 64 * 1024 * 1024 });
|
||||
return { status: result.status, stdout: result.stdout ?? '', stderr: result.error ? String(result.error) : result.stderr ?? '' };
|
||||
};
|
||||
|
||||
/** A gh failure as a named reason with the next step (DX-6). */
|
||||
export function ghFailureReason(stderr: string): string {
|
||||
if (/rate limit|secondary rate|HTTP 429/i.test(stderr)) {
|
||||
const reset = /reset[^0-9]*(\d{9,})/i.exec(stderr)?.[1];
|
||||
return `GitHub API rate limit${reset ? ` (resets at ${new Date(Number(reset) * 1000).toISOString()})` : ''}; rerun the job after the reset`;
|
||||
}
|
||||
if (/HTTP 40[13]|authentication|auth login/i.test(stderr)) return 'gh is not authorized for actions: read; check the job permissions and `gh auth status`';
|
||||
if (/ETIMEDOUT|timed out/i.test(stderr)) return 'gh timed out inside the 60 s recovery budget; rerun the job';
|
||||
return `gh failed: ${stderr.trim().split('\n')[0]?.slice(0, 160) || 'no output'}`;
|
||||
}
|
||||
|
||||
export function readCheckpoint(store: string): RecoveryCheckpoint | null {
|
||||
try {
|
||||
const parsed = JSON.parse(fs.readFileSync(path.join(store, RECOVERY_FILE), 'utf8'));
|
||||
if (parsed?.schema !== 1 || !Array.isArray(parsed.processed) || !Array.isArray(parsed.unresolved)) return null;
|
||||
return parsed as RecoveryCheckpoint;
|
||||
} catch { return null; }
|
||||
}
|
||||
|
||||
function writeCheckpoint(store: string, checkpoint: RecoveryCheckpoint): void {
|
||||
fs.mkdirSync(store, { recursive: true });
|
||||
fs.writeFileSync(path.join(store, RECOVERY_FILE), `${JSON.stringify(checkpoint, null, 2)}\n`, { mode: 0o600 });
|
||||
}
|
||||
|
||||
/** The run id that saved a restored store, from its cache key (`…-pr-<n>-<run id>-<attempt>-merged`). */
|
||||
export function storeRunId(cacheKey: string): number {
|
||||
const match = /-pr-\d+-(\d+)-\d+-(?:merged|plan)$/.exec(cacheKey);
|
||||
return match ? Number(match[1]) : 0;
|
||||
}
|
||||
|
||||
/** CEO-34 provenance: same workflow event, PR number and head repository, never this run. */
|
||||
export function acceptedRun(run: WorkflowRun, scope: { pr: number; headRepo: string; selfRunId: number }): boolean {
|
||||
return run.id !== scope.selfRunId && run.event === 'pull_request'
|
||||
&& run.head_repository?.full_name === scope.headRepo
|
||||
&& (run.pull_requests ?? []).some(pr => pr.number === scope.pr);
|
||||
}
|
||||
|
||||
/**
|
||||
* Negative receipts a cancelled run's completed work proves (ENG-5): every
|
||||
* completed FAIL panel whose trials share one input identity, and every
|
||||
* completed non-passing single-trial shard. Partial panels publish nothing.
|
||||
*/
|
||||
export function negativesFromRun(runDir: string, run: Pick<WorkflowRun, 'id' | 'head_sha' | 'updated_at'>, out: string): number {
|
||||
let manifest;
|
||||
try { manifest = parseRunManifest(fs.readFileSync(path.join(runDir, 'paid-plan', 'manifest.json'), 'utf8')); } catch { return 0; }
|
||||
const artifacts = loadSliceArtifacts(runDir);
|
||||
let written = 0;
|
||||
for (const attempt of [...new Set(artifacts.map(a => a.result.attempt ?? 1))]) {
|
||||
const results = artifacts.map(a => a.result).filter(r => (r.attempt ?? 1) === attempt);
|
||||
const completedAt = Math.max(0, ...results.map(r => r.finishedAt ?? 0)) || Date.parse(run.updated_at ?? '') || Date.now();
|
||||
const source = { runId: `${run.id}/${attempt}`, revision: run.head_sha, completedAt };
|
||||
if (!/^[a-f0-9]{40}$/.test(source.revision)) continue;
|
||||
const outcomes = results.flatMap(r => r.outcomes);
|
||||
for (const panel of panelReports(manifest, results, attempt).filter(p => p.status === 'FAIL')) {
|
||||
const keys = outcomes.filter(o => o.trial?.case === panel.case && shardFile(o.files[0] ?? '') === panel.file).map(o => o.reused ? null : o.inputKey ?? null);
|
||||
if (keys.length !== panel.panel.n || keys.some(k => k === null) || new Set(keys).size !== 1) continue;
|
||||
writeNegativeReceipt(out, { schema: 1, key: keys[0]!, source });
|
||||
written++;
|
||||
}
|
||||
for (const outcome of outcomes.filter(o => !o.trial && o.inputKey && !o.reused && o.status !== 'passed' && o.status !== 'skipped-by-diff')) {
|
||||
writeNegativeReceipt(out, { schema: 1, key: outcome.inputKey!, source });
|
||||
written++;
|
||||
}
|
||||
}
|
||||
return written;
|
||||
}
|
||||
|
||||
export interface CollectOptions {
|
||||
repo: string;
|
||||
pr: number;
|
||||
headRepo: string;
|
||||
branch: string;
|
||||
selfRunId: number;
|
||||
store: string;
|
||||
/** The restored cache key; recovery reads only runs newer than the run that saved it. */
|
||||
storeKey?: string;
|
||||
/** `dispatch`: read-only receipts of the PR's completed runs for a validation dispatch (ENG-8). */
|
||||
mode?: 'pr' | 'dispatch';
|
||||
budgetMs?: number;
|
||||
pollMs?: number;
|
||||
gh?: GhRunner;
|
||||
now?: () => number;
|
||||
sleep?: (ms: number) => void;
|
||||
}
|
||||
|
||||
export function collectRecovery(options: CollectOptions): { checkpoint: RecoveryCheckpoint; lines: string[] } {
|
||||
const gh = options.gh ?? runGh;
|
||||
const now = options.now ?? Date.now;
|
||||
const sleep = options.sleep ?? ((ms: number) => Bun.sleepSync(ms));
|
||||
const mode = options.mode ?? 'pr';
|
||||
const deadline = now() + (options.budgetMs ?? 60_000);
|
||||
const left = () => deadline - now();
|
||||
const previous = readCheckpoint(options.store);
|
||||
const processed = new Set(previous?.processed ?? []);
|
||||
const pending = new Set((previous?.unresolved ?? []).map(entry => entry.runId));
|
||||
const lines: string[] = [];
|
||||
const finish = (checkpoint: Omit<RecoveryCheckpoint, 'schema' | 'updatedAt' | 'mode'>) => {
|
||||
const full: RecoveryCheckpoint = { schema: 1, updatedAt: now(), mode, ...checkpoint, processed: checkpoint.processed.slice(-PROCESSED_KEEP) };
|
||||
writeCheckpoint(options.store, full);
|
||||
return { checkpoint: full, lines };
|
||||
};
|
||||
const degrade = (reason: string) => {
|
||||
lines.push(`receipt recovery degraded: ${reason}; no recovered receipts, this run executes unreduced`);
|
||||
return finish({ processed: [...processed], unresolved: previous?.unresolved ?? [], degraded: reason, recovered: 0 });
|
||||
};
|
||||
|
||||
const listed = gh(['api', '-X', 'GET', `repos/${options.repo}/actions/workflows/evals.yml/runs`,
|
||||
'-f', 'event=pull_request', '-f', `branch=${options.branch}`, '-f', 'per_page=50'], left());
|
||||
if (listed.status !== 0) return degrade(ghFailureReason(listed.stderr));
|
||||
let runs: WorkflowRun[];
|
||||
try { runs = (JSON.parse(listed.stdout).workflow_runs ?? []) as WorkflowRun[]; } catch { return degrade('gh returned an unreadable run list'); }
|
||||
|
||||
const floor = mode === 'pr' ? storeRunId(options.storeKey ?? '') : 0;
|
||||
const scope = { pr: options.pr, headRepo: options.headRepo, selfRunId: options.selfRunId };
|
||||
for (const rejected of runs.filter(run => run.id !== options.selfRunId && run.event === 'pull_request' && !acceptedRun(run, scope)
|
||||
&& (run.pull_requests ?? []).some(pr => pr.number === options.pr))) {
|
||||
lines.push(`rejected run ${rejected.id}: head repository ${rejected.head_repository?.full_name ?? 'unknown'} is not ${options.headRepo}`);
|
||||
}
|
||||
const older = runs.filter(run => acceptedRun(run, scope) && run.id < options.selfRunId);
|
||||
// A completed run that was not cancelled reached its report, which saved its receipts.
|
||||
const reported = (run: WorkflowRun) => run.status === 'completed' && run.conclusion !== 'cancelled';
|
||||
const fresh = (run: WorkflowRun) => pending.has(String(run.id)) || (run.id > floor && !processed.has(String(run.id)));
|
||||
const candidates = (mode === 'dispatch' ? older.filter(run => run.status === 'completed')
|
||||
: older.filter(run => fresh(run) && !reported(run))).sort((a, b) => b.id - a.id).slice(0, MAX_RECOVERED_RUNS);
|
||||
if (mode === 'pr') for (const run of older.filter(run => fresh(run) && reported(run))) processed.add(String(run.id));
|
||||
|
||||
const unresolved: RecoveryCheckpoint['unresolved'] = [];
|
||||
const work = fs.mkdtempSync(path.join(os.tmpdir(), 'recover-receipts-'));
|
||||
let recovered = 0;
|
||||
try {
|
||||
// Oldest first, so a newer FAIL always lands after the PASS it blocks.
|
||||
for (const listedRun of [...candidates].sort((a, b) => a.id - b.id)) {
|
||||
const id = String(listedRun.id);
|
||||
let run = listedRun;
|
||||
while (run.status !== 'completed' && left() > (options.pollMs ?? 5_000)) {
|
||||
sleep(options.pollMs ?? 5_000);
|
||||
const polled = gh(['api', `repos/${options.repo}/actions/runs/${id}`], left());
|
||||
if (polled.status !== 0) break;
|
||||
try { run = JSON.parse(polled.stdout) as WorkflowRun; } catch { break; }
|
||||
}
|
||||
if (run.status !== 'completed') {
|
||||
unresolved.push({ runId: id, reason: 'still in progress; its completed FAILs cannot be read yet' });
|
||||
lines.push(`run ${id} is still ${run.status}; reuse stays off until a later run reconciles it`);
|
||||
continue;
|
||||
}
|
||||
if (mode === 'pr' && run.conclusion !== 'cancelled') { processed.add(id); continue; }
|
||||
const dir = path.join(work, id);
|
||||
const names = gh(['api', '-X', 'GET', `repos/${options.repo}/actions/runs/${id}/artifacts`, '-f', 'per_page=100'], left());
|
||||
let artifacts: string[] = [];
|
||||
try {
|
||||
artifacts = names.status === 0 ? (JSON.parse(names.stdout).artifacts ?? [])
|
||||
.filter((a: { name: string; expired?: boolean }) => !a.expired && /^(?:paid-plan|paid-slice-\d+-a\d+|report-verdict-a\d+)$/.test(a.name))
|
||||
.map((a: { name: string }) => a.name) : [];
|
||||
} catch { artifacts = []; }
|
||||
if (names.status !== 0) {
|
||||
unresolved.push({ runId: id, reason: ghFailureReason(names.stderr) });
|
||||
lines.push(`run ${id}: cannot list artifacts (${ghFailureReason(names.stderr)})`);
|
||||
continue;
|
||||
}
|
||||
let failed = '';
|
||||
if (artifacts.length && left() <= 0) failed = 'the 60 s download budget ran out';
|
||||
else if (artifacts.length) {
|
||||
// gh extracts one artifact into -D itself and several into one directory per artifact.
|
||||
const target = artifacts.length === 1 ? path.join(dir, artifacts[0]!) : dir;
|
||||
const got = gh(['run', 'download', id, '-R', options.repo, ...artifacts.flatMap(name => ['-n', name]), '-D', target], left());
|
||||
if (got.status !== 0) failed = ghFailureReason(got.stderr);
|
||||
}
|
||||
if (failed) {
|
||||
unresolved.push({ runId: id, reason: failed });
|
||||
lines.push(`run ${id}: ${failed}; its evidence stays unresolved`);
|
||||
continue;
|
||||
}
|
||||
const negatives = path.join(dir, 'recovered-negatives');
|
||||
const panelNegatives = negativesFromRun(dir, run, negatives);
|
||||
const receiptDirs = [negatives, ...artifacts.filter(name => name !== 'paid-plan').flatMap(name =>
|
||||
[path.join(dir, name, 'receipts'), path.join(dir, name, 'paid-report', 'report-receipts')]),
|
||||
...(mode === 'dispatch' ? [path.join(dir, 'paid-plan', 'store'), path.join(dir, 'paid-plan', 'receipts')] : [])];
|
||||
const merged = mergeReceiptDirs(options.store, receiptDirs);
|
||||
recovered += merged;
|
||||
processed.add(id);
|
||||
lines.push(`run ${id} (${run.conclusion}): merged ${merged} receipt(s), ${panelNegatives} reconstructed negative(s) from completed panels and shards`);
|
||||
}
|
||||
} finally {
|
||||
fs.rmSync(work, { recursive: true, force: true });
|
||||
}
|
||||
if (!candidates.length) lines.push('no cancelled runs to recover');
|
||||
return finish({ processed: [...processed], unresolved, recovered });
|
||||
}
|
||||
|
||||
/** plan-slices: reuse is on only when recovery ran, did not degrade and left nothing unresolved. */
|
||||
export function recoveryDecision(store: string): { reuse: boolean; reason: string } {
|
||||
const checkpoint = readCheckpoint(store);
|
||||
if (!checkpoint) return { reuse: false, reason: 'receipt recovery unavailable (no checkpoint from the recover-receipts job); cases execute fresh' };
|
||||
if (checkpoint.degraded) return { reuse: false, reason: `receipt recovery degraded: ${checkpoint.degraded}; cases execute fresh` };
|
||||
if (checkpoint.unresolved.length) {
|
||||
return { reuse: false, reason: `cancelled run(s) ${checkpoint.unresolved.map(u => u.runId).join(', ')} not reconciled (${checkpoint.unresolved[0]!.reason}); cases execute fresh and the next push reconciles them` };
|
||||
}
|
||||
return { reuse: true, reason: `receipt reuse on; ${checkpoint.recovered} receipt(s) recovered from cancelled runs` };
|
||||
}
|
||||
|
||||
const USAGE = `usage:
|
||||
bun run scripts/recover-receipts.ts collect --repo <owner/name> --pr <n> --head-repo <owner/name> --branch <head branch>
|
||||
--run-id <this run> --store <dir> [--store-key <restored cache key>] [--mode pr|dispatch] [--budget-seconds 60]
|
||||
bun run scripts/recover-receipts.ts decide <store> [--github-output <file>]
|
||||
Recovery never fails: on any gh error it records why and the run executes unreduced.`;
|
||||
|
||||
if (import.meta.main) {
|
||||
const [command, ...rest] = process.argv.slice(2);
|
||||
const flag = (name: string) => { const i = rest.indexOf(`--${name}`); return i >= 0 ? rest[i + 1] : undefined; };
|
||||
if (!command || command === '--help' || command === '-h') { console.log(USAGE); process.exit(command ? 0 : 2); }
|
||||
if (command === 'decide' && rest[0]) {
|
||||
const decision = recoveryDecision(rest[0]);
|
||||
const out = flag('github-output');
|
||||
if (out) fs.appendFileSync(out, `reuse=${decision.reuse ? 'on' : 'off'}\n`);
|
||||
console.log(`- Receipt reuse: ${decision.reason}`);
|
||||
} else if (command === 'collect') {
|
||||
const [store, repo, headRepo, branch] = [flag('store'), flag('repo'), flag('head-repo'), flag('branch')];
|
||||
const [pr, selfRunId] = [Number(flag('pr')), Number(flag('run-id'))];
|
||||
if (!store || !repo || !headRepo || !branch || !Number.isSafeInteger(pr) || !Number.isSafeInteger(selfRunId)) {
|
||||
console.error(USAGE); process.exit(2);
|
||||
}
|
||||
const { lines } = collectRecovery({ repo, pr, headRepo, branch, selfRunId, store, storeKey: flag('store-key'),
|
||||
mode: flag('mode') === 'dispatch' ? 'dispatch' : 'pr', budgetMs: Number(flag('budget-seconds') ?? '60') * 1000 });
|
||||
console.log('### Receipt recovery');
|
||||
for (const line of lines) console.log(`- ${line}`);
|
||||
} else { console.error(USAGE); process.exit(2); }
|
||||
}
|
||||
@@ -0,0 +1,197 @@
|
||||
/**
|
||||
* CEO-23 preserved-detection replay for the PR paid lane (W1).
|
||||
*
|
||||
* Replays recorded PR runs' changed files through today's selector and
|
||||
* reports, per push, the recorded mode and case count against the new mode
|
||||
* and case count, the gate cases the new selector no longer runs, and every
|
||||
* MISS: a case that actually failed in that run, still exists, and depends
|
||||
* on the diff (today's touchfiles), yet is not selected now. Acceptance is
|
||||
* zero misses. The replay uses today's dependency graph (ENG-18): cases that
|
||||
* no longer exist are counted separately, never as misses.
|
||||
*
|
||||
* Inputs come from GitHub with `gh` (read-only): each run's `paid-plan`
|
||||
* manifest, its trial-outcomes artifact, and the three-dot compare of the
|
||||
* merge-base with the run's head SHA. Downloads are cached under --cache.
|
||||
*
|
||||
* bun run scripts/replay-pr-selection.ts --runs <runs.json> [--cache <dir>] [--repo garrytan/gstack] [--json]
|
||||
*
|
||||
* <runs.json> is `gh run list --workflow evals.yml --json databaseId,headSha,event,createdAt,conclusion -L 200` output.
|
||||
*/
|
||||
import * as fs from 'node:fs';
|
||||
import * as os from 'node:os';
|
||||
import * as path from 'node:path';
|
||||
import { spawnSync } from 'node:child_process';
|
||||
import { computePaidCaseSelection } from './test-paid-shards';
|
||||
import { packageChangeOnlyVersion } from './test-pr-profile';
|
||||
import { E2E_TIERS, E2E_TOUCHFILES, GLOBAL_TOUCHFILES } from '../test/helpers/touchfiles-data';
|
||||
import { matchGlob } from '../test/helpers/test-selection';
|
||||
import { derivedDependencies } from './pr-dependencies';
|
||||
import { PR_PROFILE_MAPS } from './test-pr-profile';
|
||||
|
||||
export interface ReplayInput {
|
||||
runId: number;
|
||||
oldMode: string;
|
||||
oldCases: number;
|
||||
changedFiles: string[];
|
||||
/** The compare API lists at most 300 files; a longer diff is replayed on that prefix and flagged. */
|
||||
truncated: boolean;
|
||||
failedCases: string[];
|
||||
packageVersionOnly: boolean;
|
||||
}
|
||||
|
||||
export interface ReplayRow {
|
||||
runId: number;
|
||||
/** Changed files in the replayed diff (the wave/ordinary split reads it). */
|
||||
changed: number;
|
||||
/** Changed files missing from today's tree (deleted or renamed since); their consumers are the live files that still reference them. */
|
||||
vanished: number;
|
||||
oldMode: string;
|
||||
newMode: string;
|
||||
oldCases: number;
|
||||
newCases: number;
|
||||
dropped: string[];
|
||||
misses: string[];
|
||||
retiredFailures: string[];
|
||||
truncated: boolean;
|
||||
error?: string;
|
||||
}
|
||||
|
||||
/**
|
||||
* Whether a case depends on any changed file under today's dependency graph:
|
||||
* its touchfiles, the globals, and the derived reference closure
|
||||
* (scripts/pr-dependencies.ts, its global runner closure included).
|
||||
*/
|
||||
export function caseTouchesDiff(id: string, changedFiles: readonly string[]): boolean {
|
||||
const patterns = [...(E2E_TOUCHFILES[id] ?? []), ...GLOBAL_TOUCHFILES];
|
||||
const derived = derivedDependencies(PR_PROFILE_MAPS, path.resolve(import.meta.dir, '..'));
|
||||
// A path absent from today's tree reaches a case through the live files that still reference it.
|
||||
const via = (file: string) => derived.tracked.has(file) ? [file] : derived.referencers(file);
|
||||
return changedFiles.some(file => patterns.some(pattern => matchGlob(file, pattern))
|
||||
|| via(file).some(source => derived.global.has(source) || !!derived.e2e.get(source)?.has(id)));
|
||||
}
|
||||
|
||||
export function replayRun(input: ReplayInput, oldSelection: readonly string[]): ReplayRow {
|
||||
const tracked = derivedDependencies(PR_PROFILE_MAPS, path.resolve(import.meta.dir, '..')).tracked;
|
||||
const base = { runId: input.runId, oldMode: input.oldMode, oldCases: input.oldCases, truncated: input.truncated,
|
||||
changed: input.changedFiles.length, vanished: input.changedFiles.filter(file => !tracked.has(file)).length };
|
||||
const retiredFailures = input.failedCases.filter(id => !Object.hasOwn(E2E_TIERS, id));
|
||||
try {
|
||||
const result = computePaidCaseSelection({ profile: 'pr', env: {}, changedFiles: input.changedFiles, packageVersionOnly: input.packageVersionOnly });
|
||||
const selected = new Set(result.selection.e2e ?? Object.keys(E2E_TIERS).filter(id => E2E_TIERS[id] === 'gate'));
|
||||
const misses = input.failedCases.filter(id => Object.hasOwn(E2E_TIERS, id) && E2E_TIERS[id] === 'gate'
|
||||
&& caseTouchesDiff(id, input.changedFiles) && !selected.has(id));
|
||||
return { ...base, newMode: result.coverage?.mode ?? 'full', newCases: selected.size,
|
||||
dropped: oldSelection.filter(id => Object.hasOwn(E2E_TIERS, id) && !selected.has(id)), misses, retiredFailures };
|
||||
} catch (error) {
|
||||
// The PR profile refuses prompts without coverage; CI fails the plan the same way, so nothing is silently dropped.
|
||||
return { ...base, newMode: 'needs-full-validation', newCases: 0, dropped: [], misses: [], retiredFailures,
|
||||
error: error instanceof Error ? error.message.slice(0, 200) : String(error) };
|
||||
}
|
||||
}
|
||||
|
||||
/** Pushes whose diff has at least this many files are wave branches (often legitimately the full gate). */
|
||||
export const WAVE_DIFF_FILES = 100;
|
||||
|
||||
function segment(label: string, rows: readonly ReplayRow[]): string {
|
||||
const pct = (n: number) => rows.length ? `${Math.round((100 * n) / rows.length)}%` : 'n/a';
|
||||
const count = (mode: string) => rows.filter(row => row.newMode === mode).length;
|
||||
const cases = rows.map(row => row.newCases).sort((a, b) => a - b);
|
||||
const median = cases.length ? cases[Math.floor((cases.length - 1) / 2)] : 0;
|
||||
return `- ${label}: ${rows.length} push(es); full-fallback ${rows.filter(row => row.oldMode === 'full-fallback').length} recorded -> ${count('full-fallback')} (${pct(count('full-fallback'))}) replayed; `
|
||||
+ `dependents ${count('dependents')}, pr ${count('pr')}; median selected gate cases ${median} (recorded median ${rows.map(row => row.oldCases).sort((a, b) => a - b)[Math.floor((rows.length - 1) / 2)] ?? 0})`;
|
||||
}
|
||||
|
||||
export function summarize(rows: readonly ReplayRow[]): string[] {
|
||||
const fallbackOld = rows.filter(row => row.oldMode === 'full-fallback').length;
|
||||
const fallbackNew = rows.filter(row => row.newMode === 'full-fallback').length;
|
||||
const misses = rows.flatMap(row => row.misses.map(id => `${row.runId}:${id}`));
|
||||
const pct = (n: number) => rows.length ? `${Math.round((100 * n) / rows.length)}%` : 'n/a';
|
||||
return [
|
||||
`PR selection replay over ${rows.length} recorded push(es) (today's dependency graph)`,
|
||||
`- full-fallback: ${fallbackOld} (${pct(fallbackOld)}) recorded -> ${fallbackNew} (${pct(fallbackNew)}) replayed`,
|
||||
segment(`wave branches (diff >= ${WAVE_DIFF_FILES} files)`, rows.filter(row => row.changed >= WAVE_DIFF_FILES)),
|
||||
segment(`ordinary pushes (diff < ${WAVE_DIFF_FILES} files)`, rows.filter(row => row.changed < WAVE_DIFF_FILES)),
|
||||
`- pushes whose diff names files missing from today's tree: ${rows.filter(row => row.vanished > 0).length} (placed through the live files that still reference them)`,
|
||||
`- needs full validation (plan refuses, as in CI): ${rows.filter(row => row.newMode === 'needs-full-validation').length}`,
|
||||
`- diffs truncated at the compare API's 300-file limit: ${rows.filter(row => row.truncated).length}`,
|
||||
`- failed cases that no longer exist: ${new Set(rows.flatMap(row => row.retiredFailures)).size}`,
|
||||
`- MISSES (failed, depends on the diff, not selected now): ${misses.length}${misses.length ? ` -> ${misses.join(', ')}` : ''}`,
|
||||
];
|
||||
}
|
||||
|
||||
function gh(args: string[]): string | null {
|
||||
const result = spawnSync('gh', args, { encoding: 'utf8', timeout: 60_000, maxBuffer: 256 * 1024 * 1024 });
|
||||
return result.status === 0 ? result.stdout : null;
|
||||
}
|
||||
|
||||
function loadInput(repo: string, run: { databaseId: number; headSha: string }, cache: string): { input: ReplayInput; oldSelection: string[] } | null {
|
||||
const dir = path.join(cache, String(run.databaseId));
|
||||
const plan = path.join(dir, 'plan', 'manifest.json');
|
||||
if (!fs.existsSync(plan)) gh(['run', 'download', String(run.databaseId), '-R', repo, '-n', 'paid-plan', '-D', path.join(dir, 'plan')]);
|
||||
if (!fs.existsSync(plan)) return null;
|
||||
const manifest = JSON.parse(fs.readFileSync(plan, 'utf8'));
|
||||
if (manifest.profile !== 'pr') return null;
|
||||
const outcomes = path.join(dir, 'outcomes');
|
||||
if (!fs.existsSync(outcomes)) {
|
||||
for (const name of ['trial-outcomes-pr-a1', 'trial-outcomes-pr']) {
|
||||
if (gh(['run', 'download', String(run.databaseId), '-R', repo, '-n', name, '-D', outcomes]) !== null) break;
|
||||
}
|
||||
}
|
||||
const failedCases = new Set<string>();
|
||||
if (fs.existsSync(outcomes)) {
|
||||
for (const file of fs.readdirSync(outcomes).filter(name => name.endsWith('.jsonl'))) {
|
||||
for (const line of fs.readFileSync(path.join(outcomes, file), 'utf8').split('\n').filter(Boolean)) {
|
||||
try { const record = JSON.parse(line); if (record.outcome === 'failed' && typeof record.case === 'string') failedCases.add(record.case); } catch { /* skip */ }
|
||||
}
|
||||
}
|
||||
}
|
||||
const compareFile = path.join(dir, 'compare.json');
|
||||
if (!fs.existsSync(compareFile)) {
|
||||
const compare = gh(['api', `repos/${repo}/compare/main...${run.headSha}`, '--jq', '{files: [.files[] | {filename, previous_filename}], merge_base: .merge_base_commit.sha}']);
|
||||
if (compare === null) return null;
|
||||
fs.writeFileSync(compareFile, compare);
|
||||
}
|
||||
const compare = JSON.parse(fs.readFileSync(compareFile, 'utf8')) as { files: Array<{ filename: string; previous_filename?: string }>; merge_base: string };
|
||||
const changedFiles = [...new Set(compare.files.flatMap(f => [f.filename, ...(f.previous_filename ? [f.previous_filename] : [])]))];
|
||||
let packageVersionOnly = false;
|
||||
if (changedFiles.includes('package.json')) {
|
||||
const read = (ref: string) => gh(['api', `repos/${repo}/contents/package.json?ref=${ref}`, '-H', 'Accept: application/vnd.github.raw']);
|
||||
const [before, after] = [read(compare.merge_base), read(run.headSha)];
|
||||
packageVersionOnly = before !== null && after !== null && packageChangeOnlyVersion(before, after);
|
||||
}
|
||||
return {
|
||||
input: { runId: run.databaseId, oldMode: manifest.prCoverage?.mode ?? 'pr', oldCases: (manifest.selection?.e2e ?? []).length,
|
||||
changedFiles, truncated: compare.files.length >= 300, failedCases: [...failedCases], packageVersionOnly },
|
||||
oldSelection: manifest.selection?.e2e ?? [],
|
||||
};
|
||||
}
|
||||
|
||||
if (import.meta.main) {
|
||||
const args = process.argv.slice(2);
|
||||
const flag = (name: string) => { const i = args.indexOf(`--${name}`); return i >= 0 ? args[i + 1] : undefined; };
|
||||
const runsFile = flag('runs');
|
||||
if (args.includes('--help') || !runsFile) {
|
||||
console.log('usage: bun run scripts/replay-pr-selection.ts --runs <runs.json> [--cache <dir>] [--repo garrytan/gstack] [--json]\n'
|
||||
+ 'runs.json: gh run list --workflow evals.yml --json databaseId,headSha,event,createdAt,conclusion -L 200');
|
||||
process.exit(runsFile || args.includes('--help') ? 0 : 2);
|
||||
}
|
||||
const repo = flag('repo') ?? 'garrytan/gstack';
|
||||
const cache = flag('cache') ?? path.join(os.tmpdir(), 'gstack-pr-replay');
|
||||
const runs = (JSON.parse(fs.readFileSync(runsFile, 'utf8')) as Array<{ databaseId: number; headSha: string; event: string; conclusion: string }>)
|
||||
.filter(run => run.event === 'pull_request' && !['action_required', 'skipped'].includes(run.conclusion));
|
||||
const rows: ReplayRow[] = [];
|
||||
for (const run of runs) {
|
||||
const loaded = loadInput(repo, run, cache);
|
||||
if (loaded) rows.push(replayRun(loaded.input, loaded.oldSelection));
|
||||
}
|
||||
if (args.includes('--json')) console.log(JSON.stringify(rows, null, 2));
|
||||
else {
|
||||
console.log('| run | recorded mode | cases | replayed mode | cases | dropped gate cases | misses |');
|
||||
console.log('|---|---|---|---|---|---|---|');
|
||||
for (const row of rows) {
|
||||
console.log(`| ${row.runId}${row.truncated ? ' (diff >300 files)' : ''} | ${row.oldMode} | ${row.oldCases} | ${row.newMode} | ${row.newCases} | ${row.dropped.length} | ${row.misses.join(', ') || '0'} |`);
|
||||
}
|
||||
console.log('');
|
||||
for (const line of summarize(rows)) console.log(line);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,49 @@
|
||||
#!/usr/bin/env bun
|
||||
/**
|
||||
* One-release stubs for retired package scripts. Each retired name in
|
||||
* package.json runs `bun run scripts/retired-command.ts <name>`, which prints
|
||||
* the replacement and exits 1, so an old command in an agent's memory or a
|
||||
* stale doc fails loudly instead of running zero cases or the wrong lane.
|
||||
*
|
||||
* CONTRIBUTING.md "Retired commands" carries the same table. TODOS.md
|
||||
* "Remove one-release command stubs" deletes this file and the stub entries
|
||||
* in the next release.
|
||||
*/
|
||||
|
||||
export interface RetiredCommand {
|
||||
replacement: string;
|
||||
reason: string;
|
||||
}
|
||||
|
||||
export const RETIRED_COMMANDS: Record<string, RetiredCommand> = {
|
||||
'test:evals': { replacement: 'bun run eval:bg:pr', reason: 'tierless run skipped every tier-gated paid file' },
|
||||
'test:evals:all': { replacement: 'bun run eval:bg:release', reason: 'tierless run skipped every tier-gated paid file' },
|
||||
'test:e2e': { replacement: 'bun run eval:bg:pr', reason: 'tierless run skipped every tier-gated paid file' },
|
||||
'test:e2e:all': { replacement: 'bun run eval:bg:release', reason: 'tierless run skipped every tier-gated paid file' },
|
||||
'test:gate': { replacement: 'bun run test:gate:sharded', reason: 'the single-process fan-out never completed a run' },
|
||||
'test:periodic': { replacement: 'bun run test:periodic:sharded', reason: 'the single-process fan-out never completed a run' },
|
||||
'test:codex': { replacement: 'bun run test:periodic:sharded', reason: 'it set no EVALS_TIER, so both periodic-tier Codex files ran zero cases' },
|
||||
'test:codex:all': { replacement: 'bun run test:periodic:sharded', reason: 'it set no EVALS_TIER, so both periodic-tier Codex files ran zero cases' },
|
||||
'eval:bg': { replacement: 'bun run eval:bg:pr', reason: 'it detached the retired tierless test:evals' },
|
||||
'eval:bg:all': { replacement: 'bun run eval:bg:release', reason: 'it detached the retired tierless test:evals:all' },
|
||||
'eval:flake-rank': { replacement: 'bun run eval:pass-rates', reason: 'it was a second name for the same script' },
|
||||
'eval:watch': { replacement: 'tail the gstack-detach log under ~/.gstack-dev/eval-runs/, or `gh run watch <run-id>` for a CI run', reason: 'it read a file only the unsharded runner wrote, so it showed nothing for sharded runs' },
|
||||
'test:audit': { replacement: 'bun run test', reason: 'test/audit-compliance.test.ts already runs in the free suite' },
|
||||
};
|
||||
|
||||
export function retiredMessage(name: string): string | null {
|
||||
const entry = RETIRED_COMMANDS[name];
|
||||
if (!entry) return null;
|
||||
return `\`bun run ${name}\` is retired (${entry.reason}).\nUse instead: ${entry.replacement}\nSee CONTRIBUTING.md "Retired commands" for the full table.`;
|
||||
}
|
||||
|
||||
if (import.meta.main) {
|
||||
const name = process.argv[2] ?? '';
|
||||
const message = retiredMessage(name);
|
||||
if (!message) {
|
||||
console.error(`retired-command: unknown name "${name}". Known: ${Object.keys(RETIRED_COMMANDS).join(', ')}`);
|
||||
process.exit(2);
|
||||
}
|
||||
console.error(message);
|
||||
process.exit(1);
|
||||
}
|
||||
+41
-402
@@ -88,6 +88,9 @@ import { createHash, randomUUID } from 'node:crypto';
|
||||
import { isPaidTestFile } from '../test/helpers/paid-test-set';
|
||||
import { resolveStateRoot } from '../lib/state-root';
|
||||
import { attributeFreeHomeWriters, guardFreeHome, sharedFreeHome, type FreeHomeGuardFactory } from './lib/free-home-guard';
|
||||
import { appendStepSummary, ciHealthSummary, SEED_REFRESH_COMMAND, unseededWarning, windowsCurationLine } from './lib/free-ci-health';
|
||||
import { curateWindowsSafe, type CurationResult } from './lib/windows-curation';
|
||||
export { curateWindowsSafe, detectWindowsFragility, KNOWN_WINDOWS_INCOMPATIBLE, type CurationResult } from './lib/windows-curation';
|
||||
import {
|
||||
BunTestOutputClassifier,
|
||||
createShardSandbox,
|
||||
@@ -142,313 +145,6 @@ export const TEST_ROOTS = [
|
||||
] as const;
|
||||
const TEST_FILE_REGEX = /\.test\.(?:[cm]?[jt]s|tsx|jsx)$/;
|
||||
|
||||
// POSIX-only patterns that indicate a test will fail on windows-latest no
|
||||
// matter how the runner shards. Codex's v1.18.0.0 review flagged the first
|
||||
// three as concrete examples in the existing free suite (test/ship-version-sync.test.ts:72,
|
||||
// test/helpers/providers/claude.ts:22, package.json:12). We scan the test's
|
||||
// own content here so the filter stays automatic as new tests land. The
|
||||
// "Windows-incompatible APIs" patterns at the bottom were added after the
|
||||
// first windows-free-tests CI run surfaced concrete failure modes.
|
||||
const WINDOWS_FRAGILE_PATTERNS: Array<{ pattern: RegExp; reason: string }> = [
|
||||
// Hardcoded POSIX shells / commands.
|
||||
{ pattern: /['"`]\/bin\/(?:ba)?sh/, reason: 'hardcoded /bin/sh or /bin/bash' },
|
||||
{ pattern: /spawnSync\(['"]sh['"],|spawn\(['"]sh['"],|exec\(['"]sh /, reason: 'spawn("sh", ...)' },
|
||||
{ pattern: /['"]bash -c['"]|['"]sh -c['"]/, reason: 'bash -c / sh -c' },
|
||||
{ pattern: /['"`]\/tmp\//, reason: 'raw /tmp/ path (use os.tmpdir())' },
|
||||
{ pattern: /['"]chmod\b/, reason: 'chmod shell command' },
|
||||
{ pattern: /['"]xargs\b/, reason: 'xargs pipeline' },
|
||||
{ pattern: /\bwhich claude\b/, reason: 'which claude (use Bun.which)' },
|
||||
// Windows-incompatible APIs.
|
||||
{ pattern: /\.mode\s*&\s*0o[0-7]+/, reason: 'POSIX file mode bitmask (mode & 0o600 etc — Windows fakes mode bits)' },
|
||||
{ pattern: /\.endsWith\(['"]\//, reason: 'hardcoded forward-slash path assertion (Windows uses \\\\)' },
|
||||
{ pattern: /['"]\.\/[a-zA-Z][^"']*['"]\)\s*\.\s*toBe\(true\)/, reason: 'forward-slash path comparison' },
|
||||
// Tests that spawn a bash shebang script in bin/ via spawnSync. Git Bash on
|
||||
// Windows can run `bash /path/to/script` but spawnSync(scriptPath, ...)
|
||||
// tries to execute the file directly via CreateProcess, which fails on the
|
||||
// shebang. The pattern matches `, 'bin'` as a path-join argument (closing
|
||||
// OR followed by another segment), which catches:
|
||||
// - path.join(ROOT, 'bin', 'script-name') — typical
|
||||
// - join(import.meta.dir, '..', 'bin', 'name') — destructured (diff-scope)
|
||||
// - path.join(ROOT, 'bin') — bare BIN constant (brain-sync)
|
||||
{ pattern: /,\s*['"]bin['"]\s*[,)]|['"]\.?\/?bin\/[a-z][\w-]+['"]/, reason: 'spawns bin/ shebang script (Windows CreateProcess does not parse shebangs)' },
|
||||
// Tests that launch a real Playwright browser. The windows-free-tests CI job
|
||||
// runs a curated subset that intentionally does NOT install Chromium —
|
||||
// browser bring-up on Windows is a separate concern (see PR #1238). Tests
|
||||
// matching `await foo.launch(` need Chromium and fail with "Executable
|
||||
// doesn't exist" on the runner.
|
||||
{ pattern: /await\s+\w+\.launch\(/, reason: 'launches Playwright browser (Chromium not installed in windows-free CI)' },
|
||||
// Tests that spawn the browse server as a subprocess via `bun run server.ts`.
|
||||
// The Bun → server.ts → Playwright path is the same one that doesn't work
|
||||
// on Windows (PR #1238 windows-pty-bun-pty-fix). Tests typically set
|
||||
// BROWSE_HEADLESS_SKIP=1 to skip the browser launch but still need a working
|
||||
// server, which they don't get on Windows.
|
||||
{ pattern: /BROWSE_HEADLESS_SKIP|spawn\(\[['"]bun['"],\s*['"]run['"]/, reason: 'spawns the browse server subprocess (Bun-driven path is Windows-broken)' },
|
||||
];
|
||||
|
||||
// Explicit known-Windows-incompatible test files that don't fit a regex
|
||||
// pattern. Listed here with the precise reason. Prefer adding a pattern above
|
||||
// when possible; this list is for environment-/runtime-specific tests where
|
||||
// the failure mode is structural rather than detectable via source-file scan.
|
||||
export const KNOWN_WINDOWS_INCOMPATIBLE: Array<{ file: string; reason: string }> = [
|
||||
{
|
||||
file: 'test/qa-evidence-producer.test.ts',
|
||||
reason: 'executes the registered Linux native actor and its inotify observer; portable capture and Windows job behavior are covered by qa-evidence.test.ts',
|
||||
},
|
||||
{
|
||||
file: 'test/qa-functional-fixture.test.ts',
|
||||
reason: 'executes graceful POSIX signal cancellation; Bun on Windows uses TerminateProcess and cannot run the fixture SIGTERM cleanup handler',
|
||||
},
|
||||
{
|
||||
file: 'test/qa-functional-observer-atomic.test.ts',
|
||||
reason: 'exercises real Linux inotify inode and directory watches through libc.so.6; Windows has no equivalent kernel interface',
|
||||
},
|
||||
{
|
||||
file: 'test/docsync-report-interface.test.ts',
|
||||
reason: 'executes registered native documentation callbacks with their real Linux inotify write observer before the model boundary',
|
||||
},
|
||||
{
|
||||
file: 'test/setup-gbrain-fixture.test.ts',
|
||||
reason: 'the fixture invokes real POSIX detector/verifier helpers through executable shebang wrappers',
|
||||
},
|
||||
{
|
||||
file: 'test/hermetic-skills-seeding.test.ts',
|
||||
reason: 'seeds the POSIX PTY skill runtime, whose embedded shell paths require a POSIX temporary root',
|
||||
},
|
||||
{
|
||||
file: 'test/hermetic-wiring.test.ts',
|
||||
reason: 'its runtime contract check seeds the POSIX PTY skill runtime; the curated Windows lane does not run that harness',
|
||||
},
|
||||
{
|
||||
file: 'test/pty-workspace-trust.test.ts',
|
||||
reason: 'launches the POSIX PTY harness with a fake executable and bound skill runtime',
|
||||
},
|
||||
{
|
||||
file: 'test/host-config.test.ts',
|
||||
reason: 'asserts "claude" binary on PATH (only true when running inside Claude Code, not on bare CI runner)',
|
||||
},
|
||||
{
|
||||
file: 'browse/test/findport.test.ts',
|
||||
reason: 'asserts Bun.serve.stop() is fire-and-forget — Bun behavior differs on Windows for this polyfill',
|
||||
},
|
||||
// First full run of the expanded lane (v1.66, 13 → ~258 files) surfaced
|
||||
// seven POSIX-bound files the content patterns cannot see (their
|
||||
// POSIX-ness is what they TEST, or arrives via a variable). Receipts:
|
||||
// PR #2593 windows-free-tests run 31918591602.
|
||||
{
|
||||
file: 'test/codex-under-codex-detection.test.ts',
|
||||
reason: 'drives the rendered preflight bash under a hardcoded POSIX PATH (/usr/bin:/bin) — bash is unreachable through that PATH on Windows, so every case sees empty output (v1.67 windows lane run 95234224148)',
|
||||
},
|
||||
{
|
||||
file: 'test/regression-pr1169-build-app-sed.test.ts',
|
||||
reason: 'tests sed escape sequences in build-app.sh — sed/bash are the subject under test',
|
||||
},
|
||||
{
|
||||
file: 'test/setup-conductor-worktree.test.ts',
|
||||
reason: 'tests ln -snf symlink semantics in the setup script — POSIX ln is the subject under test',
|
||||
},
|
||||
{
|
||||
file: 'test/artifacts-init-migration.test.ts',
|
||||
reason: 'runs a bash migration script + jq against a scaffolded git state — POSIX toolchain paths break under cmd spawn',
|
||||
},
|
||||
{
|
||||
file: 'test/gstack-decision-semantic.test.ts',
|
||||
reason: 'installs a fake gbrain SHEBANG SHIM on PATH; Windows spawn cannot exec shebang scripts',
|
||||
},
|
||||
{
|
||||
file: 'test/question-log-hook.test.ts',
|
||||
reason: 'spawns the PostToolUse hook script (bash shebang) directly; Windows spawn cannot exec it',
|
||||
},
|
||||
{
|
||||
file: 'browse/test/browser-skills-e2e.test.ts',
|
||||
reason: 'asserts forward-slash tier paths (<repo>/browser-skills/) that resolve with backslashes on Windows',
|
||||
},
|
||||
{
|
||||
file: 'design/test/variants-retry-after.test.ts',
|
||||
reason: 'wall-clock retry-timing assertions — flaky on the slow windows-latest runner even with widened bounds',
|
||||
},
|
||||
// Round-2 census (PR #2593 run 31919227507) after the first seven:
|
||||
{
|
||||
file: 'test/skill-census.test.ts',
|
||||
reason: 'census walk throws at module load on Windows (skill-census.ts:63) — the skills-tree symlink layout needs Developer Mode that CI runners lack',
|
||||
},
|
||||
{
|
||||
file: 'browse/test/browser-manager-unit.test.ts',
|
||||
reason: 'wedges the shard to its wall deadline on windows-latest (in-flight at kill); needs a Windows repro to diagnose — macOS + Linux lanes cover the file',
|
||||
},
|
||||
// Round-3 census (PR #2593 run 31919871680): the round-2 wedge had been
|
||||
// TRUNCATING its shard, so these seven only surfaced once shard 2 completed.
|
||||
// All the same POSIX-environment classes: PID/cmdline identity probing,
|
||||
// bash scripts as the subject under test, env-scrubbed child spawns.
|
||||
{
|
||||
file: 'browse/test/server-embedder-terminal-port.test.ts',
|
||||
reason: 'identity-based terminal-agent kill probes PID/cmdline with POSIX semantics; teardown asserts fail on windows-latest',
|
||||
},
|
||||
{
|
||||
file: 'design/test/daemon-discovery.test.ts',
|
||||
reason: 'verifyIdentity matches a spawned daemon via /proc-style cmdline probing — POSIX identity semantics',
|
||||
},
|
||||
{
|
||||
file: 'test/context-save-hardening.test.ts',
|
||||
reason: 'bash context-save/migration scripts (HOME-unset semantics, random-suffix path) are the subject under test',
|
||||
},
|
||||
{
|
||||
file: 'test/eval-list-cli.test.ts',
|
||||
reason: 'spawns the eval:list CLI via bun with a constructed env — bun resolution fails under Windows spawn',
|
||||
},
|
||||
{
|
||||
file: 'test/memory-cache-injection.test.ts',
|
||||
reason: 'exercises hook/deny-enforcement shell scripts — POSIX toolchain is the subject under test',
|
||||
},
|
||||
{
|
||||
file: 'test/migrations-v1.65.0.0.test.ts',
|
||||
reason: 'bash migration script (bunx re-fetch, .done markers) is the subject under test',
|
||||
},
|
||||
{
|
||||
file: 'test/question-preference-hook.test.ts',
|
||||
reason: 'spawns the PreToolUse preference hook (shebang script) directly; Windows spawn cannot exec it',
|
||||
},
|
||||
// Round-4 census (PR #2593 run 31920052810): unhandled errors with no
|
||||
// (fail) lines — attributed statically (the lane had no log artifact yet).
|
||||
{
|
||||
file: 'browse/test/browser-skill-commands.test.ts',
|
||||
reason: 'spawnSkill spawns bun with a constructed env — bun resolution fails under Windows spawn (unhandled, no (fail) line)',
|
||||
},
|
||||
{
|
||||
file: 'browse/test/security-audit-r2.test.ts',
|
||||
reason: 'symlink-attack fixtures (evil-link) need Developer Mode CI runners lack; expect(toThrow) fires unhandled on Windows',
|
||||
},
|
||||
// CSO comprehensive execution is qualified only for Linux containers behind
|
||||
// the POSIX watchdog and Unix-domain registry broker. Keep the portable
|
||||
// static/parser contracts in the Windows lane while leaving these exact
|
||||
// containment suites to the Linux and macOS gates.
|
||||
{
|
||||
file: 'test/cso-preparation-adversarial.test.ts',
|
||||
reason: 'exercises POSIX prepared-tree and archive-cache containment for qualified Linux Docker execution, which Windows does not admit',
|
||||
},
|
||||
{
|
||||
file: 'test/cso-preparation-container.test.ts',
|
||||
reason: 'asserts POSIX permission and symlink semantics for inert exports consumed by qualified Linux Docker execution',
|
||||
},
|
||||
{
|
||||
file: 'test/cso-preparation-executor.test.ts',
|
||||
reason: 'executes the Linux Docker acquisition path and its Unix-domain registry broker; comprehensive execution is unavailable on Windows',
|
||||
},
|
||||
{
|
||||
file: 'test/cso-verification-cleanup.test.ts',
|
||||
reason: 'spawns the POSIX detached watchdog used by contained repair verification, which Windows intentionally leaves unavailable',
|
||||
},
|
||||
{
|
||||
file: 'test/cso-witness.test.ts',
|
||||
reason: 'tests the contained repair witness with POSIX private-directory and compiled-helper assumptions; comprehensive execution is unavailable on Windows',
|
||||
},
|
||||
{
|
||||
file: 'test/shard-engine-equivalence.test.ts',
|
||||
reason: 'its classification golden was recorded from the POSIX runners (process-group wall kill); the win32 engine path is pinned by the mocked-platform case in shard-engine.test.ts',
|
||||
},
|
||||
{
|
||||
file: 'test/cso-scanner-cli.test.ts',
|
||||
reason: 'drives the prebuilt POSIX CSO launcher with /usr/bin/git and a POSIX-only PATH; native Windows launcher behavior is covered by the dedicated cso-windows-launcher gate',
|
||||
},
|
||||
];
|
||||
|
||||
// Force-include overrides: files a WINDOWS_FRAGILE_PATTERNS regex excludes for
|
||||
// a reason that does not actually apply to them. Each entry documents WHY the
|
||||
// pattern hit is a false positive — the point of these files is Windows
|
||||
// coverage, so auto-excluding them defeats the regression tests they carry.
|
||||
const KNOWN_WINDOWS_SAFE: Array<{ file: string; reason: string }> = [
|
||||
{ file: 'test/ship-hook-windows-paths.test.ts', reason: 'runs bin/ helpers through explicit bash and Bun argv with forward-slash paths; never executes a shebang; the path-spelling simulation is skipIf win32' },
|
||||
{ file: 'test/state-root-parity.test.ts', reason: 'runs the bash twin and lib/state-root.ts over an env table with PATH empty; no shebang execution, raw-string comparison is platform-neutral' },
|
||||
{ file: 'test/generator-eexist.test.ts',
|
||||
reason: 'E4: runs the generators through Bun argv with the Windows EEXIST emulation preload; no shebang execution' },
|
||||
{ file: 'test/gstack-config-gbrain-refresh.test.ts', reason: 'E6: runs bin/gstack-config through explicit bash; its gbrain/python3 shims are found by bash PATH lookup, never launched by CreateProcess' },
|
||||
{
|
||||
file: 'test/qa-evidence.test.ts',
|
||||
reason: 'invokes the production helper through Bun argv and exercises native Windows job cleanup, private file captures and backpressured receipt output',
|
||||
},
|
||||
{
|
||||
file: 'test/qa-evidence-selection.test.ts',
|
||||
reason: 'bin/ strings are literal dependency and Windows-selection assertions; no native actor or shebang command is launched',
|
||||
},
|
||||
{
|
||||
file: 'test/qa-deadline.test.ts',
|
||||
reason: 'launches the guard through Bun argv; mode assertions and POSIX signal cases are platform-gated, while Windows job cleanup must execute natively',
|
||||
},
|
||||
{
|
||||
file: 'test/qa-deadline-selection.test.ts',
|
||||
reason: 'bin/ strings are dependency-selection inputs; this suite never launches a shebang executable',
|
||||
},
|
||||
{
|
||||
file: 'test/shared-libs-source-reads.test.ts',
|
||||
reason: 'bin/ literal is a mocked launch assertion; actual worktree fingerprinting explicitly invokes Bash on Windows',
|
||||
},
|
||||
{
|
||||
file: 'test/claude-code-windows-job.test.ts',
|
||||
reason: 'invokes Bun directly; verifies Windows job containment at the standalone CLI boundary',
|
||||
},
|
||||
{
|
||||
file: 'test/claude-code-runner.test.ts',
|
||||
// The bin/ path is launched through process.execPath (Bun), never as a shebang; keep taskkill supervision in the Windows lane.
|
||||
reason: 'invokes the runner via Bun argv; fake CLI and timeout descendant assertions cover native Windows taskkill',
|
||||
},
|
||||
{
|
||||
file: 'test/setup-gbrain-remote-caller.test.ts',
|
||||
// bin is an expected PATH component; the adapter injects the SDK boundary
|
||||
// and never launches a shebang. Keep the native delimiter cases in CI.
|
||||
reason: 'replays the registered SDK callback with fixture-only bin paths; covers native Windows PATH composition',
|
||||
},
|
||||
{
|
||||
file: 'test/cso-windows-build-contract.test.ts',
|
||||
// bin is a temporary staging directory. The adapter injects spawnSync;
|
||||
// real PowerShell/native execution remains in cso-windows-launcher.
|
||||
reason: 'replays the native build callbacks with an injected subprocess; bin paths are staging fixtures, not shebang launches',
|
||||
},
|
||||
{
|
||||
file: 'test/setup-windows-rerun-refresh.test.ts',
|
||||
// Trips the "spawns bin/ shebang script" pattern via path.join(..., 'bin',
|
||||
// 'tool.sh') fixture paths, but every spawn goes through test/helpers/bash-script.ts
|
||||
// (bash <tempfile>) — Git Bash executes it fine on windows-latest, with no argv-length ceiling. This file IS
|
||||
// the #2444 Windows regression coverage (IS_WINDOWS=1 copy-refresh path);
|
||||
// excluding it here would keep the bug class unexercised on the one
|
||||
// platform it bites.
|
||||
reason: 'bin/ hits are fixture path segments; spawns bash explicitly — the IS_WINDOWS=1 refresh path must run on windows-latest',
|
||||
},
|
||||
{
|
||||
file: 'test/uninstall-windows-copies.test.ts',
|
||||
// Trips the "spawns bin/ shebang script" pattern via the
|
||||
// path.join(ROOT, 'bin', 'gstack-uninstall') constant, but the script is
|
||||
// always spawned through spawnSync('bash', [UNINSTALL, ...]). This file
|
||||
// carries the #2563 Windows real-dir-copy uninstall coverage — the bug
|
||||
// ONLY reproduces on the copy install shape windows-latest exercises.
|
||||
// The symlink-shape describe block self-skips on win32.
|
||||
reason: 'bin/ hit is a bash-spawned script path; #2563 real-dir uninstall coverage must run on windows-latest',
|
||||
},
|
||||
{
|
||||
file: 'browse/test/file-permissions.test.ts',
|
||||
// Trips the POSIX-mode-bitmask pattern, but every `mode & 0o777` assertion
|
||||
// is platform-guarded: win32-only tests return early, POSIX-only tests
|
||||
// guard the bitmask behind `process.platform !== 'win32'`, and the
|
||||
// symlink-skip regression test both wraps symlinkSync in try/catch
|
||||
// (runners without Developer Mode can't create symlinks) and guards its
|
||||
// bitmask — on win32 it asserts behavior (warns, skips, doesn't throw,
|
||||
// target stays usable), never fake Windows mode bits (dirs stat 0o777
|
||||
// there, so a 0o755 expectation fails on runner semantics, not our code).
|
||||
// This file carries the win32-only icacls-by-SID regression tests, which
|
||||
// can ONLY execute on windows-latest — excluding it here means the
|
||||
// machine-account ACL lockout regression is never exercised on the one
|
||||
// platform it bricks.
|
||||
reason: 'every mode-bitmask assertion is guarded off win32 (behavior asserted instead); win32-only ACL regression tests must run on windows-latest',
|
||||
},
|
||||
{
|
||||
file: 'browse/test/terminal-agent-owner-watchdog.test.ts',
|
||||
// Trips the spawn(['bun','run',...]) pattern, whose reason is the
|
||||
// Playwright-bound browse server. This test spawns terminal-agent.ts,
|
||||
// which imports only fs/path/crypto + local helpers (no Playwright, no
|
||||
// PTY at module scope) and boots under Bun on Windows — the owner-PID
|
||||
// orphan leak it pins was reported on Windows (#2019).
|
||||
reason: 'spawns terminal-agent (no Playwright), not the browse server; owner-orphan leak is a Windows defect',
|
||||
},
|
||||
];
|
||||
|
||||
export const DEFAULT_SHARD_COUNT = 20;
|
||||
// Per-test timeout passed to `bun test --timeout`. 30s matches what
|
||||
// package.json's `test` script used before it was repointed at this runner —
|
||||
@@ -536,20 +232,6 @@ export function fullSuiteJobs(platform: NodeJS.Platform = process.platform): num
|
||||
return Math.max(1, Math.min(maxFullSuiteJobs(platform), availableCpus));
|
||||
}
|
||||
|
||||
/**
|
||||
* Files that crash or wedge Bun's --parallel WORKERS but run fine in a plain
|
||||
* serial process. Full-suite mode now uses shard PROCESSES (no workers), so
|
||||
* this list is inert placement-wise — retained as the paper trail of why the
|
||||
* one-invocation --parallel strategy was abandoned, and as the exclusion list
|
||||
* should anyone re-attempt it on a newer Bun.
|
||||
*/
|
||||
export const WORKER_HOSTILE: Record<string, string> = {
|
||||
'browse/test/security-live-playwright.test.ts':
|
||||
'Bun 1.3.13 segfaults running this file in a --parallel worker ("panic: '
|
||||
+ 'Segmentation fault ... a bug in Bun"), and the crashed-worker retry then '
|
||||
+ 'wedges the whole invocation past the wall clock. Passes serially.',
|
||||
};
|
||||
|
||||
/**
|
||||
* Exclusive host-state fixtures: run in ONE serial shard AFTER the parallel
|
||||
* shards. The public name is retained for callers of the original tree-write
|
||||
@@ -568,22 +250,6 @@ export function isFreeTestFile(relativePath: string): boolean {
|
||||
return !isPaidTestFile(normalized);
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns the first POSIX-only pattern hit in the file, or null if Windows-safe.
|
||||
*/
|
||||
export function detectWindowsFragility(absolutePath: string): { reason: string } | null {
|
||||
let content: string;
|
||||
try {
|
||||
content = fs.readFileSync(absolutePath, 'utf-8');
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
for (const { pattern, reason } of WINDOWS_FRAGILE_PATTERNS) {
|
||||
if (pattern.test(content)) return { reason };
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
function walkTestFiles(dirPath: string): string[] {
|
||||
const entries = fs.readdirSync(dirPath, { withFileTypes: true });
|
||||
const files: string[] = [];
|
||||
@@ -615,37 +281,6 @@ export function collectFreeTestFiles(rootDir = ROOT): string[] {
|
||||
return [...discovered].sort();
|
||||
}
|
||||
|
||||
export interface CurationResult {
|
||||
safe: string[];
|
||||
excluded: Array<{ file: string; reason: string }>;
|
||||
}
|
||||
|
||||
export function curateWindowsSafe(files: string[], rootDir = ROOT): CurationResult {
|
||||
const safe: string[] = [];
|
||||
const excluded: Array<{ file: string; reason: string }> = [];
|
||||
const knownBad = new Map(KNOWN_WINDOWS_INCOMPATIBLE.map((e) => [e.file, e.reason]));
|
||||
const knownSafe = new Set(KNOWN_WINDOWS_SAFE.map((e) => e.file));
|
||||
for (const relativePath of files) {
|
||||
const knownReason = knownBad.get(relativePath);
|
||||
if (knownReason) {
|
||||
excluded.push({ file: relativePath, reason: knownReason });
|
||||
continue;
|
||||
}
|
||||
if (knownSafe.has(relativePath)) {
|
||||
safe.push(relativePath);
|
||||
continue;
|
||||
}
|
||||
const absolute = path.join(rootDir, relativePath);
|
||||
const fragility = detectWindowsFragility(absolute);
|
||||
if (fragility) {
|
||||
excluded.push({ file: relativePath, reason: fragility.reason });
|
||||
} else {
|
||||
safe.push(relativePath);
|
||||
}
|
||||
}
|
||||
return { safe, excluded };
|
||||
}
|
||||
|
||||
export function stableHash(input: string): number {
|
||||
let hash = 0x811c9dc5;
|
||||
for (let index = 0; index < input.length; index += 1) {
|
||||
@@ -692,8 +327,8 @@ export function assignFilesToShards(files: string[], shardCount: number): string
|
||||
// to bun's stream buffering, where silent passers print no header to
|
||||
// timestamp). GSTACK_FREE_TEST_DURATIONS overrides the path for experiments.
|
||||
// The seed is a HINT, not a contract: missing file → hash-shard fallback;
|
||||
// unknown file → 75th-percentile pessimism (placed early by LPT, bounding
|
||||
// tail risk). CI shares one plan rather than independently recomputing it.
|
||||
// unknown file → 99th-percentile pessimism (placed early by LPT, so one slow
|
||||
// new file cannot hide inside a full shard). CI shares one plan rather than independently recomputing it.
|
||||
|
||||
export const FREE_TEST_DURATIONS_FILE = 'scripts/free-test-durations.json';
|
||||
|
||||
@@ -734,9 +369,9 @@ export function packShardsByDuration(
|
||||
.map((f) => durations[normalizeRelativePath(f)])
|
||||
.filter((v): v is number => typeof v === 'number')
|
||||
.sort((a, b) => a - b);
|
||||
// Unknown files get the 75th percentile of known durations: pessimistic, so
|
||||
// LPT places them early and a surprise long-runner can't recreate the tail.
|
||||
const fallback = known.length > 0 ? known[Math.min(known.length - 1, Math.floor(known.length * 0.75))] : 1;
|
||||
// Unknown files get the 99th percentile of known durations: an unseeded
|
||||
// 115 s file packed at p75 (~1 s) once doubled the whole suite's wall.
|
||||
const fallback = known.length > 0 ? known[Math.min(known.length - 1, Math.floor(known.length * 0.99))] : 1;
|
||||
const predicted = (f: string): number => durations[normalizeRelativePath(f)] ?? fallback;
|
||||
|
||||
const ordered = [...files].sort((a, b) => predicted(b) - predicted(a) || (a < b ? -1 : 1));
|
||||
@@ -754,9 +389,9 @@ export function packShardsByDuration(
|
||||
}
|
||||
|
||||
/**
|
||||
* Files missing from the duration seed are packed at the 75th percentile, so
|
||||
* one slow new file can become the whole run's long pole without anyone
|
||||
* noticing. Name them (on stderr: --ci-plan's stdout is the CI matrix).
|
||||
* Files missing from the duration seed are packed at the 99th percentile;
|
||||
* name them (on stderr: --ci-plan's stdout is the CI matrix), and past the
|
||||
* unseeded limit also warn in the job summary with the refresh command.
|
||||
*/
|
||||
export function unseededFreeFiles(files: string[], durations: Record<string, number>): string[] {
|
||||
return files.filter((f) => durations[normalizeRelativePath(f)] === undefined);
|
||||
@@ -766,8 +401,13 @@ function warnUnseededFreeFiles(files: string[], durations: Record<string, number
|
||||
const unseeded = unseededFreeFiles(files, durations);
|
||||
if (unseeded.length === 0) return;
|
||||
const shown = unseeded.slice(0, 5).join(', ') + (unseeded.length > 5 ? `, +${unseeded.length - 5} more` : '');
|
||||
console.error(`[test:free] ${unseeded.length} file(s) have no recorded duration and are packed at the 75th-percentile estimate: ${shown}.`
|
||||
+ ' Refresh scripts/free-test-durations.json with `bun run test:ubicloud --record-durations`.');
|
||||
console.error(`[test:free] ${unseeded.length} file(s) have no recorded duration and are packed at the 99th-percentile estimate: ${shown}.`
|
||||
+ ` Refresh scripts/free-test-durations.json with \`${SEED_REFRESH_COMMAND}\`.`);
|
||||
const warning = unseededWarning(unseeded);
|
||||
if (warning) {
|
||||
console.error(`[test:free] ${warning}`);
|
||||
appendStepSummary(warning);
|
||||
}
|
||||
}
|
||||
|
||||
export interface FreeCiPlan {
|
||||
@@ -871,13 +511,6 @@ export function selectQuickFreeFiles(files: string[], durations: Record<string,
|
||||
}
|
||||
|
||||
export interface BuildShardArgsOptions {
|
||||
/**
|
||||
* Pass bun's --parallel (worker-per-file, implies --isolate). No production
|
||||
* caller today — full-suite mode uses N shard PROCESSES after the worker
|
||||
* pathologies documented in main(); retained for a future re-attempt on a
|
||||
* newer Bun (see WORKER_HOSTILE).
|
||||
*/
|
||||
parallel?: boolean;
|
||||
rootDir?: string;
|
||||
}
|
||||
|
||||
@@ -886,10 +519,7 @@ export function buildShardArgs(files: string[], options: BuildShardArgsOptions =
|
||||
// filters, so a relative `test/x.test.ts` would ALSO select
|
||||
// `browse/test/x.test.ts` — shard bleed that double-runs files.
|
||||
const selectors = exactTestFileSelectors(files, options.rootDir ?? ROOT);
|
||||
const args = ['test', ...selectors, `--timeout=${FREE_TEST_TIMEOUT_MS}`];
|
||||
if (options.parallel) args.push('--parallel');
|
||||
else args.push('--max-concurrency=1');
|
||||
return args;
|
||||
return ['test', ...selectors, `--timeout=${FREE_TEST_TIMEOUT_MS}`, '--max-concurrency=1'];
|
||||
}
|
||||
|
||||
type CliOptions = {
|
||||
@@ -965,7 +595,7 @@ export function parseCliOptions(argv: string[]): CliOptions {
|
||||
});
|
||||
|
||||
const ciModes = [paths.ciPlan, paths.ciRun, paths.ciVerify].filter(Boolean).length;
|
||||
if (ciModes > 1 || (ciModes && (quick || listOnly || dryRun || recordDurations || windowsOnly || attributeHome))) throw new Error('CI modes cannot be combined with other selection modes');
|
||||
if (ciModes > 1 || (ciModes && (quick || listOnly || dryRun || recordDurations || attributeHome))) throw new Error('CI modes cannot be combined with other selection modes');
|
||||
if (paths.ciRun && (shardIndex === null || !paths.result)) throw new Error('--ci-run requires --shard and --result');
|
||||
if (paths.ciVerify && !paths.results) throw new Error('--ci-verify requires --results');
|
||||
if (quick && (recordDurations || windowsOnly || shardIndex !== null)) throw new Error('--quick cannot change recording, Windows or shard selection');
|
||||
@@ -1382,8 +1012,6 @@ export interface RunFreeShardOptions {
|
||||
wallTimeoutMs?: number;
|
||||
rootDir?: string;
|
||||
env?: NodeJS.ProcessEnv;
|
||||
/** Pass bun's --parallel. No production caller today (see BuildShardArgsOptions.parallel). */
|
||||
parallel?: boolean;
|
||||
/** Override the spawned command. Tests inject fake pass/fail/slow commands. */
|
||||
commandFor?: (files: string[]) => ShardCommand;
|
||||
/** Suppress ALL child output from the console (tests). The classifier and the log file still see every byte. */
|
||||
@@ -1903,7 +1531,7 @@ export async function runFreeShard(
|
||||
|
||||
const rootDir = options.rootDir ?? ROOT;
|
||||
const wallTimeoutMs = options.wallTimeoutMs ?? DEFAULT_WALL_TIMEOUT_MS;
|
||||
log(`${label} (${files.length} files${options.parallel ? ', bun --parallel' : ''})`);
|
||||
log(`${label} (${files.length} files)`);
|
||||
|
||||
// Full-stream capture: EVERY child byte lands here, whatever the console
|
||||
// shows. Printed once at start so a wedged or noisy run is inspectable
|
||||
@@ -1914,7 +1542,7 @@ export async function runFreeShard(
|
||||
|
||||
const { command, args } = options.commandFor
|
||||
? options.commandFor(files)
|
||||
: { command: process.execPath, args: buildShardArgs(files, { parallel: options.parallel, rootDir }) };
|
||||
: { command: process.execPath, args: buildShardArgs(files, { rootDir }) };
|
||||
|
||||
// realpath: the browser tracker refuses a state dir whose path resolves elsewhere.
|
||||
const { stateDir, env } = createShardSandbox('gstack-free-shard-', options.env ?? process.env, { realpath: true });
|
||||
@@ -2171,6 +1799,9 @@ async function main(): Promise<number> {
|
||||
}
|
||||
|
||||
if (options.ciPlan || options.ciRun || options.ciVerify) {
|
||||
// The plan binds its exact file set, so a Windows plan cannot verify as a Linux one or vice versa.
|
||||
const curation = options.windowsOnly ? curateWindowsSafe(allFiles) : null;
|
||||
const ciFiles = curation ? curation.safe : allFiles;
|
||||
const git = spawnSync('git', ['rev-parse', 'HEAD'], { cwd: ROOT, encoding: 'utf8', timeout: 5_000 });
|
||||
if (git.status !== 0 || !git.stdout.trim()) throw new Error('Cannot bind CI plan to the checkout revision');
|
||||
const revision = git.stdout.trim();
|
||||
@@ -2182,20 +1813,29 @@ async function main(): Promise<number> {
|
||||
};
|
||||
if (options.ciPlan) {
|
||||
const durations = loadFreeTestDurations() ?? {};
|
||||
warnUnseededFreeFiles(allFiles, durations);
|
||||
const plan = createFreeCiPlan(allFiles, options.shardCount, durations, revision);
|
||||
validateFreeCiPlan(plan, allFiles, revision);
|
||||
if (curation) {
|
||||
const line = windowsCurationLine(curation);
|
||||
console.error(`[test:free] ${line}`);
|
||||
appendStepSummary(line);
|
||||
}
|
||||
warnUnseededFreeFiles(ciFiles, durations);
|
||||
const plan = createFreeCiPlan(ciFiles, options.shardCount, durations, revision);
|
||||
validateFreeCiPlan(plan, ciFiles, revision);
|
||||
writeJson(options.ciPlan, plan);
|
||||
console.log(JSON.stringify({ shard: plan.shards.map(shard => shard.shard) }));
|
||||
return 0;
|
||||
}
|
||||
const plan = JSON.parse(fs.readFileSync((options.ciRun ?? options.ciVerify)!, 'utf8')) as FreeCiPlan;
|
||||
validateFreeCiPlan(plan, allFiles, revision);
|
||||
validateFreeCiPlan(plan, ciFiles, revision);
|
||||
if (options.ciVerify) {
|
||||
const results = fs.readdirSync(options.results!).filter(file => file.endsWith('.json'))
|
||||
.map(file => JSON.parse(fs.readFileSync(path.join(options.results!, file), 'utf8')) as FreeCiResult);
|
||||
for (const section of ciHealthSummary(plan, results)) {
|
||||
console.log(`[test:free] ${section}`);
|
||||
appendStepSummary(section);
|
||||
}
|
||||
verifyFreeCiResults(plan, results);
|
||||
console.log(`[test:free] CI PASS: ${allFiles.length} files across ${results.length} isolated shards; slowest ${Math.round(Math.max(...results.map(result => result.outcome.elapsedMs + (result.retry?.elapsedMs ?? 0))) / 1000)}s including retries`);
|
||||
console.log(`[test:free] CI PASS: ${ciFiles.length} files across ${results.length} isolated shards; slowest ${Math.round(Math.max(...results.map(result => result.outcome.elapsedMs + (result.retry?.elapsedMs ?? 0))) / 1000)}s including retries`);
|
||||
return 0;
|
||||
}
|
||||
const shard = plan.shards[options.shardIndex! - 1];
|
||||
@@ -2223,7 +1863,7 @@ async function main(): Promise<number> {
|
||||
if (options.windowsOnly) {
|
||||
curationReport = curateWindowsSafe(allFiles);
|
||||
files = curationReport.safe;
|
||||
console.log(`[test:free] curated ${files.length} Windows-safe tests (${curationReport.excluded.length} excluded)`);
|
||||
console.log(`[test:free] ${windowsCurationLine(curationReport)}`);
|
||||
if (options.listOnly && curationReport.excluded.length > 0) {
|
||||
console.log('\nExcluded (POSIX-fragile):');
|
||||
for (const { file, reason } of curationReport.excluded) {
|
||||
@@ -2280,8 +1920,7 @@ async function main(): Promise<number> {
|
||||
// (compare-board), and spawn-heavy files hanging workers under load
|
||||
// (session-runner-timeout). Plain child processes have none of these:
|
||||
// proven spawn semantics, per-shard group-kill, per-shard logs, and a
|
||||
// wedge only ever costs its own shard. WORKER_HOSTILE files are moot in
|
||||
// process shards (no workers) and fold back into normal assignment.
|
||||
// wedge only ever costs its own shard.
|
||||
const jobs = fullSuiteJobs();
|
||||
// Phase split: exclusive host-state fixtures run AFTER the parallel shards,
|
||||
// so their shared process or filesystem state cannot interfere with readers.
|
||||
|
||||
@@ -0,0 +1,409 @@
|
||||
#!/usr/bin/env bun
|
||||
/**
|
||||
* test:health — the audit's success metrics as one command, read from CI
|
||||
* history (scripts/lib/ci-history.ts, the same reader pass-rates uses) plus
|
||||
* the local tree. Every metric prints its counting rule; a metric whose input
|
||||
* cannot be read prints "unavailable: <reason>; next: <step>", never 0.
|
||||
*
|
||||
* Enforced checks (`--enforce`): a free test file that flakes in more than 5%
|
||||
* of main free-tests runs over a window of at least 20 runs (W7a), and more
|
||||
* than 5 free test files missing from scripts/free-test-durations.json
|
||||
* (CEO-10). Without `--enforce` the command always exits 0; an unavailable
|
||||
* check never fails it. Needs `gh` with read access to the repository.
|
||||
*
|
||||
* Usage:
|
||||
* bun run test:health [--since-days 7] [--json] [--enforce] [--summary <file>]
|
||||
* [--repo owner/name] [--complexity-base <ref>]
|
||||
* bun run test:health flake-summary <dir> # job-summary table of downloaded flake-ledger-* artifacts
|
||||
*/
|
||||
import * as fs from 'node:fs';
|
||||
import * as path from 'node:path';
|
||||
import { spawnSync } from 'node:child_process';
|
||||
import { GH_CLIENT, downloadRunArtifacts, formatFlakeSummary, historyFetcher, listRunJobs, listWorkflowRuns, readCiFlakeLedgers,
|
||||
readFlakeLedgerDir, repoSlug, unavailableReason, type CiRun, type FlakeWindow, type GhClient } from './lib/ci-history';
|
||||
import { analyzePassRates, readTrialOutcomeDir, type TrialRecord } from './eval-flake-rank';
|
||||
import { collectFreeTestFiles, loadFreeTestDurations, unseededFreeFiles } from './test-free-shards';
|
||||
import { EVAL_POLICY } from '../test/helpers/periodic-exclude-data';
|
||||
import { resolveStateRoot } from '../lib/state-root';
|
||||
|
||||
const ROOT = path.resolve(import.meta.dir, '..');
|
||||
export const UNSEEDED_LIMIT = 5;
|
||||
export const COMPLEXITY_PATHS = ['scripts', 'test/helpers', '.github/workflows'] as const;
|
||||
|
||||
export interface Metric { id: string; title: string; rule: string; value?: string; detail?: string[]; unavailable?: string }
|
||||
export interface Check { id: string; title: string; status: 'pass' | 'fail' | 'unavailable'; detail: string; fix: string }
|
||||
export type IssueAction = 'upsert' | 'keep-open' | 'close';
|
||||
export interface HealthReport { repo: string; sinceDays: number; generatedAt: string; metrics: Metric[]; checks: Check[]; issueAction: IssueAction }
|
||||
|
||||
export interface HealthInputs {
|
||||
client: GhClient;
|
||||
repo: string;
|
||||
now: number;
|
||||
sinceDays: number;
|
||||
cacheDir: string;
|
||||
root?: string;
|
||||
complexityBase?: string;
|
||||
}
|
||||
|
||||
// --- Pure arithmetic ---
|
||||
|
||||
export function percentile(values: number[], p: number): number | null {
|
||||
if (values.length === 0) return null;
|
||||
const sorted = [...values].sort((a, b) => a - b);
|
||||
return sorted[Math.min(sorted.length - 1, Math.max(0, Math.ceil(p * sorted.length) - 1))]!;
|
||||
}
|
||||
|
||||
function choose(n: number, k: number): number {
|
||||
let out = 1;
|
||||
for (let i = 1; i <= k; i++) out = (out * (n - k + i)) / i;
|
||||
return out;
|
||||
}
|
||||
|
||||
/** P(fewer than k of n independent trials pass) when one trial fails with probability `failRate`. */
|
||||
export function panelRedProbability(failRate: number, n: number, k: number): number {
|
||||
let red = 0;
|
||||
for (let passes = 0; passes < k; passes++) red += choose(n, passes) * (1 - failRate) ** passes * failRate ** (n - passes);
|
||||
return red;
|
||||
}
|
||||
|
||||
export interface CensusStats {
|
||||
runs: number;
|
||||
trials: number;
|
||||
failed: number;
|
||||
byKind: Record<string, { trials: number; failed: number }>;
|
||||
byClass: Record<string, number>;
|
||||
/** P(lane red) with each case's pooled trial failure rate, under the recorded panels. */
|
||||
redInForce: number;
|
||||
/** The same evidence with every case judged on one trial. */
|
||||
redSingleTrial: number;
|
||||
}
|
||||
|
||||
/**
|
||||
* CEO-06/22 census arithmetic over one lane's trial records. Every observed
|
||||
* failure is treated as nondeterminism, so both red probabilities are upper
|
||||
* bounds for known-good inputs; they are reported separately from the raw
|
||||
* trial failure rate so a verdict-policy change cannot read as a reliability gain.
|
||||
*/
|
||||
export function censusStats(records: TrialRecord[], runIds: string[]): CensusStats {
|
||||
const scored = records.filter(record => record.outcome !== 'skipped');
|
||||
const byKind: CensusStats['byKind'] = {};
|
||||
const byClass: Record<string, number> = {};
|
||||
const perCase = new Map<string, { trials: number; failed: number; n: number; k: number }>();
|
||||
for (const record of scored) {
|
||||
const failed = record.outcome === 'failed';
|
||||
const kind = (byKind[record.kind] ??= { trials: 0, failed: 0 });
|
||||
kind.trials += 1;
|
||||
if (failed) { kind.failed += 1; byClass[record.failure_class ?? 'unknown'] = (byClass[record.failure_class ?? 'unknown'] ?? 0) + 1; }
|
||||
const slot = perCase.get(record.case) ?? { trials: 0, failed: 0, n: 1, k: 1 };
|
||||
slot.trials += 1;
|
||||
if (failed) slot.failed += 1;
|
||||
if (record.panel.n > slot.n) { slot.n = record.panel.n; slot.k = record.panel.k; }
|
||||
perCase.set(record.case, slot);
|
||||
}
|
||||
let greenInForce = 1, greenSingle = 1;
|
||||
for (const slot of perCase.values()) {
|
||||
const p = slot.failed / slot.trials;
|
||||
greenInForce *= 1 - panelRedProbability(p, slot.n, slot.k);
|
||||
greenSingle *= 1 - p;
|
||||
}
|
||||
return { runs: runIds.length, trials: scored.length, failed: scored.filter(record => record.outcome === 'failed').length,
|
||||
byKind, byClass, redInForce: 1 - greenInForce, redSingleTrial: 1 - greenSingle };
|
||||
}
|
||||
|
||||
const pct = (value: number) => `${(Math.round(value * 1000) / 10).toFixed(1)}%`;
|
||||
const minutes = (ms: number | null) => ms === null ? 'n/a' : `${(ms / 60_000).toFixed(1)} min`;
|
||||
const span = (from: string | null, to: string | null) => from && to ? Math.max(0, Date.parse(to) - Date.parse(from)) : null;
|
||||
|
||||
/** DX-9: close only when every enforced check has evidence and passes; missing evidence keeps the issue open. */
|
||||
export function issueActionFor(checks: Check[]): IssueAction {
|
||||
if (checks.some(check => check.status === 'fail')) return 'upsert';
|
||||
if (checks.some(check => check.status === 'unavailable')) return 'keep-open';
|
||||
return 'close';
|
||||
}
|
||||
|
||||
// --- Checks ---
|
||||
|
||||
export function flakeCheck(window: FlakeWindow | Error): Check {
|
||||
const base = { id: 'flaky-files', title: 'Free-suite flaky files (W7a)',
|
||||
fix: 'Fix the race at source (docs/TESTING_INTERNALS.md#flake-ledger); never raise the timeout or add a retry. Reproduce with `bun test <file>` under load.' };
|
||||
if (window instanceof Error) return { ...base, status: 'unavailable', detail: `unavailable: ${unavailableReason(window)}` };
|
||||
const failing = window.files.filter(file => file.failing);
|
||||
if (!window.enforceable) {
|
||||
return { ...base, status: 'pass', detail: `report-only: ${window.runs} main run(s) since ${window.since.slice(0, 10)} (< 20 needed to enforce)` };
|
||||
}
|
||||
if (failing.length === 0) return { ...base, status: 'pass', detail: `no file above 5% over ${window.runs} main runs` };
|
||||
return { ...base, status: 'fail',
|
||||
detail: failing.map(file => `${file.file}: flaky in ${file.flakyRuns}/${window.runs} main runs (${pct(file.rate)} > 5%)`).join('; ') };
|
||||
}
|
||||
|
||||
export function unseededCheck(root: string): Check {
|
||||
const base = { id: 'unseeded-free-files', title: 'Free test files missing from the duration seed (CEO-10)',
|
||||
fix: 'Refresh the seed: `bun run test:ubicloud --record-durations` (or a CI recording; a 4-core laptop recording is not acceptable), then commit scripts/free-test-durations.json (docs/TESTING_INTERNALS.md#free-suite-duration-seed).' };
|
||||
const durations = loadFreeTestDurations(root);
|
||||
if (!durations) return { ...base, status: 'unavailable', detail: 'unavailable: scripts/free-test-durations.json is missing or unreadable; next: refresh the seed' };
|
||||
const unseeded = unseededFreeFiles(collectFreeTestFiles(root), durations);
|
||||
const shown = unseeded.slice(0, 10).join(', ') + (unseeded.length > 10 ? `, +${unseeded.length - 10} more` : '');
|
||||
if (unseeded.length > UNSEEDED_LIMIT) return { ...base, status: 'fail', detail: `stale seed: ${unseeded.length} unseeded file(s) > ${UNSEEDED_LIMIT}: ${shown}` };
|
||||
return { ...base, status: 'pass', detail: `${unseeded.length} unseeded file(s) (limit ${UNSEEDED_LIMIT})${unseeded.length ? `: ${shown}` : ''}` };
|
||||
}
|
||||
|
||||
// --- Metric collection ---
|
||||
|
||||
function attempt<T>(fn: () => T): T | Error {
|
||||
try { return fn(); } catch (error) { return error instanceof Error ? error : new Error(String(error)); }
|
||||
}
|
||||
|
||||
function metric(id: string, title: string, rule: string, compute: () => { value: string; detail?: string[] }): Metric {
|
||||
const result = attempt(compute);
|
||||
if (result instanceof Error) return { id, title, rule, unavailable: unavailableReason(result) };
|
||||
return { id, title, rule, ...result };
|
||||
}
|
||||
|
||||
function records(dirs: string[]): TrialRecord[] {
|
||||
return dirs.flatMap(dir => readTrialOutcomeDir(dir).records);
|
||||
}
|
||||
|
||||
function readJson(file: string): any {
|
||||
return JSON.parse(fs.readFileSync(file, 'utf8'));
|
||||
}
|
||||
|
||||
/** Net LOC under COMPLEXITY_PATHS and the package-script count at one ref. */
|
||||
export function complexityAt(root: string, ref: string): { loc: number; scripts: number } {
|
||||
const grep = spawnSync('git', ['grep', '-c', '-I', '', ref, '--', ...COMPLEXITY_PATHS], { cwd: root, encoding: 'utf8', maxBuffer: 64 * 1024 * 1024, timeout: 60_000 });
|
||||
if (grep.status !== 0 && grep.status !== 1) throw new Error(`git grep ${ref} failed: ${grep.stderr.trim()}`);
|
||||
const loc = grep.stdout.split('\n').filter(Boolean).reduce((sum, line) => sum + Number(line.slice(line.lastIndexOf(':') + 1) || 0), 0);
|
||||
const pkg = spawnSync('git', ['show', `${ref}:package.json`], { cwd: root, encoding: 'utf8', timeout: 10_000 });
|
||||
if (pkg.status !== 0) throw new Error(`git show ${ref}:package.json failed: ${pkg.stderr.trim()}`);
|
||||
return { loc, scripts: Object.keys(JSON.parse(pkg.stdout).scripts ?? {}).length };
|
||||
}
|
||||
|
||||
export function collectHealth(inputs: HealthInputs): HealthReport {
|
||||
const { client, repo, now, sinceDays, cacheDir } = inputs;
|
||||
const root = inputs.root ?? ROOT;
|
||||
const since = new Date(now - sinceDays * 86_400_000).toISOString();
|
||||
const fetcher = historyFetcher(client);
|
||||
const artifactDirs = (run: CiRun, match: (name: string) => boolean) =>
|
||||
downloadRunArtifacts({ repo, run, cacheDir, fetcher, match, maxBytes: 64 * 1024 * 1024 });
|
||||
const prRuns = attempt(() => listWorkflowRuns(client, repo, 'evals.yml', { since, event: 'pull_request' }));
|
||||
const need = <T>(value: T | Error): T => { if (value instanceof Error) throw value; return value; };
|
||||
const prRecords = attempt(() => need(prRuns).filter(run => run.conclusion !== 'cancelled')
|
||||
.map(run => ({ run, trials: records(artifactDirs(run, name => name.startsWith('trial-outcomes'))) })));
|
||||
const metrics: Metric[] = [];
|
||||
|
||||
metrics.push(metric('pr-fallback-rate', 'PR full-fallback rate', 'share of evals.yml pull_request runs whose paid-plan manifest has prCoverage.mode = full-fallback', () => {
|
||||
const modes = new Map<string, number>();
|
||||
for (const run of need(prRuns)) {
|
||||
const manifest = artifactDirs(run, name => name === 'paid-plan').flatMap(dir => [path.join(dir, 'manifest.json')]).find(file => fs.existsSync(file));
|
||||
if (!manifest) continue;
|
||||
const mode = String(readJson(manifest).prCoverage?.mode ?? 'none');
|
||||
modes.set(mode, (modes.get(mode) ?? 0) + 1);
|
||||
}
|
||||
const total = [...modes.values()].reduce((a, b) => a + b, 0);
|
||||
if (total === 0) throw new Error('no paid-plan manifests in the window');
|
||||
return { value: `${pct((modes.get('full-fallback') ?? 0) / total)} (${modes.get('full-fallback') ?? 0}/${total} runs)`,
|
||||
detail: [...modes].map(([mode, count]) => `${mode}: ${count}`) };
|
||||
}));
|
||||
|
||||
metrics.push(metric('pr-reused-trials', 'Reused PR trial records', 'trial-outcomes records with execution = reused across non-cancelled PR runs', () => {
|
||||
const all = need(prRecords).flatMap(entry => entry.trials);
|
||||
const reused = all.filter(record => record.execution === 'reused').length;
|
||||
return { value: `${reused} of ${all.length} trial records` };
|
||||
}));
|
||||
|
||||
metrics.push(metric('pr-spend', 'Recorded PR-lane spend', 'sum of executed trial cost_usd in PR runs; per merged PR = / PRs merged in the window; per validated revision = / distinct head SHAs with a success or failure verdict', () => {
|
||||
const entries = need(prRecords);
|
||||
const spend = entries.flatMap(entry => entry.trials).filter(record => record.execution === 'executed')
|
||||
.reduce((sum, record) => sum + record.cost_usd, 0);
|
||||
const revisions = new Set(entries.filter(entry => entry.run.conclusion === 'success' || entry.run.conclusion === 'failure').map(entry => entry.run.sha)).size;
|
||||
const merged = listMergedPulls(client, repo, since);
|
||||
return { value: `$${spend.toFixed(2)} recorded`, detail: [
|
||||
`per merged PR: ${merged ? `$${(spend / merged).toFixed(2)} (${merged} merged)` : 'n/a (0 merged)'}`,
|
||||
`per validated revision: ${revisions ? `$${(spend / revisions).toFixed(2)} (${revisions} revisions)` : 'n/a'}`] };
|
||||
}));
|
||||
|
||||
metrics.push(metric('cancelled-slice-minutes', 'Cancelled slice-minutes per week', 'wall minutes of eval-slices jobs in cancelled evals.yml PR runs, scaled to 7 days', () => {
|
||||
const cancelled = need(prRuns).filter(run => run.conclusion === 'cancelled');
|
||||
const total = cancelled.flatMap(run => listRunJobs(client, repo, run.id, run.attempt))
|
||||
.filter(job => job.name.startsWith('eval-slices')).reduce((sum, job) => sum + (span(job.startedAt, job.completedAt) ?? 0), 0);
|
||||
return { value: `${(total / 60_000 * 7 / sinceDays).toFixed(0)} min/week (${cancelled.length} cancelled run(s))` };
|
||||
}));
|
||||
|
||||
metrics.push(metric('push-to-verdict', 'PR lane push-to-verdict', 'created_at to updated_at of evals.yml pull_request runs that reached success or failure (includes queue and debounce)', () => {
|
||||
const walls = need(prRuns).filter(run => run.conclusion === 'success' || run.conclusion === 'failure').map(run => span(run.createdAt, run.updatedAt)!);
|
||||
if (walls.length === 0) throw new Error('no PR run reached a verdict in the window');
|
||||
return { value: `p50 ${minutes(percentile(walls, 0.5))}, p95 ${minutes(percentile(walls, 0.95))} (${walls.length} runs)` };
|
||||
}));
|
||||
|
||||
metrics.push(...censusMetrics(inputs, artifactDirs));
|
||||
|
||||
const flakes = attempt(() => readCiFlakeLedgers({ client, repo, sinceDays, now, cacheDir }));
|
||||
metrics.push(metric('free-flaky-pass-rate', 'Free-suite flaky-pass rate', 'share of main free-tests runs (success or failure) with at least one flaky-pass ledger entry; per-file rate = runs with that file / runs', () => {
|
||||
const window = need(flakes);
|
||||
return { value: `${pct(window.runs ? window.flakyRuns / window.runs : 0)} (${window.flakyRuns}/${window.runs} main runs since ${window.since.slice(0, 10)} needed the retry pass; ${window.files.length} file(s))`,
|
||||
detail: window.files.slice(0, 15).map(file => `${file.file}: ${file.flakyRuns}/${window.runs} runs (${pct(file.rate)})${file.failing ? ' FAIL > 5%' : ''}`) };
|
||||
}));
|
||||
|
||||
metrics.push(metric('free-wall', 'Free-suite and Windows wall', 'p50 of run_started_at to updated_at for successful main runs of free-tests.yml and windows-free-tests.yml', () => {
|
||||
const wall = (workflow: string) => {
|
||||
const walls = listWorkflowRuns(client, repo, workflow, { since, branch: 'main' }).filter(run => run.conclusion === 'success').map(run => span(run.startedAt, run.updatedAt)!);
|
||||
return `${minutes(percentile(walls, 0.5))} (${walls.length} runs)`;
|
||||
};
|
||||
return { value: `free-tests ${wall('free-tests.yml')}; windows-free-tests ${wall('windows-free-tests.yml')}` };
|
||||
}));
|
||||
|
||||
metrics.push(metric('free-serial', 'Free-suite serial time', 'sum of scripts/free-test-durations.json (the committed seed)', () => {
|
||||
const durations = loadFreeTestDurations(root);
|
||||
if (!durations) throw new Error('scripts/free-test-durations.json is missing or unreadable');
|
||||
return { value: `${Math.round(Object.values(durations).reduce((a, b) => a + b, 0) / 1000)} s over ${Object.keys(durations).length} files` };
|
||||
}));
|
||||
|
||||
metrics.push(metric('net-complexity', 'Net complexity (CEO-32, reported, no target)', `lines under ${COMPLEXITY_PATHS.join(', ')} and package.json script count, HEAD vs --complexity-base`, () => {
|
||||
const head = complexityAt(root, 'HEAD');
|
||||
if (!inputs.complexityBase) return { value: `HEAD: ${head.loc} lines, ${head.scripts} package scripts (pass --complexity-base <ref> for the delta)` };
|
||||
const base = complexityAt(root, inputs.complexityBase);
|
||||
const sign = (n: number) => (n >= 0 ? `+${n}` : `${n}`);
|
||||
return { value: `${sign(head.loc - base.loc)} lines (${base.loc} -> ${head.loc}), ${sign(head.scripts - base.scripts)} package scripts (${base.scripts} -> ${head.scripts}) vs ${inputs.complexityBase}` };
|
||||
}));
|
||||
|
||||
const checks = [flakeCheck(flakes), unseededCheck(root)];
|
||||
return { repo, sinceDays, generatedAt: new Date(now).toISOString(), metrics, checks, issueAction: issueActionFor(checks) };
|
||||
}
|
||||
|
||||
function listMergedPulls(client: GhClient, repo: string, since: string): number {
|
||||
let merged = 0;
|
||||
for (let page = 1; page <= 10; page++) {
|
||||
const pulls = client.getJson(`repos/${repo}/pulls?state=closed&base=main&sort=updated&direction=desc&per_page=100&page=${page}`) as any[];
|
||||
if (!Array.isArray(pulls)) throw new Error('pulls listing returned no array');
|
||||
merged += pulls.filter(pull => pull.merged_at && String(pull.merged_at) >= since).length;
|
||||
if (pulls.length < 100 || pulls.every(pull => String(pull.updated_at) < since)) break;
|
||||
}
|
||||
return merged;
|
||||
}
|
||||
|
||||
/**
|
||||
* Census metrics from main's evals-periodic history (scheduled runs plus main
|
||||
* dispatches, the W4b weekly history): CEO-06/22 per lane over the trailing 4
|
||||
* scheduled censuses, the INFRA re-dispatch count, slice wall, and DX-12
|
||||
* quarantine evidence (qualifying vs required trials) over the last 10 runs.
|
||||
*/
|
||||
function censusMetrics(inputs: HealthInputs, artifactDirs: (run: CiRun, match: (name: string) => boolean) => string[]): Metric[] {
|
||||
const { client, repo, now } = inputs;
|
||||
const history = attempt(() => listWorkflowRuns(client, repo, 'evals-periodic.yml',
|
||||
{ since: new Date(now - Math.max(inputs.sinceDays, 84) * 86_400_000).toISOString(), branch: 'main' })
|
||||
.filter(run => run.event === 'schedule' || run.event === 'workflow_dispatch'));
|
||||
const need = <T>(value: T | Error): T => { if (value instanceof Error) throw value; return value; };
|
||||
const censuses = attempt(() => {
|
||||
const out: { run: CiRun; trials: TrialRecord[] }[] = [];
|
||||
for (const run of need(history)) {
|
||||
if (out.length === 4) break;
|
||||
const trials = records(artifactDirs(run, name => name.startsWith('trial-outcomes')));
|
||||
if (trials.length) out.push({ run, trials });
|
||||
}
|
||||
return out;
|
||||
});
|
||||
const out: Metric[] = [];
|
||||
for (const lane of ['periodic', 'gate'] as const) {
|
||||
out.push(metric(`census-${lane}`, `${lane === 'gate' ? 'Gate' : 'Periodic'} census reliability (CEO-06/22)`,
|
||||
'trailing 4 main censuses with trial outcomes (scheduled runs and main dispatches): expected failed trials per run, raw trial failure rate per kind and class, and P(lane red) from pooled per-case rates under the recorded panels vs single-trial aggregation (every failure treated as nondeterminism: upper bounds for known-good inputs)', () => {
|
||||
const runs = need(censuses);
|
||||
const laneRuns = runs.filter(entry => entry.trials.some(record => (record.lane ?? '').startsWith(`${lane}/`)));
|
||||
if (laneRuns.length === 0) throw new Error(`no main census with ${lane} trial outcomes in the last 84 days`);
|
||||
const stats = censusStats(laneRuns.flatMap(entry => entry.trials).filter(record => (record.lane ?? '').startsWith(`${lane}/`)), laneRuns.map(entry => `${entry.run.id}`));
|
||||
return { value: `${(stats.failed / stats.runs).toFixed(2)} expected failed trials/run (${stats.failed}/${stats.trials} over ${stats.runs} run(s)); P(red) ${pct(stats.redInForce)} in force, ${pct(stats.redSingleTrial)} single-trial`,
|
||||
detail: [...Object.entries(stats.byKind).map(([kind, slot]) => `kind ${kind}: ${slot.failed}/${slot.trials} failed (${pct(slot.failed / Math.max(1, slot.trials))})`),
|
||||
...Object.entries(stats.byClass).map(([cls, count]) => `class ${cls}: ${count}`)] };
|
||||
}));
|
||||
}
|
||||
out.push(metric('census-redispatch', 'INFRA re-dispatches (ENG-10)', 'trailing 4 main census runs whose redispatch job succeeded; longest paid slice job wall in the same runs', () => {
|
||||
const runs = need(history).slice(0, 4);
|
||||
const jobs = runs.map(run => listRunJobs(client, repo, run.id, run.attempt));
|
||||
const redispatched = jobs.filter(list => list.some(job => job.name === 'redispatch' && job.conclusion === 'success')).length;
|
||||
const walls = jobs.flat().filter(job => /^(eval-slices|eval-codex-slices|gate-census)/.test(job.name)).map(job => span(job.startedAt, job.completedAt) ?? 0);
|
||||
return { value: `${redispatched} of ${runs.length} census(es) re-dispatched; longest paid slice job ${minutes(walls.length ? Math.max(...walls) : null)}` };
|
||||
}));
|
||||
out.push(metric('quarantine-evidence', 'Quarantine evidence (DX-12)', `blocking cases with >= ${EVAL_POLICY.quarantine.entry.minTrials} qualifying v${EVAL_POLICY.version} trials on their current series, over the last 10 main census runs`, () => {
|
||||
const runs = need(history).slice(0, 10);
|
||||
const report = analyzePassRates(runs.flatMap(run => records(artifactDirs(run, name => name.startsWith('trial-outcomes')))), { weeklyRuns: runs.map(run => run.createdAt) });
|
||||
const blocking = report.cases.filter(entry => entry.tier === 'gate' || entry.tier === 'periodic');
|
||||
const sufficient = blocking.filter(entry => (entry.current?.trials ?? 0) >= EVAL_POLICY.quarantine.entry.minTrials);
|
||||
const labels = new Map<string, number>();
|
||||
for (const entry of sufficient) labels.set(entry.label, (labels.get(entry.label) ?? 0) + 1);
|
||||
return { value: `${sufficient.length} of ${blocking.length} blocking case(s) have enough evidence; ${blocking.length - sufficient.length} insufficient evidence (not the same as healthy)`,
|
||||
detail: [...[...labels].map(([label, count]) => `${label}: ${count}`),
|
||||
...(report.newerPolicyTrials ? [`ignored ${report.newerPolicyTrials} newer-policy trial(s)`] : []),
|
||||
...(report.olderPolicyTrials ? [`${report.olderPolicyTrials} older-policy trial(s) display only`] : [])] };
|
||||
}));
|
||||
return out;
|
||||
}
|
||||
|
||||
// --- Rendering ---
|
||||
|
||||
export function formatHealth(report: HealthReport): string {
|
||||
const lines = [`test:health ${report.repo}, last ${report.sinceDays} day(s), ${report.generatedAt}`];
|
||||
for (const entry of report.metrics) {
|
||||
lines.push(`- ${entry.title}: ${entry.unavailable ? `unavailable: ${entry.unavailable}` : entry.value}`);
|
||||
lines.push(` rule: ${entry.rule}`);
|
||||
for (const detail of entry.detail ?? []) lines.push(` ${detail}`);
|
||||
}
|
||||
lines.push('Enforced checks:');
|
||||
for (const check of report.checks) {
|
||||
lines.push(`- [${check.status.toUpperCase()}] ${check.title}: ${check.detail}`);
|
||||
if (check.status !== 'pass') lines.push(` fix: ${check.fix}`);
|
||||
}
|
||||
const unavailable = report.checks.filter(check => check.status === 'unavailable').map(check => check.id);
|
||||
if (unavailable.length) lines.push(`Missing inputs (an unavailable check never fails the run, and keeps a tracking issue open): ${unavailable.join(', ')}`);
|
||||
return lines.join('\n');
|
||||
}
|
||||
|
||||
export function formatHealthMarkdown(report: HealthReport): string {
|
||||
return ['## test:health', '', '```', formatHealth(report), '```', ''].join('\n');
|
||||
}
|
||||
|
||||
export const HELP = `Usage: bun run test:health [options]
|
||||
bun run test:health flake-summary <dir>
|
||||
|
||||
Prints the test/eval/CI success metrics from GitHub Actions history (needs gh
|
||||
with read access to the repository; \`gh auth status\` checks it). A metric whose
|
||||
input cannot be read prints "unavailable: <reason>; next: <step>", never 0.
|
||||
|
||||
Options:
|
||||
--since-days <n> window in days (default 7)
|
||||
--enforce exit 1 when an enforced check fails (flaky file > 5% of
|
||||
>= 20 main free-tests runs; > 5 unseeded free files);
|
||||
without it the command always exits 0
|
||||
--json machine-readable report (includes issueAction)
|
||||
--summary <file> append a markdown report (job summary / issue body)
|
||||
--repo <owner/name> repository (default: origin)
|
||||
--complexity-base <ref> report the net-complexity delta against <ref>
|
||||
--help this text
|
||||
|
||||
flake-summary <dir> markdown table of the flake-ledger-* artifacts under <dir>`;
|
||||
|
||||
if (import.meta.main) {
|
||||
const argv = process.argv.slice(2);
|
||||
if (argv.includes('--help') || argv.includes('-h')) { console.log(HELP); process.exit(0); }
|
||||
if (argv[0] === 'flake-summary') {
|
||||
if (!argv[1]) { console.error('usage: bun run test:health flake-summary <dir>'); process.exit(2); }
|
||||
console.log(formatFlakeSummary(readFlakeLedgerDir(argv[1])));
|
||||
process.exit(0);
|
||||
}
|
||||
const valued = new Set(['--since-days', '--summary', '--repo', '--complexity-base']);
|
||||
const known = new Set([...valued, '--enforce', '--json']);
|
||||
for (let index = 0; index < argv.length; index++) {
|
||||
if (!known.has(argv[index]!)) { console.error(`test:health: unknown argument ${argv[index]}\n\n${HELP}`); process.exit(2); }
|
||||
if (valued.has(argv[index]!)) {
|
||||
if (argv[index + 1] === undefined || argv[index + 1]!.startsWith('--')) { console.error(`test:health: ${argv[index]} needs a value`); process.exit(2); }
|
||||
index++;
|
||||
}
|
||||
}
|
||||
const flag = (name: string) => { const index = argv.indexOf(name); return index === -1 ? undefined : argv[index + 1]; };
|
||||
const sinceDays = Number(flag('--since-days') ?? 7);
|
||||
if (!Number.isInteger(sinceDays) || sinceDays < 1) { console.error('test:health: --since-days must be a positive integer'); process.exit(2); }
|
||||
const repo = flag('--repo') ?? repoSlug(ROOT);
|
||||
const cacheDir = path.join(path.resolve(resolveStateRoot()), 'test-health-cache', repo.replace('/', '-'));
|
||||
const report = collectHealth({ client: GH_CLIENT, repo, now: Date.now(), sinceDays, cacheDir, complexityBase: flag('--complexity-base') });
|
||||
console.log(argv.includes('--json') ? JSON.stringify(report, null, 2) : formatHealth(report));
|
||||
const summary = flag('--summary');
|
||||
if (summary) fs.appendFileSync(summary, formatHealthMarkdown(report));
|
||||
if (argv.includes('--enforce') && report.checks.some(check => check.status === 'fail')) process.exit(1);
|
||||
}
|
||||
+151
-754
File diff suppressed because it is too large.
Load diff
+276
-21
@@ -1,7 +1,11 @@
|
||||
/** Fast PR policy. Cadence changes here never remove cases from the broad census. */
|
||||
import * as fs from 'node:fs';
|
||||
import * as path from 'node:path';
|
||||
import { E2E_TOUCHFILES, E2E_TIERS, GLOBAL_TOUCHFILES, LLM_JUDGE_TOUCHFILES } from '../test/helpers/touchfiles-data';
|
||||
import { matchGlob, TOUCHFILES_DATA_PATH } from '../test/helpers/test-selection';
|
||||
import { isPaidTestFile } from '../test/helpers/paid-test-set';
|
||||
import { FREE_FIXTURES } from '../test/helpers/free-fixtures-data';
|
||||
import { derivedDependencies, type DerivedDependencies } from './pr-dependencies';
|
||||
|
||||
/** Existing short behavioral probes; intersect with changed-input selection. */
|
||||
export const PR_PROFILE_CASE_IDS = [
|
||||
@@ -61,6 +65,62 @@ export const PR_PROFILE_FILES: Record<string, readonly string[]> = {
|
||||
'test/skill-e2e-learnings.test.ts': ['learnings-show'],
|
||||
};
|
||||
|
||||
const ROOT = path.resolve(import.meta.dir, '..');
|
||||
|
||||
/** The one paid test file whose source registers a case (its touchfiles name exactly one), else null. */
|
||||
export function caseOwnerFile(id: string, maps: Pick<PrProfileMaps, 'e2eTouchfiles'> = PR_PROFILE_MAPS): string | null {
|
||||
const owners = (maps.e2eTouchfiles[id] ?? []).filter(dep => /^test\/[^/]+\.test\.ts$/.test(dep) && isPaidTestFile(dep));
|
||||
return owners.length === 1 ? owners[0]! : null;
|
||||
}
|
||||
|
||||
/**
|
||||
* DX-11: a gate case outside the audited profile can run in the PR lane only
|
||||
* when the runner can select it by name: its owner registers a Bun test whose
|
||||
* name is the case id (testIfSelected / testConcurrentIfSelected / test with
|
||||
* the literal id), which is what the PR lane's case-name pattern matches.
|
||||
*/
|
||||
export function caseNameAddressable(id: string, source: string): boolean {
|
||||
const quoted = id.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
|
||||
const code = source.replace(/\/\*[\s\S]*?\*\//g, '').replace(/^\s*\/\/.*$/gm, '');
|
||||
return new RegExp(`(?:^|[\\s;{(])(?:test(?:Concurrent)?IfSelected|test(?:\\.concurrent)?)\\(\\s*(['"\`])${quoted}\\1`, 'm').test(code);
|
||||
}
|
||||
|
||||
const sourceCache = new Map<string, string>();
|
||||
const readRepoSource = (file: string): string => {
|
||||
if (!sourceCache.has(file)) {
|
||||
try { sourceCache.set(file, fs.readFileSync(path.join(ROOT, file), 'utf8')); } catch { sourceCache.set(file, ''); }
|
||||
}
|
||||
return sourceCache.get(file)!;
|
||||
};
|
||||
|
||||
/** Selectable outside the profile: a gate case with one owning paid file that names it as a test. */
|
||||
export function prProfileDirectCase(id: string, maps: PrProfileMaps = PR_PROFILE_MAPS, readSource = readRepoSource): string | null {
|
||||
if (maps.tiers[id] !== 'gate') return null;
|
||||
const owner = caseOwnerFile(id, maps);
|
||||
return owner && caseNameAddressable(id, readSource(owner)) ? owner : null;
|
||||
}
|
||||
|
||||
/** Whether a manifest may carry this case in the PR lane: audited profile, or a recorded name-addressable direct case. */
|
||||
export function prProfileCaseAllowed(id: string, directCases: readonly string[] = []): boolean {
|
||||
return (PR_PROFILE_CASE_IDS as readonly string[]).includes(id) || (directCases.includes(id) && prProfileDirectCase(id) !== null);
|
||||
}
|
||||
|
||||
/** File -> PR-lane case ids: the audited map plus the selected direct cases each file owns (DX-11). */
|
||||
export function prProfileFileMap(selected: readonly string[] | null): Record<string, readonly string[]> {
|
||||
const map: Record<string, string[]> = Object.fromEntries(Object.entries(PR_PROFILE_FILES).map(([file, ids]) => [file, [...ids]]));
|
||||
for (const id of selected ?? []) {
|
||||
if ((PR_PROFILE_CASE_IDS as readonly string[]).includes(id)) continue;
|
||||
const owner = prProfileDirectCase(id);
|
||||
if (owner) (map[owner] ??= []).push(id);
|
||||
}
|
||||
return map;
|
||||
}
|
||||
|
||||
/** The PR-lane case ids one test file owns under `selected`. */
|
||||
export function prProfileFileCases(file: string, selected: readonly string[] | null): readonly string[] {
|
||||
return prProfileFileMap(selected)[file] ?? [];
|
||||
}
|
||||
|
||||
export interface PrProfileMaps {
|
||||
e2eTouchfiles: Record<string, string[]>;
|
||||
judgeTouchfiles: Record<string, string[]>;
|
||||
@@ -74,11 +134,24 @@ export const PR_PROFILE_MAPS: PrProfileMaps = {
|
||||
};
|
||||
|
||||
export interface PrProfileSelection {
|
||||
mode: 'pr' | 'full-fallback';
|
||||
/**
|
||||
* pr: the fast profile on mapped inputs; dependents: every gate case whose
|
||||
* derived reference closure holds a changed file (no profile cut);
|
||||
* full-fallback: a global build/runtime input or an underivable file.
|
||||
*/
|
||||
mode: 'pr' | 'dependents' | 'full-fallback';
|
||||
e2e: string[];
|
||||
judges: string[];
|
||||
deferred: Array<{ id: string; tier: 'gate' | 'periodic' | 'marathon'; reason: string }>;
|
||||
unknownFiles: string[];
|
||||
/** One label and fix per unknown file (CEO-13/DX-6). */
|
||||
unknownFileLabels: Array<{ file: string; label: string; fix: string }>;
|
||||
/** DX-11: gate cases outside the profile selected because their own paid test file changed. */
|
||||
directCases: string[];
|
||||
/** Unmapped changed files placed by derived references (scripts/pr-dependencies.ts), with their consumer counts. */
|
||||
derivedFiles: Array<{ file: string; e2e: number; judges: number }>;
|
||||
/** Unmapped changed files under derivable directories that no paid case's reference closure reaches. */
|
||||
noConsumerFiles: string[];
|
||||
deferredPromptFiles: string[];
|
||||
missingCoverage: string[];
|
||||
needsFullValidation: boolean;
|
||||
@@ -115,31 +188,109 @@ function matches(file: string, patterns: readonly string[]): boolean {
|
||||
return patterns.some(pattern => matchGlob(file, pattern));
|
||||
}
|
||||
|
||||
/** Workflows that define or feed the paid eval lanes; every other workflow is free-lane only. */
|
||||
export const PAID_WORKFLOW_FILES = [
|
||||
'.github/workflows/evals.yml',
|
||||
'.github/workflows/evals-periodic.yml',
|
||||
'.github/workflows/evals-marathon.yml',
|
||||
] as const;
|
||||
|
||||
/**
|
||||
* Files no paid case consumes. Entries may be globs. Scheduler inputs change
|
||||
* packing and derived job timeouts, never which cases run or how they are
|
||||
* judged; the listed workflows never run a paid case (test/free-fixtures.test.ts
|
||||
* fails when a workflow is in neither this list nor PAID_WORKFLOW_FILES).
|
||||
*/
|
||||
export const FREE_ONLY_PR_FILES = [
|
||||
'scripts/test-free-shards.ts',
|
||||
'scripts/lib/free-home-guard.ts', // Imported only by the free shard runner.
|
||||
'test/helpers/auq-parallel-worker.ts',
|
||||
// Read only by free tests (context-budget ratchet, host-config goldens), never by a paid case.
|
||||
'test/fixtures/context-budget.json',
|
||||
'test/fixtures/golden/claude-ship-SKILL.md',
|
||||
'test/fixtures/golden/codex-ship-SKILL.md',
|
||||
'test/fixtures/golden/factory-ship-SKILL.md',
|
||||
'test/helpers/stored-zip.ts', // Read only by the free test-health report tests.
|
||||
'scripts/free-test-durations.json',
|
||||
'scripts/paid-test-durations.json',
|
||||
'scripts/ubicloud/**',
|
||||
'tsconfig.test.json', // Read only by `tsc -p tsconfig.test.json` (typecheck:test); Bun's runtime reads tsconfig.json.
|
||||
'.gitignore', // Changes which untracked files git reports, never tracked content a case reads.
|
||||
'scripts/retired-command.ts', // One-release stubs for retired package scripts; no paid case imports it.
|
||||
// Reporting and launch tools: they read CI history or start a lane, never run inside a paid case.
|
||||
'scripts/test-health-report.ts',
|
||||
'scripts/bump-harness-version.ts',
|
||||
'scripts/eval-bg.ts',
|
||||
'.github/workflows/actionlint.yml',
|
||||
'.github/workflows/arm-setup-smoke.yml',
|
||||
'.github/workflows/ci-image.yml',
|
||||
'.github/workflows/cso-runtime-images.yml',
|
||||
'.github/workflows/cso-runtime-promote.yml',
|
||||
'.github/workflows/cso-runtime-qualification.yml',
|
||||
'.github/workflows/cso-scanner-images.yml',
|
||||
'.github/workflows/dependency-review.yml',
|
||||
'.github/workflows/free-tests.yml',
|
||||
'.github/workflows/make-pdf-gate.yml',
|
||||
'.github/workflows/native-qualification.yml',
|
||||
'.github/workflows/osv-scanner.yml',
|
||||
'.github/workflows/platform-qualification.yml',
|
||||
'.github/workflows/pr-title-sync.yml',
|
||||
'.github/workflows/quality-gate.yml',
|
||||
'.github/workflows/scorecard.yml',
|
||||
'.github/workflows/skill-docs.yml',
|
||||
'.github/workflows/test-health.yml',
|
||||
'.github/workflows/version-gate.yml',
|
||||
'.github/workflows/windows-free-tests.yml',
|
||||
'.github/workflows/windows-setup-e2e.yml',
|
||||
] as const;
|
||||
|
||||
const FULL_GATE_PR_FILES = [
|
||||
/**
|
||||
* Inputs every paid case consumes that must always restore the full gate
|
||||
* (fail closed, even when a touchfile or a derived reference also names
|
||||
* them): dependencies and the CI image, setup and the build, the PR
|
||||
* workflow and its setup actions, the Bun test preload and compiler config,
|
||||
* line-ending policy, the skill generator with the preamble every skill
|
||||
* embeds, and the eval policy every case is planned under.
|
||||
*/
|
||||
export const PR_FULL_GATE_FILES = [
|
||||
'package.json', 'bun.lock', '.github/docker/Dockerfile.ci',
|
||||
'scripts/host-config.ts', 'scripts/discover-skills.ts', 'hosts/index.ts',
|
||||
'setup', 'scripts/build.sh', '.github/workflows/evals.yml', '.github/actions/**',
|
||||
'test-setup.ts', 'bunfig.toml', 'tsconfig.json', '.gitattributes',
|
||||
'scripts/gen-skill-docs.ts', 'scripts/resolvers/index.ts', 'scripts/resolvers/types.ts',
|
||||
'scripts/resolvers/preamble.ts', 'scripts/resolvers/preamble/**', 'scripts/resolvers/runtime-root.ts',
|
||||
'test/helpers/periodic-exclude-data.ts',
|
||||
] as const;
|
||||
|
||||
/** Periodic- and marathon-lane workflows: the PR gate never runs a case under them; their branch dispatch validates them. */
|
||||
export const PERIODIC_ONLY_PR_FILES = ['.github/workflows/evals-periodic.yml', '.github/workflows/evals-marathon.yml'] as const;
|
||||
|
||||
/** Directories the reference derivation covers: an unmapped file there with no consumer is consumed by no paid case. */
|
||||
export const DERIVABLE_PREFIXES = ['bin/', 'lib/', 'scripts/', 'browse/', 'design/', 'make-pdf/', 'hosts/', 'extension/', 'model-overlays/', 'agents/', 'test/helpers/'] as const;
|
||||
|
||||
function knownNonBehaviorFile(file: string): boolean {
|
||||
// A mapped dependency still wins over these exemptions. New helper/fixture,
|
||||
// runtime, dependency, or workflow files are deliberately not exempted.
|
||||
// A mapped dependency still wins over these exemptions. New helper,
|
||||
// runtime, dependency, or paid-workflow files are deliberately not exempted.
|
||||
return /^(?:docs\/|(?:README|CONTRIBUTING|ARCHITECTURE|CHANGELOG|TODOS)\.md$|VERSION$)/.test(file)
|
||||
// Hermetic skill views exclude checkout instructions; these are maintained
|
||||
// by free doc/generation checks and are not copied into paid fixtures.
|
||||
|| ['AGENTS.md', 'CLAUDE.md', 'agents-digest/gstack-AGENTS.md'].includes(file)
|
||||
|| FREE_ONLY_PR_FILES.some(freeOnly => freeOnly === file)
|
||||
|| (file.startsWith('test/') && /\.test\.tsx?$/.test(file) && !isPaidTestFile(file));
|
||||
|| FREE_ONLY_PR_FILES.some(pattern => matchGlob(file, pattern))
|
||||
|| (PERIODIC_ONLY_PR_FILES as readonly string[]).includes(file)
|
||||
|| (file.startsWith('test/fixtures/') && Object.keys(FREE_FIXTURES).some(pattern => matchGlob(file, pattern)))
|
||||
// Free test files anywhere (test/, browse/test/, design/test/, make-pdf/test/): no paid case runs or reads a free test.
|
||||
|| (/\.test\.tsx?$/.test(file) && !isPaidTestFile(file));
|
||||
}
|
||||
|
||||
export const FALLBACK_FIX_ANCHOR = 'docs/TESTING_INTERNALS.md#pr-paid-lane-fallback';
|
||||
|
||||
/** Why one changed file restored the full gate, and the edit that narrows it next time (DX-6). */
|
||||
export function unknownFileLabel(file: string): { label: string; fix: string } {
|
||||
if (isPaidTestFile(file) || file.startsWith('test/helpers/')) {
|
||||
return { label: 'needs touchfile entry', fix: `register ${file} under the cases that consume it in test/helpers/touchfiles-data.ts` };
|
||||
}
|
||||
if (file.startsWith('test/fixtures/')) {
|
||||
return { label: 'needs touchfile entry', fix: `register ${file} in the touchfiles of the paid cases that read it, or add it with its free consumers to FREE_FIXTURES in test/helpers/free-fixtures-data.ts` };
|
||||
}
|
||||
if (/^\.github\/workflows\/[^/]+\.ya?ml$/.test(file) && !(PAID_WORKFLOW_FILES as readonly string[]).includes(file)) {
|
||||
return { label: 'add to FREE_ONLY_PR_FILES', fix: `add ${file} to FREE_ONLY_PR_FILES in scripts/test-pr-profile.ts` };
|
||||
}
|
||||
return { label: 'real unknown dependency', fix: `register ${file} under the cases that consume it in test/helpers/touchfiles-data.ts, add it to FREE_ONLY_PR_FILES in scripts/test-pr-profile.ts if no paid case reads it, or add its directory to DERIVABLE_PREFIXES so scripts/pr-dependencies.ts derives its consumers` };
|
||||
}
|
||||
|
||||
/** Only the release version may be ignored; dependency/script changes still matter. */
|
||||
@@ -170,6 +321,10 @@ export function selectPrProfile(options: {
|
||||
profile?: readonly string[];
|
||||
/** Generated artifacts may inherit the identity of their verified source template. */
|
||||
sourceAliases?: Readonly<Record<string, string>>;
|
||||
/** Test-file source reader for the DX-11 addressability check (defaults to the checkout). */
|
||||
readSource?: (file: string) => string;
|
||||
/** Derived references; default: derived from the checkout for the real maps, none for fixture maps. */
|
||||
derived?: DerivedDependencies | null;
|
||||
}): PrProfileSelection {
|
||||
const maps = options.maps ?? PR_PROFILE_MAPS;
|
||||
const profile = options.profile ?? PR_PROFILE_CASE_IDS;
|
||||
@@ -183,13 +338,52 @@ export function selectPrProfile(options: {
|
||||
...Object.values(maps.e2eTouchfiles).flat(), ...Object.values(maps.judgeTouchfiles).flat(),
|
||||
...maps.globalTouchfiles,
|
||||
])];
|
||||
const unknownFiles = files.filter(file => file !== TOUCHFILES_DATA_PATH
|
||||
const unmapped = files.filter(file => file !== TOUCHFILES_DATA_PATH
|
||||
&& !depends(file, dependencyPatterns) && !knownNonBehaviorFile(file));
|
||||
const sharedInputs = files.filter(file => depends(file, FULL_GATE_PR_FILES));
|
||||
// Unmapped files are placed by derived references where the derivation covers them; the rest restore the full gate.
|
||||
const derived = options.derived !== undefined ? options.derived : maps === PR_PROFILE_MAPS ? derivedDependencies(maps, ROOT) : null;
|
||||
// A path absent from the head tree was deleted by the diff: its consumers are the live files that still reference it.
|
||||
const consumers = new Map<string, { e2e: Set<string>; judges: Set<string> }>();
|
||||
const derivedGlobal: string[] = [];
|
||||
const noConsumerFiles: string[] = [];
|
||||
const deletedLabels = new Map<string, { label: string; fix: string }>();
|
||||
for (const file of derived ? unmapped : []) {
|
||||
if (derived!.tracked.has(file)) {
|
||||
if (derived!.global.has(file)) { derivedGlobal.push(file); continue; }
|
||||
const found = { e2e: derived!.e2e.get(file) ?? new Set<string>(), judges: derived!.judges.get(file) ?? new Set<string>() };
|
||||
if (found.e2e.size + found.judges.size > 0) consumers.set(file, found);
|
||||
else if (DERIVABLE_PREFIXES.some(prefix => file.startsWith(prefix))) noConsumerFiles.push(file);
|
||||
continue;
|
||||
}
|
||||
const referencers = derived!.referencers(file);
|
||||
const fullGate = referencers.find(referencer => matches(referencer, PR_FULL_GATE_FILES));
|
||||
if (fullGate) {
|
||||
deletedLabels.set(file, { label: 'deleted but still referenced', fix: `${file} is deleted but ${fullGate} (a full-gate input) still references it; remove the reference or restore the file` });
|
||||
continue;
|
||||
}
|
||||
if (referencers.some(referencer => derived!.global.has(referencer))) { derivedGlobal.push(file); continue; }
|
||||
const found = { e2e: new Set(referencers.flatMap(referencer => [...derived!.e2e.get(referencer) ?? []])),
|
||||
judges: new Set(referencers.flatMap(referencer => [...derived!.judges.get(referencer) ?? []])) };
|
||||
if (found.e2e.size + found.judges.size > 0) consumers.set(file, found);
|
||||
else noConsumerFiles.push(file);
|
||||
}
|
||||
const derivedFiles = [...consumers.keys()];
|
||||
const unknownFiles = unmapped.filter(file => !derivedGlobal.includes(file) && !consumers.has(file) && !noConsumerFiles.includes(file));
|
||||
const sharedInputs = files.filter(file => depends(file, PR_FULL_GATE_FILES));
|
||||
const fallback = unknownFiles.length > 0 || sharedInputs.length > 0;
|
||||
const candidates = fallback ? Object.keys(maps.e2eTouchfiles).sort() : selectedE2E;
|
||||
const e2e = candidates.filter(id => maps.tiers[id] === 'gate' && (fallback || profile.includes(id)));
|
||||
const judges = fallback ? Object.keys(maps.judgeTouchfiles).sort() : selectedJudges;
|
||||
const dependentsMode = !fallback && derivedFiles.length > 0;
|
||||
const derivedE2E = derivedFiles.flatMap(file => [...consumers.get(file)!.e2e]);
|
||||
const derivedJudges = derivedFiles.flatMap(file => [...consumers.get(file)!.judges]);
|
||||
// A file in the global touchfiles' import closure is as broad as a global touchfile.
|
||||
const everything = derivedGlobal.length > 0;
|
||||
const candidates = fallback || everything ? Object.keys(maps.e2eTouchfiles).sort() : [...new Set([...selectedE2E, ...derivedE2E])].sort();
|
||||
// DX-11: an edited paid test file runs its own gate cases even outside the profile, when the runner can select them by name.
|
||||
const editedPaid = new Set(files.filter(file => isPaidTestFile(file)));
|
||||
const owned = candidates.filter(id => maps.tiers[id] === 'gate' && !profile.includes(id) && editedPaid.has(caseOwnerFile(id, maps) ?? ''));
|
||||
const directCases = fallback || dependentsMode ? [] : owned.filter(id => prProfileDirectCase(id, maps, options.readSource) !== null);
|
||||
const unaddressed = fallback || dependentsMode ? [] : owned.filter(id => !directCases.includes(id));
|
||||
const e2e = candidates.filter(id => maps.tiers[id] === 'gate' && (fallback || dependentsMode || profile.includes(id) || directCases.includes(id)));
|
||||
const judges = fallback || everything ? Object.keys(maps.judgeTouchfiles).sort() : [...new Set([...selectedJudges, ...derivedJudges])].sort();
|
||||
const kept = new Set(e2e);
|
||||
const deferred = candidates.filter(id => !kept.has(id)).map(id => ({
|
||||
id, tier: maps.tiers[id],
|
||||
@@ -197,7 +391,9 @@ export function selectPrProfile(options: {
|
||||
? 'Full end-to-end marathon coverage; non-blocking lane, not executed by the PR gate'
|
||||
: maps.tiers[id] === 'periodic'
|
||||
? 'Broad periodic/release coverage; not executed by the PR gate'
|
||||
: 'Broad gate census/release coverage; outside the fast PR profile',
|
||||
: unaddressed.includes(id)
|
||||
? `Its test file changed, but the PR lane cannot select it by name; name its Bun test '${id}' (testIfSelected) to run it on PRs`
|
||||
: 'Broad gate census/release coverage; outside the fast PR profile',
|
||||
}));
|
||||
const noQuickCoverage = files.filter(file => isPromptFile(file)
|
||||
&& !(depends(file, maps.globalTouchfiles) && (e2e.length > 0 || judges.length > 0))
|
||||
@@ -209,14 +405,73 @@ export function selectPrProfile(options: {
|
||||
&& Object.values(maps.e2eTouchfiles).some(patterns => depends(file, patterns)));
|
||||
const missingCoverage = noQuickCoverage.filter(file => !deferredPromptFiles.includes(file));
|
||||
const reasons: string[] = [];
|
||||
if (unknownFiles.length) reasons.push(`Unknown dependencies restore every gate case and judge: ${unknownFiles.join(', ')}`);
|
||||
if (unknownFiles.length) reasons.push(`Unknown dependencies restore every gate case and judge: ${unknownFiles.map(file => `${file} (${(deletedLabels.get(file) ?? unknownFileLabel(file)).label})`).join(', ')}`);
|
||||
if (sharedInputs.length) reasons.push(`Shared runtime/build inputs restore every gate case and judge: ${sharedInputs.join(', ')}`);
|
||||
if (!fallback) reasons.push('Changed-input selection intersected with the fast PR profile; selected judges retained');
|
||||
if (dependentsMode) reasons.push(`Derived references select every dependent gate case: ${derivedFiles.map(file => `${file} (${consumers.get(file)!.e2e.size} cases)`).join(', ')}`);
|
||||
else if (!fallback) reasons.push('Changed-input selection intersected with the fast PR profile; selected judges retained');
|
||||
if (derivedGlobal.length) reasons.push(`Paid-runner inputs (global touchfile import closure) select every case: ${derivedGlobal.join(', ')}`);
|
||||
if (noConsumerFiles.length) reasons.push(`No paid case's reference closure reaches (or, deleted, still references): ${noConsumerFiles.join(', ')}`);
|
||||
if (directCases.length) reasons.push(`Edited paid test files select their own gate cases: ${directCases.join(', ')}`);
|
||||
if (deferred.length) reasons.push(`${deferred.length} selected behaviors remain scheduled/release coverage, not PR passes`);
|
||||
if (deferredPromptFiles.length) reasons.push(`No quick live coverage; known broad prompt checks deferred: ${deferredPromptFiles.join(', ')}`);
|
||||
if (missingCoverage.length) reasons.push(`Full validation required for prompts without a relevant PR check: ${missingCoverage.join(', ')}`);
|
||||
return {
|
||||
mode: fallback ? 'full-fallback' : 'pr', e2e, judges, deferred,
|
||||
unknownFiles, deferredPromptFiles, missingCoverage, needsFullValidation: missingCoverage.length > 0, reasons,
|
||||
mode: fallback ? 'full-fallback' : dependentsMode ? 'dependents' : 'pr', e2e, judges, deferred,
|
||||
unknownFiles, unknownFileLabels: unknownFiles.map(file => ({ file, ...(deletedLabels.get(file) ?? unknownFileLabel(file)) })), directCases,
|
||||
derivedFiles: derivedFiles.map(file => ({ file, e2e: consumers.get(file)!.e2e.size, judges: consumers.get(file)!.judges.size })), noConsumerFiles,
|
||||
deferredPromptFiles, missingCoverage, needsFullValidation: missingCoverage.length > 0, reasons,
|
||||
};
|
||||
}
|
||||
|
||||
/** The manifest fields the coverage summary reads (a PaidRunManifest subset). */
|
||||
export interface CoverageSummaryInput {
|
||||
profile?: string;
|
||||
selection?: { e2e: string[] | null; judges: string[] | null };
|
||||
prCoverage?: Pick<PrProfileSelection, 'mode' | 'deferred' | 'unknownFiles'> & Partial<Pick<PrProfileSelection, 'unknownFileLabels' | 'derivedFiles' | 'noConsumerFiles'>>;
|
||||
}
|
||||
|
||||
/**
|
||||
* Job-summary block for one PR plan (CEO-13): the profile mode, selected case
|
||||
* counts, reused records once the report has them, and on fallback every
|
||||
* file that restored the full gate with its fix.
|
||||
*/
|
||||
export function formatPrCoverageSummary(manifest: CoverageSummaryInput, totals?: { total: number; reused: number }): string[] {
|
||||
const coverage = manifest.prCoverage;
|
||||
const count = (ids: string[] | null | undefined) => ids === null || ids === undefined ? 'all' : String(ids.length);
|
||||
const lines = [
|
||||
'### PR paid lane coverage',
|
||||
`- Mode: \`${coverage?.mode ?? manifest.profile ?? 'full'}\``,
|
||||
`- Selected: ${count(manifest.selection?.e2e)} E2E case(s), ${count(manifest.selection?.judges)} judge(s); ${coverage?.deferred.length ?? 0} deferred to scheduled/release coverage`,
|
||||
];
|
||||
if (totals) lines.push(`- Reused: ${totals.reused} of ${totals.total} rule/judge record(s) came from verified receipts`);
|
||||
if (coverage?.derivedFiles?.length) {
|
||||
lines.push(`- Derived dependents (scripts/pr-dependencies.ts) of ${coverage.derivedFiles.length} unmapped file(s):`);
|
||||
for (const item of coverage.derivedFiles.slice(0, 40)) lines.push(` - \`${item.file}\`: ${item.e2e} case(s), ${item.judges} judge(s)`);
|
||||
}
|
||||
if (coverage?.noConsumerFiles?.length) lines.push(`- No paid case reaches: ${coverage.noConsumerFiles.slice(0, 40).map(file => `\`${file}\``).join(', ')}`);
|
||||
if (coverage?.mode === 'full-fallback' && coverage.unknownFiles.length) {
|
||||
lines.push(`- Full gate restored by ${coverage.unknownFiles.length} file(s) (fix: ${FALLBACK_FIX_ANCHOR}):`);
|
||||
for (const file of coverage.unknownFiles) {
|
||||
const labeled = coverage.unknownFileLabels?.find(entry => entry.file === file) ?? { file, ...unknownFileLabel(file) };
|
||||
lines.push(` - \`${file}\` (${labeled.label}): ${labeled.fix}`);
|
||||
}
|
||||
}
|
||||
return lines;
|
||||
}
|
||||
|
||||
if (import.meta.main) {
|
||||
const [command, manifestPath, outcomesPath] = process.argv.slice(2);
|
||||
if (command !== 'summary' || !manifestPath) {
|
||||
console.error('usage: bun run scripts/test-pr-profile.ts summary <manifest.json> [collector-outcomes.json]');
|
||||
process.exit(2);
|
||||
}
|
||||
const manifest = JSON.parse(fs.readFileSync(manifestPath, 'utf8')) as CoverageSummaryInput;
|
||||
let totals: { total: number; reused: number } | undefined;
|
||||
if (outcomesPath) {
|
||||
try {
|
||||
const parsed = JSON.parse(fs.readFileSync(outcomesPath, 'utf8'));
|
||||
if (Number.isSafeInteger(parsed?.totals?.total) && Number.isSafeInteger(parsed?.totals?.reused)) totals = parsed.totals;
|
||||
} catch { /* No verified report: the summary omits the reuse line. */ }
|
||||
}
|
||||
console.log(formatPrCoverageSummary(manifest, totals).join('\n'));
|
||||
}
|
||||
@@ -421,7 +421,6 @@
|
||||
"test/plan-tune.test.ts\tTS2345\tArgument of type '{ skillName: string; tmplPath: string; host: 'codex'; paths: { skillRoot: string; localSkillRoot: string; binDir: string; browseDir: string; designDir: string; }; }' is not assignable to parameter of type 'TemplateContext'. Types of property 'paths' are incompatible. Property 'makePdfDir' is missing in type '{ skillRoot: string; localSkillRoot: string; binDir: string; browseDir: string; designDir: string; }' but required in type 'HostPaths'.": 1,
|
||||
"test/preamble-compose.test.ts\tTS2322\tType '{ skillName: string; tmplPath: string; host: \"claude\" | \"codex\"; paths: HostPaths; preambleTier: 1 | 2 | 3 | 4; model?: string | undefined; }' is not assignable to type 'TemplateContext'. Types of property 'model' are incompatible. Type 'string | undefined' is not assignable to type '\"claude\" | \"fable-5\" | \"gemini\" | \"gpt\" | \"gpt-5.4\" | \"gpt-5.6-sol\" | \"gpt-6-astra\" | \"o-series\" | \"opus-4-7\" | \"opus-4-8\" | \"sonnet-5\" | undefined'. Type 'string' is not assignable to type '\"claude\" | \"fable-5\" | \"gemini\" | \"gpt\" | \"gpt-5.4\" | \"gpt-5.6-sol\" | \"gpt-6-astra\" | \"o-series\" | \"opus-4-7\" | \"opus-4-8\" | \"sonnet-5\" | undefined'.": 1,
|
||||
"test/provider-model-defaults.test.ts\tTS2769\tNo overload matches this call. The last overload gave the following error. Argument of type 'string | undefined' is not assignable to parameter of type 'string'. Type 'undefined' is not assignable to type 'string'.": 1,
|
||||
"test/pty-output-wake.test.ts\tTS2352\tConversion of type '() => { exited: Promise<number>; kill: (signal: string) => void; terminal: { write(): void; }; }' to type '{ <const In extends SpawnOptions.Writable = \"ignore\", const Out extends SpawnOptions.Readable = \"pipe\", const Err extends SpawnOptions.Readable = \"inherit\">(options: SpawnOptions<In, Out, Err> & { cmd: string[]; }): Subprocess<...>; <const In extends SpawnOptions.Writable = \"ignore\", const Out extends SpawnOptions.R...' may be a mistake because neither type sufficiently overlaps with the other. If this was intentional, convert the expression to 'unknown' first. Type '{ exited: Promise<number>; kill: (signal: string) => void; terminal: { write(): void; }; }' is missing the following properties from type 'Subprocess<any, any, any>': stdin, stdout, stderr, stdio, and 11 more.": 1,
|
||||
"test/pty-workspace-trust.test.ts\tTS2345\tArgument of type '(_command: any, options: any) => any' is not assignable to parameter of type '{ <const In extends SpawnOptions.Writable = \"ignore\", const Out extends SpawnOptions.Readable = \"pipe\", const Err extends SpawnOptions.Readable = \"inherit\">(options: SpawnOptions<In, Out, Err> & { cmd: string[]; }): Subprocess<...>; <const In extends SpawnOptions.Writable = \"ignore\", const Out extends SpawnOptions.R...'. Target signature provides too few arguments. Expected 2 or more, but got 1.": 1,
|
||||
"test/pty-workspace-trust.test.ts\tTS2559\tType '5000' has no properties in common with type '{ timeoutMs?: number | undefined; pollMs?: number | undefined; since?: number | undefined; }'.": 1,
|
||||
"test/qa-browser-preservation.test.ts\tTS2769\tNo overload matches this call. The last overload gave the following error. Argument of type 'number | null' is not assignable to parameter of type 'number'. Type 'null' is not assignable to type 'number'.": 1,
|
||||
@@ -440,8 +439,6 @@
|
||||
"test/salience-allowlist.test.ts\tTS2307\tCannot find module '../bin/gstack-brain-cache' or its corresponding type declarations.": 3,
|
||||
"test/schema-version-migration.test.ts\tTS2307\tCannot find module '../bin/gstack-brain-cache' or its corresponding type declarations.": 3,
|
||||
"test/schema-version-migration.test.ts\tTS2353\tObject literal may only specify known properties, and 'timeout' does not exist in type '(done: (err?: unknown) => void) => void | Promise<unknown>'.": 3,
|
||||
"test/section-capture-native-tools.test.ts\tTS2339\tProperty 'CI' does not exist on type '{ NODE_ENV?: string | undefined; TZ?: string | undefined; PATH: string; TMPDIR: string; TMP: string; TEMP: string; EVALS_HERMETIC: string; }'.": 3,
|
||||
"test/section-capture-native-tools.test.ts\tTS2339\tProperty 'CI' does not exist on type '{ NODE_ENV?: string | undefined; TZ?: string | undefined; PATH: string; TMPDIR: string; TMP: string; TEMP: string; GSTACK_HOME: string; GSTACK_STATE_ROOT: string; EVALS_HERMETIC: string; EVALS: string; EVALS_ALL: string; GSTACK_CARVE_SKILL: string; }'.": 1,
|
||||
"test/section-capture-native-tools.test.ts\tTS2339\tProperty 'CI' does not exist on type '{ NODE_ENV?: string | undefined; TZ?: string | undefined; PATH: string; }'.": 1,
|
||||
"test/session-runner-browse-errors.test.ts\tTS2769\tNo overload matches this call. The last overload gave the following error. Argument of type 'never[]' is not assignable to parameter of type 'undefined'.": 1,
|
||||
"test/session-runner-browse-errors.test.ts\tTS7006\tParameter 'row' implicitly has an 'any' type.": 6,
|
||||
|
||||
@@ -11,7 +11,7 @@ export DEBIAN_FRONTEND=noninteractive
|
||||
sudo -E apt-get update -qq
|
||||
sudo -E apt-get install -y -qq --no-install-recommends \
|
||||
git curl unzip xz-utils ca-certificates build-essential clang python3-venv jq \
|
||||
xvfb x11-utils poppler-utils fonts-noto-color-emoji >/dev/null
|
||||
xvfb x11-utils poppler-utils fonts-noto-color-emoji zsh >/dev/null
|
||||
|
||||
# Ubuntu 24.04 blocks unprivileged user namespaces, which Chromium's sandbox
|
||||
# needs; GitHub-hosted runners ship with this relaxed.
|
||||
|
||||
@@ -1,79 +0,0 @@
|
||||
#!/usr/bin/env bun
|
||||
/**
|
||||
* Read docs/throughput-2013-vs-2026.json, replace the README anchor with the
|
||||
* computed logical-lines multiple.
|
||||
*
|
||||
* Two-string pattern (resolves the pipeline-eats-itself bug Codex caught in V1
|
||||
* planning, Pass 2 finding #10):
|
||||
* - GSTACK-THROUGHPUT-PLACEHOLDER — stable anchor, lives in README permanently.
|
||||
* Script finds this anchor and writes the number right before it, keeping
|
||||
* the anchor itself for the next run.
|
||||
* - GSTACK-THROUGHPUT-PENDING — explicit missing-build marker. If the JSON
|
||||
* isn't present, the script writes this marker at the anchor location.
|
||||
* CI rejects commits containing this string, so contributors get a clear
|
||||
* signal to run the throughput script before committing.
|
||||
*/
|
||||
import * as fs from 'fs';
|
||||
import * as path from 'path';
|
||||
|
||||
const ROOT = process.cwd();
|
||||
const README = path.join(ROOT, 'README.md');
|
||||
const JSON_PATH = path.join(ROOT, 'docs', 'throughput-2013-vs-2026.json');
|
||||
|
||||
const ANCHOR = '<!-- GSTACK-THROUGHPUT-PLACEHOLDER -->';
|
||||
const PENDING = 'GSTACK-THROUGHPUT-PENDING';
|
||||
|
||||
function main() {
|
||||
if (!fs.existsSync(README)) {
|
||||
process.stderr.write(`README.md not found at ${README}\n`);
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
const readme = fs.readFileSync(README, 'utf-8');
|
||||
if (!readme.includes(ANCHOR)) {
|
||||
// Anchor already replaced by a computed number (or was never inserted).
|
||||
// Nothing to do — silent success.
|
||||
return;
|
||||
}
|
||||
|
||||
if (!fs.existsSync(JSON_PATH)) {
|
||||
// Build hasn't produced the JSON. Write the PENDING marker at the anchor,
|
||||
// preserving the anchor so the next run can replace it.
|
||||
const replacement = `${PENDING}: run scripts/garry-output-comparison.ts ${ANCHOR}`;
|
||||
const updated = readme.replace(ANCHOR, replacement);
|
||||
fs.writeFileSync(README, updated);
|
||||
process.stderr.write(
|
||||
`${JSON_PATH} not found. Wrote ${PENDING} marker to README. Run scripts/garry-output-comparison.ts to generate it.\n`
|
||||
);
|
||||
// Non-zero exit so CI that wraps this sees the signal, but local dev workflows
|
||||
// can continue. Callers can decide whether this is fatal.
|
||||
process.exit(0);
|
||||
}
|
||||
|
||||
let parsed: { multiples?: { logical_lines_added?: number | null } } = {};
|
||||
try {
|
||||
parsed = JSON.parse(fs.readFileSync(JSON_PATH, 'utf-8'));
|
||||
} catch (err) {
|
||||
process.stderr.write(`Failed to parse ${JSON_PATH}: ${err}\n`);
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
const mult = parsed?.multiples?.logical_lines_added;
|
||||
if (mult === null || mult === undefined) {
|
||||
// JSON exists but doesn't have a computable multiple (e.g., one year inactive).
|
||||
// Write an honest pending-ish marker. Don't fall back to a bogus number.
|
||||
const replacement = `${PENDING}: multiple not yet computable (one or both years inactive in this repo) ${ANCHOR}`;
|
||||
const updated = readme.replace(ANCHOR, replacement);
|
||||
fs.writeFileSync(README, updated);
|
||||
process.stderr.write(`Multiple not computable. Wrote ${PENDING} marker.\n`);
|
||||
process.exit(0);
|
||||
}
|
||||
|
||||
// Normal flow: replace the anchor with the number + anchor (anchor stays for next run).
|
||||
const replacement = `**${mult}×** ${ANCHOR}`;
|
||||
const updated = readme.replace(ANCHOR, replacement);
|
||||
fs.writeFileSync(README, updated);
|
||||
process.stderr.write(`README throughput multiple updated: ${mult}×\n`);
|
||||
}
|
||||
|
||||
main();
|
||||
Reference in new issue
Block a user