mirror of
https://github.com/garrytan/gstack.git
synced 2026-11-05 08:23:59 +01:00
feat(evals): red-head and READY-arrival replays for the gate-cost model (GSTA-49 T0)
Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>
This commit is contained in:
garrytanandcapy-ai[bot] committed
1 parent
e9dfcf64f0
commit
b27bf7e10a
5 files changed
+911
-28
No files matched your search
@@ -0,0 +1,421 @@
|
||||
/**
|
||||
* gate-cost-history — the two history replays of GSTA-49 plan lane T0 (P-i,
|
||||
* P-k, and the READY-to-merge guardrail in its Risks). Pure logic: the CLI
|
||||
* (`scripts/gate-cost-replay.ts --history`) passes run lists, report texts,
|
||||
* head signatures and selections in; nothing here runs git or calls GitHub.
|
||||
*
|
||||
* 1. Red attribution: each red PR-gate run's failed cases split into
|
||||
* PR-caused and outside reds by GSTA-47's rule (`classifyReds`).
|
||||
* 2. Red-head replay: every PR-caused red case against the
|
||||
* screen-provisional rule on the failing head's diff from its tested base,
|
||||
* with the canary set below fixed before the replay first ran.
|
||||
* 3. READY arrivals: a READY time per merged PR from its check history, and
|
||||
* a seeded replay of those arrivals through the train policy with the
|
||||
* merge-train model's red and bisect rules.
|
||||
*/
|
||||
import { createHash } from 'node:crypto';
|
||||
import { PyRandom, policyLabel, screenedIndexes, type ScreenPolicy, type ScreenSelection } from './gate-cost';
|
||||
|
||||
const HOUR_MS = 3_600_000;
|
||||
|
||||
// --- 1. Red attribution ---
|
||||
|
||||
export interface CaseFailure { caseId: string; kind: string }
|
||||
export interface ReportVerdict { verdict: 'GREEN' | 'RED' | 'NONE'; failures: CaseFailure[] }
|
||||
|
||||
/** report-verdict's report.txt: the verdict line and each `✗ <id> <kind> <status>` line (judge ids contain single spaces; an unbound shard is named by its test file). */
|
||||
export function parseReportVerdict(text: string): ReportVerdict {
|
||||
const verdict = /VERDICT (GREEN|RED)/.exec(text)?.[1] as 'GREEN' | 'RED' | undefined;
|
||||
const failures = [...text.matchAll(/^\s+✗ (\S(?:.*?\S)?) {2,}(\w+) +\S/gm)].map(m => ({ caseId: m[1]!, kind: m[2]! }));
|
||||
return { verdict: verdict ?? 'NONE', failures };
|
||||
}
|
||||
|
||||
/** One PR-gate run that produced a report (GSTA-47's "gated" run); failures are the union over its attempts. */
|
||||
export interface GatedRun { id: number; pr: number | null; head: string; createdAt: string; green: boolean; failures: CaseFailure[] }
|
||||
/** A head's own diff: the files changed from its merge-base with main, and GSTA-47's stamp-free signature of that diff. */
|
||||
export interface HeadDiff { ownSig: string; files: string[] }
|
||||
|
||||
export const STAMP_FILES = ['VERSION', 'CHANGELOG.md', 'agents-digest/gstack-AGENTS.md'] as const;
|
||||
|
||||
/**
|
||||
* GSTA-47's own-diff signature over `git diff -U0 <merge-base> <head> -- . ':(exclude)<stamp>'...`:
|
||||
* hunk headers, index lines and the package.json version line are dropped, so
|
||||
* a main merge or a restamp keeps the signature.
|
||||
*/
|
||||
export function stampFreeSignature(diffU0: string): string {
|
||||
let current = '';
|
||||
const blocks: string[][] = [];
|
||||
for (const line of diffU0.split('\n')) {
|
||||
if (line.startsWith('diff --git')) { current = line.split(' b/')[1] ?? ''; blocks.push([`F ${current}`]); continue; }
|
||||
if (!blocks.length || /^(index |@@|--- |\+\+\+ )/.test(line)) continue;
|
||||
if (current === 'package.json' && /^[-+]\s*"version":/.test(line)) continue;
|
||||
blocks[blocks.length - 1]!.push(line);
|
||||
}
|
||||
const kept = blocks.filter(block => block.length > 1 || !block[0]!.startsWith('F package.json')).flat();
|
||||
return createHash('sha256').update(kept.join('\n')).digest('hex').slice(0, 16);
|
||||
}
|
||||
|
||||
/** A case red in another PR's run within this window is an outside red. */
|
||||
export const RED_ELSEWHERE_WINDOW_MS = 72 * HOUR_MS;
|
||||
/** Reds GSTA-47 set from transcripts: GSTA-45's own-edit reds on #3260, and the main defects fix wave 3 (#3260) diagnosed. */
|
||||
export const DIAGNOSED_REDS = {
|
||||
prCaused: [39073934326, 39075977549, 39077524802],
|
||||
outside: [38973329985, 38986120559, 38990111230, 38993076767, 39002269281, 39004466422, 39011101113, 38980005793,
|
||||
39067746095, 39070632885, 39067885089, 39070276914, 39066860900, 39053431488],
|
||||
} as const;
|
||||
|
||||
export type RedCause = 'pr' | 'outside' | 'unknown';
|
||||
export interface CaseRed extends CaseFailure { cause: RedCause; reason: string }
|
||||
export interface RedRun { runId: number; pr: number; head: string; createdAt: string; nextRunId: number | null; cause: RedCause; cases: CaseRed[] }
|
||||
|
||||
export interface ClassifyDeps {
|
||||
heads: ReadonlyMap<string, HeadDiff>;
|
||||
/** `git diff --name-only <from> <to>`; null when unreadable. */
|
||||
delta(from: string, to: string): string[] | null;
|
||||
/** The files among `files` that are inputs of the case (declared touchfiles, its test file, harness globals). */
|
||||
touches(caseId: string, files: string[]): string[];
|
||||
}
|
||||
|
||||
/**
|
||||
* GSTA-47's attribution, per failed case of a red run, first match wins:
|
||||
* a transcript diagnosis; red in another PR within 72 h (outside); no later
|
||||
* gated run on the PR (unknown); the next gated run has the same own diff
|
||||
* (outside); the push to the next run changed one of the case's inputs among
|
||||
* either head's own files (PR-caused, whether or not the case then passed);
|
||||
* otherwise outside. A run is PR-caused when any case is, outside when every
|
||||
* case is (or none was named: infra or incomplete), unknown otherwise.
|
||||
*/
|
||||
export function classifyReds(runs: GatedRun[], deps: ClassifyDeps, diagnosed: { prCaused: readonly number[]; outside: readonly number[] } = DIAGNOSED_REDS): RedRun[] {
|
||||
const reds = runs.filter(run => !run.green);
|
||||
const index = reds.flatMap(run => run.failures.map(f => ({ caseId: f.caseId, pr: run.pr, t: Date.parse(run.createdAt) })));
|
||||
const out: RedRun[] = [];
|
||||
for (const run of reds) {
|
||||
if (run.pr === null) continue;
|
||||
const next = runs.filter(r => r.pr === run.pr && r.createdAt > run.createdAt).sort((a, b) => a.createdAt.localeCompare(b.createdAt))[0] ?? null;
|
||||
const own = deps.heads.get(run.head), nextOwn = next ? deps.heads.get(next.head) : undefined;
|
||||
const t = Date.parse(run.createdAt);
|
||||
const cases: CaseRed[] = [];
|
||||
for (const failure of run.failures) {
|
||||
if (cases.some(c => c.caseId === failure.caseId)) continue;
|
||||
const elsewhere = [...new Set(index.filter(x => x.caseId === failure.caseId && x.pr !== run.pr && Math.abs(x.t - t) < RED_ELSEWHERE_WINDOW_MS).map(x => x.pr))];
|
||||
const verdict = ((): [RedCause, string] => {
|
||||
if (diagnosed.prCaused.includes(run.id)) return ['pr', 'diagnosed own edit (GSTA-45)'];
|
||||
if (diagnosed.outside.includes(run.id)) return ['outside', 'diagnosed main defect (fix wave 3)'];
|
||||
if (elsewhere.length) return ['outside', `also red in ${elsewhere.map(pr => (pr === null ? 'an unmapped run' : `#${pr}`)).join(', ')}`];
|
||||
if (!next) return ['unknown', 'no later gated run'];
|
||||
const passed = !next.failures.some(f => f.caseId === failure.caseId);
|
||||
if (!own || !nextOwn) return ['unknown', 'a head is not readable'];
|
||||
if (own.ownSig === nextOwn.ownSig) return ['outside', passed ? 'passed next with the same own diff' : 'still red with the same own diff'];
|
||||
const delta = deps.delta(run.head, next.head);
|
||||
if (!delta) return ['unknown', 'the push between runs is not readable'];
|
||||
const hit = deps.touches(failure.caseId, delta.filter(file => own.files.includes(file) || nextOwn.files.includes(file)));
|
||||
if (hit.length) return ['pr', `${passed ? 'fixed by own push' : 'own push touched its inputs, still red'}: ${hit.slice(0, 3).join(' ')}`];
|
||||
return ['outside', passed ? 'passed next after an unrelated push' : 'still red after an unrelated push'];
|
||||
})();
|
||||
cases.push({ ...failure, cause: verdict[0], reason: verdict[1] });
|
||||
}
|
||||
const cause: RedCause = cases.every(c => c.cause === 'outside') ? 'outside' : cases.some(c => c.cause === 'pr') ? 'pr' : 'unknown';
|
||||
out.push({ runId: run.id, pr: run.pr, head: run.head, createdAt: run.createdAt, nextRunId: next?.id ?? null, cause, cases });
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
// --- 2. Red-head replay ---
|
||||
|
||||
/**
|
||||
* T2's structural canary rule (PLAN.md rev 6: one case per preamble-dependent
|
||||
* surface), resolved to gate cases before this replay first ran and not
|
||||
* changed after: for each surface, the gate case with the shortest recorded
|
||||
* session time (run 39075977549) among the cases on that surface. No gate
|
||||
* case runs plan mode on the claude -p host, so that surface has none.
|
||||
* Provisional until T2's reviewed list in test/helpers/touchfiles-data.ts.
|
||||
*/
|
||||
export const SCREEN_CANARY_CASES: Readonly<Record<string, string>> = {
|
||||
'skill start, claude -p (skillmd-* cases)': 'skillmd-outside-git',
|
||||
'skill start, PTY (PTY cases outside plan mode)': 'office-hours-auto-mode',
|
||||
'plan mode, PTY (*-plan-mode and plan-mode-no-op)': 'plan-devex-review-plan-mode',
|
||||
'AskUserQuestion format': 'auq-format-gate',
|
||||
'bins every skill runs (cases declaring scripts/resolvers/preamble.ts)': 'operational-learning',
|
||||
'hermetic sentinels (1 of 2)': 'hermetic-canary',
|
||||
'hermetic sentinels (2 of 2)': 'hermetic-sentinel',
|
||||
};
|
||||
|
||||
export const JUDGE_TEST_FILE = 'test/skill-llm-eval.test.ts';
|
||||
export type ScreenHit = 'judge' | 'global' | 'touchfiles' | 'canary' | null;
|
||||
|
||||
/**
|
||||
* Would the screen select this failed case? Judges always run. A failure
|
||||
* named by its test file (an unbound shard) counts as selected when any
|
||||
* selected case lives in that file.
|
||||
*/
|
||||
export function screenHit(failure: CaseFailure, selection: ScreenSelection, caseFile: (caseId: string) => string | null,
|
||||
canaryCases: readonly string[] = Object.values(SCREEN_CANARY_CASES)): ScreenHit {
|
||||
if (failure.kind === 'judge' || caseFile(failure.caseId) === JUDGE_TEST_FILE) return 'judge';
|
||||
if (selection.global) return 'global';
|
||||
const byFile = /^test\/.+\.test\.ts$/.test(failure.caseId);
|
||||
const covers = (ids: readonly string[]) => (byFile ? ids.some(id => caseFile(id) === failure.caseId) : ids.includes(failure.caseId));
|
||||
if (covers(selection.e2e)) return 'touchfiles';
|
||||
return selection.canary && covers(canaryCases) ? 'canary' : null;
|
||||
}
|
||||
|
||||
export interface RedHeadRow extends CaseFailure {
|
||||
pr: number;
|
||||
runId: number;
|
||||
head: string;
|
||||
base: string;
|
||||
/** `tested-merge`: the first parent of the run's tested merge commit; `main-at-run`: main's first-parent tip when the run was created. */
|
||||
baseSource: 'tested-merge' | 'main-at-run';
|
||||
files: number;
|
||||
hit: ScreenHit;
|
||||
canaryReasons: string[];
|
||||
}
|
||||
|
||||
export interface RedHeadSummary {
|
||||
mergedPrs: number;
|
||||
paidRelevantPrs: number;
|
||||
/** Merged PRs with at least one PR-caused red run (P-k's numerator). */
|
||||
prsWithPrRed: number[];
|
||||
prCausedRuns: number;
|
||||
prCausedCases: number;
|
||||
/** Merged PRs with a PR-caused red case the screen would not select (P-i's numerator). */
|
||||
missedPrs: number[];
|
||||
missedCases: number;
|
||||
/** Misses if the canary set were empty, to show how much of the result the canary choice carries. */
|
||||
missedPrsWithoutCanary: number[];
|
||||
byHit: Record<string, number>;
|
||||
/** PR-caused red runs on PRs that did not merge in the window (not counted above). */
|
||||
otherPrCausedRuns: number;
|
||||
}
|
||||
|
||||
export function summarizeRedHeads(rows: RedHeadRow[], redRuns: RedRun[], merged: Array<{ pr: number; paidRelevant: boolean }>): RedHeadSummary {
|
||||
const mergedSet = new Set(merged.map(m => m.pr));
|
||||
const inCohort = rows.filter(row => mergedSet.has(row.pr));
|
||||
const prRuns = redRuns.filter(run => run.cause === 'pr');
|
||||
const prsOf = (list: Array<{ pr: number }>) => [...new Set(list.map(row => row.pr))].sort((a, b) => a - b);
|
||||
const byHit: Record<string, number> = {};
|
||||
for (const row of inCohort) byHit[row.hit ?? 'missed'] = (byHit[row.hit ?? 'missed'] ?? 0) + 1;
|
||||
return {
|
||||
mergedPrs: merged.length,
|
||||
paidRelevantPrs: merged.filter(m => m.paidRelevant).length,
|
||||
prsWithPrRed: prsOf(prRuns.filter(run => mergedSet.has(run.pr))),
|
||||
prCausedRuns: prRuns.filter(run => mergedSet.has(run.pr)).length,
|
||||
prCausedCases: inCohort.length,
|
||||
missedPrs: prsOf(inCohort.filter(row => row.hit === null)),
|
||||
missedCases: inCohort.filter(row => row.hit === null).length,
|
||||
missedPrsWithoutCanary: prsOf(inCohort.filter(row => row.hit === null || row.hit === 'canary')),
|
||||
byHit,
|
||||
otherPrCausedRuns: prRuns.filter(run => !mergedSet.has(run.pr)).length,
|
||||
};
|
||||
}
|
||||
|
||||
// --- 3. READY arrivals ---
|
||||
|
||||
export interface CheckRun { head: string; workflow: 'free' | 'evals'; createdAt: string; completedAt: string; conclusion: string | null }
|
||||
|
||||
/**
|
||||
* When each head's checks went green: its latest Free Tests run succeeded and
|
||||
* its latest E2E Evals run (if any) succeeded or was skipped; the time is the
|
||||
* later of the two completions (`updated_at`). Null: never green.
|
||||
*/
|
||||
export function headGreenAt(runs: CheckRun[]): Map<string, string | null> {
|
||||
const latest = new Map<string, Partial<Record<CheckRun['workflow'], CheckRun>>>();
|
||||
for (const run of runs) {
|
||||
const entry = latest.get(run.head) ?? {};
|
||||
latest.set(run.head, entry);
|
||||
if (!entry[run.workflow] || entry[run.workflow]!.createdAt < run.createdAt) entry[run.workflow] = run;
|
||||
}
|
||||
const out = new Map<string, string | null>();
|
||||
for (const [head, { free, evals }] of latest) {
|
||||
const green = free?.conclusion === 'success' && (!evals || ['success', 'skipped', 'neutral'].includes(evals.conclusion ?? ''));
|
||||
out.set(head, green ? [free!.completedAt, evals?.completedAt ?? ''].sort().pop()! : null);
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
export interface PrChecks {
|
||||
pr: number;
|
||||
mergedAt: string;
|
||||
finalHead: string;
|
||||
/** Every head that ran Free Tests or E2E Evals on the PR, with its own-diff signature (null: unreadable). */
|
||||
heads: Array<{ sha: string; ownSig: string | null }>;
|
||||
/** Creation times of the PR's gated E2E Evals runs. */
|
||||
gatedRuns: string[];
|
||||
}
|
||||
|
||||
export type ReadySource = 'final-own-diff' | 'final-head' | 'none';
|
||||
export interface ReadyRow { pr: number; mergedAt: string; ready: string | null; source: ReadySource; readyToMergeHours: number | null; gatesAfterReady: number }
|
||||
|
||||
/**
|
||||
* READY is approximated by the first time the PR's final own diff (its merged
|
||||
* changes modulo main merges and stamps) had green checks before the merge:
|
||||
* the earliest green time among heads with the final head's signature, which
|
||||
* is when an unstamped READY report could first be sent. Fallback: the final
|
||||
* head's own green time. A green time after the merge does not count.
|
||||
*/
|
||||
export function readyTimes(prs: PrChecks[], greenAt: ReadonlyMap<string, string | null>): ReadyRow[] {
|
||||
return prs.map(pr => {
|
||||
const finalSig = pr.heads.find(h => h.sha === pr.finalHead)?.ownSig ?? null;
|
||||
const before = (sha: string) => { const at = greenAt.get(sha) ?? null; return at !== null && at <= pr.mergedAt ? at : null; };
|
||||
const sameDiff = finalSig === null ? [] : pr.heads.filter(h => h.ownSig === finalSig).map(h => before(h.sha)).filter((at): at is string => at !== null).sort();
|
||||
const finalAt = before(pr.finalHead);
|
||||
const [ready, source]: [string | null, ReadySource] = sameDiff.length ? [sameDiff[0]!, 'final-own-diff'] : finalAt ? [finalAt, 'final-head'] : [null, 'none'];
|
||||
return {
|
||||
pr: pr.pr, mergedAt: pr.mergedAt, ready, source,
|
||||
readyToMergeHours: ready ? (Date.parse(pr.mergedAt) - Date.parse(ready)) / HOUR_MS : null,
|
||||
gatesAfterReady: ready ? pr.gatedRuns.filter(at => at > ready && at <= pr.mergedAt).length : 0,
|
||||
};
|
||||
});
|
||||
}
|
||||
|
||||
/** Nearest-rank percentile; 0 for an empty list. */
|
||||
export function percentile(values: number[], q: number): number {
|
||||
if (!values.length) return 0;
|
||||
const sorted = [...values].sort((a, b) => a - b);
|
||||
return sorted[Math.max(0, Math.ceil(q * sorted.length) - 1)]!;
|
||||
}
|
||||
|
||||
export interface LatencySummary { prs: number; p50Hours: number; p90Hours: number; meanHours: number }
|
||||
export const latencySummary = (hours: number[]): LatencySummary => ({
|
||||
prs: hours.length, p50Hours: percentile(hours, 0.5), p90Hours: percentile(hours, 0.9),
|
||||
meanHours: hours.reduce((a, b) => a + b, 0) / Math.max(1, hours.length),
|
||||
});
|
||||
|
||||
export interface ArrivalTrainInput {
|
||||
/** One per merged PR with a READY time; `screenUsd` orders the cheapest/expensive-half policies. */
|
||||
arrivals: Array<{ pr: number; readyMs: number; paidRelevant: boolean; screenUsd: number }>;
|
||||
policy: ScreenPolicy;
|
||||
windowDays: number;
|
||||
trainSize?: number;
|
||||
fillLimitHours?: number;
|
||||
gateMinutes?: number;
|
||||
bisectMinutes?: number;
|
||||
outsideRed?: number;
|
||||
redScreened?: number;
|
||||
redUnscreened?: number;
|
||||
reps?: number;
|
||||
seed?: number;
|
||||
}
|
||||
|
||||
export interface ArrivalTrainResult {
|
||||
policy: string;
|
||||
outsideRed: number;
|
||||
trainsWeekly: number;
|
||||
meanOccupancy: number;
|
||||
fullTrainShare: number;
|
||||
/** Initial trains per repetition by size, index 0 = one PR. */
|
||||
occupancy: number[];
|
||||
gateAttemptsWeekly: number;
|
||||
bisectsWeekly: number;
|
||||
/** Gates again after a bisect dropped a defective PR. */
|
||||
attributedRebuildsWeekly: number;
|
||||
/** Garry's one unchanged rebuild after an unattributed red. */
|
||||
rebuildOnceWeekly: number;
|
||||
stallsWeekly: number;
|
||||
readyToMerge: LatencySummary;
|
||||
}
|
||||
|
||||
interface Waiting { pr: number; readyMs: number; enqueuedMs: number; screened: boolean; defect: boolean }
|
||||
|
||||
/**
|
||||
* The next train, or null to keep waiting: five or more waiting screened (or
|
||||
* unscreened) PRs dispatch a full train of their own; otherwise a full mixed
|
||||
* train, screened first; otherwise everything waiting, screened first, once
|
||||
* the oldest has waited the fill limit. The train with the oldest PR goes first.
|
||||
*/
|
||||
export function pickTrain<T extends { enqueuedMs: number; screened: boolean }>(pool: T[], nowMs: number, size: number, fillMs: number): T[] | null {
|
||||
const byAge = (a: T, b: T) => a.enqueuedMs - b.enqueuedMs;
|
||||
const screened = pool.filter(x => x.screened).sort(byAge), unscreened = pool.filter(x => !x.screened).sort(byAge);
|
||||
const options = [screened, unscreened].filter(list => list.length >= size).map(list => list.slice(0, size));
|
||||
if (!options.length) {
|
||||
const mixed = [...screened, ...unscreened];
|
||||
if (mixed.length >= size) options.push(mixed.slice(0, size));
|
||||
else if (mixed.length && nowMs - Math.min(...mixed.map(x => x.enqueuedMs)) >= fillMs) options.push(mixed);
|
||||
}
|
||||
const oldest = (train: T[]) => Math.min(...train.map(x => x.enqueuedMs));
|
||||
return options.sort((a, b) => oldest(a) - oldest(b))[0] ?? null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Replay READY arrivals through trains, one train gating at a time (each
|
||||
* builds on the previous train's merge). Each paid-relevant PR carries a
|
||||
* defect at the screened or unscreened rate; each attempt flakes at the
|
||||
* outside-red rate; every attempt takes gateMinutes and every red adds a
|
||||
* bisect of bisectMinutes. A defect drops its PR, which rejoins the queue at
|
||||
* once (repair time is not modeled), and the rest gate again; an unattributed
|
||||
* red gets one unchanged rebuild, a second is a stall and gates again. PRs
|
||||
* with no paid case ride with the screened ones.
|
||||
*/
|
||||
export function simulateArrivalTrains(input: ArrivalTrainInput): ArrivalTrainResult {
|
||||
const k = input.trainSize ?? 5, fillMs = (input.fillLimitHours ?? 3) * HOUR_MS;
|
||||
const gateMs = (input.gateMinutes ?? 45) * 60_000, bisectMs = (input.bisectMinutes ?? 45) * 60_000;
|
||||
const o = input.outsideRed ?? 0.15, rs = input.redScreened ?? 0.06, ru = input.redUnscreened ?? 0.25, reps = input.reps ?? 200;
|
||||
const rnd = new PyRandom(input.seed ?? 1);
|
||||
const t = { trains: 0, full: 0, gates: 0, bisects: 0, attributed: 0, rebuildOnce: 0, stalls: 0 };
|
||||
const occupancy = Array.from({ length: k }, () => 0);
|
||||
const hours: number[] = [];
|
||||
for (let rep = 0; rep < reps; rep += 1) {
|
||||
const screened = screenedIndexes(input.arrivals, input.policy, rnd);
|
||||
const arrivals: Waiting[] = input.arrivals.map((a, i) => ({
|
||||
pr: a.pr, readyMs: a.readyMs, enqueuedMs: a.readyMs, screened: !a.paidRelevant || screened.has(i),
|
||||
defect: a.paidRelevant && rnd.random() < (screened.has(i) ? rs : ru),
|
||||
})).sort((a, b) => a.readyMs - b.readyMs);
|
||||
const pool: Waiting[] = [];
|
||||
let next = 0, freeAt = -Infinity;
|
||||
while (next < arrivals.length || pool.length) {
|
||||
let now = pool.length ? freeAt : Math.max(freeAt, arrivals[next]!.readyMs);
|
||||
let train: Waiting[] | null;
|
||||
while (true) {
|
||||
while (next < arrivals.length && arrivals[next]!.readyMs <= now) pool.push(arrivals[next++]!);
|
||||
train = pickTrain(pool, now, k, fillMs);
|
||||
if (train) break;
|
||||
now = Math.min(next < arrivals.length ? arrivals[next]!.readyMs : Infinity, Math.min(...pool.map(x => x.enqueuedMs)) + fillMs);
|
||||
}
|
||||
for (const item of train) pool.splice(pool.indexOf(item), 1);
|
||||
t.trains += 1;
|
||||
occupancy[train.length - 1]! += 1;
|
||||
if (train.length === k) t.full += 1;
|
||||
let rebuilt = false;
|
||||
while (train.length) {
|
||||
t.gates += 1;
|
||||
now += gateMs;
|
||||
const flake = rnd.random() < o;
|
||||
const bad: Waiting[] = train.filter(item => item.defect);
|
||||
if (!flake && !bad.length) { for (const item of train) hours.push((now - item.readyMs) / HOUR_MS); break; }
|
||||
t.bisects += 1;
|
||||
now += bisectMs;
|
||||
if (bad.length) {
|
||||
const culprit: Waiting = bad[0]!;
|
||||
train = train.filter(item => item !== culprit);
|
||||
Object.assign(culprit, { defect: false, enqueuedMs: now });
|
||||
pool.push(culprit);
|
||||
rebuilt = false;
|
||||
if (train.length) t.attributed += 1;
|
||||
} else if (!rebuilt) {
|
||||
rebuilt = true;
|
||||
t.rebuildOnce += 1;
|
||||
} else {
|
||||
t.stalls += 1;
|
||||
rebuilt = false;
|
||||
}
|
||||
}
|
||||
freeAt = now;
|
||||
}
|
||||
}
|
||||
const perWeek = 7 / input.windowDays / reps;
|
||||
return {
|
||||
policy: policyLabel(input.policy), outsideRed: o, trainsWeekly: t.trains * perWeek,
|
||||
meanOccupancy: occupancy.reduce((sum, n, i) => sum + n * (i + 1), 0) / Math.max(1, t.trains),
|
||||
fullTrainShare: t.trains ? t.full / t.trains : 0, occupancy: occupancy.map(n => n / reps),
|
||||
gateAttemptsWeekly: t.gates * perWeek, bisectsWeekly: t.bisects * perWeek, attributedRebuildsWeekly: t.attributed * perWeek,
|
||||
rebuildOnceWeekly: t.rebuildOnce * perWeek, stallsWeekly: t.stalls * perWeek, readyToMerge: latencySummary(hours),
|
||||
};
|
||||
}
|
||||
|
||||
/** A run's PR: its pull_requests field, else the PR whose head branch it ran on while that PR was open (GSTA-47's mapping). */
|
||||
export function prForRun(run: { branch: string; createdAt: string; prNumbers: number[] },
|
||||
pulls: Array<{ number: number; ref: string; createdAt: string; mergedAt: string | null; closedAt: string | null }>): number | null {
|
||||
if (run.prNumbers.length) return run.prNumbers[0]!;
|
||||
const open = pulls.filter(p => p.ref === run.branch && p.createdAt <= run.createdAt
|
||||
&& run.createdAt <= new Date(Date.parse(p.mergedAt ?? p.closedAt ?? '9999-12-31') + HOUR_MS).toISOString());
|
||||
return open.sort((a, b) => b.createdAt.localeCompare(a.createdAt))[0]?.number ?? null;
|
||||
}
|
||||
+34
-14
@@ -306,6 +306,15 @@ export interface SelectionDeps {
|
||||
touchfiles(files: string[]): { e2e: string[]; global: boolean };
|
||||
}
|
||||
|
||||
export interface ScreenSelection {
|
||||
/** Gate cases the declared touchfiles and harness globals select (all of them when a global changed). */
|
||||
e2e: string[];
|
||||
global: boolean;
|
||||
canary: boolean;
|
||||
canaryReasons: string[];
|
||||
preambleOnly: string[];
|
||||
}
|
||||
|
||||
export interface SelectorCost { cases: number; judges: number; usd: number }
|
||||
export interface PrSelectionCost {
|
||||
pr: number;
|
||||
@@ -354,6 +363,9 @@ export function fullGateUsd(deps: Pick<SelectionDeps, 'gateCases' | 'judgeCases'
|
||||
return costOf(deps.gateCases, table) + costOf(deps.judgeCases, table);
|
||||
}
|
||||
|
||||
export const todaySelection = (diff: PrDiff, deps: SelectionDeps): TodaySelection =>
|
||||
(diff.files.length ? deps.today(diff.files, diff.packageVersionOnly) : { e2e: [], judges: [], shadowE2e: null, mode: 'none' });
|
||||
|
||||
/**
|
||||
* The screen-provisional rule: declared touchfiles and harness globals of the
|
||||
* files that are not heuristically preamble-only, every judge, and a canary
|
||||
@@ -361,12 +373,9 @@ export function fullGateUsd(deps: Pick<SelectionDeps, 'gateCases' | 'judgeCases'
|
||||
* resolver, or files that reach cases only transitively. A package.json whose
|
||||
* only change is the version stamp is neither selected nor a canary trigger.
|
||||
*/
|
||||
export function prSelectionCost(diff: PrDiff, deps: SelectionDeps, table: CaseCostTable, options: { canaryUsd?: number } = {}): PrSelectionCost {
|
||||
const full = fullGateUsd(deps, table);
|
||||
export function screenSelection(diff: PrDiff, deps: SelectionDeps, today: TodaySelection): ScreenSelection {
|
||||
const gate = new Set(deps.gateCases);
|
||||
const stampOnly = (file: string) => file === 'package.json' && diff.packageVersionOnly;
|
||||
const today = diff.files.length ? deps.today(diff.files, diff.packageVersionOnly) : { e2e: [], judges: [], shadowE2e: null, mode: 'none' };
|
||||
const shadowE2e = today.shadowE2e ?? today.e2e;
|
||||
const preambleOnly = heuristicPreambleOnly(Object.fromEntries(Object.entries(diff.skillMdChangedLines).filter(([file]) => diff.files.includes(file))));
|
||||
const screenFiles = diff.files.filter(file => !preambleOnly.includes(file) && !stampOnly(file));
|
||||
const touch = screenFiles.length ? deps.touchfiles(screenFiles) : { e2e: [], global: false };
|
||||
@@ -377,13 +386,19 @@ export function prSelectionCost(diff: PrDiff, deps: SelectionDeps, table: CaseCo
|
||||
...diff.files.filter(file => file.startsWith('scripts/resolvers/')).map(file => `resolver ${file}`),
|
||||
...(today.e2e.length > screenE2e.length ? ['transitive-only files (today selects more cases than declared touchfiles)'] : []),
|
||||
];
|
||||
const canary = canaryReasons.length > 0;
|
||||
const screenUsd = costOf(screenE2e, table) + (canary && !touch.global ? options.canaryUsd ?? 12 : 0) + costOf(deps.judgeCases, table);
|
||||
return { e2e: screenE2e, global: touch.global, canary: canaryReasons.length > 0, canaryReasons, preambleOnly };
|
||||
}
|
||||
|
||||
export function prSelectionCost(diff: PrDiff, deps: SelectionDeps, table: CaseCostTable, options: { canaryUsd?: number } = {}): PrSelectionCost {
|
||||
const today = todaySelection(diff, deps);
|
||||
const shadowE2e = today.shadowE2e ?? today.e2e;
|
||||
const { e2e, global, canary, canaryReasons, preambleOnly } = screenSelection(diff, deps, today);
|
||||
const screenUsd = costOf(e2e, table) + (canary && !global ? options.canaryUsd ?? 12 : 0) + costOf(deps.judgeCases, table);
|
||||
return {
|
||||
pr: diff.pr, sha: diff.sha, files: diff.files.length, mode: today.mode,
|
||||
today: { cases: today.e2e.length, judges: today.judges.length, usd: costOf(today.e2e, table) + costOf(today.judges, table) },
|
||||
shadow: { cases: shadowE2e.length, judges: today.judges.length, usd: costOf(shadowE2e, table) + costOf(today.judges, table) },
|
||||
screenProvisional: { cases: screenE2e.length, judges: deps.judgeCases.length, usd: Math.min(screenUsd, full), canary, canaryReasons, preambleOnly },
|
||||
screenProvisional: { cases: e2e.length, judges: deps.judgeCases.length, usd: Math.min(screenUsd, fullGateUsd(deps, table)), canary, canaryReasons, preambleOnly },
|
||||
};
|
||||
}
|
||||
|
||||
@@ -536,6 +551,17 @@ export interface TrainModelResult {
|
||||
attemptRedShare: number;
|
||||
}
|
||||
|
||||
/** Indexes of the PRs a policy screens: only paid-relevant PRs can screen; `random` draws from rnd. */
|
||||
export function screenedIndexes(prs: ReadonlyArray<{ paidRelevant: boolean; screenUsd: number }>, policy: ScreenPolicy, rnd: PyRandom): Set<number> {
|
||||
const relevant = prs.map((pr, index) => ({ ...pr, index })).filter(pr => pr.paidRelevant);
|
||||
const byCost = [...relevant].sort((a, b) => a.screenUsd - b.screenUsd);
|
||||
const half = relevant.length >> 1;
|
||||
const chosen = policy.kind === 'all' ? relevant : policy.kind === 'none' ? []
|
||||
: policy.kind === 'random' ? rnd.sample(relevant, Math.floor(relevant.length * policy.fraction))
|
||||
: policy.kind === 'cheapest-half' ? byCost.slice(0, half) : byCost.slice(half);
|
||||
return new Set(chosen.map(pr => pr.index));
|
||||
}
|
||||
|
||||
/**
|
||||
* Every merged PR lands in a train. Each paid-relevant PR carries a defect at
|
||||
* the screened or unscreened rate, and each gate attempt can also flake
|
||||
@@ -550,15 +576,9 @@ export function simulateTrains(input: TrainModelInput): TrainModelResult {
|
||||
const full = input.fullGateUsd ?? 93.14, bisect = input.bisectUsd ?? 6, rescreen = input.rescreenFactor ?? 1.3;
|
||||
const reps = input.reps ?? 400;
|
||||
const rnd = new PyRandom(input.seed ?? 1);
|
||||
const relevant = prs.map((pr, index) => ({ ...pr, index })).filter(pr => pr.paidRelevant);
|
||||
const byCost = [...relevant].sort((a, b) => a.screenUsd - b.screenUsd);
|
||||
const half = relevant.length >> 1;
|
||||
const t = { screen: 0, gates: 0, init: 0, initRed: 0, attRed: 0, bisects: 0, rebuildOnce: 0, stalls: 0 };
|
||||
for (let rep = 0; rep < reps; rep += 1) {
|
||||
const chosen = policy.kind === 'all' ? relevant : policy.kind === 'none' ? []
|
||||
: policy.kind === 'random' ? rnd.sample(relevant, Math.floor(relevant.length * policy.fraction))
|
||||
: policy.kind === 'cheapest-half' ? byCost.slice(0, half) : byCost.slice(half);
|
||||
const screened = new Set(chosen.map(pr => pr.index));
|
||||
const screened = screenedIndexes(prs, policy, rnd);
|
||||
const queue = prs.map((pr, index) => {
|
||||
const s = screened.has(index);
|
||||
if (s) t.screen += pr.screenUsd * rescreen;
|
||||
|
||||
+271
-12
@@ -10,7 +10,12 @@
|
||||
* bun run scripts/gate-cost-replay.ts --runs <id,id,...> [--since YYYY-MM-DD --until YYYY-MM-DD | --prs 3201,3202]
|
||||
* [--ref origin/main] [--repo owner/name] [--cache .context/gate-cost-cache] [--window-days N]
|
||||
* [--usd-per-second R]
|
||||
* [--train-size 5] [--reps 400] [--seed 1] [--cases] [--json]
|
||||
* [--train-size 5] [--reps 400] [--seed 1] [--cases] [--history] [--json]
|
||||
*
|
||||
* --history (needs --since/--until) adds the red-head replay and the
|
||||
* READY-arrival train replay (lib/gate-cost-history.ts): it lists the
|
||||
* window's E2E Evals and Free Tests runs and PRs, downloads report-verdict
|
||||
* and trial-outcomes artifacts of red runs, and fetches PR heads by SHA.
|
||||
*
|
||||
* --runs names complete full-gate runs (108 rule cases + 24 judges). Bare
|
||||
* dates are UTC days; --until is inclusive. Cases without a recorded cost are
|
||||
@@ -23,14 +28,19 @@ import * as path from 'node:path';
|
||||
import { spawnSync } from 'node:child_process';
|
||||
import {
|
||||
SCREEN_LABEL, TODAY_WEEKLY_USD, AFTER_GSTA48_WEEKLY_USD, caseCostTable, changedLinesByFile, fullGateUsd, isSkillMd,
|
||||
parseEvalReport, parseTrialOutcomes, prSelectionCost, simulateTrains, summarizeGates, summarizeSelector,
|
||||
parseEvalReport, parseTrialOutcomes, prSelectionCost, screenSelection, simulateTrains, summarizeGates, summarizeSelector, todaySelection,
|
||||
type CaseCostEntry, type GateRecord, type GateSummary, type PrDiff, type PrSelectionCost, type ScreenPolicy,
|
||||
type SelectionDeps, type SelectorSummary, type TrainModelResult,
|
||||
} from '../lib/gate-cost';
|
||||
import { GH_HISTORY, downloadRunArtifacts, repoSlug } from './lib/ci-history';
|
||||
import {
|
||||
STAMP_FILES, classifyReds, headGreenAt, latencySummary, parseReportVerdict, prForRun, readyTimes, screenHit, simulateArrivalTrains,
|
||||
stampFreeSignature, summarizeRedHeads, SCREEN_CANARY_CASES,
|
||||
type ArrivalTrainResult, type CheckRun, type GatedRun, type HeadDiff, type LatencySummary, type ReadyRow, type RedHeadRow, type RedHeadSummary, type RedRun,
|
||||
} from '../lib/gate-cost-history';
|
||||
import { GH_CLIENT, GH_HISTORY, downloadRunArtifacts, repoSlug } from './lib/ci-history';
|
||||
import { computePaidCaseSelection } from './lib/paid-select';
|
||||
import { packageChangeOnlyVersion } from './test-pr-profile';
|
||||
import { E2E_TIERS, GLOBAL_TOUCHFILES, LLM_JUDGE_TOUCHFILES, matchGlob } from '../test/helpers/touchfiles';
|
||||
import { E2E_TIERS, E2E_TOUCHFILES, GLOBAL_TOUCHFILES, LLM_JUDGE_TOUCHFILES, matchGlob } from '../test/helpers/touchfiles';
|
||||
|
||||
const ROOT = path.resolve(import.meta.dir, '..');
|
||||
const DAY_MS = 86_400_000;
|
||||
@@ -55,12 +65,13 @@ export interface ReplayArgs {
|
||||
reps: number;
|
||||
seed: number;
|
||||
cases: boolean;
|
||||
history: boolean;
|
||||
json: boolean;
|
||||
}
|
||||
|
||||
export function parseArgs(argv: string[]): ReplayArgs {
|
||||
const args: ReplayArgs = { runs: [], prs: [], ref: 'origin/main', repo: '', cache: path.join(ROOT, '.context', 'gate-cost-cache'),
|
||||
trainSize: 5, reps: 400, seed: 1, cases: false, json: false };
|
||||
trainSize: 5, reps: 400, seed: 1, cases: false, history: false, json: false };
|
||||
const list = (value: string) => value.split(/[\s,]+/).filter(Boolean);
|
||||
const positive = (flag: string, value: string) => {
|
||||
const n = Number(value);
|
||||
@@ -71,6 +82,7 @@ export function parseArgs(argv: string[]): ReplayArgs {
|
||||
const flag = argv[i]!;
|
||||
if (flag === '--json') { args.json = true; continue; }
|
||||
if (flag === '--cases') { args.cases = true; continue; }
|
||||
if (flag === '--history') { args.history = true; continue; }
|
||||
const value = argv[i + 1];
|
||||
if (value === undefined || value.startsWith('--')) throw new Error(`${flag} needs a value`);
|
||||
i += 1;
|
||||
@@ -93,6 +105,7 @@ export function parseArgs(argv: string[]): ReplayArgs {
|
||||
if (args.prs.some(pr => !Number.isInteger(pr) || pr <= 0)) throw new Error('--prs takes PR numbers');
|
||||
if (args.prs.length && (args.since || args.until)) throw new Error('use either --prs or --since/--until, not both');
|
||||
if (!!args.since !== !!args.until) throw new Error('--since and --until go together');
|
||||
if (args.history && !args.since) throw new Error('--history needs a window: --since and --until');
|
||||
return args;
|
||||
}
|
||||
|
||||
@@ -184,20 +197,23 @@ function mergedPrs(args: ReplayArgs): { prs: MergedPr[]; windowDays: number | nu
|
||||
return { prs: args.prs.map(pr => all.get(pr)!), windowDays: args.windowDays ?? null };
|
||||
}
|
||||
|
||||
function prDiff(merge: MergedPr): PrDiff {
|
||||
const files = git(['diff', '--name-only', '--no-renames', `${merge.sha}^`, merge.sha]).split('\n').filter(Boolean);
|
||||
/** The PR diff the selectors see between two revisions. */
|
||||
function revisionDiff(pr: number, from: string, to: string): PrDiff {
|
||||
const files = git(['diff', '--name-only', '--no-renames', from, to]).split('\n').filter(Boolean);
|
||||
let packageVersionOnly = false;
|
||||
if (files.includes('package.json')) {
|
||||
const show = (rev: string) => spawnSync('git', ['show', `${rev}:package.json`], { cwd: ROOT, encoding: 'utf8', timeout: 30_000 });
|
||||
const before = show(`${merge.sha}^`), after = show(merge.sha);
|
||||
const before = show(from), after = show(to);
|
||||
packageVersionOnly = before.status === 0 && after.status === 0 && packageChangeOnlyVersion(before.stdout, after.stdout);
|
||||
}
|
||||
const skillMds = files.filter(isSkillMd);
|
||||
const skillMdChangedLines = skillMds.length >= 10
|
||||
? changedLinesByFile(git(['diff', '-U0', '--no-renames', `${merge.sha}^`, merge.sha, '--', ...skillMds], 120_000)) : {};
|
||||
return { pr: merge.pr, sha: merge.sha.slice(0, 8), files, packageVersionOnly, skillMdChangedLines };
|
||||
? changedLinesByFile(git(['diff', '-U0', '--no-renames', from, to, '--', ...skillMds], 120_000)) : {};
|
||||
return { pr, sha: to.slice(0, 8), files, packageVersionOnly, skillMdChangedLines };
|
||||
}
|
||||
|
||||
const prDiff = (merge: MergedPr): PrDiff => revisionDiff(merge.pr, `${merge.sha}^`, merge.sha);
|
||||
|
||||
/** The selectors in this checkout: today's PR profile (with R4's shadow) and the plain touchfile selection. */
|
||||
export function checkoutSelectionDeps(rootDir = ROOT): SelectionDeps {
|
||||
const gateCases = Object.keys(E2E_TIERS).filter(id => E2E_TIERS[id] === 'gate');
|
||||
@@ -236,6 +252,7 @@ export interface ReplayReport {
|
||||
prs: PrSelectionCost[];
|
||||
selection: { today: SelectorSummary; shadow: SelectorSummary; screenProvisional: SelectorSummary } | null;
|
||||
model: { screenScale: number; fullGateUsd: number; trainSize: number; reps: number; seed: number; rows: TrainModelResult[] } | null;
|
||||
history?: HistoryReport;
|
||||
}
|
||||
|
||||
export function buildReport(args: ReplayArgs, gates: GateRecord[], merges: ReturnType<typeof mergedPrs>, deps: SelectionDeps,
|
||||
@@ -333,6 +350,244 @@ export function formatReport(report: ReplayReport, options: { cases?: boolean }
|
||||
return out.join('\n');
|
||||
}
|
||||
|
||||
// --- History (--history): red-head replay and READY-arrival trains ---
|
||||
|
||||
const REPORT_ARTIFACT = /^report-verdict-a\d+$/;
|
||||
const TRIALS_ARTIFACT = /^trial-outcomes[A-Za-z0-9_-]*?-a\d+$/;
|
||||
const TRAIN_POLICIES: ScreenPolicy[] = [{ kind: 'all' }, { kind: 'random', fraction: 0.5 }, { kind: 'none' }];
|
||||
const GH_RUN_PAGE = 100;
|
||||
const HISTORY_GIT_TIMEOUT_MS = 120_000;
|
||||
const FETCH_TIMEOUT_MS = 300_000;
|
||||
|
||||
interface HistoryRun { id: number; sha: string; branch: string; createdAt: string; updatedAt: string; conclusion: string | null; prNumbers: number[] }
|
||||
interface PullRow { number: number; ref: string; head: string; createdAt: string; mergedAt: string | null; closedAt: string | null }
|
||||
type Merges = ReturnType<typeof mergedPrs>;
|
||||
|
||||
export interface HistoryReport {
|
||||
window: { since: string; until: string };
|
||||
runs: { evals: number; gated: number; red: number; free: number; unmapped: number };
|
||||
redRuns: RedRun[];
|
||||
redHeads: RedHeadRow[];
|
||||
redSummary: RedHeadSummary;
|
||||
perPr: Array<{ pr: number; paidRelevant: boolean; prCausedRed: boolean; screenedMiss: boolean }>;
|
||||
ready: ReadyRow[];
|
||||
baseline: LatencySummary & { gatesAfterReady: number; sources: Record<string, number> };
|
||||
trainParams: { trainSize: number; fillLimitHours: number; gateMinutes: number; bisectMinutes: number; redScreened: number; redUnscreened: number; reps: number; seed: number };
|
||||
trains: ArrivalTrainResult[];
|
||||
}
|
||||
|
||||
function tryGit(args: string[], timeout = HISTORY_GIT_TIMEOUT_MS): string | null {
|
||||
const result = spawnSync('git', args, { cwd: ROOT, encoding: 'utf8', timeout, maxBuffer: 256 * 1024 * 1024 });
|
||||
return result.status === 0 ? result.stdout : null;
|
||||
}
|
||||
|
||||
function cachedJson<T>(file: string, load: () => T): T {
|
||||
if (fs.existsSync(file)) return JSON.parse(fs.readFileSync(file, 'utf8')) as T;
|
||||
const value = load();
|
||||
fs.writeFileSync(file, JSON.stringify(value));
|
||||
return value;
|
||||
}
|
||||
|
||||
/** Completed runs of one workflow, one UTC day per query: GitHub caps a filtered run listing at 1,000 results. */
|
||||
function listRunsByDay(repo: string, workflow: string, since: Date, until: Date, event?: string): HistoryRun[] {
|
||||
const out: HistoryRun[] = [];
|
||||
for (let day = new Date(since.toISOString().slice(0, 10)); day < until; day = new Date(day.getTime() + DAY_MS)) {
|
||||
const date = day.toISOString().slice(0, 10);
|
||||
for (let page = 1; page <= 10; page += 1) {
|
||||
const query = [`status=completed`, `per_page=${GH_RUN_PAGE}`, `page=${page}`, `created=${date}..${date}`, ...(event ? [`event=${event}`] : [])];
|
||||
const body = GH_CLIENT.getJson(`repos/${repo}/actions/workflows/${workflow}/runs?${query.join('&')}`) as { workflow_runs?: any[] };
|
||||
const runs = Array.isArray(body?.workflow_runs) ? body.workflow_runs : [];
|
||||
out.push(...runs.map(run => ({ id: Number(run.id), sha: String(run.head_sha ?? ''), branch: String(run.head_branch ?? ''), createdAt: String(run.created_at ?? ''),
|
||||
updatedAt: String(run.updated_at ?? ''), conclusion: run.conclusion ?? null,
|
||||
prNumbers: Array.isArray(run.pull_requests) ? run.pull_requests.map((pr: any) => Number(pr.number)).filter(Number.isFinite) : [] })));
|
||||
if (runs.length < GH_RUN_PAGE) break;
|
||||
}
|
||||
}
|
||||
return out.filter(run => run.createdAt >= since.toISOString() && run.createdAt < until.toISOString());
|
||||
}
|
||||
|
||||
function listPulls(repo: string, since: Date): PullRow[] {
|
||||
const out: PullRow[] = [];
|
||||
for (let page = 1; page <= 30; page += 1) {
|
||||
const body = GH_CLIENT.getJson(`repos/${repo}/pulls?state=all&sort=updated&direction=desc&per_page=100&page=${page}`) as any[];
|
||||
if (!Array.isArray(body)) break;
|
||||
out.push(...body.map(p => ({ number: Number(p.number), ref: String(p.head?.ref ?? ''), head: String(p.head?.sha ?? ''), createdAt: String(p.created_at ?? ''),
|
||||
mergedAt: p.merged_at ?? null, closedAt: p.closed_at ?? null })));
|
||||
if (body.length < 100 || String(body[body.length - 1]?.updated_at ?? '') < since.toISOString()) break;
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
/** Artifacts of one run matching a name pattern, downloaded once (a .history marker records that the run was listed). */
|
||||
function runArtifactDirs(runId: number, repo: string, cache: string, match: RegExp): string[] {
|
||||
const dir = path.join(cache, String(runId));
|
||||
const marker = path.join(dir, `.history-${match.source.replace(/[^a-z]/gi, '')}`);
|
||||
if (!fs.existsSync(marker)) {
|
||||
downloadRunArtifacts({ repo, run: { id: runId }, cacheDir: cache, match: name => match.test(name), maxBytes: 16 * 1024 * 1024, fetcher: GH_HISTORY });
|
||||
fs.mkdirSync(dir, { recursive: true });
|
||||
fs.writeFileSync(marker, '');
|
||||
}
|
||||
return fs.readdirSync(dir).filter(name => match.test(name)).sort((a, b) => Number(a.split('-a').pop()) - Number(b.split('-a').pop())).map(name => path.join(dir, name));
|
||||
}
|
||||
|
||||
/** The run's report verdict over its attempts: the last attempt's verdict, every attempt's failures; null when no report exists. */
|
||||
function runVerdict(runId: number, repo: string, cache: string): { green: boolean; failures: GatedRun['failures'] } | null {
|
||||
const reports = runArtifactDirs(runId, repo, cache, REPORT_ARTIFACT).map(dir => path.join(dir, 'report.txt')).filter(file => fs.existsSync(file))
|
||||
.map(file => parseReportVerdict(fs.readFileSync(file, 'utf8')));
|
||||
if (!reports.length) return null;
|
||||
return { green: reports[reports.length - 1]!.verdict === 'GREEN', failures: reports.flatMap(r => r.failures) };
|
||||
}
|
||||
|
||||
/** The tested merge commit of a run, from its trial outcomes. */
|
||||
function testedMergeSha(runId: number, repo: string, cache: string): string | null {
|
||||
for (const dir of runArtifactDirs(runId, repo, cache, TRIALS_ARTIFACT)) {
|
||||
const file = path.join(dir, 'trial-outcomes.jsonl');
|
||||
const first = fs.existsSync(file) ? fs.readFileSync(file, 'utf8').split('\n').find(Boolean) : undefined;
|
||||
try { const sha = first ? JSON.parse(first).sha : null; if (typeof sha === 'string' && /^[0-9a-f]{40}$/.test(sha)) return sha; } catch { /* torn line */ }
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
/** Fetch commits this clone lacks by SHA (GitHub serves PR heads and merge refs that no branch holds anymore). */
|
||||
function ensureCommits(shas: string[]): void {
|
||||
const missing = [...new Set(shas)].filter(sha => tryGit(['cat-file', '-e', `${sha}^{commit}`], 30_000) === null);
|
||||
for (let i = 0; i < missing.length; i += 20) spawnSync('git', ['fetch', '-q', 'origin', ...missing.slice(i, i + 20)], { cwd: ROOT, timeout: FETCH_TIMEOUT_MS });
|
||||
}
|
||||
|
||||
function headDiff(sha: string, ref: string): HeadDiff | null {
|
||||
const mergeBase = tryGit(['merge-base', ref, sha])?.trim();
|
||||
if (!mergeBase) return null;
|
||||
const files = tryGit(['diff', '--name-only', '--no-renames', mergeBase, sha]);
|
||||
const diff = tryGit(['diff', '-U0', '--no-color', '--no-renames', '--no-ext-diff', mergeBase, sha, '--', '.', ...STAMP_FILES.map(file => `:(exclude)${file}`)]);
|
||||
return files === null || diff === null ? null : { ownSig: stampFreeSignature(diff), files: files.split('\n').filter(Boolean) };
|
||||
}
|
||||
|
||||
/** The base a red run tested: its merge commit's first parent, else main's first-parent tip when the run was created. */
|
||||
function testedBase(run: RedRun, args: ReplayArgs, repo: string): { base: string; source: RedHeadRow['baseSource'] } | null {
|
||||
const merge = testedMergeSha(run.runId, repo, args.cache);
|
||||
if (merge) {
|
||||
ensureCommits([merge]);
|
||||
const parents = (tryGit(['rev-list', '--parents', '-n1', merge]) ?? '').trim().split(' ').slice(1);
|
||||
if (parents.length === 2 && parents[1] === run.head) return { base: parents[0]!, source: 'tested-merge' };
|
||||
}
|
||||
const tip = tryGit(['rev-list', '-1', '--first-parent', `--before=${run.createdAt}`, args.ref])?.trim();
|
||||
return tip ? { base: tip, source: 'main-at-run' } : null;
|
||||
}
|
||||
|
||||
function collectHistory(args: ReplayArgs, repo: string, merges: Merges, body: Omit<ReplayReport, 'repo' | 'ref' | 'head'>, deps: SelectionDeps, gates: GateRecord[]): HistoryReport {
|
||||
const since = merges.since!, until = merges.until!;
|
||||
const key = `${since.toISOString()}_${until.toISOString()}`.replace(/[^0-9TZ_]/g, '');
|
||||
const cached = <T>(name: string, load: () => T) => cachedJson(path.join(args.cache, `history-${key}-${name}.json`), load);
|
||||
const evals = cached('evals', () => listRunsByDay(repo, 'evals.yml', since, until));
|
||||
const free = cached('free', () => listRunsByDay(repo, 'free-tests.yml', since, until, 'pull_request'));
|
||||
const pulls = cached('pulls', () => listPulls(repo, since));
|
||||
const prOf = new Map([...evals, ...free].map(run => [run.id, prForRun(run, pulls)]));
|
||||
const gated: GatedRun[] = evals.flatMap(run => {
|
||||
const pr = prOf.get(run.id) ?? null;
|
||||
if (run.conclusion === 'success') return [{ id: run.id, pr, head: run.sha, createdAt: run.createdAt, green: true, failures: [] }];
|
||||
const verdict = run.conclusion === 'failure' ? runVerdict(run.id, repo, args.cache) : null;
|
||||
return verdict ? [{ id: run.id, pr, head: run.sha, createdAt: run.createdAt, ...verdict }] : [];
|
||||
});
|
||||
const mergedSet = new Set(merges.prs.map(m => m.pr));
|
||||
const pullOf = new Map(pulls.map(p => [p.number, p]));
|
||||
const checkRuns = [...evals.map(run => ({ run, workflow: 'evals' as const })), ...free.map(run => ({ run, workflow: 'free' as const }))]
|
||||
.filter(({ run }) => mergedSet.has(prOf.get(run.id) ?? -1));
|
||||
const heads = [...new Set([...gated.filter(r => r.pr !== null).map(r => r.head), ...checkRuns.map(({ run }) => run.sha),
|
||||
...merges.prs.map(m => pullOf.get(m.pr)?.head ?? '')].filter(Boolean))];
|
||||
ensureCommits(heads);
|
||||
const diffs = cached('heads', () => Object.fromEntries(heads.map(sha => [sha, headDiff(sha, args.ref)])));
|
||||
const headMap = new Map(Object.entries(diffs).filter((e): e is [string, HeadDiff] => e[1] !== null));
|
||||
const caseFiles = new Map(gates.flatMap(g => g.trials.map(t => [t.caseId, t.file] as const)));
|
||||
const caseFile = (id: string) => caseFiles.get(id) ?? null;
|
||||
const inputsOf = (id: string) => /^test\/.+\.test\.ts$/.test(id)
|
||||
? [id, ...[...caseFiles].filter(([, file]) => file === id).flatMap(([c]) => E2E_TOUCHFILES[c] ?? [])]
|
||||
: [...(E2E_TOUCHFILES[id] ?? []), ...(caseFile(id) ? [caseFile(id)!] : [])];
|
||||
const redRuns = classifyReds(gated, {
|
||||
heads: headMap,
|
||||
delta: (from, to) => tryGit(['diff', '--name-only', '--no-renames', from, to])?.split('\n').filter(Boolean) ?? null,
|
||||
touches: (id, files) => files.filter(file => [...inputsOf(id), ...GLOBAL_TOUCHFILES].some(pattern => matchGlob(file, pattern))),
|
||||
});
|
||||
const redHeads: RedHeadRow[] = redRuns.filter(run => run.cause === 'pr').flatMap(run => {
|
||||
const tested = testedBase(run, args, repo);
|
||||
const mergeBase = tested && tryGit(['merge-base', tested.base, run.head])?.trim();
|
||||
if (!tested || !mergeBase) return [];
|
||||
const diff = revisionDiff(run.pr, mergeBase, run.head);
|
||||
const selection = screenSelection(diff, deps, todaySelection(diff, deps));
|
||||
return run.cases.filter(c => c.cause === 'pr').map(c => ({ caseId: c.caseId, kind: c.kind, pr: run.pr, runId: run.runId, head: run.head.slice(0, 8),
|
||||
base: tested.base.slice(0, 8), baseSource: tested.source, files: diff.files.length, hit: screenHit(c, selection, caseFile), canaryReasons: selection.canaryReasons }));
|
||||
});
|
||||
const selected = new Map(body.prs.map(row => [row.pr, row]));
|
||||
const merged = merges.prs.map(m => ({ pr: m.pr, paidRelevant: (selected.get(m.pr)?.today.usd ?? 0) > 0 }));
|
||||
const redSummary = summarizeRedHeads(redHeads, redRuns, merged);
|
||||
const ready = readyTimes(merges.prs.flatMap(m => {
|
||||
const pull = pullOf.get(m.pr);
|
||||
if (!pull?.mergedAt) return [];
|
||||
const prHeads = [...new Set([...checkRuns.filter(({ run }) => prOf.get(run.id) === m.pr).map(({ run }) => run.sha), pull.head])];
|
||||
return [{ pr: m.pr, mergedAt: pull.mergedAt, finalHead: pull.head, heads: prHeads.map(sha => ({ sha, ownSig: headMap.get(sha)?.ownSig ?? null })),
|
||||
gatedRuns: gated.filter(r => r.pr === m.pr).map(r => r.createdAt) }];
|
||||
}), headGreenAt(checkRuns.map(({ run, workflow }): CheckRun => ({ head: run.sha, workflow, createdAt: run.createdAt, completedAt: run.updatedAt, conclusion: run.conclusion }))));
|
||||
const withReady = ready.filter(row => row.ready !== null);
|
||||
const sources: Record<string, number> = {};
|
||||
for (const row of ready) sources[row.source] = (sources[row.source] ?? 0) + 1;
|
||||
const scale = body.model?.screenScale ?? 1;
|
||||
const arrivals = withReady.map(row => ({ pr: row.pr, readyMs: Date.parse(row.ready!), paidRelevant: (selected.get(row.pr)?.today.usd ?? 0) > 0,
|
||||
screenUsd: (selected.get(row.pr)?.screenProvisional.usd ?? 0) * scale }));
|
||||
const trainParams = { trainSize: args.trainSize, fillLimitHours: 3, gateMinutes: 45, bisectMinutes: 45, redScreened: 0.06, redUnscreened: 0.25, reps: args.reps, seed: args.seed };
|
||||
return {
|
||||
window: { since: since.toISOString(), until: until.toISOString() },
|
||||
runs: { evals: evals.length, gated: gated.length, red: gated.filter(r => !r.green).length, free: free.length, unmapped: evals.filter(run => prOf.get(run.id) === null).length },
|
||||
redRuns, redHeads, redSummary,
|
||||
perPr: merged.map(m => ({ ...m, prCausedRed: redSummary.prsWithPrRed.includes(m.pr), screenedMiss: redSummary.missedPrs.includes(m.pr) })),
|
||||
ready, baseline: { ...latencySummary(withReady.map(row => row.readyToMergeHours!)), gatesAfterReady: withReady.reduce((sum, row) => sum + row.gatesAfterReady, 0), sources },
|
||||
trainParams,
|
||||
trains: TRAIN_POLICIES.flatMap(policy => MODEL_OUTSIDE_RED.map(outsideRed => simulateArrivalTrains({ ...trainParams, arrivals, policy, outsideRed, windowDays: merges.windowDays! }))),
|
||||
};
|
||||
}
|
||||
|
||||
export function formatHistory(h: HistoryReport): string {
|
||||
const s = h.redSummary;
|
||||
const label = SCREEN_LABEL;
|
||||
const share = (n: number, d: number) => `${n} of ${d} (${pct(n, d)})`;
|
||||
const causes = (cause: string) => h.redRuns.filter(run => run.cause === cause).length;
|
||||
const out = [`History replays, E2E Evals and Free Tests runs created ${h.window.since} to ${h.window.until}:`,
|
||||
` E2E Evals runs ${h.runs.evals} (${h.runs.unmapped} not mapped to a PR), gated ${h.runs.gated}, red ${h.runs.red}: PR-caused ${causes('pr')}, outside ${causes('outside')}, unknown ${causes('unknown')}; Free Tests PR runs ${h.runs.free}`,
|
||||
' Classification (GSTA-47): per failed case, first match wins: transcript diagnosis; red in another PR within 72 h -> outside;',
|
||||
' no later gated run -> unknown; next gated run has the same own diff -> outside; the push to it changed the case\'s inputs',
|
||||
' (touchfiles, test file, harness globals) among either head\'s own files -> PR-caused; else outside. A run with any PR-caused case is PR-caused.',
|
||||
'', `Red-head replay (${label} on the failing head's diff from its tested base; canary set fixed before the replay: ${Object.values(SCREEN_CANARY_CASES).join(', ')}):`,
|
||||
` ${pad(['PR', 'run', 'head', 'base', 'files', 'case', 'kind', `${label} hit`], [6, 12, 9, 22, 5, 46, 6])}`];
|
||||
for (const row of h.redHeads) {
|
||||
out.push(` ${pad([`#${row.pr}`, row.runId, row.head, `${row.base} ${row.baseSource}`, row.files, row.caseId, row.kind, row.hit ?? 'MISSED'], [6, 12, 9, 22, 5, 46, 6])}`);
|
||||
}
|
||||
out.push(` ${label} hits by kind: ${Object.entries(s.byHit).map(([k, n]) => `${k} ${n}`).join(', ') || 'none'}`,
|
||||
` ${label} misses: ${share(s.missedPrs.length, s.prsWithPrRed.length)} merged PRs with a PR-caused red (${s.missedCases} of ${s.prCausedCases} case results)${s.missedPrs.length ? `: #${s.missedPrs.join(', #')}` : ''}`,
|
||||
` ${label} misses with an empty canary set: ${s.missedPrsWithoutCanary.length}${s.missedPrsWithoutCanary.length ? ` (#${s.missedPrsWithoutCanary.join(', #')})` : ''}`,
|
||||
` P-i (${label} misses over paid-relevant merged PRs): ${share(s.missedPrs.length, s.paidRelevantPrs)}`,
|
||||
` P-k (merged PRs with any PR-caused red run): ${share(s.prsWithPrRed.length, s.mergedPrs)}${s.prsWithPrRed.length ? `: #${s.prsWithPrRed.join(', #')}` : ''}; ${s.prCausedRuns} PR-caused red runs`,
|
||||
` PR-caused red runs on PRs that did not merge in the window (not counted): ${s.otherPrCausedRuns}`,
|
||||
'', `Per merged PR (paid-relevant: today's selector picks a case on the squash; red: any PR-caused red run; miss: a ${label} miss):`);
|
||||
for (let i = 0; i < h.perPr.length; i += 6) {
|
||||
out.push(` ${h.perPr.slice(i, i + 6).map(p => `#${p.pr} ${p.paidRelevant ? 'paid' : '----'} ${p.prCausedRed ? 'red ' : '--- '}${p.screenedMiss ? 'miss' : '----'}`).join(' ')}`);
|
||||
}
|
||||
const b = h.baseline;
|
||||
const hrs = (v: number) => v.toFixed(1);
|
||||
const p = h.trainParams;
|
||||
out.push('', 'READY-to-merge. READY = when the merged own diff (modulo main merges and stamps) first had green Free Tests and E2E Evals on a',
|
||||
' head, before the merge (fallback: the final head\'s green time; READY reports are not machine-readable).',
|
||||
` READY signal: ${Object.entries(b.sources).map(([k, n]) => `${k} ${n}`).join(', ')}`,
|
||||
` today (one PR per gate): ${b.prs} PRs, READY-to-merge p50 ${hrs(b.p50Hours)} h, p90 ${hrs(b.p90Hours)} h, mean ${hrs(b.meanHours)} h; ${b.gatesAfterReady} gated E2E Evals runs after READY`,
|
||||
` trains (${p.reps} reps, seed ${p.seed}; size ${p.trainSize}, fill limit ${p.fillLimitHours} h, screened first, unscreened alone when ${p.trainSize}+ wait; ${p.gateMinutes} min a gate,`,
|
||||
` ${p.bisectMinutes} min a bisect, one train gating at a time; defects ${p.redScreened * 100}% screened / ${p.redUnscreened * 100}% unscreened; a dropped PR rejoins at once):`,
|
||||
` ${pad(['policy', 'o', 'trains/w', 'occupancy', 'full', 'sizes 1..k per window', 'gates/w', 'drop+gate/w', 'rebld/w', 'stalls/w', 'p50 h', 'p90 h'], [10, 5, 8, 9, 5, 22, 7, 11, 7, 8, 6])} mean h`);
|
||||
for (const r of h.trains) {
|
||||
out.push(` ${pad([r.policy, r.outsideRed.toFixed(2), r.trainsWeekly.toFixed(2), r.meanOccupancy.toFixed(2), pct(r.fullTrainShare, 1), r.occupancy.map(n => n.toFixed(1)).join('/'),
|
||||
r.gateAttemptsWeekly.toFixed(2), r.attributedRebuildsWeekly.toFixed(2), r.rebuildOnceWeekly.toFixed(2), r.stallsWeekly.toFixed(2), hrs(r.readyToMerge.p50Hours), hrs(r.readyToMerge.p90Hours)],
|
||||
[10, 5, 8, 9, 5, 22, 7, 11, 7, 8, 6])} ${hrs(r.readyToMerge.meanHours)}`);
|
||||
}
|
||||
out.push(` "screened" follows the ${label} policy column; PRs with no paid case ride as screened.`);
|
||||
return out.join('\n');
|
||||
}
|
||||
|
||||
export function main(argv: string[]): number {
|
||||
let args: ReplayArgs;
|
||||
try { args = parseArgs(argv); } catch (error) {
|
||||
@@ -346,8 +601,12 @@ export function main(argv: string[]): number {
|
||||
const empty = gates.filter(g => g.trials.length === 0);
|
||||
if (empty.length) throw new Error(`run(s) ${empty.map(g => g.runId).join(', ')} have no trial outcomes (expired artifacts, or not a paid gate run)`);
|
||||
const merges = mergedPrs(args);
|
||||
const report: ReplayReport = { repo, ref: args.ref, head: git(['rev-parse', args.ref]).trim(), ...buildReport(args, gates, merges, checkoutSelectionDeps()) };
|
||||
console.log(args.json ? JSON.stringify(report, null, 2) : formatReport(report, { cases: args.cases }));
|
||||
const deps = checkoutSelectionDeps();
|
||||
const body = buildReport(args, gates, merges, deps);
|
||||
const report: ReplayReport = { repo, ref: args.ref, head: git(['rev-parse', args.ref]).trim(), ...body,
|
||||
...(args.history ? { history: collectHistory(args, repo, merges, body, deps, gates) } : {}) };
|
||||
const text = formatReport(report, { cases: args.cases }) + (report.history ? `\n\n${formatHistory(report.history)}` : '');
|
||||
console.log(args.json ? JSON.stringify(report, null, 2) : text);
|
||||
return 0;
|
||||
} catch (error) {
|
||||
console.error(`gate-cost-replay: ${(error as Error).message}`);
|
||||
|
||||
+2
-1
@@ -123,7 +123,8 @@
|
||||
{ "file": "lib/autoplan-confirm.ts", "movedFrom": "(new: confirm-round plan delta, revision-qualified correlation and settled findings, field list 2 W5)" },
|
||||
{ "file": "lib/autoplan-public.ts", "movedFrom": "(new: machine-path-free public run copy and validate --public, field list 2 W6)" },
|
||||
{ "file": "lib/gate-cost.ts", "movedFrom": "(new: paid gate cost by model and token class, per-case costs, selection replays and the merge-train model, GSTA-49 T0)" },
|
||||
{ "file": "scripts/gate-cost-replay.ts", "movedFrom": "(new: CLI behind lib/gate-cost.ts: cached report artifacts, merged-PR replay and the train model table, GSTA-49 T0)" }
|
||||
{ "file": "scripts/gate-cost-replay.ts", "movedFrom": "(new: CLI behind lib/gate-cost.ts: cached report artifacts, merged-PR replay and the train model table, GSTA-49 T0)" },
|
||||
{ "file": "lib/gate-cost-history.ts", "movedFrom": "(new: red attribution, red-head screen replay, READY times and the READY-arrival train replay, GSTA-49 T0b)" }
|
||||
],
|
||||
"residualFiles": [
|
||||
{ "file": "scripts/test-free-shards.ts", "maxLines": 2361, "reason": "free lane policy after the shard engine took spawn, sandbox, logs, seeds and flags" },
|
||||
|
||||
+183
-1
@@ -6,7 +6,13 @@ import {
|
||||
PyRandom, SCREEN_LABEL, caseCostTable, changedLinesByFile, gateCostBreakdown, heuristicPreambleOnly, parseEvalReport,
|
||||
prSelectionCost, simulateTrains, summarizeGates, summarizeSelector, type GateRecord, type PrDiff, type SelectionDeps,
|
||||
} from '../lib/gate-cost';
|
||||
import { buildReport, checkoutSelectionDeps, formatReport, parseArgs, parseMergeLog, parseWindowDate, readGateRecord, type ReplayReport } from '../scripts/gate-cost-replay';
|
||||
import {
|
||||
DIAGNOSED_REDS, SCREEN_CANARY_CASES, classifyReds, headGreenAt, parseReportVerdict, percentile, pickTrain, prForRun, readyTimes, screenHit,
|
||||
simulateArrivalTrains, stampFreeSignature, summarizeRedHeads, type GatedRun, type HeadDiff, type RedHeadRow,
|
||||
} from '../lib/gate-cost-history';
|
||||
import {
|
||||
buildReport, checkoutSelectionDeps, formatHistory, formatReport, parseArgs, parseMergeLog, parseWindowDate, readGateRecord, type HistoryReport, type ReplayReport,
|
||||
} from '../scripts/gate-cost-replay';
|
||||
|
||||
/**
|
||||
* A one-run artifact tree, written to a temporary directory: tracked files
|
||||
@@ -219,3 +225,179 @@ describe('gate-cost-replay CLI', () => {
|
||||
expect(JSON.stringify(body.prs[0])).toContain('"screenProvisional"');
|
||||
});
|
||||
});
|
||||
|
||||
describe('red attribution (GSTA-47 rule)', () => {
|
||||
test('report verdicts: rule, judge (spaced id) and unbound-shard failure lines', () => {
|
||||
const text = ['[test:paid] VERDICT RED — lane gate/pr, attempt 1', '[test:paid] failures and split verdicts:',
|
||||
' ✗ tpa-absent-darwin rule FAIL assertion — expected [slice 14, attempt 1]',
|
||||
' ✗ canary/SKILL.md monitoring loop judge FAIL score 2 < 3',
|
||||
' ✗ test/skill-e2e-demo.test.ts rule shard NOT REPORTED [slice 3, attempt 1]'].join('\n');
|
||||
expect(parseReportVerdict(text)).toEqual({ verdict: 'RED', failures: [
|
||||
{ caseId: 'tpa-absent-darwin', kind: 'rule' }, { caseId: 'canary/SKILL.md monitoring loop', kind: 'judge' }, { caseId: 'test/skill-e2e-demo.test.ts', kind: 'rule' }] });
|
||||
expect(parseReportVerdict('[test:paid] VERDICT GREEN — lane gate/pr').verdict).toBe('GREEN');
|
||||
expect(parseReportVerdict('no report').verdict).toBe('NONE');
|
||||
});
|
||||
|
||||
test('the own-diff signature ignores hunk positions, index lines and the package.json version', () => {
|
||||
const diff = (hunk: string, index: string, version: string, body: string) => ['diff --git a/x.ts b/x.ts', `index ${index}`, '--- a/x.ts', '+++ b/x.ts', hunk, body,
|
||||
'diff --git a/package.json b/package.json', '--- a/package.json', '+++ b/package.json', '@@ -3 +3 @@', `- "version": "1.0.0",`, `+ "version": "${version}",`].join('\n');
|
||||
const base = stampFreeSignature(diff('@@ -1 +1 @@', 'aaa..bbb', '1.0.1', '+x'));
|
||||
expect(stampFreeSignature(diff('@@ -9 +9 @@', 'ccc..ddd', '1.0.2', '+x'))).toBe(base);
|
||||
expect(stampFreeSignature(diff('@@ -1 +1 @@', 'aaa..bbb', '1.0.1', '+y'))).not.toBe(base);
|
||||
expect(stampFreeSignature(diff('@@ -1 +1 @@', 'aaa..bbb', '1.0.1', '+x').split('\n').slice(0, 6).join('\n'))).toBe(base);
|
||||
});
|
||||
|
||||
const H = (sha: string) => `${sha}000000000000000000000000000000000000000`.slice(0, 40);
|
||||
const run = (id: number, pr: number | null, head: string, hour: number, failures: string[] = []): GatedRun =>
|
||||
({ id, pr, head: H(head), createdAt: new Date(Date.UTC(2026, 9, 5, hour)).toISOString(), green: failures.length === 0, failures: failures.map(caseId => ({ caseId, kind: 'rule' })) });
|
||||
const heads = new Map<string, HeadDiff>([
|
||||
[H('a1'), { ownSig: 's1', files: ['x/SKILL.md.tmpl'] }], [H('a2'), { ownSig: 's2', files: ['x/SKILL.md.tmpl', 'test/helpers/case-a.ts'] }],
|
||||
[H('b1'), { ownSig: 't1', files: ['docs/b.md'] }], [H('b2'), { ownSig: 't1', files: ['docs/b.md'] }],
|
||||
[H('c1'), { ownSig: 'u1', files: ['docs/c.md'] }], [H('c2'), { ownSig: 'u2', files: ['docs/c.md', 'docs/d.md'] }],
|
||||
]);
|
||||
const deps = {
|
||||
heads,
|
||||
delta: (from: string, to: string) => (from === H('a1') && to === H('a2') ? ['test/helpers/case-a.ts', 'lib/from-main.ts'] : ['docs/d.md']),
|
||||
touches: (caseId: string, files: string[]) => files.filter(file => file === `test/helpers/${caseId}.ts`),
|
||||
};
|
||||
|
||||
test('own push touching the case inputs is PR-caused; red elsewhere, same own diff, unrelated pushes and no later run are not', () => {
|
||||
const runs = [
|
||||
run(1, 10, 'a1', 0, ['case-a', 'case-shared']), run(2, 10, 'a2', 2),
|
||||
run(3, 11, 'b1', 1, ['case-b']), run(4, 11, 'b2', 3, ['case-b']),
|
||||
run(5, 12, 'c1', 4, ['case-shared', 'case-c']), run(6, 12, 'c2', 5),
|
||||
run(7, 13, 'zz', 6, ['case-z']), run(8, 14, 'zz', 7),
|
||||
];
|
||||
const reds = classifyReds(runs, deps, { prCaused: [], outside: [] });
|
||||
const byRun = Object.fromEntries(reds.map(r => [r.runId, r]));
|
||||
expect(byRun[1]!.cause).toBe('pr');
|
||||
expect(byRun[1]!.cases.map(c => [c.caseId, c.cause, c.reason])).toEqual([
|
||||
['case-a', 'pr', 'fixed by own push: test/helpers/case-a.ts'], ['case-shared', 'outside', 'also red in #12']]);
|
||||
expect(byRun[3]!.cases[0]).toMatchObject({ cause: 'outside', reason: 'still red with the same own diff' });
|
||||
expect(byRun[4]!.cases[0]).toMatchObject({ cause: 'unknown', reason: 'no later gated run' });
|
||||
expect(byRun[5]!.cases.map(c => c.reason)).toEqual(['also red in #10', 'passed next after an unrelated push']);
|
||||
expect(byRun[5]!.cause).toBe('outside');
|
||||
expect(byRun[7]!.cause).toBe('unknown');
|
||||
expect(classifyReds([run(9, 15, 'q1', 0, [])].map(r => ({ ...r, green: false })), deps)[0]).toMatchObject({ cause: 'outside', cases: [] });
|
||||
});
|
||||
|
||||
test('transcript diagnoses win, and the 72-hour window bounds "red elsewhere"', () => {
|
||||
const own = DIAGNOSED_REDS.prCaused[0], outside = DIAGNOSED_REDS.outside[0];
|
||||
const reds = classifyReds([run(own, 20, 'a1', 0, ['case-x']), run(outside, 21, 'a1', 0, ['case-y']), run(30, 22, 'b1', 1, ['case-x'])], deps);
|
||||
expect(reds.find(r => r.runId === own)!.cases[0]!.reason).toBe('diagnosed own edit (GSTA-45)');
|
||||
expect(reds.find(r => r.runId === outside)!.cases[0]!.reason).toBe('diagnosed main defect (fix wave 3)');
|
||||
const far = classifyReds([run(40, 30, 'a1', 0, ['case-a']), run(41, 30, 'a2', 1), run(42, 31, 'b1', 80, ['case-a'])], deps, { prCaused: [], outside: [] });
|
||||
expect(far.find(r => r.runId === 40)!.cause).toBe('pr');
|
||||
});
|
||||
});
|
||||
|
||||
describe('red-head replay', () => {
|
||||
const selection = (e2e: string[], canary: boolean, global = false) => ({ e2e, global, canary, canaryReasons: canary ? ['resolver x'] : [], preambleOnly: [] });
|
||||
const files: Record<string, string> = { 'case-a': 'test/skill-e2e-demo.test.ts', 'case-b': 'test/skill-e2e-other.test.ts', 'auq-format-gate': 'test/auq.test.ts', 'judge x': 'test/skill-llm-eval.test.ts' };
|
||||
const caseFile = (id: string) => files[id] ?? null;
|
||||
|
||||
test('judges always count; touchfiles, unbound shards by file, and canary cases only when the canary set is added', () => {
|
||||
expect(screenHit({ caseId: 'qa/SKILL.md workflow', kind: 'judge' }, selection([], false), caseFile)).toBe('judge');
|
||||
expect(screenHit({ caseId: 'judge x', kind: 'rule' }, selection([], false), caseFile)).toBe('judge');
|
||||
expect(screenHit({ caseId: 'case-a', kind: 'rule' }, selection(['case-a'], false), caseFile)).toBe('touchfiles');
|
||||
expect(screenHit({ caseId: 'test/skill-e2e-demo.test.ts', kind: 'rule' }, selection(['case-a'], false), caseFile)).toBe('touchfiles');
|
||||
expect(screenHit({ caseId: 'case-b', kind: 'rule' }, selection(['case-a'], false), caseFile)).toBeNull();
|
||||
expect(screenHit({ caseId: 'case-b', kind: 'rule' }, selection([], false, true), caseFile)).toBe('global');
|
||||
expect(screenHit({ caseId: 'auq-format-gate', kind: 'rule' }, selection([], true), caseFile)).toBe('canary');
|
||||
expect(screenHit({ caseId: 'auq-format-gate', kind: 'rule' }, selection([], false), caseFile)).toBeNull();
|
||||
expect(Object.values(SCREEN_CANARY_CASES)).toContain('auq-format-gate');
|
||||
const gate = checkoutSelectionDeps().gateCases;
|
||||
for (const id of Object.values(SCREEN_CANARY_CASES)) expect(gate).toContain(id);
|
||||
});
|
||||
|
||||
test('summaries count merged PRs only: P-k numerator, screened misses, and misses without the canary set', () => {
|
||||
const row = (pr: number, hit: RedHeadRow['hit']): RedHeadRow => ({ pr, runId: pr, head: 'h', base: 'b', baseSource: 'tested-merge', files: 1, caseId: 'c', kind: 'rule', hit, canaryReasons: [] });
|
||||
const redRuns = [10, 11, 12, 99].map(pr => ({ runId: pr, pr, head: 'h', createdAt: '', nextRunId: null, cause: 'pr' as const, cases: [] }));
|
||||
const s = summarizeRedHeads([row(10, null), row(10, 'touchfiles'), row(11, 'canary'), row(12, 'judge'), row(99, null)], redRuns,
|
||||
[10, 11, 12, 13].map(pr => ({ pr, paidRelevant: pr !== 13 })));
|
||||
expect(s).toMatchObject({ mergedPrs: 4, paidRelevantPrs: 3, prsWithPrRed: [10, 11, 12], prCausedRuns: 3, prCausedCases: 4, missedPrs: [10], missedCases: 1,
|
||||
missedPrsWithoutCanary: [10, 11], otherPrCausedRuns: 1, byHit: { missed: 1, touchfiles: 1, canary: 1, judge: 1 } });
|
||||
});
|
||||
});
|
||||
|
||||
describe('READY arrivals and trains', () => {
|
||||
const at = (hour: number) => new Date(Date.UTC(2026, 9, 10, hour)).toISOString();
|
||||
|
||||
test('READY is the first green head with the merged own diff before the merge; fallback the final head', () => {
|
||||
const green = headGreenAt([
|
||||
{ head: 'h1', workflow: 'free', createdAt: at(0), completedAt: at(1), conclusion: 'success' },
|
||||
{ head: 'h1', workflow: 'evals', createdAt: at(0), completedAt: at(2), conclusion: 'success' },
|
||||
{ head: 'h2', workflow: 'free', createdAt: at(3), completedAt: at(4), conclusion: 'success' },
|
||||
{ head: 'h2', workflow: 'evals', createdAt: at(3), completedAt: at(4), conclusion: 'failure' },
|
||||
{ head: 'h2', workflow: 'evals', createdAt: at(5), completedAt: at(6), conclusion: 'success' },
|
||||
{ head: 'h3', workflow: 'free', createdAt: at(7), completedAt: at(8), conclusion: 'success' },
|
||||
{ head: 'h4', workflow: 'free', createdAt: at(7), completedAt: at(8), conclusion: 'failure' },
|
||||
]);
|
||||
expect([...green]).toEqual([['h1', at(2)], ['h2', at(6)], ['h3', at(8)], ['h4', null]]);
|
||||
const rows = readyTimes([
|
||||
{ pr: 1, mergedAt: at(10), finalHead: 'h3', heads: [{ sha: 'h1', ownSig: 'old' }, { sha: 'h2', ownSig: 's' }, { sha: 'h3', ownSig: 's' }], gatedRuns: [at(0), at(3), at(5), at(7)] },
|
||||
{ pr: 2, mergedAt: at(10), finalHead: 'h1', heads: [{ sha: 'h1', ownSig: null }], gatedRuns: [] },
|
||||
{ pr: 3, mergedAt: at(7), finalHead: 'h3', heads: [{ sha: 'h3', ownSig: 's' }], gatedRuns: [] },
|
||||
], green);
|
||||
expect(rows.map(r => [r.pr, r.ready, r.source, r.readyToMergeHours, r.gatesAfterReady])).toEqual([
|
||||
[1, at(6), 'final-own-diff', 4, 1], [2, at(2), 'final-head', 8, 0], [3, null, 'none', null, 0]]);
|
||||
expect([percentile([5, 1, 3, 2, 4], 0.5), percentile([5, 1, 3, 2, 4], 0.9), percentile([], 0.5)]).toEqual([3, 5, 0]);
|
||||
});
|
||||
|
||||
test('train composition: unscreened PRs form their own train at five, else screened first, else the fill limit', () => {
|
||||
const item = (enqueuedMs: number, screened: boolean) => ({ enqueuedMs, screened });
|
||||
const pool = [item(0, true), item(1, true), ...[2, 3, 4, 5, 6].map(t => item(t, false))];
|
||||
expect(pickTrain(pool, 10, 5, 100)).toEqual(pool.slice(2));
|
||||
const mixed = [item(5, false), item(0, true), item(1, false), item(2, true), item(3, true)];
|
||||
expect(pickTrain(mixed, 10, 5, 100)?.map(x => x.screened)).toEqual([true, true, true, false, false]);
|
||||
expect(pickTrain([item(0, false), item(1, true)], 99, 5, 100)).toBeNull();
|
||||
expect(pickTrain([item(0, false), item(1, true)], 100, 5, 100)?.map(x => x.enqueuedMs)).toEqual([1, 0]);
|
||||
});
|
||||
|
||||
test('deterministic controls: a fill-limit train, a full train, and a repeated defect red', () => {
|
||||
const arrivals = (n: number, paidRelevant: boolean) => Array.from({ length: n }, (_, i) => ({ pr: i + 1, readyMs: 0, paidRelevant, screenUsd: 1 }));
|
||||
const base = { policy: { kind: 'all' } as const, windowDays: 7, outsideRed: 0, redScreened: 0, redUnscreened: 0, reps: 3 };
|
||||
const full = simulateArrivalTrains({ ...base, arrivals: arrivals(5, true) });
|
||||
expect(full).toMatchObject({ trainsWeekly: 1, meanOccupancy: 5, fullTrainShare: 1, gateAttemptsWeekly: 1, bisectsWeekly: 0, occupancy: [0, 0, 0, 0, 1] });
|
||||
expect(full.readyToMerge).toMatchObject({ prs: 15, p50Hours: 0.75, p90Hours: 0.75 });
|
||||
const short = simulateArrivalTrains({ ...base, arrivals: arrivals(2, false) });
|
||||
expect(short.readyToMerge.p90Hours).toBe(3.75);
|
||||
const red = simulateArrivalTrains({ ...base, arrivals: arrivals(2, true), redScreened: 1 });
|
||||
expect(red).toMatchObject({ trainsWeekly: 2, gateAttemptsWeekly: 3, bisectsWeekly: 2, attributedRebuildsWeekly: 1, rebuildOnceWeekly: 0, occupancy: [0, 2, 0, 0, 0] });
|
||||
expect(red.readyToMerge.p50Hours).toBe(8.25);
|
||||
const unscreened = simulateArrivalTrains({ ...base, policy: { kind: 'none' }, arrivals: arrivals(2, true), redUnscreened: 1 });
|
||||
expect(unscreened.bisectsWeekly).toBe(2);
|
||||
expect(simulateArrivalTrains({ ...base, arrivals: arrivals(7, true), outsideRed: 0.15, redUnscreened: 0.25, seed: 4 }))
|
||||
.toEqual(simulateArrivalTrains({ ...base, arrivals: arrivals(7, true), outsideRed: 0.15, redUnscreened: 0.25, seed: 4 }));
|
||||
});
|
||||
|
||||
test('runs map to PRs by pull_requests, else by head branch while the PR was open', () => {
|
||||
const pulls = [{ number: 5, ref: 'feat', createdAt: at(0), mergedAt: at(5), closedAt: at(5) }, { number: 6, ref: 'feat', createdAt: at(8), mergedAt: null, closedAt: null }];
|
||||
expect(prForRun({ branch: 'x', createdAt: at(1), prNumbers: [9] }, pulls)).toBe(9);
|
||||
expect(prForRun({ branch: 'feat', createdAt: at(2), prNumbers: [] }, pulls)).toBe(5);
|
||||
expect(prForRun({ branch: 'feat', createdAt: at(9), prNumbers: [] }, pulls)).toBe(6);
|
||||
expect(prForRun({ branch: 'feat', createdAt: at(7), prNumbers: [] }, pulls)).toBeNull();
|
||||
});
|
||||
|
||||
test('--history needs a window, and every history line naming the screen carries the screen-provisional label', () => {
|
||||
expect(() => parseArgs(['--runs', '1', '--history'])).toThrow('--history needs a window');
|
||||
expect(parseArgs(['--runs', '1', '--history', '--since', '2026-10-03', '--until', '2026-10-04']).history).toBe(true);
|
||||
const history: HistoryReport = {
|
||||
window: { since: at(0), until: at(24) }, runs: { evals: 3, gated: 2, red: 1, free: 4, unmapped: 0 },
|
||||
redRuns: [{ runId: 1, pr: 10, head: 'h', createdAt: at(0), nextRunId: 2, cause: 'pr', cases: [{ caseId: 'case-a', kind: 'rule', cause: 'pr', reason: 'fixed by own push: x' }] }],
|
||||
redHeads: [{ pr: 10, runId: 1, head: 'h', base: 'b', baseSource: 'tested-merge', files: 2, caseId: 'case-a', kind: 'rule', hit: null, canaryReasons: [] }],
|
||||
redSummary: summarizeRedHeads([{ pr: 10, runId: 1, head: 'h', base: 'b', baseSource: 'tested-merge', files: 2, caseId: 'case-a', kind: 'rule', hit: null, canaryReasons: [] }],
|
||||
[{ runId: 1, pr: 10, head: 'h', createdAt: at(0), nextRunId: 2, cause: 'pr', cases: [] }], [{ pr: 10, paidRelevant: true }, { pr: 11, paidRelevant: false }]),
|
||||
perPr: [{ pr: 10, paidRelevant: true, prCausedRed: true, screenedMiss: true }, { pr: 11, paidRelevant: false, prCausedRed: false, screenedMiss: false }],
|
||||
ready: [], baseline: { prs: 2, p50Hours: 1, p90Hours: 2, meanHours: 1.5, gatesAfterReady: 1, sources: { 'final-own-diff': 2 } },
|
||||
trainParams: { trainSize: 5, fillLimitHours: 3, gateMinutes: 45, bisectMinutes: 45, redScreened: 0.06, redUnscreened: 0.25, reps: 3, seed: 1 },
|
||||
trains: [simulateArrivalTrains({ arrivals: [{ pr: 10, readyMs: 0, paidRelevant: true, screenUsd: 1 }], policy: { kind: 'all' }, windowDays: 7, reps: 3 })],
|
||||
};
|
||||
const text = formatHistory(history);
|
||||
expect(text).toContain('P-k (merged PRs with any PR-caused red run): 1 of 2 (50.0%): #10');
|
||||
expect(text).toContain('misses: 1 of 1 (100.0%) merged PRs with a PR-caused red (1 of 1 case results): #10');
|
||||
const screenLines = text.split('\n').filter(line => /\bscreen\b/i.test(line));
|
||||
expect(screenLines.length).toBeGreaterThan(2);
|
||||
for (const line of screenLines) expect(line).toContain(SCREEN_LABEL);
|
||||
});
|
||||
});
|
||||
Reference in new issue
Block a user