mirror of
https://github.com/garrytan/gstack.git
synced 2026-11-04 15:57:33 +01:00
Merge remote-tracking branch 'origin/main' into capy/docsync-observer-accepts-read
This commit is contained in:
garrytan committed
commit
ef8474d7f0
10 files changed
+929
-31
No files matched your search
@@ -1,5 +1,18 @@
|
||||
# Changelog
|
||||
|
||||
## [1.91.138.0] - 2026-10-18
|
||||
|
||||
**`scripts/gate-cost-replay.ts --history` replays PR gates' red runs and READY times, so the merge-train plan's two estimates are measured instead of assumed.** Over Oct 3 to 18 (505 E2E Evals runs, 92 merged PRs), the screen-provisional rule run on each failing head's diff from its tested base would have missed the PR-caused red on 2 of the 13 PRs that had one (#3036, #3121): 2 of 72 paid-relevant PRs (2.8%), against the plan's 6% estimate. 13 of 92 merged PRs had a PR-caused red, matching the plan's figure. Replaying READY arrivals through trains of 5 with a 3-hour fill limit, the realized train averages about 2.3 PRs, so trains make 22 to 29 gate attempts a week rather than the 12 to 20 the train model assumed, and READY-to-merge p90 runs 5 to 8 hours. This is lane T0 of the plan to cut the paid gate bill about 4x; nothing changes what any gate runs.
|
||||
|
||||
### Added
|
||||
- **Red attribution as code.** Each failed case of a red PR-gate run is PR-caused or outside by GSTA-47's rule: a transcript diagnosis, red in another PR within 72 hours, the next gated run's own diff, or whether the push to it changed the case's inputs. It reproduces all 22 of GSTA-47's PR-caused runs and the same 13 PRs.
|
||||
- **The red-head replay.** Every PR-caused red case is checked against the screen-provisional rule on the failing head (not the merged squash), diffed from the base its run tested (the tested merge commit's first parent). The canary set is fixed in code before any replay (`SCREEN_CANARY_CASES`, one gate case per preamble-dependent surface) and is provisional until T2 lands. The report also lists misses with an empty canary set: here it made no difference, and 22 of the 28 red cases were caught only because the failing head changed a harness global, which makes the screen the full gate.
|
||||
- **The READY-arrival replay.** READY is approximated by the first time the merged own diff (modulo main merges and stamps) had green Free Tests and E2E Evals before the merge; READY reports are not machine-readable. Arrivals then run through the train policy (screened PRs first, unscreened in their own train once 5 wait, 45 minutes a gate and a bisect, the train model's red and rebuild rules), reporting occupancy, rebuilds and READY-to-merge p50/p90 beside today's (p50 0.1 h, p90 3.3 h).
|
||||
- Logic lives in `lib/gate-cost-history.ts`; the CLI reads runs, PRs and `report-verdict`/`trial-outcomes` artifacts read-only with `gh`, cached, and fetches old PR heads by SHA.
|
||||
|
||||
### If you…
|
||||
- **…want current numbers:** run `bun run scripts/gate-cost-replay.ts --runs <full-gate run ids> --since <date> --until <date> --history`. A cold run downloads the full gates once (about 9 minutes); a cached rerun takes about a minute.
|
||||
|
||||
## [1.91.137.0] - 2026-10-18
|
||||
|
||||
**`scripts/gate-cost-replay.ts` measures what the paid PR gate costs, and what each selection or merge-train policy would have cost, from recorded artifacts and for free.** On the nine full gates of Oct 17 and 18 it reproduces GSTA-49's measurement: $93.14 recorded per gate, 97.1% of it Fable 5.1 (the model under test), 49.8% cache writes and 40.3% output. Over the 92 squash merges of Oct 3 to 18, today's selector averages $78.67 per merged PR on the $106.19 full-gate scale, R4's exercised-map shadow $64.27, and the screening rule $45.17 (screen-provisional). This is lane T0 of the plan to cut the paid gate bill about 4x; nothing changes what any gate runs.
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
# gstack digest v1.91.137.0 — regenerate/re-copy after upgrading gstack
|
||||
# gstack digest v1.91.138.0 — regenerate/re-copy after upgrading gstack
|
||||
|
||||
Behavioral rules from gstack (https://github.com/garrytan/gstack), compressed
|
||||
for agent hosts without a full skill install. The full skills add workflows,
|
||||
|
||||
@@ -0,0 +1,421 @@
|
||||
/**
|
||||
* gate-cost-history — the two history replays of GSTA-49 plan lane T0 (P-i,
|
||||
* P-k, and the READY-to-merge guardrail in its Risks). Pure logic: the CLI
|
||||
* (`scripts/gate-cost-replay.ts --history`) passes run lists, report texts,
|
||||
* head signatures and selections in; nothing here runs git or calls GitHub.
|
||||
*
|
||||
* 1. Red attribution: each red PR-gate run's failed cases split into
|
||||
* PR-caused and outside reds by GSTA-47's rule (`classifyReds`).
|
||||
* 2. Red-head replay: every PR-caused red case against the
|
||||
* screen-provisional rule on the failing head's diff from its tested base,
|
||||
* with the canary set below fixed before the replay first ran.
|
||||
* 3. READY arrivals: a READY time per merged PR from its check history, and
|
||||
* a seeded replay of those arrivals through the train policy with the
|
||||
* merge-train model's red and bisect rules.
|
||||
*/
|
||||
import { createHash } from 'node:crypto';
|
||||
import { PyRandom, policyLabel, screenedIndexes, type ScreenPolicy, type ScreenSelection } from './gate-cost';
|
||||
|
||||
const HOUR_MS = 3_600_000;
|
||||
|
||||
// --- 1. Red attribution ---
|
||||
|
||||
export interface CaseFailure { caseId: string; kind: string }
|
||||
export interface ReportVerdict { verdict: 'GREEN' | 'RED' | 'NONE'; failures: CaseFailure[] }
|
||||
|
||||
/** report-verdict's report.txt: the verdict line and each `✗ <id> <kind> <status>` line (judge ids contain single spaces; an unbound shard is named by its test file). */
|
||||
export function parseReportVerdict(text: string): ReportVerdict {
|
||||
const verdict = /VERDICT (GREEN|RED)/.exec(text)?.[1] as 'GREEN' | 'RED' | undefined;
|
||||
const failures = [...text.matchAll(/^\s+✗ (\S(?:.*?\S)?) {2,}(\w+) +\S/gm)].map(m => ({ caseId: m[1]!, kind: m[2]! }));
|
||||
return { verdict: verdict ?? 'NONE', failures };
|
||||
}
|
||||
|
||||
/** One PR-gate run that produced a report (GSTA-47's "gated" run); failures are the union over its attempts. */
|
||||
export interface GatedRun { id: number; pr: number | null; head: string; createdAt: string; green: boolean; failures: CaseFailure[] }
|
||||
/** A head's own diff: the files changed from its merge-base with main, and GSTA-47's stamp-free signature of that diff. */
|
||||
export interface HeadDiff { ownSig: string; files: string[] }
|
||||
|
||||
export const STAMP_FILES = ['VERSION', 'CHANGELOG.md', 'agents-digest/gstack-AGENTS.md'] as const;
|
||||
|
||||
/**
|
||||
* GSTA-47's own-diff signature over `git diff -U0 <merge-base> <head> -- . ':(exclude)<stamp>'...`:
|
||||
* hunk headers, index lines and the package.json version line are dropped, so
|
||||
* a main merge or a restamp keeps the signature.
|
||||
*/
|
||||
export function stampFreeSignature(diffU0: string): string {
|
||||
let current = '';
|
||||
const blocks: string[][] = [];
|
||||
for (const line of diffU0.split('\n')) {
|
||||
if (line.startsWith('diff --git')) { current = line.split(' b/')[1] ?? ''; blocks.push([`F ${current}`]); continue; }
|
||||
if (!blocks.length || /^(index |@@|--- |\+\+\+ )/.test(line)) continue;
|
||||
if (current === 'package.json' && /^[-+]\s*"version":/.test(line)) continue;
|
||||
blocks[blocks.length - 1]!.push(line);
|
||||
}
|
||||
const kept = blocks.filter(block => block.length > 1 || !block[0]!.startsWith('F package.json')).flat();
|
||||
return createHash('sha256').update(kept.join('\n')).digest('hex').slice(0, 16);
|
||||
}
|
||||
|
||||
/** A case red in another PR's run within this window is an outside red. */
|
||||
export const RED_ELSEWHERE_WINDOW_MS = 72 * HOUR_MS;
|
||||
/** Reds GSTA-47 set from transcripts: GSTA-45's own-edit reds on #3260, and the main defects fix wave 3 (#3260) diagnosed. */
|
||||
export const DIAGNOSED_REDS = {
|
||||
prCaused: [39073934326, 39075977549, 39077524802],
|
||||
outside: [38973329985, 38986120559, 38990111230, 38993076767, 39002269281, 39004466422, 39011101113, 38980005793,
|
||||
39067746095, 39070632885, 39067885089, 39070276914, 39066860900, 39053431488],
|
||||
} as const;
|
||||
|
||||
export type RedCause = 'pr' | 'outside' | 'unknown';
|
||||
export interface CaseRed extends CaseFailure { cause: RedCause; reason: string }
|
||||
export interface RedRun { runId: number; pr: number; head: string; createdAt: string; nextRunId: number | null; cause: RedCause; cases: CaseRed[] }
|
||||
|
||||
export interface ClassifyDeps {
|
||||
heads: ReadonlyMap<string, HeadDiff>;
|
||||
/** `git diff --name-only <from> <to>`; null when unreadable. */
|
||||
delta(from: string, to: string): string[] | null;
|
||||
/** The files among `files` that are inputs of the case (declared touchfiles, its test file, harness globals). */
|
||||
touches(caseId: string, files: string[]): string[];
|
||||
}
|
||||
|
||||
/**
|
||||
* GSTA-47's attribution, per failed case of a red run, first match wins:
|
||||
* a transcript diagnosis; red in another PR within 72 h (outside); no later
|
||||
* gated run on the PR (unknown); the next gated run has the same own diff
|
||||
* (outside); the push to the next run changed one of the case's inputs among
|
||||
* either head's own files (PR-caused, whether or not the case then passed);
|
||||
* otherwise outside. A run is PR-caused when any case is, outside when every
|
||||
* case is (or none was named: infra or incomplete), unknown otherwise.
|
||||
*/
|
||||
export function classifyReds(runs: GatedRun[], deps: ClassifyDeps, diagnosed: { prCaused: readonly number[]; outside: readonly number[] } = DIAGNOSED_REDS): RedRun[] {
|
||||
const reds = runs.filter(run => !run.green);
|
||||
const index = reds.flatMap(run => run.failures.map(f => ({ caseId: f.caseId, pr: run.pr, t: Date.parse(run.createdAt) })));
|
||||
const out: RedRun[] = [];
|
||||
for (const run of reds) {
|
||||
if (run.pr === null) continue;
|
||||
const next = runs.filter(r => r.pr === run.pr && r.createdAt > run.createdAt).sort((a, b) => a.createdAt.localeCompare(b.createdAt))[0] ?? null;
|
||||
const own = deps.heads.get(run.head), nextOwn = next ? deps.heads.get(next.head) : undefined;
|
||||
const t = Date.parse(run.createdAt);
|
||||
const cases: CaseRed[] = [];
|
||||
for (const failure of run.failures) {
|
||||
if (cases.some(c => c.caseId === failure.caseId)) continue;
|
||||
const elsewhere = [...new Set(index.filter(x => x.caseId === failure.caseId && x.pr !== run.pr && Math.abs(x.t - t) < RED_ELSEWHERE_WINDOW_MS).map(x => x.pr))];
|
||||
const verdict = ((): [RedCause, string] => {
|
||||
if (diagnosed.prCaused.includes(run.id)) return ['pr', 'diagnosed own edit (GSTA-45)'];
|
||||
if (diagnosed.outside.includes(run.id)) return ['outside', 'diagnosed main defect (fix wave 3)'];
|
||||
if (elsewhere.length) return ['outside', `also red in ${elsewhere.map(pr => (pr === null ? 'an unmapped run' : `#${pr}`)).join(', ')}`];
|
||||
if (!next) return ['unknown', 'no later gated run'];
|
||||
const passed = !next.failures.some(f => f.caseId === failure.caseId);
|
||||
if (!own || !nextOwn) return ['unknown', 'a head is not readable'];
|
||||
if (own.ownSig === nextOwn.ownSig) return ['outside', passed ? 'passed next with the same own diff' : 'still red with the same own diff'];
|
||||
const delta = deps.delta(run.head, next.head);
|
||||
if (!delta) return ['unknown', 'the push between runs is not readable'];
|
||||
const hit = deps.touches(failure.caseId, delta.filter(file => own.files.includes(file) || nextOwn.files.includes(file)));
|
||||
if (hit.length) return ['pr', `${passed ? 'fixed by own push' : 'own push touched its inputs, still red'}: ${hit.slice(0, 3).join(' ')}`];
|
||||
return ['outside', passed ? 'passed next after an unrelated push' : 'still red after an unrelated push'];
|
||||
})();
|
||||
cases.push({ ...failure, cause: verdict[0], reason: verdict[1] });
|
||||
}
|
||||
const cause: RedCause = cases.every(c => c.cause === 'outside') ? 'outside' : cases.some(c => c.cause === 'pr') ? 'pr' : 'unknown';
|
||||
out.push({ runId: run.id, pr: run.pr, head: run.head, createdAt: run.createdAt, nextRunId: next?.id ?? null, cause, cases });
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
// --- 2. Red-head replay ---
|
||||
|
||||
/**
|
||||
* T2's structural canary rule (PLAN.md rev 6: one case per preamble-dependent
|
||||
* surface), resolved to gate cases before this replay first ran and not
|
||||
* changed after: for each surface, the gate case with the shortest recorded
|
||||
* session time (run 39075977549) among the cases on that surface. No gate
|
||||
* case runs plan mode on the claude -p host, so that surface has none.
|
||||
* Provisional until T2's reviewed list in test/helpers/touchfiles-data.ts.
|
||||
*/
|
||||
export const SCREEN_CANARY_CASES: Readonly<Record<string, string>> = {
|
||||
'skill start, claude -p (skillmd-* cases)': 'skillmd-outside-git',
|
||||
'skill start, PTY (PTY cases outside plan mode)': 'office-hours-auto-mode',
|
||||
'plan mode, PTY (*-plan-mode and plan-mode-no-op)': 'plan-devex-review-plan-mode',
|
||||
'AskUserQuestion format': 'auq-format-gate',
|
||||
'bins every skill runs (cases declaring scripts/resolvers/preamble.ts)': 'operational-learning',
|
||||
'hermetic sentinels (1 of 2)': 'hermetic-canary',
|
||||
'hermetic sentinels (2 of 2)': 'hermetic-sentinel',
|
||||
};
|
||||
|
||||
export const JUDGE_TEST_FILE = 'test/skill-llm-eval.test.ts';
|
||||
export type ScreenHit = 'judge' | 'global' | 'touchfiles' | 'canary' | null;
|
||||
|
||||
/**
|
||||
* Would the screen select this failed case? Judges always run. A failure
|
||||
* named by its test file (an unbound shard) counts as selected when any
|
||||
* selected case lives in that file.
|
||||
*/
|
||||
export function screenHit(failure: CaseFailure, selection: ScreenSelection, caseFile: (caseId: string) => string | null,
|
||||
canaryCases: readonly string[] = Object.values(SCREEN_CANARY_CASES)): ScreenHit {
|
||||
if (failure.kind === 'judge' || caseFile(failure.caseId) === JUDGE_TEST_FILE) return 'judge';
|
||||
if (selection.global) return 'global';
|
||||
const byFile = /^test\/.+\.test\.ts$/.test(failure.caseId);
|
||||
const covers = (ids: readonly string[]) => (byFile ? ids.some(id => caseFile(id) === failure.caseId) : ids.includes(failure.caseId));
|
||||
if (covers(selection.e2e)) return 'touchfiles';
|
||||
return selection.canary && covers(canaryCases) ? 'canary' : null;
|
||||
}
|
||||
|
||||
export interface RedHeadRow extends CaseFailure {
|
||||
pr: number;
|
||||
runId: number;
|
||||
head: string;
|
||||
base: string;
|
||||
/** `tested-merge`: the first parent of the run's tested merge commit; `main-at-run`: main's first-parent tip when the run was created. */
|
||||
baseSource: 'tested-merge' | 'main-at-run';
|
||||
files: number;
|
||||
hit: ScreenHit;
|
||||
canaryReasons: string[];
|
||||
}
|
||||
|
||||
export interface RedHeadSummary {
|
||||
mergedPrs: number;
|
||||
paidRelevantPrs: number;
|
||||
/** Merged PRs with at least one PR-caused red run (P-k's numerator). */
|
||||
prsWithPrRed: number[];
|
||||
prCausedRuns: number;
|
||||
prCausedCases: number;
|
||||
/** Merged PRs with a PR-caused red case the screen would not select (P-i's numerator). */
|
||||
missedPrs: number[];
|
||||
missedCases: number;
|
||||
/** Misses if the canary set were empty, to show how much of the result the canary choice carries. */
|
||||
missedPrsWithoutCanary: number[];
|
||||
byHit: Record<string, number>;
|
||||
/** PR-caused red runs on PRs that did not merge in the window (not counted above). */
|
||||
otherPrCausedRuns: number;
|
||||
}
|
||||
|
||||
export function summarizeRedHeads(rows: RedHeadRow[], redRuns: RedRun[], merged: Array<{ pr: number; paidRelevant: boolean }>): RedHeadSummary {
|
||||
const mergedSet = new Set(merged.map(m => m.pr));
|
||||
const inCohort = rows.filter(row => mergedSet.has(row.pr));
|
||||
const prRuns = redRuns.filter(run => run.cause === 'pr');
|
||||
const prsOf = (list: Array<{ pr: number }>) => [...new Set(list.map(row => row.pr))].sort((a, b) => a - b);
|
||||
const byHit: Record<string, number> = {};
|
||||
for (const row of inCohort) byHit[row.hit ?? 'missed'] = (byHit[row.hit ?? 'missed'] ?? 0) + 1;
|
||||
return {
|
||||
mergedPrs: merged.length,
|
||||
paidRelevantPrs: merged.filter(m => m.paidRelevant).length,
|
||||
prsWithPrRed: prsOf(prRuns.filter(run => mergedSet.has(run.pr))),
|
||||
prCausedRuns: prRuns.filter(run => mergedSet.has(run.pr)).length,
|
||||
prCausedCases: inCohort.length,
|
||||
missedPrs: prsOf(inCohort.filter(row => row.hit === null)),
|
||||
missedCases: inCohort.filter(row => row.hit === null).length,
|
||||
missedPrsWithoutCanary: prsOf(inCohort.filter(row => row.hit === null || row.hit === 'canary')),
|
||||
byHit,
|
||||
otherPrCausedRuns: prRuns.filter(run => !mergedSet.has(run.pr)).length,
|
||||
};
|
||||
}
|
||||
|
||||
// --- 3. READY arrivals ---
|
||||
|
||||
export interface CheckRun { head: string; workflow: 'free' | 'evals'; createdAt: string; completedAt: string; conclusion: string | null }
|
||||
|
||||
/**
|
||||
* When each head's checks went green: its latest Free Tests run succeeded and
|
||||
* its latest E2E Evals run (if any) succeeded or was skipped; the time is the
|
||||
* later of the two completions (`updated_at`). Null: never green.
|
||||
*/
|
||||
export function headGreenAt(runs: CheckRun[]): Map<string, string | null> {
|
||||
const latest = new Map<string, Partial<Record<CheckRun['workflow'], CheckRun>>>();
|
||||
for (const run of runs) {
|
||||
const entry = latest.get(run.head) ?? {};
|
||||
latest.set(run.head, entry);
|
||||
if (!entry[run.workflow] || entry[run.workflow]!.createdAt < run.createdAt) entry[run.workflow] = run;
|
||||
}
|
||||
const out = new Map<string, string | null>();
|
||||
for (const [head, { free, evals }] of latest) {
|
||||
const green = free?.conclusion === 'success' && (!evals || ['success', 'skipped', 'neutral'].includes(evals.conclusion ?? ''));
|
||||
out.set(head, green ? [free!.completedAt, evals?.completedAt ?? ''].sort().pop()! : null);
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
export interface PrChecks {
|
||||
pr: number;
|
||||
mergedAt: string;
|
||||
finalHead: string;
|
||||
/** Every head that ran Free Tests or E2E Evals on the PR, with its own-diff signature (null: unreadable). */
|
||||
heads: Array<{ sha: string; ownSig: string | null }>;
|
||||
/** Creation times of the PR's gated E2E Evals runs. */
|
||||
gatedRuns: string[];
|
||||
}
|
||||
|
||||
export type ReadySource = 'final-own-diff' | 'final-head' | 'none';
|
||||
export interface ReadyRow { pr: number; mergedAt: string; ready: string | null; source: ReadySource; readyToMergeHours: number | null; gatesAfterReady: number }
|
||||
|
||||
/**
|
||||
* READY is approximated by the first time the PR's final own diff (its merged
|
||||
* changes modulo main merges and stamps) had green checks before the merge:
|
||||
* the earliest green time among heads with the final head's signature, which
|
||||
* is when an unstamped READY report could first be sent. Fallback: the final
|
||||
* head's own green time. A green time after the merge does not count.
|
||||
*/
|
||||
export function readyTimes(prs: PrChecks[], greenAt: ReadonlyMap<string, string | null>): ReadyRow[] {
|
||||
return prs.map(pr => {
|
||||
const finalSig = pr.heads.find(h => h.sha === pr.finalHead)?.ownSig ?? null;
|
||||
const before = (sha: string) => { const at = greenAt.get(sha) ?? null; return at !== null && at <= pr.mergedAt ? at : null; };
|
||||
const sameDiff = finalSig === null ? [] : pr.heads.filter(h => h.ownSig === finalSig).map(h => before(h.sha)).filter((at): at is string => at !== null).sort();
|
||||
const finalAt = before(pr.finalHead);
|
||||
const [ready, source]: [string | null, ReadySource] = sameDiff.length ? [sameDiff[0]!, 'final-own-diff'] : finalAt ? [finalAt, 'final-head'] : [null, 'none'];
|
||||
return {
|
||||
pr: pr.pr, mergedAt: pr.mergedAt, ready, source,
|
||||
readyToMergeHours: ready ? (Date.parse(pr.mergedAt) - Date.parse(ready)) / HOUR_MS : null,
|
||||
gatesAfterReady: ready ? pr.gatedRuns.filter(at => at > ready && at <= pr.mergedAt).length : 0,
|
||||
};
|
||||
});
|
||||
}
|
||||
|
||||
/** Nearest-rank percentile; 0 for an empty list. */
|
||||
export function percentile(values: number[], q: number): number {
|
||||
if (!values.length) return 0;
|
||||
const sorted = [...values].sort((a, b) => a - b);
|
||||
return sorted[Math.max(0, Math.ceil(q * sorted.length) - 1)]!;
|
||||
}
|
||||
|
||||
export interface LatencySummary { prs: number; p50Hours: number; p90Hours: number; meanHours: number }
|
||||
export const latencySummary = (hours: number[]): LatencySummary => ({
|
||||
prs: hours.length, p50Hours: percentile(hours, 0.5), p90Hours: percentile(hours, 0.9),
|
||||
meanHours: hours.reduce((a, b) => a + b, 0) / Math.max(1, hours.length),
|
||||
});
|
||||
|
||||
export interface ArrivalTrainInput {
|
||||
/** One per merged PR with a READY time; `screenUsd` orders the cheapest/expensive-half policies. */
|
||||
arrivals: Array<{ pr: number; readyMs: number; paidRelevant: boolean; screenUsd: number }>;
|
||||
policy: ScreenPolicy;
|
||||
windowDays: number;
|
||||
trainSize?: number;
|
||||
fillLimitHours?: number;
|
||||
gateMinutes?: number;
|
||||
bisectMinutes?: number;
|
||||
outsideRed?: number;
|
||||
redScreened?: number;
|
||||
redUnscreened?: number;
|
||||
reps?: number;
|
||||
seed?: number;
|
||||
}
|
||||
|
||||
export interface ArrivalTrainResult {
|
||||
policy: string;
|
||||
outsideRed: number;
|
||||
trainsWeekly: number;
|
||||
meanOccupancy: number;
|
||||
fullTrainShare: number;
|
||||
/** Initial trains per repetition by size, index 0 = one PR. */
|
||||
occupancy: number[];
|
||||
gateAttemptsWeekly: number;
|
||||
bisectsWeekly: number;
|
||||
/** Gates again after a bisect dropped a defective PR. */
|
||||
attributedRebuildsWeekly: number;
|
||||
/** Garry's one unchanged rebuild after an unattributed red. */
|
||||
rebuildOnceWeekly: number;
|
||||
stallsWeekly: number;
|
||||
readyToMerge: LatencySummary;
|
||||
}
|
||||
|
||||
interface Waiting { pr: number; readyMs: number; enqueuedMs: number; screened: boolean; defect: boolean }
|
||||
|
||||
/**
|
||||
* The next train, or null to keep waiting: five or more waiting screened (or
|
||||
* unscreened) PRs dispatch a full train of their own; otherwise a full mixed
|
||||
* train, screened first; otherwise everything waiting, screened first, once
|
||||
* the oldest has waited the fill limit. The train with the oldest PR goes first.
|
||||
*/
|
||||
export function pickTrain<T extends { enqueuedMs: number; screened: boolean }>(pool: T[], nowMs: number, size: number, fillMs: number): T[] | null {
|
||||
const byAge = (a: T, b: T) => a.enqueuedMs - b.enqueuedMs;
|
||||
const screened = pool.filter(x => x.screened).sort(byAge), unscreened = pool.filter(x => !x.screened).sort(byAge);
|
||||
const options = [screened, unscreened].filter(list => list.length >= size).map(list => list.slice(0, size));
|
||||
if (!options.length) {
|
||||
const mixed = [...screened, ...unscreened];
|
||||
if (mixed.length >= size) options.push(mixed.slice(0, size));
|
||||
else if (mixed.length && nowMs - Math.min(...mixed.map(x => x.enqueuedMs)) >= fillMs) options.push(mixed);
|
||||
}
|
||||
const oldest = (train: T[]) => Math.min(...train.map(x => x.enqueuedMs));
|
||||
return options.sort((a, b) => oldest(a) - oldest(b))[0] ?? null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Replay READY arrivals through trains, one train gating at a time (each
|
||||
* builds on the previous train's merge). Each paid-relevant PR carries a
|
||||
* defect at the screened or unscreened rate; each attempt flakes at the
|
||||
* outside-red rate; every attempt takes gateMinutes and every red adds a
|
||||
* bisect of bisectMinutes. A defect drops its PR, which rejoins the queue at
|
||||
* once (repair time is not modeled), and the rest gate again; an unattributed
|
||||
* red gets one unchanged rebuild, a second is a stall and gates again. PRs
|
||||
* with no paid case ride with the screened ones.
|
||||
*/
|
||||
export function simulateArrivalTrains(input: ArrivalTrainInput): ArrivalTrainResult {
|
||||
const k = input.trainSize ?? 5, fillMs = (input.fillLimitHours ?? 3) * HOUR_MS;
|
||||
const gateMs = (input.gateMinutes ?? 45) * 60_000, bisectMs = (input.bisectMinutes ?? 45) * 60_000;
|
||||
const o = input.outsideRed ?? 0.15, rs = input.redScreened ?? 0.06, ru = input.redUnscreened ?? 0.25, reps = input.reps ?? 200;
|
||||
const rnd = new PyRandom(input.seed ?? 1);
|
||||
const t = { trains: 0, full: 0, gates: 0, bisects: 0, attributed: 0, rebuildOnce: 0, stalls: 0 };
|
||||
const occupancy = Array.from({ length: k }, () => 0);
|
||||
const hours: number[] = [];
|
||||
for (let rep = 0; rep < reps; rep += 1) {
|
||||
const screened = screenedIndexes(input.arrivals, input.policy, rnd);
|
||||
const arrivals: Waiting[] = input.arrivals.map((a, i) => ({
|
||||
pr: a.pr, readyMs: a.readyMs, enqueuedMs: a.readyMs, screened: !a.paidRelevant || screened.has(i),
|
||||
defect: a.paidRelevant && rnd.random() < (screened.has(i) ? rs : ru),
|
||||
})).sort((a, b) => a.readyMs - b.readyMs);
|
||||
const pool: Waiting[] = [];
|
||||
let next = 0, freeAt = -Infinity;
|
||||
while (next < arrivals.length || pool.length) {
|
||||
let now = pool.length ? freeAt : Math.max(freeAt, arrivals[next]!.readyMs);
|
||||
let train: Waiting[] | null;
|
||||
while (true) {
|
||||
while (next < arrivals.length && arrivals[next]!.readyMs <= now) pool.push(arrivals[next++]!);
|
||||
train = pickTrain(pool, now, k, fillMs);
|
||||
if (train) break;
|
||||
now = Math.min(next < arrivals.length ? arrivals[next]!.readyMs : Infinity, Math.min(...pool.map(x => x.enqueuedMs)) + fillMs);
|
||||
}
|
||||
for (const item of train) pool.splice(pool.indexOf(item), 1);
|
||||
t.trains += 1;
|
||||
occupancy[train.length - 1]! += 1;
|
||||
if (train.length === k) t.full += 1;
|
||||
let rebuilt = false;
|
||||
while (train.length) {
|
||||
t.gates += 1;
|
||||
now += gateMs;
|
||||
const flake = rnd.random() < o;
|
||||
const bad: Waiting[] = train.filter(item => item.defect);
|
||||
if (!flake && !bad.length) { for (const item of train) hours.push((now - item.readyMs) / HOUR_MS); break; }
|
||||
t.bisects += 1;
|
||||
now += bisectMs;
|
||||
if (bad.length) {
|
||||
const culprit: Waiting = bad[0]!;
|
||||
train = train.filter(item => item !== culprit);
|
||||
Object.assign(culprit, { defect: false, enqueuedMs: now });
|
||||
pool.push(culprit);
|
||||
rebuilt = false;
|
||||
if (train.length) t.attributed += 1;
|
||||
} else if (!rebuilt) {
|
||||
rebuilt = true;
|
||||
t.rebuildOnce += 1;
|
||||
} else {
|
||||
t.stalls += 1;
|
||||
rebuilt = false;
|
||||
}
|
||||
}
|
||||
freeAt = now;
|
||||
}
|
||||
}
|
||||
const perWeek = 7 / input.windowDays / reps;
|
||||
return {
|
||||
policy: policyLabel(input.policy), outsideRed: o, trainsWeekly: t.trains * perWeek,
|
||||
meanOccupancy: occupancy.reduce((sum, n, i) => sum + n * (i + 1), 0) / Math.max(1, t.trains),
|
||||
fullTrainShare: t.trains ? t.full / t.trains : 0, occupancy: occupancy.map(n => n / reps),
|
||||
gateAttemptsWeekly: t.gates * perWeek, bisectsWeekly: t.bisects * perWeek, attributedRebuildsWeekly: t.attributed * perWeek,
|
||||
rebuildOnceWeekly: t.rebuildOnce * perWeek, stallsWeekly: t.stalls * perWeek, readyToMerge: latencySummary(hours),
|
||||
};
|
||||
}
|
||||
|
||||
/** A run's PR: its pull_requests field, else the PR whose head branch it ran on while that PR was open (GSTA-47's mapping). */
|
||||
export function prForRun(run: { branch: string; createdAt: string; prNumbers: number[] },
|
||||
pulls: Array<{ number: number; ref: string; createdAt: string; mergedAt: string | null; closedAt: string | null }>): number | null {
|
||||
if (run.prNumbers.length) return run.prNumbers[0]!;
|
||||
const open = pulls.filter(p => p.ref === run.branch && p.createdAt <= run.createdAt
|
||||
&& run.createdAt <= new Date(Date.parse(p.mergedAt ?? p.closedAt ?? '9999-12-31') + HOUR_MS).toISOString());
|
||||
return open.sort((a, b) => b.createdAt.localeCompare(a.createdAt))[0]?.number ?? null;
|
||||
}
|
||||
+34
-14
@@ -306,6 +306,15 @@ export interface SelectionDeps {
|
||||
touchfiles(files: string[]): { e2e: string[]; global: boolean };
|
||||
}
|
||||
|
||||
export interface ScreenSelection {
|
||||
/** Gate cases the declared touchfiles and harness globals select (all of them when a global changed). */
|
||||
e2e: string[];
|
||||
global: boolean;
|
||||
canary: boolean;
|
||||
canaryReasons: string[];
|
||||
preambleOnly: string[];
|
||||
}
|
||||
|
||||
export interface SelectorCost { cases: number; judges: number; usd: number }
|
||||
export interface PrSelectionCost {
|
||||
pr: number;
|
||||
@@ -354,6 +363,9 @@ export function fullGateUsd(deps: Pick<SelectionDeps, 'gateCases' | 'judgeCases'
|
||||
return costOf(deps.gateCases, table) + costOf(deps.judgeCases, table);
|
||||
}
|
||||
|
||||
export const todaySelection = (diff: PrDiff, deps: SelectionDeps): TodaySelection =>
|
||||
(diff.files.length ? deps.today(diff.files, diff.packageVersionOnly) : { e2e: [], judges: [], shadowE2e: null, mode: 'none' });
|
||||
|
||||
/**
|
||||
* The screen-provisional rule: declared touchfiles and harness globals of the
|
||||
* files that are not heuristically preamble-only, every judge, and a canary
|
||||
@@ -361,12 +373,9 @@ export function fullGateUsd(deps: Pick<SelectionDeps, 'gateCases' | 'judgeCases'
|
||||
* resolver, or files that reach cases only transitively. A package.json whose
|
||||
* only change is the version stamp is neither selected nor a canary trigger.
|
||||
*/
|
||||
export function prSelectionCost(diff: PrDiff, deps: SelectionDeps, table: CaseCostTable, options: { canaryUsd?: number } = {}): PrSelectionCost {
|
||||
const full = fullGateUsd(deps, table);
|
||||
export function screenSelection(diff: PrDiff, deps: SelectionDeps, today: TodaySelection): ScreenSelection {
|
||||
const gate = new Set(deps.gateCases);
|
||||
const stampOnly = (file: string) => file === 'package.json' && diff.packageVersionOnly;
|
||||
const today = diff.files.length ? deps.today(diff.files, diff.packageVersionOnly) : { e2e: [], judges: [], shadowE2e: null, mode: 'none' };
|
||||
const shadowE2e = today.shadowE2e ?? today.e2e;
|
||||
const preambleOnly = heuristicPreambleOnly(Object.fromEntries(Object.entries(diff.skillMdChangedLines).filter(([file]) => diff.files.includes(file))));
|
||||
const screenFiles = diff.files.filter(file => !preambleOnly.includes(file) && !stampOnly(file));
|
||||
const touch = screenFiles.length ? deps.touchfiles(screenFiles) : { e2e: [], global: false };
|
||||
@@ -377,13 +386,19 @@ export function prSelectionCost(diff: PrDiff, deps: SelectionDeps, table: CaseCo
|
||||
...diff.files.filter(file => file.startsWith('scripts/resolvers/')).map(file => `resolver ${file}`),
|
||||
...(today.e2e.length > screenE2e.length ? ['transitive-only files (today selects more cases than declared touchfiles)'] : []),
|
||||
];
|
||||
const canary = canaryReasons.length > 0;
|
||||
const screenUsd = costOf(screenE2e, table) + (canary && !touch.global ? options.canaryUsd ?? 12 : 0) + costOf(deps.judgeCases, table);
|
||||
return { e2e: screenE2e, global: touch.global, canary: canaryReasons.length > 0, canaryReasons, preambleOnly };
|
||||
}
|
||||
|
||||
export function prSelectionCost(diff: PrDiff, deps: SelectionDeps, table: CaseCostTable, options: { canaryUsd?: number } = {}): PrSelectionCost {
|
||||
const today = todaySelection(diff, deps);
|
||||
const shadowE2e = today.shadowE2e ?? today.e2e;
|
||||
const { e2e, global, canary, canaryReasons, preambleOnly } = screenSelection(diff, deps, today);
|
||||
const screenUsd = costOf(e2e, table) + (canary && !global ? options.canaryUsd ?? 12 : 0) + costOf(deps.judgeCases, table);
|
||||
return {
|
||||
pr: diff.pr, sha: diff.sha, files: diff.files.length, mode: today.mode,
|
||||
today: { cases: today.e2e.length, judges: today.judges.length, usd: costOf(today.e2e, table) + costOf(today.judges, table) },
|
||||
shadow: { cases: shadowE2e.length, judges: today.judges.length, usd: costOf(shadowE2e, table) + costOf(today.judges, table) },
|
||||
screenProvisional: { cases: screenE2e.length, judges: deps.judgeCases.length, usd: Math.min(screenUsd, full), canary, canaryReasons, preambleOnly },
|
||||
screenProvisional: { cases: e2e.length, judges: deps.judgeCases.length, usd: Math.min(screenUsd, fullGateUsd(deps, table)), canary, canaryReasons, preambleOnly },
|
||||
};
|
||||
}
|
||||
|
||||
@@ -536,6 +551,17 @@ export interface TrainModelResult {
|
||||
attemptRedShare: number;
|
||||
}
|
||||
|
||||
/** Indexes of the PRs a policy screens: only paid-relevant PRs can screen; `random` draws from rnd. */
|
||||
export function screenedIndexes(prs: ReadonlyArray<{ paidRelevant: boolean; screenUsd: number }>, policy: ScreenPolicy, rnd: PyRandom): Set<number> {
|
||||
const relevant = prs.map((pr, index) => ({ ...pr, index })).filter(pr => pr.paidRelevant);
|
||||
const byCost = [...relevant].sort((a, b) => a.screenUsd - b.screenUsd);
|
||||
const half = relevant.length >> 1;
|
||||
const chosen = policy.kind === 'all' ? relevant : policy.kind === 'none' ? []
|
||||
: policy.kind === 'random' ? rnd.sample(relevant, Math.floor(relevant.length * policy.fraction))
|
||||
: policy.kind === 'cheapest-half' ? byCost.slice(0, half) : byCost.slice(half);
|
||||
return new Set(chosen.map(pr => pr.index));
|
||||
}
|
||||
|
||||
/**
|
||||
* Every merged PR lands in a train. Each paid-relevant PR carries a defect at
|
||||
* the screened or unscreened rate, and each gate attempt can also flake
|
||||
@@ -550,15 +576,9 @@ export function simulateTrains(input: TrainModelInput): TrainModelResult {
|
||||
const full = input.fullGateUsd ?? 93.14, bisect = input.bisectUsd ?? 6, rescreen = input.rescreenFactor ?? 1.3;
|
||||
const reps = input.reps ?? 400;
|
||||
const rnd = new PyRandom(input.seed ?? 1);
|
||||
const relevant = prs.map((pr, index) => ({ ...pr, index })).filter(pr => pr.paidRelevant);
|
||||
const byCost = [...relevant].sort((a, b) => a.screenUsd - b.screenUsd);
|
||||
const half = relevant.length >> 1;
|
||||
const t = { screen: 0, gates: 0, init: 0, initRed: 0, attRed: 0, bisects: 0, rebuildOnce: 0, stalls: 0 };
|
||||
for (let rep = 0; rep < reps; rep += 1) {
|
||||
const chosen = policy.kind === 'all' ? relevant : policy.kind === 'none' ? []
|
||||
: policy.kind === 'random' ? rnd.sample(relevant, Math.floor(relevant.length * policy.fraction))
|
||||
: policy.kind === 'cheapest-half' ? byCost.slice(0, half) : byCost.slice(half);
|
||||
const screened = new Set(chosen.map(pr => pr.index));
|
||||
const screened = screenedIndexes(prs, policy, rnd);
|
||||
const queue = prs.map((pr, index) => {
|
||||
const s = screened.has(index);
|
||||
if (s) t.screen += pr.screenUsd * rescreen;
|
||||
|
||||
+1
-1
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "gstack",
|
||||
"version": "1.91.137",
|
||||
"version": "1.91.138",
|
||||
"description": "Garry's Stack — Claude Code skills + fast headless browser. One repo, one install, entire AI engineering workflow.",
|
||||
"license": "MIT",
|
||||
"type": "module",
|
||||
|
||||
+271
-12
@@ -10,7 +10,12 @@
|
||||
* bun run scripts/gate-cost-replay.ts --runs <id,id,...> [--since YYYY-MM-DD --until YYYY-MM-DD | --prs 3201,3202]
|
||||
* [--ref origin/main] [--repo owner/name] [--cache .context/gate-cost-cache] [--window-days N]
|
||||
* [--usd-per-second R]
|
||||
* [--train-size 5] [--reps 400] [--seed 1] [--cases] [--json]
|
||||
* [--train-size 5] [--reps 400] [--seed 1] [--cases] [--history] [--json]
|
||||
*
|
||||
* --history (needs --since/--until) adds the red-head replay and the
|
||||
* READY-arrival train replay (lib/gate-cost-history.ts): it lists the
|
||||
* window's E2E Evals and Free Tests runs and PRs, downloads report-verdict
|
||||
* and trial-outcomes artifacts of red runs, and fetches PR heads by SHA.
|
||||
*
|
||||
* --runs names complete full-gate runs (108 rule cases + 24 judges). Bare
|
||||
* dates are UTC days; --until is inclusive. Cases without a recorded cost are
|
||||
@@ -23,14 +28,19 @@ import * as path from 'node:path';
|
||||
import { spawnSync } from 'node:child_process';
|
||||
import {
|
||||
SCREEN_LABEL, TODAY_WEEKLY_USD, AFTER_GSTA48_WEEKLY_USD, caseCostTable, changedLinesByFile, fullGateUsd, isSkillMd,
|
||||
parseEvalReport, parseTrialOutcomes, prSelectionCost, simulateTrains, summarizeGates, summarizeSelector,
|
||||
parseEvalReport, parseTrialOutcomes, prSelectionCost, screenSelection, simulateTrains, summarizeGates, summarizeSelector, todaySelection,
|
||||
type CaseCostEntry, type GateRecord, type GateSummary, type PrDiff, type PrSelectionCost, type ScreenPolicy,
|
||||
type SelectionDeps, type SelectorSummary, type TrainModelResult,
|
||||
} from '../lib/gate-cost';
|
||||
import { GH_HISTORY, downloadRunArtifacts, repoSlug } from './lib/ci-history';
|
||||
import {
|
||||
STAMP_FILES, classifyReds, headGreenAt, latencySummary, parseReportVerdict, prForRun, readyTimes, screenHit, simulateArrivalTrains,
|
||||
stampFreeSignature, summarizeRedHeads, SCREEN_CANARY_CASES,
|
||||
type ArrivalTrainResult, type CheckRun, type GatedRun, type HeadDiff, type LatencySummary, type ReadyRow, type RedHeadRow, type RedHeadSummary, type RedRun,
|
||||
} from '../lib/gate-cost-history';
|
||||
import { GH_CLIENT, GH_HISTORY, downloadRunArtifacts, repoSlug } from './lib/ci-history';
|
||||
import { computePaidCaseSelection } from './lib/paid-select';
|
||||
import { packageChangeOnlyVersion } from './test-pr-profile';
|
||||
import { E2E_TIERS, GLOBAL_TOUCHFILES, LLM_JUDGE_TOUCHFILES, matchGlob } from '../test/helpers/touchfiles';
|
||||
import { E2E_TIERS, E2E_TOUCHFILES, GLOBAL_TOUCHFILES, LLM_JUDGE_TOUCHFILES, matchGlob } from '../test/helpers/touchfiles';
|
||||
|
||||
const ROOT = path.resolve(import.meta.dir, '..');
|
||||
const DAY_MS = 86_400_000;
|
||||
@@ -55,12 +65,13 @@ export interface ReplayArgs {
|
||||
reps: number;
|
||||
seed: number;
|
||||
cases: boolean;
|
||||
history: boolean;
|
||||
json: boolean;
|
||||
}
|
||||
|
||||
export function parseArgs(argv: string[]): ReplayArgs {
|
||||
const args: ReplayArgs = { runs: [], prs: [], ref: 'origin/main', repo: '', cache: path.join(ROOT, '.context', 'gate-cost-cache'),
|
||||
trainSize: 5, reps: 400, seed: 1, cases: false, json: false };
|
||||
trainSize: 5, reps: 400, seed: 1, cases: false, history: false, json: false };
|
||||
const list = (value: string) => value.split(/[\s,]+/).filter(Boolean);
|
||||
const positive = (flag: string, value: string) => {
|
||||
const n = Number(value);
|
||||
@@ -71,6 +82,7 @@ export function parseArgs(argv: string[]): ReplayArgs {
|
||||
const flag = argv[i]!;
|
||||
if (flag === '--json') { args.json = true; continue; }
|
||||
if (flag === '--cases') { args.cases = true; continue; }
|
||||
if (flag === '--history') { args.history = true; continue; }
|
||||
const value = argv[i + 1];
|
||||
if (value === undefined || value.startsWith('--')) throw new Error(`${flag} needs a value`);
|
||||
i += 1;
|
||||
@@ -93,6 +105,7 @@ export function parseArgs(argv: string[]): ReplayArgs {
|
||||
if (args.prs.some(pr => !Number.isInteger(pr) || pr <= 0)) throw new Error('--prs takes PR numbers');
|
||||
if (args.prs.length && (args.since || args.until)) throw new Error('use either --prs or --since/--until, not both');
|
||||
if (!!args.since !== !!args.until) throw new Error('--since and --until go together');
|
||||
if (args.history && !args.since) throw new Error('--history needs a window: --since and --until');
|
||||
return args;
|
||||
}
|
||||
|
||||
@@ -184,20 +197,23 @@ function mergedPrs(args: ReplayArgs): { prs: MergedPr[]; windowDays: number | nu
|
||||
return { prs: args.prs.map(pr => all.get(pr)!), windowDays: args.windowDays ?? null };
|
||||
}
|
||||
|
||||
function prDiff(merge: MergedPr): PrDiff {
|
||||
const files = git(['diff', '--name-only', '--no-renames', `${merge.sha}^`, merge.sha]).split('\n').filter(Boolean);
|
||||
/** The PR diff the selectors see between two revisions. */
|
||||
function revisionDiff(pr: number, from: string, to: string): PrDiff {
|
||||
const files = git(['diff', '--name-only', '--no-renames', from, to]).split('\n').filter(Boolean);
|
||||
let packageVersionOnly = false;
|
||||
if (files.includes('package.json')) {
|
||||
const show = (rev: string) => spawnSync('git', ['show', `${rev}:package.json`], { cwd: ROOT, encoding: 'utf8', timeout: 30_000 });
|
||||
const before = show(`${merge.sha}^`), after = show(merge.sha);
|
||||
const before = show(from), after = show(to);
|
||||
packageVersionOnly = before.status === 0 && after.status === 0 && packageChangeOnlyVersion(before.stdout, after.stdout);
|
||||
}
|
||||
const skillMds = files.filter(isSkillMd);
|
||||
const skillMdChangedLines = skillMds.length >= 10
|
||||
? changedLinesByFile(git(['diff', '-U0', '--no-renames', `${merge.sha}^`, merge.sha, '--', ...skillMds], 120_000)) : {};
|
||||
return { pr: merge.pr, sha: merge.sha.slice(0, 8), files, packageVersionOnly, skillMdChangedLines };
|
||||
? changedLinesByFile(git(['diff', '-U0', '--no-renames', from, to, '--', ...skillMds], 120_000)) : {};
|
||||
return { pr, sha: to.slice(0, 8), files, packageVersionOnly, skillMdChangedLines };
|
||||
}
|
||||
|
||||
const prDiff = (merge: MergedPr): PrDiff => revisionDiff(merge.pr, `${merge.sha}^`, merge.sha);
|
||||
|
||||
/** The selectors in this checkout: today's PR profile (with R4's shadow) and the plain touchfile selection. */
|
||||
export function checkoutSelectionDeps(rootDir = ROOT): SelectionDeps {
|
||||
const gateCases = Object.keys(E2E_TIERS).filter(id => E2E_TIERS[id] === 'gate');
|
||||
@@ -236,6 +252,7 @@ export interface ReplayReport {
|
||||
prs: PrSelectionCost[];
|
||||
selection: { today: SelectorSummary; shadow: SelectorSummary; screenProvisional: SelectorSummary } | null;
|
||||
model: { screenScale: number; fullGateUsd: number; trainSize: number; reps: number; seed: number; rows: TrainModelResult[] } | null;
|
||||
history?: HistoryReport;
|
||||
}
|
||||
|
||||
export function buildReport(args: ReplayArgs, gates: GateRecord[], merges: ReturnType<typeof mergedPrs>, deps: SelectionDeps,
|
||||
@@ -333,6 +350,244 @@ export function formatReport(report: ReplayReport, options: { cases?: boolean }
|
||||
return out.join('\n');
|
||||
}
|
||||
|
||||
// --- History (--history): red-head replay and READY-arrival trains ---
|
||||
|
||||
const REPORT_ARTIFACT = /^report-verdict-a\d+$/;
|
||||
const TRIALS_ARTIFACT = /^trial-outcomes[A-Za-z0-9_-]*?-a\d+$/;
|
||||
const TRAIN_POLICIES: ScreenPolicy[] = [{ kind: 'all' }, { kind: 'random', fraction: 0.5 }, { kind: 'none' }];
|
||||
const GH_RUN_PAGE = 100;
|
||||
const HISTORY_GIT_TIMEOUT_MS = 120_000;
|
||||
const FETCH_TIMEOUT_MS = 300_000;
|
||||
|
||||
interface HistoryRun { id: number; sha: string; branch: string; createdAt: string; updatedAt: string; conclusion: string | null; prNumbers: number[] }
|
||||
interface PullRow { number: number; ref: string; head: string; createdAt: string; mergedAt: string | null; closedAt: string | null }
|
||||
type Merges = ReturnType<typeof mergedPrs>;
|
||||
|
||||
export interface HistoryReport {
|
||||
window: { since: string; until: string };
|
||||
runs: { evals: number; gated: number; red: number; free: number; unmapped: number };
|
||||
redRuns: RedRun[];
|
||||
redHeads: RedHeadRow[];
|
||||
redSummary: RedHeadSummary;
|
||||
perPr: Array<{ pr: number; paidRelevant: boolean; prCausedRed: boolean; screenedMiss: boolean }>;
|
||||
ready: ReadyRow[];
|
||||
baseline: LatencySummary & { gatesAfterReady: number; sources: Record<string, number> };
|
||||
trainParams: { trainSize: number; fillLimitHours: number; gateMinutes: number; bisectMinutes: number; redScreened: number; redUnscreened: number; reps: number; seed: number };
|
||||
trains: ArrivalTrainResult[];
|
||||
}
|
||||
|
||||
function tryGit(args: string[], timeout = HISTORY_GIT_TIMEOUT_MS): string | null {
|
||||
const result = spawnSync('git', args, { cwd: ROOT, encoding: 'utf8', timeout, maxBuffer: 256 * 1024 * 1024 });
|
||||
return result.status === 0 ? result.stdout : null;
|
||||
}
|
||||
|
||||
function cachedJson<T>(file: string, load: () => T): T {
|
||||
if (fs.existsSync(file)) return JSON.parse(fs.readFileSync(file, 'utf8')) as T;
|
||||
const value = load();
|
||||
fs.writeFileSync(file, JSON.stringify(value));
|
||||
return value;
|
||||
}
|
||||
|
||||
/** Completed runs of one workflow, one UTC day per query: GitHub caps a filtered run listing at 1,000 results. */
|
||||
function listRunsByDay(repo: string, workflow: string, since: Date, until: Date, event?: string): HistoryRun[] {
|
||||
const out: HistoryRun[] = [];
|
||||
for (let day = new Date(since.toISOString().slice(0, 10)); day < until; day = new Date(day.getTime() + DAY_MS)) {
|
||||
const date = day.toISOString().slice(0, 10);
|
||||
for (let page = 1; page <= 10; page += 1) {
|
||||
const query = [`status=completed`, `per_page=${GH_RUN_PAGE}`, `page=${page}`, `created=${date}..${date}`, ...(event ? [`event=${event}`] : [])];
|
||||
const body = GH_CLIENT.getJson(`repos/${repo}/actions/workflows/${workflow}/runs?${query.join('&')}`) as { workflow_runs?: any[] };
|
||||
const runs = Array.isArray(body?.workflow_runs) ? body.workflow_runs : [];
|
||||
out.push(...runs.map(run => ({ id: Number(run.id), sha: String(run.head_sha ?? ''), branch: String(run.head_branch ?? ''), createdAt: String(run.created_at ?? ''),
|
||||
updatedAt: String(run.updated_at ?? ''), conclusion: run.conclusion ?? null,
|
||||
prNumbers: Array.isArray(run.pull_requests) ? run.pull_requests.map((pr: any) => Number(pr.number)).filter(Number.isFinite) : [] })));
|
||||
if (runs.length < GH_RUN_PAGE) break;
|
||||
}
|
||||
}
|
||||
return out.filter(run => run.createdAt >= since.toISOString() && run.createdAt < until.toISOString());
|
||||
}
|
||||
|
||||
function listPulls(repo: string, since: Date): PullRow[] {
|
||||
const out: PullRow[] = [];
|
||||
for (let page = 1; page <= 30; page += 1) {
|
||||
const body = GH_CLIENT.getJson(`repos/${repo}/pulls?state=all&sort=updated&direction=desc&per_page=100&page=${page}`) as any[];
|
||||
if (!Array.isArray(body)) break;
|
||||
out.push(...body.map(p => ({ number: Number(p.number), ref: String(p.head?.ref ?? ''), head: String(p.head?.sha ?? ''), createdAt: String(p.created_at ?? ''),
|
||||
mergedAt: p.merged_at ?? null, closedAt: p.closed_at ?? null })));
|
||||
if (body.length < 100 || String(body[body.length - 1]?.updated_at ?? '') < since.toISOString()) break;
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
/** Artifacts of one run matching a name pattern, downloaded once (a .history marker records that the run was listed). */
|
||||
function runArtifactDirs(runId: number, repo: string, cache: string, match: RegExp): string[] {
|
||||
const dir = path.join(cache, String(runId));
|
||||
const marker = path.join(dir, `.history-${match.source.replace(/[^a-z]/gi, '')}`);
|
||||
if (!fs.existsSync(marker)) {
|
||||
downloadRunArtifacts({ repo, run: { id: runId }, cacheDir: cache, match: name => match.test(name), maxBytes: 16 * 1024 * 1024, fetcher: GH_HISTORY });
|
||||
fs.mkdirSync(dir, { recursive: true });
|
||||
fs.writeFileSync(marker, '');
|
||||
}
|
||||
return fs.readdirSync(dir).filter(name => match.test(name)).sort((a, b) => Number(a.split('-a').pop()) - Number(b.split('-a').pop())).map(name => path.join(dir, name));
|
||||
}
|
||||
|
||||
/** The run's report verdict over its attempts: the last attempt's verdict, every attempt's failures; null when no report exists. */
|
||||
function runVerdict(runId: number, repo: string, cache: string): { green: boolean; failures: GatedRun['failures'] } | null {
|
||||
const reports = runArtifactDirs(runId, repo, cache, REPORT_ARTIFACT).map(dir => path.join(dir, 'report.txt')).filter(file => fs.existsSync(file))
|
||||
.map(file => parseReportVerdict(fs.readFileSync(file, 'utf8')));
|
||||
if (!reports.length) return null;
|
||||
return { green: reports[reports.length - 1]!.verdict === 'GREEN', failures: reports.flatMap(r => r.failures) };
|
||||
}
|
||||
|
||||
/** The tested merge commit of a run, from its trial outcomes. */
|
||||
function testedMergeSha(runId: number, repo: string, cache: string): string | null {
|
||||
for (const dir of runArtifactDirs(runId, repo, cache, TRIALS_ARTIFACT)) {
|
||||
const file = path.join(dir, 'trial-outcomes.jsonl');
|
||||
const first = fs.existsSync(file) ? fs.readFileSync(file, 'utf8').split('\n').find(Boolean) : undefined;
|
||||
try { const sha = first ? JSON.parse(first).sha : null; if (typeof sha === 'string' && /^[0-9a-f]{40}$/.test(sha)) return sha; } catch { /* torn line */ }
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
/** Fetch commits this clone lacks by SHA (GitHub serves PR heads and merge refs that no branch holds anymore). */
|
||||
function ensureCommits(shas: string[]): void {
|
||||
const missing = [...new Set(shas)].filter(sha => tryGit(['cat-file', '-e', `${sha}^{commit}`], 30_000) === null);
|
||||
for (let i = 0; i < missing.length; i += 20) spawnSync('git', ['fetch', '-q', 'origin', ...missing.slice(i, i + 20)], { cwd: ROOT, timeout: FETCH_TIMEOUT_MS });
|
||||
}
|
||||
|
||||
function headDiff(sha: string, ref: string): HeadDiff | null {
|
||||
const mergeBase = tryGit(['merge-base', ref, sha])?.trim();
|
||||
if (!mergeBase) return null;
|
||||
const files = tryGit(['diff', '--name-only', '--no-renames', mergeBase, sha]);
|
||||
const diff = tryGit(['diff', '-U0', '--no-color', '--no-renames', '--no-ext-diff', mergeBase, sha, '--', '.', ...STAMP_FILES.map(file => `:(exclude)${file}`)]);
|
||||
return files === null || diff === null ? null : { ownSig: stampFreeSignature(diff), files: files.split('\n').filter(Boolean) };
|
||||
}
|
||||
|
||||
/** The base a red run tested: its merge commit's first parent, else main's first-parent tip when the run was created. */
|
||||
function testedBase(run: RedRun, args: ReplayArgs, repo: string): { base: string; source: RedHeadRow['baseSource'] } | null {
|
||||
const merge = testedMergeSha(run.runId, repo, args.cache);
|
||||
if (merge) {
|
||||
ensureCommits([merge]);
|
||||
const parents = (tryGit(['rev-list', '--parents', '-n1', merge]) ?? '').trim().split(' ').slice(1);
|
||||
if (parents.length === 2 && parents[1] === run.head) return { base: parents[0]!, source: 'tested-merge' };
|
||||
}
|
||||
const tip = tryGit(['rev-list', '-1', '--first-parent', `--before=${run.createdAt}`, args.ref])?.trim();
|
||||
return tip ? { base: tip, source: 'main-at-run' } : null;
|
||||
}
|
||||
|
||||
function collectHistory(args: ReplayArgs, repo: string, merges: Merges, body: Omit<ReplayReport, 'repo' | 'ref' | 'head'>, deps: SelectionDeps, gates: GateRecord[]): HistoryReport {
|
||||
const since = merges.since!, until = merges.until!;
|
||||
const key = `${since.toISOString()}_${until.toISOString()}`.replace(/[^0-9TZ_]/g, '');
|
||||
const cached = <T>(name: string, load: () => T) => cachedJson(path.join(args.cache, `history-${key}-${name}.json`), load);
|
||||
const evals = cached('evals', () => listRunsByDay(repo, 'evals.yml', since, until));
|
||||
const free = cached('free', () => listRunsByDay(repo, 'free-tests.yml', since, until, 'pull_request'));
|
||||
const pulls = cached('pulls', () => listPulls(repo, since));
|
||||
const prOf = new Map([...evals, ...free].map(run => [run.id, prForRun(run, pulls)]));
|
||||
const gated: GatedRun[] = evals.flatMap(run => {
|
||||
const pr = prOf.get(run.id) ?? null;
|
||||
if (run.conclusion === 'success') return [{ id: run.id, pr, head: run.sha, createdAt: run.createdAt, green: true, failures: [] }];
|
||||
const verdict = run.conclusion === 'failure' ? runVerdict(run.id, repo, args.cache) : null;
|
||||
return verdict ? [{ id: run.id, pr, head: run.sha, createdAt: run.createdAt, ...verdict }] : [];
|
||||
});
|
||||
const mergedSet = new Set(merges.prs.map(m => m.pr));
|
||||
const pullOf = new Map(pulls.map(p => [p.number, p]));
|
||||
const checkRuns = [...evals.map(run => ({ run, workflow: 'evals' as const })), ...free.map(run => ({ run, workflow: 'free' as const }))]
|
||||
.filter(({ run }) => mergedSet.has(prOf.get(run.id) ?? -1));
|
||||
const heads = [...new Set([...gated.filter(r => r.pr !== null).map(r => r.head), ...checkRuns.map(({ run }) => run.sha),
|
||||
...merges.prs.map(m => pullOf.get(m.pr)?.head ?? '')].filter(Boolean))];
|
||||
ensureCommits(heads);
|
||||
const diffs = cached('heads', () => Object.fromEntries(heads.map(sha => [sha, headDiff(sha, args.ref)])));
|
||||
const headMap = new Map(Object.entries(diffs).filter((e): e is [string, HeadDiff] => e[1] !== null));
|
||||
const caseFiles = new Map(gates.flatMap(g => g.trials.map(t => [t.caseId, t.file] as const)));
|
||||
const caseFile = (id: string) => caseFiles.get(id) ?? null;
|
||||
const inputsOf = (id: string) => /^test\/.+\.test\.ts$/.test(id)
|
||||
? [id, ...[...caseFiles].filter(([, file]) => file === id).flatMap(([c]) => E2E_TOUCHFILES[c] ?? [])]
|
||||
: [...(E2E_TOUCHFILES[id] ?? []), ...(caseFile(id) ? [caseFile(id)!] : [])];
|
||||
const redRuns = classifyReds(gated, {
|
||||
heads: headMap,
|
||||
delta: (from, to) => tryGit(['diff', '--name-only', '--no-renames', from, to])?.split('\n').filter(Boolean) ?? null,
|
||||
touches: (id, files) => files.filter(file => [...inputsOf(id), ...GLOBAL_TOUCHFILES].some(pattern => matchGlob(file, pattern))),
|
||||
});
|
||||
const redHeads: RedHeadRow[] = redRuns.filter(run => run.cause === 'pr').flatMap(run => {
|
||||
const tested = testedBase(run, args, repo);
|
||||
const mergeBase = tested && tryGit(['merge-base', tested.base, run.head])?.trim();
|
||||
if (!tested || !mergeBase) return [];
|
||||
const diff = revisionDiff(run.pr, mergeBase, run.head);
|
||||
const selection = screenSelection(diff, deps, todaySelection(diff, deps));
|
||||
return run.cases.filter(c => c.cause === 'pr').map(c => ({ caseId: c.caseId, kind: c.kind, pr: run.pr, runId: run.runId, head: run.head.slice(0, 8),
|
||||
base: tested.base.slice(0, 8), baseSource: tested.source, files: diff.files.length, hit: screenHit(c, selection, caseFile), canaryReasons: selection.canaryReasons }));
|
||||
});
|
||||
const selected = new Map(body.prs.map(row => [row.pr, row]));
|
||||
const merged = merges.prs.map(m => ({ pr: m.pr, paidRelevant: (selected.get(m.pr)?.today.usd ?? 0) > 0 }));
|
||||
const redSummary = summarizeRedHeads(redHeads, redRuns, merged);
|
||||
const ready = readyTimes(merges.prs.flatMap(m => {
|
||||
const pull = pullOf.get(m.pr);
|
||||
if (!pull?.mergedAt) return [];
|
||||
const prHeads = [...new Set([...checkRuns.filter(({ run }) => prOf.get(run.id) === m.pr).map(({ run }) => run.sha), pull.head])];
|
||||
return [{ pr: m.pr, mergedAt: pull.mergedAt, finalHead: pull.head, heads: prHeads.map(sha => ({ sha, ownSig: headMap.get(sha)?.ownSig ?? null })),
|
||||
gatedRuns: gated.filter(r => r.pr === m.pr).map(r => r.createdAt) }];
|
||||
}), headGreenAt(checkRuns.map(({ run, workflow }): CheckRun => ({ head: run.sha, workflow, createdAt: run.createdAt, completedAt: run.updatedAt, conclusion: run.conclusion }))));
|
||||
const withReady = ready.filter(row => row.ready !== null);
|
||||
const sources: Record<string, number> = {};
|
||||
for (const row of ready) sources[row.source] = (sources[row.source] ?? 0) + 1;
|
||||
const scale = body.model?.screenScale ?? 1;
|
||||
const arrivals = withReady.map(row => ({ pr: row.pr, readyMs: Date.parse(row.ready!), paidRelevant: (selected.get(row.pr)?.today.usd ?? 0) > 0,
|
||||
screenUsd: (selected.get(row.pr)?.screenProvisional.usd ?? 0) * scale }));
|
||||
const trainParams = { trainSize: args.trainSize, fillLimitHours: 3, gateMinutes: 45, bisectMinutes: 45, redScreened: 0.06, redUnscreened: 0.25, reps: args.reps, seed: args.seed };
|
||||
return {
|
||||
window: { since: since.toISOString(), until: until.toISOString() },
|
||||
runs: { evals: evals.length, gated: gated.length, red: gated.filter(r => !r.green).length, free: free.length, unmapped: evals.filter(run => prOf.get(run.id) === null).length },
|
||||
redRuns, redHeads, redSummary,
|
||||
perPr: merged.map(m => ({ ...m, prCausedRed: redSummary.prsWithPrRed.includes(m.pr), screenedMiss: redSummary.missedPrs.includes(m.pr) })),
|
||||
ready, baseline: { ...latencySummary(withReady.map(row => row.readyToMergeHours!)), gatesAfterReady: withReady.reduce((sum, row) => sum + row.gatesAfterReady, 0), sources },
|
||||
trainParams,
|
||||
trains: TRAIN_POLICIES.flatMap(policy => MODEL_OUTSIDE_RED.map(outsideRed => simulateArrivalTrains({ ...trainParams, arrivals, policy, outsideRed, windowDays: merges.windowDays! }))),
|
||||
};
|
||||
}
|
||||
|
||||
export function formatHistory(h: HistoryReport): string {
|
||||
const s = h.redSummary;
|
||||
const label = SCREEN_LABEL;
|
||||
const share = (n: number, d: number) => `${n} of ${d} (${pct(n, d)})`;
|
||||
const causes = (cause: string) => h.redRuns.filter(run => run.cause === cause).length;
|
||||
const out = [`History replays, E2E Evals and Free Tests runs created ${h.window.since} to ${h.window.until}:`,
|
||||
` E2E Evals runs ${h.runs.evals} (${h.runs.unmapped} not mapped to a PR), gated ${h.runs.gated}, red ${h.runs.red}: PR-caused ${causes('pr')}, outside ${causes('outside')}, unknown ${causes('unknown')}; Free Tests PR runs ${h.runs.free}`,
|
||||
' Classification (GSTA-47): per failed case, first match wins: transcript diagnosis; red in another PR within 72 h -> outside;',
|
||||
' no later gated run -> unknown; next gated run has the same own diff -> outside; the push to it changed the case\'s inputs',
|
||||
' (touchfiles, test file, harness globals) among either head\'s own files -> PR-caused; else outside. A run with any PR-caused case is PR-caused.',
|
||||
'', `Red-head replay (${label} on the failing head's diff from its tested base; canary set fixed before the replay: ${Object.values(SCREEN_CANARY_CASES).join(', ')}):`,
|
||||
` ${pad(['PR', 'run', 'head', 'base', 'files', 'case', 'kind', `${label} hit`], [6, 12, 9, 22, 5, 46, 6])}`];
|
||||
for (const row of h.redHeads) {
|
||||
out.push(` ${pad([`#${row.pr}`, row.runId, row.head, `${row.base} ${row.baseSource}`, row.files, row.caseId, row.kind, row.hit ?? 'MISSED'], [6, 12, 9, 22, 5, 46, 6])}`);
|
||||
}
|
||||
out.push(` ${label} hits by kind: ${Object.entries(s.byHit).map(([k, n]) => `${k} ${n}`).join(', ') || 'none'}`,
|
||||
` ${label} misses: ${share(s.missedPrs.length, s.prsWithPrRed.length)} merged PRs with a PR-caused red (${s.missedCases} of ${s.prCausedCases} case results)${s.missedPrs.length ? `: #${s.missedPrs.join(', #')}` : ''}`,
|
||||
` ${label} misses with an empty canary set: ${s.missedPrsWithoutCanary.length}${s.missedPrsWithoutCanary.length ? ` (#${s.missedPrsWithoutCanary.join(', #')})` : ''}`,
|
||||
` P-i (${label} misses over paid-relevant merged PRs): ${share(s.missedPrs.length, s.paidRelevantPrs)}`,
|
||||
` P-k (merged PRs with any PR-caused red run): ${share(s.prsWithPrRed.length, s.mergedPrs)}${s.prsWithPrRed.length ? `: #${s.prsWithPrRed.join(', #')}` : ''}; ${s.prCausedRuns} PR-caused red runs`,
|
||||
` PR-caused red runs on PRs that did not merge in the window (not counted): ${s.otherPrCausedRuns}`,
|
||||
'', `Per merged PR (paid-relevant: today's selector picks a case on the squash; red: any PR-caused red run; miss: a ${label} miss):`);
|
||||
for (let i = 0; i < h.perPr.length; i += 6) {
|
||||
out.push(` ${h.perPr.slice(i, i + 6).map(p => `#${p.pr} ${p.paidRelevant ? 'paid' : '----'} ${p.prCausedRed ? 'red ' : '--- '}${p.screenedMiss ? 'miss' : '----'}`).join(' ')}`);
|
||||
}
|
||||
const b = h.baseline;
|
||||
const hrs = (v: number) => v.toFixed(1);
|
||||
const p = h.trainParams;
|
||||
out.push('', 'READY-to-merge. READY = when the merged own diff (modulo main merges and stamps) first had green Free Tests and E2E Evals on a',
|
||||
' head, before the merge (fallback: the final head\'s green time; READY reports are not machine-readable).',
|
||||
` READY signal: ${Object.entries(b.sources).map(([k, n]) => `${k} ${n}`).join(', ')}`,
|
||||
` today (one PR per gate): ${b.prs} PRs, READY-to-merge p50 ${hrs(b.p50Hours)} h, p90 ${hrs(b.p90Hours)} h, mean ${hrs(b.meanHours)} h; ${b.gatesAfterReady} gated E2E Evals runs after READY`,
|
||||
` trains (${p.reps} reps, seed ${p.seed}; size ${p.trainSize}, fill limit ${p.fillLimitHours} h, screened first, unscreened alone when ${p.trainSize}+ wait; ${p.gateMinutes} min a gate,`,
|
||||
` ${p.bisectMinutes} min a bisect, one train gating at a time; defects ${p.redScreened * 100}% screened / ${p.redUnscreened * 100}% unscreened; a dropped PR rejoins at once):`,
|
||||
` ${pad(['policy', 'o', 'trains/w', 'occupancy', 'full', 'sizes 1..k per window', 'gates/w', 'drop+gate/w', 'rebld/w', 'stalls/w', 'p50 h', 'p90 h'], [10, 5, 8, 9, 5, 22, 7, 11, 7, 8, 6])} mean h`);
|
||||
for (const r of h.trains) {
|
||||
out.push(` ${pad([r.policy, r.outsideRed.toFixed(2), r.trainsWeekly.toFixed(2), r.meanOccupancy.toFixed(2), pct(r.fullTrainShare, 1), r.occupancy.map(n => n.toFixed(1)).join('/'),
|
||||
r.gateAttemptsWeekly.toFixed(2), r.attributedRebuildsWeekly.toFixed(2), r.rebuildOnceWeekly.toFixed(2), r.stallsWeekly.toFixed(2), hrs(r.readyToMerge.p50Hours), hrs(r.readyToMerge.p90Hours)],
|
||||
[10, 5, 8, 9, 5, 22, 7, 11, 7, 8, 6])} ${hrs(r.readyToMerge.meanHours)}`);
|
||||
}
|
||||
out.push(` "screened" follows the ${label} policy column; PRs with no paid case ride as screened.`);
|
||||
return out.join('\n');
|
||||
}
|
||||
|
||||
export function main(argv: string[]): number {
|
||||
let args: ReplayArgs;
|
||||
try { args = parseArgs(argv); } catch (error) {
|
||||
@@ -346,8 +601,12 @@ export function main(argv: string[]): number {
|
||||
const empty = gates.filter(g => g.trials.length === 0);
|
||||
if (empty.length) throw new Error(`run(s) ${empty.map(g => g.runId).join(', ')} have no trial outcomes (expired artifacts, or not a paid gate run)`);
|
||||
const merges = mergedPrs(args);
|
||||
const report: ReplayReport = { repo, ref: args.ref, head: git(['rev-parse', args.ref]).trim(), ...buildReport(args, gates, merges, checkoutSelectionDeps()) };
|
||||
console.log(args.json ? JSON.stringify(report, null, 2) : formatReport(report, { cases: args.cases }));
|
||||
const deps = checkoutSelectionDeps();
|
||||
const body = buildReport(args, gates, merges, deps);
|
||||
const report: ReplayReport = { repo, ref: args.ref, head: git(['rev-parse', args.ref]).trim(), ...body,
|
||||
...(args.history ? { history: collectHistory(args, repo, merges, body, deps, gates) } : {}) };
|
||||
const text = formatReport(report, { cases: args.cases }) + (report.history ? `\n\n${formatHistory(report.history)}` : '');
|
||||
console.log(args.json ? JSON.stringify(report, null, 2) : text);
|
||||
return 0;
|
||||
} catch (error) {
|
||||
console.error(`gate-cost-replay: ${(error as Error).message}`);
|
||||
|
||||
@@ -164,6 +164,8 @@ const SCANNER_EXEMPT: Record<string, string> = {
|
||||
'served-page JS talking to its own loopback server (same-origin relative fetch)',
|
||||
'design/src/compare.ts':
|
||||
'served-page JS talking to its own loopback server (relative ./api fetch)',
|
||||
'scripts/gate-cost-replay.ts':
|
||||
'developer-invoked read-only measurement (GSTA-49 T0): git fetch of gstack\'s own origin by SHA (old PR heads and tested merge commits) so past PR gates can be replayed; a download only, no gstack or user state leaves the machine',
|
||||
'scripts/ubicloud':
|
||||
'developer-invoked test infrastructure: Ubicloud API calls (VM create/show/destroy) with the developer\'s own token, the checkout streamed over SSH to that developer\'s own ephemeral VM, and toolchain downloads run on the VM; no installed-gstack user state is sent',
|
||||
// Skill prose templates: these render agent-executed instructions (the
|
||||
|
||||
+2
-1
@@ -123,7 +123,8 @@
|
||||
{ "file": "lib/autoplan-confirm.ts", "movedFrom": "(new: confirm-round plan delta, revision-qualified correlation and settled findings, field list 2 W5)" },
|
||||
{ "file": "lib/autoplan-public.ts", "movedFrom": "(new: machine-path-free public run copy and validate --public, field list 2 W6)" },
|
||||
{ "file": "lib/gate-cost.ts", "movedFrom": "(new: paid gate cost by model and token class, per-case costs, selection replays and the merge-train model, GSTA-49 T0)" },
|
||||
{ "file": "scripts/gate-cost-replay.ts", "movedFrom": "(new: CLI behind lib/gate-cost.ts: cached report artifacts, merged-PR replay and the train model table, GSTA-49 T0)" }
|
||||
{ "file": "scripts/gate-cost-replay.ts", "movedFrom": "(new: CLI behind lib/gate-cost.ts: cached report artifacts, merged-PR replay and the train model table, GSTA-49 T0)" },
|
||||
{ "file": "lib/gate-cost-history.ts", "movedFrom": "(new: red attribution, red-head screen replay, READY times and the READY-arrival train replay, GSTA-49 T0b)" }
|
||||
],
|
||||
"residualFiles": [
|
||||
{ "file": "scripts/test-free-shards.ts", "maxLines": 2361, "reason": "free lane policy after the shard engine took spawn, sandbox, logs, seeds and flags" },
|
||||
|
||||
+183
-1
@@ -6,7 +6,13 @@ import {
|
||||
PyRandom, SCREEN_LABEL, caseCostTable, changedLinesByFile, gateCostBreakdown, heuristicPreambleOnly, parseEvalReport,
|
||||
prSelectionCost, simulateTrains, summarizeGates, summarizeSelector, type GateRecord, type PrDiff, type SelectionDeps,
|
||||
} from '../lib/gate-cost';
|
||||
import { buildReport, checkoutSelectionDeps, formatReport, parseArgs, parseMergeLog, parseWindowDate, readGateRecord, type ReplayReport } from '../scripts/gate-cost-replay';
|
||||
import {
|
||||
DIAGNOSED_REDS, SCREEN_CANARY_CASES, classifyReds, headGreenAt, parseReportVerdict, percentile, pickTrain, prForRun, readyTimes, screenHit,
|
||||
simulateArrivalTrains, stampFreeSignature, summarizeRedHeads, type GatedRun, type HeadDiff, type RedHeadRow,
|
||||
} from '../lib/gate-cost-history';
|
||||
import {
|
||||
buildReport, checkoutSelectionDeps, formatHistory, formatReport, parseArgs, parseMergeLog, parseWindowDate, readGateRecord, type HistoryReport, type ReplayReport,
|
||||
} from '../scripts/gate-cost-replay';
|
||||
|
||||
/**
|
||||
* A one-run artifact tree, written to a temporary directory: tracked files
|
||||
@@ -219,3 +225,179 @@ describe('gate-cost-replay CLI', () => {
|
||||
expect(JSON.stringify(body.prs[0])).toContain('"screenProvisional"');
|
||||
});
|
||||
});
|
||||
|
||||
describe('red attribution (GSTA-47 rule)', () => {
|
||||
test('report verdicts: rule, judge (spaced id) and unbound-shard failure lines', () => {
|
||||
const text = ['[test:paid] VERDICT RED — lane gate/pr, attempt 1', '[test:paid] failures and split verdicts:',
|
||||
' ✗ tpa-absent-darwin rule FAIL assertion — expected [slice 14, attempt 1]',
|
||||
' ✗ canary/SKILL.md monitoring loop judge FAIL score 2 < 3',
|
||||
' ✗ test/skill-e2e-demo.test.ts rule shard NOT REPORTED [slice 3, attempt 1]'].join('\n');
|
||||
expect(parseReportVerdict(text)).toEqual({ verdict: 'RED', failures: [
|
||||
{ caseId: 'tpa-absent-darwin', kind: 'rule' }, { caseId: 'canary/SKILL.md monitoring loop', kind: 'judge' }, { caseId: 'test/skill-e2e-demo.test.ts', kind: 'rule' }] });
|
||||
expect(parseReportVerdict('[test:paid] VERDICT GREEN — lane gate/pr').verdict).toBe('GREEN');
|
||||
expect(parseReportVerdict('no report').verdict).toBe('NONE');
|
||||
});
|
||||
|
||||
test('the own-diff signature ignores hunk positions, index lines and the package.json version', () => {
|
||||
const diff = (hunk: string, index: string, version: string, body: string) => ['diff --git a/x.ts b/x.ts', `index ${index}`, '--- a/x.ts', '+++ b/x.ts', hunk, body,
|
||||
'diff --git a/package.json b/package.json', '--- a/package.json', '+++ b/package.json', '@@ -3 +3 @@', `- "version": "1.0.0",`, `+ "version": "${version}",`].join('\n');
|
||||
const base = stampFreeSignature(diff('@@ -1 +1 @@', 'aaa..bbb', '1.0.1', '+x'));
|
||||
expect(stampFreeSignature(diff('@@ -9 +9 @@', 'ccc..ddd', '1.0.2', '+x'))).toBe(base);
|
||||
expect(stampFreeSignature(diff('@@ -1 +1 @@', 'aaa..bbb', '1.0.1', '+y'))).not.toBe(base);
|
||||
expect(stampFreeSignature(diff('@@ -1 +1 @@', 'aaa..bbb', '1.0.1', '+x').split('\n').slice(0, 6).join('\n'))).toBe(base);
|
||||
});
|
||||
|
||||
const H = (sha: string) => `${sha}000000000000000000000000000000000000000`.slice(0, 40);
|
||||
const run = (id: number, pr: number | null, head: string, hour: number, failures: string[] = []): GatedRun =>
|
||||
({ id, pr, head: H(head), createdAt: new Date(Date.UTC(2026, 9, 5, hour)).toISOString(), green: failures.length === 0, failures: failures.map(caseId => ({ caseId, kind: 'rule' })) });
|
||||
const heads = new Map<string, HeadDiff>([
|
||||
[H('a1'), { ownSig: 's1', files: ['x/SKILL.md.tmpl'] }], [H('a2'), { ownSig: 's2', files: ['x/SKILL.md.tmpl', 'test/helpers/case-a.ts'] }],
|
||||
[H('b1'), { ownSig: 't1', files: ['docs/b.md'] }], [H('b2'), { ownSig: 't1', files: ['docs/b.md'] }],
|
||||
[H('c1'), { ownSig: 'u1', files: ['docs/c.md'] }], [H('c2'), { ownSig: 'u2', files: ['docs/c.md', 'docs/d.md'] }],
|
||||
]);
|
||||
const deps = {
|
||||
heads,
|
||||
delta: (from: string, to: string) => (from === H('a1') && to === H('a2') ? ['test/helpers/case-a.ts', 'lib/from-main.ts'] : ['docs/d.md']),
|
||||
touches: (caseId: string, files: string[]) => files.filter(file => file === `test/helpers/${caseId}.ts`),
|
||||
};
|
||||
|
||||
test('own push touching the case inputs is PR-caused; red elsewhere, same own diff, unrelated pushes and no later run are not', () => {
|
||||
const runs = [
|
||||
run(1, 10, 'a1', 0, ['case-a', 'case-shared']), run(2, 10, 'a2', 2),
|
||||
run(3, 11, 'b1', 1, ['case-b']), run(4, 11, 'b2', 3, ['case-b']),
|
||||
run(5, 12, 'c1', 4, ['case-shared', 'case-c']), run(6, 12, 'c2', 5),
|
||||
run(7, 13, 'zz', 6, ['case-z']), run(8, 14, 'zz', 7),
|
||||
];
|
||||
const reds = classifyReds(runs, deps, { prCaused: [], outside: [] });
|
||||
const byRun = Object.fromEntries(reds.map(r => [r.runId, r]));
|
||||
expect(byRun[1]!.cause).toBe('pr');
|
||||
expect(byRun[1]!.cases.map(c => [c.caseId, c.cause, c.reason])).toEqual([
|
||||
['case-a', 'pr', 'fixed by own push: test/helpers/case-a.ts'], ['case-shared', 'outside', 'also red in #12']]);
|
||||
expect(byRun[3]!.cases[0]).toMatchObject({ cause: 'outside', reason: 'still red with the same own diff' });
|
||||
expect(byRun[4]!.cases[0]).toMatchObject({ cause: 'unknown', reason: 'no later gated run' });
|
||||
expect(byRun[5]!.cases.map(c => c.reason)).toEqual(['also red in #10', 'passed next after an unrelated push']);
|
||||
expect(byRun[5]!.cause).toBe('outside');
|
||||
expect(byRun[7]!.cause).toBe('unknown');
|
||||
expect(classifyReds([run(9, 15, 'q1', 0, [])].map(r => ({ ...r, green: false })), deps)[0]).toMatchObject({ cause: 'outside', cases: [] });
|
||||
});
|
||||
|
||||
test('transcript diagnoses win, and the 72-hour window bounds "red elsewhere"', () => {
|
||||
const own = DIAGNOSED_REDS.prCaused[0], outside = DIAGNOSED_REDS.outside[0];
|
||||
const reds = classifyReds([run(own, 20, 'a1', 0, ['case-x']), run(outside, 21, 'a1', 0, ['case-y']), run(30, 22, 'b1', 1, ['case-x'])], deps);
|
||||
expect(reds.find(r => r.runId === own)!.cases[0]!.reason).toBe('diagnosed own edit (GSTA-45)');
|
||||
expect(reds.find(r => r.runId === outside)!.cases[0]!.reason).toBe('diagnosed main defect (fix wave 3)');
|
||||
const far = classifyReds([run(40, 30, 'a1', 0, ['case-a']), run(41, 30, 'a2', 1), run(42, 31, 'b1', 80, ['case-a'])], deps, { prCaused: [], outside: [] });
|
||||
expect(far.find(r => r.runId === 40)!.cause).toBe('pr');
|
||||
});
|
||||
});
|
||||
|
||||
describe('red-head replay', () => {
|
||||
const selection = (e2e: string[], canary: boolean, global = false) => ({ e2e, global, canary, canaryReasons: canary ? ['resolver x'] : [], preambleOnly: [] });
|
||||
const files: Record<string, string> = { 'case-a': 'test/skill-e2e-demo.test.ts', 'case-b': 'test/skill-e2e-other.test.ts', 'auq-format-gate': 'test/auq.test.ts', 'judge x': 'test/skill-llm-eval.test.ts' };
|
||||
const caseFile = (id: string) => files[id] ?? null;
|
||||
|
||||
test('judges always count; touchfiles, unbound shards by file, and canary cases only when the canary set is added', () => {
|
||||
expect(screenHit({ caseId: 'qa/SKILL.md workflow', kind: 'judge' }, selection([], false), caseFile)).toBe('judge');
|
||||
expect(screenHit({ caseId: 'judge x', kind: 'rule' }, selection([], false), caseFile)).toBe('judge');
|
||||
expect(screenHit({ caseId: 'case-a', kind: 'rule' }, selection(['case-a'], false), caseFile)).toBe('touchfiles');
|
||||
expect(screenHit({ caseId: 'test/skill-e2e-demo.test.ts', kind: 'rule' }, selection(['case-a'], false), caseFile)).toBe('touchfiles');
|
||||
expect(screenHit({ caseId: 'case-b', kind: 'rule' }, selection(['case-a'], false), caseFile)).toBeNull();
|
||||
expect(screenHit({ caseId: 'case-b', kind: 'rule' }, selection([], false, true), caseFile)).toBe('global');
|
||||
expect(screenHit({ caseId: 'auq-format-gate', kind: 'rule' }, selection([], true), caseFile)).toBe('canary');
|
||||
expect(screenHit({ caseId: 'auq-format-gate', kind: 'rule' }, selection([], false), caseFile)).toBeNull();
|
||||
expect(Object.values(SCREEN_CANARY_CASES)).toContain('auq-format-gate');
|
||||
const gate = checkoutSelectionDeps().gateCases;
|
||||
for (const id of Object.values(SCREEN_CANARY_CASES)) expect(gate).toContain(id);
|
||||
});
|
||||
|
||||
test('summaries count merged PRs only: P-k numerator, screened misses, and misses without the canary set', () => {
|
||||
const row = (pr: number, hit: RedHeadRow['hit']): RedHeadRow => ({ pr, runId: pr, head: 'h', base: 'b', baseSource: 'tested-merge', files: 1, caseId: 'c', kind: 'rule', hit, canaryReasons: [] });
|
||||
const redRuns = [10, 11, 12, 99].map(pr => ({ runId: pr, pr, head: 'h', createdAt: '', nextRunId: null, cause: 'pr' as const, cases: [] }));
|
||||
const s = summarizeRedHeads([row(10, null), row(10, 'touchfiles'), row(11, 'canary'), row(12, 'judge'), row(99, null)], redRuns,
|
||||
[10, 11, 12, 13].map(pr => ({ pr, paidRelevant: pr !== 13 })));
|
||||
expect(s).toMatchObject({ mergedPrs: 4, paidRelevantPrs: 3, prsWithPrRed: [10, 11, 12], prCausedRuns: 3, prCausedCases: 4, missedPrs: [10], missedCases: 1,
|
||||
missedPrsWithoutCanary: [10, 11], otherPrCausedRuns: 1, byHit: { missed: 1, touchfiles: 1, canary: 1, judge: 1 } });
|
||||
});
|
||||
});
|
||||
|
||||
describe('READY arrivals and trains', () => {
|
||||
const at = (hour: number) => new Date(Date.UTC(2026, 9, 10, hour)).toISOString();
|
||||
|
||||
test('READY is the first green head with the merged own diff before the merge; fallback the final head', () => {
|
||||
const green = headGreenAt([
|
||||
{ head: 'h1', workflow: 'free', createdAt: at(0), completedAt: at(1), conclusion: 'success' },
|
||||
{ head: 'h1', workflow: 'evals', createdAt: at(0), completedAt: at(2), conclusion: 'success' },
|
||||
{ head: 'h2', workflow: 'free', createdAt: at(3), completedAt: at(4), conclusion: 'success' },
|
||||
{ head: 'h2', workflow: 'evals', createdAt: at(3), completedAt: at(4), conclusion: 'failure' },
|
||||
{ head: 'h2', workflow: 'evals', createdAt: at(5), completedAt: at(6), conclusion: 'success' },
|
||||
{ head: 'h3', workflow: 'free', createdAt: at(7), completedAt: at(8), conclusion: 'success' },
|
||||
{ head: 'h4', workflow: 'free', createdAt: at(7), completedAt: at(8), conclusion: 'failure' },
|
||||
]);
|
||||
expect([...green]).toEqual([['h1', at(2)], ['h2', at(6)], ['h3', at(8)], ['h4', null]]);
|
||||
const rows = readyTimes([
|
||||
{ pr: 1, mergedAt: at(10), finalHead: 'h3', heads: [{ sha: 'h1', ownSig: 'old' }, { sha: 'h2', ownSig: 's' }, { sha: 'h3', ownSig: 's' }], gatedRuns: [at(0), at(3), at(5), at(7)] },
|
||||
{ pr: 2, mergedAt: at(10), finalHead: 'h1', heads: [{ sha: 'h1', ownSig: null }], gatedRuns: [] },
|
||||
{ pr: 3, mergedAt: at(7), finalHead: 'h3', heads: [{ sha: 'h3', ownSig: 's' }], gatedRuns: [] },
|
||||
], green);
|
||||
expect(rows.map(r => [r.pr, r.ready, r.source, r.readyToMergeHours, r.gatesAfterReady])).toEqual([
|
||||
[1, at(6), 'final-own-diff', 4, 1], [2, at(2), 'final-head', 8, 0], [3, null, 'none', null, 0]]);
|
||||
expect([percentile([5, 1, 3, 2, 4], 0.5), percentile([5, 1, 3, 2, 4], 0.9), percentile([], 0.5)]).toEqual([3, 5, 0]);
|
||||
});
|
||||
|
||||
test('train composition: unscreened PRs form their own train at five, else screened first, else the fill limit', () => {
|
||||
const item = (enqueuedMs: number, screened: boolean) => ({ enqueuedMs, screened });
|
||||
const pool = [item(0, true), item(1, true), ...[2, 3, 4, 5, 6].map(t => item(t, false))];
|
||||
expect(pickTrain(pool, 10, 5, 100)).toEqual(pool.slice(2));
|
||||
const mixed = [item(5, false), item(0, true), item(1, false), item(2, true), item(3, true)];
|
||||
expect(pickTrain(mixed, 10, 5, 100)?.map(x => x.screened)).toEqual([true, true, true, false, false]);
|
||||
expect(pickTrain([item(0, false), item(1, true)], 99, 5, 100)).toBeNull();
|
||||
expect(pickTrain([item(0, false), item(1, true)], 100, 5, 100)?.map(x => x.enqueuedMs)).toEqual([1, 0]);
|
||||
});
|
||||
|
||||
test('deterministic controls: a fill-limit train, a full train, and a repeated defect red', () => {
|
||||
const arrivals = (n: number, paidRelevant: boolean) => Array.from({ length: n }, (_, i) => ({ pr: i + 1, readyMs: 0, paidRelevant, screenUsd: 1 }));
|
||||
const base = { policy: { kind: 'all' } as const, windowDays: 7, outsideRed: 0, redScreened: 0, redUnscreened: 0, reps: 3 };
|
||||
const full = simulateArrivalTrains({ ...base, arrivals: arrivals(5, true) });
|
||||
expect(full).toMatchObject({ trainsWeekly: 1, meanOccupancy: 5, fullTrainShare: 1, gateAttemptsWeekly: 1, bisectsWeekly: 0, occupancy: [0, 0, 0, 0, 1] });
|
||||
expect(full.readyToMerge).toMatchObject({ prs: 15, p50Hours: 0.75, p90Hours: 0.75 });
|
||||
const short = simulateArrivalTrains({ ...base, arrivals: arrivals(2, false) });
|
||||
expect(short.readyToMerge.p90Hours).toBe(3.75);
|
||||
const red = simulateArrivalTrains({ ...base, arrivals: arrivals(2, true), redScreened: 1 });
|
||||
expect(red).toMatchObject({ trainsWeekly: 2, gateAttemptsWeekly: 3, bisectsWeekly: 2, attributedRebuildsWeekly: 1, rebuildOnceWeekly: 0, occupancy: [0, 2, 0, 0, 0] });
|
||||
expect(red.readyToMerge.p50Hours).toBe(8.25);
|
||||
const unscreened = simulateArrivalTrains({ ...base, policy: { kind: 'none' }, arrivals: arrivals(2, true), redUnscreened: 1 });
|
||||
expect(unscreened.bisectsWeekly).toBe(2);
|
||||
expect(simulateArrivalTrains({ ...base, arrivals: arrivals(7, true), outsideRed: 0.15, redUnscreened: 0.25, seed: 4 }))
|
||||
.toEqual(simulateArrivalTrains({ ...base, arrivals: arrivals(7, true), outsideRed: 0.15, redUnscreened: 0.25, seed: 4 }));
|
||||
});
|
||||
|
||||
test('runs map to PRs by pull_requests, else by head branch while the PR was open', () => {
|
||||
const pulls = [{ number: 5, ref: 'feat', createdAt: at(0), mergedAt: at(5), closedAt: at(5) }, { number: 6, ref: 'feat', createdAt: at(8), mergedAt: null, closedAt: null }];
|
||||
expect(prForRun({ branch: 'x', createdAt: at(1), prNumbers: [9] }, pulls)).toBe(9);
|
||||
expect(prForRun({ branch: 'feat', createdAt: at(2), prNumbers: [] }, pulls)).toBe(5);
|
||||
expect(prForRun({ branch: 'feat', createdAt: at(9), prNumbers: [] }, pulls)).toBe(6);
|
||||
expect(prForRun({ branch: 'feat', createdAt: at(7), prNumbers: [] }, pulls)).toBeNull();
|
||||
});
|
||||
|
||||
test('--history needs a window, and every history line naming the screen carries the screen-provisional label', () => {
|
||||
expect(() => parseArgs(['--runs', '1', '--history'])).toThrow('--history needs a window');
|
||||
expect(parseArgs(['--runs', '1', '--history', '--since', '2026-10-03', '--until', '2026-10-04']).history).toBe(true);
|
||||
const history: HistoryReport = {
|
||||
window: { since: at(0), until: at(24) }, runs: { evals: 3, gated: 2, red: 1, free: 4, unmapped: 0 },
|
||||
redRuns: [{ runId: 1, pr: 10, head: 'h', createdAt: at(0), nextRunId: 2, cause: 'pr', cases: [{ caseId: 'case-a', kind: 'rule', cause: 'pr', reason: 'fixed by own push: x' }] }],
|
||||
redHeads: [{ pr: 10, runId: 1, head: 'h', base: 'b', baseSource: 'tested-merge', files: 2, caseId: 'case-a', kind: 'rule', hit: null, canaryReasons: [] }],
|
||||
redSummary: summarizeRedHeads([{ pr: 10, runId: 1, head: 'h', base: 'b', baseSource: 'tested-merge', files: 2, caseId: 'case-a', kind: 'rule', hit: null, canaryReasons: [] }],
|
||||
[{ runId: 1, pr: 10, head: 'h', createdAt: at(0), nextRunId: 2, cause: 'pr', cases: [] }], [{ pr: 10, paidRelevant: true }, { pr: 11, paidRelevant: false }]),
|
||||
perPr: [{ pr: 10, paidRelevant: true, prCausedRed: true, screenedMiss: true }, { pr: 11, paidRelevant: false, prCausedRed: false, screenedMiss: false }],
|
||||
ready: [], baseline: { prs: 2, p50Hours: 1, p90Hours: 2, meanHours: 1.5, gatesAfterReady: 1, sources: { 'final-own-diff': 2 } },
|
||||
trainParams: { trainSize: 5, fillLimitHours: 3, gateMinutes: 45, bisectMinutes: 45, redScreened: 0.06, redUnscreened: 0.25, reps: 3, seed: 1 },
|
||||
trains: [simulateArrivalTrains({ arrivals: [{ pr: 10, readyMs: 0, paidRelevant: true, screenUsd: 1 }], policy: { kind: 'all' }, windowDays: 7, reps: 3 })],
|
||||
};
|
||||
const text = formatHistory(history);
|
||||
expect(text).toContain('P-k (merged PRs with any PR-caused red run): 1 of 2 (50.0%): #10');
|
||||
expect(text).toContain('misses: 1 of 1 (100.0%) merged PRs with a PR-caused red (1 of 1 case results): #10');
|
||||
const screenLines = text.split('\n').filter(line => /\bscreen\b/i.test(line));
|
||||
expect(screenLines.length).toBeGreaterThan(2);
|
||||
for (const line of screenLines) expect(line).toContain(SCREEN_LABEL);
|
||||
});
|
||||
});
|
||||
Reference in new issue
Block a user