mirror of
https://github.com/garrytan/gstack.git
synced 2026-11-15 12:57:22 +01:00
v1.93.22.0 stamp: merge main e0531db793 into #3887 (land)
This commit is contained in:
garrytan committed
commit
142dbd0676
104 files changed
+3822
-385
No files matched your search
+21
-3
@@ -39,7 +39,7 @@ export function preregTemplate(title = 'Eval preregistration'): string {
|
||||
<!-- One line per metric: name, the pass bar, the direction, and what a miss means for the decision. -->
|
||||
|
||||
## Arms
|
||||
<!-- One line per arm: the arm identity first (\`<model>\` or \`<model>#<loadout>\`, the model in lib/pricing.ts, e.g. \`claude-opus-5-5#no-gstack\`), settings, and the per-item cost from \`gstack-eval-plan arms\`. -->
|
||||
<!-- One line per arm: the arm identity first (\`<model>\` or \`<model>#<loadout>\`, the model in lib/pricing.ts, e.g. \`claude-opus-5-5#no-gstack\`), settings, and the per-item cost from \`gstack-eval-plan arms\`. Models under test are frontier models (today \`claude-opus-5-5\`, \`gpt-6.1-sol\`) plus at most one named low-cost primary; a Haiku or Sonnet arm only with a stated tier question. -->
|
||||
|
||||
## Comparison
|
||||
<!-- \`stack: loaded\` when any arm loads gstack or another skill stack (review variants, preamble loadouts, skill diets, a model panel under a stack); otherwise \`stack: none\` and a \`why:\` line. -->
|
||||
@@ -292,10 +292,16 @@ export interface DryRunResult {
|
||||
reservation_fits_cap?: { ok: boolean; cap: number; max_input_tokens: number; per_arm: Record<string, number | null> };
|
||||
/** Present when the caller linted the arms against .gstack/eval-models.json (lib/eval-models.ts). */
|
||||
model_rules?: { ok: boolean; source: string; refused: string[] };
|
||||
/** R54: Haiku and Sonnet are not models under test; arms on them alone are refused unless the plan asks about that tier. */
|
||||
frontier: { ok: boolean; small_tier: string[]; tier_question: string | null; recost_usd: Record<string, number> };
|
||||
};
|
||||
per_item_usd: Record<string, number | null>; excluded_billing: readonly string[]; pass: boolean; stub: 'stub-model (no network, $0)';
|
||||
}
|
||||
export interface DryRunOptions { input: string; arms: string[]; fields?: string[]; outputBudget?: number; retryPrompt?: string; maxRows?: number; cap?: number; modelRules?: ModelRulesCheck }
|
||||
export interface DryRunOptions { input: string; arms: string[]; fields?: string[]; outputBudget?: number; retryPrompt?: string; maxRows?: number; cap?: number; modelRules?: ModelRulesCheck; tierQuestion?: string }
|
||||
|
||||
/** Today's frontier models under test (R54, docs/TESTING_INTERNALS.md#frontier-models-under-test); a small-tier plan is re-costed on these. */
|
||||
export const FRONTIER_MODELS = ['claude-opus-5-5', 'gpt-6.1-sol'] as const;
|
||||
export const isSmallTierModel = (model: string) => /(^|[-/])(haiku|sonnet)(-|$)/i.test(model);
|
||||
|
||||
/** Four bytes per token, the estimator every gstack admission uses. */
|
||||
export const estimateTokens = (text: string) => Math.ceil(Buffer.byteLength(text, 'utf8') / 4);
|
||||
@@ -335,9 +341,12 @@ export function dryRun(o: DryRunOptions): DryRunResult {
|
||||
checks.reservation_fits_cap = { ok: Object.values(perArm).every(v => v === null || v <= o.cap!), cap: o.cap, max_input_tokens: maxInput, per_arm: perArm };
|
||||
}
|
||||
if (o.modelRules) checks.model_rules = { ok: o.modelRules.ok, source: o.modelRules.source, refused: o.modelRules.errors.map(e => e.message) };
|
||||
const smallTier = o.arms.filter(a => isSmallTierModel(armModel(a)));
|
||||
const recost = Object.fromEntries(FRONTIER_MODELS.map(m => [m, rows * o.arms.length * estimateCostUsd({ input: avgIn, output: budget }, m)]));
|
||||
checks.frontier = { ok: smallTier.length < o.arms.length || !!o.tierQuestion?.trim(), small_tier: smallTier, tier_question: o.tierQuestion?.trim() || null, recost_usd: recost };
|
||||
return {
|
||||
schema_version: 1, input: o.input, rows, bad_rows: bad, est_input_tokens: inputTokens, est_output_tokens: rows * budget, arms: o.arms, checks, per_item_usd: perItem,
|
||||
excluded_billing: EXCLUDED_BILLING_CATEGORIES, pass: rows > 0 && bad.length === 0 && checks.price_table.ok && checks.output_budget.ok && checks.retry_prompt.ok && (checks.reservation_fits_cap?.ok ?? true) && (checks.model_rules?.ok ?? true), stub: 'stub-model (no network, $0)',
|
||||
excluded_billing: EXCLUDED_BILLING_CATEGORIES, pass: rows > 0 && bad.length === 0 && checks.price_table.ok && checks.output_budget.ok && checks.retry_prompt.ok && (checks.reservation_fits_cap?.ok ?? true) && (checks.model_rules?.ok ?? true) && checks.frontier.ok, stub: 'stub-model (no network, $0)',
|
||||
};
|
||||
}
|
||||
|
||||
@@ -349,6 +358,7 @@ export function renderDryRun(r: DryRunResult): string {
|
||||
`CHECK: output-budget ${r.checks.output_budget.ok ? 'ok' : 'too small'} (${r.checks.output_budget.budget} tokens; reasoning floor ${r.checks.output_budget.floor})`,
|
||||
`CHECK: retry-prompt ${r.checks.retry_prompt.ok ? `ok — ${JSON.stringify(r.checks.retry_prompt.text)}` : 'missing (pass --retry-prompt <file> with the exact text the run will send)'}`,
|
||||
...(r.checks.reservation_fits_cap ? [renderReservationCheck(r.checks.reservation_fits_cap, r.checks.output_budget.budget)] : []),
|
||||
...(r.checks.frontier.small_tier.length ? [renderFrontierCheck(r.checks.frontier, r.arms.length)] : []),
|
||||
...(r.checks.model_rules ? [`CHECK: model-rules ${r.checks.model_rules.ok ? 'ok' : `refused ${r.checks.model_rules.refused.join('; ') || 'invalid rules'}`} (source ${r.checks.model_rules.source})`] : []),
|
||||
...Object.entries(r.per_item_usd).map(([m, usd]) => `ESTIMATE: ${m} per_item=${usd === null ? 'unknown (no price)' : `$${usd.toFixed(4)}`} (estimate: excludes ${r.excluded_billing.length} billing categories, see lib/pricing.ts)`),
|
||||
`DRY_RUN_RESULT: ${r.pass ? 'pass' : 'fail'}`,
|
||||
@@ -356,6 +366,13 @@ export function renderDryRun(r: DryRunResult): string {
|
||||
return lines.join('\n') + '\n';
|
||||
}
|
||||
|
||||
function renderFrontierCheck(c: DryRunResult['checks']['frontier'], arms: number): string {
|
||||
const verdict = !c.ok ? 'refused (every arm is Haiku or Sonnet; pass --tier-question "<what the preregistration asks about that tier>" or test a frontier model)'
|
||||
: c.tier_question ? `ok (tier question: ${JSON.stringify(c.tier_question)})` : 'warn (Haiku or Sonnet arms are not models under test)';
|
||||
const recost = Object.entries(c.recost_usd).map(([m, usd]) => `${m} $${usd.toFixed(2)}`).join(', ');
|
||||
return `CHECK: frontier ${verdict} small_tier=${c.small_tier.join(',')}; all ${arms} arm(s) re-costed on frontier prices: ${recost} (docs/TESTING_INTERNALS.md#frontier-models-under-test)`;
|
||||
}
|
||||
|
||||
function renderReservationCheck(c: NonNullable<DryRunResult['checks']['reservation_fits_cap']>, budget: number): string {
|
||||
const worst = Math.max(0, ...Object.values(c.per_arm).filter((v): v is number => v !== null));
|
||||
return `CHECK: reservation-fits-cap ${c.ok ? 'ok' : 'exceeds'} ($${worst.toFixed(4)} ${c.ok ? '<=' : '>'} cap $${c.cap.toFixed(2)}) one call = ${c.max_input_tokens} in + ${budget} out tokens`;
|
||||
@@ -376,6 +393,7 @@ export interface PilotStartOptions { out: string; cap: number; n: number; arms:
|
||||
export function startPilot(o: PilotStartOptions): { record: PilotRecord; error?: { code: ResultCodeName; message: string } } {
|
||||
const dry = JSON.parse(fs.readFileSync(o.dryRun, 'utf8')) as DryRunResult;
|
||||
if (!dry.pass) return { record: null as unknown as PilotRecord, error: { code: 'EVAL_DRY_RUN_FAILED', message: `${o.dryRun} did not pass; the pilot never starts on a failed dry run` } };
|
||||
if (o.arms.length && o.arms.every(a => isSmallTierModel(armModel(a))) && !dry.checks.frontier?.tier_question) return { record: null as unknown as PilotRecord, error: { code: 'EVAL_DRY_RUN_FAILED', message: `every pilot arm is Haiku or Sonnet and ${o.dryRun} records no --tier-question (R54, docs/TESTING_INTERNALS.md#frontier-models-under-test)` } };
|
||||
const tokens = o.tokens ?? { input: dry.rows ? Math.ceil(dry.est_input_tokens / dry.rows) : DEFAULT_TOKENS.input, output: dry.checks.output_budget.budget };
|
||||
const seat = o.seatMetered ?? [];
|
||||
const stray = seat.filter(a => !o.arms.includes(a));
|
||||
|
||||
@@ -70,7 +70,7 @@ export interface ScorecardOptions {
|
||||
export const DEFAULTS = {
|
||||
base: 'main', sinceDays: 7, gateWorkflow: 'evals.yml', gateEvent: 'workflow_dispatch', freeWorkflow: 'free-tests.yml',
|
||||
trainPrefix: 'train/', readyMarker: 'READY #\\d+|SHIP_RECEIPT:|gstack-ship-receipt',
|
||||
passMarker: '"passes"\\s*:\\s*(\\d+)|\\bpasses=(\\d+)|\\bREVIEW_PASS(?:ES)?[:= ]\\s*(\\d+)', passCap: 3, maxArtifactRuns: 60,
|
||||
passMarker: '"passes"\\s*:\\s*(\\d+)|\\bpasses=(\\d+)|\\bREVIEW_PASS(?:ES)?[:= ]\\s*(\\d+)', passCap: 2, maxArtifactRuns: 60,
|
||||
} as const;
|
||||
|
||||
export const CREW_ASSOCIATIONS = new Set(['OWNER', 'MEMBER', 'COLLABORATOR']);
|
||||
@@ -399,7 +399,7 @@ export const BENCHMARKS: Record<string, string> = {
|
||||
merged: 'several merges a day, every day; outside contributions land in the same week',
|
||||
lead_time: 'median under 24h, p90 under 3 days (DORA elite: change lead time under a day)',
|
||||
ready_to_merge: 'under 2h from READY to merge; every merged PR carries a READY comment',
|
||||
review_passes: 'median 2 passes, none at the cap (a pass that finds nothing new is the exit; what is left at the cap is named in the PR body)',
|
||||
review_passes: 'median 1 pass, few at the cap (a pass that finds nothing blocking is the exit; a PR at the cap ships with what is left named in the PR body)',
|
||||
gates: 'green on 95%+ of runs, cancelled near 0, cost known for every run',
|
||||
cost_per_merge: 'known, stable or falling week over week',
|
||||
red_train: 'under 5% of trains red (each red train costs a bisect and a rebuild)',
|
||||
|
||||
+107
-23
@@ -29,15 +29,23 @@
|
||||
* omits (bun files that fail to load), the run is red and nothing is rerun.
|
||||
* The first report is restored at the --junit path when the run ends; each
|
||||
* rerun's report is kept beside it as <name>.rerun-<n>.xml.
|
||||
* - Quarantine (lib/quarantine.ts) is a larger attempt budget, never a skip:
|
||||
* a live quarantined test is listed and ledgered on every run and never
|
||||
* counts as a pass. When every test a whole-command attempt failed is
|
||||
* quarantined and nothing else is wrong, the command may run again, up to
|
||||
* QUARANTINE_ATTEMPTS in total; the run is green only on an attempt with the
|
||||
* runner's full green verdict, so a quarantined failure never turns a red
|
||||
* attempt green. One that fails every attempt is a hard break. A quarantined
|
||||
* id missing from a run that included its file is red.
|
||||
*/
|
||||
import * as fs from 'fs';
|
||||
import * as path from 'path';
|
||||
import { spawn as spawnChild, spawnSync } from 'child_process';
|
||||
import { finalAttempt, JunitError, readJunit, type JunitReport, type JunitRunner, type JunitTest } from './junit';
|
||||
import { finalAttempt, JunitError, parseTestId, readJunit, type Attempt, type JunitReport, type JunitRunner, type JunitTest } from './junit';
|
||||
import { redactFindingSpans } from './redact-engine';
|
||||
import { resolveStateRoot } from './state-root';
|
||||
|
||||
export type LedgerKind = 'flaky-pass' | 'rerun-fail';
|
||||
export type LedgerKind = 'flaky-pass' | 'rerun-fail' | 'quarantined-pass' | 'quarantined-fail' | 'hard-break';
|
||||
|
||||
export interface LedgerLine {
|
||||
ts: string;
|
||||
@@ -64,6 +72,8 @@ export interface RunOptions {
|
||||
maxReruns: number;
|
||||
/** 'command' (default): rerun the whole command. 'test': rerun the failed test alone when the first run is proven complete. */
|
||||
rerunScope?: 'command' | 'test';
|
||||
/** Live quarantined test ids (lib/quarantine.ts liveQuarantine): up to QUARANTINE_ATTEMPTS attempts, never a skip. */
|
||||
quarantined?: ReadonlySet<string>;
|
||||
cwd?: string;
|
||||
env?: NodeJS.ProcessEnv;
|
||||
/** Runs one attempt; resolves with its exit status and the tail of its combined output. */
|
||||
@@ -73,7 +83,7 @@ export interface RunOptions {
|
||||
|
||||
export interface TestOutcome {
|
||||
test: JunitTest;
|
||||
status: 'flaky-pass' | 'fail' | 'rerun-fail';
|
||||
status: 'flaky-pass' | 'fail' | 'rerun-fail' | 'quarantined-pass' | 'quarantined-fail' | 'hard-break';
|
||||
attempts: string[];
|
||||
reason?: string;
|
||||
}
|
||||
@@ -88,6 +98,8 @@ export interface RunResult {
|
||||
commandExit: number;
|
||||
outcomes: TestOutcome[];
|
||||
reruns: number;
|
||||
/** Outcomes of quarantined tests (listed, never counted as passes). */
|
||||
quarantined: number;
|
||||
/** Why the rerun was the whole command (the default, or why a requested targeted rerun was refused); absent for targeted reruns. */
|
||||
wholeRerun?: string;
|
||||
ledger: LedgerLine[];
|
||||
@@ -243,7 +255,7 @@ function rerunProblem(again: Attempted & { report?: JunitReport }, first: JunitR
|
||||
}
|
||||
|
||||
/** A rerun clears a test only when its command exited 0 and the test ran once and passed on its first attempt. */
|
||||
function rerunVerdict(again: Attempted & { report?: JunitReport; error?: JunitError }, t: JunitTest, problem?: string): { status: 'flaky-pass' | 'rerun-fail'; attempts: string[]; reason?: string } {
|
||||
function rerunVerdict(again: Attempted & { report?: JunitReport; error?: JunitError }, t: JunitTest, problem?: string): { status: 'flaky-pass' | 'rerun-fail'; attempts: Attempt[]; reason?: string } {
|
||||
const match = again.report?.tests.find(x => x.id === t.id);
|
||||
if (!match) return { status: 'rerun-fail', attempts: [...t.attempts, 'fail'], reason: again.error ? `rerun: ${again.error.message}` : 'rerun: the report does not contain this test' };
|
||||
const attempts = [...t.attempts, ...match.attempts];
|
||||
@@ -254,11 +266,47 @@ function rerunVerdict(again: Attempted & { report?: JunitReport; error?: JunitEr
|
||||
return { status: 'flaky-pass', attempts };
|
||||
}
|
||||
|
||||
export const QUARANTINE_ATTEMPTS = 3;
|
||||
|
||||
const PASSING: ReadonlySet<TestOutcome['status']> = new Set(['flaky-pass', 'quarantined-pass']);
|
||||
|
||||
type Ran = Attempted & { report?: JunitReport; error?: JunitError };
|
||||
|
||||
/** The runner's own full green verdict for a whole-command attempt: exit 0, no failed or missing test, nothing counted outside the report. */
|
||||
function fullGreen(again: Ran, first: JunitReport): boolean {
|
||||
return !!again.report && !again.signal && again.exit === 0 && !rerunProblem(again, first, true);
|
||||
}
|
||||
|
||||
/**
|
||||
* Whether a red whole-command attempt may be followed by another: only within
|
||||
* the quarantine budget, and only when every test it failed is a live
|
||||
* quarantined test that ran once and nothing else is wrong (a kill, a missing
|
||||
* report, failures outside the report, a first-run test missing).
|
||||
*/
|
||||
function mayRunAgain(again: Ran, first: JunitReport, quarantined: ReadonlySet<string>, attemptsSoFar: number): boolean {
|
||||
if (attemptsSoFar >= QUARANTINE_ATTEMPTS || again.signal || !again.report) return false;
|
||||
const failedNow = again.report.tests.filter(x => finalAttempt(x) === 'fail');
|
||||
if (!failedNow.length || !failedNow.every(x => quarantined.has(x.id) && x.executions === 1 && x.attempts.length === 1)) return false;
|
||||
if (unreportedFailures(first.runner, again.output, failedNow.length)) return false;
|
||||
const seen = new Set(again.report.tests.map(x => x.id));
|
||||
return first.tests.every(x => seen.has(x.id));
|
||||
}
|
||||
|
||||
/** A test's attempts over the first run and the given whole-command reruns; a rerun that did not report it counts as a failure. */
|
||||
const history = (t: JunitTest, runs: Ran[]): Attempt[] =>
|
||||
[...t.attempts, ...runs.flatMap(r => r.report?.tests.find(x => x.id === t.id)?.attempts ?? ['fail' as const])];
|
||||
|
||||
/** A quarantined test passes only when it never failed or an attempt with the runner's full verdict cleared it. */
|
||||
const quarantinedStatus = (attempts: string[], cleared: boolean): 'quarantined-pass' | 'quarantined-fail' | 'hard-break' =>
|
||||
cleared || !attempts.includes('fail') ? 'quarantined-pass'
|
||||
: attempts.length >= QUARANTINE_ATTEMPTS && attempts.every(a => a === 'fail') ? 'hard-break' : 'quarantined-fail';
|
||||
|
||||
export async function runFlaky(opts: RunOptions): Promise<RunResult> {
|
||||
const cwd = opts.cwd ?? process.cwd();
|
||||
const env = { ...(opts.env ?? process.env), PLAYWRIGHT_JUNIT_INCLUDE_RETRIES: '1' };
|
||||
const now = opts.now ?? (() => new Date());
|
||||
const empty = { tests: 0, outcomes: [], reruns: 0, ledger: [] };
|
||||
const quarantined = opts.quarantined ?? new Set<string>();
|
||||
const empty = { tests: 0, outcomes: [], reruns: 0, quarantined: 0, ledger: [] };
|
||||
const first = await attempt(opts, opts.command, cwd, env);
|
||||
if (first.signal) return { exitCode: 3, code: 'FLAKY_COMMAND_KILLED', message: `the command was killed by ${first.signal}`, commandExit: first.exit, ...empty };
|
||||
if (!first.report) return { exitCode: 3, code: first.error!.code, message: first.error!.message, commandExit: first.exit, ...empty };
|
||||
@@ -282,15 +330,39 @@ export async function runFlaky(opts: RunOptions): Promise<RunResult> {
|
||||
const prov = provenance(cwd, env);
|
||||
const line = (t: JunitTest, kind: LedgerKind, by: LedgerLine['rerun_by'], attempts: string[], scope?: LedgerLine['rerun_scope']): LedgerLine =>
|
||||
({ ts: now().toISOString(), runner: 'junit', kind, id: safeText(t.id), file: safeText(t.file), rerun_by: by, ...(scope ? { rerun_scope: scope } : {}), attempts, ...prov, excerpt: safeExcerpt(t.excerpt) });
|
||||
const ledger: LedgerLine[] = runnerFlaky.map(t => line(t, 'flaky-pass', 'runner', t.attempts));
|
||||
const outcomes: TestOutcome[] = runnerFlaky.map(t => ({ test: t, status: 'flaky-pass' as const, attempts: t.attempts }));
|
||||
const isQ = (t: JunitTest) => quarantined.has(t.id);
|
||||
const outcomes: TestOutcome[] = [];
|
||||
const ledger: LedgerLine[] = [];
|
||||
for (const t of runnerFlaky.filter(x => !isQ(x))) {
|
||||
outcomes.push({ test: t, status: 'flaky-pass', attempts: t.attempts });
|
||||
ledger.push(line(t, 'flaky-pass', 'runner', t.attempts));
|
||||
}
|
||||
const settleQ = (t: JunitTest, attempts: string[], cleared: boolean, reason?: string, scope?: LedgerLine['rerun_scope']) => {
|
||||
const status = quarantinedStatus(attempts, cleared);
|
||||
const why = status === 'hard-break' ? `failed all ${attempts.length} attempts: a hard break, not a flake` : status === 'quarantined-fail' ? reason : undefined;
|
||||
outcomes.push({ test: t, status, attempts, ...(why ? { reason: why } : {}) });
|
||||
ledger.push(line(t, status, 'gstack-flaky', attempts, scope));
|
||||
};
|
||||
const settle = (t: JunitTest, attempts: string[], reason: string) => isQ(t) ? settleQ(t, attempts, false, reason) : outcomes.push({ test: t, status: 'fail', attempts, reason });
|
||||
const present = new Set(report.tests.map(t => t.id));
|
||||
const included = new Set(report.tests.map(t => `${t.file}::${t.project}`));
|
||||
for (const id of quarantined) {
|
||||
const p = parseTestId(id);
|
||||
if (!p || present.has(id) || !included.has(`${p.file}::${p.project}`)) continue;
|
||||
const t: JunitTest = { id, file: p.file, project: p.project, path: p.title.split(' > '), attempts: [], executions: 0, excerpt: [] };
|
||||
outcomes.push({ test: t, status: 'quarantined-fail', attempts: [], reason: 'quarantined, and missing from a run that included its file' });
|
||||
ledger.push(line(t, 'quarantined-fail', 'gstack-flaky', []));
|
||||
}
|
||||
const others = failed.filter(t => !isQ(t)).length;
|
||||
const eligible: JunitTest[] = [];
|
||||
for (const t of failed) {
|
||||
if (t.executions > 1) outcomes.push({ test: t, status: 'fail', attempts: t.attempts, reason: `it ran ${t.executions} times in this report and one failed; not rerun` });
|
||||
else if (t.attempts.length > 1) outcomes.push({ test: t, status: 'fail', attempts: t.attempts, reason: `the runner already retried (${t.attempts.length} attempts)` });
|
||||
else if (failed.length > opts.maxReruns) outcomes.push({ test: t, status: 'fail', attempts: t.attempts, reason: `${failed.length} tests failed, more than --max-reruns ${opts.maxReruns}; not rerun` });
|
||||
const why = t.executions > 1 ? `it ran ${t.executions} times in this report and one failed; not rerun`
|
||||
: t.attempts.length > 1 ? `the runner already retried (${t.attempts.length} attempts)`
|
||||
: !isQ(t) && others > opts.maxReruns ? `${others} tests failed, more than --max-reruns ${opts.maxReruns}; not rerun` : undefined;
|
||||
if (why) settle(t, t.attempts, why);
|
||||
else eligible.push(t);
|
||||
}
|
||||
const passedQ = report.tests.filter(t => isQ(t) && finalAttempt(t) !== 'fail' && t.attempts.some(a => a !== 'skip'));
|
||||
const wholeRerun = !eligible.length ? undefined
|
||||
: opts.rerunScope !== 'test' ? 'the default; --rerun-scope test reruns the failed test alone'
|
||||
: incompleteRun(report.runner, first.output, first.exit, opts.command, runnerFlaky.length) ?? undefined;
|
||||
@@ -304,30 +376,42 @@ export async function runFlaky(opts: RunOptions): Promise<RunResult> {
|
||||
return again;
|
||||
};
|
||||
const record = (t: JunitTest, v: ReturnType<typeof rerunVerdict>, scope: 'test' | 'command') => {
|
||||
if (isQ(t)) return settleQ(t, v.attempts, v.status === 'flaky-pass', v.reason, scope);
|
||||
outcomes.push({ test: t, status: v.status, attempts: v.attempts, ...(v.reason ? { reason: v.reason } : {}) });
|
||||
ledger.push(line(t, v.status, 'gstack-flaky', v.attempts, scope));
|
||||
};
|
||||
const runs: Ran[] = [];
|
||||
try {
|
||||
if (wholeRerun && outcomes.some(o => o.status === 'fail')) {
|
||||
for (const t of eligible) outcomes.push({ test: t, status: 'fail', attempts: t.attempts, reason: `not rerun: the run is already red, and a rerun would repeat the whole command (${wholeRerun})` });
|
||||
if (wholeRerun && outcomes.some(o => !PASSING.has(o.status))) {
|
||||
for (const t of eligible) settle(t, t.attempts, `not rerun: the run is already red, and a rerun would repeat the whole command (${wholeRerun})`);
|
||||
} else if (wholeRerun) {
|
||||
const again = await rerun(opts.command);
|
||||
const problem = rerunProblem(again, report, true);
|
||||
for (const t of eligible) record(t, rerunVerdict(again, t, problem), 'command');
|
||||
do {
|
||||
runs.push(await rerun(opts.command));
|
||||
} while (!fullGreen(runs.at(-1)!, report) && mayRunAgain(runs.at(-1)!, report, quarantined, 1 + runs.length));
|
||||
const decider = runs.at(-1)!;
|
||||
const problem = rerunProblem(decider, report, true);
|
||||
for (const t of eligible) record(t, rerunVerdict(decider, { ...t, attempts: history(t, runs.slice(0, -1)) }, problem), 'command');
|
||||
} else if (eligible.length) {
|
||||
let interrupted = false;
|
||||
for (const t of eligible) {
|
||||
if (interrupted) { outcomes.push({ test: t, status: 'fail', attempts: t.attempts, reason: 'not rerun: an earlier rerun was killed' }); continue; }
|
||||
const again = await rerun([...opts.command, ...rerunSelection(report.runner as 'playwright' | 'bun', t)]);
|
||||
interrupted = !!again.signal;
|
||||
record(t, rerunVerdict(again, t, rerunProblem(again, report, false)), 'test');
|
||||
if (interrupted) { settle(t, t.attempts, 'not rerun: an earlier rerun was killed'); continue; }
|
||||
let v: ReturnType<typeof rerunVerdict> = { status: 'rerun-fail', attempts: t.attempts };
|
||||
for (let budget = isQ(t) ? QUARANTINE_ATTEMPTS - t.attempts.length : 1; budget > 0 && v.status !== 'flaky-pass' && !interrupted; budget--) {
|
||||
const again = await rerun([...opts.command, ...rerunSelection(report.runner as 'playwright' | 'bun', t)]);
|
||||
interrupted = !!again.signal;
|
||||
v = rerunVerdict(again, { ...t, attempts: v.attempts }, rerunProblem(again, report, false));
|
||||
}
|
||||
record(t, v, 'test');
|
||||
}
|
||||
}
|
||||
} finally {
|
||||
if (reruns) fs.writeFileSync(junit, firstXml);
|
||||
}
|
||||
const red = outcomes.some(o => o.status !== 'flaky-pass');
|
||||
return { exitCode: red ? 1 : 0, tests: report.tests.length, commandExit: first.exit, outcomes, reruns, ...(wholeRerun && reruns ? { wholeRerun } : {}), ledger };
|
||||
const green = runs.length ? fullGreen(runs.at(-1)!, report) : reruns > 0 || first.exit === 0;
|
||||
for (const t of passedQ) settleQ(t, history(t, runs), green, 'the attempt that decided the run was not a full green run', runs.length ? 'command' : undefined);
|
||||
const red = !green || outcomes.some(o => !PASSING.has(o.status));
|
||||
const q = outcomes.filter(o => o.status === 'quarantined-pass' || o.status === 'quarantined-fail' || o.status === 'hard-break').length;
|
||||
return { exitCode: red ? 1 : 0, tests: report.tests.length, commandExit: first.exit, outcomes, reruns, quarantined: q, ...(wholeRerun && reruns ? { wholeRerun } : {}), ledger };
|
||||
}
|
||||
|
||||
export function ledgerPath(flag: string | undefined, env: NodeJS.ProcessEnv = process.env): string {
|
||||
@@ -352,9 +436,9 @@ export function renderResult(r: RunResult, ledger: string): string[] {
|
||||
const out: string[] = [];
|
||||
for (const o of r.outcomes) out.push(`FLAKY: ${o.status} ${safeText(o.test.id)} attempts=${o.attempts.join(',')}${o.reason ? ` (${o.reason})` : ''}`);
|
||||
const flaky = r.outcomes.filter(o => o.status === 'flaky-pass').length;
|
||||
const failed = r.outcomes.length - flaky;
|
||||
const failed = r.outcomes.filter(o => !PASSING.has(o.status)).length;
|
||||
if (r.wholeRerun) out.push(`FLAKY_RERUN: scope=command (${r.wholeRerun})`);
|
||||
out.push(`FLAKY_RESULT: tests=${r.tests} failed=${failed} flaky=${flaky} reruns=${r.reruns} command_exit=${r.commandExit} verdict=${r.exitCode === 0 ? 'pass' : 'fail'} ledger=${r.ledger.length ? ledger : 'none'}`);
|
||||
out.push(`FLAKY_RESULT: tests=${r.tests} failed=${failed} flaky=${flaky} quarantined=${r.quarantined} reruns=${r.reruns} command_exit=${r.commandExit} verdict=${r.exitCode === 0 ? 'pass' : 'fail'} ledger=${r.ledger.length ? ledger : 'none'}`);
|
||||
return out;
|
||||
}
|
||||
|
||||
|
||||
@@ -137,6 +137,14 @@ function excerptOf(node: XmlNode | undefined): string[] {
|
||||
|
||||
export const testId = (file: string, project: string, path: string[]) => `${file}::${project}::${path.join(' > ')}`;
|
||||
|
||||
/** The parts of a test id, or null when it is not `<file>::<project>::<title>`. */
|
||||
export function parseTestId(id: string): { file: string; project: string; title: string } | null {
|
||||
const i = id.indexOf('::');
|
||||
const j = i < 0 ? -1 : id.indexOf('::', i + 2);
|
||||
if (i <= 0 || j < 0 || j + 2 >= id.length) return null;
|
||||
return { file: id.slice(0, i), project: id.slice(i + 2, j), title: id.slice(j + 2) };
|
||||
}
|
||||
|
||||
function readCase(tc: XmlNode, runner: JunitRunner): Omit<JunitTest, 'id' | 'executions'> & { hidden: boolean } {
|
||||
const suite = tc.parent!;
|
||||
let file: string, project = '', path: string[];
|
||||
|
||||
+81
-15
@@ -27,6 +27,7 @@
|
||||
* Incidents: p0-4-pipe-hidden-failure (`false | tail` read as green),
|
||||
* p0-4-windows-lane-unrun (a Windows-only change merged with its lane unrun).
|
||||
*/
|
||||
import { spawnSync } from 'node:child_process';
|
||||
import * as fs from 'node:fs';
|
||||
import * as path from 'node:path';
|
||||
import { baseInputs, sha256 } from '../context';
|
||||
@@ -43,17 +44,21 @@ import { verifyFromRun } from '../../run-evidence';
|
||||
import { matchesAny } from '../../ship-policy';
|
||||
import type { CheckDef, CheckResult, PregateContext, RemoteObligation } from '../types';
|
||||
|
||||
/** The test command with `{files}`; detected from the repo when the config does not name one. */
|
||||
/** The runner a `.py`-only selection gets; it needs the pytest module importable by python3 (`pytestMissing`). */
|
||||
const PYTEST = 'python3 -m pytest {files}';
|
||||
|
||||
/** The test command with `{files}`; detected from the repo when the config does not name one. The repo's bun, vitest or jest claims JS/TS files only. */
|
||||
export function detectRunner(ctx: Pick<PregateContext, 'config' | 'repoRoot'>, files: readonly string[]): string | null {
|
||||
if (ctx.config.runner) return ctx.config.runner;
|
||||
const has = (f: string) => fs.existsSync(path.join(ctx.repoRoot, f));
|
||||
let pkg: { devDependencies?: Record<string, string>; dependencies?: Record<string, string>; scripts?: Record<string, string> } = {};
|
||||
try { pkg = JSON.parse(fs.readFileSync(path.join(ctx.repoRoot, 'package.json'), 'utf8')); } catch { /* no package.json */ }
|
||||
const deps = { ...pkg.dependencies, ...pkg.devDependencies };
|
||||
if (has('bunfig.toml') || has('bun.lock') || has('bun.lockb')) return 'bun test {files}';
|
||||
if (deps.vitest) return 'npx vitest run {files}';
|
||||
if (deps.jest) return 'npx jest {files}';
|
||||
if (files.every(f => /\.py$/.test(f)) && files.length) return 'python3 -m pytest {files}';
|
||||
const jsts = files.every(f => /\.[cm]?[jt]sx?$/.test(f));
|
||||
if (jsts && (has('bunfig.toml') || has('bun.lock') || has('bun.lockb'))) return 'bun test {files}';
|
||||
if (jsts && deps.vitest) return 'npx vitest run {files}';
|
||||
if (jsts && deps.jest) return 'npx jest {files}';
|
||||
if (files.every(f => /\.py$/.test(f)) && files.length) return PYTEST;
|
||||
if (files.every(f => /\.[cm]?js$/.test(f)) && files.length) return 'node --test {files}';
|
||||
if (files.every(f => /\.[cm]?[jt]sx?$/.test(f)) && files.length && Bun.which('bun')) return 'bun test {files}';
|
||||
return null;
|
||||
@@ -113,6 +118,18 @@ export function dartRunner(repoRoot: string, root: string, env: NodeJS.ProcessEn
|
||||
return { runner: `${fvm ? 'fvm ' : ''}${tool} test {files}` };
|
||||
}
|
||||
|
||||
/**
|
||||
* The tool `python3 -m pytest` lacks: `python3` off the PATH of `env`, or `pytest` when that python3, run from the repo root as the
|
||||
* runner is, cannot import it (a `pytest` script on PATH may belong to another interpreter); null when both are there, or when the
|
||||
* probe itself times out or dies, so the run decides.
|
||||
*/
|
||||
function pytestMissing(repoRoot: string, env: NodeJS.ProcessEnv): string | null {
|
||||
const python = Bun.which('python3', { PATH: env.PATH ?? '' });
|
||||
if (!python) return 'python3';
|
||||
const r = spawnSync(python, ['-c', 'import importlib.util, sys; sys.exit(3 if importlib.util.find_spec("pytest") is None else 0)'], { cwd: repoRoot, env, stdio: 'ignore', timeout: 10_000 });
|
||||
return r.status === 3 ? 'pytest' : null;
|
||||
}
|
||||
|
||||
/**
|
||||
* `cargo test -p <name>` for a crate (cargo runs a crate's tests, never a file list), or the missing tool when cargo is
|
||||
* not on PATH. The name comes from the repo's Cargo.toml and reaches a shell and the `{files}` expansion: a name outside
|
||||
@@ -163,6 +180,23 @@ function packageRunner(repoRoot: string, dir: string, env: NodeJS.ProcessEnv): {
|
||||
return { runner: `${PM_EXEC[pm]} ${tool} {files}` };
|
||||
}
|
||||
|
||||
/**
|
||||
* Whether a JS/TS test imports a value from Playwright's runner in a statement that starts a line: `import … from '@playwright/test'`
|
||||
* (one line or several), a bare import, or `require`. A type-only import (`import type …`, or every specifier `type X`) and a string or
|
||||
* fixture body that mentions the import mid-line do not count.
|
||||
*/
|
||||
function importsPlaywright(text: string): boolean {
|
||||
const code = text.replace(/\/\*[\s\S]*?\*\/|\/\/[^\n]*/g, '');
|
||||
if (/^[\t ]*(?:import[\t ]*|(?:const|let|var)\b[^=\n]*=[\t ]*require\([\t ]*)['"]@playwright\/test['"]/m.test(code)) return true;
|
||||
for (const m of code.matchAll(/^[\t ]*import(?=[\s{*])([^'"`;]*?)\bfrom[\t ]*['"]@playwright\/test['"]/gm)) {
|
||||
const spec = m[1]!.trim();
|
||||
if (!/^type(?=[\s{*])/.test(spec) && spec.replace(/[{}]/g, '').split(',').some(s => s.trim() && !/^type\s/.test(s.trim()))) return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
/** Playwright's own text when its browsers are not installed on this machine. */
|
||||
const PLAYWRIGHT_NO_BROWSERS = /^.*(?:Executable doesn't exist at|download new browsers).*$/m;
|
||||
|
||||
/** The run plan: the groups that can run, the files no runner claims, tools missing from PATH, and `.dart` files with no pubspec.yaml above. */
|
||||
export interface RunPlan { groups: RunGroup[]; unclaimed: string[]; missing: string[]; noPubspec: string[] }
|
||||
|
||||
@@ -171,8 +205,11 @@ export interface RunPlan { groups: RunGroup[]; unclaimed: string[]; missing: str
|
||||
* Dart tests run from their own package root (a sub-package's imports resolve
|
||||
* against its own pubspec); Rust tests run per crate as `cargo test -p <name>`
|
||||
* from the crate's workspace root; JS/TS tests owned by a nested package that
|
||||
* declares vitest or jest run from that package; the rest go to the detected
|
||||
* runner from the repo root. Files no runner claims (a tool missing from PATH,
|
||||
* declares vitest or jest run from that package; a JS/TS test that imports
|
||||
* `@playwright/test` runs through `playwright test` from its nearest package;
|
||||
* the rest go to the detected runner from the repo root, split by language
|
||||
* when no one runner fits them all (`.mjs` + `.py` runs both), and pytest only
|
||||
* when python3 can import it. Files no runner claims (a tool missing from PATH,
|
||||
* a `.dart` file outside any package, a `.rs` file outside any crate or in a
|
||||
* crate whose name Cargo would reject, no detected runner) are `unclaimed`: the lane is `incomplete`, never a pass.
|
||||
*/
|
||||
@@ -182,6 +219,7 @@ export function planRuns(ctx: PregateContext, files: readonly string[]): RunPlan
|
||||
const dart = new Map<string, string[]>();
|
||||
const cargo = new Map<string, { name: string; cwd: string; files: string[] }>();
|
||||
const packages = new Map<string, string[]>();
|
||||
const playwright = new Map<string, string[]>();
|
||||
const seen = new Map<string, string | null>();
|
||||
const seenPackage = new Map<string, string | null>();
|
||||
let model: CargoModel | null = null;
|
||||
@@ -204,6 +242,7 @@ export function planRuns(ctx: PregateContext, files: readonly string[]): RunPlan
|
||||
continue;
|
||||
}
|
||||
const pkg = /\.[cm]?[jt]sx?$/.test(f) ? nearestRoot(ctx.repoRoot, f, 'package.json', seenPackage) : null;
|
||||
if (pkg !== null && importsPlaywright(readRepoFile(ctx.repoRoot, f) ?? '')) { add(playwright, pkg, under(pkg, f)); continue; }
|
||||
if (pkg) {
|
||||
if (!pkgRunners.has(pkg)) pkgRunners.set(pkg, packageRunner(ctx.repoRoot, pkg, ctx.env));
|
||||
const r = pkgRunners.get(pkg);
|
||||
@@ -225,12 +264,24 @@ export function planRuns(ctx: PregateContext, files: readonly string[]): RunPlan
|
||||
if (r) missing.add(r.missing);
|
||||
unclaimed.push(...c.files.map(f => (c.cwd ? `${c.cwd}/${f}` : f)));
|
||||
}
|
||||
if (rest.length) {
|
||||
const runner = detectRunner(ctx, rest);
|
||||
if (runner) groups.push({ runner, cwd: '', files: rest });
|
||||
else unclaimed.push(...rest);
|
||||
const language = (f: string) => (/\.py$/.test(f) ? 'py' : /\.[cm]?js$/.test(f) ? 'js' : /\.[cm]?[jt]sx?$/.test(f) ? 'ts' : f);
|
||||
const parts = !rest.length ? [] : detectRunner(ctx, rest) ? [rest] : [...Map.groupBy(rest, language).values()];
|
||||
const byRunner = new Map<string, string[]>();
|
||||
for (const part of parts) {
|
||||
const runner = detectRunner(ctx, part);
|
||||
const lacking = runner === PYTEST ? pytestMissing(ctx.repoRoot, ctx.env) : null;
|
||||
if (lacking) missing.add(lacking);
|
||||
if (runner && !lacking) for (const f of part) add(byRunner, runner, f);
|
||||
else unclaimed.push(...part);
|
||||
}
|
||||
for (const [runner, list] of byRunner) groups.push({ runner, cwd: '', files: list });
|
||||
for (const [root, list] of [...packages].sort(([a], [b]) => a.localeCompare(b))) groups.push({ runner: (pkgRunners.get(root) as { runner: string }).runner, cwd: root, files: list });
|
||||
for (const [root, list] of [...playwright].sort(([a], [b]) => a.localeCompare(b))) {
|
||||
const pm = packageManager(ctx.repoRoot, root);
|
||||
const bin = pm === 'npm' ? 'npx' : pm;
|
||||
if (onPath(bin, ctx.env)) groups.push({ runner: `${PM_EXEC[pm]} playwright test --reporter=list {files}`, cwd: root, files: list });
|
||||
else { missing.add(bin); unclaimed.push(...list.map(f => (root ? `${root}/${f}` : f))); }
|
||||
}
|
||||
return { groups, unclaimed: [...unclaimed, ...noPubspec], missing: [...missing].sort(), noPubspec };
|
||||
}
|
||||
|
||||
@@ -269,11 +320,20 @@ function parseCargo(output: string): RunnerSummary | null {
|
||||
return { ran: lines.reduce((n, m) => n + Number(m[1]), 0) + failed, failed };
|
||||
}
|
||||
|
||||
/** The runner's own summary line: bun, node --test, vitest, jest, pytest, bats, dart/flutter test, cargo test. A bats, dart/flutter or cargo runner's summary is tried first (cargo's only there: its `N passed` lines would misread as another runner's); for any other runner the bats and dart formats are a last resort. `ran: null` when not recognised. Colour codes are dropped and a carriage return ends a line, so a coloured or redrawn verdict still reads. */
|
||||
/** playwright test: the last `N failed`, `N interrupted`, `N flaky`, `N passed` and `N skipped` lines (its closing summary, with test names between them; a test's own output earlier does not count); skipped tests did not run. */
|
||||
function parsePlaywright(output: string): RunnerSummary | null {
|
||||
if (!/^\s*\d+ (?:passed|failed|flaky|interrupted|skipped)\b/m.test(output)) return null;
|
||||
const count = (word: string) => Number([...output.matchAll(new RegExp(`^\\s*(\\d+) ${word}\\b`, 'gm'))].pop()?.[1] ?? 0);
|
||||
const failed = count('failed') + count('interrupted');
|
||||
return { ran: count('passed') + count('flaky') + failed, failed };
|
||||
}
|
||||
|
||||
/** The runner's own summary line: bun, node --test, vitest, jest, pytest, bats, dart/flutter test, cargo test, playwright test. A bats, dart/flutter, playwright or cargo runner's summary is tried first (cargo's only there: its `N passed` lines would misread as another runner's); for any other runner the bats and dart formats are a last resort. `ran: null` when not recognised. Colour codes are dropped and a carriage return ends a line, so a coloured or redrawn verdict still reads. */
|
||||
export function parseSummary(raw: string, runner = ''): RunnerSummary {
|
||||
const output = raw.replace(/\x1b\[[0-9;?]*[A-Za-z]/g, '').replace(/\r/g, '\n');
|
||||
if (/(?<![\w-])cargo\s+test(?![\w-])/.test(runner)) return parseCargo(output) ?? { ran: null, failed: null };
|
||||
if (/(?<![\w-])bats(?![\w-])/.test(runner)) { const b = parseBats(output); if (b) return b; }
|
||||
if (/(?<![\w-])playwright\s+test(?![\w-])/.test(runner)) { const p = parsePlaywright(output); if (p) return p; }
|
||||
const dartRun = /(?<![\w-])(?:dart|flutter)(?![\w-])/.test(runner);
|
||||
if (dartRun) { const d = parseDart(output); if (d) return d; }
|
||||
let m: RegExpExecArray | null;
|
||||
@@ -287,19 +347,22 @@ export function parseSummary(raw: string, runner = ''): RunnerSummary {
|
||||
return (dartRun ? null : parseDart(output)) ?? parseBats(output) ?? { ran: null, failed: null };
|
||||
}
|
||||
|
||||
interface LocalRun { exit: number | null; outcome: Supervised['outcome']; signal: string | null; observedStart: boolean; ran: number | null; failed: number | null; tail: string[]; runner: string; command: string; files: string[]; lastFile: string | null; ms: number; error?: string }
|
||||
interface LocalRun { exit: number | null; outcome: Supervised['outcome']; signal: string | null; observedStart: boolean; ran: number | null; failed: number | null; tail: string[]; runner: string; command: string; files: string[]; lastFile: string | null; ms: number; error?: string; noBrowsers?: string }
|
||||
|
||||
const quote = (f: string) => `'${f.replace(/'/g, `'\\''`)}'`;
|
||||
|
||||
async function runOnce(ctx: PregateContext, env: NodeJS.ProcessEnv, g: RunGroup, files: string[], log: string, counter: Counter): Promise<LocalRun> {
|
||||
const joined = files.map(quote).join(' ');
|
||||
// playwright test reads each file argument as a case-insensitive regex: escape it and anchor it to a path segment, so `a[1].spec.ts` runs that file, not `a1.spec.ts`.
|
||||
const args = /(?<![\w-])playwright\s+test(?![\w-])/.test(g.runner) ? files.map(f => `(?:^|/)${f.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')}$`) : files;
|
||||
const joined = args.map(quote).join(' ');
|
||||
const run = g.runner.replace('{files}', () => joined);
|
||||
const command = g.cwd ? `(cd ${quote(g.cwd)} && ${run})` : run;
|
||||
const shown = g.cwd ? files.map(f => `${g.cwd}/${f}`) : files;
|
||||
const r = await supervise({ command: run, cwd: path.join(ctx.repoRoot, g.cwd), env: { ...env, CI: env.CI ?? '1' }, log, deadline: Date.now() + ctx.timeoutMs, stallMs: ctx.stallTimeoutMs, label: `lanes (${shown.length === 1 ? shown[0] : `${shown.length} files`})`, files, adapter: adapterFor(g.runner), counter, progress: ctx.progress });
|
||||
const output = readRange(log, r.start, r.end);
|
||||
const summary = parseSummary(output, g.runner);
|
||||
return { exit: r.exit, outcome: r.outcome, signal: r.signal, observedStart: r.observedStart, ...summary, tail: output.trim().split('\n').slice(-25), runner: g.runner, command, files: shown, lastFile: r.lastFile && g.cwd ? `${g.cwd}/${r.lastFile}` : r.lastFile, ms: r.ms, ...(r.error ? { error: r.error } : {}) };
|
||||
const noBrowsers = /(?<![\w-])playwright\s+test(?![\w-])/.test(g.runner) ? PLAYWRIGHT_NO_BROWSERS.exec(output)?.[0].trim().slice(0, 200) : undefined;
|
||||
return { ...(noBrowsers ? { noBrowsers } : {}), exit: r.exit, outcome: r.outcome, signal: r.signal, observedStart: r.observedStart, ...summary, tail: output.trim().split('\n').slice(-25), runner: g.runner, command, files: shown, lastFile: r.lastFile && g.cwd ? `${g.cwd}/${r.lastFile}` : r.lastFile, ms: r.ms, ...(r.error ? { error: r.error } : {}) };
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -496,6 +559,9 @@ async function runSelected(ctx: PregateContext, sel: Selection, base: CheckResul
|
||||
// only when the runner's binary is off PATH.
|
||||
const environment = r.ran === null && r.bad && !r.bad.observedStart ? classifyCommandEnvironment(r.bad.runner, r.tail.join('\n'), r.exit, runtime.env) : null;
|
||||
if (environment) return environmentResult(withRun, environment, `the runner exited ${r.exit} before any test was counted (${total} files): ${environment.evidence}`);
|
||||
// Playwright counts a test whose browser is not installed as failed; when every test of every failed run failed and Playwright says so, the machine lacks the browsers.
|
||||
const failedRuns = r.runs.filter(x => x.exit !== 0);
|
||||
if (failedRuns.every(x => x.noBrowsers && x.failed !== null && x.failed === x.ran)) return environmentResult(withRun, { kind: 'missing-binary', evidence: `Playwright browsers not installed (npx playwright install): ${failedRuns[0]!.noBrowsers}`, proof: 'output' }, `every Playwright test failed to launch its browser (${r.failed} of ${r.ran}; ${total} files)`);
|
||||
return { ...withRun, status: 'fail', cause: 'diff', code: 'PREGATE_LANE_FAILED', detail: `local lane exited ${r.exit} (${r.failed ?? '?'} failed of ${r.ran ?? '?'} ran; ${total} files; pipefail)`, fix: `read ${r.log}, fix the in-branch failure, rerun gstack-pregate --stage tests` };
|
||||
}
|
||||
if (r.ran === 0) return { ...withRun, status: 'fail', cause: 'diff', code: 'PREGATE_LANE_FAILED', detail: `ZERO-RUN: the runner reported 0 tests${r.zero ? ` in ${r.zero}` : ''} (${total} selected file(s))`, fix: 'a bad selector, a missing file or a skipped describe; exit 0 is not a pass' };
|
||||
|
||||
@@ -69,7 +69,12 @@ function unquote(v: string): string { return /^['"`]/.test(v) ? v.slice(1, -1) :
|
||||
* `/retry.ts.snap`, or `lib/install.ts` for `install.sh`, is not the module (GST-820): a bare stem such as `install` or `retry` is an
|
||||
* English word in half the test tree, and a one-character old value (`2`)
|
||||
* sits in every test, so that value needs the constant's name (a wrapper
|
||||
* version 2 → 3 flagged 575 consumers through the stem alone).
|
||||
* version 2 → 3 flagged 575 consumers through the stem alone), and counts only
|
||||
* in a statement that names it: on the name's line, or on a line that
|
||||
* continues it (the line before ends open, `(` `,` `.` `=`, or the line
|
||||
* starts with `.` `)` `]`), comments excluded (#3492: `$2`, "from revision 2",
|
||||
* `export default 2`). A call-site name (`attempts`, `limit`) is an ordinary
|
||||
* word, so for it the file must also name the module (gstack-evals#97).
|
||||
*
|
||||
* A version-history row (`[45, "<digest>"]`, the old value first in a
|
||||
* bracketed row) is skipped when the same table, a run of consecutive
|
||||
@@ -94,12 +99,23 @@ export function consumersOf(repoRoot: string, testFiles: readonly string[], chan
|
||||
const moduleRe = new RegExp(`(?:^|[/\\\\])${escape(stem)}(?=[^\\w.-]|$|\\.${dts}(?:${exts})(?![\\w.-])|\\.(?!\\w))|(?<![\\w-])${escape(base)}(?![\\w-]|\\.\\w)`, 'm');
|
||||
const nameRe = new RegExp(`\\b${escape(change.name)}\\b`);
|
||||
const nameRequired = value.length < 2;
|
||||
const callSite = !/^[A-Z]/.test(change.name);
|
||||
const comment = (l: string) => /^\s*(?:\/\/|#|\*|\/\*|<!--)/.test(l);
|
||||
const continues = (prev: string, cur: string) => /[([{,.=+\-*/&|?:\\]\s*$/.test(prev) || /^\s*[.)\]}?:&|+]/.test(cur);
|
||||
const hits: string[] = [];
|
||||
for (const rel of testFiles) {
|
||||
let text: string;
|
||||
try { text = fs.readFileSync(path.join(repoRoot, rel), 'utf8'); } catch { continue; }
|
||||
if (!(nameRe.test(text) || (!nameRequired && moduleRe.test(text)))) continue;
|
||||
if (nameRequired ? !(nameRe.test(text) && (!callSite || moduleRe.test(text))) : !(nameRe.test(text) || moduleRe.test(text))) continue;
|
||||
const lines = text.split('\n');
|
||||
const named = (i: number) => {
|
||||
if (comment(lines[i]!)) return false;
|
||||
for (let j = i; j >= 0 && i - j < 4; j--) {
|
||||
if (j < i && (comment(lines[j]!) || !continues(lines[j]!, lines[j + 1]!))) return false;
|
||||
if (nameRe.test(lines[j]!)) return true;
|
||||
}
|
||||
return false;
|
||||
};
|
||||
const table: number[] = [];
|
||||
const tableHasNew: boolean[] = [];
|
||||
for (let i = 0; i < lines.length; i++) {
|
||||
@@ -115,7 +131,7 @@ export function consumersOf(repoRoot: string, testFiles: readonly string[], chan
|
||||
if (extended === undefined) { const base = baseText?.(rel); extended = base === null || (base !== undefined && !newRowRe.test(base)); }
|
||||
return extended;
|
||||
};
|
||||
for (let i = 0; i < lines.length; i++) if (valueRe.test(lines[i]!) && !skip(i)) { hits.push(`${rel}:${i + 1}`); break; }
|
||||
for (let i = 0; i < lines.length; i++) if (valueRe.test(lines[i]!) && (!nameRequired || oldRowRe.test(lines[i]!) || named(i)) && !skip(i)) { hits.push(`${rel}:${i + 1}`); break; }
|
||||
}
|
||||
return hits;
|
||||
}
|
||||
|
||||
@@ -40,9 +40,11 @@
|
||||
* run_profiles name → `{ workflow, events, jobs, artifact, max_age_hours?, os? }`:
|
||||
* the GitHub Actions runs `clear --run` accepts as evidence
|
||||
* (lib/run-evidence.ts); a profile named after a lane clears it
|
||||
* remote_gate_runs `{ profiles, inventory }`: the run profiles that together
|
||||
* clear the remote gate obligation on the identical tree, and
|
||||
* the command listing the gate's test files they must cover
|
||||
* remote_gate_runs `{ profiles, inventory, skipped? }`: the run profiles that
|
||||
* together clear the remote gate obligation on the identical
|
||||
* tree, the command listing the gate's test files they must
|
||||
* cover, and the command naming the selected files the gate
|
||||
* itself never runs (gstack: paid files)
|
||||
* allowlist `{ finding, paths?, pattern?, reason }` entries the `secrets`
|
||||
* check accepts as benign: a gstack-redact finding id or a
|
||||
* non-secret category, scoped by path globs and/or a regex
|
||||
@@ -354,7 +356,7 @@ export function explainConfig(): string {
|
||||
'timeout_s: seconds each runner process may take (the unit-file group, each serial file, each guard without its own timeout_s, and the remote gate\'s deadline) when --timeout is not passed; 1 to 3600, default 600',
|
||||
'remote_gate: the repo\'s remote test gate command; under 8 cores the test tier becomes a requires-remote obligation naming it, and gstack-pregate --remote runs it (it must print PREGATE_REMOTE: tree=<tree> tests=<n> failed=<n>) (docs/pregate.md#remote-gate)',
|
||||
'run_profiles: name → { workflow, events, jobs, artifact, max_age_hours?, os? }; the GitHub Actions runs gstack-pregate clear --run verifies as evidence on the identical tree (docs/pregate.md#run-evidence)',
|
||||
'remote_gate_runs: { profiles, inventory }; the run profiles that together clear the remote gate obligation, and the command listing the gate\'s test files they must cover (docs/pregate.md#run-evidence)',
|
||||
'remote_gate_runs: { profiles, inventory, skipped? }; the run profiles that together clear the remote gate obligation, the command listing the gate\'s test files they must cover, and an optional command printing which selected files (PREGATE_FILES_FILE) the gate itself does not run, which then need no run (docs/pregate.md#run-evidence)',
|
||||
`allowlist: [{ finding, paths?, pattern?, reason }]: benign findings the secrets check accepts and names as allowlisted. finding is a gstack-redact id (env.kv, google.api_key, pii.phone.e164, …) or a category (${ALLOWLISTABLE_CATEGORIES.join(', ')}); paths are globs; pattern is a regex matched at the finding's position in the added line; one of the two is required, reason always. Never ${NON_ALLOWLISTABLE.join(' or ')} or engine.* (docs/pregate.md#allowlist)`,
|
||||
'no .gstack/pregate.json on the base: <state root>/projects/<slug>/pregate.json is read instead (state root: GSTACK_STATE_ROOT, else GSTACK_HOME, else GSTACK_STATE_DIR, else a gstack plugin data dir, else ~/.gstack, per docs/state-root.md; run --explain prints the resolved path; outside the repo, so a PR cannot reach it; a path inside the checkout is ignored)',
|
||||
].join('\n') + '\n';
|
||||
|
||||
+13
-3
@@ -205,7 +205,7 @@ function userPolicyChange(file: PregateFile): string | null {
|
||||
|
||||
export interface ClearInput { runUrl?: string; bundleLane?: string; receiptPath?: string; runs?: string[]; evidence?: RunCheck }
|
||||
/** What a run clear needs: the base branch's run profiles, the verifier's dependencies, the clock, and the remote gate's inventory. */
|
||||
export interface RunCheck { repo: string; deps: RunDeps; profiles: Record<string, RunProfile>; gateRuns: RemoteGateRuns | null; now: Date; inventory: (command: string, tree: string) => string[] | { error: string } }
|
||||
export interface RunCheck { repo: string; deps: RunDeps; profiles: Record<string, RunProfile>; gateRuns: RemoteGateRuns | null; now: Date; inventory: (command: string, tree: string, selected?: string[]) => string[] | { error: string } }
|
||||
export type ClearDecision = { receipt: string; attested: boolean; substitution?: PregateReceiptSubstitution } | { refused: 'usage' | 'PREGATE_RECEIPT_INVALID'; message: string; fix?: string };
|
||||
|
||||
/**
|
||||
@@ -312,7 +312,8 @@ function treeSubstitution(file: PregateFile, gatedTree: string, lane: string, re
|
||||
/**
|
||||
* `clear --run`: GitHub Actions runs as the obligation's evidence (lib/run-evidence.ts). The remote gate needs one
|
||||
* verified run per `remote_gate_runs` profile, all on one tree, whose tree.json file lists together cover every file
|
||||
* the gate's inventory command prints; any other lane needs one verified run of the profile named after it. The run
|
||||
* the gate's inventory command prints and every selected file except those its skipped command says the gate itself
|
||||
* does not run (never an inventory file); any other lane needs one verified run of the profile named after it. The run
|
||||
* tree must be pregate.json's tree or a stamp-only predecessor of it, exactly as a gate report's tree must.
|
||||
*/
|
||||
function clearByRuns(file: PregateFile, lane: string, refs: string[], check: RunCheck | undefined, repoRoot: string): ClearDecision {
|
||||
@@ -359,10 +360,19 @@ function clearByRuns(file: PregateFile, lane: string, refs: string[], check: Run
|
||||
const idle = evidence.filter(e => (e.tests ?? e.files.length) === 0);
|
||||
if (idle.length) return { refused: 'PREGATE_RECEIPT_INVALID', message: `REUSE: refused reason=no_tests — ${idle.map(e => `run ${e.run_id} (${e.profile})`).join(', ')} records no tests; ${lane} stays uncleared` };
|
||||
const required = new Set(file.requires_remote?.find(r => r.lane === lane)?.tests ?? []);
|
||||
let skippedCount = 0;
|
||||
if (isGate) {
|
||||
const listed = check.inventory(check.gateRuns!.inventory, file.tree);
|
||||
if ('error' in listed) return { refused: 'PREGATE_RECEIPT_INVALID', message: `REUSE: refused reason=inventory_failed — the remote gate's inventory command failed: ${listed.error}; ${lane} stays uncleared` };
|
||||
if (!listed.length) return { refused: 'PREGATE_RECEIPT_INVALID', message: `REUSE: refused reason=inventory_empty — the remote gate's inventory command listed no test files; ${lane} stays uncleared` };
|
||||
const skipCommand = check.gateRuns!.skipped;
|
||||
if (skipCommand && required.size) {
|
||||
const skipped = check.inventory(skipCommand, file.tree, [...required]);
|
||||
if ('error' in skipped) return { refused: 'PREGATE_RECEIPT_INVALID', message: `REUSE: refused reason=skipped_failed — the remote gate's skipped command failed: ${skipped.error}; ${lane} stays uncleared` };
|
||||
const inventory = new Set(listed);
|
||||
for (const f of skipped) if (!inventory.has(f) && required.delete(f)) skippedCount++;
|
||||
if (!required.size) return { refused: 'PREGATE_RECEIPT_INVALID', message: `REUSE: refused reason=paid_only — the remote gate skips every selected test file (${skippedCount}), so no selected test ran; like the gate's own tests=0 report, that never clears; ${lane} stays uncleared` };
|
||||
}
|
||||
for (const f of listed) required.add(f);
|
||||
}
|
||||
const missing = uncoveredFiles([...required], evidence);
|
||||
@@ -375,7 +385,7 @@ function clearByRuns(file: PregateFile, lane: string, refs: string[], check: Run
|
||||
}
|
||||
const tests = evidence.reduce((n, e) => n + (e.tests ?? e.files.length), 0);
|
||||
const accepted = sub.substitution ? ` accepted-for=${file.tree.slice(0, 12)} stamp-only=${sub.substitution.files.length}` : '';
|
||||
return { receipt: `${isGate ? 'remote-gate' : 'run'} tree=${trees[0]!.slice(0, 12)} tests=${tests}${accepted} ${runReceiptFields(evidence, check.now)}`, attested: false, substitution: sub.substitution };
|
||||
return { receipt: `${isGate ? 'remote-gate' : 'run'} tree=${trees[0]!.slice(0, 12)} tests=${tests}${skippedCount ? ` skipped=${skippedCount}` : ''}${accepted} ${runReceiptFields(evidence, check.now)}`, attested: false, substitution: sub.substitution };
|
||||
}
|
||||
|
||||
/** Record the receipt that clears one obligation (a run URL, an evidence-bundle lane label, or a remote gate report, with the stamp-only substitution when one was accepted). */
|
||||
|
||||
@@ -79,8 +79,7 @@ export function draftConfig(repoRoot: string): InitDraft {
|
||||
const runner = detectRunner({ repoRoot, config: DEFAULT_CONFIG }, list);
|
||||
return runner ? `${lang} tests detected at run time: ${runner}` : `no runner detected for ${list.length} ${lang} test file(s); set runner (with {files})`;
|
||||
});
|
||||
const mixed = [...groups.keys()].filter(l => l !== 'dart' && l !== 'rust').length > 1 ? '; a selection mixing these languages needs runner set' : '';
|
||||
const runnerNote = notes.length ? notes.join('; ') + mixed : 'no test files found';
|
||||
const runnerNote = notes.length ? notes.join('; ') : 'no test files found';
|
||||
|
||||
const selectionTodo = discoverLanes(repoRoot).filter(l => l.runsTests && l.matrixExpression && l.testFiles.length === 0).map(l => l.id);
|
||||
const config = { $schema: PREGATE_CONFIG_SCHEMA, runner: null, dependencies, empty_lanes: [], selection: {}, warn: [], strays: { allow: [] } };
|
||||
|
||||
@@ -1,12 +1,16 @@
|
||||
/**
|
||||
* The run-evidence inputs one consumer needs (GSTA-156): the base branch's run
|
||||
* profiles, this repository on GitHub, the verifier's gh- and git-backed
|
||||
* dependencies, and the remote gate's inventory command (a pin-file command,
|
||||
* so it runs only with --allow-repo-commands). Shared by `gstack-pregate
|
||||
* dependencies, and the remote gate's inventory and skipped commands (pin-file
|
||||
* commands, so they run only with --allow-repo-commands; skipped reads the
|
||||
* selected files from PREGATE_FILES_FILE). Shared by `gstack-pregate
|
||||
* clear` and `gstack-pregate --from-run`, so both read profiles from the
|
||||
* base and nowhere else.
|
||||
*/
|
||||
import { spawnSync } from 'node:child_process';
|
||||
import * as fs from 'node:fs';
|
||||
import * as os from 'node:os';
|
||||
import * as path from 'node:path';
|
||||
import { ghRunDeps, runInventory } from '../run-evidence';
|
||||
import { wtreeFingerprint } from '../script-invocation';
|
||||
import { resolveBaseBranch } from '../ship-policy';
|
||||
@@ -28,13 +32,19 @@ export function buildRunCheck(root: string, lane: string | null, allowRepoComman
|
||||
if (!repo) return { error: 'origin is not a github.com repository; run evidence is read from GitHub Actions', profiled: lane === null ? false : lane === REMOTE_GATE_LANE ? !!loaded.config.remote_gate_runs : !!loaded.config.run_profiles[lane] };
|
||||
return {
|
||||
repo, deps: ghRunDeps(repo, root), profiles: loaded.config.run_profiles, gateRuns: loaded.config.remote_gate_runs, now: new Date(),
|
||||
inventory: (command, tree) => {
|
||||
inventory: (command, tree, selected) => {
|
||||
if (!allowRepoCommands) return { error: 'the inventory command is a pin-file command; pass --allow-repo-commands' };
|
||||
const at = () => { const fp = wtreeFingerprint({ cwd: root }); return 'tree' in fp ? fp.tree : null; };
|
||||
if (at() !== tree) return { error: `the working tree is not pregate.json's tree ${tree.slice(0, 12)}; the inventory must list that tree's gate files` };
|
||||
const listed = runInventory(command, root);
|
||||
if (at() !== tree) return { error: `the working tree changed while ${command} ran` };
|
||||
return listed;
|
||||
const dir = selected && fs.mkdtempSync(path.join(os.tmpdir(), 'gstack-pregate-skipped-'));
|
||||
try {
|
||||
if (dir) fs.writeFileSync(path.join(dir, 'selected.txt'), selected!.map(f => `${f}\n`).join(''));
|
||||
const listed = runInventory(command, root, dir ? { PREGATE_FILES_FILE: path.join(dir, 'selected.txt') } : undefined);
|
||||
if (at() !== tree) return { error: `the working tree changed while ${command} ran` };
|
||||
return listed;
|
||||
} finally {
|
||||
if (dir) fs.rmSync(dir, { recursive: true, force: true });
|
||||
}
|
||||
},
|
||||
};
|
||||
}
|
||||
@@ -149,3 +149,66 @@ export function estimateCostUsd(
|
||||
const outputCost = tokens.output * row.output_per_mtok / 1_000_000;
|
||||
return +(inputCost + cachedCost + outputCost).toFixed(6);
|
||||
}
|
||||
|
||||
/** Token classes of one Anthropic Messages usage block, as Claude Code's stream-json and session transcripts carry it. */
|
||||
export interface AnthropicUsage {
|
||||
input_tokens?: number;
|
||||
output_tokens?: number;
|
||||
cache_read_input_tokens?: number;
|
||||
cache_creation_input_tokens?: number;
|
||||
cache_creation?: { ephemeral_5m_input_tokens?: number; ephemeral_1h_input_tokens?: number };
|
||||
}
|
||||
|
||||
/** Anthropic bills a prompt-cache write at 1.25x input for the 5-minute TTL and 2x for the 1-hour TTL. */
|
||||
export const CACHE_WRITE_MULTIPLIER = { ephemeral_5m: 1.25, ephemeral_1h: 2 } as const;
|
||||
|
||||
/**
|
||||
* Cost of one usage block at its model's row, prompt-cache writes included
|
||||
* (unlike estimateCostUsd). Undefined when the model has no price row: an
|
||||
* unpriced model's usage is unknown cost, never zero.
|
||||
*/
|
||||
export function anthropicUsageCostUsd(usage: AnthropicUsage, model: string | undefined): number | undefined {
|
||||
const row = pricingRow(model);
|
||||
if (!row) return undefined;
|
||||
const count = (value: unknown) => typeof value === 'number' && Number.isFinite(value) && value > 0 ? value : 0;
|
||||
const oneHour = count(usage.cache_creation?.ephemeral_1h_input_tokens);
|
||||
const fiveMinute = typeof usage.cache_creation?.ephemeral_5m_input_tokens === 'number'
|
||||
? count(usage.cache_creation.ephemeral_5m_input_tokens) : Math.max(0, count(usage.cache_creation_input_tokens) - oneHour);
|
||||
const cachedRate = row.cached_input_per_mtok ?? row.input_per_mtok * 0.1;
|
||||
return (count(usage.input_tokens) * row.input_per_mtok
|
||||
+ count(usage.cache_read_input_tokens) * cachedRate
|
||||
+ fiveMinute * row.input_per_mtok * CACHE_WRITE_MULTIPLIER.ephemeral_5m
|
||||
+ oneHour * row.input_per_mtok * CACHE_WRITE_MULTIPLIER.ephemeral_1h
|
||||
+ count(usage.output_tokens) * row.output_per_mtok) / 1_000_000;
|
||||
}
|
||||
|
||||
/**
|
||||
* Billed cost of a session's assistant messages, subagent (sidechain) turns
|
||||
* included. A message streams as several events sharing one message id whose
|
||||
* usage grows, so each id counts once at its largest output. Each message is
|
||||
* priced at its own model, else `fallbackModel`. Undefined when no message
|
||||
* carries usage or any message's model has no price row.
|
||||
*/
|
||||
export function streamedUsageCostUsd(
|
||||
events: ReadonlyArray<{ message?: { id?: unknown; model?: unknown; usage?: unknown } } | null | undefined>,
|
||||
fallbackModel?: string,
|
||||
): number | undefined {
|
||||
const byId = new Map<string, { model?: string; usage: AnthropicUsage }>();
|
||||
for (const event of events) {
|
||||
const message = event?.message;
|
||||
const usage = message?.usage as AnthropicUsage | undefined;
|
||||
if (!usage || typeof usage !== 'object' || typeof message!.id !== 'string' || message!.model === '<synthetic>') continue;
|
||||
const seen = byId.get(message!.id);
|
||||
if (!seen || (usage.output_tokens ?? 0) >= (seen.usage.output_tokens ?? 0)) {
|
||||
byId.set(message!.id, { model: typeof message!.model === 'string' ? message!.model : fallbackModel, usage });
|
||||
}
|
||||
}
|
||||
if (byId.size === 0) return undefined;
|
||||
let total = 0;
|
||||
for (const { model, usage } of byId.values()) {
|
||||
const cost = anthropicUsageCostUsd(usage, model);
|
||||
if (cost === undefined) return undefined;
|
||||
total += cost;
|
||||
}
|
||||
return +total.toFixed(6);
|
||||
}
|
||||
@@ -0,0 +1,447 @@
|
||||
/**
|
||||
* gstack-flaky quarantine (GSTA-158 P2): a registry of known-flaky tests at
|
||||
* `.gstack/quarantine.json`. A quarantined test still runs; it gets a larger
|
||||
* attempt budget (lib/flaky-run.ts, QUARANTINE_ATTEMPTS) under the same full
|
||||
* green verdict, and it is never counted as a pass. This file holds the
|
||||
* registry: schema, validation, add/renew/release, `check` against the base
|
||||
* branch, and `report` over downloaded ledgers.
|
||||
*
|
||||
* Rules (docs/flaky.md § Quarantine):
|
||||
* - Every entry has an owner, a ledger link, a failure class (detector,
|
||||
* harness, timing, model-latency; product defects and infra are refused),
|
||||
* evidence and an expiry: 7 days by default, at most 14 from entry, and one
|
||||
* renewal of at most 7 days with new evidence and the owner named again.
|
||||
* - An expired or invalid entry is not quarantined; nothing needs editing.
|
||||
* - Live entries are capped at max(1, 10% of the suite's tests), and 5 for a
|
||||
* suite under 50 tests.
|
||||
* - `run --base` reads the registry from the base branch so a change cannot
|
||||
* quarantine the test it breaks; `check --base` refuses an entry added in
|
||||
* the change that edits its test's file, an entry that disappears without a
|
||||
* release record, a release record that was removed, and rewritten history.
|
||||
* - gstack's own CASE_QUARANTINE / EVAL_POLICY (paid evals) is a separate
|
||||
* registry; the two share no schema.
|
||||
*/
|
||||
import * as fs from 'fs';
|
||||
import * as path from 'path';
|
||||
import { spawnSync } from 'child_process';
|
||||
import { parseTestId, readJunit, type JunitRunner } from './junit';
|
||||
import { safeText } from './flaky-run';
|
||||
|
||||
export const REGISTRY_PATH = '.gstack/quarantine.json';
|
||||
export const FAILURE_CLASSES = ['detector', 'harness', 'timing', 'model-latency'] as const;
|
||||
export type FailureClass = typeof FAILURE_CLASSES[number];
|
||||
export const DEFAULT_DAYS = 7;
|
||||
export const MAX_DAYS = 14;
|
||||
export const RENEW_MAX_DAYS = 7;
|
||||
export const EXIT_RULE_PASSES = 10;
|
||||
const DAY = 86_400_000;
|
||||
|
||||
export interface Renewal { renewedAt: string; previousExpiresAt: string; expiresAt: string; owner: string; evidence: string[] }
|
||||
|
||||
export interface QuarantineEntry {
|
||||
owner: string;
|
||||
/** Link to the flaky ledger line or issue that tracks the fix. */
|
||||
ledger: string;
|
||||
failureClass: FailureClass;
|
||||
reason: string;
|
||||
evidence: string[];
|
||||
enteredAt: string;
|
||||
expiresAt: string;
|
||||
renewals: Renewal[];
|
||||
/** The suite's test count when the entry was added (`add --junit`), the base of the cap. */
|
||||
suiteTests: number;
|
||||
}
|
||||
|
||||
export type ReleasedEntry = Omit<QuarantineEntry, 'reason'> & { id: string; releasedAt: string; reason: string; pr: string };
|
||||
|
||||
export interface Registry { schema: 1; entries: Record<string, QuarantineEntry>; released: ReleasedEntry[] }
|
||||
|
||||
export class QuarantineError extends Error {
|
||||
constructor(readonly code: string, message: string, readonly exitCode: 1 | 2 | 3 = 3) { super(message); }
|
||||
}
|
||||
|
||||
export const emptyRegistry = (): Registry => ({ schema: 1, entries: {}, released: [] });
|
||||
|
||||
/** Live entries allowed for a suite of `tests` tests. */
|
||||
export const suiteCap = (tests: number) => tests < 50 ? 5 : Math.max(1, Math.floor(tests / 10));
|
||||
|
||||
function git(args: string[], cwd: string): { ok: boolean; out: string } {
|
||||
const r = spawnSync('git', args, { cwd, encoding: 'utf-8', timeout: 15_000 });
|
||||
return { ok: r.status === 0, out: r.stdout ?? '' };
|
||||
}
|
||||
|
||||
export function repoRoot(cwd: string): string {
|
||||
const r = git(['rev-parse', '--show-toplevel'], cwd);
|
||||
return r.ok && r.out.trim() ? r.out.trim() : cwd;
|
||||
}
|
||||
|
||||
export function parseRegistry(text: string, source: string): Registry {
|
||||
let data: unknown;
|
||||
try { data = JSON.parse(text); } catch (e) {
|
||||
throw new QuarantineError('FLAKY_QUARANTINE_INVALID', `${source} is not valid JSON (${(e as Error).message})`);
|
||||
}
|
||||
const r = data as Partial<Registry> | null;
|
||||
if (!r || typeof r !== 'object' || r.schema !== 1 || !r.entries || typeof r.entries !== 'object' || Array.isArray(r.entries) || !Array.isArray(r.released)) {
|
||||
throw new QuarantineError('FLAKY_QUARANTINE_INVALID', `${source} is not a schema 1 quarantine registry ({"schema":1,"entries":{},"released":[]})`);
|
||||
}
|
||||
return r as Registry;
|
||||
}
|
||||
|
||||
function resolveCommit(ref: string, cwd: string): string {
|
||||
if (!ref || ref.startsWith('-')) throw new QuarantineError('FLAKY_QUARANTINE_BASE', `--base ${JSON.stringify(ref)} is not a git ref`, 2);
|
||||
const r = git(['rev-parse', '--verify', '--quiet', `${ref}^{commit}`], cwd);
|
||||
if (!r.ok) throw new QuarantineError('FLAKY_QUARANTINE_BASE', `--base ${ref} does not resolve to a commit in ${cwd}`);
|
||||
return r.out.trim();
|
||||
}
|
||||
|
||||
function registryAt(commit: string, root: string, label: string): Registry {
|
||||
if (!git(['cat-file', '-e', `${commit}:${REGISTRY_PATH}`], root).ok) return emptyRegistry();
|
||||
const shown = git(['show', `${commit}:${REGISTRY_PATH}`], root);
|
||||
if (!shown.ok) throw new QuarantineError('FLAKY_QUARANTINE_BASE', `cannot read ${REGISTRY_PATH} at ${label}`);
|
||||
return parseRegistry(shown.out, `${REGISTRY_PATH} at ${label}`);
|
||||
}
|
||||
|
||||
/** The registry in the working tree, or at `base` when given; a missing file is an empty registry. */
|
||||
export function readRegistry(cwd: string, base?: string): Registry {
|
||||
const root = repoRoot(cwd);
|
||||
if (base !== undefined) return registryAt(resolveCommit(base, root), root, base);
|
||||
const file = path.join(root, REGISTRY_PATH);
|
||||
if (!fs.existsSync(file)) return emptyRegistry();
|
||||
return parseRegistry(fs.readFileSync(file, 'utf-8'), REGISTRY_PATH);
|
||||
}
|
||||
|
||||
export function writeRegistry(cwd: string, reg: Registry): string {
|
||||
const file = path.join(repoRoot(cwd), REGISTRY_PATH);
|
||||
const entries = Object.fromEntries(Object.keys(reg.entries).sort().map(id => [id, reg.entries[id]!]));
|
||||
fs.mkdirSync(path.dirname(file), { recursive: true });
|
||||
fs.writeFileSync(file, JSON.stringify({ schema: 1, entries, released: reg.released }, null, 2) + '\n');
|
||||
return file;
|
||||
}
|
||||
|
||||
const filled = (s: unknown): s is string => typeof s === 'string' && s.trim().length > 0;
|
||||
const filledList = (l: unknown): l is string[] => Array.isArray(l) && l.length > 0 && l.every(filled);
|
||||
|
||||
export function classRefusal(c: unknown): string | undefined {
|
||||
if (FAILURE_CLASSES.includes(c as FailureClass)) return undefined;
|
||||
if (c === 'product') return 'class product is refused: a product defect is a real red; fix it instead of quarantining it';
|
||||
if (c === 'infra') return 'class infra is refused: an infrastructure red is rerun or fixed as infra, never quarantined';
|
||||
return `class ${JSON.stringify(c)} is not one of ${FAILURE_CLASSES.join(', ')}`;
|
||||
}
|
||||
|
||||
/** What makes one entry invalid; an invalid entry is never quarantined and `check` refuses it. */
|
||||
export function entryProblems(id: string, e: QuarantineEntry, now: Date): string[] {
|
||||
if (!e || typeof e !== 'object') return ['not an object'];
|
||||
const p: string[] = [];
|
||||
if (!parseTestId(id)) p.push('not a test id (<file>::<project>::<title>; gstack-flaky ids prints them)');
|
||||
if (!filled(e.owner)) p.push('no owner');
|
||||
if (!filled(e.ledger)) p.push('no ledger link');
|
||||
if (!filled(e.reason)) p.push('no reason');
|
||||
const cls = classRefusal(e.failureClass);
|
||||
if (cls) p.push(cls);
|
||||
if (!filledList(e.evidence)) p.push('no evidence');
|
||||
if (!(Number.isInteger(e.suiteTests) && e.suiteTests > 0)) p.push('no suiteTests (add records the suite size from --junit)');
|
||||
const entered = Date.parse(e.enteredAt);
|
||||
const expires = Date.parse(e.expiresAt);
|
||||
if (Number.isNaN(entered) || Number.isNaN(expires)) return [...p, 'enteredAt or expiresAt is not a date'];
|
||||
if (entered > now.getTime() + DAY) p.push('enteredAt is in the future');
|
||||
if (!Array.isArray(e.renewals)) return [...p, 'renewals is not a list'];
|
||||
if (e.renewals.length > 1) p.push(`renewed ${e.renewals.length} times; one renewal is allowed, then fix the test or release it`);
|
||||
const firstExpiry = e.renewals.length ? Date.parse(e.renewals[0]?.previousExpiresAt) : expires;
|
||||
if (!(firstExpiry > entered)) p.push('expires before it entered');
|
||||
if (firstExpiry - entered > MAX_DAYS * DAY) p.push(`expires more than ${MAX_DAYS} days after it entered`);
|
||||
for (const r of e.renewals) {
|
||||
if (!r || typeof r !== 'object') { p.push('a renewal is not an object'); continue; }
|
||||
const renewed = Date.parse(r.renewedAt);
|
||||
if (!(renewed >= entered && renewed <= now.getTime() + DAY)) p.push('a renewal is dated before the entry or in the future');
|
||||
if (!filled(r?.owner)) p.push('a renewal names no owner');
|
||||
if (!filledList(r?.evidence) || r.evidence.every(x => filledList(e.evidence) && e.evidence.includes(x))) p.push('a renewal carries no new evidence');
|
||||
const from = Math.max(Date.parse(r?.renewedAt), Date.parse(r?.previousExpiresAt));
|
||||
const until = Date.parse(r?.expiresAt);
|
||||
if (Number.isNaN(from) || Number.isNaN(until) || until - from > RENEW_MAX_DAYS * DAY) p.push(`a renewal is not a date range of at most ${RENEW_MAX_DAYS} days`);
|
||||
}
|
||||
const last = e.renewals.at(-1);
|
||||
if (last && last.expiresAt !== e.expiresAt) p.push('expiresAt differs from the last renewal');
|
||||
return p;
|
||||
}
|
||||
|
||||
export const isLive = (e: QuarantineEntry, now: Date) => Date.parse(e.expiresAt) > now.getTime();
|
||||
|
||||
/** Ids `run` gives the quarantine budget, plus one notice per entry it ignores (expired or invalid). */
|
||||
export function liveQuarantine(reg: Registry, now: Date): { ids: Set<string>; notices: string[] } {
|
||||
const ids = new Set<string>();
|
||||
const notices: string[] = [];
|
||||
for (const [id, e] of Object.entries(reg.entries)) {
|
||||
const problems = entryProblems(id, e, now);
|
||||
if (problems.length) notices.push(`FLAKY_QUARANTINE_IGNORED: ${safeText(id)} is not quarantined: ${problems.join('; ')}`);
|
||||
else if (!isLive(e, now)) notices.push(expiredNotice(id, e));
|
||||
else ids.add(id);
|
||||
}
|
||||
return { ids, notices };
|
||||
}
|
||||
|
||||
const shq = (s: string) => "'" + s.split("'").join(`'"'"'`) + "'";
|
||||
export const renewCommand = (id: string) => `gstack-flaky quarantine renew ${shq(safeText(id))} --owner <you> --evidence <new evidence url>`;
|
||||
export const releaseCommand = (id: string) => `gstack-flaky quarantine release ${shq(safeText(id))} --reason <why> --pr <fix pr url>`;
|
||||
|
||||
export function expiredNotice(id: string, e: QuarantineEntry): string {
|
||||
return `FLAKY_QUARANTINE_EXPIRED: ${safeText(id)} expired ${e.expiresAt.slice(0, 10)} and is not quarantined; renew: ${renewCommand(id)} | release: ${releaseCommand(id)}`;
|
||||
}
|
||||
|
||||
export interface AddInput { owner?: string; ledger?: string; failureClass?: string; reason?: string; evidence: string[]; days?: number; suiteIds: string[] }
|
||||
|
||||
const refuse = (msg: string): never => { throw new QuarantineError('FLAKY_QUARANTINE_REFUSED', msg, 1); };
|
||||
|
||||
export function addEntry(reg: Registry, id: string, a: AddInput, now: Date): QuarantineEntry {
|
||||
const days = a.days ?? DEFAULT_DAYS;
|
||||
if (!Number.isInteger(days) || days < 1 || days > MAX_DAYS) refuse(`--days must be a whole number from 1 to ${MAX_DAYS}, got ${a.days}`);
|
||||
const cls = classRefusal(a.failureClass);
|
||||
if (cls) refuse(cls);
|
||||
if (!a.evidence.length) refuse('no evidence: pass --evidence <url> (a failing run, a ledger line) at least once');
|
||||
if (!a.suiteIds.includes(id)) refuse(`${safeText(id)} is not a test in the --junit report (gstack-flaky ids --junit <file> prints the ids)`);
|
||||
if (reg.entries[id]) refuse(`${safeText(id)} already has an entry; renew or release it`);
|
||||
const live = Object.values(reg.entries).filter(e => isLive(e, now)).length;
|
||||
const cap = suiteCap(a.suiteIds.length);
|
||||
if (live + 1 > cap) refuse(`the registry holds ${live} live entries and the cap for a ${a.suiteIds.length}-test suite is ${cap}; fix or release one first`);
|
||||
const entry: QuarantineEntry = {
|
||||
owner: a.owner ?? '', ledger: a.ledger ?? '', failureClass: a.failureClass as FailureClass, reason: a.reason ?? '', evidence: a.evidence,
|
||||
enteredAt: now.toISOString(), expiresAt: new Date(now.getTime() + days * DAY).toISOString(), renewals: [], suiteTests: a.suiteIds.length,
|
||||
};
|
||||
const problems = entryProblems(id, entry, now);
|
||||
if (problems.length) refuse(problems.join('; '));
|
||||
reg.entries[id] = entry;
|
||||
return entry;
|
||||
}
|
||||
|
||||
export function renewEntry(reg: Registry, id: string, r: { owner?: string; evidence: string[]; days?: number }, now: Date): QuarantineEntry {
|
||||
const e = reg.entries[id] ?? refuse(`${safeText(id)} has no entry to renew`);
|
||||
if (e.renewals.length) refuse(`${safeText(id)} was renewed once already; a second renewal is refused: fix the test or release it`);
|
||||
if (!filled(r.owner)) refuse('renew names the owner again: pass --owner');
|
||||
if (!r.evidence.some(x => filled(x) && !e.evidence.includes(x))) refuse('renew needs new evidence: pass --evidence <url> not already on the entry');
|
||||
const days = r.days ?? RENEW_MAX_DAYS;
|
||||
if (!Number.isInteger(days) || days < 1 || days > RENEW_MAX_DAYS) refuse(`a renewal is 1 to ${RENEW_MAX_DAYS} days, got ${r.days}`);
|
||||
const from = Math.max(now.getTime(), Date.parse(e.expiresAt));
|
||||
const expiresAt = new Date(from + days * DAY).toISOString();
|
||||
e.renewals.push({ renewedAt: now.toISOString(), previousExpiresAt: e.expiresAt, expiresAt, owner: r.owner!, evidence: r.evidence });
|
||||
e.expiresAt = expiresAt;
|
||||
return e;
|
||||
}
|
||||
|
||||
export function releaseEntry(reg: Registry, id: string, r: { reason?: string; pr?: string }, now: Date): ReleasedEntry {
|
||||
const e = reg.entries[id] ?? refuse(`${safeText(id)} has no entry to release`);
|
||||
if (!filled(r.reason) || !filled(r.pr)) refuse('release needs --reason and --pr (the fix, or the decision to stop quarantining)');
|
||||
const record: ReleasedEntry = { id, ...e, releasedAt: now.toISOString(), reason: r.reason!, pr: r.pr! };
|
||||
reg.released.push(record);
|
||||
delete reg.entries[id];
|
||||
return record;
|
||||
}
|
||||
|
||||
const liveEntries = (reg: Registry, now: Date) => Object.values(reg.entries).filter(e => e && typeof e === 'object' && isLive(e, now));
|
||||
const smallestSuite = (live: QuarantineEntry[]) => Math.min(...live.map(e => e.suiteTests).filter(n => Number.isInteger(n) && n > 0));
|
||||
|
||||
/** Structural problems of a whole registry plus the cap, for `check`. */
|
||||
export function registryProblems(reg: Registry, now: Date, suiteTests?: number): string[] {
|
||||
const p: string[] = [];
|
||||
for (const [id, e] of Object.entries(reg.entries)) for (const x of entryProblems(id, e, now)) p.push(`${safeText(id)}: ${x}`);
|
||||
for (const r of reg.released) {
|
||||
if (!r || !filled(r.id) || Number.isNaN(Date.parse(r.releasedAt)) || !filled(r.reason) || !filled(r.pr)) p.push(`released record ${JSON.stringify(r?.id)}: needs id, releasedAt, reason and pr`);
|
||||
}
|
||||
const live = liveEntries(reg, now);
|
||||
const tests = suiteTests ?? smallestSuite(live);
|
||||
if (live.length && Number.isFinite(tests) && live.length > suiteCap(tests)) p.push(`${live.length} live entries, over the cap of ${suiteCap(tests)} for a ${tests}-test suite`);
|
||||
return p;
|
||||
}
|
||||
|
||||
const editsFile = (changed: string[], file: string) => changed.some(f => f === file || f.endsWith(`/${file}`));
|
||||
|
||||
/** What a change did to the registry that `check --base` refuses. */
|
||||
export function baseProblems(base: Registry, head: Registry, changed: string[]): string[] {
|
||||
const p: string[] = [];
|
||||
for (const [id, e] of Object.entries(head.entries)) {
|
||||
const was = base.entries[id];
|
||||
const t = parseTestId(id);
|
||||
if (!was) {
|
||||
if (t && editsFile(changed, t.file)) p.push(`${safeText(id)}: added in the same change that edits ${t.file}; a change cannot quarantine the test it touches (land the entry on its own, or fix the test)`);
|
||||
continue;
|
||||
}
|
||||
if (was.enteredAt !== e?.enteredAt) p.push(`${safeText(id)}: enteredAt changed from ${was.enteredAt}; release the entry instead of restarting it`);
|
||||
if (JSON.stringify((e?.renewals ?? []).slice(0, was.renewals.length)) !== JSON.stringify(was.renewals)) p.push(`${safeText(id)}: its renewal history was rewritten`);
|
||||
}
|
||||
for (const [id, was] of Object.entries(base.entries)) {
|
||||
if (!(id in head.entries) && !head.released.some(r => r?.id === id && r.enteredAt === was.enteredAt)) p.push(`${safeText(id)}: removed without a release record (use: ${releaseCommand(id)})`);
|
||||
}
|
||||
for (const r of base.released) {
|
||||
if (!head.released.some(h => h?.id === r.id && h.releasedAt === r.releasedAt)) p.push(`${safeText(r.id)}: its release record of ${r.releasedAt} was removed`);
|
||||
}
|
||||
return p;
|
||||
}
|
||||
|
||||
/** `check`: the working-tree registry alone, and with `base` against what this change did since the merge base. */
|
||||
export function checkRegistry(cwd: string, o: { base?: string; suiteTests?: number }, now: Date): { problems: string[]; live: number; cap?: number } {
|
||||
const head = readRegistry(cwd);
|
||||
const problems = registryProblems(head, now, o.suiteTests);
|
||||
if (o.base !== undefined) {
|
||||
const root = repoRoot(cwd);
|
||||
const commit = resolveCommit(o.base, root);
|
||||
const mb = git(['merge-base', commit, 'HEAD'], root);
|
||||
if (!mb.ok) throw new QuarantineError('FLAKY_QUARANTINE_BASE', `no merge base between ${o.base} and HEAD (a shallow clone?)`);
|
||||
const changed = git(['-c', 'core.quotePath=false', 'diff', '--name-only', '-z', mb.out.trim()], root);
|
||||
if (!changed.ok) throw new QuarantineError('FLAKY_QUARANTINE_BASE', `cannot diff against the merge base of ${o.base}`);
|
||||
problems.push(...baseProblems(registryAt(mb.out.trim(), root, `the merge base of ${o.base}`), head, changed.out.split('\0').filter(Boolean)));
|
||||
}
|
||||
const live = liveEntries(head, now);
|
||||
const tests = o.suiteTests ?? smallestSuite(live);
|
||||
return { problems, live: live.length, ...(Number.isFinite(tests) ? { cap: suiteCap(tests) } : {}) };
|
||||
}
|
||||
|
||||
export interface LedgerRecord { ts: string; kind: string; id: string; attempts?: string[] }
|
||||
|
||||
/** Read every JSON line of the given ledger files, and of every .jsonl file under the given directories. */
|
||||
export function readLedgers(paths: string[]): { records: LedgerRecord[]; skipped: number } {
|
||||
const files: string[] = [];
|
||||
const walk = (p: string) => {
|
||||
const st = fs.statSync(p, { throwIfNoEntry: false });
|
||||
if (!st) throw new QuarantineError('FLAKY_QUARANTINE_USAGE', `--ledger ${p} does not exist`, 2);
|
||||
if (!st.isDirectory()) { files.push(p); return; }
|
||||
for (const n of fs.readdirSync(p).sort()) {
|
||||
const c = path.join(p, n);
|
||||
if (fs.statSync(c).isDirectory() || n.endsWith('.jsonl')) walk(c);
|
||||
}
|
||||
};
|
||||
paths.forEach(walk);
|
||||
const records: LedgerRecord[] = [];
|
||||
let skipped = 0;
|
||||
for (const f of files) {
|
||||
for (const l of fs.readFileSync(f, 'utf-8').split('\n')) {
|
||||
if (!l.trim()) continue;
|
||||
try {
|
||||
const r = JSON.parse(l) as LedgerRecord;
|
||||
if (filled(r?.id) && filled(r.kind) && filled(r.ts)) records.push(r); else skipped++;
|
||||
} catch { skipped++; }
|
||||
}
|
||||
}
|
||||
return { records, skipped };
|
||||
}
|
||||
|
||||
const cleanPass = (r: LedgerRecord) => r.kind === 'quarantined-pass' && !(r.attempts ?? []).includes('fail');
|
||||
const shortKind = (r: LedgerRecord) => cleanPass(r) ? 'pass' : r.kind === 'quarantined-pass' ? 'flaky' : r.kind === 'quarantined-fail' ? 'fail' : r.kind;
|
||||
|
||||
/** `report`: expired entries first, then live entries soonest-expiring first, with ledger history when given. */
|
||||
export function reportLines(reg: Registry, now: Date, ledgers?: LedgerRecord[]): string[] {
|
||||
const out: string[] = [];
|
||||
const all = Object.entries(reg.entries).sort((a, b) => Date.parse(a[1].expiresAt) - Date.parse(b[1].expiresAt));
|
||||
const expired = all.filter(([, e]) => !isLive(e, now));
|
||||
const live = all.filter(([, e]) => isLive(e, now));
|
||||
for (const [id, e] of expired) out.push(expiredNotice(id, e));
|
||||
let expiring = 0, ready = 0;
|
||||
for (const [id, e] of live) {
|
||||
const daysLeft = Math.ceil((Date.parse(e.expiresAt) - now.getTime()) / DAY);
|
||||
if (daysLeft <= 2) expiring++;
|
||||
let history = '';
|
||||
if (ledgers) {
|
||||
const sid = safeText(id);
|
||||
const mine = ledgers.filter(r => r.id === sid && Date.parse(r.ts) >= Date.parse(e.enteredAt)).sort((a, b) => Date.parse(a.ts) - Date.parse(b.ts));
|
||||
let streak = 0;
|
||||
for (let i = mine.length - 1; i >= 0 && cleanPass(mine[i]!); i--) streak++;
|
||||
const isReady = streak >= EXIT_RULE_PASSES;
|
||||
if (isReady) ready++;
|
||||
history = ` recent=${mine.slice(-5).map(shortKind).join(',') || 'none'} clean_streak=${streak}${isReady ? ` READY_TO_RELEASE: ${releaseCommand(id)}` : ''}`;
|
||||
}
|
||||
const renewal = e.renewals.length ? 'renewed once; fix or release' : `renew: ${renewCommand(id)}`;
|
||||
out.push(`QUARANTINE: ${safeText(id)} owner=${safeText(e.owner)} class=${e.failureClass} days_left=${daysLeft} expires=${e.expiresAt.slice(0, 10)} ledger=${safeText(e.ledger)}${history}${daysLeft <= 2 ? ` EXPIRING (${renewal})` : ''}`);
|
||||
}
|
||||
out.push(`QUARANTINE_REPORT: live=${live.length} expired=${expired.length} expiring=${expiring} ready=${ledgers ? ready : 'unchecked'} exit_rule=${ledgers ? `checked (${EXIT_RULE_PASSES} consecutive clean passes)` : 'not checked (no --ledger given)'}`);
|
||||
return out;
|
||||
}
|
||||
|
||||
export const QUARANTINE_USAGE = `Usage:
|
||||
gstack-flaky quarantine add <id> --owner <who> --ledger <url> --class detector|harness|timing|model-latency
|
||||
--reason <text> --evidence <url> [--evidence <url>...] --junit <file> [--runner <r>] [--days <1-${MAX_DAYS}>]
|
||||
gstack-flaky quarantine renew <id> --owner <who> --evidence <new url> [--days <1-${RENEW_MAX_DAYS}>]
|
||||
gstack-flaky quarantine release <id> --reason <text> --pr <url>
|
||||
gstack-flaky quarantine check [--base <ref>] [--junit <file>]
|
||||
gstack-flaky quarantine report [--ledger <file|dir>...] [--base <ref>]
|
||||
|
||||
add/renew/release edit ${REGISTRY_PATH} at the repository root. check validates it
|
||||
(owner, ledger, class, evidence, expiry, one renewal, the cap) and, with --base,
|
||||
refuses an entry added in the change that edits its test's file, an entry removed
|
||||
without a release record, and a removed release record. report lists expired and
|
||||
live entries, soonest-expiring first; with --ledger (files or directories of
|
||||
downloaded ledgers) it shows recent outcomes and marks entries with ${EXIT_RULE_PASSES}
|
||||
consecutive clean passes READY_TO_RELEASE.
|
||||
Exit: 0 ok, 1 refused, 2 usage, 3 unreadable registry or base.
|
||||
`;
|
||||
|
||||
interface CliIO { cwd: string; out: (s: string) => void; now: Date }
|
||||
|
||||
const usage = (msg: string): never => { throw new QuarantineError('FLAKY_QUARANTINE_USAGE', msg, 2); };
|
||||
|
||||
function parseArgs(argv: string[]) {
|
||||
const flags: Record<string, string[]> = {};
|
||||
const positional: string[] = [];
|
||||
for (let i = 0; i < argv.length; i++) {
|
||||
const a = argv[i]!;
|
||||
if (!a.startsWith('--')) { positional.push(a); continue; }
|
||||
const v = argv[++i];
|
||||
if (v === undefined) usage(`${a} needs a value`);
|
||||
(flags[a.slice(2)] ??= []).push(v!);
|
||||
}
|
||||
return { flags, positional };
|
||||
}
|
||||
|
||||
function intFlag(v: string | undefined, name: string): number | undefined {
|
||||
if (v === undefined) return undefined;
|
||||
const n = Number(v);
|
||||
if (!Number.isInteger(n)) usage(`--${name} must be a whole number`);
|
||||
return n;
|
||||
}
|
||||
|
||||
const ALLOWED: Record<string, string[]> = {
|
||||
add: ['owner', 'ledger', 'class', 'reason', 'evidence', 'junit', 'runner', 'days'], renew: ['owner', 'evidence', 'days'],
|
||||
release: ['reason', 'pr'], check: ['base', 'junit', 'runner'], report: ['ledger', 'base'],
|
||||
};
|
||||
|
||||
/** The `gstack-flaky quarantine` subcommands; returns the exit code. A JunitError from --junit propagates to the bin. */
|
||||
export function quarantineMain(argv: string[], io: CliIO): number {
|
||||
const [sub, ...rest] = argv;
|
||||
if (!sub || sub === '--help' || sub === '-h') { io.out(QUARANTINE_USAGE); return sub ? 0 : 2; }
|
||||
if (!ALLOWED[sub]) usage(`unknown quarantine command ${sub}\n${QUARANTINE_USAGE}`);
|
||||
const { flags, positional } = parseArgs(rest);
|
||||
const unknown = Object.keys(flags).find(f => !ALLOWED[sub]!.includes(f));
|
||||
if (unknown) usage(`${sub} does not take --${unknown}`);
|
||||
const one = (f: string) => flags[f]?.at(-1);
|
||||
const runner = one('runner') as JunitRunner | undefined;
|
||||
if (runner !== undefined && !['playwright', 'bun', 'generic'].includes(runner)) usage('--runner must be playwright, bun or generic');
|
||||
const needsId = sub === 'add' || sub === 'renew' || sub === 'release';
|
||||
if (needsId !== (positional.length === 1) || positional.length > 1) usage(needsId ? `${sub} takes one test id` : `${sub} takes no test id`);
|
||||
if (sub === 'check') {
|
||||
const junit = one('junit');
|
||||
const r = checkRegistry(io.cwd, { base: one('base'), suiteTests: junit ? readJunit(junit, runner).tests.length : undefined }, io.now);
|
||||
for (const p of r.problems) io.out(`QUARANTINE_REFUSED: ${p}\n`);
|
||||
io.out(`QUARANTINE_CHECK: live=${r.live} cap=${r.cap ?? 'none'} base=${one('base') ?? 'none'} verdict=${r.problems.length ? 'fail' : 'pass'}\n`);
|
||||
return r.problems.length ? 1 : 0;
|
||||
}
|
||||
if (sub === 'report') {
|
||||
const ledgers = flags.ledger ? readLedgers(flags.ledger) : undefined;
|
||||
if (ledgers?.skipped) io.out(`QUARANTINE_LEDGER: skipped ${ledgers.skipped} unreadable line(s)\n`);
|
||||
for (const l of reportLines(readRegistry(io.cwd, one('base')), io.now, ledgers?.records)) io.out(`${l}\n`);
|
||||
return 0;
|
||||
}
|
||||
const id = positional[0]!;
|
||||
const reg = readRegistry(io.cwd);
|
||||
let line: string;
|
||||
if (sub === 'add') {
|
||||
const junit = one('junit') ?? refuse('add needs --junit <file> to count the suite and confirm the id exists');
|
||||
const suiteIds = readJunit(junit, runner).tests.map(t => t.id);
|
||||
const e = addEntry(reg, id, { owner: one('owner'), ledger: one('ledger'), failureClass: one('class'), reason: one('reason'), evidence: flags.evidence ?? [], days: intFlag(one('days'), 'days'), suiteIds }, io.now);
|
||||
line = `QUARANTINE_ADDED: ${safeText(id)} owner=${safeText(e.owner)} class=${e.failureClass} expires=${e.expiresAt}`;
|
||||
} else if (sub === 'renew') {
|
||||
const e = renewEntry(reg, id, { owner: one('owner'), evidence: flags.evidence ?? [], days: intFlag(one('days'), 'days') }, io.now);
|
||||
line = `QUARANTINE_RENEWED: ${safeText(id)} owner=${safeText(e.renewals[0]!.owner)} expires=${e.expiresAt} (no further renewal)`;
|
||||
} else {
|
||||
const r = releaseEntry(reg, id, { reason: one('reason'), pr: one('pr') }, io.now);
|
||||
line = `QUARANTINE_RELEASED: ${safeText(id)} pr=${safeText(r.pr)}`;
|
||||
}
|
||||
writeRegistry(io.cwd, reg);
|
||||
io.out(`${line} registry=${REGISTRY_PATH}\n`);
|
||||
return 0;
|
||||
}
|
||||
+36
-11
@@ -55,9 +55,11 @@ const ANY_LABEL = new RegExp(`(?<![A-Za-z0-9])(${LABELS})(?![A-Za-z0-9])`);
|
||||
const LABEL_HEADING = new RegExp(`^(?:#{1,6}\\s*|\\*\\*)(${LABELS})(?:\\s+(?:findings?|severity|issues?))?\\s*:?\\**\\s*$`);
|
||||
const HEADING = /^(?:#{1,6}\s|\*\*[^*]+\*\*:?$)/;
|
||||
const LIST_ITEM = /^(?:[-*+]|\d+[.)])\s+\S/;
|
||||
const NUMBERED_HEAD = /^(?:#{1,6}\s+|\*\*\s*)\d+[.)]\s/;
|
||||
const stronger = (a: string, b: string) => ((rank(a) ?? 0) >= (rank(b) ?? 0) ? a : b);
|
||||
/** A prose voice's `Blocks: yes` (GSTA-167 normalizer v2): the finding is CRITICAL whatever its label; `Blocks: no` never lowers one. */
|
||||
const BLOCKS_YES = /(?<![A-Za-z0-9])Blocks:?\**:?\s*\**\s*yes(?![A-Za-z0-9])/i;
|
||||
const BLOCKS_NO = /(?<![A-Za-z0-9])Blocks:?\**:?\s*\**\s*no(?![A-Za-z0-9-])/i;
|
||||
const ALL_LABELS = new RegExp(ANY_LABEL.source, 'g');
|
||||
/** A file:line locus ("auth.ts:9"): prose naming one after a finding's fields starts the next finding. */
|
||||
const LOCUS = /[\w./-]+\.\w+:\d+/;
|
||||
@@ -102,9 +104,11 @@ export interface MergedLine { id: string; reader: string; label: string; stated?
|
||||
* starts the next finding, except an unlabeled, unnumbered bullet with no file:line under a paragraph finding whose fields have not
|
||||
* come yet, which is its detail and joins its text (GSTA-196: 4 findings with detail bullets collected as 8). An inline
|
||||
* `Blocks: yes` is CRITICAL too. The native voice's FIXABLE collects as CRITICAL; other labels stay as written for lead judgment. When in doubt the parser merges rather than drops: a lost blocker costs the gate.
|
||||
* With fixableRaises false (a code or docs review with no risk, error, history or floor reason, GST-987) a native FIXABLE finding that
|
||||
* states `Blocks: no` collects at its own Severity field; one with no Blocks: no, no Severity field or only a FIXABLE field stays CRITICAL.
|
||||
*/
|
||||
export function parseFindings(reader: string, output: string): Array<{ label: string; raw: string }> {
|
||||
type Finding = { label: string; raw: string; fields: Set<string>; list: boolean; lead?: string };
|
||||
export function parseFindings(reader: string, output: string, fixableRaises = true): Array<{ label: string; raw: string }> {
|
||||
type Finding = { label: string; raw: string; fields: Set<string>; list: boolean; lead?: string; fixable?: boolean; heading?: boolean };
|
||||
const out: Finding[] = [];
|
||||
let section: string | null = null;
|
||||
let current: Finding | null = null;
|
||||
@@ -115,9 +119,9 @@ export function parseFindings(reader: string, output: string): Array<{ label: st
|
||||
const named = (l: string) => (role === 'adversarial-native' && l.toUpperCase() === 'FIXABLE' ? 'CRITICAL' : l);
|
||||
const strongestOf = (labels: Array<string | undefined>) => labels.reduce<string | undefined>((a, b) => (a === undefined ? b : b === undefined ? a : stronger(a, b)), undefined);
|
||||
const strongest = (t: string) => strongestOf([...t.matchAll(ALL_LABELS)].map(m => named(m[1]!)));
|
||||
const fieldOf = (t: string) => { const ls = [...t.matchAll(SEVERITY_FIELD)].flatMap(m => (m[1] ?? m[2]!).match(FIELD_LABELS)!); return ls.length === 1 ? ls[0] : undefined; };
|
||||
const critical = (t: string) => BLOCKS_YES.test(t) || (role === 'adversarial-native' && /(?<![A-Za-z0-9])FIXABLE(?![A-Za-z0-9])/.test(t));
|
||||
const leadOf = (t: string) => { const l = LEAD_LABEL.exec(t)?.[1]; return l === undefined ? undefined : named(l); };
|
||||
const fieldOf = (t: string) => { const ls = [...t.matchAll(SEVERITY_FIELD)].flatMap(m => (m[1] ?? m[2]!).match(FIELD_LABELS)!); return ls.length === 1 && (fixableRaises || role !== 'adversarial-native' || rank(ls[0]!)) ? ls[0] : undefined; };
|
||||
const critical = (t: string) => BLOCKS_YES.test(t) || (fixableRaises && role === 'adversarial-native' && /(?<![A-Za-z0-9])FIXABLE(?![A-Za-z0-9])/.test(t));
|
||||
const leadOf = (t: string) => { const l = LEAD_LABEL.exec(t)?.[1]; return l === undefined || !fixableRaises ? l : named(l); };
|
||||
const labelOf = (t: string) => {
|
||||
const field = fieldOf(t);
|
||||
if (field === undefined) return strongest(t);
|
||||
@@ -150,7 +154,7 @@ export function parseFindings(reader: string, output: string): Array<{ label: st
|
||||
continue;
|
||||
}
|
||||
const heading = LABEL_HEADING.exec(t);
|
||||
if (heading) { section = named(heading[1]!); current = null; head = ''; continue; }
|
||||
if (heading) { section = fixableRaises ? named(heading[1]!) : heading[1]!; current = null; head = ''; continue; }
|
||||
if (!indented && HEADING.test(t)) { section = null; current = null; }
|
||||
if (t.startsWith('{')) {
|
||||
try {
|
||||
@@ -160,12 +164,20 @@ export function parseFindings(reader: string, output: string): Array<{ label: st
|
||||
}
|
||||
const detail = listItem && /^[-*+]\s/.test(t) && !locus && !label && !section && !!current && !current.list && !current.fields.size;
|
||||
if (detail && !blocks) { current!.raw += ` | ${t}`; continue; }
|
||||
if (afterGap && current?.heading && !current.fields.size && !LIST_ITEM.test(t) && !HEADING.test(t) && !NUMBERED_HEAD.test(t) && !LEAD_LABEL.test(t) && !LOCUS.test(t)) {
|
||||
current.raw += ` | ${t}`;
|
||||
if (blocks) current.label = 'CRITICAL';
|
||||
else if (label) current.label = stronger(current.label, label);
|
||||
continue;
|
||||
}
|
||||
const fresh = (listItem && !detail) || (!indented && !detail && (t.startsWith('**') || afterGap)) || (!!current?.fields.size && locus);
|
||||
if (fresh) { current = null; head = t; }
|
||||
const inherited = section && listItem ? section : undefined;
|
||||
const found = label && inherited ? stronger(inherited, label) : label ?? inherited;
|
||||
if (found) {
|
||||
current = push(blocks ? 'CRITICAL' : found, t, [], strongestOf([leadOf(t), inherited, fieldOf(t)]));
|
||||
current.fixable = inherited?.toUpperCase() === 'FIXABLE';
|
||||
current.heading = !indented && NUMBERED_HEAD.test(t);
|
||||
if (!indented) head = t;
|
||||
continue;
|
||||
}
|
||||
@@ -176,7 +188,20 @@ export function parseFindings(reader: string, output: string): Array<{ label: st
|
||||
}
|
||||
if (!fresh && !current && (!LIST_ITEM.test(head) || locus)) head = t;
|
||||
}
|
||||
return out.map(({ label, raw }) => ({ label, raw }));
|
||||
const unblocked = (f: Finding) => fixableRaises || role !== 'adversarial-native' || !(f.fixable || f.label.toUpperCase() === 'FIXABLE' || /(?<![A-Za-z0-9])FIXABLE(?![A-Za-z0-9])/.test(f.raw)) || BLOCKS_NO.test(f.raw);
|
||||
return out.map(f => ({ label: unblocked(f) ? f.label : 'CRITICAL', raw: f.raw }));
|
||||
}
|
||||
|
||||
/**
|
||||
* Whether the native voice's FIXABLE raises a finding with its own Severity field to CRITICAL (GST-987): true on the gate class, on any
|
||||
* risk reason (security, secret, auth, money, user data), a depth error, a history or floor escalation, and whenever the plan's
|
||||
* REVIEW_CLASS or REVIEW_DEPTH reasons cannot be read: only a code or docs plan whose every reason is `runtime=<n>` relaxes it.
|
||||
*/
|
||||
export function fixableRaises(planLines?: readonly string[]): boolean {
|
||||
const cls = planLines?.map(l => /^REVIEW_CLASS: (\S+)/.exec(l)?.[1]).find(Boolean);
|
||||
const reasons = planLines?.map(l => /^REVIEW_DEPTH: .*?\breasons=(\S+)/.exec(l)?.[1]).find(Boolean);
|
||||
if (!cls || !reasons) return true;
|
||||
return !['code', 'docs'].includes(cls) || !reasons.split(',').every(r => /^runtime=\d+$/.test(r));
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -201,17 +226,17 @@ const hasCleanMarker = (reader: string, output: string) => output.replace(/\r\n/
|
||||
* review: false. With unavailable (the outside voice only) the output is its failure text, recorded as missing coverage. With a
|
||||
* failed tree check (GST-482: the reply reviewed another checkout, or names none) nothing is merged and the output is no review.
|
||||
*/
|
||||
export function collect(ledger: string, reader: string, output: string, mergedFile: string, outputFile?: string, unavailable?: string, tree?: TreeCheck): { sha: string; lines: MergedLine[]; review: boolean } {
|
||||
export function collect(ledger: string, reader: string, output: string, mergedFile: string, outputFile?: string, unavailable?: string, tree?: TreeCheck, raises = true): { sha: string; lines: MergedLine[]; review: boolean } {
|
||||
const sha = promptHash(output);
|
||||
const prior = readLedger(ledger).filter(e => e.event === 'output' && e.reader === reader).length;
|
||||
const wrongTree = !unavailable && tree !== undefined && !tree.ok;
|
||||
const lines = unavailable || wrongTree ? [] : parseFindings(reader, output).map((f, i) => {
|
||||
const lines = unavailable || wrongTree ? [] : parseFindings(reader, output, raises).map((f, i) => {
|
||||
const s = stated(f.raw);
|
||||
return { id: `${reader}#${prior ? `${prior + 1}.` : ''}${i + 1}`, reader, label: f.label, ...(s ? { stated: s } : {}), raw: f.raw, output_sha256: sha };
|
||||
});
|
||||
const review = !unavailable && !wrongTree && (lines.length > 0 || hasCleanMarker(reader, output));
|
||||
fs.appendFileSync(mergedFile, lines.map(l => JSON.stringify(l) + '\n').join(''), { mode: 0o600 });
|
||||
appendEvent(ledger, { event: 'output', reader, sha256: sha, file: outputFile ? path.resolve(outputFile) : null, merged: path.resolve(mergedFile), ids: lines.map(l => l.id), review, ...(unavailable ? { unavailable } : {}),
|
||||
appendEvent(ledger, { event: 'output', reader, sha256: sha, file: outputFile ? path.resolve(outputFile) : null, merged: path.resolve(mergedFile), ids: lines.map(l => l.id), review, ...(unavailable ? { unavailable } : {}), ...(raises ? {} : { fixable_raises: false }),
|
||||
...(tree && !unavailable ? { tree: tree.expected, reported_tree: tree.reported, tree_ok: tree.ok } : {}) });
|
||||
return { sha, lines, review };
|
||||
}
|
||||
@@ -408,7 +433,7 @@ export function verify(ledger: string): { rows: VerifyRow[]; clean: boolean; bri
|
||||
notes.push(typeof o.file === 'string' ? 'output_file_missing' : 'output_file_unrecorded');
|
||||
differs.push(...ids);
|
||||
} else if (promptHash(text) === o.sha256) {
|
||||
const parsed = parseFindings(reader, text);
|
||||
const parsed = parseFindings(reader, text, o.fixable_raises !== false);
|
||||
const off = ids.filter((id, k) => { const m = lines.find(l => l.id === id); return !m || parsed[k]?.label !== m.label || parsed[k]?.raw !== m.raw; });
|
||||
if (unavailableOutput(o) && parsed.length) { status = 'MISMATCH'; notes.push('unavailable_output_has_findings'); }
|
||||
else if (!unavailableOutput(o)) {
|
||||
|
||||
+7
-6
@@ -43,7 +43,7 @@ import { sha256Hex, writeReceipt } from './egress-receipt';
|
||||
import { runBashSync } from './script-invocation';
|
||||
|
||||
export interface RunProfile { workflow: string; events: string[]; jobs: string[]; artifact: string; max_age_hours?: number; os?: string }
|
||||
export interface RemoteGateRuns { profiles: string[]; inventory: string }
|
||||
export interface RemoteGateRuns { profiles: string[]; inventory: string; skipped?: string }
|
||||
export interface ApiRun {
|
||||
id: number; run_attempt: number; status: string; conclusion: string | null; event: string; head_sha: string;
|
||||
path?: string; created_at?: string; run_started_at?: string; updated_at?: string;
|
||||
@@ -108,15 +108,16 @@ export function runProfileErrors(profiles: unknown, gateRuns: unknown): Array<{
|
||||
}
|
||||
if (gateRuns !== undefined && gateRuns !== null) {
|
||||
const g = gateRuns as Record<string, unknown>;
|
||||
if (!g || typeof g !== 'object' || Array.isArray(g)) errors.push({ path: 'remote_gate_runs', message: 'must be { profiles: [names], inventory: command }' });
|
||||
if (!g || typeof g !== 'object' || Array.isArray(g)) errors.push({ path: 'remote_gate_runs', message: 'must be { profiles: [names], inventory: command, skipped?: command }' });
|
||||
else {
|
||||
for (const k of Object.keys(g)) if (k !== 'profiles' && k !== 'inventory') errors.push({ path: `remote_gate_runs.${k}`, message: 'unknown key; valid keys: profiles, inventory' });
|
||||
for (const k of Object.keys(g)) if (k !== 'profiles' && k !== 'inventory' && k !== 'skipped') errors.push({ path: `remote_gate_runs.${k}`, message: 'unknown key; valid keys: profiles, inventory, skipped' });
|
||||
if (!Array.isArray(g.profiles) || g.profiles.length === 0 || g.profiles.some(p => typeof p !== 'string')) errors.push({ path: 'remote_gate_runs.profiles', message: 'must be a non-empty array of run_profiles names' });
|
||||
else {
|
||||
for (const p of g.profiles as string[]) if (!names.has(p)) errors.push({ path: 'remote_gate_runs.profiles', message: `${JSON.stringify(p)} is not a run_profiles name` });
|
||||
if (new Set(g.profiles).size !== g.profiles.length) errors.push({ path: 'remote_gate_runs.profiles', message: 'names each profile once' });
|
||||
}
|
||||
if (typeof g.inventory !== 'string' || !g.inventory.trim() || g.inventory.length > 512 || /[\0\r\n]/.test(g.inventory)) errors.push({ path: 'remote_gate_runs.inventory', message: 'must be one command line under 512 characters that prints the remote gate\'s test files, one per line' });
|
||||
if ('skipped' in g && (typeof g.skipped !== 'string' || !g.skipped.trim() || g.skipped.length > 512 || /[\0\r\n]/.test(g.skipped))) errors.push({ path: 'remote_gate_runs.skipped', message: 'must be one command line under 512 characters that prints which selected test files (PREGATE_FILES_FILE) the remote gate itself does not run, one per line' });
|
||||
}
|
||||
}
|
||||
return errors;
|
||||
@@ -201,9 +202,9 @@ export function runChanged(deps: RunDeps, id: number, attempt: number): string |
|
||||
return again.run_attempt === attempt && again.status === 'completed' && again.conclusion === 'success' ? null : `run ${id} moved to attempt ${again.run_attempt} (${again.status === 'completed' ? again.conclusion : again.status}) while it was being verified`;
|
||||
}
|
||||
|
||||
/** The remote gate's inventory: `command`'s non-empty output lines under pipefail, so a pipeline whose producer fails refuses instead of listing part of the gate. */
|
||||
export function runInventory(command: string, cwd: string): string[] | { error: string } {
|
||||
const r = runBashSync(`set -o pipefail\n${command}`, { cwd, encoding: 'utf8', timeout: 300_000, maxBuffer: 64 * 1024 * 1024 });
|
||||
/** The remote gate's inventory (or skipped list, with PREGATE_FILES_FILE in `env`): `command`'s non-empty output lines under pipefail, so a pipeline whose producer fails refuses instead of listing part of the gate. */
|
||||
export function runInventory(command: string, cwd: string, env?: Record<string, string>): string[] | { error: string } {
|
||||
const r = runBashSync(`set -o pipefail\n${command}`, { cwd, encoding: 'utf8', timeout: 300_000, maxBuffer: 64 * 1024 * 1024, ...(env && { env: { ...process.env, ...env } }) });
|
||||
if (r.status !== 0 || r.error) return { error: `${command} exited ${r.status ?? r.error}: ${(r.stderr ?? '').trim().slice(0, 200)}` };
|
||||
return [...new Set(r.stdout.split('\n').map(l => l.trim()).filter(Boolean))];
|
||||
}
|
||||
|
||||
+38
-22
@@ -27,7 +27,7 @@ import { versionAtRef } from './restamp';
|
||||
import { globToRegExp, type ShipPolicy } from './ship-policy';
|
||||
import { ciAtHead, type QueueContext } from './ship-queue';
|
||||
import { git, sha256, type QueueFailure } from './ship-queue-gate';
|
||||
import { fetchBase, mergeCommit, outFailed, outOk, outRefused, parsePins, trainEnv, type TrainEnv, type TrainOut } from './ship-queue-train';
|
||||
import { fetchBase, mergeCommit, mergeStep, outFailed, outOk, outRefused, parsePins, trainEnv, type TrainEnv, type TrainOut } from './ship-queue-train';
|
||||
import { prInfo, type PrInfo } from './ship-queue-train-github';
|
||||
import { listWaves, loadWave, nowIso, persist, queueRoot, rebuildFromNotes, redactCause, wavePath, type ReadyReceipt, type WaveRecord } from './ship-queue-train-record';
|
||||
|
||||
@@ -67,7 +67,8 @@ export function areaKey(file: string, isSkillDir: (dir: string) => boolean): str
|
||||
return parts.length > 1 ? top : 'root';
|
||||
}
|
||||
|
||||
export interface PlanPr { pr: string; head: string; files: string[]; lines: number }
|
||||
/** renamed maps a path the PR renamed to its new path; files lists both. */
|
||||
export interface PlanPr { pr: string; head: string; files: string[]; lines: number; renamed?: Record<string, string> }
|
||||
export interface WaveGroup { area: string; prs: PlanPr[]; integrator: string }
|
||||
|
||||
/** The most common area across files, ties broken by name; 'misc' with no files. */
|
||||
@@ -91,11 +92,13 @@ export const isHubFile = (file: string) => file.startsWith('docs/') || /\.(md|js
|
||||
* Group PRs into waves: PRs whose changes to a common code file conflict always travel together
|
||||
* (one integrator resolves the conflict once); hub files never link. conflicts(a, b) names the
|
||||
* files a merge of the two heads cannot resolve, or null when unknown, which counts every shared
|
||||
* file as conflicting. Each cluster takes its largest PR's area, clusters of one area are packed
|
||||
* in PR order into waves of at most max, and a wave below min rides solo. The integrator is the
|
||||
* PR with the most changed lines.
|
||||
* file as conflicting; a conflict at the path either PR renamed the shared file to counts as one
|
||||
* on that file. Each cluster takes its largest PR's area and is packed whole, first fit in PR
|
||||
* order, into the waves of its area of at most max; a cluster larger than max is oversize (never
|
||||
* split, so its conflicts still meet one integrator), and a wave below min rides solo. The
|
||||
* integrator is the PR with the most changed lines.
|
||||
*/
|
||||
export function groupWaves(prs: PlanPr[], areaOf: (f: string) => string, min = WAVE_MIN, max = WAVE_MAX, conflicts: (a: PlanPr, b: PlanPr) => string[] | null = () => null): { groups: WaveGroup[]; solo: string[] } {
|
||||
export function groupWaves(prs: PlanPr[], areaOf: (f: string) => string, min = WAVE_MIN, max = WAVE_MAX, conflicts: (a: PlanPr, b: PlanPr) => string[] | null = () => null): { groups: WaveGroup[]; solo: string[]; oversize: Array<{ area: string; prs: PlanPr[] }> } {
|
||||
const sorted = [...prs].sort((a, b) => Number(a.pr) - Number(b.pr));
|
||||
const root = new Map(sorted.map(p => [p.pr, p.pr]));
|
||||
const find = (x: string): string => { const r = root.get(x)!; return r === x ? x : find(r); };
|
||||
@@ -106,7 +109,8 @@ export function groupWaves(prs: PlanPr[], areaOf: (f: string) => string, min = W
|
||||
const key = `${a.pr} ${b.pr}`;
|
||||
if (!clash.has(key)) clash.set(key, conflicts(a, b));
|
||||
const hit = clash.get(key);
|
||||
if ((!hit || hit.includes(f)) && find(a.pr) !== find(b.pr)) root.set(find(b.pr), find(a.pr));
|
||||
const at = [f, a.renamed?.[f], b.renamed?.[f]];
|
||||
if ((!hit || at.some(p => p && hit.includes(p))) && find(a.pr) !== find(b.pr)) root.set(find(b.pr), find(a.pr));
|
||||
}));
|
||||
const clusters = new Map<string, PlanPr[]>();
|
||||
for (const p of sorted) clusters.set(find(p.pr), [...(clusters.get(find(p.pr)) ?? []), p]);
|
||||
@@ -117,17 +121,20 @@ export function groupWaves(prs: PlanPr[], areaOf: (f: string) => string, min = W
|
||||
}
|
||||
const groups: WaveGroup[] = [];
|
||||
const solo: string[] = [];
|
||||
const oversize: Array<{ area: string; prs: PlanPr[] }> = [];
|
||||
for (const [area, list] of [...byArea].sort((a, b) => a[0].localeCompare(b[0]))) {
|
||||
let cur: PlanPr[] = [];
|
||||
const flush = () => {
|
||||
if (cur.length >= min) groups.push({ area, prs: cur, integrator: [...cur].sort(largestFirst)[0]!.pr });
|
||||
else solo.push(...cur.map(p => p.pr));
|
||||
cur = [];
|
||||
};
|
||||
for (const c of list) for (const p of c) { if (cur.length >= max) flush(); cur.push(p); }
|
||||
flush();
|
||||
const waves: PlanPr[][] = [];
|
||||
for (const c of list) {
|
||||
if (c.length > max) { oversize.push({ area, prs: c }); continue; }
|
||||
const fit = waves.find(w => w.length + c.length <= max);
|
||||
if (fit) fit.push(...c); else waves.push([...c]);
|
||||
}
|
||||
for (const w of waves) {
|
||||
if (w.length >= min) groups.push({ area, prs: w, integrator: [...w].sort(largestFirst)[0]!.pr });
|
||||
else solo.push(...w.map(p => p.pr));
|
||||
}
|
||||
}
|
||||
return { groups, solo: solo.sort((a, b) => Number(a) - Number(b)) };
|
||||
return { groups, solo: solo.sort((a, b) => Number(a) - Number(b)), oversize };
|
||||
}
|
||||
|
||||
export interface WaveChain { integrator: string[]; merges: string[]; stamps: string[] }
|
||||
@@ -225,10 +232,14 @@ function readyReceipts(env: TrainEnv): ReadyReceipt[] {
|
||||
return [...newest.values()];
|
||||
}
|
||||
|
||||
function changed(env: TrainEnv, head: string): { files: string[]; lines: number } | null {
|
||||
function changed(env: TrainEnv, head: string): { files: string[]; lines: number; renamed: Record<string, string> } | null {
|
||||
const fork = git(env.ctx.repoRoot, ['merge-base', `${env.remote}/${env.ctx.base}`, head]);
|
||||
const stat = fork.ok ? git(env.ctx.repoRoot, ['diff', '--numstat', '--no-renames', fork.out, head]) : fork;
|
||||
if (!stat.ok) return null;
|
||||
const moves = fork.ok ? git(env.ctx.repoRoot, ['diff', '--name-status', '-z', '-M', '--diff-filter=R', fork.out, head]) : fork;
|
||||
if (!stat.ok || !moves.ok) return null;
|
||||
const renamed: Record<string, string> = {};
|
||||
const fields = moves.out.split('\0');
|
||||
for (let i = 0; i + 2 < fields.length; i += 3) if (!isReleasePath(fields[i + 1]!, env.policy.policy)) renamed[fields[i + 1]!] = fields[i + 2]!;
|
||||
let lines = 0;
|
||||
const files: string[] = [];
|
||||
for (const row of stat.out.split('\n').filter(Boolean)) {
|
||||
@@ -237,7 +248,7 @@ function changed(env: TrainEnv, head: string): { files: string[]; lines: number
|
||||
files.push(file);
|
||||
lines += (Number(add) || 0) + (Number(del) || 0);
|
||||
}
|
||||
return { files, lines };
|
||||
return { files, lines, renamed };
|
||||
}
|
||||
|
||||
const areaOf = (env: TrainEnv) => (f: string) => areaKey(f, dir => fs.existsSync(path.join(env.ctx.repoRoot, dir, 'SKILL.md.tmpl')));
|
||||
@@ -269,12 +280,15 @@ export function wavePlan(ctx: QueueContext, o: { trigger: number; min: number; m
|
||||
if (ci.line.startsWith('unknown')) { unreadable.push(`#${r.pr} (CI ${ci.line})`); continue; }
|
||||
if (!ci.green) { lines.push(`NOT_GREEN: #${r.pr} (${ci.line}; left out until CI is green at the READY head)`); continue; }
|
||||
if (!ensureCommit(env, r.head, r.pr)) { unreadable.push(`#${r.pr} (head ${short(r.head)} not fetchable)`); continue; }
|
||||
const onBase = mergeStep(env.ctx.repoRoot, `${env.remote}/${env.ctx.base}`, r.head, env.policy.policy);
|
||||
if ('conflicts' in onBase && onBase.conflicts.some(f => f.startsWith('('))) { unreadable.push(`#${r.pr} (merge onto ${env.ctx.base} unreadable: ${onBase.conflicts.join(', ')})`); continue; }
|
||||
if ('conflicts' in onBase) { lines.push(`NOT_MERGEABLE: #${r.pr} (conflicts with ${env.remote}/${env.ctx.base} at ${short(r.head)} in ${onBase.conflicts.join(', ')}; left out until the head merges cleanly onto ${env.ctx.base})`); continue; }
|
||||
const c = changed(env, r.head);
|
||||
if (!c) { unreadable.push(`#${r.pr} (diff unreadable)`); continue; }
|
||||
live.push({ pr: r.pr, head: r.head, ...c });
|
||||
}
|
||||
if (unreadable.length) return outFailed(fail('QUEUE_PROOF_MISSING', `cannot read ${unreadable.length} READY PR(s): ${unreadable.join('; ')}; a plan from a partial queue would mislead`), lines, `${Q} wave plan (after gh and git can read every PR)`, 'none');
|
||||
lines.push(`QUEUE: ${live.length} live READY PR(s) (open, head equal to its READY receipt, CI green there); wave trigger ${o.trigger}`);
|
||||
lines.push(`QUEUE: ${live.length} live READY PR(s) (open, head equal to its READY receipt, CI green there, merging cleanly onto ${env.ctx.base}); wave trigger ${o.trigger}`);
|
||||
if (live.length <= o.trigger) return outOk([...lines, `WAVE: not needed (${live.length} <= ${o.trigger}); build trains of single PRs`], `${Q} train build --prs <pins>`, 'none');
|
||||
const plan = groupWaves(live, areaOf(env), o.min, o.max, (a, b) => {
|
||||
const m = git(env.ctx.repoRoot, ['merge-tree', '--write-tree', '--name-only', '--no-messages', a.head, b.head]);
|
||||
@@ -285,6 +299,7 @@ export function wavePlan(ctx: QueueContext, o: { trigger: number; min: number; m
|
||||
lines.push(`WAVE_GROUP: area=${g.area} prs=${g.prs.map(p => '#' + p.pr).join(',')} integrator=#${g.integrator} (most changed lines)`);
|
||||
lines.push(` build: ${buildCmd(g.area, g.prs)}`);
|
||||
}
|
||||
for (const c of plan.oversize) lines.push(`WAVE_OVERSIZE: area=${c.area} prs=${c.prs.map(p => '#' + p.pr).join(',')} (${c.prs.length} PRs whose changes conflict, over the wave maximum ${o.max}; kept whole with no build command: land part of it as single slots, then run wave plan again)`);
|
||||
if (plan.solo.length) lines.push(`SOLO: ${plan.solo.map(p => '#' + p).join(',')} (no area group of ${o.min}+; they ride as single slots)`);
|
||||
const first = plan.groups[0];
|
||||
return outOk(lines, first ? `#${first.integrator}'s owner integrates: ${buildCmd(first.area, first.prs)}` : `${Q} train build --prs <pins>`, 'none');
|
||||
@@ -577,8 +592,9 @@ export function waveSettle(ctx: QueueContext, o: { wave: string; dryRun: boolean
|
||||
export const WAVE_HELP = `gstack-ship-queue wave: wave PRs for merge trains (docs/ship-policy.md#wave-prs)
|
||||
|
||||
wave plan [--trigger <n>] [--min <n>] [--max <n>]
|
||||
read-only: live READY PRs; over the trigger (default ${WAVE_TRIGGER}) WAVE_ADVISED and
|
||||
WAVE_GROUP lines (area, PRs, integrator) with each group's build command
|
||||
read-only: live READY PRs (NOT_GREEN and NOT_MERGEABLE heads left out); over the
|
||||
trigger (default ${WAVE_TRIGGER}) WAVE_ADVISED and WAVE_GROUP lines (area, PRs, integrator)
|
||||
with each group's build command; WAVE_OVERSIZE for linked PRs over --max (no command)
|
||||
wave build --prs <n>@<sha>,... [--area <name>] [--min <n>] [--max <n>] [--dry-run]
|
||||
the integrator branch wave/<nonce>: main plus each pin merged unchanged (train merge
|
||||
rules), the wave record in refs/notes/gstack-trains, the branch pushed, a frozen notice
|
||||
|
||||
+2
-2
@@ -226,8 +226,8 @@ export function ready(ctx: QueueContext, o: { pr: string; pregate?: string; gate
|
||||
if (info.headRefOid !== head) return fail({ code: 'QUEUE_NOT_READY', message: `HEAD ${head.slice(0, 12)} is not PR #${o.pr}'s pushed head ${info.headRefOid.slice(0, 12)}` }, [], run);
|
||||
const atHead = versionAtRef(ctx.repoRoot, head);
|
||||
const baseSha = git(ctx.repoRoot, ['rev-parse', `origin/${ctx.base}`]).out;
|
||||
const atBase = versionAtRef(ctx.repoRoot, baseSha);
|
||||
const version = 'version' in atHead && 'version' in atBase && atHead.version !== atBase.version ? atHead.version : 'unstamped';
|
||||
const atFork = versionAtRef(ctx.repoRoot, git(ctx.repoRoot, ['merge-base', baseSha, head]).out || baseSha);
|
||||
const version = 'version' in atHead && 'version' in atFork && atHead.version !== atFork.version ? atHead.version : 'unstamped';
|
||||
const ci = ciAtHead(ctx, head);
|
||||
const pregateFile = path.resolve(ctx.repoRoot, o.pregate ?? PREGATE_OUT_REL);
|
||||
let pregateLine = `PREGATE: missing (${path.relative(ctx.repoRoot, pregateFile)}; run gstack-pregate run --allow-repo-commands)`;
|
||||
|
||||
@@ -7,6 +7,8 @@
|
||||
*/
|
||||
import { APPROVALS_FOOTER_END, APPROVALS_TABLE_HEADER } from './approval-record';
|
||||
import { RECEIPT_FENCE } from './ship-receipt';
|
||||
import { parseReviewLoopException, reviewLoopExceptionLine } from './ship-receipt-depth';
|
||||
import { docsExceptionLine, parseReceiptDocs } from './ship-receipt-docs';
|
||||
|
||||
export interface ProseClaim { field: string; prose: number; receipt: number }
|
||||
export interface ProseCheck { checked: number; mismatches: ProseClaim[] }
|
||||
@@ -28,6 +30,32 @@ export function receiptCounters(receipt: Record<string, unknown>): Map<string, n
|
||||
const BLOCK_RE = new RegExp('```' + RECEIPT_FENCE + '[ \\t]*\\r?\\n[\\s\\S]*?\\r?\\n```', 'g');
|
||||
const ESCAPE = /[.*+?^${}()|[\]\\]/g;
|
||||
const FENCE_OPEN = /^ {0,3}(`{3,}|~{3,})/;
|
||||
const fenceClose = (fence: string) => new RegExp(`^ {0,3}${fence[0] === '`' ? '`' : '~'}{${fence.length},}\\s*$`);
|
||||
|
||||
/**
|
||||
* The body without its closed code fences (GST-984): pasted test output (`113 pass, 0 fail`) and quoted code
|
||||
* (`the first 35 failing`) are evidence, not this PR's counters. The fence rules are withoutApprovalsBlocks' (a fence
|
||||
* closes only on a run of its own character at least as long as the opener with nothing after it). A backtick run with
|
||||
* another backtick later on its line (```npm test``` inline code) opens no fence, as in CommonMark. A fence that never
|
||||
* closes stays prose, and so does the rest of the body, so an unclosed opener cannot hide the claims after it or let a
|
||||
* shorter fence inside it hide them. A marker line takes each fence's place so the text on either side never joins
|
||||
* into a claim.
|
||||
*/
|
||||
function withoutFences(body: string): string {
|
||||
const lines = body.split(/(?<=\n)/);
|
||||
const out: string[] = [];
|
||||
for (let i = 0; i < lines.length; i++) {
|
||||
const line = lines[i]!.replace(/\r?\n$/, '');
|
||||
const open = FENCE_OPEN.exec(line);
|
||||
if (!open || (open[1]![0] === '`' && line.slice(open[0].length).includes('`'))) { out.push(lines[i]!); continue; }
|
||||
const close = fenceClose(open[1]!);
|
||||
const end = lines.findIndex((l, j) => j > i && close.test(l.replace(/\r?\n$/, '')));
|
||||
if (end < 0) return out.join('') + lines.slice(i).join('');
|
||||
out.push('(code fence)\n');
|
||||
i = end;
|
||||
}
|
||||
return out.join('');
|
||||
}
|
||||
|
||||
/**
|
||||
* The body without the `## Approvals` blocks gstack-approvals renders (lib/approval-record.ts renderRecord): the
|
||||
@@ -54,7 +82,7 @@ function withoutApprovalsBlocks(body: string): string {
|
||||
for (let i = 0; i < lines.length; i++) {
|
||||
const line = bare(i);
|
||||
if (fence !== null) {
|
||||
if (new RegExp(`^ {0,3}${fence[0] === '`' ? '`' : '~'}{${fence.length},}\\s*$`).test(line)) fence = null;
|
||||
if (fenceClose(fence).test(line)) fence = null;
|
||||
} else if (FENCE_OPEN.test(line)) fence = FENCE_OPEN.exec(line)![1]!;
|
||||
else if (/^ {0,3}## Approvals\s*$/.test(line)) {
|
||||
const end = blockEnd(i);
|
||||
@@ -65,9 +93,27 @@ function withoutApprovalsBlocks(body: string): string {
|
||||
return out.join('');
|
||||
}
|
||||
|
||||
/**
|
||||
* The body without the exception lines the receipt's own exceptions require: the `Review-loop exception: …` line
|
||||
* (GST-977) and the `Documentation exception: …` line (GST-983). write and the receipt check want each word for word,
|
||||
* and each quotes the owner's reason (`outside 1 and 2 skip detector`), not this PR's counters. Only a line whose
|
||||
* trimmed text is a rendered line, the test write applies, is left out; the same words inside a longer line stay
|
||||
* prose. A marker line takes its place so the text on either side never joins into a claim.
|
||||
*/
|
||||
function withoutExceptionLines(body: string, receipt: Record<string, unknown>): string {
|
||||
const lines = new Set<string>();
|
||||
const loop = receipt.review_loop_exception === undefined ? undefined : parseReviewLoopException(receipt.review_loop_exception);
|
||||
if (loop?.ok) lines.add(reviewLoopExceptionLine(loop.exception));
|
||||
const docs = receipt.docs === undefined ? undefined : parseReceiptDocs(receipt.docs);
|
||||
const docsLine = docs?.ok ? docsExceptionLine(docs.docs) : null;
|
||||
if (docsLine) lines.add(docsLine);
|
||||
if (!lines.size) return body;
|
||||
return body.split(/(?<=\n)/).map(l => (lines.has(l.trim()) ? '(exception line)\n' : l)).join('');
|
||||
}
|
||||
|
||||
/**
|
||||
* Every `<name>=<n>`, `<name>: <n>` and `<n> <name>` in the prose (the body
|
||||
* without its receipt block and its rendered Approvals block), where `<name>` is a counter's leaf key with `_`
|
||||
* without its receipt block, its code fences, its rendered Approvals block and its exception lines), where `<name>` is a counter's leaf key with `_`
|
||||
* matched as `_`, space or `-` and not part of a longer word (`3 pass-through`
|
||||
* is not a `pass` claim), compared with the receipt. A leaf key shared by
|
||||
* several counters (`pass` under every gate lane, `passed` under gate and ci)
|
||||
@@ -77,7 +123,7 @@ function withoutApprovalsBlocks(body: string): string {
|
||||
* `<n> <name>` claim about the next key (#3513).
|
||||
*/
|
||||
export function checkReceiptProse(receipt: Record<string, unknown>, body: string): ProseCheck {
|
||||
const prose = withoutApprovalsBlocks(body.replace(BLOCK_RE, ''));
|
||||
const prose = withoutExceptionLines(withoutApprovalsBlocks(withoutFences(body.replace(BLOCK_RE, ''))), receipt);
|
||||
const byLeaf = new Map<string, Array<{ field: string; value: number }>>();
|
||||
for (const [field, value] of receiptCounters(receipt)) {
|
||||
const leaf = field.split('.').at(-1)!;
|
||||
|
||||
Reference in new issue
Block a user