mirror of
https://github.com/garrytan/gstack.git
synced 2026-11-05 08:23:59 +01:00
feat(evals): the gate reruns failed cases once and reports a flaky green (GSTA-49 F1)
Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>
This commit is contained in:
garrytanandcapy-ai[bot] committed
1 parent
85b0f531f5
commit
c4099b4e49
21 files changed
+1327
-77
No files matched your search
+19
-6
@@ -9,9 +9,13 @@
|
||||
* report verdict and its plan's selection fingerprint. Reused runs
|
||||
* bind nothing, so reuse never chains off a reuse.
|
||||
* find (recover-receipts, BASE-REF code, actions: read) takes the newest
|
||||
* earlier completed run of this PR that bound a verdict. It must be
|
||||
* GREEN with reconcile exit 0 and a successful conclusion, ran the
|
||||
* PR profile, and gated a different head. Then one of:
|
||||
* earlier completed run of this PR that bound a verdict (its
|
||||
* newest attempt's binding). It must be GREEN with reconcile exit 0
|
||||
* and a successful conclusion, ran the PR profile, and gated a
|
||||
* different head. A flaky-annotated GREEN (rule 16: a red that
|
||||
* passed its one in-run rerun, or a `gh run rerun --failed`
|
||||
* attempt that passed every earlier red) is such a GREEN; a run
|
||||
* still red after its rerun binds RED and is never reused. Then one of:
|
||||
* restamp-only (R2a): same base; the tree receipt
|
||||
* (lib/tree-receipt.ts, the base's ship policy) says
|
||||
* `TREE: same-modulo-stamps` and every differing file is a stamp
|
||||
@@ -78,6 +82,8 @@ export interface GateBinding {
|
||||
/** Cases accepted manually (unscored); a gate with any is never reused as a clean pass. */
|
||||
manual_accepted: number;
|
||||
selection_sha256: string;
|
||||
/** Rule 16: cases that failed once and passed on their one rerun; the GREEN is reusable and carries their names. */
|
||||
flaky_cases?: string[];
|
||||
/** Dispatched screens only: the purpose, the refs/pull/<n>/merge commit it tested, and the screen policy. */
|
||||
purpose?: 'screen';
|
||||
tested_sha?: string;
|
||||
@@ -112,6 +118,8 @@ export interface GateReuse {
|
||||
own_diff_sha256?: string;
|
||||
/** The reused run was a dispatched screen with this policy. */
|
||||
reused_screen_policy?: ScreenPolicy;
|
||||
/** The reused GREEN was flaky-annotated (rule 16): these cases failed once and passed on their one rerun. */
|
||||
reused_flaky?: string[];
|
||||
}
|
||||
|
||||
/** The cases, trial plans and verdict-relevant plan fields a run executes; slice packing and receipts are excluded. */
|
||||
@@ -133,6 +141,7 @@ export function parseBinding(value: unknown): GateBinding | null {
|
||||
if (!b || b.schema !== 1 || typeof b.repository !== 'string' || !Number.isSafeInteger(b.pr) || !Number.isSafeInteger(b.run_id)
|
||||
|| !Number.isSafeInteger(b.run_attempt) || !SHA.test(b.head_sha ?? '') || !SHA.test(b.base_sha ?? '') || typeof b.profile !== 'string'
|
||||
|| !Number.isSafeInteger(b.reconcile_exit) || typeof b.verdict !== 'string' || !Number.isSafeInteger(b.manual_accepted) || b.manual_accepted < 0 || !SHA256.test(b.selection_sha256 ?? '')) return null;
|
||||
if (b.flaky_cases !== undefined && (!Array.isArray(b.flaky_cases) || b.flaky_cases.some(id => typeof id !== 'string' || !id))) return null;
|
||||
if (b.purpose !== undefined && (b.purpose !== 'screen' || !SHA.test(b.tested_sha ?? '') || (b.screen_policy !== 'narrow' && b.screen_policy !== 'broad'))) return null;
|
||||
return b;
|
||||
}
|
||||
@@ -278,13 +287,15 @@ export function buildBinding(o: { repository: string; pr: number; runId: number;
|
||||
if (!o.baseSha) return { binding: null, reason: 'the tested base is unknown' };
|
||||
const manifest = readJson<PaidRunManifest>(path.join(o.reportDir, 'manifest.json'));
|
||||
if (!manifest?.entries) return { binding: null, reason: 'no readable manifest' };
|
||||
const outcomes = readJson<{ verdict?: { verdict?: string }; totals?: { manual_accepted?: number } }>(path.join(o.reportDir, 'collector-outcomes.json'));
|
||||
const outcomes = readJson<{ verdict?: { verdict?: string }; totals?: { manual_accepted?: number }; flaky?: Array<{ case?: string }> }>(path.join(o.reportDir, 'collector-outcomes.json'));
|
||||
const flaky = (outcomes?.flaky ?? []).map(record => record.case ?? '');
|
||||
const binding: GateBinding = { schema: 1, repository: o.repository, pr: o.pr, run_id: o.runId, run_attempt: o.runAttempt,
|
||||
head_sha: o.headSha, base_sha: o.baseSha, profile: o.profile, reconcile_exit: o.reconcileExit,
|
||||
verdict: outcomes?.verdict?.verdict ?? 'MISSING', manual_accepted: outcomes?.totals?.manual_accepted ?? -1, selection_sha256: selectionFingerprint(manifest),
|
||||
...(flaky.length ? { flaky_cases: flaky } : {}),
|
||||
...(o.purpose === 'screen' ? { purpose: 'screen' as const, tested_sha: o.testedSha ?? '', screen_policy: o.screenPolicy as ScreenPolicy } : {}) };
|
||||
return parseBinding(binding)
|
||||
? { binding, reason: `bound ${binding.verdict} at ${binding.head_sha.slice(0, 12)} on base ${binding.base_sha.slice(0, 12)} (reconcile exit ${binding.reconcile_exit})` }
|
||||
? { binding, reason: `bound ${binding.verdict}${binding.flaky_cases ? ` (flaky: ${binding.flaky_cases.join(', ')})` : ''} at ${binding.head_sha.slice(0, 12)} on base ${binding.base_sha.slice(0, 12)} (reconcile exit ${binding.reconcile_exit})` }
|
||||
: { binding: null, reason: 'binding fields are incomplete' };
|
||||
}
|
||||
|
||||
@@ -419,7 +430,8 @@ export function findReusableGate(o: FindOptions): { candidate: GateReuse | null;
|
||||
if (policy.errors.length) return refuse(`the base's ship policy is invalid (${policy.errors[0]!.message})`);
|
||||
const common = { schema: 1 as const, pr: o.pr, reused_run_id: run.id, reused_run_attempt: binding.run_attempt, gated_head: binding.head_sha,
|
||||
current_head: o.headSha, gated_base: binding.base_sha, base_sha: baseSha, selection_sha256: binding.selection_sha256,
|
||||
...(binding.screen_policy ? { reused_screen_policy: binding.screen_policy } : {}) };
|
||||
...(binding.screen_policy ? { reused_screen_policy: binding.screen_policy } : {}),
|
||||
...(binding.flaky_cases?.length ? { reused_flaky: binding.flaky_cases } : {}) };
|
||||
|
||||
let candidate: GateReuse;
|
||||
if (binding.base_sha === baseSha) {
|
||||
@@ -484,6 +496,7 @@ export function reuseReport(reuse: GateReuse, repo: string): string {
|
||||
`GATE REUSED (${reuse.kind === 'restamp-only' ? 'R2a, restamp-only' : 'R2, main moved'}): no paid case ran on ${reuse.current_head}`,
|
||||
`reused verdict: GREEN from run ${reuse.reused_run_id} attempt ${reuse.reused_run_attempt}${reuse.reused_screen_policy ? ` (a ${reuse.reused_screen_policy} screen)` : ''} at ${reuse.gated_head} (https://github.com/${repo}/actions/runs/${reuse.reused_run_id})`,
|
||||
`why: ${reuse.why}`,
|
||||
...(reuse.reused_flaky?.length ? [`flaky (rule 16): ${reuse.reused_flaky.join(', ')} failed once and passed on their one rerun in the reused gate`] : []),
|
||||
`base: ${reuse.gated_base} -> ${reuse.base_sha}`,
|
||||
...evidence,
|
||||
`selection: ${reuse.selection_sha256} (identical to the reused gate)`,
|
||||
|
||||
@@ -12,11 +12,11 @@
|
||||
],
|
||||
"files": {
|
||||
"scripts/lib/paid-cases.ts": {"blob":"f862cf5241580c7593b9ee62eacf5ef046248d26","since":6,"nonBehavioral":"Claude-host verdicts are unchanged: the Codex host (EVALS_HOST=codex, codex-host-runner) runs only when selected and records its own per-model series; the Codex runner refactor (execCodex extraction, onSpawn hook) keeps its events and verdicts and adds token cost accounting; host fields on the plan/report are absent for claude"},
|
||||
"scripts/lib/paid-plan.ts": {"blob":"95e1798e7e2495ace350d2624e59f83412f341bc","since":6,"nonBehavioral":"Claude-host verdicts are unchanged: the Codex host (EVALS_HOST=codex, codex-host-runner) runs only when selected and records its own per-model series; the Codex runner refactor (execCodex extraction, onSpawn hook) keeps its events and verdicts and adds token cost accounting; host fields on the plan/report are absent for claude"},
|
||||
"scripts/lib/paid-report.ts": {"blob":"f92c6bb723b158e328aa040fa5de6a0ee6d3916d","since":6,"nonBehavioral":"Claude-host verdicts are unchanged: the Codex host (EVALS_HOST=codex, codex-host-runner) runs only when selected and records its own per-model series; the Codex runner refactor (execCodex extraction, onSpawn hook) keeps its events and verdicts and adds token cost accounting; host fields on the plan/report are absent for claude"},
|
||||
"scripts/lib/paid-plan.ts": {"blob":"e8ce3cb98474dded38c5128a631d283b0ca35b63","since":6,"nonBehavioral":"F1 rule-16 flaky rerun: report aggregation, rerun planning and the gate slice exit code; no case's execution, trial outcome, trial record or panel verdict changes, so no pass-rate series resets"},
|
||||
"scripts/lib/paid-report.ts": {"blob":"df19605183b8c6bfabd01bbe23fbe821d97a405e","since":6,"nonBehavioral":"F1 rule-16 flaky rerun: report aggregation, rerun planning and the gate slice exit code; no case's execution, trial outcome, trial record or panel verdict changes, so no pass-rate series resets"},
|
||||
"scripts/lib/published-text.ts": {"blob":"3f2e4d616b5dd0204035710c66b64d49055c5714","since":6},
|
||||
"scripts/lib/shard-engine.ts": {"blob":"3cd0b1aaa84018b00bf9ea0ce48093f3c8492f66","since":6},
|
||||
"scripts/test-paid-shards.ts": {"blob":"7a362c457cd0536dad60aecc675d67599759c8a3","since":6,"nonBehavioral":"GSTA-56: only the shebang changed, to the isolated form (--no-env-file --config=/dev/null); the paid runner is launched via bun run, which ignores the shebang, so no verdict can change"},
|
||||
"scripts/test-paid-shards.ts": {"blob":"b3b4786e406ca6793df2eb24fafac27cfb99a48e","since":6,"nonBehavioral":"F1 rule-16 flaky rerun: report aggregation, rerun planning and the gate slice exit code; no case's execution, trial outcome, trial record or panel verdict changes, so no pass-rate series resets"},
|
||||
"scripts/test-pr-profile.ts": {"blob":"d334195b0d0393622e2c380dc3e2feebe099b6e6","since":6,"nonBehavioral":"classify the new evals-main-backstop.yml workflow as free-lane only: it dispatches evals.yml on main and runs inside no case, so no case's selection on a PR, input or verdict changes"},
|
||||
"scripts/test-strict-output.ts": {"blob":"01185d20dc9761b04b9b2f751ec98664fe0eaf93","since":6},
|
||||
"test/helpers/agent-sdk-runner.ts": {"blob":"210bba92d241a90df578b2c7fc943ac5755d0025","since":6},
|
||||
|
||||
+34
-10
@@ -66,14 +66,17 @@ export interface PaidRunManifest {
|
||||
selection?: PaidCaseSelection;
|
||||
prCoverage?: PrProfileSelection;
|
||||
plan?: PaidSlicePlan;
|
||||
/** A rule-16 rerun manifest (scripts/lib/paid-rerun.ts): the run attempt whose reds it reruns once, and those cases. */
|
||||
rerun?: { attempt: number; cases: string[] };
|
||||
entries: ManifestEntry[];
|
||||
}
|
||||
|
||||
/**
|
||||
* Paid evals never retry (approved 2026-09-29): a failed verdict is final for
|
||||
* its run, and trials are fixed by kind before the run (EVAL_POLICY). The
|
||||
* function stays the single statement of that policy for the Bun arguments
|
||||
* and the reuse identity.
|
||||
* Bun never retries a paid case inside a shard (approved 2026-09-29): trials
|
||||
* are fixed by kind before the run (EVAL_POLICY). The one rule-16 rerun of a
|
||||
* red case (CREW rule 16, scripts/lib/paid-rerun.ts) is a separate,
|
||||
* reported execution, never a Bun retry. The function stays the single
|
||||
* statement of that policy for the Bun arguments and the reuse identity.
|
||||
*/
|
||||
export function retriesForFiles(_files: string[]): number {
|
||||
return 0;
|
||||
@@ -514,6 +517,10 @@ export function parseRunManifest(raw: string): PaidRunManifest {
|
||||
throw new Error('manifest deferred case is outside the broad census');
|
||||
}
|
||||
}
|
||||
if (parsed.rerun !== undefined && (!Number.isSafeInteger(parsed.rerun?.attempt) || parsed.rerun.attempt < 1 || !Array.isArray(parsed.rerun.cases)
|
||||
|| parsed.rerun.cases.length === 0 || parsed.rerun.cases.some(id => typeof id !== 'string') || new Set(parsed.rerun.cases).size !== parsed.rerun.cases.length)) {
|
||||
throw new Error('manifest rerun block invalid');
|
||||
}
|
||||
if (!Number.isInteger(parsed.sliceCount) || parsed.sliceCount <= 0) throw new Error('manifest sliceCount invalid');
|
||||
if (!Array.isArray(parsed.entries)) throw new Error('manifest entries missing');
|
||||
for (const entry of parsed.entries) {
|
||||
@@ -665,6 +672,8 @@ export interface SliceResult {
|
||||
finishedAt?: number;
|
||||
/** A progress checkpoint the executor never replaced: the job ended (ceiling or cancellation) before its final result. */
|
||||
checkpoint?: true;
|
||||
/** Produced from a rule-16 rerun manifest (`manifest.rerun`), never a first-attempt slice. */
|
||||
rerun?: true;
|
||||
outcomes: Array<Pick<ShardOutcome, 'files' | 'status' | 'exitCode' | 'elapsedMs' | 'executedTests' | 'skippedTests' | 'budget' | 'reused' | 'runnerError' | 'trial' | 'inputKey' | 'sliceDeadline'>>;
|
||||
}
|
||||
|
||||
@@ -674,11 +683,18 @@ export interface SliceResult {
|
||||
* isolated trial shard fails it only when the harness produced no trial
|
||||
* record. Failed, timed-out or crashed trials are verdict input for the
|
||||
* report's panelVerdict(), so a 2/3 PASS panel never reds its runner.
|
||||
* `caseVerdicts` (the gate lane's CI slices, rule 16): a rule shard that ran
|
||||
* to completion and reported executed cases is verdict input too, so its red
|
||||
* cases stay in the report (which still fails closed on them) and the run's
|
||||
* conclusion follows the report, where a flaky case can resolve green.
|
||||
*/
|
||||
export function sliceExitCode(outcomes: ReadonlyArray<Pick<ShardOutcome, 'status' | 'trial'>>): number {
|
||||
export function sliceExitCode(outcomes: ReadonlyArray<Pick<ShardOutcome, 'status' | 'trial'> & Partial<Pick<ShardOutcome, 'exitCode' | 'executedTests' | 'skippedTests' | 'runnerError' | 'sliceDeadline'>>>,
|
||||
options: { caseVerdicts?: boolean } = {}): number {
|
||||
const completedRed = (o: (typeof outcomes)[number]) => !!options.caseVerdicts && o.status === 'failed' && o.exitCode !== null && o.exitCode !== undefined
|
||||
&& typeof o.executedTests === 'number' && o.executedTests - (o.skippedTests ?? 0) > 0 && o.runnerError === undefined && !o.sliceDeadline;
|
||||
return outcomes.every(outcome => outcome.trial !== undefined
|
||||
? outcome.trial.outcome !== null
|
||||
: outcome.status === 'passed' || outcome.status === 'skipped-by-diff') ? 0 : 1;
|
||||
: outcome.status === 'passed' || outcome.status === 'skipped-by-diff' || completedRed(outcome)) ? 0 : 1;
|
||||
}
|
||||
|
||||
/** A hollow-guarded trial shard has no trial record: the guard's verdict is a harness problem. */
|
||||
@@ -697,8 +713,16 @@ export function guardTrialRecords<T extends Pick<ShardOutcome, 'status' | 'trial
|
||||
export function verifySliceResults(
|
||||
manifest: PaidRunManifest,
|
||||
results: SliceResult[],
|
||||
/**
|
||||
* `flaky`: non-trial shards whose every failed case passed its one rule-16
|
||||
* rerun (scripts/lib/paid-rerun.ts); their non-zero exit and failed status
|
||||
* are not problems. `partial`: a later run attempt that re-ran only some
|
||||
* slices, so a slice it did not re-run is no problem.
|
||||
*/
|
||||
options: { flaky?: ReadonlySet<string>; partial?: boolean } = {},
|
||||
): { ok: boolean; problems: string[] } {
|
||||
const problems: string[] = [];
|
||||
const exitOk = (file: string, exitCode: number | null) => exitCode === 0 || !!options.flaky?.has(file);
|
||||
try { parseRunManifest(JSON.stringify(manifest)); }
|
||||
catch (error) { problems.push(`Invalid run manifest: ${error instanceof Error ? error.message : String(error)}`); }
|
||||
const byIndex = new Map<number, SliceResult>();
|
||||
@@ -713,7 +737,7 @@ export function verifySliceResults(
|
||||
byIndex.set(result.sliceIndex, result);
|
||||
}
|
||||
for (let index = 1; index <= manifest.sliceCount; index += 1) {
|
||||
if (!byIndex.has(index)) problems.push(`slice ${index}/${manifest.sliceCount} reported NO result — cancelled/crashed executor, not a pass`);
|
||||
if (!byIndex.has(index) && !options.partial) problems.push(`slice ${index}/${manifest.sliceCount} reported NO result — cancelled/crashed executor, not a pass`);
|
||||
}
|
||||
|
||||
const reported = new Map<string, { slice: number; status: ShardStatus; trial?: ShardTrialRecord; sliceDeadline?: ShardOutcome['sliceDeadline'] }>();
|
||||
@@ -731,7 +755,7 @@ export function verifySliceResults(
|
||||
? -1 : outcome.executedTests - outcome.skippedTests;
|
||||
// A trial's exit status is its verdict (panelVerdict decides); its
|
||||
// completeness is the trial record checked below.
|
||||
if ((!trialPlan && outcome.exitCode !== 0) || expected < 1 || (!trialPlan && executed !== expected)) {
|
||||
if ((!trialPlan && !exitOk(file, outcome.exitCode)) || expected < 1 || (!trialPlan && executed !== expected)) {
|
||||
problems.push(`PR profile expected ${expected} executed cases in ${file}, received ${executed}`);
|
||||
}
|
||||
}
|
||||
@@ -767,7 +791,7 @@ export function verifySliceResults(
|
||||
if (finding) {
|
||||
// Full-census runs must account for every registered case. A manifest
|
||||
// explicitly marked selective may report its executed subset.
|
||||
if (outcome.exitCode !== 0 || !Number.isInteger(outcome.executedTests) ||
|
||||
if (!exitOk(file, outcome.exitCode) || !Number.isInteger(outcome.executedTests) ||
|
||||
outcome.executedTests! < 1 || outcome.executedTests! > finding.cases ||
|
||||
(manifest.evalsAll !== false && outcome.executedTests !== finding.cases) || outcome.skippedTests !== 0) {
|
||||
problems.push(`Finding workflow must execute real unskipped cases with exit zero: ${file}`);
|
||||
@@ -794,7 +818,7 @@ export function verifySliceResults(
|
||||
// Isolated trial shards: harness health only; the panel verdict gates.
|
||||
if (entry.trial) {
|
||||
if (got.trial?.outcome === null) problems.push(`${entry.file}: no trial record (${got.trial.harness ?? 'unknown'})`);
|
||||
} else if (got.status !== 'passed') {
|
||||
} else if (got.status !== 'passed' && !(got.status === 'failed' && options.flaky?.has(normalizeRelativePath(entry.file)))) {
|
||||
// The slice deadline distinguishes work that never started (INFRA) from work killed in flight (TIMEOUT).
|
||||
const why = got.sliceDeadline === 'not_run' ? ' (not_run: the slice deadline passed before it started)'
|
||||
: got.sliceDeadline === 'hung' ? ' (hung: killed at the slice deadline)' : '';
|
||||
|
||||
+54
-22
@@ -19,13 +19,14 @@ import { writeNegativeReceipt, writePanelReceipt } from '../e2e-shard-reuse';
|
||||
import { E2E_TOUCHFILES, E2E_TIERS, LLM_JUDGE_TOUCHFILES } from '../../test/helpers/touchfiles';
|
||||
import { type EvalHost, hostSkipReason } from '../../test/helpers/e2e-hosts';
|
||||
import { CASE_TEST_NAMES, type CaseTrialPlan, caseTestNamePattern, caseTrialPlan, fileCaseRegistration, shardCaseId, shardFile, trialShardKey } from './paid-cases';
|
||||
import { type ManifestEntry, PAID_TEST_DURATIONS_FILE, type PaidRunManifest, type SliceResult, collectorOutcomeCounts, formatProfileCoverage, loadPaidTestDurations, mergePaidTestDurations, parseRunManifest, trialPanelKey, verifySliceResults, writePaidTestDurations } from './paid-plan';
|
||||
import { type ManifestEntry, PAID_TEST_DURATIONS_FILE, type PaidRunManifest, type SliceResult, collectorOutcomeCounts, formatProfileCoverage, loadPaidTestDurations, mergePaidTestDurations, parseRunManifest, trialPanelKey, writePaidTestDurations } from './paid-plan';
|
||||
import { DEFAULT_JOBS, type PaidCaseSelection, type PaidTier, ROOT, type ShardTrialRecord } from './paid-types';
|
||||
import { collectPaidTestFiles, isAllSkippedPass, shardSlug, paidSelectionEnv } from './paid-select';
|
||||
import type { RunShardsOptions } from '../test-paid-shards';
|
||||
import { attributeCommands, formatMapGap, mapGaps, readShardCommands } from '../exercised-map';
|
||||
import { checkoutExercisedConsumers } from '../pr-dependencies';
|
||||
import { PR_FULL_GATE_FILES, PR_PROFILE_MAPS } from '../test-pr-profile';
|
||||
import { FLAKE_RECORDS_FILE, flakeRecords, flakyShards, formatFlakyLine, gateProblems, rerunSpend, rerunState } from './paid-rerun';
|
||||
|
||||
// ─── Local diagnosis: one case through the CI panel runner (A9) ────────────
|
||||
|
||||
@@ -215,7 +216,7 @@ export function attributeJUnitCase(tc: JUnitCase, ctx: JUnitShardContext,
|
||||
return { kind: 'case', id };
|
||||
}
|
||||
|
||||
interface RuleCase { id: string; kind: EvalCaseKind; outcome: TrialOutcome; line?: string }
|
||||
export interface RuleCase { id: string; kind: EvalCaseKind; outcome: TrialOutcome; key: string; slice: number; failureClass?: TrialFailureClass; error?: string; costUsd: number; costKnown: boolean; line?: string }
|
||||
|
||||
/** JUnit evidence of the non-trial shards of one attempt. */
|
||||
export interface JUnitCensus {
|
||||
@@ -246,7 +247,7 @@ export const CENSUS_RED_GUIDE = 'docs/evals/census-red.md';
|
||||
|
||||
export function sliceArtifactName(root: string, slice: number, attempt: number): string {
|
||||
const name = path.basename(root);
|
||||
return /^(?:paid-slice|gate-census)-\d+-a\d+(?:-[a-z0-9-]+)?$/.test(name) ? name : `slice ${slice} attempt ${attempt}`;
|
||||
return /^(?:paid-slice|paid-rerun|gate-census)-\d+-a\d+(?:-[a-z0-9-]+)?$/.test(name) ? name : `slice ${slice} attempt ${attempt}`;
|
||||
}
|
||||
|
||||
export function junitCensus(manifest: PaidRunManifest, artifacts: ReportArtifact[], recordsByShard: Map<string, any[]>,
|
||||
@@ -288,7 +289,8 @@ export function junitCensus(manifest: PaidRunManifest, artifacts: ReportArtifact
|
||||
bump(reasons, RUNTIME_SKIP_REASON);
|
||||
census.skipped.push(`◌ ${id} ${key} (${cases.map(tc => tc.name).join('; ')}): ${RUNTIME_SKIP_REASON}`);
|
||||
}
|
||||
census.ruleCases.push({ id, kind, outcome: caseOutcome,
|
||||
census.ruleCases.push({ id, kind, outcome: caseOutcome, key, slice: result.sliceIndex, ...(failureClass ? { failureClass } : {}), ...(error && failed ? { error } : {}),
|
||||
costUsd: Math.round(mine.reduce((sum: number, r: any) => sum + (Number(r.cost_usd) || 0), 0) * 100) / 100, costKnown: trialCostKnown(mine, sessions).cost_known !== false,
|
||||
...(failed ? { line: `✗ ${id} ${kind} FAIL ${failureClass}${failedRecord?.exit_reason === 'timeout' && failedRecord?.timeout_at_turn !== undefined ? ` at turn ${failedRecord.timeout_at_turn}` : ''}${error ? ` — ${error}` : ''}`
|
||||
+ `${diagnosisText(diagnosis, error)} [slice ${result.sliceIndex}, attempt ${result.attempt ?? 1}]`
|
||||
+ `${evidencePointer(sliceArtifactName(root, result.sliceIndex, result.attempt ?? 1), key, typeof base.run_id === 'string' ? base.run_id : undefined, id)}`
|
||||
@@ -318,15 +320,20 @@ export function formatJUnitCensus(census: JUnitCensus): string[] {
|
||||
];
|
||||
}
|
||||
|
||||
interface ReportArtifact { root: string; result: SliceResult }
|
||||
export interface ReportArtifact { root: string; result: SliceResult }
|
||||
|
||||
/** Every slice result under the report dir: flat (merged) or one directory per attempt-scoped artifact. */
|
||||
export function loadSliceArtifacts(reportDir: string): ReportArtifact[] {
|
||||
/**
|
||||
* Every slice result under the report dir: flat (merged) or one directory per
|
||||
* attempt-scoped artifact. Rule-16 rerun results (scripts/lib/paid-rerun.ts)
|
||||
* are returned only with `{ reruns: true }`, and then only they are.
|
||||
*/
|
||||
export function loadSliceArtifacts(reportDir: string, options: { reruns?: boolean } = {}): ReportArtifact[] {
|
||||
const found: ReportArtifact[] = [];
|
||||
for (const name of fs.readdirSync(reportDir, { recursive: true }) as string[]) {
|
||||
const rel = normalizeRelativePath(name);
|
||||
if (!/^slice-\d+\.json$/.test(path.basename(rel)) || rel.split('/').includes('shards') || rel.split('/').includes('receipts')) continue;
|
||||
found.push({ root: path.join(reportDir, path.dirname(rel)), result: JSON.parse(fs.readFileSync(path.join(reportDir, rel), 'utf8')) as SliceResult });
|
||||
const result = JSON.parse(fs.readFileSync(path.join(reportDir, rel), 'utf8')) as SliceResult;
|
||||
if ((result.rerun === true) === (options.reruns === true)) found.push({ root: path.join(reportDir, path.dirname(rel)), result });
|
||||
}
|
||||
return found.sort((a, b) => (a.result.attempt ?? 1) - (b.result.attempt ?? 1) || a.result.sliceIndex - b.result.sliceIndex);
|
||||
}
|
||||
@@ -452,7 +459,11 @@ export interface ReportHeadline {
|
||||
infra: number;
|
||||
incomplete: number;
|
||||
unattributed: number;
|
||||
/** Cases that failed once and passed on their one rule-16 rerun (scripts/lib/paid-rerun.ts). */
|
||||
flaky?: number;
|
||||
};
|
||||
/** Re-executions of once-red cases (rule-16 reruns and later attempts) and their cost, when any ran. */
|
||||
reruns?: { executions: number; usd: number; unknown: number };
|
||||
actionRequired: number;
|
||||
wallMs: number | null;
|
||||
costUsd: number;
|
||||
@@ -474,6 +485,8 @@ export function formatHeadline(h: ReportHeadline, cost: { unknownTrials: number;
|
||||
+ ` · judge ${c.judge.passed}/${c.judge.total} · quarantined ${c.quarantined.total} (${c.quarantined.failingLane} failing the lane)`,
|
||||
` SKIPPED ${c.skipped} · INFRA ${c.infra} · INCOMPLETE ${c.incomplete} · unattributed ${c.unattributed} · ACTION REQUIRED ${h.actionRequired}`,
|
||||
` wall ${minutes} · ${formatCost(h.costUsd, cost.unknownTrials, cost.trials)}${h.redispatchEligible ? ' · every red is machine-classified INFRA/INCOMPLETE: eligible for ONE re-dispatch as a new run (EVAL_POLICY.infraRedispatch); report both runs' : ''}`,
|
||||
...(c.flaky || h.reruns ? [` FLAKY ${c.flaky ?? 0} (failed once, passed on its one rule-16 rerun) · ${h.reruns?.executions ?? 0} rerun execution(s), `
|
||||
+ `${formatCost(h.reruns?.usd ?? 0, h.reruns?.unknown ?? 0, h.reruns?.executions ?? 0)}`] : []),
|
||||
];
|
||||
}
|
||||
|
||||
@@ -527,16 +540,20 @@ export function runPaidReport(reportDir: string, options: { writeDurations?: boo
|
||||
const summaryPath = path.join(reportDir, 'collector-outcomes.json');
|
||||
const summaryMdPath = path.join(reportDir, 'report-summary.md');
|
||||
const trialOutcomesPath = path.join(reportDir, TRIAL_OUTCOMES_FILE);
|
||||
for (const file of [summaryPath, summaryMdPath, trialOutcomesPath]) fs.rmSync(file, { force: true });
|
||||
for (const file of [summaryPath, summaryMdPath, trialOutcomesPath, path.join(reportDir, FLAKE_RECORDS_FILE)]) fs.rmSync(file, { force: true });
|
||||
const manifest = parseRunManifest(fs.readFileSync(path.join(reportDir, 'manifest.json'), 'utf-8'));
|
||||
const artifacts = loadSliceArtifacts(reportDir);
|
||||
const attempts = [...new Set(artifacts.map(a => a.result.attempt ?? 1))].sort((a, b) => a - b);
|
||||
const primary = attempts[0] ?? 1;
|
||||
const results = artifacts.filter(a => (a.result.attempt ?? 1) === primary).map(a => a.result);
|
||||
const verdict = verifySliceResults(manifest, results);
|
||||
// Rule 16 (scripts/lib/paid-rerun.ts): every attempt and each attempt's one rerun form one history per case; a flaky case's own red is no problem.
|
||||
const rerun = rerunState(reportDir, manifest, artifacts);
|
||||
const flakyFirst = flakyShards(rerun.observations.filter(o => o.attempt === primary && o.phase === 'first'), rerun.flaky);
|
||||
const verdict = { problems: gateProblems(rerun.sets, rerun.observations, rerun.flaky) };
|
||||
const planned = manifest.entries.filter((e) => e.status === 'planned').length;
|
||||
const lane = `${manifest.tier}/${manifest.profile ?? 'full'}${manifest.evalsAll ? ' census' : ''}${manifest.host ? ` on ${manifest.host}` : ''}`;
|
||||
console.log(`[test:paid] report: ${results.length}/${manifest.sliceCount} slices, ${planned} planned shards, tier=${manifest.tier}, attempt ${primary}${attempts.length > 1 ? ` (later attempts ${attempts.slice(1).join(', ')} reported, never replacing it)` : ''}`);
|
||||
console.log(`[test:paid] report: ${results.length}/${manifest.sliceCount} slices, ${planned} planned shards, tier=${manifest.tier}, attempt ${primary}${attempts.length > 1 ? ` (later attempts ${attempts.slice(1).join(', ')} reported, never replacing it); a later attempt resolves a red only by the rule-16 flaky rule and never hides a new red` : ''}`);
|
||||
for (const line of rerun.ignored) console.log(`[test:paid] rule-16 rerun ignored: ${line}`);
|
||||
for (const line of formatProfileCoverage(manifest)) console.log(line);
|
||||
for (const result of [...results].sort((a, b) => a.sliceIndex - b.sliceIndex)) {
|
||||
for (const outcome of result.outcomes) {
|
||||
@@ -555,6 +572,8 @@ export function runPaidReport(reportDir: string, options: { writeDurations?: boo
|
||||
const roots = artifacts.map(a => ({ root: path.resolve(a.root), attempt: a.result.attempt ?? 1 }))
|
||||
.sort((a, b) => b.root.length - a.root.length);
|
||||
const attemptOf = (abs: string) => roots.find(r => abs === r.root || abs.startsWith(r.root + path.sep))?.attempt ?? primary;
|
||||
const rerunRoots = loadSliceArtifacts(reportDir, { reruns: true }).map(a => path.resolve(a.root));
|
||||
const inRerun = (abs: string) => rerunRoots.some(root => abs === root || abs.startsWith(root + path.sep));
|
||||
const entryBySlug = new Map(manifest.entries.map(entry => [shardSlug([entry.file]), entry]));
|
||||
const shardOf = (rel: string) => {
|
||||
const parts = normalizeRelativePath(rel).split('/');
|
||||
@@ -571,10 +590,11 @@ export function runPaidReport(reportDir: string, options: { writeDurations?: boo
|
||||
const manualClaims = new Map<string, string>();
|
||||
const recordsByShard = new Map<string, any[]>();
|
||||
let costUsd = 0;
|
||||
let flakyCollectorFailures = 0;
|
||||
for (const name of fs.readdirSync(reportDir, { recursive: true }) as string[]) {
|
||||
const rel = normalizeRelativePath(name);
|
||||
if (!isFinalizedEvalResultFile(rel) || rel.split('/').includes('receipts') || rel === 'collector-outcomes.json') continue;
|
||||
if (attemptOf(path.resolve(reportDir, rel)) !== primary) continue;
|
||||
if (inRerun(path.resolve(reportDir, rel)) || attemptOf(path.resolve(reportDir, rel)) !== primary) continue;
|
||||
const shard = shardOf(rel);
|
||||
try {
|
||||
const parsed = JSON.parse(fs.readFileSync(path.join(reportDir, rel), 'utf-8'));
|
||||
@@ -611,6 +631,7 @@ export function runPaidReport(reportDir: string, options: { writeDurations?: boo
|
||||
}
|
||||
collectors.push(parsed);
|
||||
const counts = collectorOutcomeCounts([parsed]);
|
||||
if (shard && flakyFirst.has(normalizeRelativePath(shard.file))) flakyCollectorFailures += counts.failed;
|
||||
files.push({ file: rel, tier: parsed.tier ?? 'unknown', shard: parsed.shard ?? '-',
|
||||
cost: parsed.total_cost_usd ?? 0, flaky: parsed.flaky_retries?.length ?? 0,
|
||||
total: counts.passed + counts.failed + counts.manual_accepted, ...counts });
|
||||
@@ -626,7 +647,7 @@ export function runPaidReport(reportDir: string, options: { writeDurations?: boo
|
||||
for (const f of flaky) console.log(` ⚠ ${f.name} (x${f.attempts}) — ${f.file}`);
|
||||
}
|
||||
if (manualProblems.length) verdict.problems.push(...manualProblems);
|
||||
if (evidence.failed > 0) verdict.problems.push(`${evidence.failed} unapproved final collector failure(s)`);
|
||||
if (evidence.failed > flakyCollectorFailures) verdict.problems.push(`${evidence.failed - flakyCollectorFailures} unapproved final collector failure(s)`);
|
||||
if (files.reduce((sum, file) => sum + file.total, 0) !== evidence.passed + evidence.failed + evidence.manual_accepted
|
||||
|| files.reduce((sum, file) => sum + file.executed + file.reused, 0) !== evidence.executed + evidence.reused) {
|
||||
verdict.problems.push('Collector summary totals are inconsistent');
|
||||
@@ -634,9 +655,6 @@ export function runPaidReport(reportDir: string, options: { writeDurations?: boo
|
||||
|
||||
// Panel verdicts: one function, computed here only.
|
||||
const panels = panelReports(manifest, results, primary);
|
||||
for (const panel of panels.filter(p => p.failsLane)) {
|
||||
verdict.problems.push(`PANEL ${panel.case} ${panel.status}${panel.redClass === 'INFRA' ? ' (INFRA)' : ''} ${panel.passed}/${panel.panel.n} (${panel.marks}): ${panel.reason}`);
|
||||
}
|
||||
const laterPanels = attempts.slice(1).flatMap(attempt => panelReports(manifest, artifacts.map(a => a.result), attempt)
|
||||
.filter(panel => panel.trials.length > 0));
|
||||
|
||||
@@ -719,17 +737,22 @@ export function runPaidReport(reportDir: string, options: { writeDurations?: boo
|
||||
const behaviorPanels = panels.filter(p => !p.quarantined && p.kind === 'behavior');
|
||||
const lanePanels = panels.filter(p => !p.quarantined);
|
||||
const count = (kind: EvalCaseKind) => ({
|
||||
passed: ruleCases.filter(c => c.kind === kind && c.outcome === 'passed').length
|
||||
+ lanePanels.filter(p => p.kind === kind && p.status === 'PASS').length,
|
||||
passed: ruleCases.filter(c => c.kind === kind && (c.outcome === 'passed' || (c.outcome === 'failed' && rerun.flaky.has(c.id)))).length
|
||||
+ lanePanels.filter(p => p.kind === kind && (p.status === 'PASS' || rerun.flaky.has(p.case))).length,
|
||||
total: ruleCases.filter(c => c.kind === kind).length + lanePanels.filter(p => p.kind === kind).length,
|
||||
});
|
||||
const primaryTrials = results.flatMap(r => r.outcomes.map(o => o.trial)).filter((t): t is ShardTrialRecord => !!t);
|
||||
const wall = results.filter(r => Number.isSafeInteger(r.startedAt) && Number.isSafeInteger(r.finishedAt));
|
||||
const failureLines = [
|
||||
...ruleShardFailures,
|
||||
...ruleCases.filter(c => c.line).map(c => c.line!),
|
||||
...panels.filter(p => p.status !== 'PASS' || p.split).map(p => formatPanelLine(p, manifest.tier, { artifacts: sliceArtifacts, runId })),
|
||||
...ruleCases.filter(c => c.line && !rerun.flaky.has(c.id)).map(c => c.line!),
|
||||
...panels.filter(p => (p.status !== 'PASS' || p.split) && !rerun.flaky.has(p.case)).map(p => formatPanelLine(p, manifest.tier, { artifacts: sliceArtifacts, runId })),
|
||||
...[...rerun.resolutions.values()].filter(r => r.status === 'red' && r.next)
|
||||
.map(r => `✗ ${r.case} ${r.kind} FAILED TWICE (rule 16: a real red): ${r.first.where}; again in ${r.next!.where}${r.next!.symptom ? ` — ${r.next!.symptom}` : ''}`),
|
||||
];
|
||||
const flakes = flakeRecords(rerun.resolutions.values(), { lane, env });
|
||||
const flakyLines = flakes.map(formatFlakyLine);
|
||||
const spend = rerunSpend(rerun.observations);
|
||||
const red = verdict.problems.length > 0;
|
||||
const headline: ReportHeadline = {
|
||||
lane, verdict: red ? 'RED' : 'GREEN', attempt: primary,
|
||||
@@ -743,7 +766,9 @@ export function runPaidReport(reportDir: string, options: { writeDurations?: boo
|
||||
+ results.flatMap(r => r.outcomes).filter(o => !o.trial && o.runnerError !== undefined).length,
|
||||
incomplete: panels.filter(p => p.status === 'INCOMPLETE').length,
|
||||
unattributed: census.unattributed.length,
|
||||
...(flakes.length ? { flaky: flakes.length } : {}),
|
||||
},
|
||||
...(spend.executions ? { reruns: spend } : {}),
|
||||
actionRequired: verdict.problems.length,
|
||||
wallMs: wall.length ? Math.max(...wall.map(r => r.finishedAt!)) - Math.min(...wall.map(r => r.startedAt!)) : null,
|
||||
costUsd: Math.round(costUsd * 100) / 100,
|
||||
@@ -756,11 +781,17 @@ export function runPaidReport(reportDir: string, options: { writeDurations?: boo
|
||||
console.log('[test:paid] failures and split verdicts:');
|
||||
for (const line of failureLines) console.log(` ${line}`);
|
||||
}
|
||||
for (const panel of laterPanels) console.log(` attempt ${panel.attempt} (re-run; reported, never replacing attempt ${primary}): ${formatPanelLine(panel, manifest.tier)}`);
|
||||
if (flakyLines.length) {
|
||||
console.log('[test:paid] flaky (rule 16: failed once, passed on its one rerun; green with this annotation):');
|
||||
for (const line of flakyLines) console.log(` ${line}`);
|
||||
}
|
||||
for (const panel of laterPanels) console.log(` attempt ${panel.attempt} (re-run; resolves a red only by the rule-16 flaky rule): ${formatPanelLine(panel, manifest.tier)}`);
|
||||
if (flakes.length) fs.writeFileSync(path.join(reportDir, FLAKE_RECORDS_FILE), flakes.map(record => JSON.stringify(record)).join('\n') + '\n');
|
||||
const fence = (lines: string[]) => publishedFence(lines);
|
||||
fs.writeFileSync(summaryMdPath, [
|
||||
...fence(headlineLines),
|
||||
...(failureLines.length ? ['', '**Failures and split verdicts**', '', ...fence(failureLines)] : []),
|
||||
...(flakyLines.length ? ['', '**Flaky: failed once, passed on its one rule-16 rerun**', '', ...fence(flakyLines)] : []),
|
||||
...(censusLines.length ? ['', '**Skipped, deselected and unattributed testcases**', '', ...fence(censusLines)] : []),
|
||||
...(verdict.problems.length ? ['', `**ACTION REQUIRED (${verdict.problems.length})**`, '', ...fence(verdict.problems.map(p => sanitizeTrialError(p) ?? p)), '', `Guide: ${CENSUS_RED_GUIDE}`] : []),
|
||||
].join('\n') + '\n');
|
||||
@@ -774,7 +805,8 @@ export function runPaidReport(reportDir: string, options: { writeDurations?: boo
|
||||
trials: p.trials.map(t => ({ trial: t.trial, outcome: t.outcome, ...(t.failure_class ? { failure_class: t.failure_class } : {}),
|
||||
...(t.exit_reason ? { exit_reason: t.exit_reason } : {}), ...(t.error ? { error: t.error } : {}) })) })),
|
||||
// evals.yml wraps these in a fixed fence for the PR comment.
|
||||
failures: failureLines.map(line => sanitizeFixedFenceLine(line)) }, null, 2) + '\n');
|
||||
failures: failureLines.map(line => sanitizeFixedFenceLine(line)),
|
||||
flaky: flakes, flakyLines: flakyLines.map(line => sanitizeFixedFenceLine(line)), flakyCollectorFailures }, null, 2) + '\n');
|
||||
reportMapGaps(reportDir, new Set(executed.map(record => record.case)), rootDir);
|
||||
if (verdict.problems.length) {
|
||||
console.error(`[test:paid] report: ${verdict.problems.length} problem(s):`);
|
||||
|
||||
@@ -0,0 +1,393 @@
|
||||
/**
|
||||
* Rule-16 flaky reruns for the paid gate (CREW rule 16, Garry 2026-10-19:
|
||||
* "you can just run the failed test again and note that it is flaky... You
|
||||
* don't have to do the whole test pass again for one failed test.").
|
||||
*
|
||||
* Every executed case leaves an ordered history of observations: each run
|
||||
* attempt's first execution, then that attempt's one in-run rerun. A case is
|
||||
* - FLAKY (green, annotated) when it failed exactly once, that failure was a
|
||||
* case verdict (a red rule or judge case, or a behavior panel FAIL whose
|
||||
* red class is VERDICT), and the very next observation of it passed;
|
||||
* - RED when it failed twice, or its failure was harness, INFRA,
|
||||
* INCOMPLETE or unattributed, or nothing has re-executed it yet.
|
||||
* A case that passed is never rerun and leaves no flake record.
|
||||
*
|
||||
* The rerun planner (evals.yml `rerun-plan`) reruns, once and in the same
|
||||
* run attempt, exactly the cases whose only failure is a case verdict that
|
||||
* nothing has re-executed yet, and only when their passing would turn the
|
||||
* gate green. On the gate lane a completed rule shard's red case no longer
|
||||
* fails its slice job (sliceExitCode caseVerdicts), so the run's conclusion
|
||||
* follows this report and a flaky green concludes success. The
|
||||
* report (runPaidReport) applies the same history to every attempt, so a
|
||||
* manual `gh run rerun --failed` attempt that passes every earlier red is
|
||||
* green with the same annotation, while a new red on any attempt stays red
|
||||
* unless its own next observation passes. Trials, kinds, thresholds and
|
||||
* panels never change: a behavior case reruns its whole planned panel, a
|
||||
* judge case its judge's samples, a rule case its one trial.
|
||||
*/
|
||||
import * as fs from 'node:fs';
|
||||
import * as path from 'node:path';
|
||||
import { normalizeRelativePath } from './shard-engine';
|
||||
import { E2E_TOUCHFILES, LLM_JUDGE_TOUCHFILES } from '../../test/helpers/touchfiles';
|
||||
import { isFinalizedEvalResultFile, sanitizeTrialError, type EvalCaseKind } from '../../test/helpers/eval-store';
|
||||
import { shardTrial } from './paid-cases';
|
||||
import { type PaidRunManifest, type SliceResult, codexSlicesOf, isCodexShard, loadPaidTestDurations, packBySliceBudget, parseRunManifest, trialPanelKey, verifySliceResults } from './paid-plan';
|
||||
import { type ReportArtifact, junitCensus, loadSliceArtifacts, panelReports, sliceArtifactName } from './paid-report';
|
||||
import { expectedPrCaseCount, shardSlug } from './paid-select';
|
||||
import { ROOT } from './paid-types';
|
||||
|
||||
export const FLAKE_RECORDS_FILE = 'flake-records.jsonl';
|
||||
/** Each attempt's rerun manifest lands in the report dir as `<RERUN_PLAN_ARTIFACT>-a<attempt>/manifest.json`. */
|
||||
export const RERUN_PLAN_ARTIFACT = 'rerun-plan';
|
||||
|
||||
export type Phase = 'first' | 'rerun';
|
||||
|
||||
export interface CaseObservation {
|
||||
case: string;
|
||||
kind: EvalCaseKind;
|
||||
attempt: number;
|
||||
phase: Phase;
|
||||
outcome: 'passed' | 'failed';
|
||||
/** A failed case verdict the one rerun may resolve; harness, INFRA, INCOMPLETE and unattributed failures never are. */
|
||||
eligible: boolean;
|
||||
/** Why a failure is not eligible. */
|
||||
reason?: string;
|
||||
/** The manifest keys that executed it: its non-trial shard, or its panel's trial shards. */
|
||||
keys: string[];
|
||||
/** Where it ran: attempt, phase and slice artifacts. */
|
||||
where: string;
|
||||
symptom?: string;
|
||||
costUsd: number;
|
||||
/** False when some execution captured no billing (PTY sessions): costUsd is then a lower bound. */
|
||||
costKnown: boolean;
|
||||
}
|
||||
|
||||
export interface CaseResolution {
|
||||
case: string;
|
||||
kind: EvalCaseKind;
|
||||
status: 'flaky' | 'red';
|
||||
first: CaseObservation;
|
||||
/** flaky: the passing rerun; red: the second failure, when there is one. */
|
||||
next?: CaseObservation;
|
||||
reason: string;
|
||||
}
|
||||
|
||||
export interface ObservationSet {
|
||||
attempt: number;
|
||||
phase: Phase;
|
||||
manifest: PaidRunManifest;
|
||||
artifacts: ReportArtifact[];
|
||||
/** A later attempt or a rerun that ran only some slices: panels count only when all their trials ran in it. */
|
||||
partial: boolean;
|
||||
}
|
||||
|
||||
const phaseOrder = (phase: Phase) => (phase === 'first' ? 0 : 1);
|
||||
const registered = (id: string) => Object.hasOwn(E2E_TOUCHFILES, id) || Object.hasOwn(LLM_JUDGE_TOUCHFILES, id);
|
||||
function where(set: ObservationSet, slices: number[]): string {
|
||||
const names = [...new Set(slices)].sort((a, b) => a - b)
|
||||
.map(slice => sliceArtifactName(set.artifacts.find(a => a.result.sliceIndex === slice)?.root ?? '', slice, set.attempt));
|
||||
return `attempt ${set.attempt}${set.phase === 'rerun' ? ' rerun' : ''} (${names.join(', ')})`;
|
||||
}
|
||||
|
||||
/** Collector records under each reported shard's own eval dir, keyed by shard. */
|
||||
function shardRecords(artifacts: ReportArtifact[]): Map<string, any[]> {
|
||||
const records = new Map<string, any[]>();
|
||||
for (const { root, result } of artifacts) {
|
||||
for (const outcome of result.outcomes) {
|
||||
const key = normalizeRelativePath(outcome.files[0] ?? '');
|
||||
const dir = path.join(root, 'shards', shardSlug([key]));
|
||||
if (!fs.existsSync(dir)) continue;
|
||||
for (const name of fs.readdirSync(dir, { recursive: true }) as string[]) {
|
||||
if (!isFinalizedEvalResultFile(name)) continue;
|
||||
try {
|
||||
const parsed = JSON.parse(fs.readFileSync(path.join(dir, name), 'utf8'));
|
||||
if (Array.isArray(parsed.tests)) records.set(key, [...(records.get(key) ?? []), ...parsed.tests]);
|
||||
} catch { /* a malformed collector is the report's problem, not an observation */ }
|
||||
}
|
||||
}
|
||||
}
|
||||
return records;
|
||||
}
|
||||
|
||||
type Outcome = SliceResult['outcomes'][number];
|
||||
|
||||
/** Why a non-trial shard's case results cannot count as a verdict observation, else null. */
|
||||
export function shardProblem(manifest: PaidRunManifest, key: string, got: { outcome: Outcome; checkpoint: boolean } | undefined): string | null {
|
||||
const entry = manifest.entries.find(e => normalizeRelativePath(e.file) === key);
|
||||
if (!entry || entry.status !== 'planned') return 'not a planned shard';
|
||||
if (!got) return 'never reported';
|
||||
const o = got.outcome;
|
||||
if (got.checkpoint) return 'its slice ended before its final result';
|
||||
if (o.reused) return 'a reused result';
|
||||
if (o.runnerError !== undefined) return 'runner error';
|
||||
if (o.sliceDeadline) return `slice deadline (${o.sliceDeadline})`;
|
||||
if (o.status !== 'passed' && o.status !== 'failed') return `shard ${o.status}`;
|
||||
if (o.executedTests === null || o.skippedTests === null) return 'no test summary';
|
||||
if (manifest.prCoverage?.mode === 'pr' && o.executedTests - o.skippedTests !== expectedPrCaseCount(entry.file, manifest.selection!, entry.excludeCases)) {
|
||||
return 'executed a different case count than planned';
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
/** Every case verdict one set of slice results holds: JUnit rule and judge cases, and complete trial panels. */
|
||||
export function observeCases(set: ObservationSet): CaseObservation[] {
|
||||
const { manifest, artifacts, attempt, phase } = set;
|
||||
const results = artifacts.map(a => a.result);
|
||||
const reported = new Map<string, { outcome: Outcome; checkpoint: boolean }>();
|
||||
for (const result of results) {
|
||||
for (const outcome of result.outcomes) reported.set(normalizeRelativePath(outcome.files[0] ?? ''), { outcome, checkpoint: !!result.checkpoint });
|
||||
}
|
||||
const census = junitCensus(manifest, artifacts, shardRecords(artifacts), {});
|
||||
const unattributed = new Set(census.unattributed.map(line => line.split(' :: ')[0]));
|
||||
const observations: CaseObservation[] = [];
|
||||
for (const c of census.ruleCases) {
|
||||
if (c.outcome === 'skipped') continue;
|
||||
const unsound = shardProblem(manifest, c.key, reported.get(c.key)) ?? (unattributed.has(c.key) ? 'an unattributed testcase in its shard' : null);
|
||||
if (c.outcome === 'passed' && unsound) continue;
|
||||
const reason = c.outcome !== 'failed' ? undefined : unsound ?? (c.failureClass === 'infra' ? 'INFRA failure class'
|
||||
: !registered(c.id) ? 'no registry case id (unattributed)' : undefined);
|
||||
observations.push({ case: c.id, kind: c.kind, attempt, phase, outcome: c.outcome, eligible: c.outcome === 'failed' && !reason, ...(reason ? { reason } : {}),
|
||||
keys: [c.key], where: where(set, [c.slice]),
|
||||
...(c.outcome === 'failed' ? { symptom: sanitizeTrialError(`${c.failureClass ?? 'assertion'}${c.error ? `: ${c.error}` : ''}`) } : {}),
|
||||
costUsd: c.costUsd, costKnown: c.costKnown });
|
||||
}
|
||||
const slices = new Set(results.map(r => r.sliceIndex));
|
||||
for (const panel of panelReports(manifest, results, attempt)) {
|
||||
const entries = manifest.entries.filter(e => e.status === 'planned' && e.trial && trialPanelKey(e.file) === `${panel.file}#${panel.case}`);
|
||||
if (set.partial && !entries.every(e => slices.has(e.slice))) continue;
|
||||
if (panel.status === 'SKIPPED' || (panel.status === 'FAIL' && !panel.failsLane)) continue;
|
||||
const outcomes = entries.map(e => reported.get(normalizeRelativePath(e.file))?.outcome);
|
||||
if (phase === 'rerun' && outcomes.some(o => o?.reused)) continue;
|
||||
const passed = panel.status === 'PASS';
|
||||
const reason = passed ? undefined : panel.status === 'INCOMPLETE' ? `INCOMPLETE: ${panel.reason}`
|
||||
: panel.redClass === 'INFRA' ? 'INFRA: every failed trial is infra-class' : panel.redClass !== 'VERDICT' ? `red class ${panel.redClass}` : undefined;
|
||||
const failed = panel.trials.filter(t => t.outcome === 'failed').map(t => `t${t.trial} ${t.failure_class ?? 'assertion'}${t.error ? ` ${t.error}` : ''}`);
|
||||
observations.push({ case: panel.case, kind: panel.kind, attempt, phase, outcome: passed ? 'passed' : 'failed', eligible: !passed && !reason, ...(reason ? { reason } : {}),
|
||||
keys: entries.map(e => normalizeRelativePath(e.file)), where: where(set, Object.values(panel.slices)),
|
||||
...(passed ? {} : { symptom: sanitizeTrialError(`${panel.status} ${panel.passed}/${panel.panel.n} (${panel.marks}): ${failed.join('; ')}`) }),
|
||||
costUsd: Math.round(outcomes.reduce((sum, o) => sum + (o?.trial?.cost_usd ?? 0), 0) * 100) / 100, costKnown: outcomes.every(o => o?.trial && o.trial.cost_known !== false) });
|
||||
}
|
||||
return observations;
|
||||
}
|
||||
|
||||
/**
|
||||
* The observation sets of a report dir: each attempt's first execution
|
||||
* (attempt 1 complete, later attempts partial) and each attempt's rerun,
|
||||
* bound to its own rerun manifest. A rerun result that does not match its
|
||||
* manifest is set aside and named.
|
||||
*/
|
||||
export function observationSets(reportDir: string, manifest: PaidRunManifest, firstArtifacts: ReportArtifact[]): { sets: ObservationSet[]; ignored: string[] } {
|
||||
const reported = [...new Set(firstArtifacts.map(a => a.result.attempt ?? 1))].sort((a, b) => a - b);
|
||||
const attempts = reported.length ? reported : [1];
|
||||
const sets: ObservationSet[] = attempts.map((attempt, i) => ({ attempt, phase: 'first' as const, manifest, partial: i > 0,
|
||||
artifacts: firstArtifacts.filter(a => (a.result.attempt ?? 1) === attempt) }));
|
||||
const ignored: string[] = [];
|
||||
const reruns = loadSliceArtifacts(reportDir, { reruns: true });
|
||||
for (const attempt of [...new Set(reruns.map(a => a.result.attempt ?? 1))].sort((a, b) => a - b)) {
|
||||
let rerunManifest: PaidRunManifest;
|
||||
try { rerunManifest = parseRunManifest(fs.readFileSync(path.join(reportDir, `${RERUN_PLAN_ARTIFACT}-a${attempt}`, 'manifest.json'), 'utf8')); } catch (error) {
|
||||
ignored.push(`attempt ${attempt}: rerun results without a readable rerun manifest (${error instanceof Error ? error.message : String(error)})`);
|
||||
continue;
|
||||
}
|
||||
const mine = reruns.filter(a => (a.result.attempt ?? 1) === attempt);
|
||||
const bound = mine.filter(a => rerunManifest.rerun?.attempt === attempt && a.result.sliceCount === rerunManifest.sliceCount
|
||||
&& a.result.tier === rerunManifest.tier && JSON.stringify(a.result.selection ?? null) === JSON.stringify(rerunManifest.selection ?? null));
|
||||
if (bound.length !== mine.length) ignored.push(`attempt ${attempt}: ${mine.length - bound.length} rerun result(s) do not match the rerun manifest`);
|
||||
sets.push({ attempt, phase: 'rerun', manifest: rerunManifest, partial: true, artifacts: bound });
|
||||
}
|
||||
return { sets, ignored };
|
||||
}
|
||||
|
||||
/** Every observation in order: attempt, then first execution before its rerun. A rerun counts only for the cases it was planned for. */
|
||||
export function caseHistory(sets: ObservationSet[]): CaseObservation[] {
|
||||
return sets.flatMap(set => observeCases(set).filter(o => set.phase === 'first' || (set.manifest.rerun?.cases ?? []).includes(o.case)))
|
||||
.map((o, index) => ({ o, index }))
|
||||
.sort((a, b) => a.o.attempt - b.o.attempt || phaseOrder(a.o.phase) - phaseOrder(b.o.phase) || a.index - b.index)
|
||||
.map(({ o }) => o);
|
||||
}
|
||||
|
||||
/** The rule-16 status of every case that failed at least once. */
|
||||
export function resolveCases(history: CaseObservation[]): Map<string, CaseResolution> {
|
||||
const resolved = new Map<string, CaseResolution>();
|
||||
const byCase = new Map<string, CaseObservation[]>();
|
||||
for (const o of history) byCase.set(o.case, [...(byCase.get(o.case) ?? []), o]);
|
||||
for (const [id, list] of byCase) {
|
||||
const fails = list.filter(o => o.outcome === 'failed');
|
||||
if (!fails.length) continue;
|
||||
const first = fails[0]!;
|
||||
const next = list[list.indexOf(first) + 1];
|
||||
if (fails.length === 1 && first.eligible && next?.outcome === 'passed') {
|
||||
resolved.set(id, { case: id, kind: first.kind, status: 'flaky', first, next, reason: `failed once in ${first.where}, passed on rerun in ${next.where}` });
|
||||
continue;
|
||||
}
|
||||
const reason = fails.length > 1 ? `failed twice (${first.where}; ${fails[1]!.where})`
|
||||
: !first.eligible ? `never rerun: ${first.reason}` : 'no rerun verdict';
|
||||
resolved.set(id, { case: id, kind: first.kind, status: 'red', first, ...(fails[1] ? { next: fails[1] } : {}), reason });
|
||||
}
|
||||
return resolved;
|
||||
}
|
||||
|
||||
/**
|
||||
* The cases to rerun once now: their only failure is a case verdict and
|
||||
* nothing has executed them since. Usually this attempt's own reds; on a
|
||||
* `gh run rerun --failed` attempt also an earlier attempt's red that its
|
||||
* in-run rerun never reached (another problem blocked it then).
|
||||
*/
|
||||
export function rerunCandidates(history: CaseObservation[]): CaseObservation[] {
|
||||
return [...resolveCases(history).values()].filter(r => r.status === 'red' && !r.next && r.first.eligible
|
||||
&& history.filter(o => o.case === r.case).at(-1) === r.first).map(r => r.first);
|
||||
}
|
||||
|
||||
/** Non-trial shards whose every failed case in `observations` is flaky: their non-zero exit and failed status are explained. */
|
||||
export function flakyShards(observations: CaseObservation[], flaky: ReadonlySet<string>): Set<string> {
|
||||
const failed = observations.filter(o => o.outcome === 'failed' && o.keys.length === 1 && shardTrial(o.keys[0]!) === null);
|
||||
const keys = new Set(failed.map(o => o.keys[0]!));
|
||||
return new Set([...keys].filter(key => failed.filter(o => o.keys[0] === key).every(o => flaky.has(o.case))));
|
||||
}
|
||||
|
||||
/**
|
||||
* The rerun manifest: the candidates' own shards from the plan (a behavior
|
||||
* case's whole panel, a rule or judge case's own shard, narrowed to it in PR
|
||||
* mode; otherwise the file reruns and only the planned cases count), packed
|
||||
* with the plan's budget. It runs fresh: the rerun job
|
||||
* gets no receipts and the executor never reuses for a rerun manifest.
|
||||
*/
|
||||
export function buildRerunManifest(manifest: PaidRunManifest, candidates: CaseObservation[], attempt: number,
|
||||
durations: Record<string, number> = loadPaidTestDurations(ROOT, manifest.tier)): PaidRunManifest {
|
||||
const ids = [...new Set(candidates.map(c => c.case))].sort();
|
||||
const keys = [...new Set(candidates.flatMap(c => c.keys))].sort();
|
||||
const panels = new Set(candidates.filter(c => c.keys.some(key => shardTrial(key) !== null)).map(c => c.case));
|
||||
const selection = { e2e: ids.filter(id => Object.hasOwn(E2E_TOUCHFILES, id)),
|
||||
judges: ids.filter(id => !Object.hasOwn(E2E_TOUCHFILES, id) && Object.hasOwn(LLM_JUDGE_TOUCHFILES, id)) };
|
||||
const budget = manifest.plan ?? { sliceBudgetMs: 420_000, jobs: 2 };
|
||||
const packed = packBySliceBudget(keys, budget.sliceBudgetMs, budget.jobs, durations, undefined, file => isCodexShard(file, manifest.tier));
|
||||
const entries = packed.slices.flatMap((files, index) => files.map(file => {
|
||||
const source = manifest.entries.find(e => normalizeRelativePath(e.file) === file)!;
|
||||
const exclude = (source.excludeCases ?? []).filter(id => panels.has(id));
|
||||
return { file: source.file, slice: index + 1, status: 'planned' as const, estimatedMs: packed.estimates[file]!,
|
||||
...(source.budget ? { budget: source.budget } : {}), ...(source.trial ? { trial: source.trial } : {}), ...(exclude.length ? { excludeCases: exclude } : {}) };
|
||||
})).sort((a, b) => (a.file < b.file ? -1 : 1));
|
||||
return parseRunManifest(JSON.stringify({
|
||||
version: 1, tier: manifest.tier, evalsAll: false, sliceCount: packed.slices.length,
|
||||
selectionReason: `rule-16 rerun of attempt ${attempt}: ${ids.join(', ')}`, ...(manifest.host ? { host: manifest.host } : {}),
|
||||
profile: manifest.profile ?? 'full', selection,
|
||||
...(manifest.prCoverage ? { prCoverage: { ...manifest.prCoverage, e2e: selection.e2e, judges: selection.judges } } : {}),
|
||||
plan: { sliceBudgetMs: budget.sliceBudgetMs, jobs: budget.jobs, estimatedSliceMs: packed.estimatedSliceMs, ciTimeoutMinutes: packed.ciTimeoutMinutes,
|
||||
sliceCiTimeoutMinutes: packed.sliceCiTimeoutMinutes, codexSlices: codexSlicesOf(manifest.tier, entries) },
|
||||
rerun: { attempt, cases: ids }, entries,
|
||||
} satisfies PaidRunManifest));
|
||||
}
|
||||
|
||||
export interface FlakeRecord {
|
||||
schema: 1;
|
||||
kind: 'gate-flake';
|
||||
case: string;
|
||||
case_kind: EvalCaseKind;
|
||||
repository: string | null;
|
||||
pr: number | null;
|
||||
head_sha: string | null;
|
||||
tested_sha: string | null;
|
||||
lane: string;
|
||||
run_id: string | null;
|
||||
first: { attempt: number; phase: Phase; where: string };
|
||||
rerun: { attempt: number; phase: Phase; where: string };
|
||||
symptom: string;
|
||||
/** Null when the rerun captured no billing (PTY sessions). */
|
||||
rerun_cost_usd: number | null;
|
||||
recorded_at: string;
|
||||
}
|
||||
|
||||
/** One JSONL record per flaky case, read by the daily flaky-fix pass from the run's `flake-records-a<attempt>` artifact. */
|
||||
export function flakeRecords(resolutions: Iterable<CaseResolution>, ctx: { lane: string; env: NodeJS.ProcessEnv; now?: Date }): FlakeRecord[] {
|
||||
const { env } = ctx;
|
||||
const pr = Number(env.GSTACK_GATE_PR);
|
||||
const sha = (value: string | undefined) => (/^[0-9a-f]{40}$/.test(value ?? '') ? value! : null);
|
||||
return [...resolutions].filter(r => r.status === 'flaky').sort((a, b) => (a.case < b.case ? -1 : 1)).map(r => ({
|
||||
schema: 1, kind: 'gate-flake', case: r.case, case_kind: r.kind, repository: env.GITHUB_REPOSITORY ?? null,
|
||||
pr: Number.isSafeInteger(pr) && pr > 0 ? pr : null, head_sha: sha(env.GSTACK_GATE_HEAD_SHA) ?? sha(env.GITHUB_SHA), tested_sha: sha(env.GITHUB_SHA),
|
||||
lane: ctx.lane, run_id: env.GITHUB_RUN_ID ?? null,
|
||||
first: { attempt: r.first.attempt, phase: r.first.phase, where: r.first.where },
|
||||
rerun: { attempt: r.next!.attempt, phase: r.next!.phase, where: r.next!.where },
|
||||
symptom: r.first.symptom ?? 'failed', rerun_cost_usd: r.next!.costKnown ? r.next!.costUsd : null, recorded_at: (ctx.now ?? new Date()).toISOString(),
|
||||
}));
|
||||
}
|
||||
|
||||
/** The report and PR comment line of one flaky case (the run id stays in the record: the fence sanitizer redacts long digit runs). */
|
||||
export function formatFlakyLine(record: FlakeRecord): string {
|
||||
return `⚑ ${record.case} ${record.case_kind} FLAKY: failed in ${record.first.where}, passed on its one rerun in ${record.rerun.where}; `
|
||||
+ `rerun cost ${record.rerun_cost_usd === null ? 'unknown (no billing captured)' : `$${record.rerun_cost_usd.toFixed(2)}`}; first failure: ${record.symptom}`;
|
||||
}
|
||||
|
||||
/** What re-executing once-red cases cost: every observation after a case's first failure, in-run reruns and later attempts alike. */
|
||||
export function rerunSpend(history: CaseObservation[]): { executions: number; usd: number; unknown: number } {
|
||||
const firstFail = new Map<string, number>();
|
||||
history.forEach((o, index) => { if (o.outcome === 'failed' && !firstFail.has(o.case)) firstFail.set(o.case, index); });
|
||||
const after = history.filter((o, index) => firstFail.has(o.case) && index > firstFail.get(o.case)!);
|
||||
return { executions: after.length, usd: Math.round(after.reduce((sum, o) => sum + o.costUsd, 0) * 100) / 100, unknown: after.filter(o => !o.costKnown).length };
|
||||
}
|
||||
|
||||
/** A panel whose every planned trial ran in this set (a later attempt or rerun may hold only some). */
|
||||
function completePanel(set: ObservationSet, file: string, id: string): boolean {
|
||||
const slices = new Set(set.artifacts.map(a => a.result.sliceIndex));
|
||||
return set.manifest.entries.filter(e => e.status === 'planned' && e.trial && trialPanelKey(e.file) === `${file}#${id}`).every(e => slices.has(e.slice));
|
||||
}
|
||||
|
||||
/**
|
||||
* The slice and panel problems of every first-execution set, with each flaky
|
||||
* case's own red taken out: its shard's non-zero exit and failed status and
|
||||
* its panel's FAIL. A later attempt's problems are prefixed with it, so a new
|
||||
* red there is never hidden. Collector and manual-review checks stay in the
|
||||
* report.
|
||||
*/
|
||||
export function gateProblems(sets: ObservationSet[], observations: CaseObservation[], flaky: ReadonlySet<string>): string[] {
|
||||
const problems: string[] = [];
|
||||
for (const set of sets.filter(s => s.phase === 'first')) {
|
||||
const results = set.artifacts.map(a => a.result);
|
||||
const prefix = set.partial ? `attempt ${set.attempt}: ` : '';
|
||||
const shards = flakyShards(observations.filter(o => o.attempt === set.attempt && o.phase === 'first'), flaky);
|
||||
problems.push(...verifySliceResults(set.manifest, results, { flaky: shards, partial: set.partial }).problems.map(problem => prefix + problem));
|
||||
for (const panel of panelReports(set.manifest, results, set.attempt)) {
|
||||
if (!panel.failsLane || flaky.has(panel.case) || (set.partial && !completePanel(set, panel.file, panel.case))) continue;
|
||||
problems.push(`${prefix}PANEL ${panel.case} ${panel.status}${panel.redClass === 'INFRA' ? ' (INFRA)' : ''} ${panel.passed}/${panel.panel.n} (${panel.marks}): ${panel.reason}`);
|
||||
}
|
||||
}
|
||||
return problems;
|
||||
}
|
||||
|
||||
/** Everything a report dir proves about rule-16 reruns: the ordered history, each failed case's resolution and the flaky set. */
|
||||
export function rerunState(reportDir: string, manifest: PaidRunManifest, firstArtifacts: ReportArtifact[]) {
|
||||
const { sets, ignored } = observationSets(reportDir, manifest, firstArtifacts);
|
||||
const observations = caseHistory(sets);
|
||||
const resolutions = resolveCases(observations);
|
||||
const flaky = new Set([...resolutions.values()].filter(r => r.status === 'flaky').map(r => r.case));
|
||||
return { sets, ignored, observations, resolutions, flaky };
|
||||
}
|
||||
|
||||
/**
|
||||
* evals.yml `rerun-plan`: write this attempt's rerun manifest to
|
||||
* `<out>/manifest.json` when some case verdict failed once in this attempt
|
||||
* and those cases passing would leave the gate with no other problem. A
|
||||
* harness, INFRA, INCOMPLETE or unattributed red, a case already red twice,
|
||||
* or any other problem plans nothing: the gate keeps its red.
|
||||
*/
|
||||
export function planRerun(reportDir: string, attempt: number, out: string, durations?: Record<string, number>): { manifest: PaidRunManifest | null; lines: string[] } {
|
||||
const manifest = parseRunManifest(fs.readFileSync(path.join(reportDir, 'manifest.json'), 'utf8'));
|
||||
const state = rerunState(reportDir, manifest, loadSliceArtifacts(reportDir));
|
||||
const candidates = rerunCandidates(state.observations);
|
||||
const reds = [...state.resolutions.values()].filter(r => r.status === 'red' && !candidates.includes(r.first));
|
||||
const lines = [...state.ignored.map(line => `ignored: ${line}`), ...reds.map(r => `not rerun: ${r.case} (${r.reason})`)];
|
||||
if (!candidates.length) return { manifest: null, lines: [`no rule-16 rerun: no case verdict failed once with no rerun yet`, ...lines] };
|
||||
const remaining = gateProblems(state.sets, state.observations, new Set([...state.flaky, ...candidates.map(c => c.case)]));
|
||||
if (remaining.length) {
|
||||
return { manifest: null, lines: [`no rule-16 rerun: rerunning ${candidates.map(c => c.case).join(', ')} cannot turn this gate green; ${remaining.length} other problem(s), first: ${remaining[0]}`, ...lines] };
|
||||
}
|
||||
let rerun: PaidRunManifest;
|
||||
try { rerun = buildRerunManifest(manifest, candidates, attempt, durations); } catch (error) {
|
||||
return { manifest: null, lines: [`no rule-16 rerun: the rerun manifest is invalid (${error instanceof Error ? error.message : String(error)})`, ...lines] };
|
||||
}
|
||||
fs.mkdirSync(out, { recursive: true });
|
||||
fs.writeFileSync(path.join(out, 'manifest.json'), `${JSON.stringify(rerun, null, 2)}\n`);
|
||||
return { manifest: rerun, lines: [
|
||||
`rule-16 rerun of attempt ${attempt}: ${candidates.length} case(s) once, on ${rerun.sliceCount} slice(s), same head, fresh (no reuse)`,
|
||||
...candidates.map(c => ` ${c.case} ${c.kind} failed in ${c.where}: ${c.symptom ?? 'failed'}; reruns ${c.keys.join(' ')}`), ...lines] };
|
||||
}
|
||||
@@ -77,6 +77,7 @@ import { resolveEvalHost } from '../test/helpers/e2e-hosts';
|
||||
import { scopeCodexAccess, shardFile, shardCaseId, shardTrial, trialShardKey, type CaseTrialPlan, caseTrialPlan, excludedCasesNamePattern, caseTestNamePattern, expandCaseShards, expandTrialShards, fileCaseRegistration, partitionCaseExclusions } from './lib/paid-cases';
|
||||
import { retriesForFiles, trialPanelKey, sliceExecutionOrder, buildRunManifest, parseRunManifest, type SliceResult, sliceExitCode, guardTrialRecords, formatSlicePlan, formatCapacityPreflight, sliceDeadlineMs } from './lib/paid-plan';
|
||||
import { caseSelection, runCaseDiagnosis, formatPanelLine, runPaidReport } from './lib/paid-report';
|
||||
import { planRerun } from './lib/paid-rerun';
|
||||
import {
|
||||
DEFAULT_JOBS, DEFAULT_MAX_FILES_PER_SHARD, DEFAULT_SHARD_TIMEOUT_MS, DEFAULT_TIER, DEFAULT_WITHIN_SHARD_CONCURRENCY, OVERLAY_MAX_ACTIVE_SHARDS,
|
||||
PAID_LANE_POLICY, PAID_TIERS, ROOT, SLICE_JOB_STARTED_AT_ENV, SLICE_UPLOAD_RESERVE_MS, isOverlayTestFile,
|
||||
@@ -97,6 +98,7 @@ export * from './lib/paid-select';
|
||||
export * from './lib/paid-cases';
|
||||
export * from './lib/paid-plan';
|
||||
export * from './lib/paid-report';
|
||||
export * from './lib/paid-rerun';
|
||||
|
||||
type TrialEvidence = { records: any[]; contract: string | null; sessions?: SessionLedgerRow[] };
|
||||
|
||||
@@ -722,6 +724,8 @@ type CliOptions = {
|
||||
reportDir: string | null;
|
||||
/** Report mode: merge executed shard wall times into the duration seed. */
|
||||
writeDurations: boolean;
|
||||
/** With --report: plan this attempt's rule-16 rerun into this directory instead of reporting. */
|
||||
rerunPlanDir: string | null;
|
||||
/** Planner: the workflow matrix cap, for the capacity preflight's wave count. */
|
||||
maxParallel: number | null;
|
||||
/** Local diagnosis: run one case through the panel runner CI uses (never read by CI). */
|
||||
@@ -764,6 +768,7 @@ CI modes (the eval workflows):
|
||||
--emit-plan PATH (--slices K | --slice-budget SECS --jobs N) [--skip-judges] [--max-parallel N]
|
||||
--plan PATH --slice I execute one slice of a manifest
|
||||
--report DIR [--write-durations] reconcile slice results against the manifest
|
||||
--report DIR --rerun-plan OUT plan this attempt's one rule-16 rerun of its failed cases (scripts/lib/paid-rerun.ts)
|
||||
|
||||
Background runs and CI dispatch: bun run eval:bg:<pr|gate|periodic|release> (scripts/eval-bg.ts --help).`;
|
||||
|
||||
@@ -795,6 +800,7 @@ export function parseCliOptions(argv: string[], env: NodeJS.ProcessEnv = process
|
||||
sliceIndex: null,
|
||||
reportDir: null,
|
||||
writeDurations: false,
|
||||
rerunPlanDir: null,
|
||||
maxParallel: null,
|
||||
caseId: null,
|
||||
trials: null,
|
||||
@@ -822,6 +828,7 @@ export function parseCliOptions(argv: string[], env: NodeJS.ProcessEnv = process
|
||||
'--slice': (next) => { options.sliceIndex = parsePositiveInt(next(), '--slice'); },
|
||||
'--report': pathValue('--report needs a directory', (value) => { options.reportDir = value; }),
|
||||
'--write-durations': () => { options.writeDurations = true; },
|
||||
'--rerun-plan': pathValue('--rerun-plan needs a directory', (value) => { options.rerunPlanDir = value; }),
|
||||
'--max-parallel': (next) => { options.maxParallel = parsePositiveInt(next(), '--max-parallel'); },
|
||||
'--case': (next) => {
|
||||
const value = next();
|
||||
@@ -831,6 +838,7 @@ export function parseCliOptions(argv: string[], env: NodeJS.ProcessEnv = process
|
||||
'--trials': (next) => { options.trials = parsePositiveInt(next(), '--trials'); },
|
||||
}, PAID_USAGE);
|
||||
if (options.writeDurations && !options.reportDir) throw new Error('--write-durations requires --report');
|
||||
if (options.rerunPlanDir && (!options.reportDir || options.writeDurations)) throw new Error('--rerun-plan requires --report and writes no durations');
|
||||
if (options.trials !== null && options.caseId === null) throw new Error('--trials requires --case');
|
||||
if (options.caseId !== null && (options.emitPlanPath || options.planPath || options.reportDir || options.sliceIndex !== null)) {
|
||||
throw new Error('--case is local diagnosis; it cannot combine with --emit-plan, --plan/--slice or --report');
|
||||
@@ -880,6 +888,12 @@ async function main(): Promise<number> {
|
||||
|
||||
// ── Report mode: reconcile slice artifacts against the manifest. Fail-closed:
|
||||
// a slice whose artifact never landed is a FAILURE, not an absence.
|
||||
if (options.reportDir && options.rerunPlanDir) {
|
||||
const attempt = Number(process.env.GITHUB_RUN_ATTEMPT);
|
||||
const { lines } = planRerun(options.reportDir, Number.isSafeInteger(attempt) && attempt > 0 ? attempt : 1, options.rerunPlanDir);
|
||||
for (const line of lines) console.log(`- ${line}`);
|
||||
return 0;
|
||||
}
|
||||
if (options.reportDir) return runPaidReport(options.reportDir, { writeDurations: options.writeDurations });
|
||||
|
||||
if (options.caseId && options.listOnly) {
|
||||
@@ -960,6 +974,7 @@ async function main(): Promise<number> {
|
||||
startedAt,
|
||||
finishedAt: Date.now(),
|
||||
...(checkpoint ? { checkpoint: true as const } : {}),
|
||||
...(manifest.rerun ? { rerun: true as const } : {}),
|
||||
outcomes: outcomes.map(({ files, status, exitCode, elapsedMs, executedTests, skippedTests, budget, reused, runnerError, trial, inputKey, sliceDeadline }) =>
|
||||
({ files, status, exitCode, elapsedMs, executedTests, skippedTests, ...(budget ? { budget } : {}), ...(reused ? { reused } : {}),
|
||||
...(runnerError !== undefined ? { runnerError } : {}), ...(trial ? { trial } : {}), ...(inputKey ? { inputKey } : {}),
|
||||
@@ -989,7 +1004,8 @@ async function main(): Promise<number> {
|
||||
expectedCases: Object.fromEntries(mine.map(entry => [entry.file, expectedPrCaseCount(entry.file, manifest.selection!, entry.excludeCases)])),
|
||||
casePatterns: Object.fromEntries(mine.map(entry => [entry.file, prProfileTestNamePattern(entry.file, manifest.selection!, entry.excludeCases)])),
|
||||
expectedCaseIds: Object.fromEntries(mine.map(entry => [entry.file, prProfileShardIds(entry.file, manifest.selection!, entry.excludeCases)])),
|
||||
reuseFor: e2eReuseLaneProblem(process.env, manifest.prCoverage.mode) !== null ? undefined : (files, env, budget) => {
|
||||
// A rule-16 rerun executes fresh: a receipt would replay the verdict it is rerunning.
|
||||
reuseFor: manifest.rerun || e2eReuseLaneProblem(process.env, manifest.prCoverage.mode) !== null ? undefined : (files, env, budget) => {
|
||||
const key = files[0]!;
|
||||
const file = shardFile(key);
|
||||
if (files.length !== 1 || !/^test\/skill-e2e-/.test(file)) return null;
|
||||
@@ -1027,7 +1043,7 @@ async function main(): Promise<number> {
|
||||
const t = outcome.trial!;
|
||||
console.log(` trial ${t.case} t${t.trial}/${t.panel.n}: ${t.outcome ?? `NO RECORD (${t.harness})`}${t.failure_class ? ` [${t.failure_class}]` : ''}`);
|
||||
}
|
||||
return sliceExitCode(guarded);
|
||||
return sliceExitCode(guarded, { caseVerdicts: manifest.tier === 'gate' });
|
||||
}
|
||||
|
||||
if (options.listOnly && options.sliceBudgetMs !== null) {
|
||||
|
||||
+28
-10
@@ -36,6 +36,7 @@ import { UNRECORDED_SHARD_MS, nonceScope, plannedSerialMs } from './eval-bg';
|
||||
import { caseTrialPlan, fileCaseRegistration, shardCaseId, shardFile } from './lib/paid-cases';
|
||||
import { buildRunManifest, loadPaidTestDurations, parseRunManifest, recordedShardMs, type PaidRunManifest } from './lib/paid-plan';
|
||||
import { loadSliceArtifacts } from './lib/paid-report';
|
||||
import type { FlakeRecord } from './lib/paid-rerun';
|
||||
import { paidSelectionEnv } from './lib/paid-select';
|
||||
import { ROOT } from './lib/paid-types';
|
||||
|
||||
@@ -195,11 +196,14 @@ function readJson<T>(file: string): T | null {
|
||||
}
|
||||
|
||||
/** One panel per planned case from the run's trial outcomes (trial panels from the collector's panelVerdict). */
|
||||
function casePanels(planned: Map<string, string>, outcomes: OutcomeLine[], collector: CollectorPanel[]): CasePanel[] {
|
||||
function casePanels(planned: Map<string, string>, outcomes: OutcomeLine[], collector: CollectorPanel[], flaky: FlakeRecord[] = []): CasePanel[] {
|
||||
return [...planned].sort(([a], [b]) => (a < b ? -1 : 1)).map(([id, kind]) => {
|
||||
const mine = outcomes.filter(o => o.case === id && o.execution !== 'reused');
|
||||
const cost = mine.length && mine.every(o => o.cost_known !== false && typeof o.cost_usd === 'number')
|
||||
? Math.round(mine.reduce((sum, o) => sum + o.cost_usd!, 0) * 10_000) / 10_000 : null;
|
||||
// Rule 16: the report resolved this case FLAKY (failed once, passed its one rerun): a PASS that names both runs.
|
||||
const flake = flaky.find(f => f.case === id);
|
||||
if (flake) return { case: id, kind, status: 'PASS', red_class: null, cost_usd: cost, flaky: { first: flake.first.where, rerun: flake.rerun.where } };
|
||||
const panel = collector.find(p => p.case === id);
|
||||
if (panel) return { case: id, kind, status: panel.status, red_class: panel.redClass, cost_usd: cost };
|
||||
const failed = mine.filter(o => o.outcome === 'failed');
|
||||
@@ -219,8 +223,8 @@ export function buildTrainRecord(o: { reportDir: string; nonce: string; head: st
|
||||
let lines: OutcomeLine[] = [];
|
||||
try { lines = fs.readFileSync(path.join(o.reportDir, 'trial-outcomes.jsonl'), 'utf8').split('\n').filter(Boolean).map(line => JSON.parse(line) as OutcomeLine); } catch { lines = []; }
|
||||
const primary = Math.min(...lines.map(l => l.attempt ?? 1), Infinity);
|
||||
const collector = readJson<{ verdict?: { verdict?: string }; census?: { unattributed?: unknown[] }; panels?: CollectorPanel[] }>(path.join(o.reportDir, 'collector-outcomes.json'));
|
||||
const panels = casePanels(planned, lines.filter(l => (l.attempt ?? 1) === primary), collector?.panels ?? []);
|
||||
const collector = readJson<{ verdict?: { verdict?: string }; census?: { unattributed?: unknown[] }; panels?: CollectorPanel[]; flaky?: FlakeRecord[] }>(path.join(o.reportDir, 'collector-outcomes.json'));
|
||||
const panels = casePanels(planned, lines.filter(l => (l.attempt ?? 1) === primary), collector?.panels ?? [], collector?.flaky ?? []);
|
||||
const artifacts = loadSliceArtifacts(o.reportDir);
|
||||
const first = Math.min(...artifacts.map(a => a.result.attempt ?? 1), Infinity);
|
||||
const results = artifacts.filter(a => (a.result.attempt ?? 1) === first).map(a => a.result);
|
||||
@@ -246,7 +250,7 @@ export function buildTrainRecord(o: { reportDir: string; nonce: string; head: st
|
||||
|
||||
export interface BisectPlan { schema: 1; nonce: string; cases: Array<{ id: string; kind: string }>; prefixes: string[] }
|
||||
|
||||
/** One executor per prefix; each case runs one trial, so a slice's ceiling grows with the case count. */
|
||||
/** One executor per prefix; each case runs one trial (and at most one rule-16 rerun inside its 30 minutes), so a slice's ceiling grows with the case count. */
|
||||
export function bisectMatrix(plan: BisectPlan): { slices: number[]; timeouts: number[]; refs: string[] } {
|
||||
const minutes = Math.min(350, 20 + 30 * plan.cases.length);
|
||||
return { slices: plan.prefixes.map((_, i) => i + 1), timeouts: [0, ...plan.prefixes.map(() => minutes)], refs: ['', ...plan.prefixes] };
|
||||
@@ -254,7 +258,13 @@ export function bisectMatrix(plan: BisectPlan): { slices: number[]; timeouts: nu
|
||||
|
||||
export type CaseRunner = (id: string, kind: string, evalDir: string) => Promise<{ status: string; redClass: string | null }>;
|
||||
|
||||
/** Run every red case once on the checked-out prefix; a case that leaves no verdict is INCOMPLETE (T1 relaunches it). */
|
||||
/**
|
||||
* Run every red case once on the checked-out prefix; a case that leaves no
|
||||
* verdict is INCOMPLETE (T1 relaunches it). Rule 16 holds here as in the
|
||||
* gate: a case verdict that fails (red class VERDICT) reruns once on the same
|
||||
* prefix; passing makes it a flaky PASS naming both runs, failing again keeps
|
||||
* it red. INFRA and INCOMPLETE are never rerun.
|
||||
*/
|
||||
export async function runBisectPrefix(plan: BisectPlan, index: number, o: { rootDir: string; evalDir: string; checkedOut: string; runCase?: CaseRunner }): Promise<BisectPrefixRecord> {
|
||||
const sha = plan.prefixes[index - 1];
|
||||
if (!sha) throw new Error(`bisect slice ${index} names no prefix (${plan.prefixes.length} planned)`);
|
||||
@@ -262,11 +272,19 @@ export async function runBisectPrefix(plan: BisectPlan, index: number, o: { root
|
||||
const runCase = o.runCase ?? ((id, kind, evalDir) => runOneTrial(id, kind, evalDir, o.rootDir));
|
||||
const panels: CasePanel[] = [];
|
||||
for (const c of plan.cases) {
|
||||
let verdict: { status: string; redClass: string | null };
|
||||
try { verdict = await runCase(c.id, c.kind, path.join(o.evalDir, c.id.replace(/[^A-Za-z0-9._-]/g, '_'))); }
|
||||
catch (error) { console.error(`[train-gate] ${c.id}: ${error instanceof Error ? error.message : String(error)}`); verdict = { status: 'INCOMPLETE', redClass: 'INCOMPLETE' }; }
|
||||
panels.push({ case: c.id, kind: c.kind, status: verdict.status, red_class: verdict.redClass, cost_usd: null });
|
||||
console.log(`[train-gate] prefix ${sha.slice(0, 12)} (slice ${index}): ${c.id} ${verdict.status}${verdict.redClass ? ` (${verdict.redClass})` : ''}`);
|
||||
const dir = path.join(o.evalDir, c.id.replace(/[^A-Za-z0-9._-]/g, '_'));
|
||||
const run = async (evalDir: string): Promise<{ status: string; redClass: string | null }> => {
|
||||
try { return await runCase(c.id, c.kind, evalDir); }
|
||||
catch (error) { console.error(`[train-gate] ${c.id}: ${error instanceof Error ? error.message : String(error)}`); return { status: 'INCOMPLETE', redClass: 'INCOMPLETE' }; }
|
||||
};
|
||||
const verdict = await run(dir);
|
||||
const rerun = verdict.status === 'FAIL' && verdict.redClass === 'VERDICT' ? await run(`${dir}.rerun`) : null;
|
||||
const flaky = rerun?.status === 'PASS';
|
||||
const final = flaky ? rerun! : verdict;
|
||||
panels.push({ case: c.id, kind: c.kind, status: final.status, red_class: final.redClass, cost_usd: null,
|
||||
...(flaky ? { flaky: { first: `prefix ${sha.slice(0, 12)} slice ${index}`, rerun: `prefix ${sha.slice(0, 12)} slice ${index} rerun` } } : {}) });
|
||||
console.log(`[train-gate] prefix ${sha.slice(0, 12)} (slice ${index}): ${c.id} ${verdict.status}${verdict.redClass ? ` (${verdict.redClass})` : ''}`
|
||||
+ `${rerun ? `; rule-16 rerun ${rerun.status}${rerun.redClass ? ` (${rerun.redClass})` : ''}${flaky ? ': FLAKY, counted as PASS' : ': failed twice, red'}` : ''}`);
|
||||
}
|
||||
return { schema: 1, kind: 'train-bisect-prefix', nonce: plan.nonce, prefix_sha: sha, checked_out_sha: o.checkedOut, panels };
|
||||
}
|
||||
|
||||
Reference in new issue
Block a user