diff --git a/scripts/test-paid-shards.ts b/scripts/test-paid-shards.ts index c4f7031b8..4fdd84b8e 100644 --- a/scripts/test-paid-shards.ts +++ b/scripts/test-paid-shards.ts @@ -66,9 +66,14 @@ import { type ShardChildResult, } from './test-strict-output'; import { PAID_TEST_GLOBS, isPaidTestFile } from '../test/helpers/paid-test-set'; -import { CASE_CI_EXCLUDE, PERIODIC_CI_EXCLUDE } from '../test/helpers/periodic-exclude-data'; +import { CASE_CI_EXCLUDE, CASE_QUARANTINE, EVAL_POLICY, PERIODIC_CI_EXCLUDE } from '../test/helpers/periodic-exclude-data'; import { FILE_RETRY_BUDGETS, STRICT_RETRY_CASE_BUDGETS } from '../test/helpers/eval-budgets'; -import { getProjectEvalDir, getClaudeCliVersion, isFinalizedEvalResultFile, evalEntryOutcome } from '../test/helpers/eval-store'; +import { + getProjectEvalDir, getClaudeCliVersion, isFinalizedEvalResultFile, evalEntryOutcome, failureClassOf, panelVerdict, + sanitizeTrialError, formatTrialOutcomes, CONTRACT_VIOLATIONS_FILE, TRIAL_ENV, TRIAL_OUTCOME_SCHEMA, TRIAL_OUTCOMES_FILE, + type EvalCaseKind, type PanelShape, type PanelVerdict, type TrialFailureClass, type TrialOutcome, type TrialOutcomeRecord, +} from '../test/helpers/eval-store'; +import { E2E_KINDS } from '../test/helpers/touchfiles-data'; import { manualReviewProblem } from '../test/helpers/cookie-workflow-manual-review'; import { preflightAnthropicApi } from '../test/helpers/anthropic-preflight'; import { OVERLAY_MIN_FILE_WALL_MS } from '../test/helpers/overlay-case-policy'; @@ -146,16 +151,60 @@ export const CASE_TEST_NAMES: Record = { }; const CASE_KEY_SEPARATOR = '#'; +const TRIAL_SUFFIX = /~t([1-9][0-9]*)$/; -/** The test file behind a shard key (`` or `#`). */ +/** The test file behind a shard key (``, `#` or `#~t`). */ export function shardFile(key: string): string { return normalizeRelativePath(key).split(CASE_KEY_SEPARATOR)[0]!; } -/** The E2E case id of a case shard key, else null. */ +/** The E2E case id of a case or trial shard key, else null. */ export function shardCaseId(key: string): string | null { const [, id] = normalizeRelativePath(key).split(CASE_KEY_SEPARATOR); - return id ?? null; + return id === undefined ? null : id.replace(TRIAL_SUFFIX, ''); +} + +/** The 1-based trial index of an isolated trial shard key, else null. */ +export function shardTrial(key: string): number | null { + const [, id] = normalizeRelativePath(key).split(CASE_KEY_SEPARATOR); + const match = id === undefined ? null : TRIAL_SUFFIX.exec(id); + return match ? Number(match[1]) : null; +} + +/** Shard key of one trial of an isolated case. */ +export function trialShardKey(file: string, id: string, trial: number): string { + return `${normalizeRelativePath(file)}${CASE_KEY_SEPARATOR}${id}~t${trial}`; +} + +/** Trial policy of one case, fixed from the registries before the run. */ +export interface CaseTrialPlan { kind: EvalCaseKind; panel: PanelShape; quarantined: boolean } + +/** + * `behavior` cases run EVAL_POLICY.panel; a quarantined case runs a full panel + * whose k keeps its kind's meaning (k = n for rule); everything else runs one + * trial. Only behavior and quarantined cases are isolated into trial shards. + */ +export function caseTrialPlan(id: string, kinds: Record = E2E_KINDS, + quarantine: Record = CASE_QUARANTINE): CaseTrialPlan { + const kind = kinds[id] ?? 'rule'; + const quarantined = Object.hasOwn(quarantine, id); + if (kind === 'behavior') return { kind, panel: { ...EVAL_POLICY.panel }, quarantined }; + if (quarantined) return { kind, panel: { n: EVAL_POLICY.panel.n, k: EVAL_POLICY.panel.n }, quarantined }; + return { kind, panel: { n: 1, k: 1 }, quarantined }; +} + +export function isIsolatedCase(plan: CaseTrialPlan): boolean { + return plan.kind === 'behavior' || plan.quarantined; +} + +function sameTrialPlan(a: CaseTrialPlan | undefined, b: CaseTrialPlan | undefined): boolean { + return !!a && !!b && a.kind === b.kind && a.quarantined === b.quarantined && a.panel?.n === b.panel?.n && a.panel?.k === b.panel?.k; +} + +/** Bun name pattern that runs every case of a file except `ids` (their trial shards run them). */ +export function excludedCasesNamePattern(ids: string[]): string { + const escaped = ids.map(id => (CASE_TEST_NAMES[id] ?? id).replace(/[.*+?^${}()|[\]\\]/g, '\\$&')); + return `^(?!.*(?:^|\\s)(?:${escaped.join('|')})$)`; } /** Exact Bun name pattern for a set of case ids (labels where the test name differs). */ @@ -180,6 +229,61 @@ export function expandCaseShards(files: string[], tier: PaidTier, rootDir = ROOT }); } +export interface TrialExpansion { + keys: string[]; + /** Trial policy per trial shard key. */ + trials: Record; + /** File shard key -> isolated case ids its name pattern excludes. */ + excludeCases: Record; +} + +/** + * Isolate every behavior or quarantined case of `tier` into its panel of trial + * shards (`#~t1..tn`), each selected by EVALS_SELECTION_JSON=[id] and + * its exact test name. The file shard keeps the remaining ids of the tier and + * excludes the isolated ones by name; with none remaining it is dropped. A case + * may be isolated only when its file's registration is statically known. + */ +export function expandTrialShards(keys: string[], tier: PaidTier, rootDir = ROOT, opts: { + kinds?: Record; quarantine?: Record; + touchfiles?: Record; tiers?: Record; +} = {}): TrialExpansion { + const touchfiles = opts.touchfiles ?? E2E_TOUCHFILES; + const tiers = opts.tiers ?? E2E_TIERS; + const planOf = (id: string) => caseTrialPlan(id, opts.kinds, opts.quarantine); + const out: TrialExpansion = { keys: [], trials: {}, excludeCases: {} }; + const addPanel = (file: string, id: string) => { + const plan = planOf(id); + for (let trial = 1; trial <= plan.panel.n; trial++) { + const key = trialShardKey(file, id, trial); + out.keys.push(key); + out.trials[key] = plan; + } + }; + for (const key of keys) { + const file = shardFile(key); + const caseId = shardCaseId(key); + if (caseId !== null) { + if (isIsolatedCase(planOf(caseId))) addPanel(file, caseId); + else out.keys.push(key); + continue; + } + const { registered, known } = fileCaseRegistration(file, fs.readFileSync(path.join(rootDir, file), 'utf8'), touchfiles, tiers); + const inTier = registered.filter(id => tiers[id] === tier); + const isolated = inTier.filter(id => isIsolatedCase(planOf(id))); + if (isolated.length === 0) { out.keys.push(key); continue; } + if (!known) { + throw new Error(`${file}: behavior or quarantined case(s) ${isolated.join(', ')} need a statically known case registration`); + } + for (const id of isolated) addPanel(file, id); + if (inTier.length > isolated.length) { + out.keys.push(key); + out.excludeCases[normalizeRelativePath(key)] = [...isolated].sort(); + } + } + return out; +} + /** * Split expanded shard keys into runnable keys and CI-unrunnable cases * (CASE_CI_EXCLUDE), each with its surfaced reason; never an empty shard. @@ -484,26 +588,27 @@ function packageVersionOnlySinceBase(rootDir: string, baseRef: string): boolean /** Only audited per-case files, plus the separately selected judge, enter the fast profile. */ /** The selected PR-profile case ids a shard key owns (a case key owns at most its own case). */ -function prProfileShardIds(key: string, selection: PaidCaseSelection): string[] { +/** `exclude`: isolated case ids a file shard leaves to their trial shards. */ +function prProfileShardIds(key: string, selection: PaidCaseSelection, exclude: readonly string[] = []): string[] { const caseId = shardCaseId(key); return (PR_PROFILE_FILES[shardFile(key)] ?? []) - .filter(id => (caseId === null || id === caseId) && (selection.e2e === null || selection.e2e.includes(id))); + .filter(id => (caseId === null || id === caseId) && (selection.e2e === null || selection.e2e.includes(id)) && !exclude.includes(id)); } -export function prProfileFileSelected(file: string, selection: PaidCaseSelection): boolean { +export function prProfileFileSelected(file: string, selection: PaidCaseSelection, exclude: readonly string[] = []): boolean { if (file === 'test/skill-llm-eval.test.ts') return selection.judges === null || selection.judges.length > 0; - return prProfileShardIds(file, selection).length > 0; + return prProfileShardIds(file, selection, exclude).length > 0; } -export function expectedPrCaseCount(file: string, selection: PaidCaseSelection): number { +export function expectedPrCaseCount(file: string, selection: PaidCaseSelection, exclude: readonly string[] = []): number { if (file === 'test/skill-llm-eval.test.ts') return selection.judges?.length ?? Object.keys(LLM_JUDGE_TOUCHFILES).length; - return prProfileShardIds(file, selection).length; + return prProfileShardIds(file, selection, exclude).length; } -export function prProfileTestNamePattern(file: string, selection: PaidCaseSelection): string { +export function prProfileTestNamePattern(file: string, selection: PaidCaseSelection, exclude: readonly string[] = []): string { const ids = file === 'test/skill-llm-eval.test.ts' ? selection.judges ?? Object.keys(LLM_JUDGE_TOUCHFILES) - : prProfileShardIds(file, selection); + : prProfileShardIds(file, selection, exclude); if (ids.length === 0) throw new Error(`No selected PR cases for ${file}`); return caseTestNamePattern(ids); } @@ -527,6 +632,8 @@ export interface DiffSkipOptions { allNames?: string[]; /** Injectable registration map (default: E2E_TOUCHFILES). */ e2eTouchfiles?: Record; + /** File shard -> isolated case ids its trial shards run instead. */ + excludeCases?: Record; } /** @@ -569,7 +676,8 @@ export function diffSkipDecisionForFile( const touchfiles = options.e2eTouchfiles ?? E2E_TOUCHFILES; const quoted = knownTestNamesInSource(source, allNames); const registered = Object.keys(touchfiles).filter((k) => touchfiles[k].includes(rel)); - const mapped = [...new Set([...quoted, ...registered])]; + const isolated = options.excludeCases?.[rel] ?? []; + const mapped = [...new Set([...quoted, ...registered])].filter(name => !isolated.includes(name)); if (mapped.length === 0) { return { file, kept: true, reason: 'no mappable test names — fail-open, child self-skip authoritative' }; } @@ -679,7 +787,8 @@ export function buildPaidShardArgs( export function shardSlug(files: string[]): string { return files .map((file) => path.basename(shardFile(file)).replace(/\.test\.(?:[cm]?[jt]s|tsx|jsx)$/, '') - + (shardCaseId(file) === null ? '' : `--${shardCaseId(file)}`)) + + (shardCaseId(file) === null ? '' : `--${shardCaseId(file)}`) + + (shardTrial(file) === null ? '' : `.t${shardTrial(file)}`)) .join('+') .replace(/[^a-zA-Z0-9._+-]/g, '-'); } @@ -715,6 +824,91 @@ export interface ShardOutcome { budget?: PaidShardBudget; /** Present when a verified receipt replaced execution (PR lane only). */ reused?: { inputKey: string; runId: string; revision: string; completedAt: number }; + /** The parent could not run the shard at all (a runner error, never a trial verdict). */ + runnerError?: string; + /** Isolated trial shards only: the trial record this shard produced. */ + trial?: ShardTrialRecord; +} + +/** + * One isolated trial's record, derived from its shard status and the records + * in its own eval dir. `outcome` null means the harness produced no trial + * (never started, hollow, isolation broken, runner error): the panel is then + * INCOMPLETE and the slice exits non-zero. A failed, timed-out or crashed + * trial is a trial verdict; the slice still exits zero and the report decides. + */ +export interface ShardTrialRecord { + case: string; + trial: number; + kind: EvalCaseKind; + panel: PanelShape; + quarantined: boolean; + outcome: TrialOutcome | null; + harness?: string; + failure_class?: TrialFailureClass; + exit_reason?: string; + error?: string; + timeout_at_turn?: number; + cost_usd: number; + duration_ms: number; + model?: string; +} + +/** Records and contract evidence an isolated shard left in its eval dir. */ +export function readTrialEvidence(evalDir: string | undefined): { records: any[]; contract: string | null } { + if (!evalDir || !fs.existsSync(evalDir)) return { records: [], contract: null }; + const names = fs.readdirSync(evalDir); + const parse = (name: string) => { try { return JSON.parse(fs.readFileSync(path.join(evalDir, name), 'utf8')); } catch { return null; } }; + const finalized = names.filter(name => isFinalizedEvalResultFile(name) && !name.startsWith('e2e-reused-')).map(parse).filter(Boolean); + const source = finalized.length ? finalized : names.filter(name => name.startsWith('_partial') && name.endsWith('.json')).map(parse).filter(Boolean); + const records = source.flatMap((result: any) => Array.isArray(result?.tests) ? result.tests.filter((t: any) => t && typeof t === 'object') : []); + let contract: string | null = records.find((t: any) => t.failure_class === 'contract')?.error ?? null; + if (records.some((t: any) => t.failure_class === 'contract') && contract === null) contract = 'contract violation'; + try { + const line = fs.readFileSync(path.join(evalDir, CONTRACT_VIOLATIONS_FILE), 'utf8').split('\n').find(l => l.trim()); + if (line) contract = String(JSON.parse(line).message ?? 'contract violation'); + } catch { /* no sidecar */ } + return { records, contract }; +} + +/** Classify one isolated trial shard. Contract evidence always fails the trial. */ +export function classifyTrialShard( + outcome: Pick, + caseId: string, trial: number, plan: CaseTrialPlan, + evidence: { records: any[]; contract: string | null }, +): ShardTrialRecord { + const failedRecord = evidence.records.find(record => record.passed === false) ?? evidence.records[0]; + const base: ShardTrialRecord = { + case: caseId, trial, kind: plan.kind, panel: plan.panel, quarantined: plan.quarantined, outcome: null, + cost_usd: Math.round(evidence.records.reduce((sum, record) => sum + (Number(record.cost_usd) || 0), 0) * 100) / 100, + duration_ms: outcome.elapsedMs, + ...(typeof failedRecord?.model === 'string' ? { model: failedRecord.model } : {}), + }; + const failed = (failureClass: TrialFailureClass, error?: string): ShardTrialRecord => ({ + ...base, outcome: 'failed', failure_class: evidence.contract !== null ? 'contract' : failureClass, + ...(failedRecord?.exit_reason ? { exit_reason: String(failedRecord.exit_reason) } : {}), + ...(Number.isInteger(failedRecord?.timeout_at_turn) ? { timeout_at_turn: failedRecord.timeout_at_turn } : {}), + ...(sanitizeTrialError(evidence.contract ?? failedRecord?.error ?? error) ? { error: sanitizeTrialError(evidence.contract ?? failedRecord?.error ?? error) } : {}), + }); + if (outcome.runnerError !== undefined) return { ...base, harness: `runner error: ${sanitizeTrialError(outcome.runnerError) ?? 'unknown'}` }; + if (outcome.status === 'never-started') return { ...base, harness: 'never started' }; + if (outcome.status === 'passed-empty') return { ...base, harness: 'hollow: executed no case' }; + if (outcome.status === 'skipped-by-diff') return { ...base, harness: 'skipped by diff' }; + const known = outcome.executedTests !== null && outcome.skippedTests !== null; + const ran = known ? outcome.executedTests! - outcome.skippedTests! : null; + if (outcome.status === 'timed-out') { + if (ran !== null && ran > 1) return { ...base, harness: `isolation broken: ${ran} cases ran` }; + return failed('timeout', 'shard wall reached'); + } + if (ran === null) return outcome.status === 'failed' ? failed('infra', 'crashed without a test summary') : { ...base, harness: 'no test summary' }; + if (ran > 1) return { ...base, harness: `isolation broken: ${ran} cases ran` }; + if (ran === 0) { + if (outcome.status === 'passed' && outcome.skippedTests! > 0 && evidence.contract === null) return { ...base, outcome: 'skipped' }; + if (outcome.status === 'failed') return failed('infra', 'the case never ran (load or setup failure)'); + return { ...base, harness: 'hollow: executed no case' }; + } + if (outcome.status === 'passed') return evidence.contract !== null ? failed('contract') : { ...base, outcome: 'passed' }; + return failed(failedRecord ? failureClassOf(failedRecord) : 'assertion'); } /** @@ -775,6 +969,8 @@ export interface RunShardsOptions { expectedCaseIds?: Record; /** PR lane only: verified reuse for one shard's exact child environment and wall. */ reuseFor?: (files: string[], env: NodeJS.ProcessEnv, budget: PaidShardBudget) => E2EShardReuse | null; + /** Isolated trial shards: key -> the case's fixed trial plan. */ + trials?: Record; } let shardLogSequence = 0; @@ -832,6 +1028,23 @@ export async function runPaidShard( if (options.evalDirBase) { env.GSTACK_EVAL_DIR = path.join(options.evalDirBase, 'shards', shardSlug(files)); } + const trialPlan = files.length === 1 ? options.trials?.[normalizeRelativePath(files[0]!)] : undefined; + const trialIndex = files.length === 1 ? shardTrial(files[0]!) : null; + if (trialPlan && trialIndex !== null && caseId !== null) { + // One case, one trial: the selection binds the child to exactly this id, + // and eval-store stamps every record with the trial identity. + Object.assign(env, { + [TRIAL_ENV.caseId]: caseId, [TRIAL_ENV.kind]: trialPlan.kind, [TRIAL_ENV.trial]: String(trialIndex), + [TRIAL_ENV.panelN]: String(trialPlan.panel.n), [TRIAL_ENV.panelK]: String(trialPlan.panel.k), + [TRIAL_ENV.policyVersion]: String(EVAL_POLICY.version), + EVALS_SELECTION_JSON: JSON.stringify({ version: 1, selected: [caseId], reason: `trial ${trialIndex}/${trialPlan.panel.n} of ${caseId}` }), + }); + } else { + for (const name of Object.values(TRIAL_ENV)) delete env[name]; + } + const withTrial = (outcome: ShardOutcome): ShardOutcome => trialPlan && trialIndex !== null && caseId !== null + ? { ...outcome, trial: classifyTrialShard(outcome, caseId, trialIndex, trialPlan, readTrialEvidence(env.GSTACK_EVAL_DIR)) } + : outcome; // Resolve `claude --version` ONCE in the parent (cached across shards) and // hand it to every child: eval-store's fallback is a synchronous spawn on // the same thread that polls PTY sessions, so children must never pay it. @@ -859,9 +1072,9 @@ export async function runPaidShard( }, null, 2)}\n`); } log(`${label} REUSED ${files.join(' ')} — identical inputs passed in run ${reused.source.runId} at ${reusedFrom.completed_at}`); - return { shard: shardNumber, files, status: 'passed', exitCode: 0, elapsedMs: 0, groupPid: null, + return withTrial({ shard: shardNumber, files, status: 'passed', exitCode: 0, elapsedMs: 0, groupPid: null, executedTests: caseIds.length, skippedTests: 0, budget, - reused: { inputKey: reused.key, runId: reused.source.runId, revision: reused.source.revision, completedAt: reused.source.completedAt } }; + reused: { inputKey: reused.key, runId: reused.source.runId, revision: reused.source.revision, completedAt: reused.source.completedAt } }); } const { command, args } = options.commandFor ? options.commandFor(files) @@ -872,8 +1085,11 @@ export async function runPaidShard( timeoutMs, options.withinShardConcurrency ?? DEFAULT_WITHIN_SHARD_CONCURRENCY, retriesForFiles(files), - ), ...(casePattern !== undefined ? ['--test-name-pattern', casePattern] : [])], + ), ...(casePattern !== undefined ? ['--test-name-pattern', casePattern] : []), + // Per-test outcomes for pass-rate history, keyed by Bun test name. + ...(env.GSTACK_EVAL_DIR ? ['--reporter=junit', '--reporter-outfile', path.join(env.GSTACK_EVAL_DIR, 'junit.xml')] : [])], }; + if (env.GSTACK_EVAL_DIR) fs.mkdirSync(env.GSTACK_EVAL_DIR, { recursive: true }); // Per-shard temp + Chromium-profile isolation — the free runner treats // this as mandatory (test-free-shards.ts: two concurrent shards on one // profile dir kill each other's browser; shared tmp cross-contaminates), @@ -1057,7 +1273,7 @@ export async function runPaidShard( ? summary.terminalTestCounts.reduce((a, b) => a + b, 0) : null; const skippedTests = summary.terminalTestCounts.length > 0 ? summary.skippedTests : null; - return { shard: shardNumber, files, status, exitCode, elapsedMs, groupPid, executedTests, skippedTests, budget }; + return withTrial({ shard: shardNumber, files, status, exitCode, elapsedMs, groupPid, executedTests, skippedTests, budget }); } export interface RunSummary { @@ -1166,7 +1382,8 @@ export async function runPaidShards( try { outcomes[index] = await runPaidShard(shards[index], index + 1, shards.length, { ...options, jobs }); } catch (error) { - outcomes[index] = { + const runnerError = error instanceof Error ? error.message : String(error); + const failed: ShardOutcome = { shard: index + 1, files: shards[index], status: 'failed', @@ -1175,7 +1392,13 @@ export async function runPaidShards( groupPid: null, executedTests: null, skippedTests: null, + runnerError, }; + const key = shards[index].length === 1 ? normalizeRelativePath(shards[index][0]!) : ''; + const plan = options.trials?.[key]; + outcomes[index] = plan && shardCaseId(key) !== null && shardTrial(key) !== null + ? { ...failed, trial: classifyTrialShard(failed, shardCaseId(key)!, shardTrial(key)!, plan, { records: [], contract: null }) } + : failed; console.error(`[test:paid] shard ${index + 1} could not run: ${error instanceof Error ? error.message : String(error)}`); } finally { if (overlay) activeOverlayShards--; @@ -1229,6 +1452,10 @@ export interface ManifestEntry { budget?: PaidShardBudget; /** Budget-mode packing weight (recorded wall, or the whole budget when unknown). */ estimatedMs?: number; + /** Isolated trial shard (`#~t`): the case's kind, fixed panel and quarantine at plan time. */ + trial?: CaseTrialPlan; + /** File shard whose isolated cases run as trial shards: their ids, excluded by name here. */ + excludeCases?: string[]; } /** Budget-mode plan: per-executor estimate and the CI job timeout it needs. */ @@ -1293,18 +1520,48 @@ export function writePaidTestDurations(tier: PaidTier, durations: Record#`). */ +function durationKey(key: string): string { + const rel = normalizeRelativePath(key); + return shardTrial(rel) === null ? rel : `${shardFile(rel)}${CASE_KEY_SEPARATOR}${shardCaseId(rel)}`; +} + +/** Recorded wall of a shard key; an unrecorded trial falls back to its whole file's wall. */ +export function recordedShardMs(recorded: Record, key: string): number | undefined { + const rel = normalizeRelativePath(key); + return recorded[durationKey(rel)] ?? (shardTrial(rel) === null ? undefined : recorded[shardFile(rel)]); +} + +/** + * Merge a report's executed single-file outcomes into the seed; all-skipped + * shards carry no cost signal. Trials of one case record their longest wall + * under the case key. + */ export function mergePaidTestDurations(seed: Record, results: SliceResult[]): Record { const merged = { ...seed }; + const fresh = new Map(); for (const result of results) { for (const outcome of result.outcomes) { - if (outcome.files.length !== 1 || outcome.elapsedMs < 1_000 || isAllSkippedPass(outcome)) continue; - merged[normalizeRelativePath(outcome.files[0])] = outcome.elapsedMs; + if (outcome.files.length !== 1 || outcome.elapsedMs < 1_000 || isAllSkippedPass(outcome) || outcome.reused) continue; + const key = durationKey(outcome.files[0]!); + fresh.set(key, Math.max(fresh.get(key) ?? 0, outcome.elapsedMs)); } } + for (const [key, ms] of fresh) merged[key] = ms; return Object.fromEntries(Object.entries(merged).sort(([a], [b]) => (a < b ? -1 : 1))); } +/** Panel identity of a trial shard key (`#`), else null. */ +export function trialPanelKey(key: string): string | null { + return shardTrial(key) === null ? null : durationKey(key); +} + +/** True when `key` is a trial whose panel already has a trial in `planned`. */ +function sharesPanel(planned: readonly string[], key: string): boolean { + const panel = trialPanelKey(key); + return panel !== null && planned.some(other => other !== key && trialPanelKey(other) === panel); +} + /** Setup, image pull and artifact upload allowance on top of a slice's supervised wall. */ export const CI_SETUP_ALLOWANCE_MINUTES = 20; @@ -1342,12 +1599,15 @@ export function packBySliceBudget(files: string[], budgetMs: number, jobs: numbe recorded: Record, timeoutMs?: number): { slices: string[][]; estimates: Record; estimatedSliceMs: number[]; ciTimeoutMinutes: number; } { - const estimates = Object.fromEntries(files.map(file => [file, recorded[normalizeRelativePath(file)] ?? budgetMs])); + const estimates = Object.fromEntries(files.map(file => [file, recordedShardMs(recorded, file) ?? budgetMs])); const weight = (file: string) => estimates[file]!; const slices: string[][] = []; for (const file of sliceExecutionOrder(files.filter(file => !isOverlayTestFile(file)).map(file => ({ file, estimatedMs: weight(file) }))).map(entry => entry.file)) { let best = -1, bestMs = -1; slices.forEach((planned, index) => { + // Trials of one case never share a runner: independent machines, and + // the panel's wall stays one trial long. + if (sharesPanel(planned, file)) return; const ms = estimatedSliceMs([...planned, file], weight, jobs); if (ms <= budgetMs && ms > bestMs) { best = index; bestMs = ms; } }); @@ -1396,6 +1656,9 @@ export function buildRunManifest(opts: { durations?: Record; /** Weekly gate census only: LLM judges already run in the periodic census and PR gate lanes. */ skipJudges?: boolean; + /** Injectable registries (default: E2E_KINDS and CASE_QUARANTINE). */ + kinds?: Record; + quarantine?: Record; }): PaidRunManifest { const budgetMode = opts.sliceBudgetMs !== undefined; if (budgetMode === (opts.sliceCount !== undefined)) throw new Error('Plan with exactly one of --slices or --slice-budget'); @@ -1413,27 +1676,38 @@ export function buildRunManifest(opts: { const tierSelection = selectPaidTestFiles(discovered, opts.tier, rootDir, env); const judge = (file: string) => /^test\/skill-llm-eval[^/]*\.test\.ts$/.test(normalizeRelativePath(file)); const selected = opts.skipJudges ? tierSelection.selected.filter(file => !judge(file)) : tierSelection.selected; + const kinds = opts.kinds ?? E2E_KINDS; + const quarantine = opts.quarantine ?? CASE_QUARANTINE; + const notLive = [...Object.keys(kinds).filter(id => kinds[id] === 'behavior'), ...Object.keys(quarantine)].filter(id => !Object.hasOwn(E2E_TIERS, id)); + if (notLive.length) throw new Error(`Only live E2E cases can be behavior or quarantined (judges sample their panel inside the case): ${notLive.join(', ')}`); const caseKeys = partitionCaseExclusions(expandCaseShards(selected, opts.tier, rootDir)); const excluded = [...tierSelection.excluded, ...(opts.skipJudges ? tierSelection.selected.filter(judge) .map(file => ({ file, reason: 'skipped: LLM judges run in the periodic census and PR gate lanes' })) : []), ...caseKeys.excluded]; - const shards = planPaidShards(caseKeys.runnable, { maxFilesPerShard: 1 }); + const expansion = expandTrialShards(caseKeys.runnable, opts.tier, rootDir, { kinds, quarantine }); + const excludeOf = (key: string) => expansion.excludeCases[normalizeRelativePath(key)] ?? []; + const shards = planPaidShards(expansion.keys, { maxFilesPerShard: 1 }); const cases = computePaidCaseSelection({ profile, env, rootDir, changedFiles: opts.changedFiles }); const fast = cases.coverage?.mode === 'pr'; - const profileShards = fast ? shards.filter(files => prProfileFileSelected(files[0], cases.selection)) : shards; + const profileShards = fast ? shards.filter(files => prProfileFileSelected(files[0], cases.selection, excludeOf(files[0]!))) : shards; const { runnable, skipped } = partitionShardsByDiffSelection(profileShards, - cases.selection.e2e === null ? null : new Set(cases.selection.e2e), { rootDir }); + cases.selection.e2e === null ? null : new Set(cases.selection.e2e), { rootDir, excludeCases: expansion.excludeCases }); if (fast) for (const files of shards) { - if (!prProfileFileSelected(files[0], cases.selection)) skipped.push({ files, reason: 'Outside the fast PR profile; retained in broad gate/periodic coverage' }); + if (!prProfileFileSelected(files[0], cases.selection, excludeOf(files[0]!))) skipped.push({ files, reason: 'Outside the fast PR profile; retained in broad gate/periodic coverage' }); } + const extras = (key: string): Pick => { + const trial = expansion.trials[normalizeRelativePath(key)]; + const exclude = expansion.excludeCases[normalizeRelativePath(key)]; + return { ...(trial ? { trial } : {}), ...(exclude ? { excludeCases: exclude } : {}) }; + }; const entries: ManifestEntry[] = []; if (budgetMode) { const plan = packBySliceBudget(runnable.map(files => files[0]!), opts.sliceBudgetMs!, opts.jobs!, opts.durations ?? loadPaidTestDurations(rootDir, opts.tier), opts.timeoutMs); plan.slices.forEach((files, index) => files.forEach(file => entries.push({ file, slice: index + 1, status: 'planned', - estimatedMs: plan.estimates[file]!, + estimatedMs: plan.estimates[file]!, ...extras(file), ...(FILE_RETRY_BUDGETS.some(budget => budget.file === shardFile(file)) ? { budget: resolvePaidShardBudget([file], opts.timeoutMs) } : {}) }))); - for (const s of skipped) entries.push({ file: s.files[0], slice: 0, status: 'skipped-by-diff', reason: s.reason }); + for (const s of skipped) entries.push({ file: s.files[0], slice: 0, status: 'skipped-by-diff', reason: s.reason, ...extras(s.files[0]!) }); for (const e of excluded) entries.push({ file: e.file, slice: 0, status: 'excluded', reason: e.reason }); entries.sort((a, b) => (a.file < b.file ? -1 : 1)); return parseRunManifest(JSON.stringify({ @@ -1464,15 +1738,28 @@ export function buildRunManifest(opts: { for (const files of [...registered].sort(byWall).concat( ordinary.filter(files => !registeredFiles.has(files[0])))) { const lanes = registeredFiles.has(files[0]) ? longLanes : ordinarySlices; - let lane = 0; - for (let index = 1; index < lanes; index++) if (loads[index] < loads[lane]) lane = index; + const laneKeys = (index: number) => [...allocations].filter(([, lane]) => lane === index + 1).map(([key]) => key); + let lane = -1; + for (let index = 0; index < lanes; index++) { + if (sharesPanel(laneKeys(index), files[0]!)) continue; + if (lane < 0 || loads[index] < loads[lane]) lane = index; + } + if (lane < 0) lane = loads.slice(0, lanes).indexOf(Math.min(...loads.slice(0, lanes))); allocations.set(files[0], lane + 1); loads[lane] += resolvePaidShardTimeoutMs(files, opts.timeoutMs); } } let ordinaryIndex = 0; for (const files of ordinary) { - if (!allocations.has(files[0])) allocations.set(files[0], (ordinaryIndex++ % ordinarySlices) + 1); + if (allocations.has(files[0])) continue; + // Round-robin, skipping a lane that already holds a trial of the same panel. + let lane = ordinaryIndex % ordinarySlices; + for (let step = 0; step < ordinarySlices; step++) { + const candidate = (ordinaryIndex + step) % ordinarySlices; + if (!sharesPanel([...allocations].filter(([, l]) => l === candidate + 1).map(([key]) => key), files[0]!)) { lane = candidate; break; } + } + ordinaryIndex++; + allocations.set(files[0], lane + 1); } const packed = packByRecordedDuration(); function packByRecordedDuration(): Map | null { @@ -1483,10 +1770,10 @@ export function buildRunManifest(opts: { ordinary.filter(files => allocations.get(files[0]) === lane + 1).map(files => files[0])); const caps = SUPERVISED_WORKER_COUNTS.map(jobs => Math.max(...lanes.map(files => bound(files, jobs)))); const fits = (files: string[]) => SUPERVISED_WORKER_COUNTS.every((jobs, k) => bound(files, jobs) <= caps[k]); - const known = ordinary.map(files => recorded[normalizeRelativePath(files[0])]) + const known = ordinary.map(files => recordedShardMs(recorded, files[0]!)) .filter((ms): ms is number => ms !== undefined).sort((a, b) => a - b); const fallback = known.length ? known[Math.min(known.length - 1, Math.floor(known.length * 0.75))] : 1; - const weight = (file: string) => recorded[normalizeRelativePath(file)] ?? fallback; + const weight = (file: string) => recordedShardMs(recorded, file) ?? fallback; const load = (files: string[]) => files.reduce((sum, file) => sum + weight(file), 0); const registeredFiles = new Set(registered.map(files => files[0])); // Local search from the supervised baseline: move or swap a file out of @@ -1510,6 +1797,7 @@ export function buildRunManifest(opts: { const heavyAfter = lanes[heavy].filter(file => file !== a).concat(b === null ? [] : [b]); const otherAfter = lanes[other].filter(file => file !== b).concat([a]); if (!fits(heavyAfter) || !fits(otherAfter)) continue; + if (sharesPanel(otherAfter, a) || (b !== null && sharesPanel(heavyAfter, b))) continue; const [h, o] = [heavy, other]; best = { gain, apply: () => { lanes[h] = heavyAfter; lanes[o] = otherAfter; } }; } @@ -1522,11 +1810,11 @@ export function buildRunManifest(opts: { } runnable.forEach((files) => { const slice = files.some(isOverlayTestFile) ? overlaySlice : (packed ?? allocations).get(files[0])!; - entries.push({ file: files[0], slice, status: 'planned', + entries.push({ file: files[0], slice, status: 'planned', ...extras(files[0]!), ...(FILE_RETRY_BUDGETS.some(budget => budget.file === shardFile(files[0]!)) ? { budget: resolvePaidShardBudget(files, opts.timeoutMs) } : {}) }); }); - for (const s of skipped) entries.push({ file: s.files[0], slice: 0, status: 'skipped-by-diff', reason: s.reason }); + for (const s of skipped) entries.push({ file: s.files[0], slice: 0, status: 'skipped-by-diff', reason: s.reason, ...extras(s.files[0]!) }); for (const e of excluded) entries.push({ file: e.file, slice: 0, status: 'excluded', reason: e.reason }); entries.sort((a, b) => (a.file < b.file ? -1 : 1)); @@ -1603,26 +1891,70 @@ export function parseRunManifest(raw: string): PaidRunManifest { if (entry.estimatedMs !== undefined && (entry.status !== 'planned' || !Number.isSafeInteger(entry.estimatedMs) || entry.estimatedMs < 0)) { throw new Error(`manifest entry ${entry.file} has an invalid estimate`); } - if (entry.status === 'planned' && parsed.prCoverage?.mode === 'pr' && !prProfileFileSelected(entry.file, parsed.selection!)) { + if (entry.status === 'planned' && parsed.prCoverage?.mode === 'pr' && !prProfileFileSelected(entry.file, parsed.selection!, entry.excludeCases ?? [])) { throw new Error(`manifest file is outside its PR case selection: ${entry.file}`); } } for (const entry of parsed.entries) { const caseId = shardCaseId(entry.file); - if (caseId === null ? entry.status === 'planned' && CASE_SHARDED_FILES.includes(shardFile(entry.file)) + const trial = shardTrial(entry.file); + if (trial !== null) { + const plan = entry.trial; + const expected = plan && ['rule', 'behavior', 'judge'].includes(plan.kind) && typeof plan.quarantined === 'boolean' + ? caseTrialPlan(caseId!, { [caseId!]: plan.kind }, plan.quarantined ? { [caseId!]: true } : {}) : undefined; + if (!Object.hasOwn(E2E_TOUCHFILES, caseId!) || !E2E_TOUCHFILES[caseId!]!.includes(shardFile(entry.file)) + || !plan || !expected || !isIsolatedCase(expected) || !sameTrialPlan(plan, expected) || trial > plan.panel.n) { + throw new Error(`Trial shard must name a registered isolated case of its file with its fixed policy panel: ${entry.file}`); + } + } else if (entry.trial !== undefined) { + throw new Error(`Only trial shards carry a trial plan: ${entry.file}`); + } else if (caseId === null ? entry.status === 'planned' && CASE_SHARDED_FILES.includes(shardFile(entry.file)) : !CASE_SHARDED_FILES.includes(shardFile(entry.file)) || !(caseId in E2E_TOUCHFILES)) { throw new Error(`Case-sharded files plan one registered case per shard: ${entry.file}`); } + if (entry.excludeCases !== undefined && (caseId !== null || !Array.isArray(entry.excludeCases) || entry.excludeCases.length === 0 + || entry.excludeCases.some(id => !parsed.entries.some(other => shardTrial(other.file) !== null + && shardFile(other.file) === shardFile(entry.file) && shardCaseId(other.file) === id)))) { + throw new Error(`A file shard may exclude only cases that run as its trial shards: ${entry.file}`); + } if (caseId !== null && entry.status === 'planned' && parsed.selection?.e2e && !parsed.selection.e2e.includes(caseId)) { throw new Error(`Planned case shard is outside the manifest selection: ${entry.file}`); } } + // Panels are whole: exactly n trial entries per isolated case with one plan + // and one status, and planned trials on distinct slices when the ordinary + // slices allow it. + const panels = new Map(); + for (const entry of parsed.entries) { + const panel = trialPanelKey(entry.file); + if (panel !== null) panels.set(panel, [...(panels.get(panel) ?? []), entry]); + } + const reservedOverlay = parsed.sliceCount > 1 && parsed.entries.some(entry => entry.status === 'planned' && isOverlayTestFile(entry.file)); + const ordinarySliceCount = parsed.sliceCount - Number(reservedOverlay); + for (const [panel, trials] of panels) { + const n = trials[0]!.trial!.panel.n; + const indices = trials.map(entry => shardTrial(entry.file)!).sort((a, b) => a - b); + if (indices.length !== n || indices.some((index, i) => index !== i + 1) + || trials.some(entry => !sameTrialPlan(entry.trial, trials[0]!.trial) || entry.status !== trials[0]!.status) + || parsed.entries.some(entry => normalizeRelativePath(entry.file) === panel)) { + throw new Error(`Isolated case ${panel} must plan exactly its ${n} trials together`); + } + const slices = trials.filter(entry => entry.status === 'planned').map(entry => entry.slice); + if (ordinarySliceCount >= n && new Set(slices).size !== slices.length) { + throw new Error(`Trials of ${panel} share a slice; each trial needs its own runner`); + } + } if (parsed.prCoverage?.mode === 'pr') { const planned = parsed.entries.filter(entry => entry.status === 'planned').map(entry => normalizeRelativePath(entry.file)); const required: string[][] = Object.entries(PR_PROFILE_FILES).flatMap(([file, ids]) => { const selected = ids.filter(id => parsed.selection!.e2e!.includes(id)); if (!selected.length) return []; - return [CASE_SHARDED_FILES.includes(file) ? selected.map(id => `${file}#${id}`) : [file]]; + const owners = new Set(selected.flatMap(id => { + const trials = parsed.entries.filter(entry => shardTrial(entry.file) !== null && shardFile(entry.file) === file && shardCaseId(entry.file) === id); + if (trials.length) return trials.map(entry => normalizeRelativePath(entry.file)); + return [CASE_SHARDED_FILES.includes(file) ? `${file}#${id}` : file]; + })); + return [[...owners]]; }); if (parsed.selection!.judges!.length) required.push(['test/skill-llm-eval.test.ts']); for (const owners of required) { @@ -1644,6 +1976,14 @@ export function parseRunManifest(raw: string): PaidRunManifest { } const keys = parsed.entries.map(entry => normalizeRelativePath(entry.file)); if (new Set(keys).size !== keys.length) throw new Error('Duplicate manifest entry'); + // Unique result slugs: shard artifacts merge by path, so a shared slug would + // let one trial's records overwrite another's. + const slugs = new Map(); + for (const entry of parsed.entries) { + const slug = shardSlug([entry.file]); + if (slugs.has(slug)) throw new Error(`Shards ${slugs.get(slug)} and ${entry.file} share the result slug ${slug}`); + slugs.set(slug, entry.file); + } const overlaySlice = parsed.sliceCount; const plannedOverlays = parsed.entries.filter(entry => entry.status === 'planned' && isOverlayTestFile(entry.file)); if (plannedOverlays.some(entry => entry.slice !== overlaySlice)) { @@ -1655,7 +1995,9 @@ export function parseRunManifest(raw: string): PaidRunManifest { } for (const budget of FILE_RETRY_BUDGETS) { const entries = parsed.entries.filter(entry => shardFile(entry.file) === budget.file); - if (entries.length > 1 && entries.some(entry => shardCaseId(entry.file) === null)) throw new Error(`Duplicate registered manifest entry: ${budget.file}`); + const fileKeys = entries.filter(entry => shardCaseId(entry.file) === null).length; + const caseKeys = entries.filter(entry => shardCaseId(entry.file) !== null && shardTrial(entry.file) === null).length; + if (fileKeys > 1 || (fileKeys === 1 && caseKeys > 0)) throw new Error(`Duplicate registered manifest entry: ${budget.file}`); for (const entry of entries.filter(entry => entry.status === 'planned')) { if (!entry.budget) throw new Error(`Registered manifest needs an explicit budget record: ${budget.file}`); const expected = resolvePaidShardBudget([entry.file], entry.budget.source === 'explicit' ? entry.budget.timeoutMs : undefined); @@ -1673,7 +2015,31 @@ export interface SliceResult { sliceIndex: number; sliceCount: number; timeoutOverrideMs?: number; - outcomes: Array>; + /** CI run attempt (github.run_attempt) that produced this slice; absent means 1. */ + attempt?: number; + /** Epoch ms bounds of the slice's shard execution (lane wall time). */ + startedAt?: number; + finishedAt?: number; + outcomes: Array>; +} + +/** + * Slice exit = execution completeness, never the semantic verdict. A rule + * shard that did not pass fails the slice (unchanged fail-closed rule); an + * isolated trial shard fails it only when the harness produced no trial + * record. Failed, timed-out or crashed trials are verdict input for the + * report's panelVerdict(), so a 2/3 PASS panel never reds its runner. + */ +export function sliceExitCode(outcomes: ReadonlyArray>): number { + return outcomes.every(outcome => outcome.trial !== undefined + ? outcome.trial.outcome !== null + : outcome.status === 'passed' || outcome.status === 'skipped-by-diff') ? 0 : 1; +} + +/** A hollow-guarded trial shard has no trial record: the guard's verdict is a harness problem. */ +export function guardTrialRecords>(outcomes: T[]): T[] { + return outcomes.map(outcome => outcome.trial && outcome.status === 'passed-empty' && outcome.trial.outcome !== null + ? { ...outcome, trial: { ...outcome.trial, outcome: null, harness: 'hollow: executed no case' } } : outcome); } /** @@ -1704,21 +2070,35 @@ export function verifySliceResults( if (!byIndex.has(index)) problems.push(`slice ${index}/${manifest.sliceCount} reported NO result — cancelled/crashed executor, not a pass`); } - const reported = new Map(); + const reported = new Map(); + const planned = new Map(manifest.entries.map(entry => [normalizeRelativePath(entry.file), entry])); for (const result of results) { for (const outcome of result.outcomes) { if (outcome.files.some(file => FILE_RETRY_BUDGETS.some(budget => budget.file === shardFile(file)) || shardCaseId(file) !== null) && outcome.files.length !== 1) { problems.push('Registered result must report its own shard'); } const file = normalizeRelativePath(outcome.files[0] ?? ''); + const trialPlan = planned.get(file)?.trial; if (manifest.prCoverage?.mode === 'pr') { - const expected = expectedPrCaseCount(file, manifest.selection!); + const expected = expectedPrCaseCount(file, manifest.selection!, planned.get(file)?.excludeCases); const executed = outcome.executedTests === null || outcome.skippedTests === null ? -1 : outcome.executedTests - outcome.skippedTests; - if (outcome.exitCode !== 0 || expected < 1 || executed !== expected) { + // A trial's exit status is its verdict (panelVerdict decides); its + // completeness is the trial record checked below. + if ((!trialPlan && outcome.exitCode !== 0) || expected < 1 || (!trialPlan && executed !== expected)) { problems.push(`PR profile expected ${expected} executed cases in ${file}, received ${executed}`); } } + if (trialPlan) { + const t = outcome.trial; + if (!t || t.case !== shardCaseId(file) || t.trial !== shardTrial(file) || !sameTrialPlan(t, trialPlan) + || !(t.outcome === null || ['passed', 'failed', 'skipped'].includes(t.outcome)) + || (t.outcome === 'failed') !== (t.failure_class !== undefined)) { + problems.push(`${file}: trial record missing or does not match its planned trial`); + } + } else if (outcome.trial !== undefined) { + problems.push(`${file}: an unplanned trial record`); + } if (shardCaseId(file) !== null && outcome.status === 'passed' && (outcome.executedTests === null || outcome.skippedTests === null || outcome.executedTests - outcome.skippedTests !== 1)) { problems.push(`Case shard must execute exactly its one case: ${file}`); @@ -1732,7 +2112,7 @@ export function verifySliceResults( } } if (reported.has(file)) problems.push(`${file} reported by two slices`); - reported.set(file, { slice: result.sliceIndex, status: outcome.status }); + reported.set(file, { slice: result.sliceIndex, status: outcome.status, ...(outcome.trial ? { trial: outcome.trial } : {}) }); const registered = FILE_RETRY_BUDGETS.find(budget => budget.file === shardFile(file)); const finding = STRICT_RETRY_CASE_BUDGETS.find(budget => budget.file === file); if (finding) { @@ -1762,7 +2142,10 @@ export function verifySliceResults( continue; // the missing-slice problem above already covers it } if (got.slice !== entry.slice) problems.push(`${entry.file} planned for slice ${entry.slice} but reported by slice ${got.slice}`); - if (got.status !== 'passed') problems.push(`${entry.file}: ${got.status}`); + // Isolated trial shards: harness health only; the panel verdict gates. + if (entry.trial) { + if (got.trial?.outcome === null) problems.push(`${entry.file}: no trial record (${got.trial.harness ?? 'unknown'})`); + } else if (got.status !== 'passed') problems.push(`${entry.file}: ${got.status}`); } return { ok: problems.length === 0, problems }; } @@ -1783,6 +2166,27 @@ export function formatSlicePlan(manifest: PaidRunManifest): string[] { return lines; } +/** + * Capacity preflight (E-A10): what the plan asks of the runner pool and the + * API. Sessions are planned shard processes; at most jobs of them run per + * slice at once. Waves > 1 mean slices queue behind the matrix cap and the + * lane wall grows by whole slices. + */ +export function formatCapacityPreflight(manifest: PaidRunManifest, maxParallel?: number): string[] { + const planned = manifest.entries.filter(entry => entry.status === 'planned'); + const trials = planned.filter(entry => entry.trial); + const jobs = manifest.plan?.jobs ?? DEFAULT_JOBS; + const longest = [...trials].sort((a, b) => (b.estimatedMs ?? 0) - (a.estimatedMs ?? 0))[0]; + const waves = maxParallel ? Math.ceil(manifest.sliceCount / maxParallel) : null; + return [ + `[test:paid] capacity: ${manifest.sliceCount} slice(s), ${planned.length} planned shard(s) (${planned.length - trials.length} rule/judge, ${trials.length} trial shard(s) in ${new Set(trials.map(entry => trialPanelKey(entry.file))).size} panel(s)); ` + + `peak ${Math.min(manifest.sliceCount, maxParallel ?? manifest.sliceCount) * jobs} concurrent shard process(es)` + + (waves !== null ? `; wave(s) at max-parallel ${maxParallel}: ${waves}` : ''), + ...(longest ? [`[test:paid] capacity: longest indivisible trial ~${((longest.estimatedMs ?? 0) / 60_000).toFixed(1)}m (${longest.file})`] : []), + ...(waves !== null && waves > 1 ? [`[test:paid] capacity: ⚠ ${manifest.sliceCount} slices exceed max-parallel ${maxParallel}; later slices queue for a second wave`] : []), + ]; +} + export function formatProfileCoverage(manifest: PaidRunManifest): string[] { const coverage = manifest.prCoverage; return [ @@ -1791,19 +2195,15 @@ export function formatProfileCoverage(manifest: PaidRunManifest): string[] { ]; } -/** Final outcomes use each case's last attempt; the attempt total stays visible. */ +/** Every collector record counts: paid evals never retry, so a later record never replaces an earlier one. */ export function collectorOutcomeCounts(results: Array<{ tests?: Array<{ name: string; suite?: string; passed: boolean; execution?: string; manual_review?: unknown; }> }>): { executed: number; reused: number; passed: number; failed: number; manual_accepted: number; attempts: number } { const counts = { executed: 0, reused: 0, passed: 0, failed: 0, manual_accepted: 0, attempts: 0 }; for (const result of results) { - const cases = new Map[number]>(); for (const entry of result.tests ?? []) { if (!entry || typeof entry !== 'object' || typeof entry.name !== 'string' || typeof entry.passed !== 'boolean') continue; counts.attempts++; - cases.set(`${entry.suite ?? ''}\0${entry.name}`, entry); - } - for (const entry of cases.values()) { counts[entry.execution === 'reused' ? 'reused' : 'executed']++; const outcome = evalEntryOutcome(entry); counts[outcome === 'manual-review' ? 'manual_accepted' : outcome]++; @@ -1812,6 +2212,438 @@ export function collectorOutcomeCounts(results: Array<{ tests?: Array<{ return counts; } + +// ─── Report: verdicts, history records and the human readout ─────────────── + +/** One Bun JUnit testcase (`--reporter=junit`). */ +export interface JUnitCase { name: string; classname: string; outcome: TrialOutcome; timeMs: number; failureType?: string; message?: string } + +const xmlUnescape = (text: string) => text.replace(/&(lt|gt|quot|apos|amp|#(\d+)|#x([0-9a-f]+));/gi, (_, name: string, dec?: string, hex?: string) => + dec ? String.fromCodePoint(Number(dec)) : hex ? String.fromCodePoint(parseInt(hex, 16)) + : ({ lt: '<', gt: '>', quot: '"', apos: "'", amp: '&' } as Record)[name.toLowerCase()]!); + +function xmlAttributes(tag: string): Record { + return Object.fromEntries([...tag.matchAll(/([\w:-]+)="([^"]*)"/g)].map(match => [match[1]!, xmlUnescape(match[2]!)])); +} + +/** Per-test outcomes from a Bun JUnit report; unparseable input yields []. */ +export function parseJUnitCases(xml: string): JUnitCase[] { + const cases: JUnitCase[] = []; + for (const match of xml.matchAll(/]*?)(\/>|>([\s\S]*?)<\/testcase>)/g)) { + const attrs = xmlAttributes(match[1]!); + const body = match[3] ?? ''; + const failure = /<(failure|error)\b([^>]*?)(?:\/>|>)/.exec(body); + const failureAttrs = failure ? xmlAttributes(failure[2]!) : {}; + cases.push({ + name: attrs.name ?? '', classname: attrs.classname ?? '', + outcome: failure ? 'failed' : / CASE_TEST_NAMES[id] === name) ?? null; +} + +interface ReportArtifact { root: string; result: SliceResult } + +/** Every slice result under the report dir: flat (merged) or one directory per attempt-scoped artifact. */ +export function loadSliceArtifacts(reportDir: string): ReportArtifact[] { + const found: ReportArtifact[] = []; + for (const name of fs.readdirSync(reportDir, { recursive: true }) as string[]) { + const rel = normalizeRelativePath(name); + if (!/^slice-\d+\.json$/.test(path.basename(rel)) || rel.split('/').includes('shards') || rel.split('/').includes('receipts')) continue; + found.push({ root: path.join(reportDir, path.dirname(rel)), result: JSON.parse(fs.readFileSync(path.join(reportDir, rel), 'utf8')) as SliceResult }); + } + return found.sort((a, b) => (a.result.attempt ?? 1) - (b.result.attempt ?? 1) || a.result.sliceIndex - b.result.sliceIndex); +} + +export interface PanelReport extends PanelVerdict { + file: string; + /** Slice per trial index (trial n -> slice), for the rerun/artifact pointer. */ + slices: Record; +} + +/** Panel verdicts of one run attempt: exactly the planned trials, each from its reported record. */ +export function panelReports(manifest: PaidRunManifest, results: SliceResult[], attempt: number): PanelReport[] { + const reported = new Map(); + for (const result of results.filter(r => (r.attempt ?? 1) === attempt)) { + for (const outcome of result.outcomes) reported.set(normalizeRelativePath(outcome.files[0] ?? ''), { slice: result.sliceIndex, outcome }); + } + const panels = new Map(); + for (const entry of manifest.entries.filter(e => e.status === 'planned' && e.trial)) { + const key = trialPanelKey(entry.file)!; + panels.set(key, [...(panels.get(key) ?? []), entry]); + } + return [...panels.entries()].sort(([a], [b]) => (a < b ? -1 : 1)).map(([key, entries]) => { + const plan = entries[0]!.trial!; + const slices: Record = {}; + const trials = entries.flatMap(entry => { + const got = reported.get(normalizeRelativePath(entry.file)); + const t = got?.outcome.trial; + if (!got || !t || t.outcome === null) return []; + slices[t.trial] = got.slice; + return [{ trial: t.trial, outcome: t.outcome, attempt, ...(t.failure_class ? { failure_class: t.failure_class } : {}), + ...(t.exit_reason ? { exit_reason: t.exit_reason } : {}), ...(t.error ? { error: t.error } : {}), + ...(got.outcome.reused ? { execution: 'reused' as const } : {}), + ...(t.timeout_at_turn !== undefined ? { timeout_at_turn: t.timeout_at_turn } : {}) }]; + }); + const verdict = panelVerdict({ case: shardCaseId(key)!, kind: plan.kind, panel: plan.panel, trials, quarantined: plan.quarantined }); + return { ...verdict, file: shardFile(key), slices }; + }); +} + +/** Why one failed trial failed, in one line: a case timeout names its turn. */ +function trialCause(trial: PanelVerdict['trials'][number] & { timeout_at_turn?: number }): string { + const cls = trial.failure_class ?? 'assertion'; + const head = cls === 'timeout' || trial.exit_reason === 'timeout' + ? `timeout${trial.timeout_at_turn !== undefined ? ` at turn ${trial.timeout_at_turn}` : ''}` + : cls; + return `t${trial.trial}: ${head}${trial.exit_reason && trial.exit_reason !== 'timeout' ? ` (${trial.exit_reason})` : ''}${trial.error ? ` — ${trial.error}` : ''}`; +} + +export function rerunCommand(tier: PaidTier, id: string, trials: number): string { + return `bun run scripts/test-paid-shards.ts --tier ${tier} --case ${id}${trials > 1 ? ` --trials ${trials}` : ''}`; +} + +/** One line per non-PASS or split panel verdict. */ +export function formatPanelLine(panel: PanelReport, tier: PaidTier): string { + const mark = panel.status === 'PASS' ? '⚠' : panel.failsLane ? '✗' : '◌'; + const label = panel.status === 'PASS' ? `PASS ${panel.passed}/${panel.panel.n}` : `${panel.status} ${panel.passed}/${panel.panel.n}`; + const causes = panel.trials.filter(t => t.outcome !== 'passed').map(t => trialCause(t as any)); + const where = Object.entries(panel.slices).map(([trial, slice]) => `t${trial}@slice ${slice}`).join(', '); + return `${mark} ${panel.case} ${panel.kind}${panel.quarantined ? ' (quarantined)' : ''} ${label} (${panel.marks})` + + `${causes.length ? ` ${causes.join('; ')}` : ''}${panel.status === 'INCOMPLETE' ? ` [${panel.reason}]` : ''}` + + `${where ? ` [${where}, attempt ${panel.attempt}]` : ''} rerun: ${rerunCommand(tier, panel.case, panel.panel.n)}`; +} + +export interface ReportHeadline { + lane: string; + verdict: 'GREEN' | 'RED'; + attempt: number; + counts: { + rule: { passed: number; total: number }; + behavior: { passed: number; total: number; split: number }; + judge: { passed: number; total: number }; + quarantined: { total: number; failingLane: number }; + skipped: number; + infra: number; + incomplete: number; + unattributed: number; + }; + actionRequired: number; + wallMs: number | null; + costUsd: number; + redispatchEligible: boolean; +} + +export function formatHeadline(h: ReportHeadline): string[] { + const c = h.counts; + const minutes = h.wallMs === null ? 'unknown' : `${Math.floor(h.wallMs / 60_000)}m${String(Math.round((h.wallMs % 60_000) / 1000)).padStart(2, '0')}s`; + return [ + `[test:paid] VERDICT ${h.verdict} — lane ${h.lane}, attempt ${h.attempt}`, + ` rule ${c.rule.passed}/${c.rule.total} · behavior ${c.behavior.passed}/${c.behavior.total}${c.behavior.split ? ` (${c.behavior.split} split)` : ''}` + + ` · judge ${c.judge.passed}/${c.judge.total} · quarantined ${c.quarantined.total} (${c.quarantined.failingLane} failing the lane)`, + ` SKIPPED ${c.skipped} · INFRA ${c.infra} · INCOMPLETE ${c.incomplete} · unattributed ${c.unattributed} · ACTION REQUIRED ${h.actionRequired}`, + ` wall ${minutes} · cost $${h.costUsd.toFixed(2)}${h.redispatchEligible ? ' · every red is machine-classified INFRA/INCOMPLETE: eligible for ONE re-dispatch as a new run (EVAL_POLICY.infraRedispatch); report both runs' : ''}`, + ]; +} + +/** Problems a runner loss or an API/CLI failure before grading produces; nothing else qualifies for re-dispatch. */ +const INFRA_PROBLEMS = [ + /^slice \d+\/\d+ reported NO result/, + /^planned .* was never reported$/, + /: never-started$/, + /: no trial record \((?:never started|runner error: .*|no test summary)\)$/, + /^PANEL \S+ INCOMPLETE /, + /^PANEL \S+ FAIL \(INFRA\)/, +]; +export function infraOnly(problems: readonly string[]): boolean { + return problems.length > 0 && problems.every(problem => INFRA_PROBLEMS.some(re => re.test(problem))); +} + +/** + * Report mode: reconcile slice artifacts against the manifest (fail-closed), + * compute every panel verdict with panelVerdict(), write collector-outcomes + * v2, trial-outcomes.jsonl and report-summary.md, and exit non-zero when the + * lane is red. Only the earliest run attempt decides the lane; later attempts + * are reported beside it and never replace it. + */ +export function runPaidReport(reportDir: string, options: { writeDurations?: boolean; env?: NodeJS.ProcessEnv; rootDir?: string } = {}): number { + const env = options.env ?? process.env; + const rootDir = options.rootDir ?? ROOT; + const summaryPath = path.join(reportDir, 'collector-outcomes.json'); + const summaryMdPath = path.join(reportDir, 'report-summary.md'); + const trialOutcomesPath = path.join(reportDir, TRIAL_OUTCOMES_FILE); + for (const file of [summaryPath, summaryMdPath, trialOutcomesPath]) fs.rmSync(file, { force: true }); + const manifest = parseRunManifest(fs.readFileSync(path.join(reportDir, 'manifest.json'), 'utf-8')); + const artifacts = loadSliceArtifacts(reportDir); + const attempts = [...new Set(artifacts.map(a => a.result.attempt ?? 1))].sort((a, b) => a - b); + const primary = attempts[0] ?? 1; + const results = artifacts.filter(a => (a.result.attempt ?? 1) === primary).map(a => a.result); + const verdict = verifySliceResults(manifest, results); + const planned = manifest.entries.filter((e) => e.status === 'planned').length; + const lane = `${manifest.tier}/${manifest.profile ?? 'full'}${manifest.evalsAll ? ' census' : ''}`; + console.log(`[test:paid] report: ${results.length}/${manifest.sliceCount} slices, ${planned} planned shards, tier=${manifest.tier}, attempt ${primary}${attempts.length > 1 ? ` (later attempts ${attempts.slice(1).join(', ')} reported, never replacing it)` : ''}`); + for (const line of formatProfileCoverage(manifest)) console.log(line); + for (const result of [...results].sort((a, b) => a.sliceIndex - b.sliceIndex)) { + for (const outcome of result.outcomes) { + const shown = outcome.reused ? `reused (run ${outcome.reused.runId})` : outcome.trial + ? `trial ${outcome.trial.outcome ?? 'NO RECORD'}` : outcome.status; + console.log(` slice ${result.sliceIndex} ${shown.padEnd(15)} ${String(Math.round(outcome.elapsedMs / 1000)).padStart(5)}s ${outcome.files.join(' ')}`); + } + } + if (options.writeDurations) { + const durations = mergePaidTestDurations(loadPaidTestDurations(rootDir, manifest.tier), results); + writePaidTestDurations(manifest.tier, durations, rootDir); + console.log(`[test:paid] wrote ${Object.keys(durations).length} ${manifest.tier} durations to ${PAID_TEST_DURATIONS_FILE}`); + } + + // Which artifact root (attempt) and which shard (isolated or not) each file belongs to. + const roots = artifacts.map(a => ({ root: path.resolve(a.root), attempt: a.result.attempt ?? 1 })) + .sort((a, b) => b.root.length - a.root.length); + const attemptOf = (abs: string) => roots.find(r => abs === r.root || abs.startsWith(r.root + path.sep))?.attempt ?? primary; + const entryBySlug = new Map(manifest.entries.map(entry => [shardSlug([entry.file]), entry])); + const shardOf = (rel: string) => { + const parts = normalizeRelativePath(rel).split('/'); + const at = parts.lastIndexOf('shards'); + return at >= 0 && parts[at + 1] ? entryBySlug.get(parts[at + 1]!) ?? null : null; + }; + + const flaky: Array<{ name: string; attempts: number; file: string }> = []; + const collectors: Parameters[0] = []; + const files: Array<{ file: string; tier: string; shard: string | number; cost: number; + flaky: number; total: number; executed: number; reused: number; passed: number; + failed: number; manual_accepted: number; attempts: number }> = []; + const manualProblems: string[] = []; + const manualClaims = new Map(); + const recordsByShard = new Map(); + let costUsd = 0; + for (const name of fs.readdirSync(reportDir, { recursive: true }) as string[]) { + const rel = normalizeRelativePath(name); + if (!isFinalizedEvalResultFile(rel) || rel.split('/').includes('receipts') || rel === 'collector-outcomes.json') continue; + if (attemptOf(path.resolve(reportDir, rel)) !== primary) continue; + const shard = shardOf(rel); + try { + const parsed = JSON.parse(fs.readFileSync(path.join(reportDir, rel), 'utf-8')); + if (!Array.isArray(parsed.tests)) { + if (Object.hasOwn(parsed, 'tests') || parsed.total_tests !== undefined || parsed.manual_review !== undefined) { + manualProblems.push(`${rel}: malformed collector tests[]`); + } + continue; + } + costUsd += Number(parsed.total_cost_usd) || 0; + if (shard) recordsByShard.set(shard.file, [...(recordsByShard.get(shard.file) ?? []), ...parsed.tests]); + // Trial records are verdict input for panelVerdict(), never collector gates. + if (shard?.trial) continue; + const seen = new Map(); + for (const [index, entry] of parsed.tests.entries()) { + if (!entry || typeof entry !== 'object' || typeof entry.name !== 'string' || !entry.name + || typeof entry.passed !== 'boolean') { + manualProblems.push(`${rel}: attempt ${index + 1}: malformed collector entry (name/passed required)`); + continue; + } + const key = `${entry.suite ?? ''}\0${entry.name}`; + const occurrence = (seen.get(key) ?? 0) + 1; + seen.set(key, occurrence); + if (Object.hasOwn(entry, 'manual_review') && occurrence !== 1) { + manualProblems.push(`${rel}: attempt ${index + 1}: manual review is only valid on the first case attempt`); + } + if (Object.hasOwn(entry, 'manual_review')) { + const previous = manualClaims.get(key); + if (previous && previous !== rel) manualProblems.push(`${rel}: duplicate manual-review claim for ${entry.name} (also in ${previous})`); + else manualClaims.set(key, rel); + } + const problem = manualReviewProblem(entry, rootDir); + if (problem) manualProblems.push(`${rel}: attempt ${index + 1}: ${problem}`); + } + collectors.push(parsed); + const counts = collectorOutcomeCounts([parsed]); + files.push({ file: rel, tier: parsed.tier ?? 'unknown', shard: parsed.shard ?? '-', + cost: parsed.total_cost_usd ?? 0, flaky: parsed.flaky_retries?.length ?? 0, + total: counts.passed + counts.failed + counts.manual_accepted, ...counts }); + for (const f of parsed.flaky_retries ?? []) flaky.push({ ...f, file: rel }); + } catch (error) { + manualProblems.push(`${rel}: malformed collector JSON (${error instanceof Error ? error.message : String(error)})`); + } + } + const evidence = collectorOutcomeCounts(collectors); + console.log(`[test:paid] collector final outcomes: ${evidence.executed} executed, ${evidence.reused} reused; ${evidence.passed} passed, ${evidence.failed} failed, ${evidence.manual_accepted} manual accepted (unscored; no score-cache credit) (${evidence.attempts} attempt records from ${collectors.length} collectors; every record counts)`); + if (flaky.length > 0) { + console.log(`[test:paid] report: ⚠ ${flaky.length} cases with multiple attempts this run: (paid evals never retry; each record counts)`); + for (const f of flaky) console.log(` ⚠ ${f.name} (x${f.attempts}) — ${f.file}`); + } + const allSkipped = results.flatMap((r) => r.outcomes.filter(isAllSkippedPass)); + if (allSkipped.length > 0) { + console.log(`[test:paid] report: ⚠ ${allSkipped.length} shard(s) passed with EVERY test skipped — they verified nothing:`); + for (const outcome of allSkipped) { + console.log(` ⚠ ${outcome.files.join(' ')} (${outcome.executedTests} skipped — external service missing or tier mismatch)`); + } + } + if (manualProblems.length) verdict.problems.push(...manualProblems); + if (evidence.failed > 0) verdict.problems.push(`${evidence.failed} unapproved final collector failure(s)`); + if (files.reduce((sum, file) => sum + file.total, 0) !== evidence.passed + evidence.failed + evidence.manual_accepted + || files.reduce((sum, file) => sum + file.executed + file.reused, 0) !== evidence.executed + evidence.reused) { + verdict.problems.push('Collector summary totals are inconsistent'); + } + + // Panel verdicts: one function, computed here only. + const panels = panelReports(manifest, results, primary); + for (const panel of panels.filter(p => p.failsLane)) { + verdict.problems.push(`PANEL ${panel.case} ${panel.status}${panel.redClass === 'INFRA' ? ' (INFRA)' : ''} ${panel.passed}/${panel.panel.n} (${panel.marks}): ${panel.reason}`); + } + const laterPanels = attempts.slice(1).flatMap(attempt => panelReports(manifest, artifacts.map(a => a.result), attempt) + .filter(panel => panel.trials.length > 0)); + + // Quarantine policy checks on census runs: the per-tier cap and entry expiry. + if (manifest.evalsAll) { + const tierIds = Object.keys(E2E_TIERS).filter(id => E2E_TIERS[id] === manifest.tier); + const quarantined = Object.keys(CASE_QUARANTINE).filter(id => E2E_TIERS[id] === manifest.tier); + if (quarantined.length > EVAL_POLICY.quarantine.capFraction * tierIds.length) { + verdict.problems.push(`QUARANTINE over cap: ${quarantined.length} of ${tierIds.length} ${manifest.tier} cases (cap ${Math.round(EVAL_POLICY.quarantine.capFraction * 100)}%)`); + } + const expiryMs = EVAL_POLICY.quarantine.expiryWeeklyRuns * 7 * 24 * 60 * 60 * 1000; + for (const id of quarantined) { + const entered = Date.parse(CASE_QUARANTINE[id]!.enteredAt); + if (!Number.isFinite(entered) || Date.now() - entered > expiryMs) { + verdict.problems.push(`QUARANTINE expired: ${id} (entered ${CASE_QUARANTINE[id]!.enteredAt}; entries expire after ${EVAL_POLICY.quarantine.expiryWeeklyRuns} weekly runs)`); + } + } + } + + // History: one trial-outcomes line per isolated trial and per JUnit rule/judge case. + const runId = env.GITHUB_RUN_ID; + const sha = env.GITHUB_SHA; + const history: TrialOutcomeRecord[] = []; + const common = (attempt: number) => ({ schema: TRIAL_OUTCOME_SCHEMA, tier: manifest.tier, attempt, policy_version: EVAL_POLICY.version, + ...(runId ? { run_id: runId } : {}), ...(sha ? { sha } : {}), lane, recorded_at: new Date().toISOString() }); + for (const { result } of artifacts) { + const attempt = result.attempt ?? 1; + for (const outcome of result.outcomes) { + const t = outcome.trial; + if (!t || t.outcome === null) continue; + history.push({ ...common(attempt), case: t.case, file: shardFile(outcome.files[0]!), kind: t.kind, trial: t.trial, panel: t.panel, + outcome: t.outcome, ...(t.outcome === 'failed' ? { failure_class: t.failure_class ?? 'assertion' } : {}), + ...(t.exit_reason ? { exit_reason: t.exit_reason } : {}), ...(t.error ? { error: t.error } : {}), + duration_ms: t.duration_ms, cost_usd: t.cost_usd, ...(t.model ? { model: t.model } : {}), + ...(outcome.reused ? { input_identity: outcome.reused.inputKey } : {}), + quarantined: t.quarantined, execution: outcome.reused ? 'reused' : 'executed', source: 'shard' } as TrialOutcomeRecord); + } + } + const ruleCases: Array<{ id: string; kind: EvalCaseKind; outcome: TrialOutcome; line?: string }> = []; + const junitFailedShards = new Set(); + let unattributed = 0; + const cliVersion = env.GSTACK_CLAUDE_CLI_VERSION; + for (const { root, result } of artifacts.filter(a => (a.result.attempt ?? 1) === primary)) { + for (const outcome of result.outcomes) { + const key = normalizeRelativePath(outcome.files[0] ?? ''); + if (outcome.trial || outcome.files.length !== 1) continue; + let xml = ''; + try { xml = fs.readFileSync(path.join(root, 'shards', shardSlug([key]), 'junit.xml'), 'utf8'); } catch { continue; } + const records = recordsByShard.get(key) ?? []; + for (const tc of parseJUnitCases(xml)) { + const id = caseIdForTestName(tc.name); + if (id === null) { unattributed++; continue; } + const kind = (E2E_KINDS[id] ?? 'rule') as EvalCaseKind; + const mine = records.filter((r: any) => r?.name === id || r?.case_id === id); + const failedRecord = mine.find((r: any) => r.passed === false); + const failureClass: TrialFailureClass | undefined = tc.outcome !== 'failed' ? undefined + : tc.failureType === 'TimeoutError' ? 'timeout' : failedRecord ? failureClassOf(failedRecord) : 'assertion'; + const error = sanitizeTrialError(failedRecord?.error ?? tc.message); + if (tc.outcome === 'failed') junitFailedShards.add(key); + ruleCases.push({ id, kind, outcome: tc.outcome, + ...(tc.outcome === 'failed' ? { line: `✗ ${id} ${kind} FAIL ${failureClass}${failedRecord?.exit_reason === 'timeout' && failedRecord?.timeout_at_turn !== undefined ? ` at turn ${failedRecord.timeout_at_turn}` : ''}${error ? ` — ${error}` : ''} [slice ${result.sliceIndex}, attempt ${primary}] rerun: ${rerunCommand(manifest.tier, id, 1)}` } : {}) }); + history.push({ ...common(primary), case: id, file: shardFile(key), kind, trial: 1, panel: { n: 1, k: 1 }, outcome: tc.outcome, + ...(failureClass ? { failure_class: failureClass } : {}), ...(failedRecord?.exit_reason ? { exit_reason: String(failedRecord.exit_reason) } : {}), + ...(error && tc.outcome === 'failed' ? { error } : {}), duration_ms: tc.timeMs, + cost_usd: Math.round(mine.reduce((sum: number, r: any) => sum + (Number(r.cost_usd) || 0), 0) * 100) / 100, + ...(typeof mine[0]?.model === 'string' ? { model: mine[0].model } : {}), ...(cliVersion ? { cli_version: cliVersion } : {}), + quarantined: false, execution: outcome.reused ? 'reused' : 'executed', source: 'junit' } as TrialOutcomeRecord); + } + } + } + fs.writeFileSync(trialOutcomesPath, formatTrialOutcomes(history)); + + // Headline and failure block (A4): one formatter for the log, the PR comment and the weekly issue. + const ruleShardFailures = manifest.entries.filter(entry => entry.status === 'planned' && !entry.trial).flatMap(entry => { + const got = results.flatMap(r => r.outcomes.map(o => ({ o, slice: r.sliceIndex }))).find(({ o }) => normalizeRelativePath(o.files[0] ?? '') === normalizeRelativePath(entry.file)); + if (got && got.o.status === 'passed') return []; + if (got && junitFailedShards.has(normalizeRelativePath(entry.file))) return []; + const id = shardCaseId(entry.file); + return [`✗ ${entry.file} rule shard ${got ? got.o.status : 'NOT REPORTED'}${got?.o.runnerError ? ` — ${sanitizeTrialError(got.o.runnerError)}` : ''} [slice ${entry.slice}, attempt ${primary}]${id ? ` rerun: ${rerunCommand(manifest.tier, id, 1)}` : ''}`]; + }); + const behaviorPanels = panels.filter(p => !p.quarantined && p.kind === 'behavior'); + const lanePanels = panels.filter(p => !p.quarantined); + const count = (kind: EvalCaseKind) => ({ + passed: ruleCases.filter(c => c.kind === kind && c.outcome === 'passed').length + + lanePanels.filter(p => p.kind === kind && p.status === 'PASS').length, + total: ruleCases.filter(c => c.kind === kind).length + lanePanels.filter(p => p.kind === kind).length, + }); + const primaryTrials = results.flatMap(r => r.outcomes.map(o => o.trial)).filter((t): t is ShardTrialRecord => !!t); + const wall = results.filter(r => Number.isSafeInteger(r.startedAt) && Number.isSafeInteger(r.finishedAt)); + const failureLines = [ + ...ruleShardFailures, + ...ruleCases.filter(c => c.line).map(c => c.line!), + ...panels.filter(p => p.status !== 'PASS' || p.split).map(p => formatPanelLine(p, manifest.tier)), + ]; + const red = verdict.problems.length > 0; + const headline: ReportHeadline = { + lane, verdict: red ? 'RED' : 'GREEN', attempt: primary, + counts: { + rule: count('rule'), + behavior: { ...count('behavior'), split: behaviorPanels.filter(p => p.split).length }, + judge: count('judge'), + quarantined: { total: panels.filter(p => p.quarantined).length, failingLane: panels.filter(p => p.quarantined && p.failsLane).length }, + skipped: allSkipped.length + panels.filter(p => p.status === 'SKIPPED').length + ruleCases.filter(c => c.outcome === 'skipped').length, + infra: primaryTrials.filter(t => t.failure_class === 'infra').length + + results.flatMap(r => r.outcomes).filter(o => !o.trial && o.runnerError !== undefined).length, + incomplete: panels.filter(p => p.status === 'INCOMPLETE').length, + unattributed, + }, + actionRequired: verdict.problems.length, + wallMs: wall.length ? Math.max(...wall.map(r => r.finishedAt!)) - Math.min(...wall.map(r => r.startedAt!)) : null, + costUsd: Math.round(costUsd * 100) / 100, + redispatchEligible: red && infraOnly(verdict.problems), + }; + const headlineLines = formatHeadline(headline); + for (const line of headlineLines) console.log(line); + if (failureLines.length) { + console.log('[test:paid] failures and split verdicts:'); + for (const line of failureLines) console.log(` ${line}`); + } + for (const panel of laterPanels) console.log(` attempt ${panel.attempt} (re-run; reported, never replacing attempt ${primary}): ${formatPanelLine(panel, manifest.tier)}`); + const fence = (lines: string[]) => ['```', ...lines.map(line => line.replace(/```/g, "'''")), '```']; + fs.writeFileSync(summaryMdPath, [ + ...fence(headlineLines), + ...(failureLines.length ? ['', '**Failures and split verdicts**', '', ...fence(failureLines)] : []), + ...(verdict.problems.length ? ['', `**ACTION REQUIRED (${verdict.problems.length})**`, '', ...fence(verdict.problems.map(p => sanitizeTrialError(p) ?? p))] : []), + ].join('\n') + '\n'); + if (!manualProblems.length) fs.writeFileSync(summaryPath, JSON.stringify({ version: 2, files, totals: { + ...evidence, total: evidence.passed + evidence.failed + evidence.manual_accepted, + flaky: files.reduce((sum, file) => sum + file.flaky, 0), + }, verdict: headline, headline: headlineLines, + panels: panels.map(p => ({ case: p.case, kind: p.kind, status: p.status, passed: p.passed, n: p.panel.n, k: p.panel.k, + marks: p.marks, split: p.split, quarantined: p.quarantined, failsLane: p.failsLane, redClass: p.redClass, reason: p.reason, + trials: p.trials.map(t => ({ trial: t.trial, outcome: t.outcome, ...(t.failure_class ? { failure_class: t.failure_class } : {}), + ...(t.exit_reason ? { exit_reason: t.exit_reason } : {}), ...(t.error ? { error: t.error } : {}) })) })), + failures: failureLines.map(line => sanitizeTrialError(line) ?? line) }, null, 2) + '\n'); + if (verdict.problems.length) { + console.error(`[test:paid] report: ${verdict.problems.length} problem(s):`); + for (const problem of verdict.problems) console.error(` ✗ ${problem}`); + if (headline.redispatchEligible) console.error('[test:paid] report: INFRA-ONLY RED — one re-dispatch as a new run is allowed; report both runs'); + return 1; + } + console.log(evidence.manual_accepted + ? `[test:paid] report: every planned shard accounted; ${evidence.manual_accepted} manual acceptance(s), no automated-score credit` + : '[test:paid] report: every planned shard accounted and passed'); + return 0; +} + type CliOptions = { tier: PaidTier; profile: PaidProfile; @@ -1838,6 +2670,8 @@ type CliOptions = { reportDir: string | null; /** Report mode: merge executed shard wall times into the duration seed. */ writeDurations: boolean; + /** Planner: the workflow matrix cap, for the capacity preflight's wave count. */ + maxParallel: number | null; }; function parsePositiveInt(value: string | undefined, flag: string): number { @@ -1892,6 +2726,7 @@ export function parseCliOptions(argv: string[], env: NodeJS.ProcessEnv = process sliceIndex: null, reportDir: null, writeDurations: false, + maxParallel: null, }; for (let index = 0; index < argv.length; index += 1) { @@ -1929,6 +2764,7 @@ export function parseCliOptions(argv: string[], env: NodeJS.ProcessEnv = process options.reportDir = value; continue; } if (arg === '--write-durations') { options.writeDurations = true; continue; } + if (arg === '--max-parallel') { options.maxParallel = parsePositiveInt(argv[index += 1], '--max-parallel'); continue; } throw new Error(`Unknown argument: ${arg}`); } if (options.writeDurations && !options.reportDir) throw new Error('--write-durations requires --report'); @@ -1965,125 +2801,13 @@ async function main(): Promise { + `${excludedCount} excluded (${manifest.selectionReason})`, ); for (const line of formatSlicePlan(manifest)) console.log(line); + for (const line of formatCapacityPreflight(manifest, options.maxParallel ?? undefined)) console.log(line); return 0; } // ── Report mode: reconcile slice artifacts against the manifest. Fail-closed: // a slice whose artifact never landed is a FAILURE, not an absence. - const reportDir = options.reportDir; - if (reportDir) { - const summaryPath = path.join(reportDir, 'collector-outcomes.json'); - fs.rmSync(summaryPath, { force: true }); - const manifest = parseRunManifest(fs.readFileSync(path.join(reportDir, 'manifest.json'), 'utf-8')); - const results: SliceResult[] = fs.readdirSync(reportDir) - .filter((name) => /^slice-\d+\.json$/.test(name)) - .map((name) => JSON.parse(fs.readFileSync(path.join(reportDir, name), 'utf-8')) as SliceResult); - const verdict = verifySliceResults(manifest, results); - const planned = manifest.entries.filter((e) => e.status === 'planned').length; - console.log(`[test:paid] report: ${results.length}/${manifest.sliceCount} slices, ${planned} planned shards, tier=${manifest.tier}`); - for (const line of formatProfileCoverage(manifest)) console.log(line); - for (const result of results.sort((a, b) => a.sliceIndex - b.sliceIndex)) { - for (const outcome of result.outcomes) { - const shown = outcome.reused ? `reused (run ${outcome.reused.runId})` : outcome.status; - console.log(` slice ${result.sliceIndex} ${shown.padEnd(15)} ${String(Math.round(outcome.elapsedMs / 1000)).padStart(5)}s ${outcome.files.join(' ')}`); - } - } - if (options.writeDurations) { - const durations = mergePaidTestDurations(loadPaidTestDurations(ROOT, manifest.tier), results); - writePaidTestDurations(manifest.tier, durations); - console.log(`[test:paid] wrote ${Object.keys(durations).length} ${manifest.tier} durations to ${PAID_TEST_DURATIONS_FILE}`); - } - // Historical flaky_retries includes every case with multiple attempts, - // whether its final result passed or failed. Report attempts separately - // from the shard verdict; reconciliation above still controls gating. - // Source: the finalized eval-store JSONs inside the slice artifacts. - const flaky: Array<{ name: string; attempts: number; file: string }> = []; - const collectors: Parameters[0] = []; - const files: Array<{ file: string; tier: string; shard: string | number; cost: number; - flaky: number; total: number; executed: number; reused: number; passed: number; - failed: number; manual_accepted: number; attempts: number }> = []; - const manualProblems: string[] = []; - const manualClaims = new Map(); - for (const name of fs.readdirSync(reportDir, { recursive: true }) as string[]) { - if (!isFinalizedEvalResultFile(name)) continue; - try { - const parsed = JSON.parse(fs.readFileSync(path.join(reportDir, name), 'utf-8')); - if (!Array.isArray(parsed.tests)) { - if (Object.hasOwn(parsed, 'tests') || parsed.total_tests !== undefined || parsed.manual_review !== undefined) { - manualProblems.push(`${name}: malformed collector tests[]`); - } - continue; - } - const seen = new Map(); - for (const [index, entry] of parsed.tests.entries()) { - if (!entry || typeof entry !== 'object' || typeof entry.name !== 'string' || !entry.name - || typeof entry.passed !== 'boolean') { - manualProblems.push(`${name}: attempt ${index + 1}: malformed collector entry (name/passed required)`); - continue; - } - const key = `${entry.suite ?? ''}\0${entry.name}`; - const occurrence = (seen.get(key) ?? 0) + 1; - seen.set(key, occurrence); - if (Object.hasOwn(entry, 'manual_review') && occurrence !== 1) { - manualProblems.push(`${name}: attempt ${index + 1}: manual review is only valid on the first case attempt`); - } - if (Object.hasOwn(entry, 'manual_review')) { - const previous = manualClaims.get(key); - if (previous && previous !== name) manualProblems.push(`${name}: duplicate manual-review claim for ${entry.name} (also in ${previous})`); - else manualClaims.set(key, name); - } - const problem = manualReviewProblem(entry, ROOT); - if (problem) manualProblems.push(`${name}: attempt ${index + 1}: ${problem}`); - } - collectors.push(parsed); - const counts = collectorOutcomeCounts([parsed]); - files.push({ file: name, tier: parsed.tier ?? 'unknown', shard: parsed.shard ?? '-', - cost: parsed.total_cost_usd ?? 0, flaky: parsed.flaky_retries?.length ?? 0, - total: counts.passed + counts.failed + counts.manual_accepted, ...counts }); - for (const f of parsed.flaky_retries ?? []) flaky.push({ ...f, file: name }); - } catch (error) { - manualProblems.push(`${name}: malformed collector JSON (${error instanceof Error ? error.message : String(error)})`); - } - } - const evidence = collectorOutcomeCounts(collectors); - console.log(`[test:paid] collector final outcomes: ${evidence.executed} executed, ${evidence.reused} reused; ${evidence.passed} passed, ${evidence.failed} failed, ${evidence.manual_accepted} manual accepted (unscored; no score-cache credit) (${evidence.attempts} attempt records from ${collectors.length} collectors)`); - if (flaky.length > 0) { - console.log(`[test:paid] report: ⚠ ${flaky.length} cases with multiple attempts this run:`); - for (const f of flaky) console.log(` ⚠ ${f.name} (x${f.attempts}) — ${f.file}`); - } - - // Census honesty: a 'passed' shard whose every test skipped verified - // nothing (external-service binary absent on the runner). Not a failure — - // service availability is host state, not a repo regression — but the - // report must say so, or the weekly lane reads codex/gemini as covered - // on runners that never install them. - const allSkipped = results.flatMap((r) => r.outcomes.filter(isAllSkippedPass)); - if (allSkipped.length > 0) { - console.log(`[test:paid] report: ⚠ ${allSkipped.length} shard(s) passed with EVERY test skipped — they verified nothing:`); - for (const outcome of allSkipped) { - console.log(` ⚠ ${outcome.files.join(' ')} (${outcome.executedTests} skipped — external service missing or tier mismatch)`); - } - } - if (manualProblems.length) verdict.problems.push(...manualProblems); - if (evidence.failed > 0) verdict.problems.push(`${evidence.failed} unapproved final collector failure(s)`); - if (files.reduce((sum, file) => sum + file.total, 0) !== evidence.passed + evidence.failed + evidence.manual_accepted - || files.reduce((sum, file) => sum + file.executed + file.reused, 0) !== evidence.executed + evidence.reused) { - verdict.problems.push('Collector summary totals are inconsistent'); - } - if (!manualProblems.length) fs.writeFileSync(summaryPath, JSON.stringify({ version: 1, files, totals: { - ...evidence, total: evidence.passed + evidence.failed + evidence.manual_accepted, - flaky: files.reduce((sum, file) => sum + file.flaky, 0), - } }, null, 2) + '\n'); - if (verdict.problems.length) { - console.error(`[test:paid] report: ${verdict.problems.length} problem(s):`); - for (const problem of verdict.problems) console.error(` ✗ ${problem}`); - return 1; - } - console.log(evidence.manual_accepted - ? `[test:paid] report: every planned shard accounted; ${evidence.manual_accepted} manual acceptance(s), no automated-score credit` - : '[test:paid] report: every planned shard accounted and passed'); - return 0; - } + if (options.reportDir) return runPaidReport(options.reportDir, { writeDurations: options.writeDurations }); const discovered = collectPaidTestFiles(); if (discovered.length === 0) throw new Error('No paid test files were discovered.'); @@ -2119,28 +2843,36 @@ async function main(): Promise { } const evalDirBase = process.env.GSTACK_EVAL_DIR || getProjectEvalDir(); + const trials = Object.fromEntries(mine.filter(entry => entry.trial).map(entry => [normalizeRelativePath(entry.file), entry.trial!])); + const exclusionPatterns = Object.fromEntries(mine.filter(entry => entry.excludeCases).map(entry => [entry.file, + manifest.prCoverage?.mode === 'pr' ? prProfileTestNamePattern(entry.file, manifest.selection!, entry.excludeCases) + : excludedCasesNamePattern(entry.excludeCases!)])); + const startedAt = Date.now(); let summary: RunSummary; if (shards.length === 0) { summary = summarize([]); } else { preflightAnthropicApi(process.env); summary = await runPaidShards(shards, { + trials, + casePatterns: exclusionPatterns, timeoutMs: options.timeoutExplicit ? options.timeoutMs : undefined, jobs: options.jobs, withinShardConcurrency: options.withinShardConcurrency, registeredBudgets: Object.fromEntries(mine.filter(entry => entry.budget).map(entry => [normalizeRelativePath(entry.file), entry.budget!])), ...(manifest.prCoverage?.mode === 'pr' ? { - expectedCases: Object.fromEntries(mine.map(entry => [entry.file, expectedPrCaseCount(entry.file, manifest.selection!)])), - casePatterns: Object.fromEntries(mine.map(entry => [entry.file, prProfileTestNamePattern(entry.file, manifest.selection!)])), - expectedCaseIds: Object.fromEntries(mine.map(entry => [entry.file, prProfileShardIds(entry.file, manifest.selection!)])), + expectedCases: Object.fromEntries(mine.map(entry => [entry.file, expectedPrCaseCount(entry.file, manifest.selection!, entry.excludeCases)])), + casePatterns: Object.fromEntries(mine.map(entry => [entry.file, prProfileTestNamePattern(entry.file, manifest.selection!, entry.excludeCases)])), + expectedCaseIds: Object.fromEntries(mine.map(entry => [entry.file, prProfileShardIds(entry.file, manifest.selection!, entry.excludeCases)])), reuseFor: e2eReuseLaneProblem(process.env, manifest.prCoverage.mode) !== null ? undefined : (files, env, budget) => { const key = files[0]!; const file = shardFile(key); if (files.length !== 1 || !/^test\/skill-e2e-/.test(file)) return null; const { registered, known } = fileCaseRegistration(file, fs.readFileSync(path.join(ROOT, file), 'utf8')); - return prepareE2EShardReuse({ root: ROOT, key, file, caseIds: prProfileShardIds(key, manifest.selection!), + const exclude = mine.find(entry => entry.file === key)?.excludeCases; + return prepareE2EShardReuse({ root: ROOT, key, file, caseIds: prProfileShardIds(key, manifest.selection!, exclude), registeredIds: registered, registrationKnown: known, - casePattern: prProfileTestNamePattern(key, manifest.selection!), expectedCases: expectedPrCaseCount(key, manifest.selection!), + casePattern: prProfileTestNamePattern(key, manifest.selection!, exclude), expectedCases: expectedPrCaseCount(key, manifest.selection!, exclude), retries: retriesForFiles(files), timeoutMs: budget.timeoutMs, withinShardConcurrency: options.withinShardConcurrency, tier: manifest.tier, profile, env }); }, @@ -2161,8 +2893,9 @@ async function main(): Promise { evalDirBase, }); } - const guarded = applyHollowShardGuard(summary.outcomes, { evalsAll: manifest.evalsAll, requireExecuted: manifest.prCoverage?.mode === 'pr' }); + const guarded = guardTrialRecords(applyHollowShardGuard(summary.outcomes, { evalsAll: manifest.evalsAll, requireExecuted: manifest.prCoverage?.mode === 'pr' })); summary = summarize(guarded); + const attempt = Number(process.env.GITHUB_RUN_ATTEMPT); const sliceResult: SliceResult = { version: 1, tier: manifest.tier, @@ -2171,15 +2904,23 @@ async function main(): Promise { sliceIndex: options.sliceIndex, sliceCount: manifest.sliceCount, ...(options.timeoutExplicit ? { timeoutOverrideMs: options.timeoutMs } : {}), - outcomes: guarded.map(({ files, status, exitCode, elapsedMs, executedTests, skippedTests, budget, reused }) => - ({ files, status, exitCode, elapsedMs, executedTests, skippedTests, ...(budget ? { budget } : {}), ...(reused ? { reused } : {}) })), + attempt: Number.isSafeInteger(attempt) && attempt > 0 ? attempt : 1, + startedAt, + finishedAt: Date.now(), + outcomes: guarded.map(({ files, status, exitCode, elapsedMs, executedTests, skippedTests, budget, reused, runnerError, trial }) => + ({ files, status, exitCode, elapsedMs, executedTests, skippedTests, ...(budget ? { budget } : {}), ...(reused ? { reused } : {}), + ...(runnerError !== undefined ? { runnerError } : {}), ...(trial ? { trial } : {}) })), }; fs.mkdirSync(evalDirBase, { recursive: true }); const sliceResultPath = path.join(evalDirBase, `slice-${options.sliceIndex}.json`); fs.writeFileSync(sliceResultPath, `${JSON.stringify(sliceResult, null, 2)}\n`); console.log(`[test:paid] slice result: ${sliceResultPath}`); for (const line of formatSummary(summary)) console.log(line); - return summaryExitCode(summary); + for (const outcome of guarded.filter(outcome => outcome.trial)) { + const t = outcome.trial!; + console.log(` trial ${t.case} t${t.trial}/${t.panel.n}: ${t.outcome ?? `NO RECORD (${t.harness})`}${t.failure_class ? ` [${t.failure_class}]` : ''}`); + } + return sliceExitCode(guarded); } if (options.listOnly && options.sliceBudgetMs !== null) { diff --git a/test/ci-paid-coordination.test.ts b/test/ci-paid-coordination.test.ts index 18c80adfc..7405037e9 100644 --- a/test/ci-paid-coordination.test.ts +++ b/test/ci-paid-coordination.test.ts @@ -248,7 +248,8 @@ describe('dependency-free CI planner and report execution', () => { const red = run(['--report', reportDir], tier); expect(red.status).toBe(1); expect(red.stderr).toContain(`${failed.outcomes[0].files[0]}: failed`); - expect(red.stdout).toContain('3 executed, 0 reused; 1 passed, 2 failed, 0 manual accepted (unscored; no score-cache credit) (6 attempt records from 1 collectors)'); + // Paid evals never retry: every record counts, a later pass never hides an earlier failure. + expect(red.stdout).toContain('6 executed, 0 reused; 2 passed, 4 failed, 0 manual accepted (unscored; no score-cache credit) (6 attempt records from 1 collectors'); expect(red.stdout).toContain('3 cases with multiple attempts this run:'); expect(red.stdout).not.toMatch(/passed only on retry|not blocking/); diff --git a/test/paid-pr-profile.test.ts b/test/paid-pr-profile.test.ts index 34609e54f..8e46944df 100644 --- a/test/paid-pr-profile.test.ts +++ b/test/paid-pr-profile.test.ts @@ -241,7 +241,7 @@ describe('PR profile paid-runner integration', () => { expect(guarded[0].status).toBe('passed-empty'); }); - test('report distinguishes retained/deferred coverage and final executed/reused outcomes from attempts', () => { + test('report distinguishes retained/deferred coverage and counts every executed/reused record', () => { const manifest = ceoManifest(); const lines = formatProfileCoverage(manifest).join('\n'); expect(lines).toContain('profile=pr mode=pr'); @@ -252,6 +252,7 @@ describe('PR profile paid-runner integration', () => { { name: 'retry', suite: 'judge', passed: true, execution: 'executed' }, { name: 'cached', suite: 'judge', passed: true, execution: 'reused' }, { name: 'failed', suite: 'native', passed: false }, - ] }])).toEqual({ executed: 2, reused: 1, passed: 2, failed: 1, manual_accepted: 0, attempts: 4 }); + // Paid evals never retry: a later pass never replaces an earlier failed record. + ] }])).toEqual({ executed: 3, reused: 1, passed: 2, failed: 2, manual_accepted: 0, attempts: 4 }); }); }); diff --git a/test/paid-report-fail-open.test.ts b/test/paid-report-fail-open.test.ts index cc734f998..a286c737c 100644 --- a/test/paid-report-fail-open.test.ts +++ b/test/paid-report-fail-open.test.ts @@ -108,3 +108,110 @@ describe('rule shards stay fail-closed through --report', () => { expect(r.out).toContain(`${RULE_B} planned for slice 2 but reported by slice 1`); }); }); + +describe('behavior and quarantined panels through --report', () => { + const FILE = 'test/skill-e2e-review.test.ts'; + const ID = 'review-design-lite'; + const key = (trial: number) => `${FILE}#${ID}~t${trial}`; + const plan = (quarantined = false) => manifest([ + ...[1, 2, 3].map(trial => ({ file: key(trial), slice: trial, status: 'planned' as const, + trial: { kind: 'behavior' as const, panel: { n: 3, k: 2 }, quarantined } })), + { file: RULE_A, slice: 4, status: 'planned' }, + ], 4); + type TrialResult = 'passed' | 'failed' | 'contract' | 'missing' | 'harness'; + const trialOutcome = (trial: number, result: TrialResult, quarantined = false): Outcome | null => { + if (result === 'missing') return null; + const record = { case: ID, trial, kind: 'behavior' as const, panel: { n: 3, k: 2 }, quarantined, cost_usd: 0, duration_ms: 1_000 }; + if (result === 'harness') return passed(key(trial), { status: 'never-started', exitCode: null, executedTests: null, skippedTests: null, + trial: { ...record, outcome: null, harness: 'never started' } }); + if (result === 'passed') return passed(key(trial), { trial: { ...record, outcome: 'passed' } }); + return passed(key(trial), { status: 'failed', exitCode: 1, + trial: { ...record, outcome: 'failed', failure_class: result === 'contract' ? 'contract' : 'timeout', + exit_reason: 'timeout', timeout_at_turn: 14, error: result === 'contract' ? 'handoff missing' : 'no posture match' } }); + }; + const run = (results: TrialResult[], quarantined = false, dropSlice?: number) => report(plan(quarantined), [1, 2, 3, 4] + .filter(index => index !== dropSlice) + .map(index => slice(index, 4, index === 4 ? [passed(RULE_A)] + : [trialOutcome(index, results[index - 1]!, quarantined)].filter((o): o is Outcome => o !== null)))); + + test('behavior 3/3: green', () => { + const r = run(['passed', 'passed', 'passed']); + expect(r.status, r.out).toBe(0); + expect(r.out).toContain('VERDICT GREEN'); + }); + + test('behavior 2/3: green, the failed trial shown with its cause', () => { + const r = run(['passed', 'failed', 'passed']); + expect(r.status, r.out).toBe(0); + expect(r.out).toContain(`⚠ ${ID} behavior PASS 2/3 (✓✗✓)`); + expect(r.out).toContain('t2: timeout at turn 14'); + const summary = JSON.parse(fs.readFileSync(path.join(r.dir, 'collector-outcomes.json'), 'utf8')); + expect(summary.version).toBe(2); + expect(summary.panels[0]).toMatchObject({ case: ID, status: 'PASS', split: true, failsLane: false }); + const history = fs.readFileSync(path.join(r.dir, 'trial-outcomes.jsonl'), 'utf8').trim().split('\n').map(line => JSON.parse(line)); + expect(history.map(h => [h.trial, h.outcome])).toEqual([[1, 'passed'], [2, 'failed'], [3, 'passed']]); + }); + + test('behavior 1/3: red', () => { + const r = run(['passed', 'failed', 'failed']); + expect(r.status).toBe(1); + expect(r.out).toContain(`PANEL ${ID} FAIL 1/3`); + }); + + test('a missing trial record: INCOMPLETE, red', () => { + const r = run(['passed', 'missing', 'passed']); + expect(r.status).toBe(1); + expect(r.out).toContain(`PANEL ${ID} INCOMPLETE`); + }); + + test('a trial the harness never started: red, machine-classified for one re-dispatch', () => { + const r = run(['passed', 'harness', 'passed']); + expect(r.status).toBe(1); + expect(r.out).toContain('no trial record (never started)'); + expect(r.out).toContain('INFRA-ONLY RED'); + }); + + test('a contract trial at 2/3: red', () => { + const r = run(['passed', 'passed', 'contract']); + expect(r.status).toBe(1); + expect(r.out).toContain(`PANEL ${ID} FAIL 2/3`); + expect(r.out).toContain('contract violation'); + expect(r.out).not.toContain('INFRA-ONLY RED'); + }); + + test('quarantined 1/3: reported, does not fail the lane', () => { + const r = run(['passed', 'failed', 'failed'], true); + expect(r.status, r.out).toBe(0); + expect(r.out).toContain(`◌ ${ID} behavior (quarantined) FAIL 1/3`); + }); + + test('quarantined 0/3: hard break, red', () => { + const r = run(['failed', 'failed', 'failed'], true); + expect(r.status).toBe(1); + expect(r.out).toContain('quarantined hard break'); + }); + + test('quarantined contract violation: red', () => { + const r = run(['passed', 'passed', 'contract'], true); + expect(r.status).toBe(1); + }); + + test('a missing trial slice: red', () => { + const r = run(['passed', 'passed', 'passed'], false, 2); + expect(r.status).toBe(1); + expect(r.out).toContain('slice 2/4 reported NO result'); + expect(r.out).toContain(`PANEL ${ID} INCOMPLETE`); + }); + + test('a later run attempt never replaces the first attempt verdict', () => { + const r = run(['passed', 'failed', 'failed']); + expect(r.status).toBe(1); + const retry = slice(3, 4, [trialOutcome(3, 'passed')!]); + fs.mkdirSync(path.join(r.dir, 'paid-slice-3-a2'), { recursive: true }); + fs.writeFileSync(path.join(r.dir, 'paid-slice-3-a2', 'slice-3.json'), JSON.stringify({ ...retry, attempt: 2 })); + const again = spawnSync(process.execPath, [path.join(ROOT, 'scripts/test-paid-shards.ts'), '--tier', 'periodic', '--report', r.dir], + { cwd: ROOT, encoding: 'utf8', timeout: 30_000 }); + expect(again.status).toBe(1); + expect(again.stdout).toContain('attempt 1 (later attempts 2 reported, never replacing it)'); + }); +}); diff --git a/test/paid-run-manifest.test.ts b/test/paid-run-manifest.test.ts index b91da2547..74090aace 100644 --- a/test/paid-run-manifest.test.ts +++ b/test/paid-run-manifest.test.ts @@ -37,11 +37,16 @@ import { summarize, summaryExitCode, verifySliceResults, + expandTrialShards, + formatCapacityPreflight, + shardSlug, type PaidRunManifest, type ShardOutcome, type SliceResult, } from '../scripts/test-paid-shards'; +import { E2E_KINDS } from './helpers/touchfiles-data'; + const ROOT = path.resolve(__dirname, '..'); const outcome = (over: Partial): ShardOutcome => ({ @@ -508,3 +513,98 @@ describe('retry parity', () => { expect(buildPaidShardArgs(['x'], 1000, 4).join(' ')).toContain('--retry 0'); }); }); + +describe('trial planner (behavior and quarantined panels)', () => { + const REVIEW = 'test/skill-e2e-review.test.ts'; + const budgetPlan = (tier: 'gate' | 'periodic', kinds: Record, quarantine: Record = {}) => + buildRunManifest({ tier, sliceBudgetMs: 540_000, jobs: 2, evalsAll: true, env: { EVALS_ALL: '1' }, + kinds: { ...E2E_KINDS, ...kinds }, quarantine }); + + test('a behavior case becomes three trial shards on three different slices; its file shard runs the rest', () => { + const manifest = budgetPlan('gate', { 'review-sql-injection': 'behavior' }); + const trials = manifest.entries.filter(entry => entry.file.startsWith(`${REVIEW}#review-sql-injection~t`)); + expect(trials.map(entry => entry.file)).toEqual([1, 2, 3].map(n => `${REVIEW}#review-sql-injection~t${n}`)); + expect(trials.every(entry => entry.status === 'planned')).toBe(true); + expect(new Set(trials.map(entry => entry.slice)).size).toBe(3); + expect(trials[0]!.trial).toEqual({ kind: 'behavior', panel: { n: 3, k: 2 }, quarantined: false }); + const fileShard = manifest.entries.find(entry => entry.file === REVIEW)!; + expect(fileShard.excludeCases).toEqual(['review-sql-injection']); + const slugs = manifest.entries.map(entry => shardSlug([entry.file])); + expect(new Set(slugs).size).toBe(slugs.length); + expect(parseRunManifest(JSON.stringify(manifest))).toEqual(manifest); + }); + + test('a file whose only tier case is isolated drops its file shard', () => { + const manifest = budgetPlan('periodic', { 'review-design-lite': 'behavior' }); + expect(manifest.entries.some(entry => entry.file === REVIEW)).toBe(false); + expect(manifest.entries.filter(entry => entry.file.startsWith(`${REVIEW}#`)).map(entry => entry.file)) + .toEqual([1, 2, 3].map(n => `${REVIEW}#review-design-lite~t${n}`)); + }); + + test('a quarantined rule case runs a full panel with k = n', () => { + const manifest = budgetPlan('gate', {}, { 'review-enum-completeness': { reason: 'r' } }); + const trial = manifest.entries.find(entry => entry.file === `${REVIEW}#review-enum-completeness~t1`)!; + expect(trial.trial).toEqual({ kind: 'rule', panel: { n: 3, k: 3 }, quarantined: true }); + }); + + test('slice-count plans keep trials on different slices too', () => { + const manifest = buildRunManifest({ tier: 'gate', sliceCount: 5, evalsAll: true, env: { EVALS_ALL: '1' }, + kinds: { ...E2E_KINDS, 'review-sql-injection': 'behavior', 'review-enum-completeness': 'behavior' } }); + for (const id of ['review-sql-injection', 'review-enum-completeness']) { + const slices = manifest.entries.filter(entry => entry.file.startsWith(`${REVIEW}#${id}~t`)).map(entry => entry.slice); + expect(new Set(slices).size).toBe(3); + } + }); + + test('judges and unknown ids cannot be isolated; unknown registrations throw', () => { + expect(() => budgetPlan('gate', { 'review/SKILL.md workflow': 'behavior' })).toThrow(/Only live E2E cases/); + const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'trial-unknown-')); + try { + fs.mkdirSync(path.join(dir, 'test')); + fs.writeFileSync(path.join(dir, 'test/skill-e2e-x.test.ts'), 'const name = pick(); runSkillTest({ testName: name });'); + expect(() => expandTrialShards(['test/skill-e2e-x.test.ts'], 'gate', dir, { + kinds: { x: 'behavior' }, touchfiles: { x: ['test/skill-e2e-x.test.ts'] }, tiers: { x: 'gate' }, + })).toThrow(/statically known case registration/); + } finally { fs.rmSync(dir, { recursive: true, force: true }); } + }); + + test('parse rejects partial panels, shared runners, forged plans and stray exclusions', () => { + const manifest = budgetPlan('gate', { 'review-sql-injection': 'behavior' }); + const trialFiles = [1, 2, 3].map(n => `${REVIEW}#review-sql-injection~t${n}`); + const mutate = (fn: (m: PaidRunManifest) => void) => { const m = structuredClone(manifest); fn(m); return JSON.stringify(m); }; + expect(() => parseRunManifest(mutate(m => { m.entries = m.entries.filter(e => e.file !== trialFiles[1]); }))) + .toThrow(/exactly its 3 trials/); + expect(() => parseRunManifest(mutate(m => { + const [a, b] = trialFiles.map(f => m.entries.find(e => e.file === f)!); + b!.slice = a!.slice; + }))).toThrow(/share a slice/); + expect(() => parseRunManifest(mutate(m => { m.entries.find(e => e.file === trialFiles[0])!.trial!.panel.k = 1; }))) + .toThrow(/fixed policy panel/); + expect(() => parseRunManifest(mutate(m => { m.entries.find(e => e.file === trialFiles[0])!.trial = undefined; }))) + .toThrow(/fixed policy panel/); + expect(() => parseRunManifest(mutate(m => { m.entries.find(e => e.file === REVIEW)!.excludeCases = ['review-design-lite']; }))) + .toThrow(/exclude only cases/); + expect(() => parseRunManifest(mutate(m => { m.entries.find(e => e.file === REVIEW)!.trial = m.entries.find(e => e.file === trialFiles[0])!.trial; }))) + .toThrow(/Only trial shards/); + }); + + test('capacity preflight names slices, shards, waves and the longest indivisible trial', () => { + const manifest = budgetPlan('gate', { 'review-sql-injection': 'behavior' }); + const lines = formatCapacityPreflight(manifest, 16).join('\n'); + expect(lines).toContain(`${manifest.sliceCount} slice(s)`); + expect(lines).toContain('3 trial shard(s)'); + expect(lines).toContain(`wave(s) at max-parallel 16: ${Math.ceil(manifest.sliceCount / 16)}`); + expect(lines).toMatch(/longest indivisible trial ~\d+\.\dm \(test\/skill-e2e-review\.test\.ts#review-sql-injection~t\d\)/); + }); + + test('durations: trials record their longest wall under the case key and seed their own estimate', () => { + const key = `${REVIEW}#review-sql-injection`; + const merged = mergePaidTestDurations({}, [{ version: 1, tier: 'gate', sliceIndex: 1, sliceCount: 1, outcomes: [1, 2, 3].map(n => ({ + files: [`${key}~t${n}`], status: 'passed' as const, exitCode: 0, elapsedMs: n * 60_000, executedTests: 1, skippedTests: 0, + })) }]); + expect(merged).toEqual({ [key]: 180_000 }); + const packed = packBySliceBudget([1, 2, 3].map(n => `${key}~t${n}`), 540_000, 2, merged); + expect(packed.slices).toHaveLength(3); + expect(Object.values(packed.estimates)).toEqual([180_000, 180_000, 180_000]); + }); +}); diff --git a/test/paid-shards.test.ts b/test/paid-shards.test.ts index e40239457..5268eef0d 100644 --- a/test/paid-shards.test.ts +++ b/test/paid-shards.test.ts @@ -47,6 +47,14 @@ import { selectPaidTestFiles, buildRunManifest, parseRunManifest, + classifyTrialShard, + sliceExitCode, + guardTrialRecords, + parseJUnitCases, + caseIdForTestName, + shardTrial, + excludedCasesNamePattern, + type CaseTrialPlan, type ShardOutcome, } from '../scripts/test-paid-shards'; @@ -578,3 +586,92 @@ describe('all-skipped pass census', () => { expect(reviewLine).not.toContain('SKIPPED'); }); }); + +describe('isolated trial shards: record, classification and slice exit', () => { + const plan: CaseTrialPlan = { kind: 'behavior', panel: { n: 3, k: 2 }, quarantined: false }; + const key = (n: number) => `test/skill-e2e-review.test.ts#review-sql-injection~t${n}`; + const base = { status: 'passed' as const, executedTests: 1, skippedTests: 0, elapsedMs: 5 }; + const none = { records: [], contract: null }; + + test('trial keys keep their case id, file and index', () => { + expect(shardCaseId(key(2))).toBe('review-sql-injection'); + expect(shardFile(key(2))).toBe('test/skill-e2e-review.test.ts'); + expect(shardTrial(key(2))).toBe(2); + expect(shardTrial('test/skill-e2e-review.test.ts#review-sql-injection')).toBeNull(); + expect(shardSlug([key(2)])).toBe('skill-e2e-review--review-sql-injection.t2'); + }); + + test('classification: verdicts versus harness problems', () => { + const c = (over: Partial, evidence: { records: any[]; contract: string | null } = none) => + classifyTrialShard({ ...base, ...over }, 'review-sql-injection', 1, plan, evidence); + expect(c({}).outcome).toBe('passed'); + expect(c({}, { records: [], contract: 'handoff missing' })).toMatchObject({ outcome: 'failed', failure_class: 'contract', error: 'handoff missing' }); + expect(c({ status: 'failed' }, { records: [{ passed: false, exit_reason: 'timeout', timeout_at_turn: 9, error: 'x' }], contract: null })) + .toMatchObject({ outcome: 'failed', failure_class: 'timeout', timeout_at_turn: 9 }); + expect(c({ status: 'failed' }, { records: [{ passed: false, error: 'expected 3' }], contract: null })).toMatchObject({ outcome: 'failed', failure_class: 'assertion' }); + expect(c({ status: 'timed-out', executedTests: null, skippedTests: null })).toMatchObject({ outcome: 'failed', failure_class: 'timeout' }); + expect(c({ status: 'failed', executedTests: null, skippedTests: null })).toMatchObject({ outcome: 'failed', failure_class: 'infra' }); + expect(c({ status: 'failed', executedTests: 0, skippedTests: 0 })).toMatchObject({ outcome: 'failed', failure_class: 'infra' }); + expect(c({ executedTests: 1, skippedTests: 1 })).toMatchObject({ outcome: 'skipped' }); + for (const over of [{ status: 'never-started' as const }, { status: 'passed-empty' as const }, { executedTests: 2 }, + { runnerError: 'spawn failed' }, { executedTests: 0, skippedTests: 0 }]) { + expect(c(over).outcome, JSON.stringify(over)).toBeNull(); + } + }); + + test('slice exit: rule shards stay strict; failed trials never red the runner, missing records do', () => { + const trial = (outcome: 'passed' | 'failed' | null) => ({ status: outcome === 'failed' ? 'failed' as const : 'passed' as const, + trial: { case: 'c', trial: 1, ...plan, outcome, cost_usd: 0, duration_ms: 1, ...(outcome === null ? { harness: 'never started' } : {}) } }); + expect(sliceExitCode([{ status: 'passed' }, trial('failed')])).toBe(0); + expect(sliceExitCode([{ status: 'failed' }, trial('passed')])).toBe(1); + expect(sliceExitCode([{ status: 'passed' }, trial(null)])).toBe(1); + expect(sliceExitCode([{ status: 'timed-out' }])).toBe(1); + const hollow = guardTrialRecords([{ ...trial('passed'), status: 'passed-empty' as const }]); + expect(hollow[0]!.trial!.outcome).toBeNull(); + expect(sliceExitCode(hollow)).toBe(1); + }); + + test('runPaidShards binds each trial to its case, index and panel and records its outcome', async () => { + const evalDirBase = fs.mkdtempSync(path.join(os.tmpdir(), 'trial-shards-')); + try { + const script = (fail: boolean) => `const fs = require('fs'), path = require('path'); +const dir = process.env.GSTACK_EVAL_DIR; fs.mkdirSync(dir, { recursive: true }); +const env = Object.fromEntries(Object.entries(process.env).filter(([k]) => k.startsWith('GSTACK_EVAL_') || k === 'EVALS_SELECTION_JSON')); +fs.writeFileSync(path.join(dir, 'env.json'), JSON.stringify(env)); +fs.writeFileSync(path.join(dir, 'run.json'), JSON.stringify({ tests: [{ name: 'review-sql-injection', passed: ${!fail}, cost_usd: 0.5, + duration_ms: 1, exit_reason: ${fail ? "'timeout'" : "'success'"}, timeout_at_turn: 4, model: 'm' }] })); +console.log("Ran 1 tests across 1 files. [1ms]"); process.exit(${fail ? 1 : 0});`; + const summary = await runPaidShards([[key(1)], [key(2)], [key(3)]], { + jobs: 3, evalDirBase, log: () => {}, trials: { [key(1)]: plan, [key(2)]: plan, [key(3)]: plan }, + commandFor: files => ({ command: process.execPath, args: ['-e', script(files[0] === key(2))] }), + }); + const byKey = (n: number) => summary.outcomes.find(o => o.files[0] === key(n))!; + expect(byKey(1).trial).toMatchObject({ case: 'review-sql-injection', trial: 1, outcome: 'passed', cost_usd: 0.5, model: 'm' }); + expect(byKey(2).trial).toMatchObject({ trial: 2, outcome: 'failed', failure_class: 'timeout', exit_reason: 'timeout', timeout_at_turn: 4 }); + expect(sliceExitCode(summary.outcomes)).toBe(0); + const env = JSON.parse(fs.readFileSync(path.join(evalDirBase, 'shards', shardSlug([key(3)]), 'env.json'), 'utf8')); + expect(env).toMatchObject({ GSTACK_EVAL_CASE_ID: 'review-sql-injection', GSTACK_EVAL_KIND: 'behavior', GSTACK_EVAL_TRIAL: '3', + GSTACK_EVAL_PANEL_N: '3', GSTACK_EVAL_PANEL_K: '2', GSTACK_EVAL_POLICY_VERSION: '1' }); + expect(JSON.parse(env.EVALS_SELECTION_JSON).selected).toEqual(['review-sql-injection']); + } finally { fs.rmSync(evalDirBase, { recursive: true, force: true }); } + }); + + test('file shards exclude isolated names; JUnit cases map to registry ids or stay unattributed', () => { + const pattern = new RegExp(excludedCasesNamePattern(['review-sql-injection'])); + expect(pattern.test('suite > review-sql-injection')).toBe(false); + expect(pattern.test('suite > review-enum-completeness')).toBe(true); + const cases = parseJUnitCases(` + + + +`); + expect(cases).toEqual([ + { name: 'review-sql-injection', classname: 's', outcome: 'passed', timeMs: 1500 }, + { name: 'review-enum-completeness', classname: 's', outcome: 'failed', timeMs: 100, failureType: 'TimeoutError', message: 'test & timed out' }, + { name: 'plain helper', classname: '', outcome: 'skipped', timeMs: 0 }, + ]); + expect(caseIdForTestName('review-sql-injection')).toBe('review-sql-injection'); + expect(caseIdForTestName(CASE_TEST_NAMES['plan-review-report']!)).toBe('plan-review-report'); + expect(caseIdForTestName('plain helper')).toBeNull(); + }); +});