diff --git a/scripts/lib/paid-cases.ts b/scripts/lib/paid-cases.ts index bf186281c..908d2d165 100644 --- a/scripts/lib/paid-cases.ts +++ b/scripts/lib/paid-cases.ts @@ -81,6 +81,9 @@ export const CASE_TEST_NAMES: Record = { 'plan-review-report': '/plan-eng-review writes GSTACK REVIEW REPORT to plan file', 'auq-format-gate': "/plan-ceo-review's first AskUserQuestion is a compliant decision brief (7/7 + substance)", 'autoplan-dual-voice': 'both Claude + Codex voices produce output in Phase 1 (within timeout)', + 'cso-full-audit': '/cso persists supported tenant-boundary findings with redacted evidence', + 'cso-diff-mode': '/cso --diff records its base and investigates changed security paths', + 'cso-infra-scope': '/cso --infra finds an attacker-to-credential execution path', }; export const CASE_KEY_SEPARATOR = '#'; diff --git a/scripts/lib/paid-report.ts b/scripts/lib/paid-report.ts index a039d7c22..3fdc4ec4b 100644 --- a/scripts/lib/paid-report.ts +++ b/scripts/lib/paid-report.ts @@ -57,7 +57,7 @@ export { PERIODIC_CI_EXCLUDE }; type E2EShardReuse = NonNullable>; import { CASE_TEST_NAMES, type CaseTrialPlan, caseTrialPlan, shardCaseId, shardFile, trialShardKey } from './paid-cases'; import { type ManifestEntry, PAID_TEST_DURATIONS_FILE, type PaidRunManifest, type SliceResult, collectorOutcomeCounts, formatProfileCoverage, loadPaidTestDurations, mergePaidTestDurations, parseRunManifest, trialPanelKey, verifySliceResults, writePaidTestDurations } from './paid-plan'; -import { DEFAULT_JOBS, type PaidTier, ROOT, type RunShardsOptions, type ShardTrialRecord, collectPaidTestFiles, fileCaseRegistration, isAllSkippedPass, runPaidShards, shardSlug, paidSelectionEnv } from '../test-paid-shards'; +import { DEFAULT_JOBS, type PaidCaseSelection, type PaidTier, ROOT, type RunShardsOptions, type ShardTrialRecord, collectPaidTestFiles, fileCaseRegistration, isAllSkippedPass, runPaidShards, shardSlug, paidSelectionEnv } from '../test-paid-shards'; // ─── Local diagnosis: one case through the CI panel runner (A9) ──────────── @@ -115,8 +115,8 @@ export async function runCaseDiagnosis(id: string, options: { // ─── Report: verdicts, history records and the human readout ─────────────── -/** One Bun JUnit testcase (`--reporter=junit`). */ -export interface JUnitCase { name: string; classname: string; outcome: TrialOutcome; timeMs: number; failureType?: string; message?: string } +/** One Bun JUnit testcase (`--reporter=junit`); `line` is absent on Bun's hook placeholders. */ +export interface JUnitCase { name: string; classname: string; outcome: TrialOutcome; timeMs: number; line?: number; failureType?: string; message?: string } const xmlUnescape = (text: string) => text.replace(/&(lt|gt|quot|apos|amp|#(\d+)|#x([0-9a-f]+));/gi, (_, name: string, dec?: string, hex?: string) => dec ? String.fromCodePoint(Number(dec)) : hex ? String.fromCodePoint(parseInt(hex, 16)) @@ -138,6 +138,7 @@ export function parseJUnitCases(xml: string): JUnitCase[] { name: attrs.name ?? '', classname: attrs.classname ?? '', outcome: failure ? 'failed' : / CASE_TEST_NAMES[id] === name) ?? null; } +/** Where one JUnit testcase lands in the census: a case result, a zero-credit deselection with its reason, or unattributed. */ +export type JUnitAttribution = + | { kind: 'case'; id: string } + | { kind: 'deselected'; reason: string } + | { kind: 'unattributed' }; + +export interface JUnitShardContext { + /** Shard key: `` or `#`. */ + key: string; + tier: PaidTier; + /** The run's case selection; a null list selects every case of the tier. */ + selection?: PaidCaseSelection; + /** Ids this file shard leaves to their isolated trial shards. */ + excludeCases?: readonly string[]; +} + +export const DESELECTION_REASONS = { + hook: 'Bun hook placeholder of a skipped describe block (not a test)', + sibling: 'sibling case of a case shard (its own shard runs it)', + isolated: 'isolated case (its trial shards run it)', + selection: "outside the run's case selection", +} as const; +export const RUNTIME_SKIP_REASON = 'selected in this lane but skipped at runtime by the test itself (test.skip, describe.skip or an unmet prerequisite)'; + +/** + * Attribute one testcase of a non-trial shard. A case shard's own case is + * its id or label; elsewhere a name is its registry id or CASE_TEST_NAMES + * label, else the only id the file registers, else the file itself when it + * registers none (one keyless case). A skipped testcase of another tier, a + * case-shard sibling, an isolated or unselected case, or a Bun hook + * placeholder (`(unnamed)` with no line) is deselected: zero credit, never + * SKIPPED. Anything else is unattributed and reported by name. + */ +export function attributeJUnitCase(tc: JUnitCase, ctx: JUnitShardContext, + touchfiles: Record = E2E_TOUCHFILES, tiers: Record = E2E_TIERS): JUnitAttribution { + if (tc.outcome === 'skipped' && tc.name === '(unnamed)' && tc.line === undefined) return { kind: 'deselected', reason: DESELECTION_REASONS.hook }; + const file = shardFile(ctx.key); + const shardCase = shardCaseId(ctx.key); + const named = caseIdForTestName(tc.name); + if (shardCase !== null && named !== shardCase) { + return tc.outcome === 'skipped' ? { kind: 'deselected', reason: DESELECTION_REASONS.sibling } : { kind: 'unattributed' }; + } + const registered = Object.keys(touchfiles).filter(id => touchfiles[id]!.includes(file)); + const id = named ?? (registered.length === 1 ? registered[0]! : registered.length === 0 ? file : null); + if (id === null) return { kind: 'unattributed' }; + if (tc.outcome !== 'skipped') return { kind: 'case', id }; + const tier = tiers[id]; + if (tier !== undefined && tier !== ctx.tier) return { kind: 'deselected', reason: `${tier}-tier case (this lane runs ${ctx.tier})` }; + if (ctx.excludeCases?.includes(id)) return { kind: 'deselected', reason: DESELECTION_REASONS.isolated }; + const pool = tier !== undefined ? ctx.selection?.e2e : Object.hasOwn(LLM_JUDGE_TOUCHFILES, id) ? ctx.selection?.judges : null; + if (pool && !pool.includes(id)) return { kind: 'deselected', reason: DESELECTION_REASONS.selection }; + return { kind: 'case', id }; +} + +interface RuleCase { id: string; kind: EvalCaseKind; outcome: TrialOutcome; line?: string } + +/** JUnit evidence of the non-trial shards of one attempt. */ +export interface JUnitCensus { + /** One result per attributed case per shard. */ + ruleCases: RuleCase[]; + history: TrialOutcomeRecord[]; + failedShards: Set; + /** One line per selected in-lane case that skipped, with its reason. */ + skipped: string[]; + /** Zero-credit deselections: reason -> testcase count. */ + deselected: Map; + /** Per shard key: skip reason -> skipped testcase count. */ + shardSkipReasons: Map>; + /** ` :: ()` for every testcase no rule attributes. */ + unattributed: string[]; +} + +const bump = (counts: Map, key: string) => counts.set(key, (counts.get(key) ?? 0) + 1); + +export function junitCensus(manifest: PaidRunManifest, artifacts: ReportArtifact[], recordsByShard: Map, + base: Record, cliVersion?: string): JUnitCensus { + const census: JUnitCensus = { ruleCases: [], history: [], failedShards: new Set(), skipped: [], deselected: new Map(), shardSkipReasons: new Map(), unattributed: [] }; + const entries = new Map(manifest.entries.map(entry => [normalizeRelativePath(entry.file), entry])); + for (const { root, result } of artifacts) { + for (const outcome of result.outcomes) { + const key = normalizeRelativePath(outcome.files[0] ?? ''); + if (outcome.trial || outcome.files.length !== 1) continue; + let xml = ''; + try { xml = fs.readFileSync(path.join(root, 'shards', shardSlug([key]), 'junit.xml'), 'utf8'); } catch { continue; } + const ctx: JUnitShardContext = { key, tier: manifest.tier, selection: manifest.selection, excludeCases: entries.get(key)?.excludeCases }; + const byCase = new Map(); + const reasons = new Map(); + for (const tc of parseJUnitCases(xml)) { + const attributed = attributeJUnitCase(tc, ctx); + if (attributed.kind === 'case') byCase.set(attributed.id, [...(byCase.get(attributed.id) ?? []), tc]); + else if (attributed.kind === 'deselected') { bump(census.deselected, attributed.reason); bump(reasons, attributed.reason); } + else census.unattributed.push(`${key} :: ${tc.name || '(no name)'} (${tc.outcome})`); + } + const records = recordsByShard.get(key) ?? []; + for (const [id, cases] of byCase) { + const kind = (E2E_KINDS[id] ?? 'rule') as EvalCaseKind; + const failed = cases.find(tc => tc.outcome === 'failed'); + const caseOutcome: TrialOutcome = failed ? 'failed' : cases.some(tc => tc.outcome === 'passed') ? 'passed' : 'skipped'; + const mine = records.filter((r: any) => r?.name === id || r?.case_id === id); + const failedRecord = mine.find((r: any) => r.passed === false); + const failureClass: TrialFailureClass | undefined = !failed ? undefined + : failed.failureType === 'TimeoutError' ? 'timeout' : failedRecord ? failureClassOf(failedRecord) : 'assertion'; + const error = sanitizeTrialError(failedRecord?.error ?? failed?.message); + const rerun = Object.hasOwn(E2E_TIERS, id) ? rerunCommand(manifest.tier, id, 1) : `EVALS=1 EVALS_TIER=${manifest.tier} bun test ${shardFile(key)}`; + if (failed) census.failedShards.add(key); + if (caseOutcome === 'skipped') { + bump(reasons, RUNTIME_SKIP_REASON); + census.skipped.push(`◌ ${id} ${key} (${cases.map(tc => tc.name).join('; ')}): ${RUNTIME_SKIP_REASON}`); + } + census.ruleCases.push({ id, kind, outcome: caseOutcome, + ...(failed ? { line: `✗ ${id} ${kind} FAIL ${failureClass}${failedRecord?.exit_reason === 'timeout' && failedRecord?.timeout_at_turn !== undefined ? ` at turn ${failedRecord.timeout_at_turn}` : ''}${error ? ` — ${error}` : ''} [slice ${result.sliceIndex}, attempt ${result.attempt ?? 1}] rerun: ${rerun}` } : {}) }); + census.history.push({ ...base, case: id, file: shardFile(key), kind, trial: 1, panel: { n: 1, k: 1 }, outcome: caseOutcome, + ...(failureClass ? { failure_class: failureClass } : {}), ...(failedRecord?.exit_reason ? { exit_reason: String(failedRecord.exit_reason) } : {}), + ...(error && failed ? { error } : {}), duration_ms: cases.reduce((sum, tc) => sum + tc.timeMs, 0), + cost_usd: Math.round(mine.reduce((sum: number, r: any) => sum + (Number(r.cost_usd) || 0), 0) * 100) / 100, + ...(typeof mine[0]?.model === 'string' ? { model: mine[0].model } : {}), ...(cliVersion ? { cli_version: cliVersion } : {}), + quarantined: false, execution: outcome.reused ? 'reused' : 'executed', source: 'junit' } as TrialOutcomeRecord); + } + census.shardSkipReasons.set(key, reasons); + } + } + return census; +} + +/** The census block of the readout: every skipped case with its reason, deselections by reason, every unattributed testcase by name. */ +export function formatJUnitCensus(census: JUnitCensus): string[] { + const deselected = [...census.deselected.entries()].sort(([a, x], [b, y]) => y - x || (a < b ? -1 : 1)); + return [ + ...(census.skipped.length ? [`SKIPPED cases (${census.skipped.length}):`, ...census.skipped.map(line => ` ${line}`)] : []), + ...(deselected.length ? [`deselected testcases, zero credit (${deselected.reduce((sum, [, n]) => sum + n, 0)}):`, + ...deselected.map(([reason, n]) => ` ${String(n).padStart(4)} ${reason}`)] : []), + ...(census.unattributed.length ? [`UNATTRIBUTED testcases (${census.unattributed.length}; no registry id, label, single registered id or keyless file):`, + ...census.unattributed.map(line => ` ? ${line}`)] : []), + ]; +} + interface ReportArtifact { root: string; result: SliceResult } /** Every slice result under the report dir: flat (merged) or one directory per attempt-scoped artifact. */ @@ -206,6 +343,7 @@ export function panelReports(manifest: PaidRunManifest, results: SliceResult[], /** Why one failed trial failed, in one line: a case timeout names its turn. */ function trialCause(trial: PanelVerdict['trials'][number] & { timeout_at_turn?: number }): string { + if (trial.outcome === 'skipped') return `t${trial.trial}: skipped`; const cls = trial.failure_class ?? 'assertion'; const head = cls === 'timeout' || trial.exit_reason === 'timeout' ? `timeout${trial.timeout_at_turn !== undefined ? ` at turn ${trial.timeout_at_turn}` : ''}` @@ -224,7 +362,7 @@ export function formatPanelLine(panel: PanelReport, tier: PaidTier): string { const causes = panel.trials.filter(t => t.outcome !== 'passed').map(t => trialCause(t as any)); const where = Object.entries(panel.slices).map(([trial, slice]) => `t${trial}@slice ${slice}`).join(', '); return `${mark} ${panel.case} ${panel.kind}${panel.quarantined ? ' (quarantined)' : ''} ${label} (${panel.marks})` - + `${causes.length ? ` ${causes.join('; ')}` : ''}${panel.status === 'INCOMPLETE' ? ` [${panel.reason}]` : ''}` + + `${causes.length ? ` ${causes.join('; ')}` : ''}${panel.status === 'INCOMPLETE' || panel.status === 'SKIPPED' ? ` [${panel.reason}]` : ''}` + `${where ? ` [${where}, attempt ${panel.attempt}]` : ''} rerun: ${rerunCommand(tier, panel.case, panel.panel.n)}`; } @@ -384,13 +522,6 @@ export function runPaidReport(reportDir: string, options: { writeDurations?: boo console.log(`[test:paid] report: ⚠ ${flaky.length} cases with multiple attempts this run: (paid evals never retry; each record counts)`); for (const f of flaky) console.log(` ⚠ ${f.name} (x${f.attempts}) — ${f.file}`); } - const allSkipped = results.flatMap((r) => r.outcomes.filter(isAllSkippedPass)); - if (allSkipped.length > 0) { - console.log(`[test:paid] report: ⚠ ${allSkipped.length} shard(s) passed with EVERY test skipped — they verified nothing:`); - for (const outcome of allSkipped) { - console.log(` ⚠ ${outcome.files.join(' ')} (${outcome.executedTests} skipped — external service missing or tier mismatch)`); - } - } if (manualProblems.length) verdict.problems.push(...manualProblems); if (evidence.failed > 0) verdict.problems.push(`${evidence.failed} unapproved final collector failure(s)`); if (files.reduce((sum, file) => sum + file.total, 0) !== evidence.passed + evidence.failed + evidence.manual_accepted @@ -450,38 +581,22 @@ export function runPaidReport(reportDir: string, options: { writeDurations?: boo quarantined: t.quarantined, execution: outcome.reused ? 'reused' : 'executed', source: 'shard' } as TrialOutcomeRecord); } } - const ruleCases: Array<{ id: string; kind: EvalCaseKind; outcome: TrialOutcome; line?: string }> = []; - const junitFailedShards = new Set(); - let unattributed = 0; - const cliVersion = env.GSTACK_CLAUDE_CLI_VERSION; - for (const { root, result } of artifacts.filter(a => (a.result.attempt ?? 1) === primary)) { - for (const outcome of result.outcomes) { - const key = normalizeRelativePath(outcome.files[0] ?? ''); - if (outcome.trial || outcome.files.length !== 1) continue; - let xml = ''; - try { xml = fs.readFileSync(path.join(root, 'shards', shardSlug([key]), 'junit.xml'), 'utf8'); } catch { continue; } - const records = recordsByShard.get(key) ?? []; - for (const tc of parseJUnitCases(xml)) { - const id = caseIdForTestName(tc.name); - if (id === null) { unattributed++; continue; } - const kind = (E2E_KINDS[id] ?? 'rule') as EvalCaseKind; - const mine = records.filter((r: any) => r?.name === id || r?.case_id === id); - const failedRecord = mine.find((r: any) => r.passed === false); - const failureClass: TrialFailureClass | undefined = tc.outcome !== 'failed' ? undefined - : tc.failureType === 'TimeoutError' ? 'timeout' : failedRecord ? failureClassOf(failedRecord) : 'assertion'; - const error = sanitizeTrialError(failedRecord?.error ?? tc.message); - if (tc.outcome === 'failed') junitFailedShards.add(key); - ruleCases.push({ id, kind, outcome: tc.outcome, - ...(tc.outcome === 'failed' ? { line: `✗ ${id} ${kind} FAIL ${failureClass}${failedRecord?.exit_reason === 'timeout' && failedRecord?.timeout_at_turn !== undefined ? ` at turn ${failedRecord.timeout_at_turn}` : ''}${error ? ` — ${error}` : ''} [slice ${result.sliceIndex}, attempt ${primary}] rerun: ${rerunCommand(manifest.tier, id, 1)}` } : {}) }); - history.push({ ...common(primary), case: id, file: shardFile(key), kind, trial: 1, panel: { n: 1, k: 1 }, outcome: tc.outcome, - ...(failureClass ? { failure_class: failureClass } : {}), ...(failedRecord?.exit_reason ? { exit_reason: String(failedRecord.exit_reason) } : {}), - ...(error && tc.outcome === 'failed' ? { error } : {}), duration_ms: tc.timeMs, - cost_usd: Math.round(mine.reduce((sum: number, r: any) => sum + (Number(r.cost_usd) || 0), 0) * 100) / 100, - ...(typeof mine[0]?.model === 'string' ? { model: mine[0].model } : {}), ...(cliVersion ? { cli_version: cliVersion } : {}), - quarantined: false, execution: outcome.reused ? 'reused' : 'executed', source: 'junit' } as TrialOutcomeRecord); - } + const census = junitCensus(manifest, artifacts.filter(a => (a.result.attempt ?? 1) === primary), recordsByShard, + common(primary), env.GSTACK_CLAUDE_CLI_VERSION); + history.push(...census.history); + const { ruleCases } = census; + const allSkipped = results.flatMap((r) => r.outcomes.filter(isAllSkippedPass)); + const hollowWithoutJUnit = allSkipped.filter(outcome => !census.shardSkipReasons.has(normalizeRelativePath(outcome.files[0] ?? ''))); + if (allSkipped.length > 0) { + console.log(`[test:paid] report: ⚠ ${allSkipped.length} shard(s) passed with EVERY test skipped — they verified nothing:`); + for (const outcome of allSkipped) { + const reasons = census.shardSkipReasons.get(normalizeRelativePath(outcome.files[0] ?? '')); + console.log(` ⚠ ${outcome.files.join(' ')} (${outcome.executedTests} skipped — ${reasons?.size + ? [...reasons.entries()].map(([reason, n]) => `${n} ${reason}`).join('; ') : 'no JUnit report, reason unknown'})`); } } + const censusLines = formatJUnitCensus(census); + for (const line of censusLines) console.log(`[test:paid] ${line}`); // series_identity is stamped afterwards by scripts/eval-trial-series.ts (the report job's next step). fs.writeFileSync(trialOutcomesPath, formatTrialOutcomes(history)); @@ -489,7 +604,7 @@ export function runPaidReport(reportDir: string, options: { writeDurations?: boo const ruleShardFailures = manifest.entries.filter(entry => entry.status === 'planned' && !entry.trial).flatMap(entry => { const got = results.flatMap(r => r.outcomes.map(o => ({ o, slice: r.sliceIndex }))).find(({ o }) => normalizeRelativePath(o.files[0] ?? '') === normalizeRelativePath(entry.file)); if (got && got.o.status === 'passed') return []; - if (got && junitFailedShards.has(normalizeRelativePath(entry.file))) return []; + if (got && census.failedShards.has(normalizeRelativePath(entry.file))) return []; const id = shardCaseId(entry.file); return [`✗ ${entry.file} rule shard ${got ? got.o.status : 'NOT REPORTED'}${got?.o.runnerError ? ` — ${sanitizeTrialError(got.o.runnerError)}` : ''} [slice ${entry.slice}, attempt ${primary}]${id ? ` rerun: ${rerunCommand(manifest.tier, id, 1)}` : ''}`]; }); @@ -515,11 +630,11 @@ export function runPaidReport(reportDir: string, options: { writeDurations?: boo behavior: { ...count('behavior'), split: behaviorPanels.filter(p => p.split).length }, judge: count('judge'), quarantined: { total: panels.filter(p => p.quarantined).length, failingLane: panels.filter(p => p.quarantined && p.failsLane).length }, - skipped: allSkipped.length + panels.filter(p => p.status === 'SKIPPED').length + ruleCases.filter(c => c.outcome === 'skipped').length, + skipped: hollowWithoutJUnit.length + panels.filter(p => p.status === 'SKIPPED').length + ruleCases.filter(c => c.outcome === 'skipped').length, infra: primaryTrials.filter(t => t.failure_class === 'infra').length + results.flatMap(r => r.outcomes).filter(o => !o.trial && o.runnerError !== undefined).length, incomplete: panels.filter(p => p.status === 'INCOMPLETE').length, - unattributed, + unattributed: census.unattributed.length, }, actionRequired: verdict.problems.length, wallMs: wall.length ? Math.max(...wall.map(r => r.finishedAt!)) - Math.min(...wall.map(r => r.startedAt!)) : null, @@ -537,12 +652,14 @@ export function runPaidReport(reportDir: string, options: { writeDurations?: boo fs.writeFileSync(summaryMdPath, [ ...fence(headlineLines), ...(failureLines.length ? ['', '**Failures and split verdicts**', '', ...fence(failureLines)] : []), + ...(censusLines.length ? ['', '**Skipped, deselected and unattributed testcases**', '', ...fence(censusLines)] : []), ...(verdict.problems.length ? ['', `**ACTION REQUIRED (${verdict.problems.length})**`, '', ...fence(verdict.problems.map(p => sanitizeTrialError(p) ?? p))] : []), ].join('\n') + '\n'); if (!manualProblems.length) fs.writeFileSync(summaryPath, JSON.stringify({ version: 2, files, totals: { ...evidence, total: evidence.passed + evidence.failed + evidence.manual_accepted, flaky: files.reduce((sum, file) => sum + file.flaky, 0), }, verdict: headline, headline: headlineLines, + census: { skipped: census.skipped, deselected: Object.fromEntries(census.deselected), unattributed: census.unattributed }, panels: panels.map(p => ({ case: p.case, kind: p.kind, status: p.status, passed: p.passed, n: p.panel.n, k: p.panel.k, marks: p.marks, split: p.split, quarantined: p.quarantined, failsLane: p.failsLane, redClass: p.redClass, reason: p.reason, trials: p.trials.map(t => ({ trial: t.trial, outcome: t.outcome, ...(t.failure_class ? { failure_class: t.failure_class } : {}), diff --git a/scripts/test-paid-shards.ts b/scripts/test-paid-shards.ts index bc92b31ab..5fe708eb6 100644 --- a/scripts/test-paid-shards.ts +++ b/scripts/test-paid-shards.ts @@ -209,14 +209,15 @@ export function classifyPaidTestFile(source: string, tier: PaidTier): TierClassi * The E2E ids a paid file registers: the touchfile registrations that list the * file. `known` is true only when those ids are complete: no computed * registration (testName, *IfSelected, describeIfSelected with a non-literal - * argument) and every literal registration argument is among them. Quoted - * strings elsewhere (comments, skill paths) never count. + * argument) and every literal registration argument is among them + * (`unregistered` lists the literal ids that are not). Quoted strings + * elsewhere (comments, skill paths) never count. */ export function fileCaseRegistration( file: string, source: string, touchfiles: Record = E2E_TOUCHFILES, tiers: Record = E2E_TIERS, -): { registered: string[]; known: boolean } { +): { registered: string[]; known: boolean; computed: boolean; unregistered: string[] } { const rel = normalizeRelativePath(file); const registered = Object.keys(touchfiles).filter(key => touchfiles[key]!.includes(rel)); const computed = /testName\s*:\s*(?!string\b)(?:`[^`]*\$\{|[A-Za-z_$])/.test(source) @@ -230,23 +231,30 @@ export function fileCaseRegistration( ...[...source.matchAll(/\bdescribeIfSelected\s*\([^,]*,\s*\[([^\]]*)\]/g)] .flatMap(m => [...m[1]!.matchAll(/(['"`])([^'"`]+)\1/g)].map(n => n[2]!)), ].filter(id => id in tiers); - return { registered, known: registered.length > 0 && !computed && literal.every(id => registered.includes(id)) }; + const unregistered = [...new Set(literal.filter(id => !registered.includes(id)))]; + return { registered, known: registered.length > 0 && !computed && unregistered.length === 0, computed, unregistered }; } /** - * A file is skipped for a tier lane only when its registered E2E ids are fully - * known (fileCaseRegistration) and none of them has that tier. Any computed - * registration, an id missing from the file's touchfile registration, or no id at - * all keeps today's scheduling (the child's runtime filter decides). + * A file is skipped for a tier lane only when it registers E2E ids, none of + * them has that tier, and no literal registration names an id outside its + * touchfile registration. A computed registration (a name built at runtime) + * can only run in this lane by producing an id of this tier (the child's + * EVALS_TIER filter drops every other), so it is skipped too when every id of + * the tier is registered to a test file (its touchfile list names one), which + * is not this file. An id of the tier that no test file registers, or no id + * at all, keeps today's scheduling (the child's runtime filter decides). */ export function tierSkipReason( file: string, source: string, tier: PaidTier, touchfiles: Record = E2E_TOUCHFILES, tiers: Record = E2E_TIERS, ): string | null { - const { registered, known } = fileCaseRegistration(file, source, touchfiles, tiers); - if (!known || registered.some(id => tiers[id] === tier)) return null; - return `skipped: no E2E_TIERS id has tier ${tier}`; + const { registered, computed, unregistered } = fileCaseRegistration(file, source, touchfiles, tiers); + if (registered.length === 0 || unregistered.length > 0 || registered.some(id => tiers[id] === tier)) return null; + if (!computed) return `skipped: no E2E_TIERS id has tier ${tier}`; + const unowned = Object.keys(tiers).some(id => tiers[id] === tier && !(touchfiles[id] ?? []).some(dep => /^test\/[^/]+\.test\.ts$/.test(dep))); + return unowned ? null : `skipped: no E2E_TIERS id has tier ${tier} (its computed names can only produce ${tier} ids other test files register)`; } /** diff --git a/test/cookie-validation-phases.test.ts b/test/cookie-validation-phases.test.ts index 0d68b33ff..cc0653d88 100644 --- a/test/cookie-validation-phases.test.ts +++ b/test/cookie-validation-phases.test.ts @@ -44,10 +44,11 @@ test('the existing quality and behavior phases retain their complete separate sh expect(quality.evalsAll).toBe(true); expect(behavior.evalsAll).toBe(true); expect(qualityFiles).toHaveLength(1); - // 45 files (first-task-scaffold registers no gate case, so the gate lane - // skips it); the seven case-sharded files contribute one shard per gate case. - expect(new Set(behaviorFiles.map(file => file.split('#')[0])).size).toBe(45); - expect(behaviorFiles).toHaveLength(72); + // 41 files (first-task-scaffold registers no gate case, and four files whose + // computed names hold only periodic cases, so the gate lane skips them); the + // seven case-sharded files contribute one shard per gate case. + expect(new Set(behaviorFiles.map(file => file.split('#')[0])).size).toBe(41); + expect(behaviorFiles).toHaveLength(68); expect(behaviorFiles).toEqual(expect.arrayContaining([ ...['review-exploratory-small-cli', 'ship-exploratory-small-cli', 'ship-exploratory-unavailable', 'ship-exploratory-plan-checks', 'ship-exploratory-late-input'].map(id => `test/skill-e2e-qa-callers.test.ts#${id}`), diff --git a/test/fixtures/paid-report-census/gate/manifest.json b/test/fixtures/paid-report-census/gate/manifest.json new file mode 100644 index 000000000..952504408 --- /dev/null +++ b/test/fixtures/paid-report-census/gate/manifest.json @@ -0,0 +1,43 @@ +{ + "version": 1, + "tier": "gate", + "evalsAll": true, + "sliceCount": 1, + "selectionReason": "run-all", + "profile": "full", + "selection": { + "e2e": null, + "judges": null + }, + "entries": [ + { + "file": "test/skill-e2e-plan-mode-no-op.test.ts", + "slice": 1, + "status": "planned", + "estimatedMs": 217184, + "budget": { + "timeoutMs": 3120000, + "source": "registered", + "policyId": "skill-e2e-plan-mode-no-op-existing-retry-v1" + } + }, + { + "file": "test/skill-e2e-cso.test.ts", + "slice": 1, + "status": "planned", + "estimatedMs": 227394 + }, + { + "file": "test/skill-routing-e2e.test.ts", + "slice": 1, + "status": "planned", + "estimatedMs": 1000 + }, + { + "file": "test/skill-e2e-qa-bugs.test.ts", + "slice": 1, + "status": "planned", + "estimatedMs": 1000 + } + ] +} diff --git a/test/fixtures/paid-report-census/gate/paid-slice-1-a1/shards/skill-e2e-cso/junit.xml b/test/fixtures/paid-report-census/gate/paid-slice-1-a1/shards/skill-e2e-cso/junit.xml new file mode 100644 index 000000000..8bc3131c0 --- /dev/null +++ b/test/fixtures/paid-report-census/gate/paid-slice-1-a1/shards/skill-e2e-cso/junit.xml @@ -0,0 +1,18 @@ + + + + + + + + + + + + + + + + + + diff --git a/test/fixtures/paid-report-census/gate/paid-slice-1-a1/shards/skill-e2e-plan-mode-no-op/junit.xml b/test/fixtures/paid-report-census/gate/paid-slice-1-a1/shards/skill-e2e-plan-mode-no-op/junit.xml new file mode 100644 index 000000000..36c70261e --- /dev/null +++ b/test/fixtures/paid-report-census/gate/paid-slice-1-a1/shards/skill-e2e-plan-mode-no-op/junit.xml @@ -0,0 +1,12 @@ + + + + + + + + + + + + diff --git a/test/fixtures/paid-report-census/gate/paid-slice-1-a1/shards/skill-e2e-qa-bugs/junit.xml b/test/fixtures/paid-report-census/gate/paid-slice-1-a1/shards/skill-e2e-qa-bugs/junit.xml new file mode 100644 index 000000000..0fe282c7c --- /dev/null +++ b/test/fixtures/paid-report-census/gate/paid-slice-1-a1/shards/skill-e2e-qa-bugs/junit.xml @@ -0,0 +1,22 @@ + + + + + + + + + + + + + + + + + + + + + + diff --git a/test/fixtures/paid-report-census/gate/paid-slice-1-a1/shards/skill-routing-e2e/junit.xml b/test/fixtures/paid-report-census/gate/paid-slice-1-a1/shards/skill-routing-e2e/junit.xml new file mode 100644 index 000000000..03f2e78c7 --- /dev/null +++ b/test/fixtures/paid-report-census/gate/paid-slice-1-a1/shards/skill-routing-e2e/junit.xml @@ -0,0 +1,40 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/test/fixtures/paid-report-census/gate/paid-slice-1-a1/slice-1.json b/test/fixtures/paid-report-census/gate/paid-slice-1-a1/slice-1.json new file mode 100644 index 000000000..fcf88141d --- /dev/null +++ b/test/fixtures/paid-report-census/gate/paid-slice-1-a1/slice-1.json @@ -0,0 +1,74 @@ +{ + "version": 1, + "tier": "gate", + "profile": "full", + "selection": { + "e2e": null, + "judges": null + }, + "sliceIndex": 1, + "sliceCount": 1, + "attempt": 1, + "outcomes": [ + { + "files": [ + "test/skill-e2e-plan-mode-no-op.test.ts" + ], + "status": "passed", + "exitCode": 0, + "elapsedMs": 176495, + "executedTests": 5, + "skippedTests": 0, + "budget": { + "timeoutMs": 3120000, + "source": "registered", + "policyId": "skill-e2e-plan-mode-no-op-existing-retry-v1" + } + }, + { + "files": [ + "test/skill-e2e-cso.test.ts" + ], + "status": "passed", + "exitCode": 0, + "elapsedMs": 205014, + "executedTests": 3, + "skippedTests": 2, + "budget": { + "timeoutMs": 1800000, + "source": "default", + "policyId": null + } + }, + { + "files": [ + "test/skill-routing-e2e.test.ts" + ], + "status": "passed", + "exitCode": 0, + "elapsedMs": 27, + "executedTests": 11, + "skippedTests": 11, + "budget": { + "timeoutMs": 1800000, + "source": "default", + "policyId": null + } + }, + { + "files": [ + "test/skill-e2e-qa-bugs.test.ts" + ], + "status": "passed", + "exitCode": 0, + "elapsedMs": 31, + "executedTests": 5, + "skippedTests": 5, + "budget": { + "timeoutMs": 1800000, + "source": "default", + "policyId": null + } + } + ] +} diff --git a/test/fixtures/paid-report-census/periodic/manifest.json b/test/fixtures/paid-report-census/periodic/manifest.json new file mode 100644 index 000000000..64c74c295 --- /dev/null +++ b/test/fixtures/paid-report-census/periodic/manifest.json @@ -0,0 +1,56 @@ +{ + "version": 1, + "tier": "periodic", + "evalsAll": true, + "sliceCount": 1, + "selectionReason": "run-all", + "profile": "full", + "selection": { + "e2e": null, + "judges": null + }, + "entries": [ + { + "file": "test/skill-e2e-qa-workflow.test.ts", + "slice": 1, + "status": "planned", + "estimatedMs": 403953 + }, + { + "file": "test/skill-e2e-design.test.ts#design-consultation-core", + "slice": 1, + "status": "planned", + "estimatedMs": 159257 + }, + { + "file": "test/skill-e2e-benchmark-providers.test.ts", + "slice": 1, + "status": "planned", + "estimatedMs": 9495 + }, + { + "file": "test/carve-section-loading-review.test.ts", + "slice": 1, + "status": "planned", + "estimatedMs": 247902 + }, + { + "file": "test/skill-e2e-cso.test.ts", + "slice": 1, + "status": "planned", + "estimatedMs": 253498 + }, + { + "file": "test/codex-e2e-recommendation-substance.test.ts", + "slice": 1, + "status": "planned", + "estimatedMs": 1000 + }, + { + "file": "test/skill-e2e-test-value.test.ts", + "slice": 1, + "status": "planned", + "estimatedMs": 332000 + } + ] +} diff --git a/test/fixtures/paid-report-census/periodic/paid-slice-1-a1/shards/carve-section-loading-review/junit.xml b/test/fixtures/paid-report-census/periodic/paid-slice-1-a1/shards/carve-section-loading-review/junit.xml new file mode 100644 index 000000000..0dbf3530a --- /dev/null +++ b/test/fixtures/paid-report-census/periodic/paid-slice-1-a1/shards/carve-section-loading-review/junit.xml @@ -0,0 +1,8 @@ + + + + + + + + diff --git a/test/fixtures/paid-report-census/periodic/paid-slice-1-a1/shards/codex-e2e-recommendation-substance/junit.xml b/test/fixtures/paid-report-census/periodic/paid-slice-1-a1/shards/codex-e2e-recommendation-substance/junit.xml new file mode 100644 index 000000000..ad1b24d6e --- /dev/null +++ b/test/fixtures/paid-report-census/periodic/paid-slice-1-a1/shards/codex-e2e-recommendation-substance/junit.xml @@ -0,0 +1,8 @@ + + + + + + + + diff --git a/test/fixtures/paid-report-census/periodic/paid-slice-1-a1/shards/skill-e2e-benchmark-providers/junit.xml b/test/fixtures/paid-report-census/periodic/paid-slice-1-a1/shards/skill-e2e-benchmark-providers/junit.xml new file mode 100644 index 000000000..8762d2a07 --- /dev/null +++ b/test/fixtures/paid-report-census/periodic/paid-slice-1-a1/shards/skill-e2e-benchmark-providers/junit.xml @@ -0,0 +1,15 @@ + + + + + + + + + + + + + + + diff --git a/test/fixtures/paid-report-census/periodic/paid-slice-1-a1/shards/skill-e2e-cso/junit.xml b/test/fixtures/paid-report-census/periodic/paid-slice-1-a1/shards/skill-e2e-cso/junit.xml new file mode 100644 index 000000000..d8e8c1ba9 --- /dev/null +++ b/test/fixtures/paid-report-census/periodic/paid-slice-1-a1/shards/skill-e2e-cso/junit.xml @@ -0,0 +1,16 @@ + + + + + + + + + + + + + + + + diff --git a/test/fixtures/paid-report-census/periodic/paid-slice-1-a1/shards/skill-e2e-design--design-consultation-core/junit.xml b/test/fixtures/paid-report-census/periodic/paid-slice-1-a1/shards/skill-e2e-design--design-consultation-core/junit.xml new file mode 100644 index 000000000..7168ee989 --- /dev/null +++ b/test/fixtures/paid-report-census/periodic/paid-slice-1-a1/shards/skill-e2e-design--design-consultation-core/junit.xml @@ -0,0 +1,48 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/test/fixtures/paid-report-census/periodic/paid-slice-1-a1/shards/skill-e2e-qa-workflow/junit.xml b/test/fixtures/paid-report-census/periodic/paid-slice-1-a1/shards/skill-e2e-qa-workflow/junit.xml new file mode 100644 index 000000000..2dfa4437d --- /dev/null +++ b/test/fixtures/paid-report-census/periodic/paid-slice-1-a1/shards/skill-e2e-qa-workflow/junit.xml @@ -0,0 +1,35 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/test/fixtures/paid-report-census/periodic/paid-slice-1-a1/shards/skill-e2e-test-value/junit.xml b/test/fixtures/paid-report-census/periodic/paid-slice-1-a1/shards/skill-e2e-test-value/junit.xml new file mode 100644 index 000000000..0e496c74c --- /dev/null +++ b/test/fixtures/paid-report-census/periodic/paid-slice-1-a1/shards/skill-e2e-test-value/junit.xml @@ -0,0 +1,20 @@ + + + + + + + + + + + + + + + + + + + + diff --git a/test/fixtures/paid-report-census/periodic/paid-slice-1-a1/slice-1.json b/test/fixtures/paid-report-census/periodic/paid-slice-1-a1/slice-1.json new file mode 100644 index 000000000..a45af4070 --- /dev/null +++ b/test/fixtures/paid-report-census/periodic/paid-slice-1-a1/slice-1.json @@ -0,0 +1,119 @@ +{ + "version": 1, + "tier": "periodic", + "profile": "full", + "selection": { + "e2e": null, + "judges": null + }, + "sliceIndex": 1, + "sliceCount": 1, + "attempt": 1, + "outcomes": [ + { + "files": [ + "test/skill-e2e-qa-workflow.test.ts" + ], + "status": "passed", + "exitCode": 0, + "elapsedMs": 353035, + "executedTests": 8, + "skippedTests": 7, + "budget": { + "timeoutMs": 1800000, + "source": "default", + "policyId": null + } + }, + { + "files": [ + "test/skill-e2e-design.test.ts#design-consultation-core" + ], + "status": "passed", + "exitCode": 0, + "elapsedMs": 176652, + "executedTests": 4, + "skippedTests": 3, + "budget": { + "timeoutMs": 1800000, + "source": "default", + "policyId": null + } + }, + { + "files": [ + "test/skill-e2e-benchmark-providers.test.ts" + ], + "status": "passed", + "exitCode": 0, + "elapsedMs": 8552, + "executedTests": 8, + "skippedTests": 0, + "budget": { + "timeoutMs": 1800000, + "source": "default", + "policyId": null + } + }, + { + "files": [ + "test/carve-section-loading-review.test.ts" + ], + "status": "passed", + "exitCode": 0, + "elapsedMs": 143440, + "executedTests": 1, + "skippedTests": 0, + "budget": { + "timeoutMs": 1800000, + "source": "default", + "policyId": null + } + }, + { + "files": [ + "test/skill-e2e-cso.test.ts" + ], + "status": "passed", + "exitCode": 0, + "elapsedMs": 293156, + "executedTests": 3, + "skippedTests": 1, + "budget": { + "timeoutMs": 1800000, + "source": "default", + "policyId": null + } + }, + { + "files": [ + "test/codex-e2e-recommendation-substance.test.ts" + ], + "status": "passed", + "exitCode": 0, + "elapsedMs": 13090, + "executedTests": 1, + "skippedTests": 0, + "budget": { + "timeoutMs": 1800000, + "source": "default", + "policyId": null + } + }, + { + "files": [ + "test/skill-e2e-test-value.test.ts" + ], + "status": "passed", + "exitCode": 0, + "elapsedMs": 50, + "executedTests": 3, + "skippedTests": 3, + "budget": { + "timeoutMs": 1800000, + "source": "default", + "policyId": null + } + } + ] +} diff --git a/test/paid-report-census.test.ts b/test/paid-report-census.test.ts new file mode 100644 index 000000000..b57a0ae74 --- /dev/null +++ b/test/paid-report-census.test.ts @@ -0,0 +1,125 @@ +/** + * Census attribution in the paid report (I4): every JUnit testcase of a + * non-trial shard is a case result, a zero-credit deselection with its + * reason, or listed by name as unattributed. Fixtures are trimmed from the + * 2026-10-03 periodic and gate census artifacts (run 37151477069, commit + * ac20ef1), whose report counted deselections as SKIPPED 82/282 and left + * 72/24 testcases unattributed. + */ +import { afterAll, beforeAll, describe, expect, test } from 'bun:test'; +import * as fs from 'node:fs'; +import * as os from 'node:os'; +import * as path from 'node:path'; +import { DESELECTION_REASONS, RUNTIME_SKIP_REASON, attributeJUnitCase, formatPanelLine, runPaidReport, type JUnitCase } from '../scripts/test-paid-shards'; +import { panelVerdict } from './helpers/eval-store'; + +const FIXTURES = path.join(import.meta.dir, 'fixtures', 'paid-report-census'); + +let base: string; +beforeAll(() => { base = fs.mkdtempSync(path.join(os.tmpdir(), 'paid-report-census-')); }); +afterAll(() => { fs.rmSync(base, { recursive: true, force: true }); }); + +let runs = 0; +function report(lane: 'periodic' | 'gate', edit: (dir: string) => void = () => {}) { + const dir = path.join(base, `${lane}-${++runs}`); + fs.cpSync(path.join(FIXTURES, lane), dir, { recursive: true }); + edit(dir); + const lines: string[] = []; + const { log, error } = console; + console.log = (...args: unknown[]) => { lines.push(args.join(' ')); }; + console.error = console.log; + let status: number; + try { status = runPaidReport(dir, { env: {} }); } finally { console.log = log; console.error = error; } + const history = fs.readFileSync(path.join(dir, 'trial-outcomes.jsonl'), 'utf8').split('\n').filter(Boolean).map(line => JSON.parse(line)); + return { status, out: lines.join('\n'), dir, history }; +} + +const junit = (dir: string, slug: string) => path.join(dir, 'paid-slice-1-a1', 'shards', slug, 'junit.xml'); + +describe('census attribution from the 2026-10-03 artifacts', () => { + test('periodic: deselections and hook placeholders are zero-credit, every executed testcase maps to one case', () => { + const r = report('periodic'); + expect(r.status, r.out).toBe(0); + expect(r.out).toContain(' rule 7/7 · behavior 0/0 · judge 0/0 · quarantined 0 (0 failing the lane)'); + expect(r.out).toContain(' SKIPPED 0 · INFRA 0 · INCOMPLETE 0 · unattributed 0 · ACTION REQUIRED 0'); + expect(r.out).toContain(`deselected testcases, zero credit (21):`); + expect(r.out).toContain(` 10 ${DESELECTION_REASONS.sibling}`); + expect(r.out).toContain(' 7 gate-tier case (this lane runs periodic)'); + expect(r.out).toContain(` 4 ${DESELECTION_REASONS.hook}`); + expect(r.out).toContain(' ⚠ test/skill-e2e-test-value.test.ts (3 skipped — 3 gate-tier case (this lane runs periodic))'); + expect(r.out).not.toContain('UNATTRIBUTED'); + expect(r.history.map(record => `${record.case}:${record.outcome}`).sort()).toEqual([ + 'benchmark-providers-live:passed', 'carve-section-loading:passed', 'cso-full-audit:passed', 'cso-infra-scope:passed', + 'design-consultation-core:passed', 'qa-fix-loop:passed', 'test/codex-e2e-recommendation-substance.test.ts:passed', + ]); + expect(r.history.find(record => record.case === 'benchmark-providers-live')).toMatchObject({ file: 'test/skill-e2e-benchmark-providers.test.ts', source: 'junit' }); + expect(r.history.filter(record => record.file === 'test/skill-e2e-design.test.ts')) + .toEqual([expect.objectContaining({ case: 'design-consultation-core', outcome: 'passed', source: 'junit' })]); + expect(r.history.some(record => record.outcome === 'skipped')).toBe(false); + }); + + test('gate: other-tier cases and hook placeholders are deselected; a multi-test case is one result', () => { + const r = report('gate'); + expect(r.status, r.out).toBe(0); + expect(r.out).toContain(' rule 2/2 · behavior 0/0 · judge 0/0 · quarantined 0 (0 failing the lane)'); + expect(r.out).toContain(' SKIPPED 0 · INFRA 0 · INCOMPLETE 0 · unattributed 0 · ACTION REQUIRED 0'); + expect(r.out).toContain(' 16 periodic-tier case (this lane runs gate)'); + expect(r.out).toContain(` 2 ${DESELECTION_REASONS.hook}`); + expect(r.out).toContain(` ⚠ test/skill-e2e-qa-bugs.test.ts (5 skipped — 2 ${DESELECTION_REASONS.hook}; 3 periodic-tier case (this lane runs gate))`); + expect(r.history.map(record => `${record.case}:${record.outcome}`).sort()).toEqual(['cso-diff-mode:passed', 'plan-mode-no-op:passed']); + }); + + test('a selected in-lane case that skipped is SKIPPED with its reason; an unattributable testcase is listed by name', () => { + const r = report('periodic', dir => { + const qa = junit(dir, 'skill-e2e-qa-workflow'); + fs.writeFileSync(qa, fs.readFileSync(qa, 'utf8').replace(/(]*?) \/>/, '$1>')); + const cso = junit(dir, 'skill-e2e-cso'); + fs.writeFileSync(cso, fs.readFileSync(cso, 'utf8').replace('\n', + '\n\n')); + }); + expect(r.status, r.out).toBe(0); + expect(r.out).toContain(' SKIPPED 1 · INFRA 0 · INCOMPLETE 0 · unattributed 1 · ACTION REQUIRED 0'); + expect(r.out).toContain(`[test:paid] ◌ qa-fix-loop test/skill-e2e-qa-workflow.test.ts (qa-fix-loop): ${RUNTIME_SKIP_REASON}`); + expect(r.out).toContain('[test:paid] ? test/skill-e2e-cso.test.ts :: an unlabeled helper (passed)'); + const summary = fs.readFileSync(path.join(r.dir, 'report-summary.md'), 'utf8'); + expect(summary).toContain('**Skipped, deselected and unattributed testcases**'); + expect(summary).toContain('? test/skill-e2e-cso.test.ts :: an unlabeled helper (passed)'); + const collector = JSON.parse(fs.readFileSync(path.join(r.dir, 'collector-outcomes.json'), 'utf8')); + expect(collector.census.unattributed).toEqual(['test/skill-e2e-cso.test.ts :: an unlabeled helper (passed)']); + expect(collector.census.skipped).toHaveLength(1); + expect(collector.verdict.counts).toMatchObject({ skipped: 1, unattributed: 1 }); + }); +}); + +describe('attributeJUnitCase', () => { + const tc = (name: string, outcome: JUnitCase['outcome'], line: number | null = 1): JUnitCase => + ({ name, classname: 's', outcome, timeMs: 0, ...(line === null ? {} : { line }) }); + const file = 'test/skill-e2e-review.test.ts'; + + test('isolated and unselected in-lane cases are deselected; a selected skip is a case', () => { + expect(attributeJUnitCase(tc('review-design-lite', 'skipped'), { key: file, tier: 'periodic', excludeCases: ['review-design-lite'] })) + .toEqual({ kind: 'deselected', reason: DESELECTION_REASONS.isolated }); + expect(attributeJUnitCase(tc('review-sql-injection', 'skipped'), { key: file, tier: 'gate', selection: { e2e: ['review-enum-completeness'], judges: null } })) + .toEqual({ kind: 'deselected', reason: DESELECTION_REASONS.selection }); + expect(attributeJUnitCase(tc('review-sql-injection', 'skipped'), { key: file, tier: 'gate', selection: { e2e: ['review-sql-injection'], judges: null } })) + .toEqual({ kind: 'case', id: 'review-sql-injection' }); + }); + + test('only a skipped, line-less "(unnamed)" is a hook placeholder; a failing hook or a stray case-shard result is never deselected', () => { + expect(attributeJUnitCase(tc('(unnamed)', 'skipped', null), { key: file, tier: 'gate' })).toEqual({ kind: 'deselected', reason: DESELECTION_REASONS.hook }); + expect(attributeJUnitCase(tc('(unnamed)', 'failed', null), { key: file, tier: 'gate' })).toEqual({ kind: 'unattributed' }); + expect(attributeJUnitCase(tc('(unnamed)', 'skipped'), { key: file, tier: 'gate' })).toEqual({ kind: 'unattributed' }); + const shard = 'test/skill-e2e-design.test.ts#design-consultation-core'; + expect(attributeJUnitCase(tc('design-consultation-core', 'passed'), { key: shard, tier: 'periodic' })).toEqual({ kind: 'case', id: 'design-consultation-core' }); + expect(attributeJUnitCase(tc('needs Aside', 'skipped'), { key: shard, tier: 'periodic' })).toEqual({ kind: 'deselected', reason: DESELECTION_REASONS.sibling }); + expect(attributeJUnitCase(tc('needs Aside', 'passed'), { key: shard, tier: 'periodic' })).toEqual({ kind: 'unattributed' }); + }); + + test('a SKIPPED panel line names its reason and never calls a skipped trial an assertion', () => { + const verdict = panelVerdict({ case: 'review-design-lite', kind: 'behavior', panel: { n: 3, k: 2 }, + trials: [1, 2, 3].map(trial => ({ trial, outcome: 'skipped' as const })), quarantined: false }); + const line = formatPanelLine({ ...verdict, file: 'test/skill-e2e-review.test.ts', slices: {} }, 'periodic'); + expect(line).toContain('t1: skipped; t2: skipped; t3: skipped [every trial skipped (no verdict credit)]'); + expect(line).not.toContain('assertion'); + }); +}); diff --git a/test/paid-retry-supervision.test.ts b/test/paid-retry-supervision.test.ts index e5cec82d4..21109714b 100644 --- a/test/paid-retry-supervision.test.ts +++ b/test/paid-retry-supervision.test.ts @@ -210,7 +210,7 @@ test('detached PR fallback and release commands cover their actual default worke const prFloor = Math.ceil((Math.ceil(fullGateFiles.length / prWorkers) * 1_800_000 + fullGateFiles.reduce( (total, file) => total + Math.max(0, resolvePaidShardBudget([file]).timeoutMs - 1_800_000), 0, )) / 1000 * 1.05); - expect(prFloor).toBe(72_755); + expect(prFloor).toBe(68_975); expect(prWall).toBe(92_820_000); expect(prWall).toBeGreaterThanOrEqual(paidShardWallUpperBoundMs(files, prWorkers) + 120_000); @@ -229,8 +229,8 @@ test('detached PR fallback and release commands cover their actual default worke )) / 1000 * 1.05)); } const detachedReleaseWall = Number(scripts['eval:bg:release'].match(/--timeout (\d+)/)?.[1]) * 1000; - expect(releaseFloors).toEqual([21_725, 35_711]); - expect(releaseFloors.reduce((total, floor) => total + floor, 0)).toBe(57_436); + expect(releaseFloors).toEqual([19_835, 33_821]); + expect(releaseFloors.reduce((total, floor) => total + floor, 0)).toBe(53_656); expect(detachedReleaseWall).toBe(116_700_000); expect(detachedReleaseWall).toBeGreaterThanOrEqual(releaseWall + 120_000); }); diff --git a/test/paid-shards.test.ts b/test/paid-shards.test.ts index e11f0a14d..a6ec1e1b1 100644 --- a/test/paid-shards.test.ts +++ b/test/paid-shards.test.ts @@ -128,6 +128,35 @@ describe('tier lane skip (B5)', () => { expect(tierSkipReason(file, "testIfSelected('sample-gate', async () => {});", 'gate', touchfiles, tiers)).toBeNull(); }); + test('a computed registration is skipped only when every id of the lane tier is registered to another test file', () => { + const other = 'test/skill-e2e-other.test.ts'; + const owned = { 'sample-periodic': [file], 'gate-elsewhere': [other, 'x/**'] }; + const ownedTiers = { 'sample-periodic': 'periodic', 'gate-elsewhere': 'gate' }; + for (const source of [ + "testConcurrentIfSelected(`sample-${label}`, async () => {});", + "runSkillTest({ testName: `sample-${label}` });", + "for (const entry of CASES) describeIfSelected(entry.suite, [entry.id], () => { testIfSelected(entry.id, async () => {}); });", + ]) { + expect(tierSkipReason(file, source, 'gate', owned, ownedTiers)).toBe( + 'skipped: no E2E_TIERS id has tier gate (its computed names can only produce gate ids other test files register)'); + expect(tierSkipReason(file, source, 'periodic', owned, ownedTiers)).toBeNull(); + expect(tierSkipReason(file, source, 'gate', owned, { ...ownedTiers, 'gate-orphan': 'gate' })).toBeNull(); + expect(tierSkipReason(file, `${source}\ntestIfSelected('gate-elsewhere', async () => {});`, 'gate', owned, ownedTiers)).toBeNull(); + } + }); + + test('census files whose computed names hold only other-tier cases never take a runner (2026-10-03 census)', () => { + const lanes = { gate: ['test/skill-e2e-plan-decision-classification.test.ts', 'test/skill-e2e-plan-devex-peer-comparison-classification.test.ts', + 'test/skill-e2e-qa-bugs.test.ts', 'test/skill-routing-e2e.test.ts'], periodic: ['test/skill-e2e-coverage-audit.test.ts', 'test/skill-e2e-test-value.test.ts'] }; + for (const [tier, files] of Object.entries(lanes) as Array<['gate' | 'periodic', string[]]>) { + const { selected, excluded } = selectPaidTestFiles(collectPaidTestFiles(), tier, ROOT, {}); + for (const hollow of files) { + expect(selected, hollow).not.toContain(hollow); + expect(excluded.find(entry => entry.file === hollow)?.reason, hollow).toStartWith(`skipped: no E2E_TIERS id has tier ${tier}`); + } + } + }); + test('the real constructed-name diagram file stays scheduled in both lanes', () => { const source = fs.readFileSync(path.join(ROOT, 'test/skill-e2e-diagram.test.ts'), 'utf8'); for (const tier of ['gate', 'periodic'] as const) expect(tierSkipReason('test/skill-e2e-diagram.test.ts', source, tier)).toBeNull();