From 7eb85e0e3ce979fe4c4b9f7d7739fcd3d94f8135 Mon Sep 17 00:00:00 2001 From: garrytan Date: Sat, 3 Oct 2026 22:34:17 +0000 Subject: [PATCH] I4: attribute every census testcase; deselections never count as SKIPPED The paid report counted every skipped JUnit testcase as SKIPPED and every testcase whose name was not a registry id as unattributed. On the 2026-10-03 census (run 37151477069) that was SKIPPED 82 / unattributed 72 (periodic) and SKIPPED 282 / unattributed 24 (gate), and every case-shard sibling also wrote a skipped history record that reset its pass-rate series. - attributeJUnitCase: a name is its registry id or CASE_TEST_NAMES label, else the only id its file registers, else the keyless file itself. Skipped other-tier cases, case-shard siblings, isolated or unselected cases and Bun's '(unnamed)' hook placeholders are zero-credit deselections with a reason and write no history record. Anything else is listed by name as unattributed. - One result and one history record per case per shard; a genuinely skipped selected case counts as SKIPPED and is listed with its reason, also in report-summary.md and collector-outcomes.json. SKIPPED panels show their reason and skipped trials are no longer called assertions. - CASE_TEST_NAMES labels for the three /cso cases. - tierSkipReason: a computed registration whose lane-tier ids are all registered to other test files is excluded at plan time, so the four gate and two periodic files that ran with every test skipped no longer take a runner. Census rerun: periodic SKIPPED 0 / unattributed 0, gate SKIPPED 0 / unattributed 0; verdicts unchanged. --- scripts/lib/paid-cases.ts | 3 + scripts/lib/paid-report.ts | 205 ++++++++++++++---- scripts/test-paid-shards.ts | 30 ++- test/cookie-validation-phases.test.ts | 9 +- .../paid-report-census/gate/manifest.json | 43 ++++ .../shards/skill-e2e-cso/junit.xml | 18 ++ .../skill-e2e-plan-mode-no-op/junit.xml | 12 + .../shards/skill-e2e-qa-bugs/junit.xml | 22 ++ .../shards/skill-routing-e2e/junit.xml | 40 ++++ .../gate/paid-slice-1-a1/slice-1.json | 74 +++++++ .../paid-report-census/periodic/manifest.json | 56 +++++ .../carve-section-loading-review/junit.xml | 8 + .../junit.xml | 8 + .../skill-e2e-benchmark-providers/junit.xml | 15 ++ .../shards/skill-e2e-cso/junit.xml | 16 ++ .../junit.xml | 48 ++++ .../shards/skill-e2e-qa-workflow/junit.xml | 35 +++ .../shards/skill-e2e-test-value/junit.xml | 20 ++ .../periodic/paid-slice-1-a1/slice-1.json | 119 ++++++++++ test/paid-report-census.test.ts | 125 +++++++++++ test/paid-retry-supervision.test.ts | 6 +- test/paid-shards.test.ts | 29 +++ 22 files changed, 879 insertions(+), 62 deletions(-) create mode 100644 test/fixtures/paid-report-census/gate/manifest.json create mode 100644 test/fixtures/paid-report-census/gate/paid-slice-1-a1/shards/skill-e2e-cso/junit.xml create mode 100644 test/fixtures/paid-report-census/gate/paid-slice-1-a1/shards/skill-e2e-plan-mode-no-op/junit.xml create mode 100644 test/fixtures/paid-report-census/gate/paid-slice-1-a1/shards/skill-e2e-qa-bugs/junit.xml create mode 100644 test/fixtures/paid-report-census/gate/paid-slice-1-a1/shards/skill-routing-e2e/junit.xml create mode 100644 test/fixtures/paid-report-census/gate/paid-slice-1-a1/slice-1.json create mode 100644 test/fixtures/paid-report-census/periodic/manifest.json create mode 100644 test/fixtures/paid-report-census/periodic/paid-slice-1-a1/shards/carve-section-loading-review/junit.xml create mode 100644 test/fixtures/paid-report-census/periodic/paid-slice-1-a1/shards/codex-e2e-recommendation-substance/junit.xml create mode 100644 test/fixtures/paid-report-census/periodic/paid-slice-1-a1/shards/skill-e2e-benchmark-providers/junit.xml create mode 100644 test/fixtures/paid-report-census/periodic/paid-slice-1-a1/shards/skill-e2e-cso/junit.xml create mode 100644 test/fixtures/paid-report-census/periodic/paid-slice-1-a1/shards/skill-e2e-design--design-consultation-core/junit.xml create mode 100644 test/fixtures/paid-report-census/periodic/paid-slice-1-a1/shards/skill-e2e-qa-workflow/junit.xml create mode 100644 test/fixtures/paid-report-census/periodic/paid-slice-1-a1/shards/skill-e2e-test-value/junit.xml create mode 100644 test/fixtures/paid-report-census/periodic/paid-slice-1-a1/slice-1.json create mode 100644 test/paid-report-census.test.ts diff --git a/scripts/lib/paid-cases.ts b/scripts/lib/paid-cases.ts index bf186281c..908d2d165 100644 --- a/scripts/lib/paid-cases.ts +++ b/scripts/lib/paid-cases.ts @@ -81,6 +81,9 @@ export const CASE_TEST_NAMES: Record = { 'plan-review-report': '/plan-eng-review writes GSTACK REVIEW REPORT to plan file', 'auq-format-gate': "/plan-ceo-review's first AskUserQuestion is a compliant decision brief (7/7 + substance)", 'autoplan-dual-voice': 'both Claude + Codex voices produce output in Phase 1 (within timeout)', + 'cso-full-audit': '/cso persists supported tenant-boundary findings with redacted evidence', + 'cso-diff-mode': '/cso --diff records its base and investigates changed security paths', + 'cso-infra-scope': '/cso --infra finds an attacker-to-credential execution path', }; export const CASE_KEY_SEPARATOR = '#'; diff --git a/scripts/lib/paid-report.ts b/scripts/lib/paid-report.ts index a039d7c22..3fdc4ec4b 100644 --- a/scripts/lib/paid-report.ts +++ b/scripts/lib/paid-report.ts @@ -57,7 +57,7 @@ export { PERIODIC_CI_EXCLUDE }; type E2EShardReuse = NonNullable>; import { CASE_TEST_NAMES, type CaseTrialPlan, caseTrialPlan, shardCaseId, shardFile, trialShardKey } from './paid-cases'; import { type ManifestEntry, PAID_TEST_DURATIONS_FILE, type PaidRunManifest, type SliceResult, collectorOutcomeCounts, formatProfileCoverage, loadPaidTestDurations, mergePaidTestDurations, parseRunManifest, trialPanelKey, verifySliceResults, writePaidTestDurations } from './paid-plan'; -import { DEFAULT_JOBS, type PaidTier, ROOT, type RunShardsOptions, type ShardTrialRecord, collectPaidTestFiles, fileCaseRegistration, isAllSkippedPass, runPaidShards, shardSlug, paidSelectionEnv } from '../test-paid-shards'; +import { DEFAULT_JOBS, type PaidCaseSelection, type PaidTier, ROOT, type RunShardsOptions, type ShardTrialRecord, collectPaidTestFiles, fileCaseRegistration, isAllSkippedPass, runPaidShards, shardSlug, paidSelectionEnv } from '../test-paid-shards'; // ─── Local diagnosis: one case through the CI panel runner (A9) ──────────── @@ -115,8 +115,8 @@ export async function runCaseDiagnosis(id: string, options: { // ─── Report: verdicts, history records and the human readout ─────────────── -/** One Bun JUnit testcase (`--reporter=junit`). */ -export interface JUnitCase { name: string; classname: string; outcome: TrialOutcome; timeMs: number; failureType?: string; message?: string } +/** One Bun JUnit testcase (`--reporter=junit`); `line` is absent on Bun's hook placeholders. */ +export interface JUnitCase { name: string; classname: string; outcome: TrialOutcome; timeMs: number; line?: number; failureType?: string; message?: string } const xmlUnescape = (text: string) => text.replace(/&(lt|gt|quot|apos|amp|#(\d+)|#x([0-9a-f]+));/gi, (_, name: string, dec?: string, hex?: string) => dec ? String.fromCodePoint(Number(dec)) : hex ? String.fromCodePoint(parseInt(hex, 16)) @@ -138,6 +138,7 @@ export function parseJUnitCases(xml: string): JUnitCase[] { name: attrs.name ?? '', classname: attrs.classname ?? '', outcome: failure ? 'failed' : / CASE_TEST_NAMES[id] === name) ?? null; } +/** Where one JUnit testcase lands in the census: a case result, a zero-credit deselection with its reason, or unattributed. */ +export type JUnitAttribution = + | { kind: 'case'; id: string } + | { kind: 'deselected'; reason: string } + | { kind: 'unattributed' }; + +export interface JUnitShardContext { + /** Shard key: `` or `#`. */ + key: string; + tier: PaidTier; + /** The run's case selection; a null list selects every case of the tier. */ + selection?: PaidCaseSelection; + /** Ids this file shard leaves to their isolated trial shards. */ + excludeCases?: readonly string[]; +} + +export const DESELECTION_REASONS = { + hook: 'Bun hook placeholder of a skipped describe block (not a test)', + sibling: 'sibling case of a case shard (its own shard runs it)', + isolated: 'isolated case (its trial shards run it)', + selection: "outside the run's case selection", +} as const; +export const RUNTIME_SKIP_REASON = 'selected in this lane but skipped at runtime by the test itself (test.skip, describe.skip or an unmet prerequisite)'; + +/** + * Attribute one testcase of a non-trial shard. A case shard's own case is + * its id or label; elsewhere a name is its registry id or CASE_TEST_NAMES + * label, else the only id the file registers, else the file itself when it + * registers none (one keyless case). A skipped testcase of another tier, a + * case-shard sibling, an isolated or unselected case, or a Bun hook + * placeholder (`(unnamed)` with no line) is deselected: zero credit, never + * SKIPPED. Anything else is unattributed and reported by name. + */ +export function attributeJUnitCase(tc: JUnitCase, ctx: JUnitShardContext, + touchfiles: Record = E2E_TOUCHFILES, tiers: Record = E2E_TIERS): JUnitAttribution { + if (tc.outcome === 'skipped' && tc.name === '(unnamed)' && tc.line === undefined) return { kind: 'deselected', reason: DESELECTION_REASONS.hook }; + const file = shardFile(ctx.key); + const shardCase = shardCaseId(ctx.key); + const named = caseIdForTestName(tc.name); + if (shardCase !== null && named !== shardCase) { + return tc.outcome === 'skipped' ? { kind: 'deselected', reason: DESELECTION_REASONS.sibling } : { kind: 'unattributed' }; + } + const registered = Object.keys(touchfiles).filter(id => touchfiles[id]!.includes(file)); + const id = named ?? (registered.length === 1 ? registered[0]! : registered.length === 0 ? file : null); + if (id === null) return { kind: 'unattributed' }; + if (tc.outcome !== 'skipped') return { kind: 'case', id }; + const tier = tiers[id]; + if (tier !== undefined && tier !== ctx.tier) return { kind: 'deselected', reason: `${tier}-tier case (this lane runs ${ctx.tier})` }; + if (ctx.excludeCases?.includes(id)) return { kind: 'deselected', reason: DESELECTION_REASONS.isolated }; + const pool = tier !== undefined ? ctx.selection?.e2e : Object.hasOwn(LLM_JUDGE_TOUCHFILES, id) ? ctx.selection?.judges : null; + if (pool && !pool.includes(id)) return { kind: 'deselected', reason: DESELECTION_REASONS.selection }; + return { kind: 'case', id }; +} + +interface RuleCase { id: string; kind: EvalCaseKind; outcome: TrialOutcome; line?: string } + +/** JUnit evidence of the non-trial shards of one attempt. */ +export interface JUnitCensus { + /** One result per attributed case per shard. */ + ruleCases: RuleCase[]; + history: TrialOutcomeRecord[]; + failedShards: Set; + /** One line per selected in-lane case that skipped, with its reason. */ + skipped: string[]; + /** Zero-credit deselections: reason -> testcase count. */ + deselected: Map; + /** Per shard key: skip reason -> skipped testcase count. */ + shardSkipReasons: Map>; + /** ` :: ()` for every testcase no rule attributes. */ + unattributed: string[]; +} + +const bump = (counts: Map, key: string) => counts.set(key, (counts.get(key) ?? 0) + 1); + +export function junitCensus(manifest: PaidRunManifest, artifacts: ReportArtifact[], recordsByShard: Map, + base: Record, cliVersion?: string): JUnitCensus { + const census: JUnitCensus = { ruleCases: [], history: [], failedShards: new Set(), skipped: [], deselected: new Map(), shardSkipReasons: new Map(), unattributed: [] }; + const entries = new Map(manifest.entries.map(entry => [normalizeRelativePath(entry.file), entry])); + for (const { root, result } of artifacts) { + for (const outcome of result.outcomes) { + const key = normalizeRelativePath(outcome.files[0] ?? ''); + if (outcome.trial || outcome.files.length !== 1) continue; + let xml = ''; + try { xml = fs.readFileSync(path.join(root, 'shards', shardSlug([key]), 'junit.xml'), 'utf8'); } catch { continue; } + const ctx: JUnitShardContext = { key, tier: manifest.tier, selection: manifest.selection, excludeCases: entries.get(key)?.excludeCases }; + const byCase = new Map(); + const reasons = new Map(); + for (const tc of parseJUnitCases(xml)) { + const attributed = attributeJUnitCase(tc, ctx); + if (attributed.kind === 'case') byCase.set(attributed.id, [...(byCase.get(attributed.id) ?? []), tc]); + else if (attributed.kind === 'deselected') { bump(census.deselected, attributed.reason); bump(reasons, attributed.reason); } + else census.unattributed.push(`${key} :: ${tc.name || '(no name)'} (${tc.outcome})`); + } + const records = recordsByShard.get(key) ?? []; + for (const [id, cases] of byCase) { + const kind = (E2E_KINDS[id] ?? 'rule') as EvalCaseKind; + const failed = cases.find(tc => tc.outcome === 'failed'); + const caseOutcome: TrialOutcome = failed ? 'failed' : cases.some(tc => tc.outcome === 'passed') ? 'passed' : 'skipped'; + const mine = records.filter((r: any) => r?.name === id || r?.case_id === id); + const failedRecord = mine.find((r: any) => r.passed === false); + const failureClass: TrialFailureClass | undefined = !failed ? undefined + : failed.failureType === 'TimeoutError' ? 'timeout' : failedRecord ? failureClassOf(failedRecord) : 'assertion'; + const error = sanitizeTrialError(failedRecord?.error ?? failed?.message); + const rerun = Object.hasOwn(E2E_TIERS, id) ? rerunCommand(manifest.tier, id, 1) : `EVALS=1 EVALS_TIER=${manifest.tier} bun test ${shardFile(key)}`; + if (failed) census.failedShards.add(key); + if (caseOutcome === 'skipped') { + bump(reasons, RUNTIME_SKIP_REASON); + census.skipped.push(`◌ ${id} ${key} (${cases.map(tc => tc.name).join('; ')}): ${RUNTIME_SKIP_REASON}`); + } + census.ruleCases.push({ id, kind, outcome: caseOutcome, + ...(failed ? { line: `✗ ${id} ${kind} FAIL ${failureClass}${failedRecord?.exit_reason === 'timeout' && failedRecord?.timeout_at_turn !== undefined ? ` at turn ${failedRecord.timeout_at_turn}` : ''}${error ? ` — ${error}` : ''} [slice ${result.sliceIndex}, attempt ${result.attempt ?? 1}] rerun: ${rerun}` } : {}) }); + census.history.push({ ...base, case: id, file: shardFile(key), kind, trial: 1, panel: { n: 1, k: 1 }, outcome: caseOutcome, + ...(failureClass ? { failure_class: failureClass } : {}), ...(failedRecord?.exit_reason ? { exit_reason: String(failedRecord.exit_reason) } : {}), + ...(error && failed ? { error } : {}), duration_ms: cases.reduce((sum, tc) => sum + tc.timeMs, 0), + cost_usd: Math.round(mine.reduce((sum: number, r: any) => sum + (Number(r.cost_usd) || 0), 0) * 100) / 100, + ...(typeof mine[0]?.model === 'string' ? { model: mine[0].model } : {}), ...(cliVersion ? { cli_version: cliVersion } : {}), + quarantined: false, execution: outcome.reused ? 'reused' : 'executed', source: 'junit' } as TrialOutcomeRecord); + } + census.shardSkipReasons.set(key, reasons); + } + } + return census; +} + +/** The census block of the readout: every skipped case with its reason, deselections by reason, every unattributed testcase by name. */ +export function formatJUnitCensus(census: JUnitCensus): string[] { + const deselected = [...census.deselected.entries()].sort(([a, x], [b, y]) => y - x || (a < b ? -1 : 1)); + return [ + ...(census.skipped.length ? [`SKIPPED cases (${census.skipped.length}):`, ...census.skipped.map(line => ` ${line}`)] : []), + ...(deselected.length ? [`deselected testcases, zero credit (${deselected.reduce((sum, [, n]) => sum + n, 0)}):`, + ...deselected.map(([reason, n]) => ` ${String(n).padStart(4)} ${reason}`)] : []), + ...(census.unattributed.length ? [`UNATTRIBUTED testcases (${census.unattributed.length}; no registry id, label, single registered id or keyless file):`, + ...census.unattributed.map(line => ` ? ${line}`)] : []), + ]; +} + interface ReportArtifact { root: string; result: SliceResult } /** Every slice result under the report dir: flat (merged) or one directory per attempt-scoped artifact. */ @@ -206,6 +343,7 @@ export function panelReports(manifest: PaidRunManifest, results: SliceResult[], /** Why one failed trial failed, in one line: a case timeout names its turn. */ function trialCause(trial: PanelVerdict['trials'][number] & { timeout_at_turn?: number }): string { + if (trial.outcome === 'skipped') return `t${trial.trial}: skipped`; const cls = trial.failure_class ?? 'assertion'; const head = cls === 'timeout' || trial.exit_reason === 'timeout' ? `timeout${trial.timeout_at_turn !== undefined ? ` at turn ${trial.timeout_at_turn}` : ''}` @@ -224,7 +362,7 @@ export function formatPanelLine(panel: PanelReport, tier: PaidTier): string { const causes = panel.trials.filter(t => t.outcome !== 'passed').map(t => trialCause(t as any)); const where = Object.entries(panel.slices).map(([trial, slice]) => `t${trial}@slice ${slice}`).join(', '); return `${mark} ${panel.case} ${panel.kind}${panel.quarantined ? ' (quarantined)' : ''} ${label} (${panel.marks})` - + `${causes.length ? ` ${causes.join('; ')}` : ''}${panel.status === 'INCOMPLETE' ? ` [${panel.reason}]` : ''}` + + `${causes.length ? ` ${causes.join('; ')}` : ''}${panel.status === 'INCOMPLETE' || panel.status === 'SKIPPED' ? ` [${panel.reason}]` : ''}` + `${where ? ` [${where}, attempt ${panel.attempt}]` : ''} rerun: ${rerunCommand(tier, panel.case, panel.panel.n)}`; } @@ -384,13 +522,6 @@ export function runPaidReport(reportDir: string, options: { writeDurations?: boo console.log(`[test:paid] report: ⚠ ${flaky.length} cases with multiple attempts this run: (paid evals never retry; each record counts)`); for (const f of flaky) console.log(` ⚠ ${f.name} (x${f.attempts}) — ${f.file}`); } - const allSkipped = results.flatMap((r) => r.outcomes.filter(isAllSkippedPass)); - if (allSkipped.length > 0) { - console.log(`[test:paid] report: ⚠ ${allSkipped.length} shard(s) passed with EVERY test skipped — they verified nothing:`); - for (const outcome of allSkipped) { - console.log(` ⚠ ${outcome.files.join(' ')} (${outcome.executedTests} skipped — external service missing or tier mismatch)`); - } - } if (manualProblems.length) verdict.problems.push(...manualProblems); if (evidence.failed > 0) verdict.problems.push(`${evidence.failed} unapproved final collector failure(s)`); if (files.reduce((sum, file) => sum + file.total, 0) !== evidence.passed + evidence.failed + evidence.manual_accepted @@ -450,38 +581,22 @@ export function runPaidReport(reportDir: string, options: { writeDurations?: boo quarantined: t.quarantined, execution: outcome.reused ? 'reused' : 'executed', source: 'shard' } as TrialOutcomeRecord); } } - const ruleCases: Array<{ id: string; kind: EvalCaseKind; outcome: TrialOutcome; line?: string }> = []; - const junitFailedShards = new Set(); - let unattributed = 0; - const cliVersion = env.GSTACK_CLAUDE_CLI_VERSION; - for (const { root, result } of artifacts.filter(a => (a.result.attempt ?? 1) === primary)) { - for (const outcome of result.outcomes) { - const key = normalizeRelativePath(outcome.files[0] ?? ''); - if (outcome.trial || outcome.files.length !== 1) continue; - let xml = ''; - try { xml = fs.readFileSync(path.join(root, 'shards', shardSlug([key]), 'junit.xml'), 'utf8'); } catch { continue; } - const records = recordsByShard.get(key) ?? []; - for (const tc of parseJUnitCases(xml)) { - const id = caseIdForTestName(tc.name); - if (id === null) { unattributed++; continue; } - const kind = (E2E_KINDS[id] ?? 'rule') as EvalCaseKind; - const mine = records.filter((r: any) => r?.name === id || r?.case_id === id); - const failedRecord = mine.find((r: any) => r.passed === false); - const failureClass: TrialFailureClass | undefined = tc.outcome !== 'failed' ? undefined - : tc.failureType === 'TimeoutError' ? 'timeout' : failedRecord ? failureClassOf(failedRecord) : 'assertion'; - const error = sanitizeTrialError(failedRecord?.error ?? tc.message); - if (tc.outcome === 'failed') junitFailedShards.add(key); - ruleCases.push({ id, kind, outcome: tc.outcome, - ...(tc.outcome === 'failed' ? { line: `✗ ${id} ${kind} FAIL ${failureClass}${failedRecord?.exit_reason === 'timeout' && failedRecord?.timeout_at_turn !== undefined ? ` at turn ${failedRecord.timeout_at_turn}` : ''}${error ? ` — ${error}` : ''} [slice ${result.sliceIndex}, attempt ${primary}] rerun: ${rerunCommand(manifest.tier, id, 1)}` } : {}) }); - history.push({ ...common(primary), case: id, file: shardFile(key), kind, trial: 1, panel: { n: 1, k: 1 }, outcome: tc.outcome, - ...(failureClass ? { failure_class: failureClass } : {}), ...(failedRecord?.exit_reason ? { exit_reason: String(failedRecord.exit_reason) } : {}), - ...(error && tc.outcome === 'failed' ? { error } : {}), duration_ms: tc.timeMs, - cost_usd: Math.round(mine.reduce((sum: number, r: any) => sum + (Number(r.cost_usd) || 0), 0) * 100) / 100, - ...(typeof mine[0]?.model === 'string' ? { model: mine[0].model } : {}), ...(cliVersion ? { cli_version: cliVersion } : {}), - quarantined: false, execution: outcome.reused ? 'reused' : 'executed', source: 'junit' } as TrialOutcomeRecord); - } + const census = junitCensus(manifest, artifacts.filter(a => (a.result.attempt ?? 1) === primary), recordsByShard, + common(primary), env.GSTACK_CLAUDE_CLI_VERSION); + history.push(...census.history); + const { ruleCases } = census; + const allSkipped = results.flatMap((r) => r.outcomes.filter(isAllSkippedPass)); + const hollowWithoutJUnit = allSkipped.filter(outcome => !census.shardSkipReasons.has(normalizeRelativePath(outcome.files[0] ?? ''))); + if (allSkipped.length > 0) { + console.log(`[test:paid] report: ⚠ ${allSkipped.length} shard(s) passed with EVERY test skipped — they verified nothing:`); + for (const outcome of allSkipped) { + const reasons = census.shardSkipReasons.get(normalizeRelativePath(outcome.files[0] ?? '')); + console.log(` ⚠ ${outcome.files.join(' ')} (${outcome.executedTests} skipped — ${reasons?.size + ? [...reasons.entries()].map(([reason, n]) => `${n} ${reason}`).join('; ') : 'no JUnit report, reason unknown'})`); } } + const censusLines = formatJUnitCensus(census); + for (const line of censusLines) console.log(`[test:paid] ${line}`); // series_identity is stamped afterwards by scripts/eval-trial-series.ts (the report job's next step). fs.writeFileSync(trialOutcomesPath, formatTrialOutcomes(history)); @@ -489,7 +604,7 @@ export function runPaidReport(reportDir: string, options: { writeDurations?: boo const ruleShardFailures = manifest.entries.filter(entry => entry.status === 'planned' && !entry.trial).flatMap(entry => { const got = results.flatMap(r => r.outcomes.map(o => ({ o, slice: r.sliceIndex }))).find(({ o }) => normalizeRelativePath(o.files[0] ?? '') === normalizeRelativePath(entry.file)); if (got && got.o.status === 'passed') return []; - if (got && junitFailedShards.has(normalizeRelativePath(entry.file))) return []; + if (got && census.failedShards.has(normalizeRelativePath(entry.file))) return []; const id = shardCaseId(entry.file); return [`✗ ${entry.file} rule shard ${got ? got.o.status : 'NOT REPORTED'}${got?.o.runnerError ? ` — ${sanitizeTrialError(got.o.runnerError)}` : ''} [slice ${entry.slice}, attempt ${primary}]${id ? ` rerun: ${rerunCommand(manifest.tier, id, 1)}` : ''}`]; }); @@ -515,11 +630,11 @@ export function runPaidReport(reportDir: string, options: { writeDurations?: boo behavior: { ...count('behavior'), split: behaviorPanels.filter(p => p.split).length }, judge: count('judge'), quarantined: { total: panels.filter(p => p.quarantined).length, failingLane: panels.filter(p => p.quarantined && p.failsLane).length }, - skipped: allSkipped.length + panels.filter(p => p.status === 'SKIPPED').length + ruleCases.filter(c => c.outcome === 'skipped').length, + skipped: hollowWithoutJUnit.length + panels.filter(p => p.status === 'SKIPPED').length + ruleCases.filter(c => c.outcome === 'skipped').length, infra: primaryTrials.filter(t => t.failure_class === 'infra').length + results.flatMap(r => r.outcomes).filter(o => !o.trial && o.runnerError !== undefined).length, incomplete: panels.filter(p => p.status === 'INCOMPLETE').length, - unattributed, + unattributed: census.unattributed.length, }, actionRequired: verdict.problems.length, wallMs: wall.length ? Math.max(...wall.map(r => r.finishedAt!)) - Math.min(...wall.map(r => r.startedAt!)) : null, @@ -537,12 +652,14 @@ export function runPaidReport(reportDir: string, options: { writeDurations?: boo fs.writeFileSync(summaryMdPath, [ ...fence(headlineLines), ...(failureLines.length ? ['', '**Failures and split verdicts**', '', ...fence(failureLines)] : []), + ...(censusLines.length ? ['', '**Skipped, deselected and unattributed testcases**', '', ...fence(censusLines)] : []), ...(verdict.problems.length ? ['', `**ACTION REQUIRED (${verdict.problems.length})**`, '', ...fence(verdict.problems.map(p => sanitizeTrialError(p) ?? p))] : []), ].join('\n') + '\n'); if (!manualProblems.length) fs.writeFileSync(summaryPath, JSON.stringify({ version: 2, files, totals: { ...evidence, total: evidence.passed + evidence.failed + evidence.manual_accepted, flaky: files.reduce((sum, file) => sum + file.flaky, 0), }, verdict: headline, headline: headlineLines, + census: { skipped: census.skipped, deselected: Object.fromEntries(census.deselected), unattributed: census.unattributed }, panels: panels.map(p => ({ case: p.case, kind: p.kind, status: p.status, passed: p.passed, n: p.panel.n, k: p.panel.k, marks: p.marks, split: p.split, quarantined: p.quarantined, failsLane: p.failsLane, redClass: p.redClass, reason: p.reason, trials: p.trials.map(t => ({ trial: t.trial, outcome: t.outcome, ...(t.failure_class ? { failure_class: t.failure_class } : {}), diff --git a/scripts/test-paid-shards.ts b/scripts/test-paid-shards.ts index bc92b31ab..5fe708eb6 100644 --- a/scripts/test-paid-shards.ts +++ b/scripts/test-paid-shards.ts @@ -209,14 +209,15 @@ export function classifyPaidTestFile(source: string, tier: PaidTier): TierClassi * The E2E ids a paid file registers: the touchfile registrations that list the * file. `known` is true only when those ids are complete: no computed * registration (testName, *IfSelected, describeIfSelected with a non-literal - * argument) and every literal registration argument is among them. Quoted - * strings elsewhere (comments, skill paths) never count. + * argument) and every literal registration argument is among them + * (`unregistered` lists the literal ids that are not). Quoted strings + * elsewhere (comments, skill paths) never count. */ export function fileCaseRegistration( file: string, source: string, touchfiles: Record = E2E_TOUCHFILES, tiers: Record = E2E_TIERS, -): { registered: string[]; known: boolean } { +): { registered: string[]; known: boolean; computed: boolean; unregistered: string[] } { const rel = normalizeRelativePath(file); const registered = Object.keys(touchfiles).filter(key => touchfiles[key]!.includes(rel)); const computed = /testName\s*:\s*(?!string\b)(?:`[^`]*\$\{|[A-Za-z_$])/.test(source) @@ -230,23 +231,30 @@ export function fileCaseRegistration( ...[...source.matchAll(/\bdescribeIfSelected\s*\([^,]*,\s*\[([^\]]*)\]/g)] .flatMap(m => [...m[1]!.matchAll(/(['"`])([^'"`]+)\1/g)].map(n => n[2]!)), ].filter(id => id in tiers); - return { registered, known: registered.length > 0 && !computed && literal.every(id => registered.includes(id)) }; + const unregistered = [...new Set(literal.filter(id => !registered.includes(id)))]; + return { registered, known: registered.length > 0 && !computed && unregistered.length === 0, computed, unregistered }; } /** - * A file is skipped for a tier lane only when its registered E2E ids are fully - * known (fileCaseRegistration) and none of them has that tier. Any computed - * registration, an id missing from the file's touchfile registration, or no id at - * all keeps today's scheduling (the child's runtime filter decides). + * A file is skipped for a tier lane only when it registers E2E ids, none of + * them has that tier, and no literal registration names an id outside its + * touchfile registration. A computed registration (a name built at runtime) + * can only run in this lane by producing an id of this tier (the child's + * EVALS_TIER filter drops every other), so it is skipped too when every id of + * the tier is registered to a test file (its touchfile list names one), which + * is not this file. An id of the tier that no test file registers, or no id + * at all, keeps today's scheduling (the child's runtime filter decides). */ export function tierSkipReason( file: string, source: string, tier: PaidTier, touchfiles: Record = E2E_TOUCHFILES, tiers: Record = E2E_TIERS, ): string | null { - const { registered, known } = fileCaseRegistration(file, source, touchfiles, tiers); - if (!known || registered.some(id => tiers[id] === tier)) return null; - return `skipped: no E2E_TIERS id has tier ${tier}`; + const { registered, computed, unregistered } = fileCaseRegistration(file, source, touchfiles, tiers); + if (registered.length === 0 || unregistered.length > 0 || registered.some(id => tiers[id] === tier)) return null; + if (!computed) return `skipped: no E2E_TIERS id has tier ${tier}`; + const unowned = Object.keys(tiers).some(id => tiers[id] === tier && !(touchfiles[id] ?? []).some(dep => /^test\/[^/]+\.test\.ts$/.test(dep))); + return unowned ? null : `skipped: no E2E_TIERS id has tier ${tier} (its computed names can only produce ${tier} ids other test files register)`; } /** diff --git a/test/cookie-validation-phases.test.ts b/test/cookie-validation-phases.test.ts index 0d68b33ff..cc0653d88 100644 --- a/test/cookie-validation-phases.test.ts +++ b/test/cookie-validation-phases.test.ts @@ -44,10 +44,11 @@ test('the existing quality and behavior phases retain their complete separate sh expect(quality.evalsAll).toBe(true); expect(behavior.evalsAll).toBe(true); expect(qualityFiles).toHaveLength(1); - // 45 files (first-task-scaffold registers no gate case, so the gate lane - // skips it); the seven case-sharded files contribute one shard per gate case. - expect(new Set(behaviorFiles.map(file => file.split('#')[0])).size).toBe(45); - expect(behaviorFiles).toHaveLength(72); + // 41 files (first-task-scaffold registers no gate case, and four files whose + // computed names hold only periodic cases, so the gate lane skips them); the + // seven case-sharded files contribute one shard per gate case. + expect(new Set(behaviorFiles.map(file => file.split('#')[0])).size).toBe(41); + expect(behaviorFiles).toHaveLength(68); expect(behaviorFiles).toEqual(expect.arrayContaining([ ...['review-exploratory-small-cli', 'ship-exploratory-small-cli', 'ship-exploratory-unavailable', 'ship-exploratory-plan-checks', 'ship-exploratory-late-input'].map(id => `test/skill-e2e-qa-callers.test.ts#${id}`), diff --git a/test/fixtures/paid-report-census/gate/manifest.json b/test/fixtures/paid-report-census/gate/manifest.json new file mode 100644 index 000000000..952504408 --- /dev/null +++ b/test/fixtures/paid-report-census/gate/manifest.json @@ -0,0 +1,43 @@ +{ + "version": 1, + "tier": "gate", + "evalsAll": true, + "sliceCount": 1, + "selectionReason": "run-all", + "profile": "full", + "selection": { + "e2e": null, + "judges": null + }, + "entries": [ + { + "file": "test/skill-e2e-plan-mode-no-op.test.ts", + "slice": 1, + "status": "planned", + "estimatedMs": 217184, + "budget": { + "timeoutMs": 3120000, + "source": "registered", + "policyId": "skill-e2e-plan-mode-no-op-existing-retry-v1" + } + }, + { + "file": "test/skill-e2e-cso.test.ts", + "slice": 1, + "status": "planned", + "estimatedMs": 227394 + }, + { + "file": "test/skill-routing-e2e.test.ts", + "slice": 1, + "status": "planned", + "estimatedMs": 1000 + }, + { + "file": "test/skill-e2e-qa-bugs.test.ts", + "slice": 1, + "status": "planned", + "estimatedMs": 1000 + } + ] +} diff --git a/test/fixtures/paid-report-census/gate/paid-slice-1-a1/shards/skill-e2e-cso/junit.xml b/test/fixtures/paid-report-census/gate/paid-slice-1-a1/shards/skill-e2e-cso/junit.xml new file mode 100644 index 000000000..8bc3131c0 --- /dev/null +++ b/test/fixtures/paid-report-census/gate/paid-slice-1-a1/shards/skill-e2e-cso/junit.xml @@ -0,0 +1,18 @@ + + + + + + + + + + + + + + + + + + diff --git a/test/fixtures/paid-report-census/gate/paid-slice-1-a1/shards/skill-e2e-plan-mode-no-op/junit.xml b/test/fixtures/paid-report-census/gate/paid-slice-1-a1/shards/skill-e2e-plan-mode-no-op/junit.xml new file mode 100644 index 000000000..36c70261e --- /dev/null +++ b/test/fixtures/paid-report-census/gate/paid-slice-1-a1/shards/skill-e2e-plan-mode-no-op/junit.xml @@ -0,0 +1,12 @@ + + + + + + + + + + + + diff --git a/test/fixtures/paid-report-census/gate/paid-slice-1-a1/shards/skill-e2e-qa-bugs/junit.xml b/test/fixtures/paid-report-census/gate/paid-slice-1-a1/shards/skill-e2e-qa-bugs/junit.xml new file mode 100644 index 000000000..0fe282c7c --- /dev/null +++ b/test/fixtures/paid-report-census/gate/paid-slice-1-a1/shards/skill-e2e-qa-bugs/junit.xml @@ -0,0 +1,22 @@ + + + + + + + + + + + + + + + + + + + + + + diff --git a/test/fixtures/paid-report-census/gate/paid-slice-1-a1/shards/skill-routing-e2e/junit.xml b/test/fixtures/paid-report-census/gate/paid-slice-1-a1/shards/skill-routing-e2e/junit.xml new file mode 100644 index 000000000..03f2e78c7 --- /dev/null +++ b/test/fixtures/paid-report-census/gate/paid-slice-1-a1/shards/skill-routing-e2e/junit.xml @@ -0,0 +1,40 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/test/fixtures/paid-report-census/gate/paid-slice-1-a1/slice-1.json b/test/fixtures/paid-report-census/gate/paid-slice-1-a1/slice-1.json new file mode 100644 index 000000000..fcf88141d --- /dev/null +++ b/test/fixtures/paid-report-census/gate/paid-slice-1-a1/slice-1.json @@ -0,0 +1,74 @@ +{ + "version": 1, + "tier": "gate", + "profile": "full", + "selection": { + "e2e": null, + "judges": null + }, + "sliceIndex": 1, + "sliceCount": 1, + "attempt": 1, + "outcomes": [ + { + "files": [ + "test/skill-e2e-plan-mode-no-op.test.ts" + ], + "status": "passed", + "exitCode": 0, + "elapsedMs": 176495, + "executedTests": 5, + "skippedTests": 0, + "budget": { + "timeoutMs": 3120000, + "source": "registered", + "policyId": "skill-e2e-plan-mode-no-op-existing-retry-v1" + } + }, + { + "files": [ + "test/skill-e2e-cso.test.ts" + ], + "status": "passed", + "exitCode": 0, + "elapsedMs": 205014, + "executedTests": 3, + "skippedTests": 2, + "budget": { + "timeoutMs": 1800000, + "source": "default", + "policyId": null + } + }, + { + "files": [ + "test/skill-routing-e2e.test.ts" + ], + "status": "passed", + "exitCode": 0, + "elapsedMs": 27, + "executedTests": 11, + "skippedTests": 11, + "budget": { + "timeoutMs": 1800000, + "source": "default", + "policyId": null + } + }, + { + "files": [ + "test/skill-e2e-qa-bugs.test.ts" + ], + "status": "passed", + "exitCode": 0, + "elapsedMs": 31, + "executedTests": 5, + "skippedTests": 5, + "budget": { + "timeoutMs": 1800000, + "source": "default", + "policyId": null + } + } + ] +} diff --git a/test/fixtures/paid-report-census/periodic/manifest.json b/test/fixtures/paid-report-census/periodic/manifest.json new file mode 100644 index 000000000..64c74c295 --- /dev/null +++ b/test/fixtures/paid-report-census/periodic/manifest.json @@ -0,0 +1,56 @@ +{ + "version": 1, + "tier": "periodic", + "evalsAll": true, + "sliceCount": 1, + "selectionReason": "run-all", + "profile": "full", + "selection": { + "e2e": null, + "judges": null + }, + "entries": [ + { + "file": "test/skill-e2e-qa-workflow.test.ts", + "slice": 1, + "status": "planned", + "estimatedMs": 403953 + }, + { + "file": "test/skill-e2e-design.test.ts#design-consultation-core", + "slice": 1, + "status": "planned", + "estimatedMs": 159257 + }, + { + "file": "test/skill-e2e-benchmark-providers.test.ts", + "slice": 1, + "status": "planned", + "estimatedMs": 9495 + }, + { + "file": "test/carve-section-loading-review.test.ts", + "slice": 1, + "status": "planned", + "estimatedMs": 247902 + }, + { + "file": "test/skill-e2e-cso.test.ts", + "slice": 1, + "status": "planned", + "estimatedMs": 253498 + }, + { + "file": "test/codex-e2e-recommendation-substance.test.ts", + "slice": 1, + "status": "planned", + "estimatedMs": 1000 + }, + { + "file": "test/skill-e2e-test-value.test.ts", + "slice": 1, + "status": "planned", + "estimatedMs": 332000 + } + ] +} diff --git a/test/fixtures/paid-report-census/periodic/paid-slice-1-a1/shards/carve-section-loading-review/junit.xml b/test/fixtures/paid-report-census/periodic/paid-slice-1-a1/shards/carve-section-loading-review/junit.xml new file mode 100644 index 000000000..0dbf3530a --- /dev/null +++ b/test/fixtures/paid-report-census/periodic/paid-slice-1-a1/shards/carve-section-loading-review/junit.xml @@ -0,0 +1,8 @@ + + + + + + + + diff --git a/test/fixtures/paid-report-census/periodic/paid-slice-1-a1/shards/codex-e2e-recommendation-substance/junit.xml b/test/fixtures/paid-report-census/periodic/paid-slice-1-a1/shards/codex-e2e-recommendation-substance/junit.xml new file mode 100644 index 000000000..ad1b24d6e --- /dev/null +++ b/test/fixtures/paid-report-census/periodic/paid-slice-1-a1/shards/codex-e2e-recommendation-substance/junit.xml @@ -0,0 +1,8 @@ + + + + + + + + diff --git a/test/fixtures/paid-report-census/periodic/paid-slice-1-a1/shards/skill-e2e-benchmark-providers/junit.xml b/test/fixtures/paid-report-census/periodic/paid-slice-1-a1/shards/skill-e2e-benchmark-providers/junit.xml new file mode 100644 index 000000000..8762d2a07 --- /dev/null +++ b/test/fixtures/paid-report-census/periodic/paid-slice-1-a1/shards/skill-e2e-benchmark-providers/junit.xml @@ -0,0 +1,15 @@ + + + + + + + + + + + + + + + diff --git a/test/fixtures/paid-report-census/periodic/paid-slice-1-a1/shards/skill-e2e-cso/junit.xml b/test/fixtures/paid-report-census/periodic/paid-slice-1-a1/shards/skill-e2e-cso/junit.xml new file mode 100644 index 000000000..d8e8c1ba9 --- /dev/null +++ b/test/fixtures/paid-report-census/periodic/paid-slice-1-a1/shards/skill-e2e-cso/junit.xml @@ -0,0 +1,16 @@ + + + + + + + + + + + + + + + + diff --git a/test/fixtures/paid-report-census/periodic/paid-slice-1-a1/shards/skill-e2e-design--design-consultation-core/junit.xml b/test/fixtures/paid-report-census/periodic/paid-slice-1-a1/shards/skill-e2e-design--design-consultation-core/junit.xml new file mode 100644 index 000000000..7168ee989 --- /dev/null +++ b/test/fixtures/paid-report-census/periodic/paid-slice-1-a1/shards/skill-e2e-design--design-consultation-core/junit.xml @@ -0,0 +1,48 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/test/fixtures/paid-report-census/periodic/paid-slice-1-a1/shards/skill-e2e-qa-workflow/junit.xml b/test/fixtures/paid-report-census/periodic/paid-slice-1-a1/shards/skill-e2e-qa-workflow/junit.xml new file mode 100644 index 000000000..2dfa4437d --- /dev/null +++ b/test/fixtures/paid-report-census/periodic/paid-slice-1-a1/shards/skill-e2e-qa-workflow/junit.xml @@ -0,0 +1,35 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/test/fixtures/paid-report-census/periodic/paid-slice-1-a1/shards/skill-e2e-test-value/junit.xml b/test/fixtures/paid-report-census/periodic/paid-slice-1-a1/shards/skill-e2e-test-value/junit.xml new file mode 100644 index 000000000..0e496c74c --- /dev/null +++ b/test/fixtures/paid-report-census/periodic/paid-slice-1-a1/shards/skill-e2e-test-value/junit.xml @@ -0,0 +1,20 @@ + + + + + + + + + + + + + + + + + + + + diff --git a/test/fixtures/paid-report-census/periodic/paid-slice-1-a1/slice-1.json b/test/fixtures/paid-report-census/periodic/paid-slice-1-a1/slice-1.json new file mode 100644 index 000000000..a45af4070 --- /dev/null +++ b/test/fixtures/paid-report-census/periodic/paid-slice-1-a1/slice-1.json @@ -0,0 +1,119 @@ +{ + "version": 1, + "tier": "periodic", + "profile": "full", + "selection": { + "e2e": null, + "judges": null + }, + "sliceIndex": 1, + "sliceCount": 1, + "attempt": 1, + "outcomes": [ + { + "files": [ + "test/skill-e2e-qa-workflow.test.ts" + ], + "status": "passed", + "exitCode": 0, + "elapsedMs": 353035, + "executedTests": 8, + "skippedTests": 7, + "budget": { + "timeoutMs": 1800000, + "source": "default", + "policyId": null + } + }, + { + "files": [ + "test/skill-e2e-design.test.ts#design-consultation-core" + ], + "status": "passed", + "exitCode": 0, + "elapsedMs": 176652, + "executedTests": 4, + "skippedTests": 3, + "budget": { + "timeoutMs": 1800000, + "source": "default", + "policyId": null + } + }, + { + "files": [ + "test/skill-e2e-benchmark-providers.test.ts" + ], + "status": "passed", + "exitCode": 0, + "elapsedMs": 8552, + "executedTests": 8, + "skippedTests": 0, + "budget": { + "timeoutMs": 1800000, + "source": "default", + "policyId": null + } + }, + { + "files": [ + "test/carve-section-loading-review.test.ts" + ], + "status": "passed", + "exitCode": 0, + "elapsedMs": 143440, + "executedTests": 1, + "skippedTests": 0, + "budget": { + "timeoutMs": 1800000, + "source": "default", + "policyId": null + } + }, + { + "files": [ + "test/skill-e2e-cso.test.ts" + ], + "status": "passed", + "exitCode": 0, + "elapsedMs": 293156, + "executedTests": 3, + "skippedTests": 1, + "budget": { + "timeoutMs": 1800000, + "source": "default", + "policyId": null + } + }, + { + "files": [ + "test/codex-e2e-recommendation-substance.test.ts" + ], + "status": "passed", + "exitCode": 0, + "elapsedMs": 13090, + "executedTests": 1, + "skippedTests": 0, + "budget": { + "timeoutMs": 1800000, + "source": "default", + "policyId": null + } + }, + { + "files": [ + "test/skill-e2e-test-value.test.ts" + ], + "status": "passed", + "exitCode": 0, + "elapsedMs": 50, + "executedTests": 3, + "skippedTests": 3, + "budget": { + "timeoutMs": 1800000, + "source": "default", + "policyId": null + } + } + ] +} diff --git a/test/paid-report-census.test.ts b/test/paid-report-census.test.ts new file mode 100644 index 000000000..b57a0ae74 --- /dev/null +++ b/test/paid-report-census.test.ts @@ -0,0 +1,125 @@ +/** + * Census attribution in the paid report (I4): every JUnit testcase of a + * non-trial shard is a case result, a zero-credit deselection with its + * reason, or listed by name as unattributed. Fixtures are trimmed from the + * 2026-10-03 periodic and gate census artifacts (run 37151477069, commit + * ac20ef1), whose report counted deselections as SKIPPED 82/282 and left + * 72/24 testcases unattributed. + */ +import { afterAll, beforeAll, describe, expect, test } from 'bun:test'; +import * as fs from 'node:fs'; +import * as os from 'node:os'; +import * as path from 'node:path'; +import { DESELECTION_REASONS, RUNTIME_SKIP_REASON, attributeJUnitCase, formatPanelLine, runPaidReport, type JUnitCase } from '../scripts/test-paid-shards'; +import { panelVerdict } from './helpers/eval-store'; + +const FIXTURES = path.join(import.meta.dir, 'fixtures', 'paid-report-census'); + +let base: string; +beforeAll(() => { base = fs.mkdtempSync(path.join(os.tmpdir(), 'paid-report-census-')); }); +afterAll(() => { fs.rmSync(base, { recursive: true, force: true }); }); + +let runs = 0; +function report(lane: 'periodic' | 'gate', edit: (dir: string) => void = () => {}) { + const dir = path.join(base, `${lane}-${++runs}`); + fs.cpSync(path.join(FIXTURES, lane), dir, { recursive: true }); + edit(dir); + const lines: string[] = []; + const { log, error } = console; + console.log = (...args: unknown[]) => { lines.push(args.join(' ')); }; + console.error = console.log; + let status: number; + try { status = runPaidReport(dir, { env: {} }); } finally { console.log = log; console.error = error; } + const history = fs.readFileSync(path.join(dir, 'trial-outcomes.jsonl'), 'utf8').split('\n').filter(Boolean).map(line => JSON.parse(line)); + return { status, out: lines.join('\n'), dir, history }; +} + +const junit = (dir: string, slug: string) => path.join(dir, 'paid-slice-1-a1', 'shards', slug, 'junit.xml'); + +describe('census attribution from the 2026-10-03 artifacts', () => { + test('periodic: deselections and hook placeholders are zero-credit, every executed testcase maps to one case', () => { + const r = report('periodic'); + expect(r.status, r.out).toBe(0); + expect(r.out).toContain(' rule 7/7 · behavior 0/0 · judge 0/0 · quarantined 0 (0 failing the lane)'); + expect(r.out).toContain(' SKIPPED 0 · INFRA 0 · INCOMPLETE 0 · unattributed 0 · ACTION REQUIRED 0'); + expect(r.out).toContain(`deselected testcases, zero credit (21):`); + expect(r.out).toContain(` 10 ${DESELECTION_REASONS.sibling}`); + expect(r.out).toContain(' 7 gate-tier case (this lane runs periodic)'); + expect(r.out).toContain(` 4 ${DESELECTION_REASONS.hook}`); + expect(r.out).toContain(' ⚠ test/skill-e2e-test-value.test.ts (3 skipped — 3 gate-tier case (this lane runs periodic))'); + expect(r.out).not.toContain('UNATTRIBUTED'); + expect(r.history.map(record => `${record.case}:${record.outcome}`).sort()).toEqual([ + 'benchmark-providers-live:passed', 'carve-section-loading:passed', 'cso-full-audit:passed', 'cso-infra-scope:passed', + 'design-consultation-core:passed', 'qa-fix-loop:passed', 'test/codex-e2e-recommendation-substance.test.ts:passed', + ]); + expect(r.history.find(record => record.case === 'benchmark-providers-live')).toMatchObject({ file: 'test/skill-e2e-benchmark-providers.test.ts', source: 'junit' }); + expect(r.history.filter(record => record.file === 'test/skill-e2e-design.test.ts')) + .toEqual([expect.objectContaining({ case: 'design-consultation-core', outcome: 'passed', source: 'junit' })]); + expect(r.history.some(record => record.outcome === 'skipped')).toBe(false); + }); + + test('gate: other-tier cases and hook placeholders are deselected; a multi-test case is one result', () => { + const r = report('gate'); + expect(r.status, r.out).toBe(0); + expect(r.out).toContain(' rule 2/2 · behavior 0/0 · judge 0/0 · quarantined 0 (0 failing the lane)'); + expect(r.out).toContain(' SKIPPED 0 · INFRA 0 · INCOMPLETE 0 · unattributed 0 · ACTION REQUIRED 0'); + expect(r.out).toContain(' 16 periodic-tier case (this lane runs gate)'); + expect(r.out).toContain(` 2 ${DESELECTION_REASONS.hook}`); + expect(r.out).toContain(` ⚠ test/skill-e2e-qa-bugs.test.ts (5 skipped — 2 ${DESELECTION_REASONS.hook}; 3 periodic-tier case (this lane runs gate))`); + expect(r.history.map(record => `${record.case}:${record.outcome}`).sort()).toEqual(['cso-diff-mode:passed', 'plan-mode-no-op:passed']); + }); + + test('a selected in-lane case that skipped is SKIPPED with its reason; an unattributable testcase is listed by name', () => { + const r = report('periodic', dir => { + const qa = junit(dir, 'skill-e2e-qa-workflow'); + fs.writeFileSync(qa, fs.readFileSync(qa, 'utf8').replace(/(]*?) \/>/, '$1>')); + const cso = junit(dir, 'skill-e2e-cso'); + fs.writeFileSync(cso, fs.readFileSync(cso, 'utf8').replace('\n', + '\n\n')); + }); + expect(r.status, r.out).toBe(0); + expect(r.out).toContain(' SKIPPED 1 · INFRA 0 · INCOMPLETE 0 · unattributed 1 · ACTION REQUIRED 0'); + expect(r.out).toContain(`[test:paid] ◌ qa-fix-loop test/skill-e2e-qa-workflow.test.ts (qa-fix-loop): ${RUNTIME_SKIP_REASON}`); + expect(r.out).toContain('[test:paid] ? test/skill-e2e-cso.test.ts :: an unlabeled helper (passed)'); + const summary = fs.readFileSync(path.join(r.dir, 'report-summary.md'), 'utf8'); + expect(summary).toContain('**Skipped, deselected and unattributed testcases**'); + expect(summary).toContain('? test/skill-e2e-cso.test.ts :: an unlabeled helper (passed)'); + const collector = JSON.parse(fs.readFileSync(path.join(r.dir, 'collector-outcomes.json'), 'utf8')); + expect(collector.census.unattributed).toEqual(['test/skill-e2e-cso.test.ts :: an unlabeled helper (passed)']); + expect(collector.census.skipped).toHaveLength(1); + expect(collector.verdict.counts).toMatchObject({ skipped: 1, unattributed: 1 }); + }); +}); + +describe('attributeJUnitCase', () => { + const tc = (name: string, outcome: JUnitCase['outcome'], line: number | null = 1): JUnitCase => + ({ name, classname: 's', outcome, timeMs: 0, ...(line === null ? {} : { line }) }); + const file = 'test/skill-e2e-review.test.ts'; + + test('isolated and unselected in-lane cases are deselected; a selected skip is a case', () => { + expect(attributeJUnitCase(tc('review-design-lite', 'skipped'), { key: file, tier: 'periodic', excludeCases: ['review-design-lite'] })) + .toEqual({ kind: 'deselected', reason: DESELECTION_REASONS.isolated }); + expect(attributeJUnitCase(tc('review-sql-injection', 'skipped'), { key: file, tier: 'gate', selection: { e2e: ['review-enum-completeness'], judges: null } })) + .toEqual({ kind: 'deselected', reason: DESELECTION_REASONS.selection }); + expect(attributeJUnitCase(tc('review-sql-injection', 'skipped'), { key: file, tier: 'gate', selection: { e2e: ['review-sql-injection'], judges: null } })) + .toEqual({ kind: 'case', id: 'review-sql-injection' }); + }); + + test('only a skipped, line-less "(unnamed)" is a hook placeholder; a failing hook or a stray case-shard result is never deselected', () => { + expect(attributeJUnitCase(tc('(unnamed)', 'skipped', null), { key: file, tier: 'gate' })).toEqual({ kind: 'deselected', reason: DESELECTION_REASONS.hook }); + expect(attributeJUnitCase(tc('(unnamed)', 'failed', null), { key: file, tier: 'gate' })).toEqual({ kind: 'unattributed' }); + expect(attributeJUnitCase(tc('(unnamed)', 'skipped'), { key: file, tier: 'gate' })).toEqual({ kind: 'unattributed' }); + const shard = 'test/skill-e2e-design.test.ts#design-consultation-core'; + expect(attributeJUnitCase(tc('design-consultation-core', 'passed'), { key: shard, tier: 'periodic' })).toEqual({ kind: 'case', id: 'design-consultation-core' }); + expect(attributeJUnitCase(tc('needs Aside', 'skipped'), { key: shard, tier: 'periodic' })).toEqual({ kind: 'deselected', reason: DESELECTION_REASONS.sibling }); + expect(attributeJUnitCase(tc('needs Aside', 'passed'), { key: shard, tier: 'periodic' })).toEqual({ kind: 'unattributed' }); + }); + + test('a SKIPPED panel line names its reason and never calls a skipped trial an assertion', () => { + const verdict = panelVerdict({ case: 'review-design-lite', kind: 'behavior', panel: { n: 3, k: 2 }, + trials: [1, 2, 3].map(trial => ({ trial, outcome: 'skipped' as const })), quarantined: false }); + const line = formatPanelLine({ ...verdict, file: 'test/skill-e2e-review.test.ts', slices: {} }, 'periodic'); + expect(line).toContain('t1: skipped; t2: skipped; t3: skipped [every trial skipped (no verdict credit)]'); + expect(line).not.toContain('assertion'); + }); +}); diff --git a/test/paid-retry-supervision.test.ts b/test/paid-retry-supervision.test.ts index e5cec82d4..21109714b 100644 --- a/test/paid-retry-supervision.test.ts +++ b/test/paid-retry-supervision.test.ts @@ -210,7 +210,7 @@ test('detached PR fallback and release commands cover their actual default worke const prFloor = Math.ceil((Math.ceil(fullGateFiles.length / prWorkers) * 1_800_000 + fullGateFiles.reduce( (total, file) => total + Math.max(0, resolvePaidShardBudget([file]).timeoutMs - 1_800_000), 0, )) / 1000 * 1.05); - expect(prFloor).toBe(72_755); + expect(prFloor).toBe(68_975); expect(prWall).toBe(92_820_000); expect(prWall).toBeGreaterThanOrEqual(paidShardWallUpperBoundMs(files, prWorkers) + 120_000); @@ -229,8 +229,8 @@ test('detached PR fallback and release commands cover their actual default worke )) / 1000 * 1.05)); } const detachedReleaseWall = Number(scripts['eval:bg:release'].match(/--timeout (\d+)/)?.[1]) * 1000; - expect(releaseFloors).toEqual([21_725, 35_711]); - expect(releaseFloors.reduce((total, floor) => total + floor, 0)).toBe(57_436); + expect(releaseFloors).toEqual([19_835, 33_821]); + expect(releaseFloors.reduce((total, floor) => total + floor, 0)).toBe(53_656); expect(detachedReleaseWall).toBe(116_700_000); expect(detachedReleaseWall).toBeGreaterThanOrEqual(releaseWall + 120_000); }); diff --git a/test/paid-shards.test.ts b/test/paid-shards.test.ts index e11f0a14d..a6ec1e1b1 100644 --- a/test/paid-shards.test.ts +++ b/test/paid-shards.test.ts @@ -128,6 +128,35 @@ describe('tier lane skip (B5)', () => { expect(tierSkipReason(file, "testIfSelected('sample-gate', async () => {});", 'gate', touchfiles, tiers)).toBeNull(); }); + test('a computed registration is skipped only when every id of the lane tier is registered to another test file', () => { + const other = 'test/skill-e2e-other.test.ts'; + const owned = { 'sample-periodic': [file], 'gate-elsewhere': [other, 'x/**'] }; + const ownedTiers = { 'sample-periodic': 'periodic', 'gate-elsewhere': 'gate' }; + for (const source of [ + "testConcurrentIfSelected(`sample-${label}`, async () => {});", + "runSkillTest({ testName: `sample-${label}` });", + "for (const entry of CASES) describeIfSelected(entry.suite, [entry.id], () => { testIfSelected(entry.id, async () => {}); });", + ]) { + expect(tierSkipReason(file, source, 'gate', owned, ownedTiers)).toBe( + 'skipped: no E2E_TIERS id has tier gate (its computed names can only produce gate ids other test files register)'); + expect(tierSkipReason(file, source, 'periodic', owned, ownedTiers)).toBeNull(); + expect(tierSkipReason(file, source, 'gate', owned, { ...ownedTiers, 'gate-orphan': 'gate' })).toBeNull(); + expect(tierSkipReason(file, `${source}\ntestIfSelected('gate-elsewhere', async () => {});`, 'gate', owned, ownedTiers)).toBeNull(); + } + }); + + test('census files whose computed names hold only other-tier cases never take a runner (2026-10-03 census)', () => { + const lanes = { gate: ['test/skill-e2e-plan-decision-classification.test.ts', 'test/skill-e2e-plan-devex-peer-comparison-classification.test.ts', + 'test/skill-e2e-qa-bugs.test.ts', 'test/skill-routing-e2e.test.ts'], periodic: ['test/skill-e2e-coverage-audit.test.ts', 'test/skill-e2e-test-value.test.ts'] }; + for (const [tier, files] of Object.entries(lanes) as Array<['gate' | 'periodic', string[]]>) { + const { selected, excluded } = selectPaidTestFiles(collectPaidTestFiles(), tier, ROOT, {}); + for (const hollow of files) { + expect(selected, hollow).not.toContain(hollow); + expect(excluded.find(entry => entry.file === hollow)?.reason, hollow).toStartWith(`skipped: no E2E_TIERS id has tier ${tier}`); + } + } + }); + test('the real constructed-name diagram file stays scheduled in both lanes', () => { const source = fs.readFileSync(path.join(ROOT, 'test/skill-e2e-diagram.test.ts'), 'utf8'); for (const tier of ['gate', 'periodic'] as const) expect(tierSkipReason('test/skill-e2e-diagram.test.ts', source, tier)).toBeNull();