diff --git a/scripts/eval-flake-rank.ts b/scripts/eval-flake-rank.ts index 8ce5d746d..a31d95cf4 100644 --- a/scripts/eval-flake-rank.ts +++ b/scripts/eval-flake-rank.ts @@ -54,7 +54,7 @@ import { matchGlob } from '../test/helpers/test-selection'; import { CASE_TEST_NAMES } from './test-paid-shards'; import { resolveStateRoot } from '../lib/state-root'; import { GH_JOBS, PASS_RATES_USAGE, criticalPath, formatCriticalPath, formatHeadroom, formatRedLedger, headroom, headroomAlarms, parsePassRatesArgs, - redLedger, triageRun, type CriticalPath } from './lib/eval-history'; + redLedger, triageRun, CENSUS_RED_GUIDE, type CriticalPath } from './lib/eval-history'; interface TestSeries { name: string; @@ -705,7 +705,7 @@ if (import.meta.main) { const lines = view === 'headroom' ? formatHeadroom(caseHeadroom, critical) : view === 'reds' ? formatRedLedger(redLedger(scoped)) : historyError ? [] : triageRun({ repo, run: fetched.find(run => run.id === parsed.runId)!, records, caseFilter, cacheDir, fetcher: GH_HISTORY, download: (match, maxBytes) => downloadRunArtifacts({ repo, run: fetched.find(run => run.id === parsed.runId)!, match, cacheDir, maxBytes }) }) - .concat(formatCriticalPath(critical)); + .concat(formatCriticalPath(critical), historyError ? [] : [`guide: ${CENSUS_RED_GUIDE}`]); if (asJson) console.log(JSON.stringify({ scope, historyError, view, lines }, null, 2)); else { console.log(`scope: ${scope}`); diff --git a/scripts/lib/eval-history.ts b/scripts/lib/eval-history.ts index 9c9a5d968..e71c889a4 100644 --- a/scripts/lib/eval-history.ts +++ b/scripts/lib/eval-history.ts @@ -26,6 +26,9 @@ import { sanitizeFixedFenceLine } from './published-text'; import type { HistoryFetcher, WeeklyRun } from '../eval-flake-rank'; import { CASE_SHARDED_FILES, trialShardKey } from './paid-cases'; import { shardSlug } from '../test-paid-shards'; +import { CENSUS_RED_GUIDE } from './paid-report'; + +export { CENSUS_RED_GUIDE }; /** Below this many session samples a case's headroom is insufficient, never a pass. */ export const HEADROOM_MIN_SAMPLES = 3; @@ -214,6 +217,7 @@ export function formatRedLedger(ledger: RedLedger, opts: { top?: number } = {}): const g = ledger.allGreen; lines.push(g ? `all-green probability (approximation, assumes independent verdicts): ${pct(g.p)} for the latest census's ${g.verdicts} verdicts [${pct(g.lo)}–${pct(g.hi)} with the pooled rate's 95% interval as prior]; ` + 'formula Π(1 - p_i) over its cases; rerun: bun run eval:pass-rates --reds' : 'all-green probability: no post-policy verdicts'); + lines.push(`guide: ${CENSUS_RED_GUIDE}`); return lines; } diff --git a/scripts/lib/paid-report.ts b/scripts/lib/paid-report.ts index 5789675ba..bd2f7d08b 100644 --- a/scripts/lib/paid-report.ts +++ b/scripts/lib/paid-report.ts @@ -268,6 +268,9 @@ function caseSessions(rows: SessionLedgerRow[], id: string, onlyCase: boolean): } /** The artifact holding a slice's shards: the downloaded directory's own name when it is one. */ +/** The red-census guide the report and every eval:pass-rates view link. */ +export const CENSUS_RED_GUIDE = 'docs/evals/census-red.md'; + export function sliceArtifactName(root: string, slice: number, attempt: number): string { const name = path.basename(root); return /^(?:paid-slice|gate-census)-\d+-a\d+$/.test(name) ? name : `slice ${slice} attempt ${attempt}`; @@ -762,7 +765,7 @@ export function runPaidReport(reportDir: string, options: { writeDurations?: boo ...fence(headlineLines), ...(failureLines.length ? ['', '**Failures and split verdicts**', '', ...fence(failureLines)] : []), ...(censusLines.length ? ['', '**Skipped, deselected and unattributed testcases**', '', ...fence(censusLines)] : []), - ...(verdict.problems.length ? ['', `**ACTION REQUIRED (${verdict.problems.length})**`, '', ...fence(verdict.problems.map(p => sanitizeTrialError(p) ?? p))] : []), + ...(verdict.problems.length ? ['', `**ACTION REQUIRED (${verdict.problems.length})**`, '', ...fence(verdict.problems.map(p => sanitizeTrialError(p) ?? p)), '', `Guide: ${CENSUS_RED_GUIDE}`] : []), ].join('\n') + '\n'); if (!manualProblems.length) fs.writeFileSync(summaryPath, JSON.stringify({ version: 2, files, totals: { ...evidence, total: evidence.passed + evidence.failed + evidence.manual_accepted, diff --git a/test/eval-history.test.ts b/test/eval-history.test.ts index 73c46470e..b618fc8a8 100644 --- a/test/eval-history.test.ts +++ b/test/eval-history.test.ts @@ -100,6 +100,7 @@ describe('red ledger over the 8 wave censuses', () => { expect(text).toContain('rerun: bun run eval:pass-rates --reds'); expect(text).toContain('37186854666 gate/full census: 1 red of'); expect(text).toContain('class: timeout 1; cause: unrecorded 1'); + expect(formatRedLedger(ledger).at(-1)).toBe('guide: docs/evals/census-red.md'); }); test('new records group reds by machine cause', () => { diff --git a/test/paid-report-census.test.ts b/test/paid-report-census.test.ts index b5065a95b..178757325 100644 --- a/test/paid-report-census.test.ts +++ b/test/paid-report-census.test.ts @@ -200,6 +200,7 @@ describe('red lines carry their values, evidence and a one-case command (plan 0. expect(open.length).toBeGreaterThan(3); expect(lines.slice(at).find(l => /^`{3,}$/.test(l))).toBe(open); expect(summary).not.toMatch(/@team/); + if (summary.includes('**ACTION REQUIRED')) expect(summary.trimEnd().endsWith('Guide: docs/evals/census-red.md')).toBe(true); const collector = JSON.parse(fs.readFileSync(path.join(r.dir, 'collector-outcomes.json'), 'utf8')); expect(collector.failures.join('\n')).not.toContain('`'); });