0.4: census report and eval:pass-rates views link the census-red guide

scripts/eval-flake-rank.ts

@@ -54,7 +54,7 @@ import { matchGlob } from '../test/helpers/test-selection';
 import { CASE_TEST_NAMES } from './test-paid-shards';
 import { resolveStateRoot } from '../lib/state-root';
 import { GH_JOBS, PASS_RATES_USAGE, criticalPath, formatCriticalPath, formatHeadroom, formatRedLedger, headroom, headroomAlarms, parsePassRatesArgs,
-  redLedger, triageRun, type CriticalPath } from './lib/eval-history';
+  redLedger, triageRun, CENSUS_RED_GUIDE, type CriticalPath } from './lib/eval-history';
 
 interface TestSeries {
   name: string;
@@ -705,7 +705,7 @@ if (import.meta.main) {
     const lines = view === 'headroom' ? formatHeadroom(caseHeadroom, critical) : view === 'reds' ? formatRedLedger(redLedger(scoped))
       : historyError ? [] : triageRun({ repo, run: fetched.find(run => run.id === parsed.runId)!, records, caseFilter, cacheDir, fetcher: GH_HISTORY,
         download: (match, maxBytes) => downloadRunArtifacts({ repo, run: fetched.find(run => run.id === parsed.runId)!, match, cacheDir, maxBytes }) })
-        .concat(formatCriticalPath(critical));
+        .concat(formatCriticalPath(critical), historyError ? [] : [`guide: ${CENSUS_RED_GUIDE}`]);
     if (asJson) console.log(JSON.stringify({ scope, historyError, view, lines }, null, 2));
     else {
       console.log(`scope: ${scope}`);

scripts/lib/eval-history.ts

@@ -26,6 +26,9 @@ import { sanitizeFixedFenceLine } from './published-text';
 import type { HistoryFetcher, WeeklyRun } from '../eval-flake-rank';
 import { CASE_SHARDED_FILES, trialShardKey } from './paid-cases';
 import { shardSlug } from '../test-paid-shards';
+import { CENSUS_RED_GUIDE } from './paid-report';
+
+export { CENSUS_RED_GUIDE };
 
 /** Below this many session samples a case's headroom is insufficient, never a pass. */
 export const HEADROOM_MIN_SAMPLES = 3;
@@ -214,6 +217,7 @@ export function formatRedLedger(ledger: RedLedger, opts: { top?: number } = {}):
   const g = ledger.allGreen;
   lines.push(g ? `all-green probability (approximation, assumes independent verdicts): ${pct(g.p)} for the latest census's ${g.verdicts} verdicts [${pct(g.lo)}–${pct(g.hi)} with the pooled rate's 95% interval as prior]; `
     + 'formula Π(1 - p_i) over its cases; rerun: bun run eval:pass-rates --reds' : 'all-green probability: no post-policy verdicts');
+  lines.push(`guide: ${CENSUS_RED_GUIDE}`);
   return lines;
 }
 

scripts/lib/paid-report.ts

@@ -268,6 +268,9 @@ function caseSessions(rows: SessionLedgerRow[], id: string, onlyCase: boolean):
 }
 
 /** The artifact holding a slice's shards: the downloaded directory's own name when it is one. */
+/** The red-census guide the report and every eval:pass-rates view link. */
+export const CENSUS_RED_GUIDE = 'docs/evals/census-red.md';
+
 export function sliceArtifactName(root: string, slice: number, attempt: number): string {
   const name = path.basename(root);
   return /^(?:paid-slice|gate-census)-\d+-a\d+$/.test(name) ? name : `slice ${slice} attempt ${attempt}`;
@@ -762,7 +765,7 @@ export function runPaidReport(reportDir: string, options: { writeDurations?: boo
     ...fence(headlineLines),
     ...(failureLines.length ? ['', '**Failures and split verdicts**', '', ...fence(failureLines)] : []),
     ...(censusLines.length ? ['', '**Skipped, deselected and unattributed testcases**', '', ...fence(censusLines)] : []),
-    ...(verdict.problems.length ? ['', `**ACTION REQUIRED (${verdict.problems.length})**`, '', ...fence(verdict.problems.map(p => sanitizeTrialError(p) ?? p))] : []),
+    ...(verdict.problems.length ? ['', `**ACTION REQUIRED (${verdict.problems.length})**`, '', ...fence(verdict.problems.map(p => sanitizeTrialError(p) ?? p)), '', `Guide: ${CENSUS_RED_GUIDE}`] : []),
   ].join('\n') + '\n');
   if (!manualProblems.length) fs.writeFileSync(summaryPath, JSON.stringify({ version: 2, files, totals: {
     ...evidence, total: evidence.passed + evidence.failed + evidence.manual_accepted,

test/eval-history.test.ts

@@ -100,6 +100,7 @@ describe('red ledger over the 8 wave censuses', () => {
     expect(text).toContain('rerun: bun run eval:pass-rates --reds');
     expect(text).toContain('37186854666 gate/full census: 1 red of');
     expect(text).toContain('class: timeout 1; cause: unrecorded 1');
+    expect(formatRedLedger(ledger).at(-1)).toBe('guide: docs/evals/census-red.md');
   });
 
   test('new records group reds by machine cause', () => {

test/paid-report-census.test.ts

@@ -200,6 +200,7 @@ describe('red lines carry their values, evidence and a one-case command (plan 0.
     expect(open.length).toBeGreaterThan(3);
     expect(lines.slice(at).find(l => /^`{3,}$/.test(l))).toBe(open);
     expect(summary).not.toMatch(/@team/);
+    if (summary.includes('**ACTION REQUIRED')) expect(summary.trimEnd().endsWith('Guide: docs/evals/census-red.md')).toBe(true);
     const collector = JSON.parse(fs.readFileSync(path.join(r.dir, 'collector-outcomes.json'), 'utf8'));
     expect(collector.failures.join('\n')).not.toContain('`');
   });