diff --git a/scripts/lib/eval-history.ts b/scripts/lib/eval-history.ts index d02e327d0..9c9a5d968 100644 --- a/scripts/lib/eval-history.ts +++ b/scripts/lib/eval-history.ts @@ -22,7 +22,7 @@ import * as path from 'node:path'; import { spawnSync } from 'node:child_process'; import { panelVerdict, type TrialOutcomeRecord } from '../../test/helpers/eval-store'; import { HEADROOM_FAIL, HEADROOM_WARN } from '../../test/helpers/eval-budgets'; -import { sanitizePublishedText } from './published-text'; +import { sanitizeFixedFenceLine } from './published-text'; import type { HistoryFetcher, WeeklyRun } from '../eval-flake-rank'; import { CASE_SHARDED_FILES, trialShardKey } from './paid-cases'; import { shardSlug } from '../test-paid-shards'; @@ -204,7 +204,7 @@ export function formatRedLedger(ledger: RedLedger, opts: { top?: number } = {}): for (const red of c.reds) { const failed = red.trials.filter(t => t.outcome === 'failed'); lines.push(` ✗ ${red.case} ${red.status} ${failed.map(t => `t${t.trial} ${t.failure_class ?? 'assertion'}/${t.failure_cause ?? 'unrecorded'}` - + `${t.error ? ` — ${sanitizePublishedText(t.error, 160)}` : ''}`).join('; ')}`); + + `${t.error ? ` — ${sanitizeFixedFenceLine(t.error, 160)}` : ''}`).join('; ')}`); } } const p = ledger.pooled; @@ -275,7 +275,7 @@ export function triageRun(opts: { repo: string; run: WeeklyRun; records: readonl const d = t.failure_detail; const detail = !d ? '' : 'judge' in d ? ` ${d.judge.map(j => `${j.dimension} ${j.mean} < ${j.threshold} (${j.samples} samples)${j.rationale ? ` "${j.rationale}"` : ''}`).join('; ')}` : ` Expected: ${d.expected} · Received: ${d.received}`; - lines.push(sanitizePublishedText(` t${t.trial}: ${t.failure_class ?? 'assertion'} / cause ${t.failure_cause ?? 'unrecorded'}` + lines.push(sanitizeFixedFenceLine(` t${t.trial}: ${t.failure_class ?? 'assertion'} / cause ${t.failure_cause ?? 'unrecorded'}` + `${t.failure_cause_evidence ? `: ${t.failure_cause_evidence}` : ''}${t.error ? ` — ${t.error}` : ''}${detail}`, 2000)); const plan = planFor(red.lane); const slice = plan.files.get(recordShardKey(t)); diff --git a/scripts/lib/paid-report.ts b/scripts/lib/paid-report.ts index e460e78ad..eaf41c7ce 100644 --- a/scripts/lib/paid-report.ts +++ b/scripts/lib/paid-report.ts @@ -36,7 +36,7 @@ import { trialCostKnown, trialFailureFields, trialSessions, type TrialFailureDetail, } from '../../test/helpers/eval-store'; import { readSessionLedger, type SessionLedgerRow } from '../../test/helpers/session-ledger'; -import { publishedFence, sanitizePublishedText } from './published-text'; +import { publishedFence, sanitizeFixedFenceLine } from './published-text'; import { E2E_KINDS } from '../../test/helpers/touchfiles-data'; import { manualReviewProblem } from '../../test/helpers/cookie-workflow-manual-review'; import { preflightAnthropicApi } from '../../test/helpers/anthropic-preflight'; @@ -772,7 +772,8 @@ export function runPaidReport(reportDir: string, options: { writeDurations?: boo marks: p.marks, split: p.split, quarantined: p.quarantined, failsLane: p.failsLane, redClass: p.redClass, reason: p.reason, trials: p.trials.map(t => ({ trial: t.trial, outcome: t.outcome, ...(t.failure_class ? { failure_class: t.failure_class } : {}), ...(t.exit_reason ? { exit_reason: t.exit_reason } : {}), ...(t.error ? { error: t.error } : {}) })) })), - failures: failureLines.map(line => sanitizePublishedText(line)) }, null, 2) + '\n'); + // evals.yml wraps these in a fixed fence for the PR comment. + failures: failureLines.map(line => sanitizeFixedFenceLine(line)) }, null, 2) + '\n'); if (verdict.problems.length) { console.error(`[test:paid] report: ${verdict.problems.length} problem(s):`); for (const problem of verdict.problems) console.error(` ✗ ${problem}`); diff --git a/scripts/lib/published-text.ts b/scripts/lib/published-text.ts index 5cfddb2a3..3f2e4d616 100644 --- a/scripts/lib/published-text.ts +++ b/scripts/lib/published-text.ts @@ -21,10 +21,15 @@ function clean(text: string): string { return redacted.replace(/@(?=[A-Za-z0-9_-])/g, '@\u200b'); } -/** One inline line: cleaned, HTML and Markdown link brackets escaped, capped. */ +/** One inline line: cleaned, HTML, Markdown link brackets and backticks neutralized, capped. */ export function sanitizePublishedText(text: string, max = PUBLISHED_TEXT_MAX): string { return cap(clean(text).replace(/&/g, '&').replace(//g, '>') - .replace(/\[/g, '[').replace(/\]/g, ']'), max); + .replace(/\[/g, '[').replace(/\]/g, ']').replace(/`/g, "'"), max); +} + +/** One line for a consumer that wraps it in a fixed three-backtick fence (the PR comment): backticks become quotes. */ +export function sanitizeFixedFenceLine(text: string, max = PUBLISHED_TEXT_MAX): string { + return cap(clean(text).replace(/`/g, "'"), max); } /** Cleaned, capped lines inside a fence longer than the longest backtick run they contain. */ diff --git a/test/eval-history.test.ts b/test/eval-history.test.ts index 7d25b7439..73c46470e 100644 --- a/test/eval-history.test.ts +++ b/test/eval-history.test.ts @@ -143,7 +143,7 @@ describe('triageRun (stub fetcher, no network)', () => { expect(downloaded).toEqual(['paid-plan', 'paid-slice-5-a1']); expect(lines[0]).toContain('run 42 (b @ abcdef123, t): 2 verdict red(s)'); expect(lines).toContain('✗ qa-only/SKILL.md workflow periodic/full census FAIL (history: 1/2 verdicts green)'); - expect(lines.join('\n')).toContain('t1: assertion / cause assertion — expect(received).toBeGreaterThanOrEqual(expected) clarity 3.67 < 4 (3 samples) "Clarity suffers from density."'); + expect(lines.join('\n')).toContain('t1: assertion / cause assertion — expect(received).toBeGreaterThanOrEqual(expected) clarity 3.67 < 4 (3 samples) "Clarity suffers from density."'); expect(lines.join('\n')).toContain(`evidence: ${path.join(cacheDir, 'paid-slice-5-a1', 'shards', 'skill-llm-eval')}`); expect(lines).toContain(' evidence: paid-slice-7-a1 too large to fetch (70 MB)'); }); diff --git a/test/paid-report-census.test.ts b/test/paid-report-census.test.ts index 3b7e37c14..1e6d895a4 100644 --- a/test/paid-report-census.test.ts +++ b/test/paid-report-census.test.ts @@ -198,6 +198,8 @@ describe('red lines carry their values, evidence and a one-case command (plan 0. expect(open.length).toBeGreaterThan(3); expect(lines.slice(at).find(l => /^`{3,}$/.test(l))).toBe(open); expect(summary).not.toMatch(/@team/); + const collector = JSON.parse(fs.readFileSync(path.join(r.dir, 'collector-outcomes.json'), 'utf8')); + expect(collector.failures.join('\n')).not.toContain('`'); }); }); diff --git a/test/published-text.test.ts b/test/published-text.test.ts index 5754bafe6..8cf6fda30 100644 --- a/test/published-text.test.ts +++ b/test/published-text.test.ts @@ -4,7 +4,7 @@ * break out of its fence, inject HTML or images, leak a credential, or flood. */ import { describe, expect, test } from 'bun:test'; -import { publishedFence, sanitizePublishedText, PUBLISHED_TEXT_MAX } from '../scripts/lib/published-text'; +import { publishedFence, sanitizeFixedFenceLine, sanitizePublishedText, PUBLISHED_TEXT_MAX } from '../scripts/lib/published-text'; // Built at runtime so this file never contains a credential-shaped literal. const GHP = ['ghp', '_', 'A1b2C3d4E5f6G7h8I9j0K1l2M3n4O5p6Q7r8'].join(''); @@ -37,6 +37,9 @@ describe('sanitizePublishedText (inline)', () => { // eslint-disable-next-line no-control-regex expect(out[5]).not.toMatch(/[\u0000-\u001f\u007f\u202e]/); }); + test('backticks cannot open a code span or close an enclosing fence', () => { + expect(out[1]).not.toContain('`'); + }); test('length is capped', () => { expect(sanitizePublishedText('x'.repeat(10 * 1024))).toHaveLength(PUBLISHED_TEXT_MAX); }); @@ -62,3 +65,13 @@ describe('publishedFence (blocks)', () => { expect(body).toContain(''); }); }); + +describe('sanitizeFixedFenceLine (lines a workflow wraps in a fixed fence)', () => { + test('no backtick survives, and the rest is cleaned like every published line', () => { + const out = ADVERSARIAL.map(line => sanitizeFixedFenceLine(line)).join('\n'); + expect(out).not.toContain('`'); + expect(out).not.toMatch(/@[A-Za-z]/); + expect(out).not.toContain(GHP); + expect(sanitizeFixedFenceLine('z'.repeat(10 * 1024))).toHaveLength(PUBLISHED_TEXT_MAX); + }); +});