0.3: published lines cannot break a fixed fence

evals.yml wraps collector-outcomes failures in a fixed three-backtick fence
for the PR comment, so those lines (and eval:pass-rates output meant for the
tracking issue) go through sanitizeFixedFenceLine: cleaned like every
published line, with backticks turned into quotes. Inline published text
also neutralizes backticks.

scripts/lib/eval-history.ts

@@ -22,7 +22,7 @@ import * as path from 'node:path';
 import { spawnSync } from 'node:child_process';
 import { panelVerdict, type TrialOutcomeRecord } from '../../test/helpers/eval-store';
 import { HEADROOM_FAIL, HEADROOM_WARN } from '../../test/helpers/eval-budgets';
-import { sanitizePublishedText } from './published-text';
+import { sanitizeFixedFenceLine } from './published-text';
 import type { HistoryFetcher, WeeklyRun } from '../eval-flake-rank';
 import { CASE_SHARDED_FILES, trialShardKey } from './paid-cases';
 import { shardSlug } from '../test-paid-shards';
@@ -204,7 +204,7 @@ export function formatRedLedger(ledger: RedLedger, opts: { top?: number } = {}):
     for (const red of c.reds) {
       const failed = red.trials.filter(t => t.outcome === 'failed');
       lines.push(`    ✗ ${red.case} ${red.status} ${failed.map(t => `t${t.trial} ${t.failure_class ?? 'assertion'}/${t.failure_cause ?? 'unrecorded'}`
-        + `${t.error ? ` — ${sanitizePublishedText(t.error, 160)}` : ''}`).join('; ')}`);
+        + `${t.error ? ` — ${sanitizeFixedFenceLine(t.error, 160)}` : ''}`).join('; ')}`);
     }
   }
   const p = ledger.pooled;
@@ -275,7 +275,7 @@ export function triageRun(opts: { repo: string; run: WeeklyRun; records: readonl
       const d = t.failure_detail;
       const detail = !d ? '' : 'judge' in d ? `  ${d.judge.map(j => `${j.dimension} ${j.mean} < ${j.threshold} (${j.samples} samples)${j.rationale ? ` "${j.rationale}"` : ''}`).join('; ')}`
         : `  Expected: ${d.expected} · Received: ${d.received}`;
-      lines.push(sanitizePublishedText(`    t${t.trial}: ${t.failure_class ?? 'assertion'} / cause ${t.failure_cause ?? 'unrecorded'}`
+      lines.push(sanitizeFixedFenceLine(`    t${t.trial}: ${t.failure_class ?? 'assertion'} / cause ${t.failure_cause ?? 'unrecorded'}`
         + `${t.failure_cause_evidence ? `: ${t.failure_cause_evidence}` : ''}${t.error ? ` — ${t.error}` : ''}${detail}`, 2000));
       const plan = planFor(red.lane);
       const slice = plan.files.get(recordShardKey(t));

scripts/lib/paid-report.ts

@@ -36,7 +36,7 @@ import {
   trialCostKnown, trialFailureFields, trialSessions, type TrialFailureDetail,
 } from '../../test/helpers/eval-store';
 import { readSessionLedger, type SessionLedgerRow } from '../../test/helpers/session-ledger';
-import { publishedFence, sanitizePublishedText } from './published-text';
+import { publishedFence, sanitizeFixedFenceLine } from './published-text';
 import { E2E_KINDS } from '../../test/helpers/touchfiles-data';
 import { manualReviewProblem } from '../../test/helpers/cookie-workflow-manual-review';
 import { preflightAnthropicApi } from '../../test/helpers/anthropic-preflight';
@@ -772,7 +772,8 @@ export function runPaidReport(reportDir: string, options: { writeDurations?: boo
     marks: p.marks, split: p.split, quarantined: p.quarantined, failsLane: p.failsLane, redClass: p.redClass, reason: p.reason,
     trials: p.trials.map(t => ({ trial: t.trial, outcome: t.outcome, ...(t.failure_class ? { failure_class: t.failure_class } : {}),
       ...(t.exit_reason ? { exit_reason: t.exit_reason } : {}), ...(t.error ? { error: t.error } : {}) })) })),
-  failures: failureLines.map(line => sanitizePublishedText(line)) }, null, 2) + '\n');
+  // evals.yml wraps these in a fixed fence for the PR comment.
+  failures: failureLines.map(line => sanitizeFixedFenceLine(line)) }, null, 2) + '\n');
   if (verdict.problems.length) {
     console.error(`[test:paid] report: ${verdict.problems.length} problem(s):`);
     for (const problem of verdict.problems) console.error(`  ✗ ${problem}`);

scripts/lib/published-text.ts

@@ -21,10 +21,15 @@ function clean(text: string): string {
   return redacted.replace(/@(?=[A-Za-z0-9_-])/g, '@\u200b');
 }
 
-/** One inline line: cleaned, HTML and Markdown link brackets escaped, capped. */
+/** One inline line: cleaned, HTML, Markdown link brackets and backticks neutralized, capped. */
 export function sanitizePublishedText(text: string, max = PUBLISHED_TEXT_MAX): string {
   return cap(clean(text).replace(/&/g, '&amp;').replace(/</g, '&lt;').replace(/>/g, '&gt;')
-    .replace(/\[/g, '&#91;').replace(/\]/g, '&#93;'), max);
+    .replace(/\[/g, '&#91;').replace(/\]/g, '&#93;').replace(/`/g, "'"), max);
+}
+
+/** One line for a consumer that wraps it in a fixed three-backtick fence (the PR comment): backticks become quotes. */
+export function sanitizeFixedFenceLine(text: string, max = PUBLISHED_TEXT_MAX): string {
+  return cap(clean(text).replace(/`/g, "'"), max);
 }
 
 /** Cleaned, capped lines inside a fence longer than the longest backtick run they contain. */

test/eval-history.test.ts

@@ -143,7 +143,7 @@ describe('triageRun (stub fetcher, no network)', () => {
     expect(downloaded).toEqual(['paid-plan', 'paid-slice-5-a1']);
     expect(lines[0]).toContain('run 42 (b @ abcdef123, t): 2 verdict red(s)');
     expect(lines).toContain('✗ qa-only/SKILL.md workflow  periodic/full census  FAIL  (history: 1/2 verdicts green)');
-    expect(lines.join('\n')).toContain('t1: assertion / cause assertion — expect(received).toBeGreaterThanOrEqual(expected)  clarity 3.67 < 4 (3 samples) "Clarity suffers from density."');
+    expect(lines.join('\n')).toContain('t1: assertion / cause assertion — expect(received).toBeGreaterThanOrEqual(expected)  clarity 3.67 < 4 (3 samples) "Clarity suffers from density."');
     expect(lines.join('\n')).toContain(`evidence: ${path.join(cacheDir, 'paid-slice-5-a1', 'shards', 'skill-llm-eval')}`);
     expect(lines).toContain('    evidence: paid-slice-7-a1 too large to fetch (70 MB)');
   });

test/paid-report-census.test.ts

@@ -198,6 +198,8 @@ describe('red lines carry their values, evidence and a one-case command (plan 0.
     expect(open.length).toBeGreaterThan(3);
     expect(lines.slice(at).find(l => /^`{3,}$/.test(l))).toBe(open);
     expect(summary).not.toMatch(/@team/);
+    const collector = JSON.parse(fs.readFileSync(path.join(r.dir, 'collector-outcomes.json'), 'utf8'));
+    expect(collector.failures.join('\n')).not.toContain('`');
   });
 });
 

test/published-text.test.ts

@@ -4,7 +4,7 @@
  * break out of its fence, inject HTML or images, leak a credential, or flood.
  */
 import { describe, expect, test } from 'bun:test';
-import { publishedFence, sanitizePublishedText, PUBLISHED_TEXT_MAX } from '../scripts/lib/published-text';
+import { publishedFence, sanitizeFixedFenceLine, sanitizePublishedText, PUBLISHED_TEXT_MAX } from '../scripts/lib/published-text';
 
 // Built at runtime so this file never contains a credential-shaped literal.
 const GHP = ['ghp', '_', 'A1b2C3d4E5f6G7h8I9j0K1l2M3n4O5p6Q7r8'].join('');
@@ -37,6 +37,9 @@ describe('sanitizePublishedText (inline)', () => {
     // eslint-disable-next-line no-control-regex
     expect(out[5]).not.toMatch(/[\u0000-\u001f\u007f\u202e]/);
   });
+  test('backticks cannot open a code span or close an enclosing fence', () => {
+    expect(out[1]).not.toContain('`');
+  });
   test('length is capped', () => {
     expect(sanitizePublishedText('x'.repeat(10 * 1024))).toHaveLength(PUBLISHED_TEXT_MAX);
   });
@@ -62,3 +65,13 @@ describe('publishedFence (blocks)', () => {
     expect(body).toContain('</details>');
   });
 });
+
+describe('sanitizeFixedFenceLine (lines a workflow wraps in a fixed fence)', () => {
+  test('no backtick survives, and the rest is cleaned like every published line', () => {
+    const out = ADVERSARIAL.map(line => sanitizeFixedFenceLine(line)).join('\n');
+    expect(out).not.toContain('`');
+    expect(out).not.toMatch(/@[A-Za-z]/);
+    expect(out).not.toContain(GHP);
+    expect(sanitizeFixedFenceLine('z'.repeat(10 * 1024))).toHaveLength(PUBLISHED_TEXT_MAX);
+  });
+});