From d725d94ea33cbba48bf747c7f59e18a71c90b226 Mon Sep 17 00:00:00 2001 From: garrytan Date: Sun, 4 Oct 2026 17:45:35 +0000 Subject: [PATCH] 0.3: red lines name their values, evidence and an after-a-repair command; one published-text sanitizer Each red line in the census report keeps its stored error and adds the machine cause with its evidence, the failure_detail values (Expected/Received, or each failing judge dimension with its mean, sample count and the sample sentence naming it), the slice artifact and shard directory that hold the transcript, a fetch command (eval:pass-rates --run --case ), and 'after a repair:' with a command that selects only that case (E2E ids via --case, judges via their selection and an exact test-name filter). A session that completed and failed a check says so instead of 'assertion (success)'. Report text that reaches a PR comment, issue or committed file goes through scripts/lib/published-text.ts: control and bidi characters stripped, credential-scan patterns redacted, mentions neutralized, HTML and link brackets escaped inline, blocks in a fence longer than any backtick run, length capped. Verdicts and collector verdict objects are unchanged. --- scripts/lib/paid-report.ts | 78 ++++++++++++++++++++++++------- scripts/lib/published-text.ts | 36 ++++++++++++++ test/paid-report-census.test.ts | 83 +++++++++++++++++++++++++++++++-- test/published-text.test.ts | 64 +++++++++++++++++++++++++ 4 files changed, 242 insertions(+), 19 deletions(-) create mode 100644 scripts/lib/published-text.ts create mode 100644 test/published-text.test.ts diff --git a/scripts/lib/paid-report.ts b/scripts/lib/paid-report.ts index 998635ebb..748f778ef 100644 --- a/scripts/lib/paid-report.ts +++ b/scripts/lib/paid-report.ts @@ -33,9 +33,10 @@ import { getProjectEvalDir, getClaudeCliVersion, isFinalizedEvalResultFile, evalEntryOutcome, failureClassOf, panelVerdict, sanitizeTrialError, formatTrialOutcomes, CONTRACT_VIOLATIONS_FILE, TRIAL_ENV, TRIAL_OUTCOME_SCHEMA, TRIAL_OUTCOMES_FILE, type EvalCaseKind, type PanelShape, type PanelVerdict, type TrialFailureClass, type TrialOutcome, type TrialOutcomeRecord, - trialFailureFields, trialSessions, + trialFailureFields, trialSessions, type TrialFailureDetail, } from '../../test/helpers/eval-store'; import { readSessionLedger, type SessionLedgerRow } from '../../test/helpers/session-ledger'; +import { publishedFence, sanitizePublishedText } from './published-text'; import { E2E_KINDS } from '../../test/helpers/touchfiles-data'; import { manualReviewProblem } from '../../test/helpers/cookie-workflow-manual-review'; import { preflightAnthropicApi } from '../../test/helpers/anthropic-preflight'; @@ -57,7 +58,7 @@ export { PAID_TEST_GLOBS, isPaidTestFile }; export { PERIODIC_CI_EXCLUDE }; type E2EShardReuse = NonNullable>; -import { CASE_TEST_NAMES, type CaseTrialPlan, caseTrialPlan, shardCaseId, shardFile, trialShardKey } from './paid-cases'; +import { CASE_TEST_NAMES, type CaseTrialPlan, caseTestNamePattern, caseTrialPlan, shardCaseId, shardFile, trialShardKey } from './paid-cases'; import { type ManifestEntry, PAID_TEST_DURATIONS_FILE, type PaidRunManifest, type SliceResult, collectorOutcomeCounts, formatProfileCoverage, loadPaidTestDurations, mergePaidTestDurations, parseRunManifest, trialPanelKey, verifySliceResults, writePaidTestDurations } from './paid-plan'; import { DEFAULT_JOBS, type PaidCaseSelection, type PaidTier, ROOT, type RunShardsOptions, type ShardTrialRecord, collectPaidTestFiles, fileCaseRegistration, isAllSkippedPass, runPaidShards, shardSlug, paidSelectionEnv } from '../test-paid-shards'; @@ -232,6 +233,12 @@ function caseSessions(rows: SessionLedgerRow[], id: string, onlyCase: boolean): return rows.filter(row => row.case === id || (!row.case && (onlyCase || row.test_name === id || row.test_name === CASE_TEST_NAMES[id]))); } +/** The artifact holding a slice's shards: the downloaded directory's own name when it is one. */ +export function sliceArtifactName(root: string, slice: number, attempt: number): string { + const name = path.basename(root); + return /^(?:paid-slice|gate-census)-\d+-a\d+$/.test(name) ? name : `slice ${slice} attempt ${attempt}`; +} + export function junitCensus(manifest: PaidRunManifest, artifacts: ReportArtifact[], recordsByShard: Map, base: Record, cliVersion?: string): JUnitCensus { const census: JUnitCensus = { ruleCases: [], history: [], failedShards: new Set(), skipped: [], deselected: new Map(), shardSkipReasons: new Map(), unattributed: [] }; @@ -262,7 +269,6 @@ export function junitCensus(manifest: PaidRunManifest, artifacts: ReportArtifact const failureClass: TrialFailureClass | undefined = !failed ? undefined : failed.failureType === 'TimeoutError' ? 'timeout' : failedRecord ? failureClassOf(failedRecord) : 'assertion'; const error = sanitizeTrialError(failedRecord?.error ?? failed?.message); - const rerun = Object.hasOwn(E2E_TIERS, id) ? rerunCommand(manifest.tier, id, 1) : `EVALS=1 EVALS_TIER=${manifest.tier} bun test ${shardFile(key)}`; const sessions = caseSessions(ledger, id, byCase.size === 1); const diagnosis = failed && failureClass ? trialFailureFields({ failure_class: failureClass, exit_reason: failedRecord?.exit_reason, error: failedRecord?.error ?? failed.message, sessions, record: failedRecord }) : undefined; @@ -272,7 +278,10 @@ export function junitCensus(manifest: PaidRunManifest, artifacts: ReportArtifact census.skipped.push(`◌ ${id} ${key} (${cases.map(tc => tc.name).join('; ')}): ${RUNTIME_SKIP_REASON}`); } census.ruleCases.push({ id, kind, outcome: caseOutcome, - ...(failed ? { line: `✗ ${id} ${kind} FAIL ${failureClass}${failedRecord?.exit_reason === 'timeout' && failedRecord?.timeout_at_turn !== undefined ? ` at turn ${failedRecord.timeout_at_turn}` : ''}${error ? ` — ${error}` : ''} [slice ${result.sliceIndex}, attempt ${result.attempt ?? 1}] rerun: ${rerun}` } : {}) }); + ...(failed ? { line: `✗ ${id} ${kind} FAIL ${failureClass}${failedRecord?.exit_reason === 'timeout' && failedRecord?.timeout_at_turn !== undefined ? ` at turn ${failedRecord.timeout_at_turn}` : ''}${error ? ` — ${error}` : ''}` + + `${diagnosisText(diagnosis, error)} [slice ${result.sliceIndex}, attempt ${result.attempt ?? 1}]` + + `${evidencePointer(sliceArtifactName(root, result.sliceIndex, result.attempt ?? 1), key, typeof base.run_id === 'string' ? base.run_id : undefined, id)}` + + ` after a repair: ${afterRepairCommand(manifest.tier, id, shardFile(key))}` } : {}) }); census.history.push({ ...base, case: id, file: shardFile(key), kind, trial: 1, panel: { n: 1, k: 1 }, outcome: caseOutcome, ...(failureClass ? { failure_class: failureClass } : {}), ...(failedRecord?.exit_reason ? { exit_reason: String(failedRecord.exit_reason) } : {}), ...(error && failed ? { error } : {}), ...diagnosis, ...trialSessions(sessions), duration_ms: cases.reduce((sum, tc) => sum + tc.timeMs, 0), @@ -339,7 +348,9 @@ export function panelReports(manifest: PaidRunManifest, results: SliceResult[], return [{ trial: t.trial, outcome: t.outcome, attempt, ...(t.failure_class ? { failure_class: t.failure_class } : {}), ...(t.exit_reason ? { exit_reason: t.exit_reason } : {}), ...(t.error ? { error: t.error } : {}), ...(got.outcome.reused ? { execution: 'reused' as const } : {}), - ...(t.timeout_at_turn !== undefined ? { timeout_at_turn: t.timeout_at_turn } : {}) }]; + ...(t.timeout_at_turn !== undefined ? { timeout_at_turn: t.timeout_at_turn } : {}), + ...(t.failure_cause ? { failure_cause: t.failure_cause } : {}), ...(t.failure_cause_evidence ? { failure_cause_evidence: t.failure_cause_evidence } : {}), + ...(t.failure_detail ? { failure_detail: t.failure_detail } : {}) }]; }); const verdict = panelVerdict({ case: shardCaseId(key)!, kind: plan.kind, panel: plan.panel, trials, quarantined: plan.quarantined }); // Reuse is whole-panel only: every trial reused from one run, or none. @@ -352,29 +363,62 @@ export function panelReports(manifest: PaidRunManifest, results: SliceResult[], }); } +type TrialDiagnosis = { failure_cause?: string; failure_cause_evidence?: string; failure_detail?: TrialFailureDetail }; + +/** Cause (when it says more than the class), its evidence and the failure detail, appended to a red line. */ +function diagnosisText(d: TrialDiagnosis | undefined, error?: string): string { + if (!d?.failure_cause) return ''; + const evidence = d.failure_cause_evidence && d.failure_cause_evidence !== error ? `: ${d.failure_cause_evidence}` : ''; + const detail = !d.failure_detail ? '' + : 'judge' in d.failure_detail ? ` ${d.failure_detail.judge.map(j => `${j.dimension} ${j.mean} < ${j.threshold} (${j.samples} samples)${j.rationale ? ` "${j.rationale}"` : ''}`).join('; ')}` + : ` Expected: ${d.failure_detail.expected} · Received: ${d.failure_detail.received}`; + return ` · cause ${d.failure_cause}${evidence}${detail}`; +} + +/** Where a red's transcript lives and how to fetch it. */ +function evidencePointer(artifact: string, key: string, runId: string | undefined, id: string): string { + const arg = /^[\w./-]+$/.test(id) ? id : `'${id.replace(/'/g, `'\\''`)}'`; + return ` evidence: ${artifact}/shards/${shardSlug([key])}${runId ? ` (fetch: bun run eval:pass-rates --run ${runId} --case ${arg})` : ''}`; +} + /** Why one failed trial failed, in one line: a case timeout names its turn. */ -function trialCause(trial: PanelVerdict['trials'][number] & { timeout_at_turn?: number }): string { +function trialCause(trial: PanelVerdict['trials'][number] & TrialDiagnosis & { timeout_at_turn?: number }): string { if (trial.outcome === 'skipped') return `t${trial.trial}: skipped`; const cls = trial.failure_class ?? 'assertion'; const head = cls === 'timeout' || trial.exit_reason === 'timeout' ? `timeout${trial.timeout_at_turn !== undefined ? ` at turn ${trial.timeout_at_turn}` : ''}` : cls; - return `t${trial.trial}: ${head}${trial.exit_reason && trial.exit_reason !== 'timeout' ? ` (${trial.exit_reason})` : ''}${trial.error ? ` — ${trial.error}` : ''}`; + const exit = trial.exit_reason === 'success' ? ' (session completed; check failed)' + : trial.exit_reason && trial.exit_reason !== 'timeout' ? ` (${trial.exit_reason})` : ''; + return `t${trial.trial}: ${head}${exit}${trial.error ? ` — ${trial.error}` : ''}${diagnosisText(trial, trial.error)}`; } -export function rerunCommand(tier: PaidTier, id: string, trials: number): string { - return `bun run scripts/test-paid-shards.ts --tier ${tier} --case ${id}${trials > 1 ? ` --trials ${trials}` : ''}`; +/** + * A paid run of exactly one case, for after a repair (never on unchanged + * inputs): E2E cases through --case, judges through their selection and an + * exact test-name filter, a keyless file (its own case) by file. + */ +export function afterRepairCommand(tier: PaidTier, id: string, file: string, trials = 1): string { + if (Object.hasOwn(E2E_TIERS, id)) return `bun run scripts/test-paid-shards.ts --tier ${tier} --case ${id}${trials > 1 ? ` --trials ${trials}` : ''}`; + if (Object.hasOwn(LLM_JUDGE_TOUCHFILES, id)) { + return `EVALS=1 EVALS_TIER=${tier} EVALS_JUDGE_SELECTION_JSON='${JSON.stringify({ version: 1, selected: [id], reason: 'after a repair' })}' bun test ${file} -t '${caseTestNamePattern([id])}'`; + } + return `EVALS=1 EVALS_TIER=${tier} bun test ${file}`; } -/** One line per non-PASS or split panel verdict. */ -export function formatPanelLine(panel: PanelReport, tier: PaidTier): string { +/** One line per non-PASS or split panel verdict. `artifacts` names each slice's artifact (slice -> name). */ +export function formatPanelLine(panel: PanelReport, tier: PaidTier, ctx: { artifacts?: Map; runId?: string } = {}): string { const mark = panel.status === 'PASS' ? '⚠' : panel.failsLane ? '✗' : '◌'; const label = panel.status === 'PASS' ? `PASS ${panel.passed}/${panel.panel.n}` : `${panel.status} ${panel.passed}/${panel.panel.n}`; const causes = panel.trials.filter(t => t.outcome !== 'passed').map(t => trialCause(t as any)); const where = Object.entries(panel.slices).map(([trial, slice]) => `t${trial}@slice ${slice}`).join(', '); + const red = panel.trials.find(t => t.outcome === 'failed'); + const slice = red ? panel.slices[red.trial] : undefined; + const evidence = red && slice !== undefined + ? evidencePointer(ctx.artifacts?.get(slice) ?? `slice ${slice} attempt ${panel.attempt}`, trialShardKey(panel.file, panel.case, red.trial), ctx.runId, panel.case) : ''; return `${mark} ${panel.case} ${panel.kind}${panel.quarantined ? ' (quarantined)' : ''} ${label} (${panel.marks})` + `${causes.length ? ` ${causes.join('; ')}` : ''}${panel.status === 'INCOMPLETE' || panel.status === 'SKIPPED' ? ` [${panel.reason}]` : ''}` - + `${where ? ` [${where}, attempt ${panel.attempt}]` : ''} rerun: ${rerunCommand(tier, panel.case, panel.panel.n)}`; + + `${where ? ` [${where}, attempt ${panel.attempt}]` : ''}${evidence} after a repair: ${afterRepairCommand(tier, panel.case, panel.file, panel.panel.n)}`; } export interface ReportHeadline { @@ -595,6 +639,8 @@ export function runPaidReport(reportDir: string, options: { writeDurations?: boo quarantined: t.quarantined, execution: outcome.reused ? 'reused' : 'executed', source: 'shard' } as TrialOutcomeRecord); } } + const sliceArtifacts = new Map(artifacts.filter(a => (a.result.attempt ?? 1) === primary) + .map(a => [a.result.sliceIndex, sliceArtifactName(a.root, a.result.sliceIndex, primary)] as const)); const census = junitCensus(manifest, artifacts.filter(a => (a.result.attempt ?? 1) === primary), recordsByShard, common(primary), env.GSTACK_CLAUDE_CLI_VERSION); history.push(...census.history); @@ -620,7 +666,7 @@ export function runPaidReport(reportDir: string, options: { writeDurations?: boo if (got && got.o.status === 'passed') return []; if (got && census.failedShards.has(normalizeRelativePath(entry.file))) return []; const id = shardCaseId(entry.file); - return [`✗ ${entry.file} rule shard ${got ? got.o.status : 'NOT REPORTED'}${got?.o.runnerError ? ` — ${sanitizeTrialError(got.o.runnerError)}` : ''} [slice ${entry.slice}, attempt ${primary}]${id ? ` rerun: ${rerunCommand(manifest.tier, id, 1)}` : ''}`]; + return [`✗ ${entry.file} rule shard ${got ? got.o.status : 'NOT REPORTED'}${got?.o.runnerError ? ` — ${sanitizeTrialError(got.o.runnerError)}` : ''} [slice ${entry.slice}, attempt ${primary}]${id ? ` after a repair: ${afterRepairCommand(manifest.tier, id, shardFile(entry.file))}` : ''}`]; }); const behaviorPanels = panels.filter(p => !p.quarantined && p.kind === 'behavior'); const lanePanels = panels.filter(p => !p.quarantined); @@ -634,7 +680,7 @@ export function runPaidReport(reportDir: string, options: { writeDurations?: boo const failureLines = [ ...ruleShardFailures, ...ruleCases.filter(c => c.line).map(c => c.line!), - ...panels.filter(p => p.status !== 'PASS' || p.split).map(p => formatPanelLine(p, manifest.tier)), + ...panels.filter(p => p.status !== 'PASS' || p.split).map(p => formatPanelLine(p, manifest.tier, { artifacts: sliceArtifacts, runId })), ]; const red = verdict.problems.length > 0; const headline: ReportHeadline = { @@ -662,7 +708,7 @@ export function runPaidReport(reportDir: string, options: { writeDurations?: boo for (const line of failureLines) console.log(` ${line}`); } for (const panel of laterPanels) console.log(` attempt ${panel.attempt} (re-run; reported, never replacing attempt ${primary}): ${formatPanelLine(panel, manifest.tier)}`); - const fence = (lines: string[]) => ['```', ...lines.map(line => line.replace(/```/g, "'''")), '```']; + const fence = (lines: string[]) => publishedFence(lines); fs.writeFileSync(summaryMdPath, [ ...fence(headlineLines), ...(failureLines.length ? ['', '**Failures and split verdicts**', '', ...fence(failureLines)] : []), @@ -678,7 +724,7 @@ export function runPaidReport(reportDir: string, options: { writeDurations?: boo marks: p.marks, split: p.split, quarantined: p.quarantined, failsLane: p.failsLane, redClass: p.redClass, reason: p.reason, trials: p.trials.map(t => ({ trial: t.trial, outcome: t.outcome, ...(t.failure_class ? { failure_class: t.failure_class } : {}), ...(t.exit_reason ? { exit_reason: t.exit_reason } : {}), ...(t.error ? { error: t.error } : {}) })) })), - failures: failureLines.map(line => sanitizeTrialError(line) ?? line) }, null, 2) + '\n'); + failures: failureLines.map(line => sanitizePublishedText(line)) }, null, 2) + '\n'); if (verdict.problems.length) { console.error(`[test:paid] report: ${verdict.problems.length} problem(s):`); for (const problem of verdict.problems) console.error(` ✗ ${problem}`); diff --git a/scripts/lib/published-text.ts b/scripts/lib/published-text.ts new file mode 100644 index 000000000..5cfddb2a3 --- /dev/null +++ b/scripts/lib/published-text.ts @@ -0,0 +1,36 @@ +/** + * The one sanitizer for model- or fixture-written text that reaches a public + * issue, PR comment or committed file (census report, tracking issue, + * eval:pass-rates output). Control and bidi characters are stripped, the + * repo's credential-scan patterns (lib/redact-engine, the engine the quality + * gate's credential scan runs) are redacted, @-mentions are neutralized and + * length is capped. Inline text also has HTML and Markdown link brackets + * escaped; blocks go inside a code fence longer than any backtick run they + * contain, so their content stays literal and cannot close the fence. + */ +import { redactFindingSpans } from '../../lib/redact-engine'; + +export const PUBLISHED_TEXT_MAX = 1000; + +const cap = (text: string, max: number) => (text.length > max ? `${text.slice(0, max - 1)}…` : text); + +function clean(text: string): string { + // eslint-disable-next-line no-control-regex + const stripped = text.replace(/[\u0000-\u0008\u000a-\u001f\u007f-\u009f\u200b-\u200f\u202a-\u202e\u2066-\u2069]/g, ' ').slice(0, 64 * 1024); + const redacted = redactFindingSpans(stripped) ?? '[redacted: credential-shaped text that could not be isolated]'; + return redacted.replace(/@(?=[A-Za-z0-9_-])/g, '@\u200b'); +} + +/** One inline line: cleaned, HTML and Markdown link brackets escaped, capped. */ +export function sanitizePublishedText(text: string, max = PUBLISHED_TEXT_MAX): string { + return cap(clean(text).replace(/&/g, '&').replace(//g, '>') + .replace(/\[/g, '[').replace(/\]/g, ']'), max); +} + +/** Cleaned, capped lines inside a fence longer than the longest backtick run they contain. */ +export function publishedFence(lines: readonly string[], max = PUBLISHED_TEXT_MAX): string[] { + const body = lines.map(line => cap(clean(line), max)); + const longest = Math.max(0, ...body.flatMap(line => (line.match(/`+/g) ?? []).map(run => run.length))); + const fence = '`'.repeat(Math.max(3, longest + 1)); + return [fence, ...body, fence]; +} diff --git a/test/paid-report-census.test.ts b/test/paid-report-census.test.ts index 9d0b3dfcb..3b7e37c14 100644 --- a/test/paid-report-census.test.ts +++ b/test/paid-report-census.test.ts @@ -11,7 +11,7 @@ import { afterAll, beforeAll, describe, expect, test } from 'bun:test'; import * as fs from 'node:fs'; import * as os from 'node:os'; import * as path from 'node:path'; -import { DESELECTION_REASONS, RUNTIME_SKIP_REASON, attributeJUnitCase, formatPanelLine, runPaidReport, type JUnitCase } from '../scripts/test-paid-shards'; +import { DESELECTION_REASONS, RUNTIME_SKIP_REASON, attributeJUnitCase, formatPanelLine, resolvePaidShardBudget, runPaidReport, type JUnitCase } from '../scripts/test-paid-shards'; import { panelVerdict } from './helpers/eval-store'; import { trialRecordProblems as trialRecordProblemsV1 } from './fixtures/trial-record-v1-reader'; @@ -22,7 +22,7 @@ beforeAll(() => { base = fs.mkdtempSync(path.join(os.tmpdir(), 'paid-report-cens afterAll(() => { fs.rmSync(base, { recursive: true, force: true }); }); let runs = 0; -function report(lane: 'periodic' | 'gate', edit: (dir: string) => void = () => {}) { +function report(lane: 'periodic' | 'gate', edit: (dir: string) => void = () => {}, env: NodeJS.ProcessEnv = {}) { const dir = path.join(base, `${lane}-${++runs}`); fs.cpSync(path.join(FIXTURES, lane), dir, { recursive: true }); edit(dir); @@ -31,7 +31,7 @@ function report(lane: 'periodic' | 'gate', edit: (dir: string) => void = () => { console.log = (...args: unknown[]) => { lines.push(args.join(' ')); }; console.error = console.log; let status: number; - try { status = runPaidReport(dir, { env: {} }); } finally { console.log = log; console.error = error; } + try { status = runPaidReport(dir, { env }); } finally { console.log = log; console.error = error; } const history = fs.readFileSync(path.join(dir, 'trial-outcomes.jsonl'), 'utf8').split('\n').filter(Boolean).map(line => JSON.parse(line)); return { status, out: lines.join('\n'), dir, history }; } @@ -124,6 +124,83 @@ describe('record builders attach ledger sessions and failure causes (plan 0.1/0. }); }); +describe('red lines carry their values, evidence and a one-case command (plan 0.3)', () => { + const QA_ONLY = 'qa-only/SKILL.md workflow'; + const addJudgeRed = (dir: string) => { + const shard = path.join(dir, 'paid-slice-1-a1', 'shards', 'skill-llm-eval'); + fs.mkdirSync(shard, { recursive: true }); + const message = 'expect(received).toBeGreaterThanOrEqual(expected)\n\nExpected: >= 4\nReceived: 3.6666666666666665\n'; + fs.writeFileSync(path.join(shard, 'junit.xml'), `\n` + + `` + + `\n`); + // Trimmed from census 37198445662's paid-slice-5-a1 qa-only record. + fs.writeFileSync(path.join(shard, '1.91.19-x-llm-judge-2026-10-04-1129.json'), JSON.stringify({ tier: 'llm-judge', total_cost_usd: 0.06, tests: [{ + name: QA_ONLY, suite: 'Other skill evals', tier: 'llm-judge', passed: false, duration_ms: 34829, cost_usd: 0.06, execution: 'executed', + judge_scores: { clarity: 3.6666666666666665, completeness: 4.333333333333333, actionability: 3.6666666666666665 }, + judge_reasoning: '[sample 1] The bundle is thorough. Clarity suffers from extreme density and cross-file indirection. Two actionability concerns: Browser Quick allots 30 seconds.\n[sample 2] Deliverables are concrete.\n[sample 3] Execution is feasible.', + exit_reason: 'validation_failed', error: message.replace('>', '>') }] })); + const manifestPath = path.join(dir, 'manifest.json'); + const manifest = JSON.parse(fs.readFileSync(manifestPath, 'utf8')); + const budget = resolvePaidShardBudget(['test/skill-llm-eval.test.ts']); + manifest.entries.push({ file: 'test/skill-llm-eval.test.ts', slice: 1, status: 'planned', estimatedMs: 35_000, budget }); + fs.writeFileSync(manifestPath, JSON.stringify(manifest)); + const slicePath = path.join(dir, 'paid-slice-1-a1', 'slice-1.json'); + const slice = JSON.parse(fs.readFileSync(slicePath, 'utf8')); + slice.outcomes.push({ files: ['test/skill-llm-eval.test.ts'], status: 'failed', exitCode: 1, elapsedMs: 35_000, executedTests: 1, skippedTests: 0, budget }); + fs.writeFileSync(slicePath, JSON.stringify(slice)); + }; + + test('a judge red names each failing dimension, its mean, samples and rationale, its artifact and an exact one-case command', () => { + const r = report('gate', addJudgeRed, { GITHUB_RUN_ID: '37198445662' }); + const line = r.out.split('\n').find(l => l.includes(`✗ ${QA_ONLY}`))!; + expect(line).toContain('FAIL assertion — expect(received).toBeGreaterThanOrEqual(expected)'); + expect(line).toContain('· cause assertion clarity 3.67 < 4 (3 samples) "Clarity suffers from extreme density and cross-file indirection."; ' + + 'actionability 3.67 < 4 (3 samples) "Two actionability concerns: Browser Quick allots 30 seconds."'); + expect(line).toContain('evidence: paid-slice-1-a1/shards/skill-llm-eval (fetch: bun run eval:pass-rates --run 37198445662 --case \'qa-only/SKILL.md workflow\')'); + expect(line).toContain(`after a repair: EVALS=1 EVALS_TIER=gate EVALS_JUDGE_SELECTION_JSON='{"version":1,"selected":["${QA_ONLY}"],"reason":"after a repair"}' ` + + `bun test test/skill-llm-eval.test.ts -t '(?:^|\\s)(?:qa-only/SKILL\\.md workflow)$'`); + expect(line).not.toContain('rerun:'); + const record = r.history.find(x => x.case === QA_ONLY); + expect(record).toMatchObject({ failure_class: 'assertion', error: 'expect(received).toBeGreaterThanOrEqual(expected)', failure_cause: 'assertion', + failure_detail: { judge: [expect.objectContaining({ dimension: 'clarity', mean: 3.67 }), expect.objectContaining({ dimension: 'actionability', mean: 3.67 })] } }); + expect(trialRecordProblemsV1(record)).toEqual([]); + }); + + test('a behavior panel line names the failed trial\'s cause, values, slice artifact and an n-trial one-case command', () => { + const verdict = panelVerdict({ case: 'plan-review-prosons-neutral-neg', kind: 'behavior', panel: { n: 3, k: 2 }, quarantined: false, trials: [ + { trial: 1, outcome: 'failed', failure_class: 'assertion', exit_reason: 'success', error: 'expect(received).not.toMatch(expected)', + failure_cause: 'assertion', failure_cause_evidence: 'session completed; check failed', + failure_detail: { expected: 'not /taste call/i', received: '"a taste call"' } } as any, + { trial: 2, outcome: 'failed', failure_class: 'assertion' }, { trial: 3, outcome: 'passed' }] }); + const line = formatPanelLine({ ...verdict, file: 'test/skill-e2e-plan-prosons.test.ts', slices: { 1: 7, 2: 9, 3: 9 } }, 'periodic', + { artifacts: new Map([[7, 'paid-slice-7-a1']]), runId: '37193478719' }); + expect(line).toContain('t1: assertion (session completed; check failed) — expect(received).not.toMatch(expected) · cause assertion: session completed; check failed Expected: not /taste call/i · Received: "a taste call"'); + expect(line).toContain('evidence: paid-slice-7-a1/shards/skill-e2e-plan-prosons--plan-review-prosons-neutral-neg.t1 (fetch: bun run eval:pass-rates --run 37193478719 --case plan-review-prosons-neutral-neg)'); + expect(line).toContain('after a repair: bun run scripts/test-paid-shards.ts --tier periodic --case plan-review-prosons-neutral-neg --trials 3'); + }); + + test('a rule red shows Expected/Received and selects only its case; the summary fence cannot be broken', () => { + const r = report('gate', dir => { + const file = path.join(dir, 'paid-slice-1-a1', 'shards', 'skill-e2e-plan-mode-no-op', 'junit.xml'); + fs.writeFileSync(file, fs.readFileSync(file, 'utf8').replace(/(]*?) \/>/, + '$1>')); + addJudgeRed(dir); + }); + const line = r.out.split('\n').find(l => l.includes('✗ plan-mode-no-op'))!; + expect(line).toContain('Expected: "asked" · Received: "```@\u200bteam"'); + expect(line).toContain('evidence: paid-slice-1-a1/shards/skill-e2e-plan-mode-no-op'); + expect(line).not.toContain('(fetch:'); + expect(line).toContain('after a repair: bun run scripts/test-paid-shards.ts --tier gate --case plan-mode-no-op'); + const summary = fs.readFileSync(path.join(r.dir, 'report-summary.md'), 'utf8'); + const lines = summary.split('\n'); + const at = lines.findIndex(l => l.includes('✗ plan-mode-no-op')); + const open = lines.slice(0, at).findLast(l => /^`{3,}$/.test(l))!; + expect(open.length).toBeGreaterThan(3); + expect(lines.slice(at).find(l => /^`{3,}$/.test(l))).toBe(open); + expect(summary).not.toMatch(/@team/); + }); +}); + describe('attributeJUnitCase', () => { const tc = (name: string, outcome: JUnitCase['outcome'], line: number | null = 1): JUnitCase => ({ name, classname: 's', outcome, timeMs: 0, ...(line === null ? {} : { line }) }); diff --git a/test/published-text.test.ts b/test/published-text.test.ts new file mode 100644 index 000000000..5754bafe6 --- /dev/null +++ b/test/published-text.test.ts @@ -0,0 +1,64 @@ +/** + * The published-text sanitizer (plan 0.3): adversarial model or fixture text + * reaching a public issue, PR comment or committed report cannot ping people, + * break out of its fence, inject HTML or images, leak a credential, or flood. + */ +import { describe, expect, test } from 'bun:test'; +import { publishedFence, sanitizePublishedText, PUBLISHED_TEXT_MAX } from '../scripts/lib/published-text'; + +// Built at runtime so this file never contains a credential-shaped literal. +const GHP = ['ghp', '_', 'A1b2C3d4E5f6G7h8I9j0K1l2M3n4O5p6Q7r8'].join(''); +const SK = ['sk', '-ant-api03-', 'Zx9'.repeat(30)].join(''); +const ADVERSARIAL = [ + 'ping @garrytan and @gstack/maintainers', + 'close ``` the fence ```` here', + '', + '![tracker](https://evil.example/p.png) [click](https://evil.example)', + `token ${GHP} and key ${SK}`, + 'bell\u0007 escape\u001b[31m bidi\u202eevil', +]; + +describe('sanitizePublishedText (inline)', () => { + const out = ADVERSARIAL.map(line => sanitizePublishedText(line)); + test('mentions are neutralized', () => { + expect(out[0]).not.toMatch(/@[A-Za-z]/); + expect(out[0]).toContain('@\u200bgarrytan'); + }); + test('HTML and Markdown links or images cannot render', () => { + expect(out[2]).not.toContain('<'); + expect(out[2]).toContain('</details>'); + expect(out[3]).not.toMatch(/!?\[[^\]]*\]\(/); + }); + test('credential-scan patterns are redacted', () => { + expect(out[4]).not.toContain(GHP); + expect(out[4]).not.toContain(SK); + }); + test('control and bidi characters are stripped', () => { + // eslint-disable-next-line no-control-regex + expect(out[5]).not.toMatch(/[\u0000-\u001f\u007f\u202e]/); + }); + test('length is capped', () => { + expect(sanitizePublishedText('x'.repeat(10 * 1024))).toHaveLength(PUBLISHED_TEXT_MAX); + }); +}); + +describe('publishedFence (blocks)', () => { + const block = publishedFence([...ADVERSARIAL, 'y'.repeat(10 * 1024)]); + test('the fence is longer than any backtick run inside it, so nothing closes it early', () => { + const fence = block[0]!; + expect(block.at(-1)).toBe(fence); + expect(fence).toMatch(/^`{5,}$/); + for (const line of block.slice(1, -1)) { + for (const run of line.match(/`+/g) ?? []) expect(run.length).toBeLessThan(fence.length); + } + }); + test('content stays literal inside the fence but loses mentions, credentials, control characters and length', () => { + const body = block.slice(1, -1).join('\n'); + expect(body).not.toMatch(/@[A-Za-z]/); + expect(body).not.toContain(GHP); + expect(body).not.toContain(SK); + expect(body).not.toContain('\u202e'); + expect(block.at(-2)).toHaveLength(PUBLISHED_TEXT_MAX); + expect(body).toContain(''); + }); +});