mirror of
https://github.com/garrytan/gstack.git
synced 2026-10-31 22:14:25 +01:00
0.3: red lines name their values, evidence and an after-a-repair command; one published-text sanitizer
Each red line in the census report keeps its stored error and adds the machine cause with its evidence, the failure_detail values (Expected/Received, or each failing judge dimension with its mean, sample count and the sample sentence naming it), the slice artifact and shard directory that hold the transcript, a fetch command (eval:pass-rates --run <id> --case <id>), and 'after a repair:' with a command that selects only that case (E2E ids via --case, judges via their selection and an exact test-name filter). A session that completed and failed a check says so instead of 'assertion (success)'. Report text that reaches a PR comment, issue or committed file goes through scripts/lib/published-text.ts: control and bidi characters stripped, credential-scan patterns redacted, mentions neutralized, HTML and link brackets escaped inline, blocks in a fence longer than any backtick run, length capped. Verdicts and collector verdict objects are unchanged.
This commit is contained in:
garrytan committed
1 parent
80fe5de7bd
commit
d725d94ea3
4 files changed
+242
-19
No files matched your search
+62
-16
@@ -33,9 +33,10 @@ import {
|
||||
getProjectEvalDir, getClaudeCliVersion, isFinalizedEvalResultFile, evalEntryOutcome, failureClassOf, panelVerdict,
|
||||
sanitizeTrialError, formatTrialOutcomes, CONTRACT_VIOLATIONS_FILE, TRIAL_ENV, TRIAL_OUTCOME_SCHEMA, TRIAL_OUTCOMES_FILE,
|
||||
type EvalCaseKind, type PanelShape, type PanelVerdict, type TrialFailureClass, type TrialOutcome, type TrialOutcomeRecord,
|
||||
trialFailureFields, trialSessions,
|
||||
trialFailureFields, trialSessions, type TrialFailureDetail,
|
||||
} from '../../test/helpers/eval-store';
|
||||
import { readSessionLedger, type SessionLedgerRow } from '../../test/helpers/session-ledger';
|
||||
import { publishedFence, sanitizePublishedText } from './published-text';
|
||||
import { E2E_KINDS } from '../../test/helpers/touchfiles-data';
|
||||
import { manualReviewProblem } from '../../test/helpers/cookie-workflow-manual-review';
|
||||
import { preflightAnthropicApi } from '../../test/helpers/anthropic-preflight';
|
||||
@@ -57,7 +58,7 @@ export { PAID_TEST_GLOBS, isPaidTestFile };
|
||||
export { PERIODIC_CI_EXCLUDE };
|
||||
|
||||
type E2EShardReuse = NonNullable<ReturnType<typeof prepareE2EShardReuse>>;
|
||||
import { CASE_TEST_NAMES, type CaseTrialPlan, caseTrialPlan, shardCaseId, shardFile, trialShardKey } from './paid-cases';
|
||||
import { CASE_TEST_NAMES, type CaseTrialPlan, caseTestNamePattern, caseTrialPlan, shardCaseId, shardFile, trialShardKey } from './paid-cases';
|
||||
import { type ManifestEntry, PAID_TEST_DURATIONS_FILE, type PaidRunManifest, type SliceResult, collectorOutcomeCounts, formatProfileCoverage, loadPaidTestDurations, mergePaidTestDurations, parseRunManifest, trialPanelKey, verifySliceResults, writePaidTestDurations } from './paid-plan';
|
||||
import { DEFAULT_JOBS, type PaidCaseSelection, type PaidTier, ROOT, type RunShardsOptions, type ShardTrialRecord, collectPaidTestFiles, fileCaseRegistration, isAllSkippedPass, runPaidShards, shardSlug, paidSelectionEnv } from '../test-paid-shards';
|
||||
|
||||
@@ -232,6 +233,12 @@ function caseSessions(rows: SessionLedgerRow[], id: string, onlyCase: boolean):
|
||||
return rows.filter(row => row.case === id || (!row.case && (onlyCase || row.test_name === id || row.test_name === CASE_TEST_NAMES[id])));
|
||||
}
|
||||
|
||||
/** The artifact holding a slice's shards: the downloaded directory's own name when it is one. */
|
||||
export function sliceArtifactName(root: string, slice: number, attempt: number): string {
|
||||
const name = path.basename(root);
|
||||
return /^(?:paid-slice|gate-census)-\d+-a\d+$/.test(name) ? name : `slice ${slice} attempt ${attempt}`;
|
||||
}
|
||||
|
||||
export function junitCensus(manifest: PaidRunManifest, artifacts: ReportArtifact[], recordsByShard: Map<string, any[]>,
|
||||
base: Record<string, unknown>, cliVersion?: string): JUnitCensus {
|
||||
const census: JUnitCensus = { ruleCases: [], history: [], failedShards: new Set(), skipped: [], deselected: new Map(), shardSkipReasons: new Map(), unattributed: [] };
|
||||
@@ -262,7 +269,6 @@ export function junitCensus(manifest: PaidRunManifest, artifacts: ReportArtifact
|
||||
const failureClass: TrialFailureClass | undefined = !failed ? undefined
|
||||
: failed.failureType === 'TimeoutError' ? 'timeout' : failedRecord ? failureClassOf(failedRecord) : 'assertion';
|
||||
const error = sanitizeTrialError(failedRecord?.error ?? failed?.message);
|
||||
const rerun = Object.hasOwn(E2E_TIERS, id) ? rerunCommand(manifest.tier, id, 1) : `EVALS=1 EVALS_TIER=${manifest.tier} bun test ${shardFile(key)}`;
|
||||
const sessions = caseSessions(ledger, id, byCase.size === 1);
|
||||
const diagnosis = failed && failureClass ? trialFailureFields({ failure_class: failureClass, exit_reason: failedRecord?.exit_reason,
|
||||
error: failedRecord?.error ?? failed.message, sessions, record: failedRecord }) : undefined;
|
||||
@@ -272,7 +278,10 @@ export function junitCensus(manifest: PaidRunManifest, artifacts: ReportArtifact
|
||||
census.skipped.push(`◌ ${id} ${key} (${cases.map(tc => tc.name).join('; ')}): ${RUNTIME_SKIP_REASON}`);
|
||||
}
|
||||
census.ruleCases.push({ id, kind, outcome: caseOutcome,
|
||||
...(failed ? { line: `✗ ${id} ${kind} FAIL ${failureClass}${failedRecord?.exit_reason === 'timeout' && failedRecord?.timeout_at_turn !== undefined ? ` at turn ${failedRecord.timeout_at_turn}` : ''}${error ? ` — ${error}` : ''} [slice ${result.sliceIndex}, attempt ${result.attempt ?? 1}] rerun: ${rerun}` } : {}) });
|
||||
...(failed ? { line: `✗ ${id} ${kind} FAIL ${failureClass}${failedRecord?.exit_reason === 'timeout' && failedRecord?.timeout_at_turn !== undefined ? ` at turn ${failedRecord.timeout_at_turn}` : ''}${error ? ` — ${error}` : ''}`
|
||||
+ `${diagnosisText(diagnosis, error)} [slice ${result.sliceIndex}, attempt ${result.attempt ?? 1}]`
|
||||
+ `${evidencePointer(sliceArtifactName(root, result.sliceIndex, result.attempt ?? 1), key, typeof base.run_id === 'string' ? base.run_id : undefined, id)}`
|
||||
+ ` after a repair: ${afterRepairCommand(manifest.tier, id, shardFile(key))}` } : {}) });
|
||||
census.history.push({ ...base, case: id, file: shardFile(key), kind, trial: 1, panel: { n: 1, k: 1 }, outcome: caseOutcome,
|
||||
...(failureClass ? { failure_class: failureClass } : {}), ...(failedRecord?.exit_reason ? { exit_reason: String(failedRecord.exit_reason) } : {}),
|
||||
...(error && failed ? { error } : {}), ...diagnosis, ...trialSessions(sessions), duration_ms: cases.reduce((sum, tc) => sum + tc.timeMs, 0),
|
||||
@@ -339,7 +348,9 @@ export function panelReports(manifest: PaidRunManifest, results: SliceResult[],
|
||||
return [{ trial: t.trial, outcome: t.outcome, attempt, ...(t.failure_class ? { failure_class: t.failure_class } : {}),
|
||||
...(t.exit_reason ? { exit_reason: t.exit_reason } : {}), ...(t.error ? { error: t.error } : {}),
|
||||
...(got.outcome.reused ? { execution: 'reused' as const } : {}),
|
||||
...(t.timeout_at_turn !== undefined ? { timeout_at_turn: t.timeout_at_turn } : {}) }];
|
||||
...(t.timeout_at_turn !== undefined ? { timeout_at_turn: t.timeout_at_turn } : {}),
|
||||
...(t.failure_cause ? { failure_cause: t.failure_cause } : {}), ...(t.failure_cause_evidence ? { failure_cause_evidence: t.failure_cause_evidence } : {}),
|
||||
...(t.failure_detail ? { failure_detail: t.failure_detail } : {}) }];
|
||||
});
|
||||
const verdict = panelVerdict({ case: shardCaseId(key)!, kind: plan.kind, panel: plan.panel, trials, quarantined: plan.quarantined });
|
||||
// Reuse is whole-panel only: every trial reused from one run, or none.
|
||||
@@ -352,29 +363,62 @@ export function panelReports(manifest: PaidRunManifest, results: SliceResult[],
|
||||
});
|
||||
}
|
||||
|
||||
type TrialDiagnosis = { failure_cause?: string; failure_cause_evidence?: string; failure_detail?: TrialFailureDetail };
|
||||
|
||||
/** Cause (when it says more than the class), its evidence and the failure detail, appended to a red line. */
|
||||
function diagnosisText(d: TrialDiagnosis | undefined, error?: string): string {
|
||||
if (!d?.failure_cause) return '';
|
||||
const evidence = d.failure_cause_evidence && d.failure_cause_evidence !== error ? `: ${d.failure_cause_evidence}` : '';
|
||||
const detail = !d.failure_detail ? ''
|
||||
: 'judge' in d.failure_detail ? ` ${d.failure_detail.judge.map(j => `${j.dimension} ${j.mean} < ${j.threshold} (${j.samples} samples)${j.rationale ? ` "${j.rationale}"` : ''}`).join('; ')}`
|
||||
: ` Expected: ${d.failure_detail.expected} · Received: ${d.failure_detail.received}`;
|
||||
return ` · cause ${d.failure_cause}${evidence}${detail}`;
|
||||
}
|
||||
|
||||
/** Where a red's transcript lives and how to fetch it. */
|
||||
function evidencePointer(artifact: string, key: string, runId: string | undefined, id: string): string {
|
||||
const arg = /^[\w./-]+$/.test(id) ? id : `'${id.replace(/'/g, `'\\''`)}'`;
|
||||
return ` evidence: ${artifact}/shards/${shardSlug([key])}${runId ? ` (fetch: bun run eval:pass-rates --run ${runId} --case ${arg})` : ''}`;
|
||||
}
|
||||
|
||||
/** Why one failed trial failed, in one line: a case timeout names its turn. */
|
||||
function trialCause(trial: PanelVerdict['trials'][number] & { timeout_at_turn?: number }): string {
|
||||
function trialCause(trial: PanelVerdict['trials'][number] & TrialDiagnosis & { timeout_at_turn?: number }): string {
|
||||
if (trial.outcome === 'skipped') return `t${trial.trial}: skipped`;
|
||||
const cls = trial.failure_class ?? 'assertion';
|
||||
const head = cls === 'timeout' || trial.exit_reason === 'timeout'
|
||||
? `timeout${trial.timeout_at_turn !== undefined ? ` at turn ${trial.timeout_at_turn}` : ''}`
|
||||
: cls;
|
||||
return `t${trial.trial}: ${head}${trial.exit_reason && trial.exit_reason !== 'timeout' ? ` (${trial.exit_reason})` : ''}${trial.error ? ` — ${trial.error}` : ''}`;
|
||||
const exit = trial.exit_reason === 'success' ? ' (session completed; check failed)'
|
||||
: trial.exit_reason && trial.exit_reason !== 'timeout' ? ` (${trial.exit_reason})` : '';
|
||||
return `t${trial.trial}: ${head}${exit}${trial.error ? ` — ${trial.error}` : ''}${diagnosisText(trial, trial.error)}`;
|
||||
}
|
||||
|
||||
export function rerunCommand(tier: PaidTier, id: string, trials: number): string {
|
||||
return `bun run scripts/test-paid-shards.ts --tier ${tier} --case ${id}${trials > 1 ? ` --trials ${trials}` : ''}`;
|
||||
/**
|
||||
* A paid run of exactly one case, for after a repair (never on unchanged
|
||||
* inputs): E2E cases through --case, judges through their selection and an
|
||||
* exact test-name filter, a keyless file (its own case) by file.
|
||||
*/
|
||||
export function afterRepairCommand(tier: PaidTier, id: string, file: string, trials = 1): string {
|
||||
if (Object.hasOwn(E2E_TIERS, id)) return `bun run scripts/test-paid-shards.ts --tier ${tier} --case ${id}${trials > 1 ? ` --trials ${trials}` : ''}`;
|
||||
if (Object.hasOwn(LLM_JUDGE_TOUCHFILES, id)) {
|
||||
return `EVALS=1 EVALS_TIER=${tier} EVALS_JUDGE_SELECTION_JSON='${JSON.stringify({ version: 1, selected: [id], reason: 'after a repair' })}' bun test ${file} -t '${caseTestNamePattern([id])}'`;
|
||||
}
|
||||
return `EVALS=1 EVALS_TIER=${tier} bun test ${file}`;
|
||||
}
|
||||
|
||||
/** One line per non-PASS or split panel verdict. */
|
||||
export function formatPanelLine(panel: PanelReport, tier: PaidTier): string {
|
||||
/** One line per non-PASS or split panel verdict. `artifacts` names each slice's artifact (slice -> name). */
|
||||
export function formatPanelLine(panel: PanelReport, tier: PaidTier, ctx: { artifacts?: Map<number, string>; runId?: string } = {}): string {
|
||||
const mark = panel.status === 'PASS' ? '⚠' : panel.failsLane ? '✗' : '◌';
|
||||
const label = panel.status === 'PASS' ? `PASS ${panel.passed}/${panel.panel.n}` : `${panel.status} ${panel.passed}/${panel.panel.n}`;
|
||||
const causes = panel.trials.filter(t => t.outcome !== 'passed').map(t => trialCause(t as any));
|
||||
const where = Object.entries(panel.slices).map(([trial, slice]) => `t${trial}@slice ${slice}`).join(', ');
|
||||
const red = panel.trials.find(t => t.outcome === 'failed');
|
||||
const slice = red ? panel.slices[red.trial] : undefined;
|
||||
const evidence = red && slice !== undefined
|
||||
? evidencePointer(ctx.artifacts?.get(slice) ?? `slice ${slice} attempt ${panel.attempt}`, trialShardKey(panel.file, panel.case, red.trial), ctx.runId, panel.case) : '';
|
||||
return `${mark} ${panel.case} ${panel.kind}${panel.quarantined ? ' (quarantined)' : ''} ${label} (${panel.marks})`
|
||||
+ `${causes.length ? ` ${causes.join('; ')}` : ''}${panel.status === 'INCOMPLETE' || panel.status === 'SKIPPED' ? ` [${panel.reason}]` : ''}`
|
||||
+ `${where ? ` [${where}, attempt ${panel.attempt}]` : ''} rerun: ${rerunCommand(tier, panel.case, panel.panel.n)}`;
|
||||
+ `${where ? ` [${where}, attempt ${panel.attempt}]` : ''}${evidence} after a repair: ${afterRepairCommand(tier, panel.case, panel.file, panel.panel.n)}`;
|
||||
}
|
||||
|
||||
export interface ReportHeadline {
|
||||
@@ -595,6 +639,8 @@ export function runPaidReport(reportDir: string, options: { writeDurations?: boo
|
||||
quarantined: t.quarantined, execution: outcome.reused ? 'reused' : 'executed', source: 'shard' } as TrialOutcomeRecord);
|
||||
}
|
||||
}
|
||||
const sliceArtifacts = new Map(artifacts.filter(a => (a.result.attempt ?? 1) === primary)
|
||||
.map(a => [a.result.sliceIndex, sliceArtifactName(a.root, a.result.sliceIndex, primary)] as const));
|
||||
const census = junitCensus(manifest, artifacts.filter(a => (a.result.attempt ?? 1) === primary), recordsByShard,
|
||||
common(primary), env.GSTACK_CLAUDE_CLI_VERSION);
|
||||
history.push(...census.history);
|
||||
@@ -620,7 +666,7 @@ export function runPaidReport(reportDir: string, options: { writeDurations?: boo
|
||||
if (got && got.o.status === 'passed') return [];
|
||||
if (got && census.failedShards.has(normalizeRelativePath(entry.file))) return [];
|
||||
const id = shardCaseId(entry.file);
|
||||
return [`✗ ${entry.file} rule shard ${got ? got.o.status : 'NOT REPORTED'}${got?.o.runnerError ? ` — ${sanitizeTrialError(got.o.runnerError)}` : ''} [slice ${entry.slice}, attempt ${primary}]${id ? ` rerun: ${rerunCommand(manifest.tier, id, 1)}` : ''}`];
|
||||
return [`✗ ${entry.file} rule shard ${got ? got.o.status : 'NOT REPORTED'}${got?.o.runnerError ? ` — ${sanitizeTrialError(got.o.runnerError)}` : ''} [slice ${entry.slice}, attempt ${primary}]${id ? ` after a repair: ${afterRepairCommand(manifest.tier, id, shardFile(entry.file))}` : ''}`];
|
||||
});
|
||||
const behaviorPanels = panels.filter(p => !p.quarantined && p.kind === 'behavior');
|
||||
const lanePanels = panels.filter(p => !p.quarantined);
|
||||
@@ -634,7 +680,7 @@ export function runPaidReport(reportDir: string, options: { writeDurations?: boo
|
||||
const failureLines = [
|
||||
...ruleShardFailures,
|
||||
...ruleCases.filter(c => c.line).map(c => c.line!),
|
||||
...panels.filter(p => p.status !== 'PASS' || p.split).map(p => formatPanelLine(p, manifest.tier)),
|
||||
...panels.filter(p => p.status !== 'PASS' || p.split).map(p => formatPanelLine(p, manifest.tier, { artifacts: sliceArtifacts, runId })),
|
||||
];
|
||||
const red = verdict.problems.length > 0;
|
||||
const headline: ReportHeadline = {
|
||||
@@ -662,7 +708,7 @@ export function runPaidReport(reportDir: string, options: { writeDurations?: boo
|
||||
for (const line of failureLines) console.log(` ${line}`);
|
||||
}
|
||||
for (const panel of laterPanels) console.log(` attempt ${panel.attempt} (re-run; reported, never replacing attempt ${primary}): ${formatPanelLine(panel, manifest.tier)}`);
|
||||
const fence = (lines: string[]) => ['```', ...lines.map(line => line.replace(/```/g, "'''")), '```'];
|
||||
const fence = (lines: string[]) => publishedFence(lines);
|
||||
fs.writeFileSync(summaryMdPath, [
|
||||
...fence(headlineLines),
|
||||
...(failureLines.length ? ['', '**Failures and split verdicts**', '', ...fence(failureLines)] : []),
|
||||
@@ -678,7 +724,7 @@ export function runPaidReport(reportDir: string, options: { writeDurations?: boo
|
||||
marks: p.marks, split: p.split, quarantined: p.quarantined, failsLane: p.failsLane, redClass: p.redClass, reason: p.reason,
|
||||
trials: p.trials.map(t => ({ trial: t.trial, outcome: t.outcome, ...(t.failure_class ? { failure_class: t.failure_class } : {}),
|
||||
...(t.exit_reason ? { exit_reason: t.exit_reason } : {}), ...(t.error ? { error: t.error } : {}) })) })),
|
||||
failures: failureLines.map(line => sanitizeTrialError(line) ?? line) }, null, 2) + '\n');
|
||||
failures: failureLines.map(line => sanitizePublishedText(line)) }, null, 2) + '\n');
|
||||
if (verdict.problems.length) {
|
||||
console.error(`[test:paid] report: ${verdict.problems.length} problem(s):`);
|
||||
for (const problem of verdict.problems) console.error(` ✗ ${problem}`);
|
||||
|
||||
@@ -0,0 +1,36 @@
|
||||
/**
|
||||
* The one sanitizer for model- or fixture-written text that reaches a public
|
||||
* issue, PR comment or committed file (census report, tracking issue,
|
||||
* eval:pass-rates output). Control and bidi characters are stripped, the
|
||||
* repo's credential-scan patterns (lib/redact-engine, the engine the quality
|
||||
* gate's credential scan runs) are redacted, @-mentions are neutralized and
|
||||
* length is capped. Inline text also has HTML and Markdown link brackets
|
||||
* escaped; blocks go inside a code fence longer than any backtick run they
|
||||
* contain, so their content stays literal and cannot close the fence.
|
||||
*/
|
||||
import { redactFindingSpans } from '../../lib/redact-engine';
|
||||
|
||||
export const PUBLISHED_TEXT_MAX = 1000;
|
||||
|
||||
const cap = (text: string, max: number) => (text.length > max ? `${text.slice(0, max - 1)}…` : text);
|
||||
|
||||
function clean(text: string): string {
|
||||
// eslint-disable-next-line no-control-regex
|
||||
const stripped = text.replace(/[\u0000-\u0008\u000a-\u001f\u007f-\u009f\u200b-\u200f\u202a-\u202e\u2066-\u2069]/g, ' ').slice(0, 64 * 1024);
|
||||
const redacted = redactFindingSpans(stripped) ?? '[redacted: credential-shaped text that could not be isolated]';
|
||||
return redacted.replace(/@(?=[A-Za-z0-9_-])/g, '@\u200b');
|
||||
}
|
||||
|
||||
/** One inline line: cleaned, HTML and Markdown link brackets escaped, capped. */
|
||||
export function sanitizePublishedText(text: string, max = PUBLISHED_TEXT_MAX): string {
|
||||
return cap(clean(text).replace(/&/g, '&').replace(/</g, '<').replace(/>/g, '>')
|
||||
.replace(/\[/g, '[').replace(/\]/g, ']'), max);
|
||||
}
|
||||
|
||||
/** Cleaned, capped lines inside a fence longer than the longest backtick run they contain. */
|
||||
export function publishedFence(lines: readonly string[], max = PUBLISHED_TEXT_MAX): string[] {
|
||||
const body = lines.map(line => cap(clean(line), max));
|
||||
const longest = Math.max(0, ...body.flatMap(line => (line.match(/`+/g) ?? []).map(run => run.length)));
|
||||
const fence = '`'.repeat(Math.max(3, longest + 1));
|
||||
return [fence, ...body, fence];
|
||||
}
|
||||
@@ -11,7 +11,7 @@ import { afterAll, beforeAll, describe, expect, test } from 'bun:test';
|
||||
import * as fs from 'node:fs';
|
||||
import * as os from 'node:os';
|
||||
import * as path from 'node:path';
|
||||
import { DESELECTION_REASONS, RUNTIME_SKIP_REASON, attributeJUnitCase, formatPanelLine, runPaidReport, type JUnitCase } from '../scripts/test-paid-shards';
|
||||
import { DESELECTION_REASONS, RUNTIME_SKIP_REASON, attributeJUnitCase, formatPanelLine, resolvePaidShardBudget, runPaidReport, type JUnitCase } from '../scripts/test-paid-shards';
|
||||
import { panelVerdict } from './helpers/eval-store';
|
||||
import { trialRecordProblems as trialRecordProblemsV1 } from './fixtures/trial-record-v1-reader';
|
||||
|
||||
@@ -22,7 +22,7 @@ beforeAll(() => { base = fs.mkdtempSync(path.join(os.tmpdir(), 'paid-report-cens
|
||||
afterAll(() => { fs.rmSync(base, { recursive: true, force: true }); });
|
||||
|
||||
let runs = 0;
|
||||
function report(lane: 'periodic' | 'gate', edit: (dir: string) => void = () => {}) {
|
||||
function report(lane: 'periodic' | 'gate', edit: (dir: string) => void = () => {}, env: NodeJS.ProcessEnv = {}) {
|
||||
const dir = path.join(base, `${lane}-${++runs}`);
|
||||
fs.cpSync(path.join(FIXTURES, lane), dir, { recursive: true });
|
||||
edit(dir);
|
||||
@@ -31,7 +31,7 @@ function report(lane: 'periodic' | 'gate', edit: (dir: string) => void = () => {
|
||||
console.log = (...args: unknown[]) => { lines.push(args.join(' ')); };
|
||||
console.error = console.log;
|
||||
let status: number;
|
||||
try { status = runPaidReport(dir, { env: {} }); } finally { console.log = log; console.error = error; }
|
||||
try { status = runPaidReport(dir, { env }); } finally { console.log = log; console.error = error; }
|
||||
const history = fs.readFileSync(path.join(dir, 'trial-outcomes.jsonl'), 'utf8').split('\n').filter(Boolean).map(line => JSON.parse(line));
|
||||
return { status, out: lines.join('\n'), dir, history };
|
||||
}
|
||||
@@ -124,6 +124,83 @@ describe('record builders attach ledger sessions and failure causes (plan 0.1/0.
|
||||
});
|
||||
});
|
||||
|
||||
describe('red lines carry their values, evidence and a one-case command (plan 0.3)', () => {
|
||||
const QA_ONLY = 'qa-only/SKILL.md workflow';
|
||||
const addJudgeRed = (dir: string) => {
|
||||
const shard = path.join(dir, 'paid-slice-1-a1', 'shards', 'skill-llm-eval');
|
||||
fs.mkdirSync(shard, { recursive: true });
|
||||
const message = 'expect(received).toBeGreaterThanOrEqual(expected)\n\nExpected: >= 4\nReceived: 3.6666666666666665\n';
|
||||
fs.writeFileSync(path.join(shard, 'junit.xml'), `<?xml version="1.0" encoding="UTF-8"?>\n<testsuites><testsuite name="test/skill-llm-eval.test.ts">`
|
||||
+ `<testcase name="${QA_ONLY}" classname="Other skill evals" time="34.8" file="test/skill-llm-eval.test.ts" line="9">`
|
||||
+ `<failure type="AssertionError" message="${message.replace(/\n/g, ' ')}" /></testcase></testsuite></testsuites>\n`);
|
||||
// Trimmed from census 37198445662's paid-slice-5-a1 qa-only record.
|
||||
fs.writeFileSync(path.join(shard, '1.91.19-x-llm-judge-2026-10-04-1129.json'), JSON.stringify({ tier: 'llm-judge', total_cost_usd: 0.06, tests: [{
|
||||
name: QA_ONLY, suite: 'Other skill evals', tier: 'llm-judge', passed: false, duration_ms: 34829, cost_usd: 0.06, execution: 'executed',
|
||||
judge_scores: { clarity: 3.6666666666666665, completeness: 4.333333333333333, actionability: 3.6666666666666665 },
|
||||
judge_reasoning: '[sample 1] The bundle is thorough. Clarity suffers from extreme density and cross-file indirection. Two actionability concerns: Browser Quick allots 30 seconds.\n[sample 2] Deliverables are concrete.\n[sample 3] Execution is feasible.',
|
||||
exit_reason: 'validation_failed', error: message.replace('>', '>') }] }));
|
||||
const manifestPath = path.join(dir, 'manifest.json');
|
||||
const manifest = JSON.parse(fs.readFileSync(manifestPath, 'utf8'));
|
||||
const budget = resolvePaidShardBudget(['test/skill-llm-eval.test.ts']);
|
||||
manifest.entries.push({ file: 'test/skill-llm-eval.test.ts', slice: 1, status: 'planned', estimatedMs: 35_000, budget });
|
||||
fs.writeFileSync(manifestPath, JSON.stringify(manifest));
|
||||
const slicePath = path.join(dir, 'paid-slice-1-a1', 'slice-1.json');
|
||||
const slice = JSON.parse(fs.readFileSync(slicePath, 'utf8'));
|
||||
slice.outcomes.push({ files: ['test/skill-llm-eval.test.ts'], status: 'failed', exitCode: 1, elapsedMs: 35_000, executedTests: 1, skippedTests: 0, budget });
|
||||
fs.writeFileSync(slicePath, JSON.stringify(slice));
|
||||
};
|
||||
|
||||
test('a judge red names each failing dimension, its mean, samples and rationale, its artifact and an exact one-case command', () => {
|
||||
const r = report('gate', addJudgeRed, { GITHUB_RUN_ID: '37198445662' });
|
||||
const line = r.out.split('\n').find(l => l.includes(`✗ ${QA_ONLY}`))!;
|
||||
expect(line).toContain('FAIL assertion — expect(received).toBeGreaterThanOrEqual(expected)');
|
||||
expect(line).toContain('· cause assertion clarity 3.67 < 4 (3 samples) "Clarity suffers from extreme density and cross-file indirection."; '
|
||||
+ 'actionability 3.67 < 4 (3 samples) "Two actionability concerns: Browser Quick allots 30 seconds."');
|
||||
expect(line).toContain('evidence: paid-slice-1-a1/shards/skill-llm-eval (fetch: bun run eval:pass-rates --run 37198445662 --case \'qa-only/SKILL.md workflow\')');
|
||||
expect(line).toContain(`after a repair: EVALS=1 EVALS_TIER=gate EVALS_JUDGE_SELECTION_JSON='{"version":1,"selected":["${QA_ONLY}"],"reason":"after a repair"}' `
|
||||
+ `bun test test/skill-llm-eval.test.ts -t '(?:^|\\s)(?:qa-only/SKILL\\.md workflow)$'`);
|
||||
expect(line).not.toContain('rerun:');
|
||||
const record = r.history.find(x => x.case === QA_ONLY);
|
||||
expect(record).toMatchObject({ failure_class: 'assertion', error: 'expect(received).toBeGreaterThanOrEqual(expected)', failure_cause: 'assertion',
|
||||
failure_detail: { judge: [expect.objectContaining({ dimension: 'clarity', mean: 3.67 }), expect.objectContaining({ dimension: 'actionability', mean: 3.67 })] } });
|
||||
expect(trialRecordProblemsV1(record)).toEqual([]);
|
||||
});
|
||||
|
||||
test('a behavior panel line names the failed trial\'s cause, values, slice artifact and an n-trial one-case command', () => {
|
||||
const verdict = panelVerdict({ case: 'plan-review-prosons-neutral-neg', kind: 'behavior', panel: { n: 3, k: 2 }, quarantined: false, trials: [
|
||||
{ trial: 1, outcome: 'failed', failure_class: 'assertion', exit_reason: 'success', error: 'expect(received).not.toMatch(expected)',
|
||||
failure_cause: 'assertion', failure_cause_evidence: 'session completed; check failed',
|
||||
failure_detail: { expected: 'not /taste call/i', received: '"a taste call"' } } as any,
|
||||
{ trial: 2, outcome: 'failed', failure_class: 'assertion' }, { trial: 3, outcome: 'passed' }] });
|
||||
const line = formatPanelLine({ ...verdict, file: 'test/skill-e2e-plan-prosons.test.ts', slices: { 1: 7, 2: 9, 3: 9 } }, 'periodic',
|
||||
{ artifacts: new Map([[7, 'paid-slice-7-a1']]), runId: '37193478719' });
|
||||
expect(line).toContain('t1: assertion (session completed; check failed) — expect(received).not.toMatch(expected) · cause assertion: session completed; check failed Expected: not /taste call/i · Received: "a taste call"');
|
||||
expect(line).toContain('evidence: paid-slice-7-a1/shards/skill-e2e-plan-prosons--plan-review-prosons-neutral-neg.t1 (fetch: bun run eval:pass-rates --run 37193478719 --case plan-review-prosons-neutral-neg)');
|
||||
expect(line).toContain('after a repair: bun run scripts/test-paid-shards.ts --tier periodic --case plan-review-prosons-neutral-neg --trials 3');
|
||||
});
|
||||
|
||||
test('a rule red shows Expected/Received and selects only its case; the summary fence cannot be broken', () => {
|
||||
const r = report('gate', dir => {
|
||||
const file = path.join(dir, 'paid-slice-1-a1', 'shards', 'skill-e2e-plan-mode-no-op', 'junit.xml');
|
||||
fs.writeFileSync(file, fs.readFileSync(file, 'utf8').replace(/(<testcase name="plan-ceo-review reaches a terminal outcome outside plan mode"[^>]*?) \/>/,
|
||||
'$1><failure type="AssertionError" message="expect(received).toBe(expected) Expected: "asked" Received: "```@team</details>"" /></testcase>'));
|
||||
addJudgeRed(dir);
|
||||
});
|
||||
const line = r.out.split('\n').find(l => l.includes('✗ plan-mode-no-op'))!;
|
||||
expect(line).toContain('Expected: "asked" · Received: "```@\u200bteam</details>"');
|
||||
expect(line).toContain('evidence: paid-slice-1-a1/shards/skill-e2e-plan-mode-no-op');
|
||||
expect(line).not.toContain('(fetch:');
|
||||
expect(line).toContain('after a repair: bun run scripts/test-paid-shards.ts --tier gate --case plan-mode-no-op');
|
||||
const summary = fs.readFileSync(path.join(r.dir, 'report-summary.md'), 'utf8');
|
||||
const lines = summary.split('\n');
|
||||
const at = lines.findIndex(l => l.includes('✗ plan-mode-no-op'));
|
||||
const open = lines.slice(0, at).findLast(l => /^`{3,}$/.test(l))!;
|
||||
expect(open.length).toBeGreaterThan(3);
|
||||
expect(lines.slice(at).find(l => /^`{3,}$/.test(l))).toBe(open);
|
||||
expect(summary).not.toMatch(/@team/);
|
||||
});
|
||||
});
|
||||
|
||||
describe('attributeJUnitCase', () => {
|
||||
const tc = (name: string, outcome: JUnitCase['outcome'], line: number | null = 1): JUnitCase =>
|
||||
({ name, classname: 's', outcome, timeMs: 0, ...(line === null ? {} : { line }) });
|
||||
|
||||
@@ -0,0 +1,64 @@
|
||||
/**
|
||||
* The published-text sanitizer (plan 0.3): adversarial model or fixture text
|
||||
* reaching a public issue, PR comment or committed report cannot ping people,
|
||||
* break out of its fence, inject HTML or images, leak a credential, or flood.
|
||||
*/
|
||||
import { describe, expect, test } from 'bun:test';
|
||||
import { publishedFence, sanitizePublishedText, PUBLISHED_TEXT_MAX } from '../scripts/lib/published-text';
|
||||
|
||||
// Built at runtime so this file never contains a credential-shaped literal.
|
||||
const GHP = ['ghp', '_', 'A1b2C3d4E5f6G7h8I9j0K1l2M3n4O5p6Q7r8'].join('');
|
||||
const SK = ['sk', '-ant-api03-', 'Zx9'.repeat(30)].join('');
|
||||
const ADVERSARIAL = [
|
||||
'ping @garrytan and @gstack/maintainers',
|
||||
'close ``` the fence ```` here',
|
||||
'</details><img src=x onerror=alert(1)>',
|
||||
' [click](https://evil.example)',
|
||||
`token ${GHP} and key ${SK}`,
|
||||
'bell\u0007 escape\u001b[31m bidi\u202eevil',
|
||||
];
|
||||
|
||||
describe('sanitizePublishedText (inline)', () => {
|
||||
const out = ADVERSARIAL.map(line => sanitizePublishedText(line));
|
||||
test('mentions are neutralized', () => {
|
||||
expect(out[0]).not.toMatch(/@[A-Za-z]/);
|
||||
expect(out[0]).toContain('@\u200bgarrytan');
|
||||
});
|
||||
test('HTML and Markdown links or images cannot render', () => {
|
||||
expect(out[2]).not.toContain('<');
|
||||
expect(out[2]).toContain('</details>');
|
||||
expect(out[3]).not.toMatch(/!?\[[^\]]*\]\(/);
|
||||
});
|
||||
test('credential-scan patterns are redacted', () => {
|
||||
expect(out[4]).not.toContain(GHP);
|
||||
expect(out[4]).not.toContain(SK);
|
||||
});
|
||||
test('control and bidi characters are stripped', () => {
|
||||
// eslint-disable-next-line no-control-regex
|
||||
expect(out[5]).not.toMatch(/[\u0000-\u001f\u007f\u202e]/);
|
||||
});
|
||||
test('length is capped', () => {
|
||||
expect(sanitizePublishedText('x'.repeat(10 * 1024))).toHaveLength(PUBLISHED_TEXT_MAX);
|
||||
});
|
||||
});
|
||||
|
||||
describe('publishedFence (blocks)', () => {
|
||||
const block = publishedFence([...ADVERSARIAL, 'y'.repeat(10 * 1024)]);
|
||||
test('the fence is longer than any backtick run inside it, so nothing closes it early', () => {
|
||||
const fence = block[0]!;
|
||||
expect(block.at(-1)).toBe(fence);
|
||||
expect(fence).toMatch(/^`{5,}$/);
|
||||
for (const line of block.slice(1, -1)) {
|
||||
for (const run of line.match(/`+/g) ?? []) expect(run.length).toBeLessThan(fence.length);
|
||||
}
|
||||
});
|
||||
test('content stays literal inside the fence but loses mentions, credentials, control characters and length', () => {
|
||||
const body = block.slice(1, -1).join('\n');
|
||||
expect(body).not.toMatch(/@[A-Za-z]/);
|
||||
expect(body).not.toContain(GHP);
|
||||
expect(body).not.toContain(SK);
|
||||
expect(body).not.toContain('\u202e');
|
||||
expect(block.at(-2)).toHaveLength(PUBLISHED_TEXT_MAX);
|
||||
expect(body).toContain('</details>');
|
||||
});
|
||||
});
|
||||
Reference in new issue
Block a user