test(evals): add trial records, panelVerdict, expectContract and trial-outcomes JSONL

EvalTestEntry gains case_id, kind, trial, panel, failure_class and
policy_version, stamped from the runner's TRIAL_ENV on isolated trial
shards. panelVerdict() is the single verdict function (INCOMPLETE on
missing or duplicate trials, contract veto at any count, quarantine
hard-break rule, INFRA/INCOMPLETE machine classification). expectContract()
records failure_class 'contract' on the collector entry and a sidecar
before throwing. trial-outcomes JSONL has a fail-closed writer and a
data-only reader.

test/helpers/eval-store.test.ts

@@ -14,8 +14,20 @@ import {
   formatComparison,
   generateCommentary,
   judgePassed,
+  CONTRACT_VIOLATIONS_FILE,
+  ContractViolation,
+  TRIAL_ENV,
+  TRIAL_OUTCOME_SCHEMA,
+  expectContract,
+  failureClassOf,
+  formatTrialOutcomes,
+  panelVerdict,
+  parseTrialOutcomes,
+  sanitizeTrialError,
+  trialContextFromEnv,
 } from './eval-store';
-import type { EvalResult, EvalTestEntry, ComparisonResult } from './eval-store';
+import type { EvalResult, EvalTestEntry, ComparisonResult, PanelTrial, TrialOutcomeRecord } from './eval-store';
+import { EVAL_POLICY } from './periodic-exclude-data';
 import { manualReviewFixture } from './manual-judge-review-fixture';
 
 let tmpDir: string;
@@ -957,3 +969,191 @@ describe('generateCommentary', () => {
     expect(notes.some(n => n.includes('Stable run'))).toBe(true);
   });
 });
+
+// --- Trials, panel verdicts and contract vetoes (eval reliability policy) ---
+
+const PANEL = EVAL_POLICY.panel;
+const pass = (trial: number, extra: Partial<PanelTrial> = {}): PanelTrial => ({ trial, outcome: 'passed', ...extra });
+const fail = (trial: number, extra: Partial<PanelTrial> = {}): PanelTrial => ({ trial, outcome: 'failed', ...extra });
+const behavior = (trials: PanelTrial[], quarantined = false) =>
+  panelVerdict({ case: 'case-x', kind: 'behavior', panel: PANEL, trials, quarantined });
+
+describe('panelVerdict', () => {
+  test('policy constants are the approved pre-registration', () => {
+    expect(EVAL_POLICY.panel).toEqual({ n: 3, k: 2 });
+    expect(EVAL_POLICY.quarantine).toEqual({
+      entry: { rate: 0.95, minTrials: 10 },
+      exit: { rate: 0.97, minTrials: 10 },
+      capFraction: 0.1,
+      expiryWeeklyRuns: 8,
+    });
+    expect(EVAL_POLICY.infraRedispatch).toBe(1);
+  });
+
+  test('rule: one trial, any failure fails the lane', () => {
+    const ok = panelVerdict({ case: 'r', kind: 'rule', panel: { n: 1, k: 1 }, trials: [pass(1)] });
+    expect(ok).toMatchObject({ status: 'PASS', split: false, failsLane: false, coverage: true, marks: '✓' });
+    const bad = panelVerdict({ case: 'r', kind: 'rule', panel: { n: 1, k: 1 }, trials: [fail(1)] });
+    expect(bad).toMatchObject({ status: 'FAIL', failsLane: true, coverage: false, redClass: 'VERDICT', marks: '✗' });
+  });
+
+  test('behavior 3/3 is a clean PASS', () => {
+    expect(behavior([pass(1), pass(2), pass(3)])).toMatchObject({ status: 'PASS', split: false, passed: 3, failsLane: false });
+  });
+
+  test('behavior 2/3 is a split PASS that shows its failed trial', () => {
+    const v = behavior([pass(1), fail(2, { exit_reason: 'timeout' }), pass(3)]);
+    expect(v).toMatchObject({ status: 'PASS', split: true, passed: 2, failed: 1, failsLane: false, coverage: true, marks: '✓✗✓', reason: 'PASS 2/3' });
+    expect(v.trials[1].exit_reason).toBe('timeout');
+  });
+
+  test('behavior 1/3 and 0/3 fail the lane', () => {
+    expect(behavior([pass(1), fail(2), fail(3)])).toMatchObject({ status: 'FAIL', failsLane: true, redClass: 'VERDICT' });
+    expect(behavior([fail(1), fail(2), fail(3)])).toMatchObject({ status: 'FAIL', failsLane: true });
+  });
+
+  test('a contract trial fails the panel even at 2/3', () => {
+    const v = behavior([pass(1), pass(2), fail(3, { failure_class: 'contract' })]);
+    expect(v).toMatchObject({ status: 'FAIL', contract: true, failsLane: true, redClass: 'VERDICT', reason: 'contract violation' });
+  });
+
+  test('a missing trial is INCOMPLETE and fails the lane', () => {
+    const v = behavior([pass(1), pass(3)]);
+    expect(v).toMatchObject({ status: 'INCOMPLETE', failsLane: true, coverage: false, redClass: 'INCOMPLETE', marks: '✓·✓' });
+    expect(v.reason).toContain('missing trial t2');
+  });
+
+  test('duplicate or out-of-range trial records are INCOMPLETE, never deduplicated', () => {
+    expect(behavior([pass(1), pass(2), pass(2), fail(3)]).status).toBe('INCOMPLETE');
+    expect(behavior([pass(1), pass(2), pass(3), pass(4)]).reason).toContain('unexpected trial t4');
+    const v = panelVerdict({ case: 'c', kind: 'behavior', panel: { n: 11, k: 6 }, trials: Array.from({ length: 10 }, (_, i) => pass(i + 2)) });
+    expect(v.reason).toContain('missing trial t1');
+  });
+
+  test('timeout and infra trials count as failed, never passing', () => {
+    const v = behavior([pass(1), fail(2, { exit_reason: 'timeout' }), fail(3, { failure_class: 'infra' })]);
+    expect(v).toMatchObject({ status: 'FAIL', passed: 1, failed: 2, failsLane: true, redClass: 'VERDICT' });
+    const infra = behavior([pass(1), fail(2, { failure_class: 'infra' }), fail(3, { failure_class: 'infra' })]);
+    expect(infra).toMatchObject({ status: 'FAIL', redClass: 'INFRA' });
+    expect(failureClassOf({ exit_reason: 'timeout' })).toBe('timeout');
+    expect(failureClassOf({})).toBe('assertion');
+  });
+
+  test('quarantined: 1/3 does not fail the lane, 0/3 and contract do, no coverage credit', () => {
+    expect(behavior([pass(1), fail(2), fail(3)], true)).toMatchObject({ status: 'FAIL', failsLane: false, coverage: false, redClass: null });
+    expect(behavior([fail(1), fail(2), fail(3)], true)).toMatchObject({ status: 'FAIL', failsLane: true });
+    expect(behavior([pass(1), pass(2), fail(3, { failure_class: 'contract' })], true)).toMatchObject({ status: 'FAIL', failsLane: true });
+    expect(behavior([pass(1), pass(2), pass(3)], true)).toMatchObject({ status: 'PASS', coverage: false, failsLane: false });
+    expect(behavior([pass(1), pass(2)], true)).toMatchObject({ status: 'INCOMPLETE', failsLane: true });
+  });
+
+  test('quarantined rule keeps rule meaning (k = n)', () => {
+    const v = panelVerdict({ case: 'r', kind: 'rule', panel: { n: 3, k: 3 }, trials: [pass(1), pass(2), fail(3)], quarantined: true });
+    expect(v).toMatchObject({ status: 'FAIL', failsLane: false });
+  });
+
+  test('all-skipped panel is SKIPPED with no credit; partly skipped is INCOMPLETE', () => {
+    const skip = (trial: number): PanelTrial => ({ trial, outcome: 'skipped' });
+    expect(behavior([skip(1), skip(2), skip(3)])).toMatchObject({ status: 'SKIPPED', coverage: false, failsLane: false });
+    expect(behavior([pass(1), pass(2), skip(3)])).toMatchObject({ status: 'INCOMPLETE', failsLane: true });
+  });
+
+  test('trials of different run attempts are never merged into one verdict', () => {
+    expect(() => behavior([pass(1), pass(2), fail(3, { attempt: 2 })])).toThrow(/run attempts/);
+    expect(behavior([pass(1, { attempt: 2 }), pass(2, { attempt: 2 }), pass(3, { attempt: 2 })]).attempt).toBe(2);
+  });
+
+  test('invalid panels throw', () => {
+    expect(() => panelVerdict({ case: 'c', kind: 'behavior', panel: { n: 3, k: 4 }, trials: [] })).toThrow(/invalid panel/);
+    expect(() => panelVerdict({ case: 'c', kind: 'nope' as any, panel: { n: 1, k: 1 }, trials: [] })).toThrow(/unknown kind/);
+  });
+});
+
+describe('trial context and expectContract', () => {
+  let dir: string;
+  const saved: Record<string, string | undefined> = {};
+  const keys = [...Object.values(TRIAL_ENV), 'GSTACK_EVAL_DIR'];
+  beforeEach(() => {
+    dir = fs.mkdtempSync(path.join(os.tmpdir(), 'panel-verdict-'));
+    for (const key of keys) saved[key] = process.env[key];
+  });
+  afterEach(() => {
+    for (const key of keys) {
+      if (saved[key] === undefined) delete process.env[key];
+      else process.env[key] = saved[key];
+    }
+    fs.rmSync(dir, { recursive: true, force: true });
+  });
+  const setTrial = () => Object.assign(process.env, {
+    [TRIAL_ENV.caseId]: 'case-x', [TRIAL_ENV.kind]: 'behavior', [TRIAL_ENV.trial]: '2',
+    [TRIAL_ENV.panelN]: '3', [TRIAL_ENV.panelK]: '2', [TRIAL_ENV.policyVersion]: String(EVAL_POLICY.version),
+    GSTACK_EVAL_DIR: dir,
+  });
+
+  test('trialContextFromEnv: absent, complete, and malformed', () => {
+    for (const key of Object.values(TRIAL_ENV)) delete process.env[key];
+    expect(trialContextFromEnv()).toBeNull();
+    setTrial();
+    expect(trialContextFromEnv()).toEqual({ case_id: 'case-x', kind: 'behavior', trial: 2, panel: { n: 3, k: 2 }, policy_version: EVAL_POLICY.version });
+    process.env[TRIAL_ENV.trial] = '4';
+    expect(() => trialContextFromEnv()).toThrow(/Malformed trial context/);
+  });
+
+  test('passing contract is a no-op', () => {
+    setTrial();
+    expect(() => expectContract(true, 'fine')).not.toThrow();
+    expect(fs.existsSync(path.join(dir, CONTRACT_VIOLATIONS_FILE))).toBe(false);
+  });
+
+  test('failed contract stamps the recorded entry and the sidecar before throwing', () => {
+    setTrial();
+    const collector = new EvalCollector('e2e', dir);
+    collector.addTest({ name: 'case-x', suite: 's', tier: 'e2e', passed: true, duration_ms: 1, cost_usd: 0 });
+    expect(() => expectContract(false, 'handoff missing', { collector, name: 'case-x' })).toThrow(ContractViolation);
+    const partial = JSON.parse(fs.readFileSync(path.join(dir, '_partial-e2e.json'), 'utf-8'));
+    expect(partial.tests[0]).toMatchObject({ passed: false, failure_class: 'contract', case_id: 'case-x', trial: 2, kind: 'behavior', panel: { n: 3, k: 2 } });
+    const sidecar = fs.readFileSync(path.join(dir, CONTRACT_VIOLATIONS_FILE), 'utf-8').trim().split('\n').map((l) => JSON.parse(l));
+    expect(sidecar).toEqual([expect.objectContaining({ case_id: 'case-x', trial: 2, message: 'handoff missing' })]);
+  });
+
+  test('a contract marked before recording stamps the later record, or becomes its own at finalize', async () => {
+    setTrial();
+    const collector = new EvalCollector('e2e', dir);
+    expect(() => expectContract(0, 'no question asked', { collector, name: 'later' })).toThrow('CONTRACT: no question asked');
+    collector.addTest({ name: 'later', suite: 's', tier: 'e2e', passed: true, duration_ms: 1, cost_usd: 0 });
+    expect(() => expectContract(null, 'never recorded', { collector, name: 'orphan' })).toThrow();
+    const file = await collector.finalize();
+    const tests = JSON.parse(fs.readFileSync(file, 'utf-8')).tests;
+    expect(tests.find((t: any) => t.name === 'later')).toMatchObject({ passed: false, failure_class: 'contract' });
+    expect(tests.find((t: any) => t.name === 'orphan')).toMatchObject({ passed: false, failure_class: 'contract', error: 'never recorded' });
+  });
+});
+
+describe('trial-outcomes JSONL', () => {
+  const record = (extra: Partial<TrialOutcomeRecord> = {}): TrialOutcomeRecord => ({
+    schema: TRIAL_OUTCOME_SCHEMA, case: 'case-x', file: 'test/x.test.ts', tier: 'gate', kind: 'behavior',
+    trial: 1, panel: { n: 3, k: 2 }, attempt: 1, outcome: 'passed', duration_ms: 10, cost_usd: 0.1,
+    policy_version: EVAL_POLICY.version, quarantined: false, execution: 'executed', source: 'shard', ...extra,
+  });
+
+  test('round-trips valid records', () => {
+    const records = [record(), record({ trial: 2, outcome: 'failed', failure_class: 'timeout', exit_reason: 'timeout' })];
+    expect(parseTrialOutcomes(formatTrialOutcomes(records))).toEqual({ records, errors: [] });
+  });
+
+  test('writer fails closed; reader reports bad lines as data errors', () => {
+    expect(() => formatTrialOutcomes([record({ outcome: 'failed' })])).toThrow(/failed without failure_class/);
+    expect(() => formatTrialOutcomes([record({ trial: 4 })])).toThrow(/trial invalid/);
+    const text = `${JSON.stringify(record())}\nnot json\n${JSON.stringify({ ...record(), schema: 'other' })}\n`;
+    const parsed = parseTrialOutcomes(text);
+    expect(parsed.records).toHaveLength(1);
+    expect(parsed.errors).toEqual(['line 2: not JSON', 'line 3: schema other']);
+    expect(parseTrialOutcomes(text, { maxBytes: 10 }).errors[0]).toContain('exceed');
+  });
+
+  test('sanitizeTrialError keeps one capped line without mentions', () => {
+    expect(sanitizeTrialError('\n  expected @garrytan to `see`\nsecond')).toBe("expected @\u200bgarrytan to 'see'");
+    expect(sanitizeTrialError('x'.repeat(1000))!.length).toBe(300);
+    expect(sanitizeTrialError('')).toBeUndefined();
+  });
+});

test/helpers/eval-store.ts

@@ -76,6 +76,18 @@ export interface EvalTestEntry {
    *  its body again and re-records under the same name. Set by addTest. */
   attempt?: number;
 
+  // Trial identity (eval reliability policy). Stamped by addTest from the
+  // TRIAL_ENV variables the paid runner sets on an isolated trial shard.
+  /** Registry id (E2E_TIERS / LLM_JUDGE_TOUCHFILES key) this record belongs to. */
+  case_id?: string;
+  kind?: EvalCaseKind;
+  /** 1-based trial index within the case's panel. */
+  trial?: number;
+  panel?: PanelShape;
+  /** Why a failed record failed; 'contract' comes only from expectContract. */
+  failure_class?: TrialFailureClass;
+  policy_version?: number;
+
   // E2E
   transcript?: any[];
   prompt?: string;
@@ -131,6 +143,326 @@ export function evalEntryOutcome(entry: unknown): 'passed' | 'failed' | 'manual-
   return result.passed === true ? 'passed' : 'failed';
 }
 
+// --- Trials and panel verdicts ---
+//
+// Paid evals never retry. Each case's kind (E2E_KINDS) fixes its trials before
+// the run; a panel verdict is computed once, by panelVerdict(), from exactly
+// panel.n trial records of one run attempt. The report, collector-outcomes,
+// the PR comment and pass-rates all read that one function.
+
+export type EvalCaseKind = 'rule' | 'behavior' | 'judge';
+/** assertion: an ordinary failed expectation. contract: expectContract() fired
+ *  (fails the panel at any count). timeout: the case budget ran out.
+ *  infra: API/CLI/runner failure before the model could be graded. */
+export type TrialFailureClass = 'assertion' | 'contract' | 'timeout' | 'infra';
+export type TrialOutcome = 'passed' | 'failed' | 'skipped';
+export interface PanelShape { n: number; k: number }
+
+/** Environment the paid runner sets on an isolated trial shard. */
+export const TRIAL_ENV = {
+  caseId: 'GSTACK_EVAL_CASE_ID',
+  kind: 'GSTACK_EVAL_KIND',
+  trial: 'GSTACK_EVAL_TRIAL',
+  panelN: 'GSTACK_EVAL_PANEL_N',
+  panelK: 'GSTACK_EVAL_PANEL_K',
+  policyVersion: 'GSTACK_EVAL_POLICY_VERSION',
+} as const;
+
+/** Sidecar every expectContract() failure appends to (in GSTACK_EVAL_DIR), so a
+ *  contract veto survives a test that throws before recording its entry. */
+export const CONTRACT_VIOLATIONS_FILE = 'contract-violations.jsonl';
+
+export interface TrialContext {
+  case_id: string;
+  kind: EvalCaseKind;
+  trial: number;
+  panel: PanelShape;
+  policy_version: number;
+}
+
+const EVAL_KINDS: readonly EvalCaseKind[] = ['rule', 'behavior', 'judge'];
+const FAILURE_CLASSES: readonly TrialFailureClass[] = ['assertion', 'contract', 'timeout', 'infra'];
+
+function positiveInt(raw: string | undefined): number | null {
+  if (raw === undefined || !/^[1-9][0-9]*$/.test(raw)) return null;
+  return Number(raw);
+}
+
+/** Trial context of this process, or null outside an isolated trial shard.
+ *  A partial or malformed context throws: a mislabeled trial is fail-open. */
+export function trialContextFromEnv(env: NodeJS.ProcessEnv = process.env): TrialContext | null {
+  const caseId = env[TRIAL_ENV.caseId];
+  if (!caseId) return null;
+  const kind = env[TRIAL_ENV.kind] as EvalCaseKind | undefined;
+  const trial = positiveInt(env[TRIAL_ENV.trial]);
+  const n = positiveInt(env[TRIAL_ENV.panelN]);
+  const k = positiveInt(env[TRIAL_ENV.panelK]);
+  const policy = positiveInt(env[TRIAL_ENV.policyVersion]);
+  if (!kind || !EVAL_KINDS.includes(kind) || trial === null || n === null || k === null || policy === null || k > n || trial > n) {
+    throw new Error(`Malformed trial context for ${caseId}: ${Object.values(TRIAL_ENV).map((name) => `${name}=${env[name] ?? ''}`).join(' ')}`);
+  }
+  return { case_id: caseId, kind, trial, panel: { n, k }, policy_version: policy };
+}
+
+/** Failure class of a failed record: an explicit class wins, then the exit reason. */
+export function failureClassOf(entry: Pick<EvalTestEntry, 'failure_class' | 'exit_reason'>): TrialFailureClass {
+  if (entry.failure_class && FAILURE_CLASSES.includes(entry.failure_class)) return entry.failure_class;
+  return entry.exit_reason === 'timeout' ? 'timeout' : 'assertion';
+}
+
+export class ContractViolation extends Error {
+  constructor(message: string) {
+    super(`CONTRACT: ${message}`);
+    this.name = 'ContractViolation';
+  }
+}
+
+/**
+ * Assert a contract: an outcome the product must meet on every run. On failure
+ * it records failure_class 'contract' before throwing, both on the collector
+ * entry named `record.name` (now or when the test records it) and in the
+ * GSTACK_EVAL_DIR sidecar, so panelVerdict() fails the panel even at 2 of 3.
+ */
+export function expectContract(
+  condition: unknown,
+  message: string,
+  record?: { collector: EvalCollector | null; name: string },
+): asserts condition {
+  if (condition) return;
+  record?.collector?.markContractViolation(record.name, message);
+  const evalDir = process.env.GSTACK_EVAL_DIR;
+  if (evalDir) {
+    const context = trialContextFromEnv();
+    fs.mkdirSync(evalDir, { recursive: true });
+    fs.appendFileSync(path.join(evalDir, CONTRACT_VIOLATIONS_FILE), JSON.stringify({
+      case_id: context?.case_id ?? record?.name ?? null,
+      name: record?.name ?? null,
+      trial: context?.trial ?? null,
+      message,
+      at: new Date().toISOString(),
+    }) + '\n');
+  }
+  throw new ContractViolation(message);
+}
+
+export interface PanelTrial {
+  trial: number;
+  outcome: TrialOutcome;
+  /** Required meaning for a failed trial; absent reads as 'assertion'. */
+  failure_class?: TrialFailureClass;
+  /** CI run attempt (github.run_attempt); absent means 1. */
+  attempt?: number;
+  exit_reason?: string;
+  error?: string;
+  execution?: 'executed' | 'reused';
+}
+
+export interface PanelVerdictInput {
+  case: string;
+  kind: EvalCaseKind;
+  panel: PanelShape;
+  trials: readonly PanelTrial[];
+  quarantined?: boolean;
+}
+
+export type PanelStatus = 'PASS' | 'FAIL' | 'INCOMPLETE' | 'SKIPPED';
+
+export interface PanelVerdict {
+  case: string;
+  kind: EvalCaseKind;
+  panel: PanelShape;
+  attempt: number;
+  quarantined: boolean;
+  status: PanelStatus;
+  passed: number;
+  failed: number;
+  /** A failed trial carried failure_class 'contract'. */
+  contract: boolean;
+  /** PASS with at least one failed trial: shown as `PASS k/n`, never clean. */
+  split: boolean;
+  /** Whether this verdict makes the lane red. */
+  failsLane: boolean;
+  /** Whether it counts as passing coverage (never for quarantined or skipped). */
+  coverage: boolean;
+  /** Machine classification of a lane-failing verdict: INCOMPLETE (missing or
+   *  malformed trial records), INFRA (every failed trial is infra-class), or
+   *  VERDICT (a real red). Null when the verdict does not fail the lane. */
+  redClass: 'INCOMPLETE' | 'INFRA' | 'VERDICT' | null;
+  /** One glyph per trial index: ✓ pass, ✗ fail, – skipped, · missing. */
+  marks: string;
+  reason: string;
+  trials: PanelTrial[];
+}
+
+/**
+ * The single verdict function. `rule`/`judge` cases run panel {1,1}; `behavior`
+ * cases run EVAL_POLICY.panel; a quarantined case runs a full panel whose k
+ * keeps its kind's meaning (k = n for rule). Verdict: INCOMPLETE unless
+ * exactly one record per trial index 1..n; SKIPPED when every trial skipped;
+ * FAIL on any contract trial; otherwise PASS iff passed >= k. A quarantined
+ * FAIL fails the lane only on a hard break (0 of n) or a contract violation.
+ */
+export function panelVerdict(input: PanelVerdictInput): PanelVerdict {
+  const { n, k } = input.panel;
+  if (!Number.isInteger(n) || !Number.isInteger(k) || n < 1 || k < 1 || k > n) {
+    throw new Error(`${input.case}: invalid panel {n:${n}, k:${k}}`);
+  }
+  if (!EVAL_KINDS.includes(input.kind)) throw new Error(`${input.case}: unknown kind ${String(input.kind)}`);
+  const attempts = new Set(input.trials.map((t) => t.attempt ?? 1));
+  if (attempts.size > 1) {
+    throw new Error(`${input.case}: trials from run attempts ${[...attempts].join(', ')}; compute one verdict per attempt`);
+  }
+  const attempt = [...attempts][0] ?? 1;
+  const quarantined = input.quarantined === true;
+  const trials = [...input.trials].sort((a, b) => a.trial - b.trial);
+  const byIndex = new Map<number, PanelTrial>();
+  const problems: string[] = [];
+  for (const t of trials) {
+    if (!Number.isInteger(t.trial) || t.trial < 1 || t.trial > n) problems.push(`unexpected trial t${t.trial}`);
+    else if (byIndex.has(t.trial)) problems.push(`duplicate trial t${t.trial}`);
+    else if (t.outcome !== 'passed' && t.outcome !== 'failed' && t.outcome !== 'skipped') problems.push(`t${t.trial} has outcome ${String(t.outcome)}`);
+    else byIndex.set(t.trial, t);
+  }
+  for (let i = 1; i <= n; i++) if (!trials.some((t) => t.trial === i)) problems.push(`missing trial t${i}`);
+  const marks = Array.from({ length: n }, (_, i) => {
+    const t = byIndex.get(i + 1);
+    return !t ? '·' : t.outcome === 'passed' ? '✓' : t.outcome === 'failed' ? '✗' : '–';
+  }).join('');
+  const passed = [...byIndex.values()].filter((t) => t.outcome === 'passed').length;
+  const failedTrials = [...byIndex.values()].filter((t) => t.outcome === 'failed');
+  const skipped = [...byIndex.values()].filter((t) => t.outcome === 'skipped').length;
+  const contract = failedTrials.some((t) => failureClassOf(t) === 'contract');
+  const base = { case: input.case, kind: input.kind, panel: { n, k }, attempt, quarantined, passed, failed: failedTrials.length, contract, marks, trials };
+
+  if (problems.length === 0 && skipped === n) {
+    return { ...base, status: 'SKIPPED', split: false, failsLane: false, coverage: false, redClass: null, reason: 'every trial skipped (no verdict credit)' };
+  }
+  if (problems.length === 0 && skipped > 0) problems.push(`${skipped} of ${n} trials skipped`);
+  if (problems.length > 0) {
+    return { ...base, status: 'INCOMPLETE', split: false, failsLane: true, coverage: false, redClass: 'INCOMPLETE', reason: problems.join('; ') };
+  }
+  if (!contract && passed >= k) {
+    const split = failedTrials.length > 0;
+    return {
+      ...base, status: 'PASS', split, failsLane: false, coverage: !quarantined, redClass: null,
+      reason: split ? `PASS ${passed}/${n}` : `${passed}/${n} passed`,
+    };
+  }
+  const hardBreak = passed === 0;
+  const failsLane = !quarantined || contract || hardBreak;
+  const allInfra = !contract && failedTrials.length > 0 && failedTrials.every((t) => failureClassOf(t) === 'infra');
+  const why = contract ? 'contract violation' : `${passed}/${n} passed, needs ${k}`;
+  return {
+    ...base, status: 'FAIL', split: false, failsLane, coverage: false,
+    redClass: failsLane ? (allInfra ? 'INFRA' : 'VERDICT') : null,
+    reason: !quarantined ? why
+      : contract ? `${why}; quarantine never excuses a contract`
+        : hardBreak ? `${why}; quarantined hard break`
+          : `${why}; quarantined, does not fail the lane`,
+  };
+}
+
+// --- trial-outcomes JSONL (one line per trial; pass-rate history input) ---
+
+export const TRIAL_OUTCOME_SCHEMA = 'gstack-trial-outcome/v1';
+export const TRIAL_OUTCOMES_FILE = 'trial-outcomes.jsonl';
+/** Cap on a stored `error` line (sanitized first line of the failure). */
+export const TRIAL_ERROR_MAX = 300;
+
+export interface TrialOutcomeRecord {
+  schema: typeof TRIAL_OUTCOME_SCHEMA;
+  /** Registry id. */
+  case: string;
+  file: string;
+  tier: string;
+  kind: EvalCaseKind;
+  trial: number;
+  panel: PanelShape;
+  /** CI run attempt (github.run_attempt); 1 locally and for pre-policy backfill. */
+  attempt: number;
+  outcome: TrialOutcome;
+  /** Present exactly when outcome is 'failed'. */
+  failure_class?: TrialFailureClass;
+  exit_reason?: string;
+  error?: string;
+  duration_ms: number;
+  cost_usd: number;
+  model?: string;
+  cli_version?: string;
+  /** Reuse input key of the trial's shard, when known. */
+  input_identity?: string;
+  /** EVAL_POLICY.version; 0 marks pre-policy backfill. */
+  policy_version: number;
+  quarantined: boolean;
+  execution: 'executed' | 'reused';
+  /** shard: isolated trial shard status. junit: a rule file shard's per-test
+   *  JUnit outcome. backfill: imported pre-policy artifact record. */
+  source: 'shard' | 'junit' | 'backfill';
+  run_id?: string;
+  sha?: string;
+  lane?: string;
+  recorded_at?: string;
+}
+
+/** First line of free text, stripped of @-mentions and control characters, capped. */
+export function sanitizeTrialError(text: string | undefined): string | undefined {
+  if (!text) return undefined;
+  const first = text.split('\n').map((l) => l.trim()).find((l) => l.length > 0);
+  if (!first) return undefined;
+  // eslint-disable-next-line no-control-regex
+  const clean = first.replace(/[\u0000-\u001f\u007f]/g, ' ').replace(/`/g, "'").replace(/@(?=[A-Za-z0-9_-])/g, '@\u200b');
+  return clean.length > TRIAL_ERROR_MAX ? `${clean.slice(0, TRIAL_ERROR_MAX - 1)}…` : clean;
+}
+
+function trialRecordProblems(r: any): string[] {
+  const problems: string[] = [];
+  if (!r || typeof r !== 'object' || Array.isArray(r)) return ['not an object'];
+  if (r.schema !== TRIAL_OUTCOME_SCHEMA) problems.push(`schema ${String(r.schema)}`);
+  for (const key of ['case', 'file', 'tier'] as const) if (typeof r[key] !== 'string' || r[key].length === 0) problems.push(`${key} missing`);
+  if (!EVAL_KINDS.includes(r.kind)) problems.push(`kind ${String(r.kind)}`);
+  const n = r.panel?.n, k = r.panel?.k;
+  if (!Number.isInteger(n) || !Number.isInteger(k) || n < 1 || k < 1 || k > n) problems.push('panel invalid');
+  if (!Number.isInteger(r.trial) || r.trial < 1 || (Number.isInteger(n) && r.trial > n)) problems.push('trial invalid');
+  if (!Number.isInteger(r.attempt) || r.attempt < 1) problems.push('attempt invalid');
+  if (!['passed', 'failed', 'skipped'].includes(r.outcome)) problems.push(`outcome ${String(r.outcome)}`);
+  if (r.outcome === 'failed' && !FAILURE_CLASSES.includes(r.failure_class)) problems.push('failed without failure_class');
+  if (r.outcome !== 'failed' && r.failure_class !== undefined) problems.push('failure_class on a non-failed trial');
+  if (typeof r.duration_ms !== 'number' || !Number.isFinite(r.duration_ms) || r.duration_ms < 0) problems.push('duration_ms invalid');
+  if (typeof r.cost_usd !== 'number' || !Number.isFinite(r.cost_usd) || r.cost_usd < 0) problems.push('cost_usd invalid');
+  if (!Number.isInteger(r.policy_version) || r.policy_version < 0) problems.push('policy_version invalid');
+  if (typeof r.quarantined !== 'boolean') problems.push('quarantined invalid');
+  if (r.execution !== 'executed' && r.execution !== 'reused') problems.push('execution invalid');
+  if (!['shard', 'junit', 'backfill'].includes(r.source)) problems.push('source invalid');
+  if (r.error !== undefined && (typeof r.error !== 'string' || r.error.length > TRIAL_ERROR_MAX)) problems.push('error invalid');
+  return problems;
+}
+
+/** Serialize records as JSONL; throws on any invalid record (writers fail closed). */
+export function formatTrialOutcomes(records: readonly TrialOutcomeRecord[]): string {
+  return records.map((r) => {
+    const problems = trialRecordProblems(r);
+    if (problems.length > 0) throw new Error(`invalid trial record ${r?.case}~t${r?.trial}: ${problems.join(', ')}`);
+    return JSON.stringify(r);
+  }).join('\n') + (records.length > 0 ? '\n' : '');
+}
+
+/** Parse downloaded JSONL as data only: invalid lines are reported, never guessed. */
+export function parseTrialOutcomes(text: string, opts: { maxBytes?: number } = {}): { records: TrialOutcomeRecord[]; errors: string[] } {
+  const maxBytes = opts.maxBytes ?? 16 * 1024 * 1024;
+  if (Buffer.byteLength(text) > maxBytes) return { records: [], errors: [`trial outcomes exceed ${maxBytes} bytes`] };
+  const records: TrialOutcomeRecord[] = [];
+  const errors: string[] = [];
+  text.split('\n').forEach((line, i) => {
+    if (line.trim() === '') return;
+    let parsed: unknown;
+    try { parsed = JSON.parse(line); } catch { errors.push(`line ${i + 1}: not JSON`); return; }
+    const problems = trialRecordProblems(parsed);
+    if (problems.length > 0) errors.push(`line ${i + 1}: ${problems.join(', ')}`);
+    else records.push(parsed as TrialOutcomeRecord);
+  });
+  return { records, errors };
+}
+
 export interface EvalResult {
   schema_version: number;
   version: string;
@@ -887,6 +1219,7 @@ export class EvalCollector {
   private shard: string | null;
   private fileNamespace?: string;
   private createdAt = Date.now();
+  private pendingContract = new Map<string, string>();
 
   constructor(tier: 'e2e' | 'llm-judge', evalDir?: string, fileNamespace?: string) {
     if (fileNamespace !== undefined && !/^[a-z0-9]+(?:-[a-z0-9]+)*$/.test(fileNamespace)) {
@@ -903,7 +1236,29 @@ export class EvalCollector {
     // names are unique by convention). Stamp the 1-based attempt so a
     // pass-on-attempt-2 stays visible forever — the stream hides it.
     const prior = this.tests.filter((t) => t.name === entry.name).length;
-    this.tests.push({ ...entry, attempt: prior + 1 });
+    const context = trialContextFromEnv();
+    const record: EvalTestEntry = { ...(context ?? {}), ...entry, attempt: prior + 1 };
+    const contract = this.pendingContract.get(entry.name);
+    if (contract !== undefined) {
+      this.pendingContract.delete(entry.name);
+      Object.assign(record, { passed: false, failure_class: 'contract', error: record.error ?? contract });
+    }
+    this.tests.push(record);
+    this.savePartial();
+  }
+
+  /** expectContract() hook: mark `name`'s latest record (or its next one) as a
+   *  contract failure. An unmatched mark becomes its own failed record at
+   *  finalize, so the veto is never lost. */
+  markContractViolation(name: string, message: string): void {
+    const existing = this.tests.filter((t) => t.name === name).at(-1);
+    if (!existing) {
+      this.pendingContract.set(name, message);
+      return;
+    }
+    existing.passed = false;
+    existing.failure_class = 'contract';
+    existing.error = existing.error ?? message;
     this.savePartial();
   }
 
@@ -959,6 +1314,14 @@ export class EvalCollector {
   async finalize(): Promise<string> {
     if (this.finalized) return '';
     this.finalized = true;
+    for (const [name, message] of this.pendingContract) {
+      this.tests.push({
+        ...(trialContextFromEnv() ?? {}),
+        name, suite: 'contract', tier: this.tier, passed: false, duration_ms: 0, cost_usd: 0,
+        failure_class: 'contract', error: message, attempt: 1,
+      });
+    }
+    this.pendingContract.clear();
 
     const git = getGitInfo();
     const version = getVersion();

test/setup-gbrain-fixture.test.ts

@@ -169,7 +169,7 @@ describe('setup-gbrain owned Path 4 fixture', () => {
           pathToClaudeCodeExecutable: '/nonexistent/free-test-never-spawn-claude',
           signal: controller.signal,
         }, () => { validated = true; }, mode === 'deadline' ? 250 : 1000, {
-          collector: { addTest: (row: EvalTestEntry) => rows.push(row) } as EvalCollector,
+          collector: { addTest: (row: EvalTestEntry) => rows.push(row) } as unknown as EvalCollector,
           name: 'setup-gbrain-path4-local-pglite', suite: 'setup-gbrain',
         });
       } catch (error) { failure = String(error); }
@@ -214,7 +214,7 @@ describe('setup-gbrain owned Path 4 fixture', () => {
         await Promise.resolve();
         controller.abort(new Error('caller cancelled during validation'));
       }, 1000, {
-        collector: { addTest: (row: EvalTestEntry) => rows.push(row) } as EvalCollector,
+        collector: { addTest: (row: EvalTestEntry) => rows.push(row) } as unknown as EvalCollector,
         name: 'setup-gbrain-path4-local-pglite', suite: 'setup-gbrain',
       })).rejects.toThrow('caller cancelled during validation');
       expect(rows).toHaveLength(1);
@@ -288,7 +288,7 @@ describe('setup-gbrain owned Path 4 fixture', () => {
               throw new Error(`assertion diagnostic ${fixture.token} ${credentialUrl}`);
             }
           }, undefined, {
-            collector: { addTest: (row: EvalTestEntry) => rows.push(row) } as EvalCollector,
+            collector: { addTest: (row: EvalTestEntry) => rows.push(row) } as unknown as EvalCollector,
             name: 'setup-gbrain-path4-local-pglite', suite: 'setup-gbrain',
           });
         } catch (error) { thrown = String(error); }