From 30bcb700c16c529a3e855ef06ffc3581fa147c7c Mon Sep 17 00:00:00 2001 From: garrytan Date: Wed, 7 Oct 2026 01:18:09 +0000 Subject: [PATCH] feat(ship-measure): the measurement bar and the weekly off-ship sweep CLI Decide each diagnostic measurement with one bar (scripts/lib/measure-bar.ts): MEETS at 9/10; MEETS-qualified at 8/10 when every red is a mechanical provider error, judge noise or a cited model miss; EXTEND at 7/10 (or an unqualified 8/10) for one more batch on identical inputs, decided once on the pooled 20; BELOW otherwise and for any timeout, hang, regression or fixable cause. Behavior uses 12 trials (11/10/9; 22/20 of 24); judge counts median-panel outputs. A batch with more than 30% provider-evidence failures is void and redispatched once. Below-strict reds nobody classified return needs-classify. ship-measure gains classify (append-only records; provider refused without evidence), decide and extend (refuses a changed identity, names the field, never a third batch), exit codes 5 and 6, and a pooling identity (frozen tree, runner, CLI, model, CI image, Bun, policy and harness versions). ship-measure sweep ranks main's gate cases by their cost to the all-green probability from eval:pass-rates history, measures the top K with the same runner and bar under a weekly cap shared with the last 7 days of sweep reports (ship_measure_sweep_cases, ship_measure_sweep_budget_usd), and writes a report with the predicted all-green probability before and after. Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> --- bin/gstack-config | 26 +- scripts/lib/measure-bar.ts | 205 ++++++++++++ scripts/ship-measure-sweep.ts | 349 ++++++++++++++++++++ scripts/ship-measure.ts | 404 ++++++++++++++++------- test/fixtures/module-size-ratchet.json | 2 + test/helpers/touchfiles-data.ts | 2 +- test/ship-measure-bar.test.ts | 238 +++++++++++++ test/ship-measure-seeded-fixture.test.ts | 14 +- test/ship-measure-sweep.test.ts | 198 +++++++++++ test/ship-measure.test.ts | 84 ++--- 10 files changed, 1337 insertions(+), 185 deletions(-) create mode 100644 scripts/lib/measure-bar.ts create mode 100644 scripts/ship-measure-sweep.ts create mode 100644 test/ship-measure-bar.test.ts create mode 100644 test/ship-measure-sweep.test.ts diff --git a/bin/gstack-config b/bin/gstack-config index f1aebac24..19e53abec 100755 --- a/bin/gstack-config +++ b/bin/gstack-config @@ -188,15 +188,19 @@ CONFIG_HEADER='# gstack configuration — edit freely, changes take effect on ne # ─── /ship measure-then-fix loop ───────────────────────────────────── # A red eval case is measured alone before the full gate reruns # (scripts/ship-measure.ts; docs/TESTING_INTERNALS.md#ship-measure). -# ship_measure_rule_trials: 10 # rule cases: trials; pass at 90% (9 of 10) -# ship_measure_behavior_panels: 4 # behavior cases: panels of 3; every panel -# # at 2 of 3 and 90% of trials (11 of 12) -# ship_measure_judge_outputs: 10 # judge cases: outputs, each scored by a -# # 3-sample panel; pass at 90% of outputs +# Bar (scripts/lib/measure-bar.ts): MEETS at 9/10 of the trials, MEETS-qualified +# at 8/10 with every red qualifying, EXTEND at 7/10 (one more batch, pooled). +# ship_measure_rule_trials: 10 # rule cases: trials per batch (MEETS 9 of 10) +# ship_measure_behavior_panels: 4 # behavior cases: panels of 3 per batch +# # (12 trials; MEETS 11 of 12) +# ship_measure_judge_outputs: 10 # judge cases: outputs per batch, each scored +# # by its median 3-sample panel (MEETS 9 of 10) # ship_measure_ask_per_trial_usd: 2 # ask first above this estimated cost per trial # ship_measure_budget_usd: 25 # estimated admission budget per red case -# # (baseline, repair rounds and judge scoring) +# # (baseline, extension, redispatch, repair rounds) # ship_measure_max_rounds: 3 # repair rounds before a named-red stop +# ship_measure_sweep_cases: 5 # weekly off-ship sweep: cases measured (K) +# ship_measure_sweep_budget_usd: 150 # weekly off-ship sweep: spend cap per 7 days # ship_rerun_backend: local # local | ubicloud — where free-suite flake reruns # # run; ubicloud also needs UBICLOUD_API_KEY. # # Invalid values are REJECTED (value kept). @@ -271,6 +275,8 @@ lookup_default() { ship_measure_ask_per_trial_usd) echo "2" ;; ship_measure_budget_usd) echo "25" ;; ship_measure_max_rounds) echo "3" ;; + ship_measure_sweep_cases) echo "5" ;; + ship_measure_sweep_budget_usd) echo "150" ;; ship_rerun_backend) echo "local" ;; # local | ubicloud (Ubicloud spend is opt-in) # Unknown key: exit non-zero instead of printing "". The fallback pattern # the preambles use, @@ -675,12 +681,12 @@ case "${1:-}" in # The measure-loop controls bound paid spend: reject a typo instead of # coercing it, so a bad value never changes how much /ship may spend. case "$KEY" in - ship_measure_rule_trials|ship_measure_behavior_panels|ship_measure_judge_outputs|ship_measure_max_rounds) + ship_measure_rule_trials|ship_measure_behavior_panels|ship_measure_judge_outputs|ship_measure_max_rounds|ship_measure_sweep_cases) if ! printf '%s' "$VALUE" | LC_ALL=C grep -qE '^[1-9][0-9]*$'; then echo "Error: $KEY '$VALUE' is not a positive integer. Existing value left unchanged." >&2 exit 1 fi ;; - ship_measure_ask_per_trial_usd|ship_measure_budget_usd) + ship_measure_ask_per_trial_usd|ship_measure_budget_usd|ship_measure_sweep_budget_usd) if ! printf '%s' "$VALUE" | LC_ALL=C grep -qE '^([0-9]+(\.[0-9]+)?|\.[0-9]+)$' || ! printf '%s' "$VALUE" | LC_ALL=C grep -qE '[1-9]'; then echo "Error: $KEY '$VALUE' is not a positive amount in USD (e.g. 25 or 2.50). Existing value left unchanged." >&2 exit 1 @@ -876,7 +882,7 @@ case "${1:-}" in timeline_stop_hook design_detector design_detector_install_prompted memorable_recall \ browse_extension_id ship_measure_rule_trials ship_measure_behavior_panels \ ship_measure_judge_outputs ship_measure_ask_per_trial_usd ship_measure_budget_usd \ - ship_measure_max_rounds ship_rerun_backend; do + ship_measure_max_rounds ship_measure_sweep_cases ship_measure_sweep_budget_usd ship_rerun_backend; do gstack_config_select "$KEY" VALUE="$_gstack_cfg_value" SOURCE="default" @@ -899,7 +905,7 @@ case "${1:-}" in timeline_stop_hook design_detector design_detector_install_prompted memorable_recall \ browse_extension_id ship_measure_rule_trials ship_measure_behavior_panels \ ship_measure_judge_outputs ship_measure_ask_per_trial_usd ship_measure_budget_usd \ - ship_measure_max_rounds ship_rerun_backend; do + ship_measure_max_rounds ship_measure_sweep_cases ship_measure_sweep_budget_usd ship_rerun_backend; do printf ' %-24s %s\n' "$KEY:" "$(lookup_default "$KEY")" done ;; diff --git a/scripts/lib/measure-bar.ts b/scripts/lib/measure-bar.ts new file mode 100644 index 000000000..0428a1750 --- /dev/null +++ b/scripts/lib/measure-bar.ts @@ -0,0 +1,205 @@ +/** + * The ship-measure measurement bar (approved by Garry 2026-10-06): one pure + * decision over a case's diagnostic trials, shared by /ship's measure loop + * (scripts/ship-measure.ts) and the weekly off-ship sweep + * (scripts/ship-measure-sweep.ts). Nothing here runs a trial or reads a file. + * + * For N trials per batch (rule and judge N = 10; behavior N = 12, four panels + * of 3; a judge trial is one output scored by its median 3-sample panel), the + * counts round up from 9/10, 8/10 and 7/10 of N: + * + * MEETS passes >= 9/10 of N. + * MEETS-qualified passes >= 8/10 of N and every failed trial is in a + * qualifying class: provider (affirmative provider or + * transport evidence only), judge-noise, or model-miss + * citing its evidence (path:line). + * EXTEND passes >= 7/10 of N without meeting either bar, and no red + * has a known fixable cause: run N more trials on identical + * inputs, then decide once on the pooled 2N (>= 9/10 strict, + * >= 8/10 qualified). There is never a third batch. + * BELOW everything else, and any red in a fix-round class + * (timeout, hang, regression, fixable) or a contract + * violation: a fix round. + * + * Before the counts: a batch where more than 30% of trials fail with + * affirmative provider evidence is void (an outage, not a measurement) and may + * be redispatched once; both batches are reported. A batch below strict whose + * failed trials are not all classified returns needs-classify when the + * classification decides the outcome (passes at or above the 7/10 floor). + */ + +export type BarKind = 'rule' | 'behavior' | 'judge'; +export type TrialSet = 'initial' | 'redispatch' | 'extend' | 'extend-redispatch'; +export const QUALIFYING_CLASSES = ['provider', 'judge-noise', 'model-miss'] as const; +export const FIX_ROUND_CLASSES = ['timeout', 'hang', 'regression', 'fixable'] as const; +export const FAILURE_CLASSES = [...QUALIFYING_CLASSES, ...FIX_ROUND_CLASSES, 'unclassified'] as const; +export type FailureClass = (typeof FAILURE_CLASSES)[number]; + +/** A batch is void when more than this share of its trials fail on affirmative provider evidence. */ +export const VOID_SHARE = { num: 3, den: 10 } as const; + +export interface BarTrial { + trial: number; + set: TrialSet; + passed: boolean; + contract?: boolean; + /** Judge outputs: each sample's pass; the output passes at 2 of 3. */ + samples?: boolean[]; + failureCause?: string; + failureDetail?: string; + failureEvidence?: string; +} + +export interface Classification { trial: number; class: FailureClass; evidence: string; by: 'machine' | 'agent'; at: string } + +export type BarDecision = 'MEETS' | 'MEETS-qualified' | 'EXTEND' | 'BELOW' | 'needs-classify' | 'void' | 'incomplete'; +export type BarNext = 'none' | 'redispatch' | 'extend' | 'classify' | 'fix' | 'stop'; + +export interface BarThresholds { trials: number; strict: number; qualified: number; floor: number } +export interface SetSummary { set: TrialSet; trials: number; passes: number; providerFailures: number; void: boolean } + +export interface BarOutcome { + decision: BarDecision; + next: BarNext; + passes: number; + thresholds: BarThresholds; + pooled: boolean; + sets: SetSummary[]; + /** Failed counted trials that still need a class. */ + needsClassify: number[]; + reason: string; +} + +const atLeast = (n: number, num: number) => Math.ceil((n * num) / 10); + +/** Strict 9/10, qualified 8/10 and extend floor 7/10 of n, rounded up (10: 9/8/7; 12: 11/10/9; 20: 18/16/14; 24: 22/20/17). */ +export function barThresholds(n: number): BarThresholds { + return { trials: n, strict: atLeast(n, 9), qualified: atLeast(n, 8), floor: atLeast(n, 7) }; +} + +/** A trial's pass under the bar: a judge output at 2 of its 3 samples; a contract violation never passes. */ +export function trialPassed(kind: BarKind, t: BarTrial): boolean { + if (t.contract) return false; + if (kind === 'judge' && t.samples) return t.samples.length === 3 && t.samples.filter(Boolean).length >= 2; + return t.passed; +} + +const PROVIDER_TEXT = /\b(?:429|5\d\d)\b|rate[ _-]?limit|overloaded|api_error|ECONNRESET|connection reset|socket hang up|ETIMEDOUT|EAI_AGAIN|service unavailable|bad gateway/i; +const TIMEOUT_CAUSES = new Set(['session_timeout', 'observer_timeout', 'provider_stall']); + +/** + * The affirmative provider or transport evidence for a failed trial, or null. + * `api_error` is provider evidence by itself; `pre_turn_infra` only when its + * evidence or detail names a 429/5xx, rate limit, overload or connection reset + * (a crash or module-load failure is a harness defect, never an outage). + */ +export function providerEvidence(t: BarTrial): string | null { + if (t.passed && !t.contract) return null; + const text = [t.failureEvidence, t.failureDetail].filter(Boolean).join(' '); + if (t.failureCause === 'api_error') return text || 'failure_cause api_error'; + if (t.failureCause === 'pre_turn_infra' && PROVIDER_TEXT.test(text)) return text; + return null; +} + +/** The class a failed trial gets without review: provider on affirmative evidence; timeout or hang from its cause. */ +export function machineClass(t: BarTrial): FailureClass | null { + if (providerEvidence(t)) return 'provider'; + if (t.failureCause === 'provider_stall') return 'hang'; + if (t.failureCause && TIMEOUT_CAUSES.has(t.failureCause)) return 'timeout'; + return null; +} + +const CITATION = /[\w./-]+:\d+/; + +/** Why an agent classification is refused, or null when it is accepted. */ +export function classificationProblem(t: BarTrial | undefined, kind: BarKind, cls: string, evidence: string): string | null { + if (!t) return 'no such trial in this measurement'; + if (trialPassed(kind, t)) return `trial ${t.trial} passed; only failed trials are classified`; + if (!(FAILURE_CLASSES as readonly string[]).includes(cls)) return `class must be one of ${FAILURE_CLASSES.join(', ')}. Received: ${cls}`; + if (!evidence.trim()) return 'every classification needs --evidence (what you read: a capture path:line, a failure line, a transcript line)'; + if (cls === 'provider' && !providerEvidence(t)) { + return `provider needs affirmative provider or transport evidence (failure_cause api_error, or pre_turn_infra naming a 429/5xx, rate limit or connection reset); trial ${t.trial}'s failure_cause is ${t.failureCause ?? 'unrecorded'}`; + } + if ((QUALIFYING_CLASSES as readonly string[]).includes(cls) && t.failureCause && TIMEOUT_CAUSES.has(t.failureCause)) { + return `trial ${t.trial} failed on ${t.failureCause}: a timeout or hang never qualifies; it goes to a fix round`; + } + return null; +} + +/** Each failed trial's effective class: its latest agent record, else its machine class. */ +export function effectiveClasses(trials: readonly BarTrial[], records: readonly Classification[]): Map { + const out = new Map(); + for (const t of trials) { + const cls = machineClass(t); + if (cls) out.set(t.trial, { trial: t.trial, class: cls, evidence: providerEvidence(t) ?? t.failureCause ?? '', by: 'machine', at: '' }); + } + for (const r of records) if (r.by === 'agent') out.set(r.trial, r); + return out; +} + +const qualifies = (c: Classification) => (QUALIFYING_CLASSES as readonly string[]).includes(c.class) && (c.class !== 'model-miss' || CITATION.test(c.evidence)); +const fixRound = (c: Classification) => (FIX_ROUND_CLASSES as readonly string[]).includes(c.class); + +function summarize(kind: BarKind, set: TrialSet, trials: BarTrial[]): SetSummary { + const providerFailures = trials.filter(t => !trialPassed(kind, t) && providerEvidence(t)).length; + return { set, trials: trials.length, passes: trials.filter(t => trialPassed(kind, t)).length, providerFailures, + void: trials.length > 0 && providerFailures * VOID_SHARE.den > trials.length * VOID_SHARE.num }; +} + +/** The counted batch of a pair (first try, its one redispatch), or a void/incomplete stop. */ +function countedBatch(n: number, first: SetSummary, retry: SetSummary, firstTrials: BarTrial[], retryTrials: BarTrial[], label: string): + { trials: BarTrial[] } | { stop: Pick } { + const use = first.void ? retry : first; + if (use.trials === 0) return { stop: { decision: 'void', next: 'redispatch', reason: `${label} batch is void: ${first.providerFailures} of ${first.trials} trials failed on provider evidence (> 30%); redispatch it once` } }; + if (use.void) return { stop: { decision: 'void', next: 'stop', reason: `${label} batch and its one redispatch are both void (provider evidence in ${first.providerFailures}/${first.trials} and ${retry.providerFailures}/${retry.trials}); stop with a named infrastructure red` } }; + if (use.trials < n) return { stop: { decision: 'incomplete', next: 'stop', reason: `${label} batch ran ${use.trials} of ${n} trials` } }; + return { trials: first.void ? retryTrials : firstTrials }; +} + +/** + * The bar's decision over a measurement's trials and its classification + * records (append-only; the latest agent record for a trial wins). + */ +export function decide(kind: BarKind, n: number, trials: readonly BarTrial[], records: readonly Classification[] = []): BarOutcome { + const of = (set: TrialSet) => trials.filter(t => t.set === set); + const sets = (['initial', 'redispatch', 'extend', 'extend-redispatch'] as const).map(set => summarize(kind, set, of(set))); + const [initial, redispatch, extend, extendRedispatch] = sets as [SetSummary, SetSummary, SetSummary, SetSummary]; + const reported = sets.filter(s => s.trials > 0); + const base = countedBatch(n, initial, redispatch, of('initial'), of('redispatch'), 'the first'); + const empty = { passes: 0, pooled: false, sets: reported, needsClassify: [] as number[] }; + if ('stop' in base) return { ...empty, thresholds: barThresholds(n), ...base.stop }; + const pooled = extend.trials > 0; + const ext = pooled ? countedBatch(n, extend, extendRedispatch, of('extend'), of('extend-redispatch'), 'the extension') : { trials: [] }; + if ('stop' in ext) return { ...empty, pooled, thresholds: barThresholds(2 * n), ...ext.stop }; + const counted = [...base.trials, ...ext.trials]; + const thresholds = barThresholds(counted.length); + const passes = counted.filter(t => trialPassed(kind, t)).length; + const failed = counted.filter(t => !trialPassed(kind, t)); + const classes = effectiveClasses(failed, records); + const result = (decision: BarDecision, next: BarNext, reason: string, needsClassify: number[] = []): BarOutcome => + ({ decision, next, passes, thresholds, pooled, sets: reported, needsClassify, reason }); + const count = `${passes}/${counted.length}${pooled ? ' pooled' : ''}`; + const contract = failed.find(t => t.contract); + if (contract) return result('BELOW', 'fix', `${count}; trial ${contract.trial} violated a contract: a fix round, whatever the count`); + if (passes >= thresholds.strict) return result('MEETS', 'none', `${count} meets the strict bar (${thresholds.strict}/${counted.length})`); + if (passes < (pooled ? thresholds.qualified : thresholds.floor)) { + return result('BELOW', 'fix', `${count} is under the ${pooled ? `qualified pooled bar (${thresholds.qualified}/${counted.length})` : `extend floor (${thresholds.floor}/${counted.length})`}: a fix round`); + } + const unclassified = failed.filter(t => !classes.has(t.trial)).map(t => t.trial); + if (unclassified.length) return result('needs-classify', 'classify', `${count} is below strict; classify trial(s) ${unclassified.map(t => `t${t}`).join(', ')}, then decide`, unclassified); + const fix = failed.find(t => fixRound(classes.get(t.trial)!)); + if (fix) return result('BELOW', 'fix', `${count}; trial ${fix.trial} is ${classes.get(fix.trial)!.class}, which never qualifies: a fix round`); + if (passes >= thresholds.qualified && failed.every(t => qualifies(classes.get(t.trial)!))) { + return result('MEETS-qualified', 'none', `${count} meets the qualified bar (${thresholds.qualified}/${counted.length}); every red qualifies (${failed.map(t => `t${t.trial} ${classes.get(t.trial)!.class}`).join(', ')})`); + } + if (pooled) return result('BELOW', 'fix', `${count} does not meet the pooled bar (${thresholds.strict} strict, or ${thresholds.qualified} with every red qualifying); there is no third batch: a fix round`); + return result('EXTEND', 'extend', `${count} is at or above the extend floor (${thresholds.floor}/${counted.length}) without meeting a bar: run ${n} more trials on identical inputs, then decide once on the pooled ${2 * n}`); +} + +/** The first identity field that differs between two batches, or null when they may pool. */ +export function identityMismatch(a: Record, b: Record): string | null { + for (const key of [...new Set([...Object.keys(a), ...Object.keys(b)])].sort()) { + if (a[key] !== b[key]) return `${key} changed (${a[key] ?? 'absent'} -> ${b[key] ?? 'absent'})`; + } + return null; +} diff --git a/scripts/ship-measure-sweep.ts b/scripts/ship-measure-sweep.ts new file mode 100644 index 000000000..564df72d0 --- /dev/null +++ b/scripts/ship-measure-sweep.ts @@ -0,0 +1,349 @@ +/** + * ship-measure sweep — the weekly off-ship qualification sweep on main + * (approved by Garry 2026-10-06; docs/TESTING_INTERNALS.md#ship-measure-sweep). + * + * With ~130 gate verdicts, an all-green gate is unlikely even when every case + * is healthy, so flaky cases are measured on main before any ship hits them. + * The sweep reads eval:pass-rates history (evals-periodic.yml on main plus the + * branch census runs that pool into it), ranks the gate cases with a failed + * trial by how much each costs the gate's all-green probability Π(1 − p_i) + * (p_i: its red-verdict rate shrunk toward the pooled rate, as + * `eval:pass-rates --reds` computes it; ties broken by the lower per-trial + * pass rate), and measures the top K at their kind's N on the + * checked-out head with the same runner and measurement bar /ship uses + * (scripts/ship-measure.ts, scripts/lib/measure-bar.ts). + * + * Spend: ship_measure_sweep_budget_usd (default $150) is a weekly cap enforced + * through the same admission budget as /ship's loop, shared across the + * sweep's cases and every sweep report of the last 7 days found under --prior + * (CI downloads the earlier runs' reports there) and the local sweep root. + * When the cap is exhausted the sweep stops and lists what it skipped. + * + * Every trial is diagnostic: it never becomes a verdict and never enters + * EVAL_POLICY pooling or series history. The sweep never pushes, commits or + * opens a pull request; an agent fixes from its report. + */ +import * as fs from 'node:fs'; +import * as path from 'node:path'; +import { spawnSync } from 'node:child_process'; +import { CASE_QUARANTINE } from '../test/helpers/periodic-exclude-data'; +import { E2E_KINDS, E2E_TIERS, LLM_JUDGE_TOUCHFILES } from '../test/helpers/touchfiles-data'; +import { parseCliFlags } from './lib/shard-engine'; +import { RED_RATE_SHRINK, verdictsOf } from './lib/eval-history'; +import { repoSlug } from './lib/ci-history'; +import { loadPassRateHistory, type TrialRecord as HistoryRecord } from './eval-flake-rank'; +import { + caseSelectionPreflight, caseSlug, commandRunner, gstackRunner, kindPlan, measureCase, measurementIdentity, readMeasureConfig, + type MeasureConfig, type MeasureKind, type Measurement, type SpendPool, type TrialRunner, +} from './ship-measure'; + +const ROOT = path.resolve(import.meta.dir, '..'); +const WEEK_MS = 7 * 24 * 3600_000; +export const SWEEP_SCHEMA = 'gstack-ship-measure-sweep/1'; + +export interface SweepCandidate { + case: string; + kind: MeasureKind; + /** Red-verdict rate over gate history, shrunk toward the pooled rate (K = RED_RATE_SHRINK, as eval:pass-rates --reds). */ + redRate: number; + reds: number; + verdicts: number; + /** Per-trial pass rate over the history window, or null without scored trials. */ + passRate: number | null; + scoredTrials: number; + /** All-green probability gained if this case never went red: P / (1 − p_i) − P. */ + contribution: number; + /** Mean known cost per trial from history (else the gate-wide mean), or null. */ + costPerTrialUsd: number | null; +} + +export interface SweepPlan { + candidates: SweepCandidate[]; + picked: SweepCandidate[]; + /** All-green probability over the latest census's gate verdicts, from history. */ + allGreen: number | null; + censusCases: string[]; + pooledRate: number; + /** Per-case shrunk red rates for every case in the latest census. */ + rates: Record; +} + +const liveCase = (id: string) => Object.hasOwn(E2E_TIERS, id) || Object.hasOwn(LLM_JUDGE_TOUCHFILES, id); +export const sweepKind = (id: string): MeasureKind => (E2E_KINDS[id] as MeasureKind | undefined) ?? (Object.hasOwn(E2E_TIERS, id) ? 'rule' : 'judge'); + +/** + * Rank the gate cases history flags: every live, non-quarantined case of the + * latest gate census with a failed trial in the window, by its contribution + * to the all-green probability, then by the lower per-trial pass rate. A red + * verdict here is a lane-failing verdict with at least one failed trial (an + * INCOMPLETE record-count verdict is a reporting fault, not a flake). Picks + * the top k. + */ +export function planSweep(records: readonly HistoryRecord[], k: number, opts: { quarantine?: Record } = {}): SweepPlan { + const quarantine = opts.quarantine ?? CASE_QUARANTINE; + const gate = records.filter(r => r.tier === 'gate' && r.policy_version >= 1); + const verdicts = verdictsOf(gate).filter(v => v.status !== 'SKIPPED'); + const red = (v: (typeof verdicts)[number]) => v.failsLane && v.trials.some(t => t.outcome === 'failed'); + const pooledRate = verdicts.length ? verdicts.filter(red).length / verdicts.length : 0; + const byCase = new Map(); + for (const v of verdicts) { + const e = byCase.get(v.case) ?? { verdicts: 0, reds: 0 }; + e.verdicts++; if (red(v)) e.reds++; + byCase.set(v.case, e); + } + const rates: Record = Object.fromEntries([...byCase].map(([id, e]) => [id, (e.reds + RED_RATE_SHRINK * pooledRate) / (e.verdicts + RED_RATE_SHRINK)])); + const latestRun = verdicts.map(v => v.run).filter(run => run !== 'local').sort().at(-1) ?? 'local'; + const censusCases = [...new Set(verdicts.filter(v => v.run === latestRun).map(v => v.case))].sort(); + const allGreen = censusCases.length ? censusCases.reduce((p, id) => p * (1 - rates[id]!), 1) : null; + const scored = gate.filter(r => r.execution === 'executed' && (r.outcome === 'passed' || r.outcome === 'failed')); + const knownCost = (r: HistoryRecord) => r.execution === 'executed' && r.cost_known !== false && r.cost_usd > 0; + const mean = (xs: number[]) => xs.length ? xs.reduce((a, b) => a + b, 0) / xs.length : null; + const gateCost = mean(records.filter(r => r.tier === 'gate' && knownCost(r)).map(r => r.cost_usd)); + const candidates: SweepCandidate[] = []; + for (const id of censusCases) { + if (!liveCase(id) || Object.hasOwn(quarantine, id)) continue; + const trials = scored.filter(r => r.case === id); + const failedTrials = trials.filter(r => r.outcome === 'failed').length; + const e = byCase.get(id)!; + if (failedTrials === 0) continue; + candidates.push({ + case: id, kind: sweepKind(id), redRate: rates[id]!, reds: e.reds, verdicts: e.verdicts, + passRate: trials.length ? (trials.length - failedTrials) / trials.length : null, scoredTrials: trials.length, + contribution: allGreen === null ? 0 : allGreen / (1 - rates[id]!) - allGreen, + costPerTrialUsd: mean(records.filter(r => r.case === id && knownCost(r)).map(r => r.cost_usd)) ?? gateCost, + }); + } + candidates.sort((a, b) => b.contribution - a.contribution || b.redRate - a.redRate || (a.passRate ?? 1) - (b.passRate ?? 1) || (a.case < b.case ? -1 : 1)); + return { candidates, picked: candidates.slice(0, k), allGreen, censusCases, pooledRate, rates }; +} + +/** + * A measured case's predicted red-verdict rate: its measured per-trial failure + * fraction shrunk toward the pooled rate with the same K as history, and for a + * behavior case the 3-trial panel's red risk 1 − P(at least 2 of 3 pass). + */ +export function measuredRedRate(kind: MeasureKind, failures: number, trials: number, pooledRate: number): number { + const q = (failures + RED_RATE_SHRINK * pooledRate) / (trials + RED_RATE_SHRINK); + return kind === 'behavior' ? 1 - ((1 - q) ** 3 + 3 * (1 - q) ** 2 * q) : q; +} + +/** Π(1 − p_i) over the census cases, with measured cases' rates replacing history's. */ +export function predictedAllGreen(plan: SweepPlan, measured: Record): number | null { + if (plan.allGreen === null) return null; + return plan.censusCases.reduce((p, id) => p * (1 - (measured[id] ?? plan.rates[id] ?? plan.pooledRate)), 1); +} + +export interface SweepRow { + rank: number; + case: string; + kind: MeasureKind; + historyPassRate: number | null; + historyTrials: number; + reds: number; + verdicts: number; + redRate: number; + status: 'planned' | 'measured' | 'budget_exhausted' | 'skipped' | 'error'; + decision: string; + measuredPasses: number | null; + measuredTrials: number | null; + estimatedUsd: number | null; + actualUsd: number; + costUnknownTrials: number; + captures: string | null; + note: string; +} + +export interface SweepReport { + schema: typeof SWEEP_SCHEMA; + label: 'diagnostic'; + verdict: null; + sweepId: string; + startedAt: string; + finishedAt: string | null; + ref: string; + sha: string; + dryRun: boolean; + k: number; + capUsd: number; + priorWeekUsd: number; + /** Charged against the weekly cap by this sweep: known costs plus the estimate for trials whose cost is unknown. */ + chargedUsd: number; + historyScope: string; + allGreenBefore: number | null; + allGreenAfter: number | null; + censusVerdicts: number; + rows: SweepRow[]; +} + +/** Spend charged by sweep reports under dirs started within the last 7 days (deduplicated by sweep id). */ +export function weeklySpent(dirs: readonly string[], now: number, excludeId?: string): number { + const seen = new Map(); + for (const dir of dirs) { + if (!fs.existsSync(dir)) continue; + for (const name of fs.readdirSync(dir, { recursive: true }) as string[]) { + if (path.basename(name) !== 'sweep-report.json') continue; + let report: SweepReport; + try { report = JSON.parse(fs.readFileSync(path.join(dir, name), 'utf8')) as SweepReport; } catch { continue; } + if (report.schema !== SWEEP_SCHEMA || report.sweepId === excludeId || report.dryRun) continue; + const started = Date.parse(report.startedAt); + if (!Number.isFinite(started) || now - started > WEEK_MS || started > now) continue; + seen.set(report.sweepId, Math.max(seen.get(report.sweepId) ?? 0, report.chargedUsd)); + } + } + return [...seen.values()].reduce((a, b) => a + b, 0); +} + +const pct = (x: number | null) => x === null ? 'n/a' : `${Math.round(x * 1000) / 10}%`; +const prob = (x: number | null) => x === null || x >= 0.001 || x === 0 ? pct(x) : `${(x * 100).toPrecision(2)}%`; +const money = (x: number | null) => x === null ? 'unknown' : `$${x.toFixed(2)}`; +const cell = (text: string) => text.replace(/\|/g, '/').replace(/\n/g, ' '); + +export function formatSweepReport(r: SweepReport): string { + const rows = r.rows.map(row => `| ${row.rank} | ${cell(row.case)} | ${row.kind} | ${pct(row.historyPassRate)} of ${row.historyTrials} trials; ${row.reds}/${row.verdicts} red verdicts (rate ${pct(row.redRate)}) | ` + + `${row.measuredTrials === null ? '—' : `${row.measuredPasses}/${row.measuredTrials}`} | ${cell(row.decision)} | ${money(row.estimatedUsd)} / ${money(row.actualUsd)}${row.costUnknownTrials ? ` (+${row.costUnknownTrials} unknown)` : ''} | ${row.captures ? `\`${cell(row.captures)}\`` : '—'} |`); + const noted = r.rows.filter(row => row.status !== 'measured'); + return [ + `## ship-measure sweep${r.dryRun ? ' (dry run: nothing ran, $0)' : ''}: diagnostic measurements, not verdicts`, + '', + `Ref ${r.ref} at ${r.sha.slice(0, 12)}; K=${r.k}; weekly cap $${r.capUsd} (charged earlier this week $${r.priorWeekUsd.toFixed(2)}, this sweep $${r.chargedUsd.toFixed(2)}). History: ${r.historyScope}.`, + '', + `Predicted gate all-green probability (approximation, assumes independent verdicts, over the latest census's ${r.censusVerdicts} gate verdicts): before ${prob(r.allGreenBefore)} → after ${prob(r.allGreenAfter)}${r.dryRun ? ' (no measurements yet)' : ' (measured cases use this sweep\'s rates)'}.`, + '', + '| # | Case | Kind | History rate | Measured | Decision | Cost (est / actual) | Captures |', + '|---|---|---|---|---|---|---|---|', + ...(rows.length ? rows : ['| — | no case in the latest gate census has a failed trial or red verdict | | | | | | |']), + ...(noted.length ? ['', r.dryRun ? 'Plan:' : 'Skipped or stopped:', ...noted.map(row => `- ${row.case}: ${row.note}`)] : []), + '', + 'Every trial is diagnostic: it never becomes a verdict and never enters pass-rate history. The sweep never pushes or opens a pull request. ' + + 'Fix a BELOW case at its cause, classify a needs-classify case (`ship-measure classify --out --case --round baseline ...`), and measure again on the fix.', + ].join('\n'); +} + +export interface SweepOptions { + records: readonly HistoryRecord[]; + historyScope: string; + k: number; + capUsd: number; + config: MeasureConfig; + outDir: string; + priorDirs: string[]; + dryRun: boolean; + parallel: number; + ref: string; + sha: string; + now?: number; + /** One runner for every case (default: gstack's single-case runner after a selection preflight). */ + runner?: TrialRunner; + runnerId?: string; + preflight?: (caseId: string) => { ok: boolean; detail: string }; + identity?: (kind: MeasureKind, trials: number) => Record; + evalDir?: string; + log?: (line: string) => void; +} + +/** Plan, measure in rank order under the weekly cap, and write sweep-report.json/.md after every case. */ +export async function runSweep(o: SweepOptions): Promise { + const log = o.log ?? ((line: string) => console.log(line)); + const now = o.now ?? Date.now(); + const sweepId = new Date(now).toISOString().replace(/[:.]/g, '-'); + const dir = path.join(o.outDir, `sweep-${sweepId}`); + fs.mkdirSync(dir, { recursive: true }); + const plan = planSweep(o.records, o.k); + const priorWeekUsd = weeklySpent([...o.priorDirs, o.outDir], now, sweepId); + const pool: SpendPool = { label: `weekly sweep cap`, capUsd: o.capUsd, spentUsd: priorWeekUsd }; + const report: SweepReport = { + schema: SWEEP_SCHEMA, label: 'diagnostic', verdict: null, sweepId, startedAt: new Date(now).toISOString(), finishedAt: null, ref: o.ref, sha: o.sha, + dryRun: o.dryRun, k: o.k, capUsd: o.capUsd, priorWeekUsd, chargedUsd: 0, historyScope: o.historyScope, + allGreenBefore: plan.allGreen, allGreenAfter: plan.allGreen, censusVerdicts: plan.censusCases.length, rows: [], + }; + const save = () => { + report.chargedUsd = Math.max(0, pool.spentUsd - priorWeekUsd); + fs.writeFileSync(path.join(dir, 'sweep-report.json'), `${JSON.stringify(report, null, 2)}\n`); + fs.writeFileSync(path.join(dir, 'sweep-report.md'), `${formatSweepReport(report)}\n`); + }; + const measuredRates: Record = {}; + for (const [index, c] of plan.picked.entries()) { + const n = kindPlan(c.kind, o.config).trials; + const row: SweepRow = { + rank: index + 1, case: c.case, kind: c.kind, historyPassRate: c.passRate, historyTrials: c.scoredTrials, reds: c.reds, verdicts: c.verdicts, redRate: c.redRate, + status: 'planned', decision: 'planned', measuredPasses: null, measuredTrials: null, + estimatedUsd: c.costPerTrialUsd === null ? null : c.costPerTrialUsd * n, actualUsd: 0, costUnknownTrials: 0, captures: null, note: '', + }; + report.rows.push(row); + if (o.dryRun) { row.note = `would measure ${n} trials at ${c.costPerTrialUsd === null ? 'an unknown cost (one calibration trial first)' : `${money(c.costPerTrialUsd)}/trial`}`; save(); continue; } + if (pool.spentUsd >= pool.capUsd) { + Object.assign(row, { status: 'skipped', decision: 'skipped', note: `weekly cap $${o.capUsd} exhausted ($${pool.spentUsd.toFixed(2)} charged in the last 7 days)` }); + save(); + continue; + } + const selected = (o.preflight ?? caseSelectionPreflight)(c.case); + if (!selected.ok) { Object.assign(row, { status: 'skipped', decision: 'skipped', note: `selection preflight failed (no paid call made): ${selected.detail}` }); save(); continue; } + let m: Measurement; + try { + m = await measureCase({ + caseId: c.case, kind: c.kind, round: 'baseline', config: o.config, outDir: dir, parallel: o.parallel, costPerTrialUsd: c.costPerTrialUsd, approved: true, + runner: o.runner ?? gstackRunner(), pool, log, + identity: () => (o.identity ?? ((kind, trials) => measurementIdentity(kind, trials, o.runnerId ?? 'gstack')))(c.kind, n), + ...(o.evalDir ? { evalDir: o.evalDir } : {}), + }); + } catch (error) { + Object.assign(row, { status: 'error', decision: 'error', note: (error as Error).message }); + save(); + continue; + } + const captures = path.relative(o.outDir, path.join(dir, caseSlug(c.case), 'baseline')); + Object.assign(row, { + status: m.status === 'measured' ? 'measured' : 'budget_exhausted', decision: m.status === 'measured' ? m.decision : m.status.replace('_', ' '), + measuredPasses: m.passes, measuredTrials: m.counted || m.trials.length, actualUsd: m.actualUsd, costUnknownTrials: m.costUnknownTrials, captures, + note: m.status === 'measured' ? m.decisionReason : m.reason ?? '', + }); + if (m.status === 'measured' && m.decision !== 'void' && m.decision !== 'incomplete') { + measuredRates[c.case] = measuredRedRate(c.kind, m.counted - m.passes, m.counted, plan.pooledRate); + report.allGreenAfter = predictedAllGreen(plan, measuredRates); + } + save(); + } + report.finishedAt = new Date().toISOString(); + save(); + log(formatSweepReport(report)); + log(`[ship-measure] sweep report: ${path.join(dir, 'sweep-report.md')}`); + return report; +} + +function gitText(args: string[]): string { + const r = spawnSync('git', args, { cwd: ROOT, encoding: 'utf8', timeout: 30_000 }); + return r.status === 0 ? r.stdout.trim() : 'unknown'; +} + +/** `ship-measure sweep` CLI. Exits 0 when the sweep completed (whatever it measured), 1 when history was unavailable, 4 on usage errors. */ +export async function sweepMain(argv: string[], env: NodeJS.ProcessEnv = process.env): Promise { + const flags: Record = {}; + const historyDirs: string[] = []; + const priorDirs: string[] = []; + const value = (name: string) => (next: () => string | undefined) => { const v = next(); if (v === undefined) throw new Error(`${name} needs a value`); flags[name] = v; }; + const list = (into: string[], name: string) => (next: () => string | undefined) => { const v = next(); if (v === undefined) throw new Error(`${name} needs a value`); into.push(path.resolve(v)); }; + parseCliFlags(argv, { + ...Object.fromEntries(['--k', '--cap-usd', '--command', '--jobs', '--out', '--runs', '--repo'].map(name => [name, value(name)])), + '--history-dir': list(historyDirs, '--history-dir'), '--prior': list(priorDirs, '--prior'), '--dry-run': () => { flags['--dry-run'] = '1'; }, + }, 'Usage: ship-measure sweep [--k K] [--cap-usd USD] [--dry-run] [--history-dir DIR]... [--prior DIR]... [--runs N] [--command \'CMD {case}\'] [--jobs N] [--out DIR]'); + const config = readMeasureConfig(); + const int = (raw: string | undefined, flag: string, fallback: number) => { + if (raw === undefined) return fallback; + if (!/^[1-9][0-9]*$/.test(raw)) throw new Error(`${flag} needs a positive integer. Received: ${raw}`); + return Number(raw); + }; + const capUsd = flags['--cap-usd'] === undefined ? config.sweepBudgetUsd : Number(flags['--cap-usd']); + if (!Number.isFinite(capUsd) || capUsd <= 0) throw new Error(`--cap-usd needs a positive amount in USD. Received: ${flags['--cap-usd']}`); + const repo = flags['--repo'] ?? repoSlug(ROOT); + const history = loadPassRateHistory({ repo, workflow: 'evals-periodic.yml', branch: 'main', runsLimit: int(flags['--runs'], '--runs', 10), dirs: historyDirs, sinceDays: 60 }); + if (history.historyError) { console.error(`[ship-measure] sweep: pass-rate history unavailable (${history.historyError}); nothing measured`); return 1; } + const outDir = path.resolve(flags['--out'] ?? path.join(process.cwd(), '.context', 'ship-measure')); + const report = await runSweep({ + records: history.records, historyScope: history.scope, k: int(flags['--k'], '--k', config.sweepCases), capUsd, config, outDir, priorDirs, + dryRun: flags['--dry-run'] === '1', parallel: int(flags['--jobs'], '--jobs', Number(env.EVALS_JOBS) || 4), + ref: env.GITHUB_REF_NAME ?? gitText(['rev-parse', '--abbrev-ref', 'HEAD']), sha: env.GITHUB_SHA ?? gitText(['rev-parse', 'HEAD']), + ...(flags['--command'] ? { runner: commandRunner(flags['--command']), runnerId: `command:${flags['--command']}`, preflight: () => ({ ok: true, detail: 'documented command' }) } : {}), + }); + if (env.GITHUB_STEP_SUMMARY) fs.appendFileSync(env.GITHUB_STEP_SUMMARY, `${formatSweepReport(report)}\n`); + return 0; +} diff --git a/scripts/ship-measure.ts b/scripts/ship-measure.ts index 581e2b7ca..31be5368d 100644 --- a/scripts/ship-measure.ts +++ b/scripts/ship-measure.ts @@ -1,43 +1,53 @@ #!/usr/bin/env bun /** * ship-measure — /ship's measure-then-fix runner for a red eval case or a red - * free-suite shard. Every trial it runs is DIAGNOSTIC: it never changes a - * recorded verdict, its artifacts are labeled `diagnostic` with `verdict: - * null`, and they live under `.context/ship-measure/`, never in the project - * eval dir that pass-rate history and CI uploads read. The lane verdict still - * comes from the one full gate run /ship makes after the case measures at or - * above target (docs/TESTING_INTERNALS.md#ship-measure). + * free-suite shard, and the weekly off-ship qualification sweep. Every trial + * it runs is DIAGNOSTIC: it never changes a recorded verdict, its artifacts + * are labeled `diagnostic` with `verdict: null`, and they live under + * `.context/ship-measure/`, never in the project eval dir that pass-rate + * history and CI uploads read. The lane verdict still comes from the one full + * gate run /ship makes after the case measures at or above the bar + * (docs/TESTING_INTERNALS.md#ship-measure). * - * Per-kind aggregation (config keys in bin/gstack-config, CEO-9 defaults): - * rule N trials (ship_measure_rule_trials, 10); meets at >= 90% of N. - * behavior P panels of 3 (ship_measure_behavior_panels, 4); every panel - * passes at its own 2 of 3 with no contract violation, and >= 90% - * of all 3P trials pass (11 of 12). - * judge O outputs (ship_measure_judge_outputs, 10), each scored by its - * 3-sample panel (passes at 2 of 3); meets at >= 90% of O. + * Trials per batch (config keys in bin/gstack-config): rule + * ship_measure_rule_trials (10); behavior ship_measure_behavior_panels panels + * of 3 (4, so 12 trials); judge ship_measure_judge_outputs outputs (10), each + * scored by its median 3-sample panel. The decision is the measurement bar in + * scripts/lib/measure-bar.ts: MEETS at 9/10, MEETS-qualified at 8/10 with + * every red in a qualifying class, EXTEND at 7/10 (N more trials on identical + * inputs, decided once on the pooled 2N), else BELOW (a fix round). A batch + * with more than 30% provider-evidence failures is void and is redispatched + * once. * * Spend: ship_measure_budget_usd is an estimated ADMISSION budget per red case - * across the baseline and every repair round. Before each batch the runner - * reserves the estimated cost of every concurrent trial and admits only what - * fits beside what was already spent; actual costs are reconciled after the - * batch. With no per-trial estimate it asks once ("estimate unknown"), then - * runs one calibration trial alone. ship_measure_ask_per_trial_usd asks first - * above that per-trial estimate. ship_measure_max_rounds caps repair rounds. + * across the baseline, extension, redispatch and every repair round. Before + * each batch the runner reserves the estimated cost of every concurrent trial + * and admits only what fits beside what was already spent; actual costs are + * reconciled after the batch. With no per-trial estimate it asks once + * ("estimate unknown"), then runs one calibration trial alone. + * ship_measure_ask_per_trial_usd asks first above that per-trial estimate. + * ship_measure_max_rounds caps repair rounds. * * Usage: * bun run scripts/ship-measure.ts table * bun run scripts/ship-measure.ts measure --case ID --round baseline|round-N * [--kind rule|behavior|judge] [--command 'CMD {case}'] [--cost-per-trial USD] * [--approved] [--fix TEXT] [--jobs N] [--out DIR] + * bun run scripts/ship-measure.ts classify --case ID --round R --trials 2,5 --class C --evidence TEXT [--out DIR] + * bun run scripts/ship-measure.ts decide --case ID --round R [--out DIR] + * bun run scripts/ship-measure.ts extend --case ID --round R [--command ..] [--cost-per-trial USD] [--jobs N] [--out DIR] * bun run scripts/ship-measure.ts skip --case ID --reason TEXT [--out DIR] * bun run scripts/ship-measure.ts report [--out DIR] * bun run scripts/ship-measure.ts free (--files a,b,... | --shard I) [--reruns N] * [--concurrency C] [--backend local|ubicloud] [--wall-cap SECS] [--out DIR] + * bun run scripts/ship-measure.ts sweep [--k K] [--cap-usd USD] [--dry-run] [--history-dir DIR]... + * [--prior DIR] [--command 'CMD {case}'] [--jobs N] [--out DIR] (scripts/ship-measure-sweep.ts) * - * measure exits 0 at or above target, 1 below target, 2 when it needs the - * user's approval (nothing ran), 3 on a named-red stop (budget exhausted or - * round limit), 4 on a usage or setup error. free exits 0 when every - * completed rerun passed, else 1. + * measure, extend, classify and decide exit 0 at MEETS or MEETS-qualified, 1 + * at BELOW (a fix round), 2 when it needs the user's approval (nothing ran), 3 + * on a named-red stop (budget exhausted, round limit, a void batch twice), 4 on + * a usage, setup or refused-classification error, 5 at needs-classify, 6 at + * EXTEND. free exits 0 when every completed rerun passed, else 1. */ import * as fs from 'node:fs'; import * as os from 'node:os'; @@ -45,11 +55,16 @@ import * as path from 'node:path'; import { spawn, spawnSync } from 'node:child_process'; import { EVAL_POLICY } from '../test/helpers/periodic-exclude-data'; import { E2E_KINDS, E2E_TIERS, LLM_JUDGE_TOUCHFILES } from '../test/helpers/touchfiles-data'; -import { getProjectEvalDir, isFinalizedEvalResultFile } from '../test/helpers/eval-store'; +import { getClaudeCliVersion, getProjectEvalDir, isFinalizedEvalResultFile } from '../test/helpers/eval-store'; import { parseCliFlags } from './lib/shard-engine'; import { collectPaidTestFiles, paidSelectionEnv } from './lib/paid-select'; import { DEFAULT_JOBS } from './lib/paid-types'; import { privateFreeHome } from './lib/free-home-guard'; +import { + FAILURE_CLASSES, barThresholds, classificationProblem, decide, identityMismatch, + type BarDecision, type BarNext, type BarThresholds, type Classification, type FailureClass, type SetSummary, type TrialSet, +} from './lib/measure-bar'; +import harnessManifest from './harness-version.json'; import { TREE_MUTATING, assignFilesToShards, collectFreeTestFiles, fullSuiteJobs, loadFreeTestDurations, packShardsByDuration, runFreeShard, wallTimeoutForShard, type FreeShardOutcome, @@ -65,12 +80,16 @@ export const JUDGE_SAMPLES = EVAL_POLICY.judge.samples; export interface MeasureConfig { ruleTrials: number; behaviorPanels: number; judgeOutputs: number; askPerTrialUsd: number; budgetUsd: number; maxRounds: number; rerunBackend: RerunBackend; + sweepCases: number; sweepBudgetUsd: number; } export const MEASURE_DEFAULTS: MeasureConfig = { ruleTrials: 10, behaviorPanels: 4, judgeOutputs: 10, askPerTrialUsd: 2, budgetUsd: 25, maxRounds: 3, rerunBackend: 'local', + sweepCases: 5, sweepBudgetUsd: 150, }; +const USD_FIELDS = new Set(['askPerTrialUsd', 'budgetUsd', 'sweepBudgetUsd']); + function gstackConfigGet(key: string): string | undefined { const r = spawnSync(path.join(ROOT, 'bin', 'gstack-config'), ['get', key], { encoding: 'utf8', timeout: 10_000 }); return r.status === 0 ? r.stdout.trim() : undefined; @@ -86,6 +105,8 @@ export function readMeasureConfig(configGet: (key: string) => string | undefined ['budgetUsd', 'ship_measure_budget_usd', configGet('ship_measure_budget_usd')], ['maxRounds', 'ship_measure_max_rounds', configGet('ship_measure_max_rounds')], ['rerunBackend', 'ship_rerun_backend', configGet('ship_rerun_backend')], + ['sweepCases', 'ship_measure_sweep_cases', configGet('ship_measure_sweep_cases')], + ['sweepBudgetUsd', 'ship_measure_sweep_budget_usd', configGet('ship_measure_sweep_budget_usd')], ]; const config = { ...MEASURE_DEFAULTS }; for (const [field, key, rawValue] of raw) { @@ -96,7 +117,7 @@ export function readMeasureConfig(configGet: (key: string) => string | undefined config.rerunBackend = value; continue; } - const usd = field === 'askPerTrialUsd' || field === 'budgetUsd'; + const usd = USD_FIELDS.has(field); if (!(usd ? Number.isFinite(Number(value)) && Number(value) > 0 : /^[1-9][0-9]*$/.test(value))) { throw new Error(`${key} '${value}' is not a ${usd ? 'positive amount in USD' : 'positive integer'}. Fix: gstack-config set ${key} ${MEASURE_DEFAULTS[field]}`); } @@ -105,51 +126,48 @@ export function readMeasureConfig(configGet: (key: string) => string | undefined return config; } -const ninetyPercent = (n: number) => Math.ceil((n * 9) / 10); - export interface KindPlan { kind: MeasureKind; - /** Scored units: trials (rule), panels (behavior), outputs (judge). */ - units: number; + /** Behavior trials launch in panels of 3; rule and judge trials one at a time. */ trialsPerUnit: number; + /** Trials per batch (outputs for judge). */ trials: number; - /** Units that must pass. */ - unitTarget: number; - /** Trials (outputs for judge) that must pass. */ - trialTarget: number; + bar: BarThresholds; + pooledBar: BarThresholds; } export function kindPlan(kind: MeasureKind, config: MeasureConfig): KindPlan { - if (kind === 'behavior') { - const trials = config.behaviorPanels * PANEL.n; - return { kind, units: config.behaviorPanels, trialsPerUnit: PANEL.n, trials, unitTarget: config.behaviorPanels, trialTarget: ninetyPercent(trials) }; - } - const units = kind === 'judge' ? config.judgeOutputs : config.ruleTrials; - return { kind, units, trialsPerUnit: 1, trials: units, unitTarget: ninetyPercent(units), trialTarget: ninetyPercent(units) }; + const trials = kind === 'behavior' ? config.behaviorPanels * PANEL.n : kind === 'judge' ? config.judgeOutputs : config.ruleTrials; + return { kind, trialsPerUnit: kind === 'behavior' ? PANEL.n : 1, trials, bar: barThresholds(trials), pooledBar: barThresholds(2 * trials) }; } /** The per-kind table /ship prints before it runs anything. */ export function formatKindTable(config: MeasureConfig): string { - const rule = kindPlan('rule', config); - const behavior = kindPlan('behavior', config); - const judge = kindPlan('judge', config); const budget = `$${config.budgetUsd} total; asks above $${config.askPerTrialUsd}/trial or with no estimate`; + const row = (kind: MeasureKind, label: string) => { + const { trials: n, bar, pooledBar } = kindPlan(kind, config); + const unit = kind === 'judge' ? ' outputs' : ''; + return `| ${kind} | ${label} | ${bar.strict} of ${n}${unit} | ${bar.qualified} of ${n}, every red qualifying | ${bar.floor} of ${n} (or ${bar.qualified} that does not qualify): ${n} more on identical inputs, then ${pooledBar.strict} of ${2 * n}, or ${pooledBar.qualified} of ${2 * n} qualified | ${budget} |`; + }; return [ - '| Kind | Trials | Pass bar | Panels | Budget per red case |', - '|---|---|---|---|---|', - `| rule | ${rule.trials} | at least ${rule.trialTarget} of ${rule.trials} trials | none | ${budget} |`, - `| behavior | ${behavior.trials} | every panel at ${PANEL.k} of ${PANEL.n}, no contract violation, at least ${behavior.trialTarget} of ${behavior.trials} trials | ${behavior.units} of ${PANEL.n} trials | ${budget} |`, - `| judge | ${judge.trials} outputs | at least ${judge.unitTarget} of ${judge.trials} outputs | ${JUDGE_SAMPLES} samples per output, passes at 2 of ${JUDGE_SAMPLES} | ${budget} |`, + '| Kind | Trials per batch | MEETS | MEETS-qualified | EXTEND, then the pooled decision | Budget per red case |', + '|---|---|---|---|---|---|', + row('rule', String(config.ruleTrials)), + row('behavior', `${config.behaviorPanels * PANEL.n} (${config.behaviorPanels} panels of ${PANEL.n})`), + row('judge', `${config.judgeOutputs} outputs, each its median ${JUDGE_SAMPLES}-sample panel`), '', + 'Qualifying reds: provider (affirmative provider or transport evidence only), judge noise at the threshold, a model miss citing its evidence.', + 'Never qualifying (a fix round): a timeout, a hang, a regression, a contract violation, any known fixable cause. Below the EXTEND floor: BELOW, a fix round.', + 'A batch with more than 30% of trials failing on provider evidence is void and is redispatched once; both batches are reported. There is never a third batch.', `Repair rounds: at most ${config.maxRounds}. Free-suite rerun backend: ${config.rerunBackend}. Every trial is diagnostic and never changes a recorded verdict.`, ].join('\n'); } -// ─── Trials and aggregation ──────────────────────────────────────────────── +// ─── Trials ──────────────────────────────────────────────────────────────── export interface TrialResult { passed: boolean; - /** A contract violation fails its unit at any count. */ + /** A contract violation fails the measurement at any count. */ contract?: boolean; /** Billed cost; absent means unknown. */ costUsd?: number; @@ -157,38 +175,21 @@ export interface TrialResult { samples?: boolean[]; failureCause?: string; failureDetail?: string; + /** The eval store's one-line failure_cause_evidence. */ + failureEvidence?: string; } export interface TrialRequest { caseId: string; kind: MeasureKind; round: string; trial: number; dir: string } export type TrialRunner = (request: TrialRequest) => Promise; -export interface TrialRecord extends TrialResult { trial: number; unit: number; batch: number; dir: string } - -export interface Aggregate { unitPasses: number; trialPasses: number; complete: boolean; meets: boolean; unitResults: boolean[] } - -function unitPassed(kind: MeasureKind, trials: TrialRecord[]): boolean { - if (trials.some(t => t.contract)) return false; - if (kind === 'behavior') return trials.length === PANEL.n && trials.filter(t => t.passed).length >= PANEL.k; - const [only] = trials; - if (!only) return false; - if (kind === 'judge' && only.samples) return only.samples.length === JUDGE_SAMPLES && only.samples.filter(Boolean).length >= 2; - return only.passed; -} - -export function aggregate(plan: KindPlan, trials: TrialRecord[]): Aggregate { - const unitResults = Array.from({ length: plan.units }, (_, i) => unitPassed(plan.kind, trials.filter(t => t.unit === i + 1))); - const unitPasses = unitResults.filter(Boolean).length; - const trialPasses = plan.kind === 'behavior' ? trials.filter(t => t.passed && !t.contract).length : unitPasses; - const complete = trials.length === plan.trials; - return { unitPasses, trialPasses, complete, unitResults, meets: complete && unitPasses >= plan.unitTarget && trialPasses >= plan.trialTarget }; -} +export interface TrialRecord extends TrialResult { trial: number; set: TrialSet; unit: number; batch: number; dir: string } // ─── One measurement: baseline or a repair round ─────────────────────────── export type MeasureStatus = 'measured' | 'needs_approval' | 'budget_exhausted' | 'round_limit'; export interface Measurement { - schema: 'gstack-ship-measure/1'; + schema: 'gstack-ship-measure/2'; label: 'diagnostic'; verdict: null; case: string; @@ -198,10 +199,17 @@ export interface Measurement { status: MeasureStatus; reason?: string; plan: KindPlan; + /** Inputs every pooled batch must share: tree, runner, CLI, model, image, policy and harness versions. */ + identity: Record; trials: TrialRecord[]; - unitPasses: number; - trialPasses: number; - meets: boolean; + decision: BarDecision; + next: BarNext; + decisionReason: string; + passes: number; + counted: number; + sets: SetSummary[]; + /** The per-trial estimate later batches reserve; null until known. */ + costPerTrialUsd: number | null; /** Reserved before admission; null when a batch ran without an estimate. */ estimatedUsd: number | null; actualUsd: number; @@ -211,6 +219,9 @@ export interface Measurement { interface Ledger { case: string; kind: MeasureKind; approved: boolean; spentUsd: number; rounds: string[] } +/** A spend cap shared across cases (the sweep's weekly cap), checked beside each case's own budget. */ +export interface SpendPool { label: string; capUsd: number; spentUsd: number } + export interface MeasureOptions { caseId: string; kind: MeasureKind; @@ -224,6 +235,9 @@ export interface MeasureOptions { fix?: string; /** Pass-rate history directory diagnostic artifacts must stay out of (default: the project eval dir). */ evalDir?: string; + /** The batch identity (default: measurementIdentity of the working tree). */ + identity?: () => Record; + pool?: SpendPool; log?: (line: string) => void; } @@ -244,6 +258,59 @@ function roundNumber(round: string): number { return Number(match[1]); } +/** The git tree of the working tree as it would be committed now (tracked and unignored files), without touching the real index. */ +export function frozenTree(cwd: string): string { + const git = (args: string[], env: NodeJS.ProcessEnv = process.env) => spawnSync('git', args, { cwd, env, encoding: 'utf8', timeout: 120_000, maxBuffer: 64 * 1024 * 1024 }); + const top = git(['rev-parse', '--show-toplevel']); + if (top.status !== 0) return 'no-git'; + const tmp = fs.mkdtempSync(path.join(os.tmpdir(), 'ship-measure-index-')); + try { + const index = path.resolve(cwd, git(['rev-parse', '--git-path', 'index']).stdout.trim()); + if (fs.existsSync(index)) fs.copyFileSync(index, path.join(tmp, 'index')); + const env = { ...process.env, GIT_INDEX_FILE: path.join(tmp, 'index') }; + const add = git(['-C', top.stdout.trim(), 'add', '-A'], env); + const tree = add.status === 0 ? git(['write-tree'], env) : add; + if (tree.status !== 0) throw new Error(`ship-measure: could not snapshot the working tree: ${tree.stderr.trim()}`); + return tree.stdout.trim(); + } finally { fs.rmSync(tmp, { recursive: true, force: true }); } +} + +/** Everything a pooled batch must share with the batch it extends. */ +export function measurementIdentity(kind: MeasureKind, trials: number, runner: string, cwd = process.cwd(), env: NodeJS.ProcessEnv = process.env): Record { + return { + tree: frozenTree(cwd), kind, trials: String(trials), runner, policy: String(EVAL_POLICY.version), harness: String(harnessManifest.version), + claude_cli: getClaudeCliVersion(), model: env.EVALS_MODEL ?? 'default', image: env.GSTACK_CI_IMAGE ?? 'local', bun: Bun.version, + }; +} + +const roundDirOf = (outDir: string, caseId: string, round: string) => path.join(outDir, caseSlug(caseId), round); +const classificationsPath = (roundDir: string) => path.join(roundDir, 'classifications.jsonl'); + +export function readClassifications(roundDir: string): Classification[] { + if (!fs.existsSync(classificationsPath(roundDir))) return []; + return fs.readFileSync(classificationsPath(roundDir), 'utf8').split('\n').filter(Boolean).map(line => JSON.parse(line) as Classification); +} + +/** Apply the bar to the measurement's trials and its classification records. */ +function withDecision(m: Measurement, roundDir: string): Measurement { + const outcome = decide(m.kind, m.plan.trials, m.trials, readClassifications(roundDir)); + return { ...m, decision: outcome.decision, next: outcome.next, decisionReason: outcome.reason, passes: outcome.passes, counted: outcome.thresholds.trials, sets: outcome.sets }; +} + +function loadMeasurement(outDir: string, caseId: string, round: string): { m: Measurement; roundDir: string } { + const roundDir = roundDirOf(outDir, caseId, round); + const m = readJson(path.join(roundDir, 'measurement.json')); + if (!m) throw new Error(`ship-measure: no measurement at ${roundDir}; run measure --case ${caseId} --round ${round} first`); + return { m, roundDir }; +} + +function saveMeasurement(m: Measurement, roundDir: string, log: (line: string) => void): Measurement { + const decided = withDecision(m, roundDir); + writeJson(path.join(roundDir, 'measurement.json'), decided); + log(formatMeasurementLine(decided)); + return decided; +} + export async function measureCase(o: MeasureOptions): Promise { const log = o.log ?? ((line: string) => console.log(line)); const plan = kindPlan(o.kind, o.config); @@ -252,9 +319,9 @@ export async function measureCase(o: MeasureOptions): Promise { const ledgerPath = path.join(caseDir, 'ledger.json'); const ledger: Ledger = readJson(ledgerPath) ?? { case: o.caseId, kind: o.kind, approved: false, spentUsd: 0, rounds: [] }; const base: Measurement = { - schema: 'gstack-ship-measure/1', label: 'diagnostic', verdict: null, case: o.caseId, kind: o.kind, round: o.round, - ...(o.fix ? { fix: o.fix } : {}), status: 'measured', plan, trials: [], unitPasses: 0, trialPasses: 0, meets: false, - estimatedUsd: 0, actualUsd: 0, costUnknownTrials: 0, parallel: Math.max(1, o.parallel), + schema: 'gstack-ship-measure/2', label: 'diagnostic', verdict: null, case: o.caseId, kind: o.kind, round: o.round, + ...(o.fix ? { fix: o.fix } : {}), status: 'measured', plan, identity: {}, trials: [], decision: 'incomplete', next: 'stop', decisionReason: '', + passes: 0, counted: 0, sets: [], costPerTrialUsd: o.costPerTrialUsd, estimatedUsd: 0, actualUsd: 0, costUnknownTrials: 0, parallel: Math.max(1, o.parallel), }; if (roundNumber(o.round) > o.config.maxRounds) { return { ...base, status: 'round_limit', reason: `repair-round limit ${o.config.maxRounds} reached; stop with the named-red report` }; @@ -269,104 +336,167 @@ export async function measureCase(o: MeasureOptions): Promise { const roundDir = path.join(caseDir, o.round); if (fs.existsSync(roundDir)) throw new Error(`ship-measure: ${roundDir} already exists; measurements are never overwritten. Use the next round.`); fs.mkdirSync(roundDir, { recursive: true }); - const result = await runBatches(o, plan, roundDir, ledger, estimate, log); - const totals = aggregate(plan, result.trials); - const measurement: Measurement = { ...base, ...result, unitPasses: totals.unitPasses, trialPasses: totals.trialPasses, meets: totals.meets }; + let m: Measurement = { ...base, identity: (o.identity ?? (() => measurementIdentity(o.kind, plan.trials, 'gstack')))() }; + m = withDecision(await runSet(o, m, 'initial', roundDir, ledger, log), roundDir); + if (m.status === 'measured' && m.next === 'redispatch') { + log(`[ship-measure] ${o.caseId} ${o.round}: ${m.decisionReason}`); + m = await runSet(o, m, 'redispatch', roundDir, ledger, log); + } ledger.rounds.push(o.round); - writeJson(path.join(roundDir, 'measurement.json'), measurement); writeJson(ledgerPath, ledger); - log(formatMeasurementLine(measurement)); - return measurement; + return saveMeasurement(m, roundDir, log); } -async function runBatches(o: MeasureOptions, plan: KindPlan, roundDir: string, ledger: Ledger, initialEstimate: number | null, - log: (line: string) => void): Promise> { +export interface ExtendOptions extends Omit { costPerTrialUsd?: number | null } + +/** EXTEND: N more trials on identical inputs (the identity must match field for field), decided once on the pooled 2N. */ +export async function extendCase(o: ExtendOptions): Promise { + const log = o.log ?? ((line: string) => console.log(line)); + assertDiagnosticDir(o.outDir, o.evalDir ?? getProjectEvalDir()); + let { m, roundDir } = loadMeasurement(o.outDir, o.caseId, o.round); + m = withDecision(m, roundDir); + if (m.trials.some(t => t.set === 'extend')) throw new Error(`ship-measure: ${o.caseId} ${o.round} was already extended; there is never a third batch`); + if (m.decision !== 'EXTEND') throw new Error(`ship-measure: ${o.caseId} ${o.round} is ${m.decision}, not EXTEND (${m.decisionReason})`); + const now = (o.identity ?? (() => measurementIdentity(m.kind, m.plan.trials, m.identity.runner ?? 'gstack')))(); + const changed = identityMismatch(m.identity, now); + if (changed) throw new Error(`ship-measure: refusing to pool ${o.caseId} ${o.round}: ${changed}. EXTEND needs identical inputs; measure the changed tree as a new round.`); + const ledgerPath = path.join(o.outDir, caseSlug(o.caseId), 'ledger.json'); + const ledger = readJson(ledgerPath) ?? { case: o.caseId, kind: m.kind, approved: true, spentUsd: m.actualUsd, rounds: [o.round] }; + const opts: MeasureOptions = { ...o, kind: m.kind, approved: true, costPerTrialUsd: o.costPerTrialUsd ?? m.costPerTrialUsd }; + m = withDecision(await runSet(opts, { ...m, status: 'measured' }, 'extend', roundDir, ledger, log), roundDir); + if (m.status === 'measured' && m.next === 'redispatch') { + log(`[ship-measure] ${o.caseId} ${o.round}: ${m.decisionReason}`); + m = await runSet(opts, m, 'extend-redispatch', roundDir, ledger, log); + } + writeJson(ledgerPath, ledger); + return saveMeasurement(m, roundDir, log); +} + +/** Append one immutable classification record per trial (a later record supersedes, never edits), then re-decide. */ +export function classifyTrials(outDir: string, caseId: string, round: string, trials: number[], cls: string, evidence: string, log = (line: string) => console.log(line)): Measurement { + const { m, roundDir } = loadMeasurement(outDir, caseId, round); + for (const trial of trials) { + const problem = classificationProblem(m.trials.find(t => t.trial === trial), m.kind, cls, evidence); + if (problem) throw new Error(`ship-measure: classify t${trial} refused: ${problem}`); + } + const at = new Date().toISOString(); + fs.appendFileSync(classificationsPath(roundDir), trials.map(trial => `${JSON.stringify({ trial, class: cls as FailureClass, evidence: evidence.trim(), by: 'agent', at } satisfies Classification)}\n`).join('')); + return saveMeasurement(m, roundDir, log); +} + +export function decideRound(outDir: string, caseId: string, round: string, log = (line: string) => console.log(line)): Measurement { + const { m, roundDir } = loadMeasurement(outDir, caseId, round); + return saveMeasurement(m, roundDir, log); +} + +/** Run one set of N trials (initial, redispatch, extend or extend-redispatch) under the admission budget. */ +async function runSet(o: MeasureOptions, m: Measurement, set: TrialSet, roundDir: string, ledger: Ledger, log: (line: string) => void): Promise { + const plan = m.plan; + const first = m.trials.length; const trials: TrialRecord[] = []; + const initialEstimate = m.costPerTrialUsd; let estimate = initialEstimate; - let estimatedUsd: number | null = 0; - let actualUsd = 0; - let costUnknownTrials = 0; - let knownCosts = 0; + let estimatedUsd = m.estimatedUsd; + let actualUsd = m.actualUsd; + let costUnknownTrials = m.costUnknownTrials; + let knownCosts = m.trials.filter(t => typeof t.costUsd === 'number').length; let batch = 0; const parallel = Math.max(1, o.parallel); + const done = (status: MeasureStatus, reason?: string): Measurement => ({ + ...m, status, ...(reason ? { reason } : {}), trials: [...m.trials, ...trials], estimatedUsd, actualUsd, costUnknownTrials, parallel, + costPerTrialUsd: estimate, + }); while (trials.length < plan.trials) { const remaining = plan.trials - trials.length; - const headroom = o.config.budgetUsd - ledger.spentUsd; + const headroom = Math.min(o.config.budgetUsd - ledger.spentUsd, o.pool ? o.pool.capUsd - o.pool.spentUsd : Infinity); let size = Math.min(parallel, remaining); if (estimate === null) size = knownCosts === 0 && trials.length === 0 ? 1 : size; else size = Math.min(size, Math.floor((headroom + 1e-9) / estimate)); if (size < 1 || headroom <= 0) { - return { trials, status: 'budget_exhausted', estimatedUsd, actualUsd, costUnknownTrials, parallel, - reason: `budget $${o.config.budgetUsd} per red case: spent $${ledger.spentUsd.toFixed(2)}, next trial reserves $${(estimate ?? 0).toFixed(2)}; ${trials.length} of ${plan.trials} trials ran` }; + const cap = o.pool && o.pool.capUsd - o.pool.spentUsd <= o.config.budgetUsd - ledger.spentUsd + ? `${o.pool.label} $${o.pool.capUsd}: spent $${o.pool.spentUsd.toFixed(2)}` + : `budget $${o.config.budgetUsd} per red case: spent $${ledger.spentUsd.toFixed(2)}`; + return done('budget_exhausted', `${cap}, next trial reserves $${(estimate ?? 0).toFixed(2)}; ${trials.length} of ${plan.trials} ${set} trials ran`); } batch += 1; estimatedUsd = estimate === null || estimatedUsd === null ? null : estimatedUsd + size * estimate; - log(`[ship-measure] ${o.caseId} ${o.round}: batch ${batch} admits ${size} trial(s)${estimate === null ? ' (calibration, no estimate)' : `, reserves $${(size * estimate).toFixed(2)}`}; spent $${ledger.spentUsd.toFixed(2)} of $${o.config.budgetUsd}`); - const start = trials.length; + log(`[ship-measure] ${o.caseId} ${o.round} ${set}: batch ${batch} admits ${size} trial(s)${estimate === null ? ' (calibration, no estimate)' : `, reserves $${(size * estimate).toFixed(2)}`}; spent $${ledger.spentUsd.toFixed(2)} of $${o.config.budgetUsd}`); + const start = first + trials.length; const results = await Promise.all(Array.from({ length: size }, async (_, i) => { const trial = start + i + 1; const dir = path.join(roundDir, `t${String(trial).padStart(2, '0')}`); fs.mkdirSync(dir, { recursive: false }); let result: TrialResult; - try { result = await o.runner({ caseId: o.caseId, kind: o.kind, round: o.round, trial, dir }); } + try { result = await o.runner({ caseId: o.caseId, kind: m.kind, round: o.round, trial, dir }); } catch (error) { result = { passed: false, failureCause: 'unknown', failureDetail: `runner error: ${(error as Error).message}` }; } - return { ...result, trial, unit: Math.ceil(trial / plan.trialsPerUnit), batch, dir } satisfies TrialRecord; + return { ...result, trial, set, unit: Math.ceil((trial - first) / plan.trialsPerUnit), batch, dir } satisfies TrialRecord; })); for (const record of results) { writeJson(path.join(record.dir, 'trial.json'), { label: 'diagnostic', verdict: null, ...record }); trials.push(record); - if (typeof record.costUsd === 'number' && Number.isFinite(record.costUsd)) { - actualUsd += record.costUsd; ledger.spentUsd += record.costUsd; knownCosts += 1; - } else { - costUnknownTrials += 1; - if (estimate !== null) ledger.spentUsd += estimate; - } + const charge = typeof record.costUsd === 'number' && Number.isFinite(record.costUsd) ? record.costUsd : estimate; + if (typeof record.costUsd === 'number' && Number.isFinite(record.costUsd)) { actualUsd += record.costUsd; knownCosts += 1; } + else costUnknownTrials += 1; + if (charge !== null) { ledger.spentUsd += charge; if (o.pool) o.pool.spentUsd += charge; } } if (initialEstimate === null && knownCosts > 0) estimate = actualUsd / knownCosts; } - return { trials, status: 'measured', estimatedUsd, actualUsd, costUnknownTrials, parallel }; + return done('measured'); } +const DECISION_WORDS: Record = { + MEETS: 'MEETS', 'MEETS-qualified': 'MEETS-qualified', EXTEND: 'EXTEND', BELOW: 'BELOW (fix round)', + 'needs-classify': 'needs-classify', void: 'void', incomplete: 'incomplete', +}; + export function formatMeasurementLine(m: Measurement): string { - const observed = m.kind === 'behavior' - ? `${m.trialPasses}/${m.plan.trials} trials, ${m.unitPasses}/${m.plan.units} panels` - : `${m.unitPasses}/${m.plan.units} ${m.kind === 'judge' ? 'outputs' : 'trials'}`; const when = m.fix ? ` after fix at ${m.fix}` : m.round === 'baseline' ? ' (baseline)' : ''; - const verdict = m.status === 'measured' ? (m.meets ? 'at or above target' : 'below target') : m.status.replace('_', ' '); - return `[ship-measure] DIAGNOSTIC ${m.case} ${m.round}: observed ${observed}${when}; ${verdict}; est ${usd(m.estimatedUsd)}, actual ${usd(m.actualUsd)}${m.costUnknownTrials ? ` (+${m.costUnknownTrials} trial(s) cost unknown)` : ''}`; + const verdict = m.status === 'measured' ? DECISION_WORDS[m.decision] : m.status.replace('_', ' '); + const sets = m.sets.length > 1 ? `; batches ${m.sets.map(s => `${s.set} ${s.passes}/${s.trials}${s.void ? ' void' : ''}`).join(', ')}` : ''; + return `[ship-measure] DIAGNOSTIC ${m.case} ${m.round}: observed ${m.passes}/${m.counted} ${m.kind === 'judge' ? 'outputs' : 'trials'}${when}; ${verdict}${m.status === 'measured' ? ` — ${m.decisionReason}` : m.reason ? ` — ${m.reason}` : ''}${sets}; est ${usd(m.estimatedUsd)}, actual ${usd(m.actualUsd)}${m.costUnknownTrials ? ` (+${m.costUnknownTrials} trial(s) cost unknown)` : ''}`; } const usd = (value: number | null) => value === null ? 'unknown' : `$${value.toFixed(2)}`; +/** measure/extend/classify/decide exit code for a measurement (see the header). */ +export function measurementExit(m: Measurement): number { + if (m.status === 'needs_approval') return 2; + if (m.status !== 'measured') return 3; + return ({ MEETS: 0, 'MEETS-qualified': 0, BELOW: 1, 'needs-classify': 5, EXTEND: 6, void: 3, incomplete: 3 } as const)[m.decision]; +} + // ─── Unmeasured skip and the PR-body report ──────────────────────────────── export function recordUnmeasured(outDir: string, caseId: string, reason: string): string { if (!reason.trim()) throw new Error('skip needs --reason: why this red is infrastructure and is not measured'); const file = path.join(outDir, caseSlug(caseId), 'unmeasured.json'); - writeJson(file, { schema: 'gstack-ship-measure/1', label: 'diagnostic', verdict: null, case: caseId, status: 'unmeasured', reason: reason.trim() }); + writeJson(file, { schema: 'gstack-ship-measure/2', label: 'diagnostic', verdict: null, case: caseId, status: 'unmeasured', reason: reason.trim() }); return file; } /** The per-case table for the PR body: every measurement, estimated and actual spend, never a pass for an unmeasured case. */ export function formatReport(outDir: string): string { - const rows = ['| Case | Kind | Measurement | Observed | Target | Est. spend | Actual spend | Status |', '|---|---|---|---|---|---|---|---|']; + const rows = ['| Case | Kind | Measurement | Observed | Bar | Decision | Est. spend | Actual spend |', '|---|---|---|---|---|---|---|---|']; const cases = fs.existsSync(outDir) ? fs.readdirSync(outDir).filter(name => fs.statSync(path.join(outDir, name)).isDirectory()).sort() : []; + const qualified: string[] = []; for (const name of cases) { const dir = path.join(outDir, name); const skip = readJson<{ case: string; reason: string }>(path.join(dir, 'unmeasured.json')); - if (skip) { rows.push(`| ${skip.case} | — | — | — | — | — | — | unmeasured (${skip.reason.replace(/\|/g, '/')}); not a pass |`); continue; } + if (skip) { rows.push(`| ${skip.case} | — | — | — | — | unmeasured (${skip.reason.replace(/\|/g, '/')}); not a pass | — | — |`); continue; } const ledger = readJson(path.join(dir, 'ledger.json')); for (const round of ledger?.rounds ?? []) { const m = readJson(path.join(dir, round, 'measurement.json')); if (!m) continue; - const observed = m.kind === 'behavior' ? `${m.trialPasses}/${m.trials.length} trials, ${m.unitPasses}/${m.plan.units} panels` : `${m.unitPasses}/${m.trials.length}`; - const target = m.kind === 'behavior' ? `all ${m.plan.units} panels, ${m.plan.trialTarget}/${m.plan.trials}` : `${m.plan.unitTarget}/${m.plan.units}`; + const bar = m.counted > m.plan.trials ? m.plan.pooledBar : m.plan.bar; const what = m.fix ? `${m.round}: after fix at ${m.fix.replace(/\|/g, '/')}` : m.round; - const status = m.status === 'measured' ? (m.meets ? 'at or above target' : 'below target') : m.status.replace('_', ' '); - rows.push(`| ${m.case} | ${m.kind} | ${what} | observed ${observed} | ${target} | ${usd(m.estimatedUsd)} | ${usd(m.actualUsd)}${m.costUnknownTrials ? ` (${m.costUnknownTrials} unknown)` : ''} | ${status} |`); + const batches = m.sets.length > 1 ? ` (${m.sets.map(s => `${s.set} ${s.passes}/${s.trials}${s.void ? ' void' : ''}`).join(', ')})` : ''; + const decision = m.status === 'measured' ? DECISION_WORDS[m.decision] : m.status.replace('_', ' '); + rows.push(`| ${m.case} | ${m.kind} | ${what} | observed ${m.passes}/${m.counted}${batches} | ${bar.strict}/${bar.trials} strict, ${bar.qualified}/${bar.trials} qualified | ${decision} | ${usd(m.estimatedUsd)} | ${usd(m.actualUsd)}${m.costUnknownTrials ? ` (${m.costUnknownTrials} unknown)` : ''} |`); + if (m.status === 'measured' && m.decision === 'MEETS-qualified') qualified.push(`- ${m.case} ${m.round}: ${m.decisionReason.replace(/\|/g, '/')}`); } } - return [...rows, '', 'Diagnostic measurements: they never change a recorded verdict; the lane verdict is the full gate run.'].join('\n'); + return [...rows, ...(qualified.length ? ['', 'Qualified MEETS (every red classified into a qualifying class):', ...qualified] : []), + '', 'Diagnostic measurements: they never change a recorded verdict; the lane verdict is the full gate run.'].join('\n'); } // ─── Trial runners ───────────────────────────────────────────────────────── @@ -454,6 +584,7 @@ function parseTrialOutput(output: string): Partial { return { ...(field('failure_cause') ? { failureCause: field('failure_cause') } : {}), ...(field('failure_detail') ? { failureDetail: field('failure_detail') } : {}), + ...(field('failure_evidence') ? { failureEvidence: field('failure_evidence') } : {}), ...(cost !== undefined && Number.isFinite(Number(cost)) ? { costUsd: Number(cost) } : {}), ...(samples ? { samples: samples.split(/[,\s]+/).filter(Boolean).map(s => s === '1' || s === 'pass' || s === 'true') } : {}), ...(/^\s*contract_violation\b/m.test(output) ? { contract: true } : {}), @@ -481,6 +612,7 @@ export function commandRunner(template: string, cwd = process.cwd()): TrialRunne ...(parsed.costUsd === undefined && costOf(records) !== undefined ? { costUsd: costOf(records) } : {}), ...(code !== 0 && !parsed.failureCause && failed?.failure_cause ? { failureCause: String(failed.failure_cause) } : {}), ...(code !== 0 && !parsed.failureDetail && failed?.failure_detail ? { failureDetail: JSON.stringify(failed.failure_detail).slice(0, 300) } : {}), + ...(code !== 0 && !parsed.failureEvidence && failed?.failure_cause_evidence ? { failureEvidence: String(failed.failure_cause_evidence).slice(0, 300) } : {}), }; }; } @@ -526,6 +658,7 @@ export function gstackRunner(rootDir = ROOT): TrialRunner { ...(own.some(r => r.failure_class === 'contract') ? { contract: true } : {}), ...(passed ? {} : { failureCause: failed?.failure_cause ? String(failed.failure_cause) : own.length ? 'assertion' : 'unknown', + ...(failed?.failure_cause_evidence ? { failureEvidence: String(failed.failure_cause_evidence).slice(0, 300) } : {}), failureDetail: failed?.failure_detail ? JSON.stringify(failed.failure_detail).slice(0, 300) : own.length ? String(failed?.error ?? '').slice(0, 300) : `no trial record; exit ${code}; ${output.trim().split('\n').slice(-1)[0] ?? ''}`.slice(0, 300), }), @@ -627,10 +760,11 @@ function positive(raw: string | undefined, flag: string, integer = true): number export async function main(argv: string[], env: NodeJS.ProcessEnv = process.env): Promise { const [command, ...rest] = argv; + if (command === 'sweep') return (await import('./ship-measure-sweep')).sweepMain(rest, env); const flags: Record = {}; const value = (name: string) => (next: () => string | undefined) => { const v = next(); if (v === undefined) throw new Error(`${name} needs a value`); flags[name] = v; }; const handlers = Object.fromEntries(['--case', '--round', '--kind', '--command', '--cost-per-trial', '--fix', '--jobs', '--out', '--reason', - '--files', '--shard', '--reruns', '--concurrency', '--backend', '--wall-cap'].map(name => [name, value(name)])); + '--files', '--shard', '--reruns', '--concurrency', '--backend', '--wall-cap', '--trials', '--class', '--evidence'].map(name => [name, value(name)])); parseCliFlags(rest, { ...handlers, '--approved': () => { flags['--approved'] = '1'; } }, 'Usage: see the header of scripts/ship-measure.ts'); const config = readMeasureConfig(); const outDir = path.resolve(flags['--out'] ?? path.join(process.cwd(), '.context', 'ship-measure')); @@ -638,29 +772,43 @@ export async function main(argv: string[], env: NodeJS.ProcessEnv = process.env) if (command === 'report') { console.log(formatReport(outDir)); return 0; } if (command === 'skip') { console.log(`[ship-measure] ${flags['--case']} labeled unmeasured: ${recordUnmeasured(outDir, flags['--case'] ?? '', flags['--reason'] ?? '')}`); return 0; } if (command === 'free') return runFreeCli(flags, config, outDir, env); - if (command !== 'measure') throw new Error(`Unknown command: ${command ?? '(none)'}. Use table, measure, skip, report or free.`); + if (command !== 'measure' && command !== 'classify' && command !== 'decide' && command !== 'extend') { + throw new Error(`Unknown command: ${command ?? '(none)'}. Use table, measure, classify, decide, extend, skip, report, free or sweep.`); + } const caseId = flags['--case']; - if (!caseId) throw new Error('measure needs --case'); - const kind = (flags['--kind'] ?? E2E_KINDS[caseId] ?? 'rule') as MeasureKind; - if (!['rule', 'behavior', 'judge'].includes(kind)) throw new Error(`--kind must be rule, behavior or judge. Received: ${kind}`); + if (!caseId) throw new Error(`${command} needs --case`); + const round = flags['--round'] ?? 'baseline'; + if (command === 'classify') { + const trials = (flags['--trials'] ?? '').split(',').map(t => t.trim().replace(/^t/, '')).filter(Boolean).map(t => positive(t, '--trials')); + if (!trials.length || !flags['--class']) throw new Error(`classify needs --trials 2,5 and --class (${FAILURE_CLASSES.join(', ')})`); + return measurementExit(classifyTrials(outDir, caseId, round, trials, flags['--class'], flags['--evidence'] ?? '')); + } + if (command === 'decide') return measurementExit(decideRound(outDir, caseId, round)); const isGstack = path.resolve(process.cwd()) === ROOT; if (!flags['--command'] && !isGstack) throw new Error('No single-case eval command: pass --command \' {case}\' (ask the user once and record the answer).'); + const runner = flags['--command'] ? commandRunner(flags['--command']) : gstackRunner(); + const runnerId = flags['--command'] ? `command:${flags['--command']}` : 'gstack'; + const parallel = flags['--jobs'] ? positive(flags['--jobs'], '--jobs') : Number(env.EVALS_JOBS) || DEFAULT_JOBS; + const costPerTrialUsd = flags['--cost-per-trial'] !== undefined ? positive(flags['--cost-per-trial'], '--cost-per-trial', false) : null; + if (command === 'extend') { + return measurementExit(await extendCase({ caseId, round, config, outDir, runner, parallel, ...(costPerTrialUsd !== null ? { costPerTrialUsd } : {}) })); + } + const kind = (flags['--kind'] ?? E2E_KINDS[caseId] ?? 'rule') as MeasureKind; + if (!['rule', 'behavior', 'judge'].includes(kind)) throw new Error(`--kind must be rule, behavior or judge. Received: ${kind}`); console.log(formatKindTable(config)); if (!flags['--command']) { const selected = caseSelectionPreflight(caseId); if (!selected.ok) throw new Error(`--case ${caseId} does not select its test (no paid call made): ${selected.detail}`); console.log(`[ship-measure] preflight: ${selected.detail}`); } + const plan = kindPlan(kind, config); const m = await measureCase({ - caseId, kind, round: flags['--round'] ?? 'baseline', config, outDir, fix: flags['--fix'], - runner: flags['--command'] ? commandRunner(flags['--command']) : gstackRunner(), - parallel: flags['--jobs'] ? positive(flags['--jobs'], '--jobs') : Number(env.EVALS_JOBS) || DEFAULT_JOBS, - costPerTrialUsd: flags['--cost-per-trial'] !== undefined ? positive(flags['--cost-per-trial'], '--cost-per-trial', false) : null, - approved: flags['--approved'] === '1', + caseId, kind, round, config, outDir, fix: flags['--fix'], runner, parallel, costPerTrialUsd, approved: flags['--approved'] === '1', + identity: () => measurementIdentity(kind, plan.trials, runnerId), }); - if (m.status === 'needs_approval') { console.log(`[ship-measure] NEEDS APPROVAL ${caseId}: ${m.reason}`); return 2; } - if (m.status !== 'measured') { console.log(`[ship-measure] NAMED RED ${caseId}: ${m.reason}`); return 3; } - return m.meets ? 0 : 1; + if (m.status === 'needs_approval') console.log(`[ship-measure] NEEDS APPROVAL ${caseId}: ${m.reason}`); + else if (m.status !== 'measured' || m.next === 'stop') console.log(`[ship-measure] NAMED RED ${caseId}: ${m.reason ?? m.decisionReason}`); + return measurementExit(m); } async function runFreeCli(flags: Record, config: MeasureConfig, outDir: string, env: NodeJS.ProcessEnv): Promise { diff --git a/test/fixtures/module-size-ratchet.json b/test/fixtures/module-size-ratchet.json index d2f883539..5d7228b59 100644 --- a/test/fixtures/module-size-ratchet.json +++ b/test/fixtures/module-size-ratchet.json @@ -2,6 +2,8 @@ "newModules": [ { "file": "scripts/lib/shard-engine.ts", "movedFrom": "scripts/test-strict-output.ts" }, { "file": "scripts/lib/eval-history.ts", "movedFrom": "(new: headroom, red ledger and run triage, kept out of scripts/eval-flake-rank.ts)" }, + { "file": "scripts/lib/measure-bar.ts", "movedFrom": "(new: the ship-measure measurement bar, kept out of scripts/ship-measure.ts)" }, + { "file": "scripts/ship-measure-sweep.ts", "movedFrom": "(new: the weekly off-ship qualification sweep, kept out of scripts/ship-measure.ts)" }, { "file": "scripts/lib/published-text.ts", "movedFrom": "(new: one sanitizer for model-written text reaching public reports)" }, { "file": "test/helpers/session-ledger.ts", "movedFrom": "(new: per-session ledger every paid runner appends to)" }, { "file": "scripts/lib/paid-types.ts", "movedFrom": "scripts/test-paid-shards.ts" }, diff --git a/test/helpers/touchfiles-data.ts b/test/helpers/touchfiles-data.ts index 92708be65..ccaae5e66 100644 --- a/test/helpers/touchfiles-data.ts +++ b/test/helpers/touchfiles-data.ts @@ -389,7 +389,7 @@ export const E2E_TOUCHFILES: Record = { 'tpa-apple-ban': [ 'scripts/resolvers/third-party-actions.ts', 'ship/SKILL.md.tmpl', 'ship/sections/apple-release.md.tmpl', 'scripts/gen-skill-docs.ts', 'test/helpers/session-runner.ts', 'test/skill-e2e-third-party-actions.test.ts', 'test/helpers/third-party-actions.ts', 'lib/eval-model.ts' ], - 'ship-measure-seeded-flake': ['ship/**', 'scripts/ship-measure.ts', 'scripts/test-free-shards.ts', 'scripts/lib/shard-engine.ts', 'scripts/lib/paid-select.ts', 'scripts/lib/free-home-guard.ts', 'bin/gstack-config', 'bin/gstack-state-root.sh', 'lib/state-root.ts', 'scripts/resolvers/sections.ts', 'scripts/resolvers/testing.ts', 'scripts/gen-skill-docs.ts', 'test/skill-e2e-ship-measure-loop.test.ts', 'test/helpers/ship-measure-seeded-fixture.ts', 'test/helpers/agent-sdk-runner.ts', 'test/helpers/skill-fixture.ts'], + 'ship-measure-seeded-flake': ['ship/**', 'scripts/ship-measure.ts', 'scripts/lib/measure-bar.ts', 'scripts/test-free-shards.ts', 'scripts/lib/shard-engine.ts', 'scripts/lib/paid-select.ts', 'scripts/lib/free-home-guard.ts', 'bin/gstack-config', 'bin/gstack-state-root.sh', 'lib/state-root.ts', 'scripts/resolvers/sections.ts', 'scripts/resolvers/testing.ts', 'scripts/gen-skill-docs.ts', 'test/skill-e2e-ship-measure-loop.test.ts', 'test/helpers/ship-measure-seeded-fixture.ts', 'test/helpers/agent-sdk-runner.ts', 'test/helpers/skill-fixture.ts'], 'ship-section-loading': [ 'ship/**', 'scripts/resolvers/sections.ts', 'scripts/gen-skill-docs.ts', 'test/helpers/auq-sdk-capture.ts', 'test/helpers/session-runner.ts', 'test/skill-e2e-ship-section-loading.test.ts', 'scripts/resolvers/testing.ts', 'test/helpers/agent-sdk-runner.ts', 'test/helpers/auq-native-capture.ts', 'test/helpers/auto-decision-state.ts', 'test/helpers/autoplan-artifact-digest.ts', 'test/helpers/autoplan-artifact-permission.ts', 'test/helpers/autoplan-artifact-recorder.ts', 'test/helpers/claude-pty-runner.ts', 'test/helpers/pty/**', 'test/helpers/dx-selected-navigation.ts', 'test/helpers/e2e-gate.ts', 'test/helpers/eng-cache-writer-decision.ts', 'test/helpers/hermetic-skill-runtime.ts', 'test/helpers/llm-judge.ts', 'test/helpers/native-auto-decide.ts', 'test/helpers/owned-claude-transcript.ts', 'test/helpers/plan-count-artifacts.ts', 'test/helpers/plan-count-file-permission.ts', 'test/helpers/plan-count-fixture.ts', 'test/helpers/plan-count-pending-exit.ts', 'test/helpers/plan-count-pending-question.ts', 'test/helpers/plan-count-transcript.ts', 'test/helpers/plan-floor-review.ts', 'test/helpers/plan-floor-target.ts', 'test/helpers/plan-scope-selection.ts', 'test/helpers/plan-seed-submission.ts', 'test/helpers/plan-skill-question-events.ts', 'test/helpers/plan-skill-question-hook-scope.ts', 'test/helpers/plan-skill-questions.ts', 'test/helpers/pty-screen.ts', 'test/helpers/pty-trust-dialog.ts', 'test/helpers/skill-census.ts'], 'plan-ceo-section-loading': [ diff --git a/test/ship-measure-bar.test.ts b/test/ship-measure-bar.test.ts new file mode 100644 index 000000000..c268c054f --- /dev/null +++ b/test/ship-measure-bar.test.ts @@ -0,0 +1,238 @@ +/** + * The measurement bar's decision table (scripts/lib/measure-bar.ts), free and + * deterministic, plus the runner paths that apply it (void redispatch, EXTEND + * on identical inputs, classification records) with fake trial runners. + * Rules (approved 2026-10-06): MEETS >= 9/10; MEETS-qualified 8/10 with every + * red qualifying (provider on affirmative evidence, judge noise, a cited model + * miss); EXTEND 7/10 or an unqualified 8/10, then one pooled decision on 20 + * (18 strict, 16 qualified); BELOW otherwise. Behavior uses 12 (11, 10, 9; + * pooled 22 and 20 of 24); a judge trial is one output scored by its median + * 3-sample panel. More than 30% provider-evidence failures voids a batch, which + * may be redispatched once. + */ +import { afterAll, describe, expect, test } from 'bun:test'; +import * as fs from 'node:fs'; +import * as os from 'node:os'; +import * as path from 'node:path'; +import { + barThresholds, classificationProblem, decide, identityMismatch, machineClass, providerEvidence, + type BarKind, type BarTrial, type Classification, type FailureClass, type TrialSet, +} from '../scripts/lib/measure-bar'; +import { MEASURE_DEFAULTS, classifyTrials, decideRound, extendCase, measureCase, type TrialRequest, type TrialResult } from '../scripts/ship-measure'; + +const temps: string[] = []; +const tmp = (prefix: string) => { const dir = fs.mkdtempSync(path.join(os.tmpdir(), prefix)); temps.push(dir); return dir; }; +afterAll(() => { for (const dir of temps) fs.rmSync(dir, { recursive: true, force: true }); }); + +const ASSERTION = { failureCause: 'assertion', failureDetail: 'expected sorted' }; +const API = { failureCause: 'api_error', failureEvidence: 'API Error: 529 overloaded' }; +const TIMEOUT = { failureCause: 'session_timeout', failureEvidence: 'armed 300s timeout fired' }; + +/** A batch of n trials in one set; `fails` maps a 1-based slot to that trial's failure fields. */ +function batch(set: TrialSet, n: number, fails: Record> = {}, start = 0): BarTrial[] { + return Array.from({ length: n }, (_, i) => { + const failure = fails[i + 1]; + return { trial: start + i + 1, set, passed: !failure, ...(failure ?? {}) }; + }); +} +const agent = (trial: number, cls: FailureClass, evidence = 'test/x.test.ts:12 read the capture'): Classification => ({ trial, class: cls, evidence, by: 'agent', at: '2026-10-06T00:00:00Z' }); +const fails = (slots: number[], fields: Partial = ASSERTION) => Object.fromEntries(slots.map(s => [s, fields])); + +describe('thresholds round up from 9/10, 8/10 and 7/10', () => { + test('N = 10, 12, 20 and 24', () => { + expect(barThresholds(10)).toEqual({ trials: 10, strict: 9, qualified: 8, floor: 7 }); + expect(barThresholds(12)).toEqual({ trials: 12, strict: 11, qualified: 10, floor: 9 }); + expect(barThresholds(20)).toMatchObject({ strict: 18, qualified: 16 }); + expect(barThresholds(24)).toMatchObject({ strict: 22, qualified: 20 }); + }); +}); + +describe('one batch of 10 (rule)', () => { + const d = (trials: BarTrial[], records: Classification[] = [], kind: BarKind = 'rule') => decide(kind, kind === 'behavior' ? 12 : 10, trials, records); + + test('9/10 and 10/10 MEETS with no classification', () => { + expect(d(batch('initial', 10, fails([4])))).toMatchObject({ decision: 'MEETS', next: 'none', passes: 9 }); + expect(d(batch('initial', 10))).toMatchObject({ decision: 'MEETS', passes: 10 }); + }); + + test('a contract violation is never MEETS, even at 10/10 assertions passed', () => { + const trials = batch('initial', 10).map(t => t.trial === 3 ? { ...t, contract: true } : t); + expect(d(trials)).toMatchObject({ decision: 'BELOW', next: 'fix' }); + }); + + test('8/10 with only qualifying reds is MEETS-qualified: machine provider, judge noise, a cited model miss', () => { + expect(d(batch('initial', 10, fails([2, 5], API)))).toMatchObject({ decision: 'MEETS-qualified', needsClassify: [] }); + const trials = batch('initial', 10, fails([2, 5])); + expect(d(trials, [agent(2, 'judge-noise', 'median 3 at threshold 4'), agent(5, 'model-miss', 'transcript.jsonl:88 skipped the sort step')]).decision).toBe('MEETS-qualified'); + }); + + test('a sub-strict batch with unreviewed reds is needs-classify, listing exactly those trials', () => { + const out = d(batch('initial', 10, fails([2, 5])), [agent(2, 'judge-noise', 'at threshold')]); + expect(out).toMatchObject({ decision: 'needs-classify', next: 'classify', needsClassify: [5] }); + expect(d(batch('initial', 10, fails([1, 2, 3]))).decision).toBe('needs-classify'); + }); + + test('8/10 with a timeout, a hang, a regression or a known fixable cause is a fix round', () => { + expect(d(batch('initial', 10, { 2: TIMEOUT, 5: API })).decision).toBe('BELOW'); + expect(d(batch('initial', 10, { 2: { failureCause: 'provider_stall' }, 5: API })).decision).toBe('BELOW'); + for (const cls of ['regression', 'fixable', 'timeout', 'hang'] as const) { + expect(d(batch('initial', 10, fails([2, 5])), [agent(2, cls), agent(5, 'judge-noise')])).toMatchObject({ decision: 'BELOW', next: 'fix' }); + } + }); + + test('8/10 with a red reviewed as unclassified, or a model miss without a citation, is EXTEND', () => { + expect(d(batch('initial', 10, fails([2, 5])), [agent(2, 'unclassified', 'reviewed: no evidence for any route'), agent(5, 'judge-noise')])) + .toMatchObject({ decision: 'EXTEND', next: 'extend' }); + expect(d(batch('initial', 10, fails([2, 5])), [agent(2, 'model-miss', 'the model went off script'), agent(5, 'judge-noise')]).decision).toBe('EXTEND'); + }); + + test('7/10 EXTEND when every red qualifies or is reviewed; 7/10 with a fixable red is a fix round; 6/10 BELOW without classification', () => { + expect(d(batch('initial', 10, fails([1, 2, 3], API))).decision).toBe('EXTEND'); + expect(d(batch('initial', 10, fails([1, 2, 3])), [agent(1, 'judge-noise'), agent(2, 'judge-noise'), agent(3, 'fixable')]).decision).toBe('BELOW'); + expect(d(batch('initial', 10, fails([1, 2, 3, 4])))).toMatchObject({ decision: 'BELOW', needsClassify: [] }); + }); + + test('a later classification record supersedes an earlier one for the same trial', () => { + const trials = batch('initial', 10, fails([2, 5])); + expect(d(trials, [agent(2, 'judge-noise'), agent(5, 'judge-noise'), agent(5, 'fixable')]).decision).toBe('BELOW'); + expect(d(trials, [agent(2, 'judge-noise'), agent(5, 'fixable'), agent(5, 'judge-noise')]).decision).toBe('MEETS-qualified'); + }); +}); + +describe('behavior (12) and judge (outputs as median panels)', () => { + test('behavior: 11/12 MEETS, 10/12 qualified, 9/12 EXTEND, 8/12 BELOW', () => { + expect(decide('behavior', 12, batch('initial', 12, fails([7]))).decision).toBe('MEETS'); + expect(decide('behavior', 12, batch('initial', 12, fails([1, 2], API))).decision).toBe('MEETS-qualified'); + expect(decide('behavior', 12, batch('initial', 12, fails([1, 2, 3], API))).decision).toBe('EXTEND'); + expect(decide('behavior', 12, batch('initial', 12, fails([1, 2, 3, 4]))).decision).toBe('BELOW'); + }); + + test('behavior pooled over 24: 22 MEETS, 20 qualified, 19 BELOW', () => { + const base = batch('initial', 12, fails([1, 2, 3], API)); + expect(decide('behavior', 12, [...base, ...batch('extend', 12, {}, 12)]).decision).toBe('MEETS-qualified'); + expect(decide('behavior', 12, [...batch('initial', 12, fails([1])), ...batch('extend', 12, fails([5]), 12)]).decision).toBe('MEETS'); + expect(decide('behavior', 12, [...base, ...batch('extend', 12, fails([1, 2]), 12)], [agent(16, 'unclassified'), agent(17, 'unclassified')]).decision).toBe('BELOW'); + }); + + test('judge: each trial is one output, passing at 2 of its 3 samples whatever the runner claimed', () => { + const outputs = batch('initial', 10).map(t => ({ ...t, samples: t.trial === 2 ? [true, false, false] : [true, false, true] })); + expect(decide('judge', 10, outputs)).toMatchObject({ decision: 'MEETS', passes: 9 }); + const short = batch('initial', 10).map(t => ({ ...t, samples: [true, true] })); + expect(decide('judge', 10, short).passes).toBe(0); + }); +}); + +describe('the pooled decision after EXTEND (once, never a third batch)', () => { + const base = batch('initial', 10, fails([1, 2, 3], API)); + test('pooled 18/20 strict and 16/20 qualified MEET; 17/20 with an unqualified red is BELOW, never EXTEND', () => { + expect(decide('rule', 10, [...batch('initial', 10, fails([1, 2])), ...batch('extend', 10, {}, 10)], [agent(1, 'unclassified'), agent(2, 'unclassified')]).decision).toBe('MEETS'); + expect(decide('rule', 10, [...base, ...batch('extend', 10, { 1: API }, 10)])).toMatchObject({ decision: 'MEETS-qualified', pooled: true, passes: 16 }); + expect(decide('rule', 10, [...base, ...batch('extend', 10, { 1: API, 2: API, 3: API, 4: ASSERTION }, 10)]).decision).toBe('BELOW'); + const unqualified = decide('rule', 10, [...batch('initial', 10, fails([1, 2])), ...batch('extend', 10, { 1: ASSERTION }, 10)], [agent(1, 'unclassified'), agent(2, 'judge-noise'), agent(11, 'unclassified')]); + expect(unqualified).toMatchObject({ decision: 'BELOW', pooled: true, passes: 17 }); + }); + + test('an extension red nobody classified is needs-classify; one with a fixable cause cancels the extension into a fix round', () => { + expect(decide('rule', 10, [...base, ...batch('extend', 10, fails([1]), 10)])).toMatchObject({ decision: 'needs-classify', needsClassify: [11] }); + expect(decide('rule', 10, [...base, ...batch('extend', 10, fails([1]), 10)], [agent(11, 'fixable')]).decision).toBe('BELOW'); + }); +}); + +describe('provider evidence is mechanical; void batches', () => { + test('provider needs affirmative evidence: api_error, or pre_turn_infra naming a 429/5xx or reset; a refusal or crash never qualifies', () => { + expect(providerEvidence({ trial: 1, set: 'initial', passed: false, ...API })).toContain('529'); + expect(providerEvidence({ trial: 1, set: 'initial', passed: false, failureCause: 'pre_turn_infra', failureEvidence: 'HTTP 429 rate limit' })).toContain('429'); + expect(providerEvidence({ trial: 1, set: 'initial', passed: false, failureCause: 'pre_turn_infra', failureEvidence: 'ECONNRESET while streaming' })).not.toBeNull(); + expect(providerEvidence({ trial: 1, set: 'initial', passed: false, failureCause: 'pre_turn_infra', failureEvidence: 'Cannot find module ./entry' })).toBeNull(); + expect(providerEvidence({ trial: 1, set: 'initial', passed: false, failureCause: 'refusal', failureEvidence: 'stop_reason refusal' })).toBeNull(); + expect(machineClass({ trial: 1, set: 'initial', passed: false, failureCause: 'refusal' })).toBeNull(); + expect(machineClass({ trial: 1, set: 'initial', passed: false, ...TIMEOUT })).toBe('timeout'); + }); + + test('provider cannot be set by hand when failure_cause disagrees; a timeout never takes a qualifying class; evidence is required', () => { + const assertion: BarTrial = { trial: 3, set: 'initial', passed: false, ...ASSERTION }; + expect(classificationProblem(assertion, 'rule', 'provider', 'I think the API flaked')).toContain('affirmative provider or transport evidence'); + expect(classificationProblem({ trial: 3, set: 'initial', passed: false, failureCause: 'pre_turn_infra', failureEvidence: 'module load failed' }, 'rule', 'provider', 'x')).toContain('pre_turn_infra'); + expect(classificationProblem({ trial: 3, set: 'initial', passed: false, ...TIMEOUT }, 'rule', 'model-miss', 'a.ts:1')).toContain('never qualifies'); + expect(classificationProblem(assertion, 'rule', 'fixable', ' ')).toContain('--evidence'); + expect(classificationProblem(assertion, 'rule', 'flaky', 'x')).toContain('class must be one of'); + expect(classificationProblem({ trial: 3, set: 'initial', passed: true }, 'rule', 'fixable', 'x')).toContain('passed'); + expect(classificationProblem({ trial: 3, set: 'initial', passed: false, ...API }, 'rule', 'provider', 'api 529')).toBeNull(); + }); + + test('more than 30% provider-evidence failures voids the batch: one redispatch, then decided on it; void twice stops', () => { + expect(decide('rule', 10, batch('initial', 10, fails([1, 2, 3, 4], API)))).toMatchObject({ decision: 'void', next: 'redispatch' }); + const redispatched = decide('rule', 10, [...batch('initial', 10, fails([1, 2, 3, 4], API)), ...batch('redispatch', 10, fails([5]), 10)]); + expect(redispatched).toMatchObject({ decision: 'MEETS', passes: 9 }); + expect(redispatched.sets.map(s => [s.set, s.void])).toEqual([['initial', true], ['redispatch', false]]); + expect(decide('rule', 10, [...batch('initial', 10, fails([1, 2, 3, 4], API)), ...batch('redispatch', 10, fails([1, 2, 3, 4], API), 10)])) + .toMatchObject({ decision: 'void', next: 'stop' }); + expect(decide('rule', 10, batch('initial', 10, fails([1, 2, 3], API))).decision).not.toBe('void'); + }); + + test('module-load failures (pre_turn_infra without provider evidence) never void a batch', () => { + const crash = { failureCause: 'pre_turn_infra', failureEvidence: 'SyntaxError: Export named x not found' }; + expect(decide('rule', 10, batch('initial', 10, fails([1, 2, 3, 4, 5, 6], crash)))).toMatchObject({ decision: 'BELOW' }); + }); + + test('a void extension gets its own one redispatch', () => { + const base = batch('initial', 10, fails([1, 2, 3], API)); + expect(decide('rule', 10, [...base, ...batch('extend', 10, fails([1, 2, 3, 4], API), 10)])).toMatchObject({ decision: 'void', next: 'redispatch', pooled: true }); + expect(decide('rule', 10, [...base, ...batch('extend', 10, fails([1, 2, 3, 4], API), 10), ...batch('extend-redispatch', 10, {}, 20)]).decision).toBe('MEETS-qualified'); + }); +}); + +describe('pooling identity', () => { + test('a changed field is named, including the CI image digest', () => { + const a = { tree: 't1', image: 'ghcr.io/x/ci@sha256:aaa', model: 'm' }; + expect(identityMismatch(a, { ...a })).toBeNull(); + expect(identityMismatch(a, { ...a, image: 'ghcr.io/x/ci@sha256:bbb' })).toContain('image changed'); + expect(identityMismatch(a, { ...a, tree: 't2' })).toContain('tree changed'); + }); +}); + +describe('the runner applies the bar: redispatch, EXTEND, classify', () => { + /** A fake runner; outcomes are keyed by trial number across every set. */ + const runner = (outcome: (trial: number) => TrialResult) => { + const calls: TrialRequest[] = []; + return { calls, run: async (r: TrialRequest) => { calls.push(r); return outcome(r.trial); } }; + }; + const opts = (outDir: string, run: (r: TrialRequest) => Promise, identity = { tree: 't1', image: 'img-a' }) => ({ + caseId: 'bar-case', round: 'baseline', config: MEASURE_DEFAULTS, runner: run, outDir, parallel: 5, costPerTrialUsd: 0.1, + evalDir: tmp('bar-history-'), log: () => {}, identity: () => identity, + }); + + test('a void batch is redispatched once by measure, and both batches are reported', async () => { + const outDir = tmp('bar-void-'); + const r = runner(trial => trial <= 4 ? { passed: false, costUsd: 0.1, ...API } : { passed: true, costUsd: 0.1 }); + const m = await measureCase({ ...opts(outDir, r.run), kind: 'rule', approved: true }); + expect(r.calls).toHaveLength(20); + expect(m).toMatchObject({ decision: 'MEETS', passes: 10, counted: 10 }); + expect(m.sets.map(s => `${s.set}:${s.passes}/${s.trials}${s.void ? ':void' : ''}`)).toEqual(['initial:6/10:void', 'redispatch:10/10']); + }); + + test('classify records reds, refuses a hand-set provider, and EXTEND pools once on identical inputs', async () => { + const outDir = tmp('bar-extend-'); + const r = runner(trial => [2, 5, 9].includes(trial) ? { passed: false, costUsd: 0.1, ...ASSERTION } : { passed: true, costUsd: 0.1 }); + const m = await measureCase({ ...opts(outDir, r.run), kind: 'rule', approved: true }); + expect(m.decision).toBe('needs-classify'); + expect(() => classifyTrials(outDir, 'bar-case', 'baseline', [2], 'provider', 'looked like an outage', () => {})).toThrow('affirmative provider'); + expect(classifyTrials(outDir, 'bar-case', 'baseline', [2, 5, 9], 'unclassified', 'reviewed captures t02 t05 t09: no route', () => {}).decision).toBe('EXTEND'); + const records = fs.readFileSync(path.join(outDir, 'bar-case', 'baseline', 'classifications.jsonl'), 'utf8').trim().split('\n'); + expect(records).toHaveLength(3); + await expect(extendCase({ ...opts(outDir, r.run, { tree: 't1', image: 'img-b' }) })).rejects.toThrow('image changed'); + const pooled = await extendCase({ ...opts(outDir, r.run) }); + expect(r.calls.map(c => c.trial).slice(10)).toEqual([11, 12, 13, 14, 15, 16, 17, 18, 19, 20]); + // 7/10 + 10/10 = 17/20: under the pooled strict bar, and its reds are not in a qualifying class. + expect(pooled).toMatchObject({ decision: 'BELOW', counted: 20, passes: 17 }); + expect(decideRound(outDir, 'bar-case', 'baseline', () => {}).decision).toBe('BELOW'); + await expect(extendCase({ ...opts(outDir, r.run) })).rejects.toThrow('never a third batch'); + }); + + test('EXTEND is refused unless the round decided EXTEND', async () => { + const outDir = tmp('bar-noextend-'); + const r = runner(() => ({ passed: true, costUsd: 0.1 })); + await measureCase({ ...opts(outDir, r.run), kind: 'rule', approved: true }); + await expect(extendCase({ ...opts(outDir, r.run) })).rejects.toThrow('is MEETS, not EXTEND'); + }); +}); diff --git a/test/ship-measure-seeded-fixture.test.ts b/test/ship-measure-seeded-fixture.test.ts index 47afd99ef..ef0faf10f 100644 --- a/test/ship-measure-seeded-fixture.test.ts +++ b/test/ship-measure-seeded-fixture.test.ts @@ -1,7 +1,8 @@ /** * Free checks of the seeded-flake fixture the paid case drives (CEO-3, * ENG-13), measured through the real scripts/ship-measure.ts CLI: unsorted it - * measures 7 of 10 with failures that name the line, parallel trials claim + * measures 7 of 10 with failures that name the line, which the bar sends to + * needs-classify and, classified fixable, to a fix round; parallel trials claim * unique slots that reset per round, an unrelated edit still measures 7 of 10 * and leaves the gate red, and only the sorting fix measures 10 of 10 and turns * the gate green. @@ -31,8 +32,8 @@ describe('seeded-flake fixture (free)', () => { expect(gate().status).toBe(1); const baseline = measure('baseline'); - expect(baseline.stdout).toContain(`DIAGNOSTIC ${SEEDED_CASE} baseline: observed 7/10 trials`); - expect(baseline.status).toBe(1); + expect(baseline.stdout).toContain(`DIAGNOSTIC ${SEEDED_CASE} baseline: observed 7/10 trials (baseline); needs-classify`); + expect(baseline.status).toBe(5); const failed = fs.readdirSync(roundDir('baseline')).filter(t => /^t\d+$/.test(t)) .map(t => JSON.parse(fs.readFileSync(path.join(roundDir('baseline'), t, 'trial.json'), 'utf8'))).filter(t => !t.passed); expect(failed).toHaveLength(3); @@ -43,17 +44,22 @@ describe('seeded-flake fixture (free)', () => { const slotsOf = (dir: string) => readFixtureRuns(log).filter(r => r.kind === 'case' && r.measureDir?.startsWith(dir)).map(r => r.slot).sort((a, b) => a! - b!); expect(slotsOf(roundDir('baseline'))).toEqual([1, 2, 3, 4, 5, 6, 7, 8, 9, 10]); expect(readFixtureRuns(log).filter(r => r.kind === 'case' && r.result === 'fail').map(r => r.slot).sort()).toEqual([2, 5, 9]); + const classify = spawnSync(process.execPath, ['run', path.join(ROOT, 'scripts/ship-measure.ts'), 'classify', '--case', SEEDED_CASE, '--round', 'baseline', + '--trials', failed.map(t => t.trial).join(','), '--class', 'fixable', '--evidence', `src/priorities.js:${SEEDED_LINE} returns storage order`], { cwd: repo, env, encoding: 'utf8', timeout: 60_000 }); + expect(classify.stdout).toContain('BELOW (fix round)'); + expect(classify.status).toBe(1); fs.writeFileSync(path.join(repo, 'src/priorities.js'), UNRELATED_EDIT); const control = measure('round-1'); expect(control.stdout).toContain(`DIAGNOSTIC ${SEEDED_CASE} round-1: observed 7/10 trials`); - expect(control.status).toBe(1); + expect(control.status).toBe(5); expect(slotsOf(roundDir('round-1'))).toEqual([1, 2, 3, 4, 5, 6, 7, 8, 9, 10]); expect(gate().status).toBe(1); fs.writeFileSync(path.join(repo, 'src/priorities.js'), SORTED_FIX); const fixed = measure('round-2'); expect(fixed.stdout).toContain(`DIAGNOSTIC ${SEEDED_CASE} round-2: observed 10/10 trials`); + expect(fixed.stdout).toContain('; MEETS'); expect(fixed.status).toBe(0); expect(gate().status).toBe(0); const runs = readFixtureRuns(log); diff --git a/test/ship-measure-sweep.test.ts b/test/ship-measure-sweep.test.ts new file mode 100644 index 000000000..d4f963130 --- /dev/null +++ b/test/ship-measure-sweep.test.ts @@ -0,0 +1,198 @@ +/** + * Free contracts for the weekly off-ship qualification sweep + * (scripts/ship-measure-sweep.ts): which gate cases it picks from pass-rate + * history, the predicted all-green probability before and after, the weekly + * spend cap enforced through the admission budget across earlier sweep + * reports, the report it writes after every case, and a $0 CLI run (dry run, + * and a fake single-case command). Nothing here makes a paid call. + */ +import { afterAll, describe, expect, test } from 'bun:test'; +import * as fs from 'node:fs'; +import * as os from 'node:os'; +import * as path from 'node:path'; +import { spawnSync } from 'node:child_process'; +import { EVAL_POLICY } from '../test/helpers/periodic-exclude-data'; +import { TRIAL_OUTCOME_SCHEMA, type TrialOutcomeRecord } from '../test/helpers/eval-store'; +import { MEASURE_DEFAULTS, type TrialRequest, type TrialResult } from '../scripts/ship-measure'; +import { SWEEP_SCHEMA, measuredRedRate, planSweep, predictedAllGreen, runSweep, weeklySpent, type SweepReport } from '../scripts/ship-measure-sweep'; + +const ROOT = path.resolve(import.meta.dir, '..'); +const temps: string[] = []; +const tmp = (prefix: string) => { const dir = fs.mkdtempSync(path.join(os.tmpdir(), prefix)); temps.push(dir); return dir; }; +afterAll(() => { for (const dir of temps) fs.rmSync(dir, { recursive: true, force: true }); }); + +const A = 'ship-skipped-queued-finding'; +const B = 'investigate-owned-completion'; +const C = 'investigate-owned-abort'; +const Q = 'investigate-owned-ending-error'; + +function record(id: string, run: number, failed: boolean, extra: Partial = {}): TrialOutcomeRecord { + return { + schema: TRIAL_OUTCOME_SCHEMA, case: id, file: `test/${id}.test.ts`, tier: 'gate', kind: 'rule', trial: 1, panel: { n: 1, k: 1 }, attempt: 1, + outcome: failed ? 'failed' : 'passed', ...(failed ? { failure_class: 'assertion' as const } : {}), duration_ms: 1000, cost_usd: 1, + policy_version: EVAL_POLICY.version, quarantined: false, execution: 'executed', source: 'shard', run_id: String(run), lane: 'gate', + series_identity: 's1', model: 'm', cli_version: 'c', recorded_at: `2026-09-${10 + run}T00:00:00Z`, ...extra, + } as TrialOutcomeRecord; +} + +/** Three weekly runs: A red twice, B red once, C never, a periodic case red (not gate), a retired id red. */ +function history(): TrialOutcomeRecord[] { + const out: TrialOutcomeRecord[] = []; + for (const run of [1, 2, 3]) { + out.push(record(A, run, run !== 3), record(B, run, run === 2), record(C, run, false)); + out.push(record('retired-case-id', run, true), record('some-periodic-case', run, true, { tier: 'periodic', lane: 'periodic' })); + } + return out; +} + +describe('picking cases from gate history', () => { + test('ranks live, non-quarantined gate cases with a red by their cost to the all-green probability, and picks K', () => { + const withQuarantined = [...history(), ...[1, 2, 3].map(run => record(Q, run, true))]; + expect(planSweep(withQuarantined, 3, { quarantine: {} }).picked.map(c => c.case)).toEqual([Q, A, B]); + const plan = planSweep(withQuarantined, 2, { quarantine: { [Q]: {} } }); + expect(plan.candidates.map(c => c.case)).toEqual([A, B]); + expect(plan.picked.map(c => c.case)).toEqual([A, B]); + const [a, b] = plan.candidates; + expect(a).toMatchObject({ reds: 2, verdicts: 3, scoredTrials: 3, costPerTrialUsd: 1, kind: 'rule' }); + expect(a!.passRate).toBeCloseTo(1 / 3, 5); + expect(a!.redRate).toBeGreaterThan(b!.redRate); + expect(a!.contribution).toBeGreaterThan(b!.contribution); + expect(plan.censusCases).toEqual([A, B, C, Q, 'retired-case-id'].sort()); + const product = plan.censusCases.reduce((p, id) => p * (1 - plan.rates[id]!), 1); + expect(plan.allGreen).toBeCloseTo(product, 10); + expect(planSweep(history(), 1).picked.map(c => c.case)).toEqual([A]); + }); + + test('a measured case replaces its history rate; a behavior rate converts to the 3-trial panel red risk', () => { + expect(measuredRedRate('rule', 0, 10, 0)).toBe(0); + expect(measuredRedRate('rule', 1, 10, 0.05)).toBeCloseTo((1 + 4 * 0.05) / 14, 10); + const q = 2 / 16; + expect(measuredRedRate('behavior', 2, 12, 0)).toBeCloseTo(1 - ((1 - q) ** 3 + 3 * (1 - q) ** 2 * q), 10); + const plan = planSweep(history(), 2); + const after = predictedAllGreen(plan, { [A]: 0 })!; + expect(after).toBeCloseTo(plan.allGreen! / (1 - plan.rates[A]!), 10); + }); +}); + +describe('the weekly cap', () => { + const report = (dir: string, id: string, startedAt: string, chargedUsd: number, dryRun = false) => { + fs.mkdirSync(path.join(dir, id), { recursive: true }); + fs.writeFileSync(path.join(dir, id, 'sweep-report.json'), JSON.stringify({ schema: SWEEP_SCHEMA, sweepId: id, startedAt, chargedUsd, dryRun })); + }; + test('sums the last 7 days of sweep reports once each, skipping dry runs, older reports and this sweep', () => { + const dir = tmp('sweep-prior-'); + const now = Date.parse('2026-10-12T12:00:00Z'); + report(dir, 'a', '2026-10-10T00:00:00Z', 40); + report(path.join(dir, 'copy'), 'a', '2026-10-10T00:00:00Z', 40); + report(dir, 'b', '2026-10-06T00:00:00Z', 25); + report(dir, 'old', '2026-10-01T00:00:00Z', 100); + report(dir, 'dry', '2026-10-11T00:00:00Z', 0, true); + report(dir, 'self', '2026-10-12T11:00:00Z', 9); + expect(weeklySpent([dir], now, 'self')).toBe(65); + expect(weeklySpent([path.join(dir, 'missing')], now)).toBe(0); + }); +}); + +describe('running the sweep with a fake runner', () => { + const fake = (fail: (r: TrialRequest) => boolean) => { + const calls: TrialRequest[] = []; + const runner = async (r: TrialRequest): Promise => { calls.push(r); return fail(r) ? { passed: false, costUsd: 1, failureCause: 'assertion' } : { passed: true, costUsd: 1 }; }; + return { calls, runner }; + }; + const base = (outDir: string, runner: (r: TrialRequest) => Promise, extra: Partial[0]> = {}) => runSweep({ + records: history(), historyScope: 'fixture history', k: 2, capUsd: 150, config: MEASURE_DEFAULTS, outDir, priorDirs: [], dryRun: false, parallel: 5, + ref: 'main', sha: 'a'.repeat(40), runner, runnerId: 'fake', preflight: () => ({ ok: true, detail: 'fake' }), identity: () => ({ tree: 'fixed' }), + evalDir: tmp('sweep-history-'), log: () => {}, now: Date.parse('2026-10-12T12:00:00Z'), ...extra, + }); + + test('measures each picked case at its kind\'s N with the bar, writes the report, and predicts the all-green probability after', async () => { + const outDir = tmp('sweep-out-'); + const run = fake(r => r.caseId === A && r.trial <= 4); + const report = await base(outDir, run.runner); + expect(run.calls.filter(c => c.caseId === A)).toHaveLength(10); + expect(run.calls.filter(c => c.caseId === B)).toHaveLength(10); + expect(report.rows.map(r => [r.case, r.status, r.decision, `${r.measuredPasses}/${r.measuredTrials}`])).toEqual([[A, 'measured', 'BELOW', '6/10'], [B, 'measured', 'MEETS', '10/10']]); + expect(report).toMatchObject({ label: 'diagnostic', verdict: null, chargedUsd: 20, priorWeekUsd: 0, k: 2 }); + expect(report.allGreenAfter).not.toBe(report.allGreenBefore); + const dir = path.join(outDir, `sweep-${report.sweepId}`); + const saved = JSON.parse(fs.readFileSync(path.join(dir, 'sweep-report.json'), 'utf8')) as SweepReport; + expect(saved.rows[0]!.captures).toBe(path.join(`sweep-${report.sweepId}`, A, 'baseline')); + expect(fs.existsSync(path.join(outDir, saved.rows[0]!.captures!, 't01', 'trial.json'))).toBe(true); + const md = fs.readFileSync(path.join(dir, 'sweep-report.md'), 'utf8'); + expect(md).toContain('| # | Case | Kind | History rate | Measured | Decision | Cost (est / actual) | Captures |'); + expect(md).toMatch(/Predicted gate all-green probability .*before \d+(\.\d)?% → after \d+(\.\d)?%/); + expect(md).toContain(`| 1 | ${A} | rule | 33.3% of 3 trials; 2/3 red verdicts`); + expect(md).toContain('never pushes or opens a pull request'); + }); + + test('stops at the weekly cap: a partly funded case stops at the budget, the rest are skipped and listed', async () => { + const outDir = tmp('sweep-cap-'); + const prior = tmp('sweep-cap-prior-'); + fs.writeFileSync(path.join(prior, 'sweep-report.json'), JSON.stringify({ schema: SWEEP_SCHEMA, sweepId: 'earlier', startedAt: '2026-10-08T00:00:00Z', chargedUsd: 136, dryRun: false })); + const run = fake(() => false); + const report = await base(outDir, run.runner, { priorDirs: [prior], capUsd: 150, k: 3, records: history().map(r => r.case === C && r.run_id === '1' ? record(C, 1, true) : r) }); + expect(report.priorWeekUsd).toBe(136); + // B and C tie (one red in three); ties fall to the case id, so C ranks before B. + expect(report.rows.map(r => [r.case, r.status])).toEqual([[A, 'measured'], [C, 'budget_exhausted'], [B, 'skipped']]); + expect(run.calls).toHaveLength(14); + expect(report.chargedUsd).toBe(14); + expect(report.rows[2]!.note).toContain('weekly cap $150 exhausted'); + expect(fs.readFileSync(path.join(outDir, `sweep-${report.sweepId}`, 'sweep-report.md'), 'utf8')).toContain(`- ${B}: weekly cap $150 exhausted`); + }); + + test('a dry run plans and estimates without running anything or charging the cap', async () => { + const outDir = tmp('sweep-dry-'); + const run = fake(() => false); + const report = await base(outDir, run.runner, { dryRun: true }); + expect(run.calls).toHaveLength(0); + expect(report.rows.map(r => [r.case, r.status, r.estimatedUsd])).toEqual([[A, 'planned', 10], [B, 'planned', 10]]); + expect(report.chargedUsd).toBe(0); + expect(weeklySpent([outDir], Date.parse('2026-10-12T13:00:00Z'))).toBe(0); + }); + + test('a case whose selection preflight fails is skipped before any paid call', async () => { + const run = fake(() => false); + const report = await base(tmp('sweep-preflight-'), run.runner, { k: 1, preflight: () => ({ ok: false, detail: '--list planned nothing' }) }); + expect(report.rows[0]).toMatchObject({ status: 'skipped', note: 'selection preflight failed (no paid call made): --list planned nothing' }); + expect(run.calls).toHaveLength(0); + }); +}); + +describe('ship-measure sweep CLI ($0)', () => { + const historyDir = () => { + const dir = tmp('sweep-cli-history-'); + fs.writeFileSync(path.join(dir, 'trial-outcomes.jsonl'), `${history().map(r => JSON.stringify(r)).join('\n')}\n`); + return dir; + }; + const cli = (cwd: string, ...args: string[]) => spawnSync(process.execPath, ['run', path.join(ROOT, 'scripts/ship-measure.ts'), 'sweep', ...args], { + cwd, encoding: 'utf8', timeout: 120_000, env: { ...process.env, GSTACK_STATE_ROOT: tmp('sweep-cli-state-'), GITHUB_STEP_SUMMARY: '' }, + }); + + test('--dry-run reads local history, prints the plan and the before probability, and spends nothing', () => { + const cwd = tmp('sweep-cli-'); + const r = cli(cwd, '--history-dir', historyDir(), '--k', '2', '--dry-run', '--out', path.join(cwd, 'out')); + expect(r.status).toBe(0); + expect(r.stdout).toContain('(dry run: nothing ran, $0)'); + expect(r.stdout).toContain(`| 1 | ${A} | rule |`); + expect(r.stdout).toContain('would measure 10 trials'); + }); + + test('a fake single-case command runs the whole sweep end to end at $0', () => { + const cwd = tmp('sweep-cli-fake-'); + const script = path.join(cwd, 'fake-trial.sh'); + fs.writeFileSync(script, '#!/usr/bin/env bash\necho "cost_usd: 0"\nif [ "$GSTACK_SHIP_MEASURE_TRIAL" = 3 ]; then echo "failure_cause: assertion"; exit 1; fi\necho "PASS $1"\n', { mode: 0o755 }); + const r = cli(cwd, '--history-dir', historyDir(), '--k', '1', '--cap-usd', '1', '--command', `${script} {case}`, '--jobs', '5', '--out', path.join(cwd, 'out')); + expect(r.status).toBe(0); + expect(r.stdout).toContain(`DIAGNOSTIC ${A} baseline: observed 9/10 trials (baseline); MEETS`); + expect(r.stdout).toMatch(new RegExp(`\\| 1 \\| ${A} \\| rule \\| [^|]+ \\| 9/10 \\| MEETS \\|`)); + }); + + test('history that cannot be read fails closed with nothing measured', () => { + const cwd = tmp('sweep-cli-nohistory-'); + const r = spawnSync(process.execPath, ['run', path.join(ROOT, 'scripts/ship-measure.ts'), 'sweep', '--dry-run', '--repo', 'garrytan/no-such-repo-xyz', '--out', path.join(cwd, 'out')], { + cwd, encoding: 'utf8', timeout: 120_000, env: { ...process.env, PATH: path.dirname(process.execPath), GSTACK_STATE_ROOT: tmp('sweep-cli-state-') }, + }); + expect(r.status).toBe(1); + expect(r.stderr).toContain('pass-rate history unavailable'); + }); +}); diff --git a/test/ship-measure.test.ts b/test/ship-measure.test.ts index 3c9d25707..e6305b196 100644 --- a/test/ship-measure.test.ts +++ b/test/ship-measure.test.ts @@ -1,7 +1,8 @@ /** * Free contracts for scripts/ship-measure.ts, /ship's measure-then-fix runner: - * per-kind aggregation (rule trials, behavior panels of 3 at their own 2 of 3, - * judge outputs scored by 3-sample panels), unique artifact directories, the + * per-kind batches decided by the measurement bar (rule trials, behavior + * trials in panels of 3, judge outputs scored by 3-sample panels; the pure + * decision table lives in test/ship-measure-bar.test.ts), unique artifact directories, the * diagnostic label, the estimated admission budget, the ask-once rule, the * repair-round limit, the unmeasured skip, the PR table and the free-suite * reruns with the flaky retry off. Every runner here is a fake; nothing paid. @@ -12,7 +13,7 @@ import * as os from 'node:os'; import * as path from 'node:path'; import { spawnSync } from 'node:child_process'; import { - MEASURE_DEFAULTS, aggregate, assertDiagnosticDir, formatKindTable, formatReport, freeParallelism, judgeFile, kindPlan, + MEASURE_DEFAULTS, assertDiagnosticDir, formatKindTable, formatReport, freeParallelism, judgeFile, kindPlan, measureCase, measureFreeShard, readMeasureConfig, recordUnmeasured, type MeasureConfig, type MeasureKind, type TrialRecord, type TrialRequest, type TrialResult, } from '../scripts/ship-measure'; @@ -43,29 +44,32 @@ const measure = (kind: MeasureKind, run: FakeRun, extra: Partial {}, ...extra, + costPerTrialUsd: 0.5, approved: false, evalDir: tmp('ship-measure-history-'), log: () => {}, identity: () => ({ tree: 'fixed' }), ...extra, }); }; describe('per-kind plans and the printed table (CEO-9, DX-7)', () => { - test('defaults: rule 10 trials at 9, behavior 4 panels of 3 at 11 of 12, judge 10 outputs at 9', () => { - expect(kindPlan('rule', MEASURE_DEFAULTS)).toEqual({ kind: 'rule', units: 10, trialsPerUnit: 1, trials: 10, unitTarget: 9, trialTarget: 9 }); - expect(kindPlan('behavior', MEASURE_DEFAULTS)).toEqual({ kind: 'behavior', units: 4, trialsPerUnit: 3, trials: 12, unitTarget: 4, trialTarget: 11 }); - expect(kindPlan('judge', MEASURE_DEFAULTS)).toEqual({ kind: 'judge', units: 10, trialsPerUnit: 1, trials: 10, unitTarget: 9, trialTarget: 9 }); + test('defaults: rule 10, behavior 4 panels of 3 (12), judge 10 outputs; the table prints the bar per kind', () => { + expect(kindPlan('rule', MEASURE_DEFAULTS)).toMatchObject({ trials: 10, trialsPerUnit: 1, bar: { strict: 9, qualified: 8, floor: 7 }, pooledBar: { trials: 20, strict: 18, qualified: 16 } }); + expect(kindPlan('behavior', MEASURE_DEFAULTS)).toMatchObject({ trials: 12, trialsPerUnit: 3, bar: { strict: 11, qualified: 10, floor: 9 }, pooledBar: { trials: 24, strict: 22, qualified: 20 } }); + expect(kindPlan('judge', MEASURE_DEFAULTS)).toMatchObject({ trials: 10, trialsPerUnit: 1, bar: { strict: 9, qualified: 8, floor: 7 } }); const table = formatKindTable(MEASURE_DEFAULTS); - expect(table).toContain('| rule | 10 | at least 9 of 10 trials |'); - expect(table).toContain('every panel at 2 of 3, no contract violation, at least 11 of 12 trials | 4 of 3 trials'); - expect(table).toContain('| judge | 10 outputs | at least 9 of 10 outputs | 3 samples per output, passes at 2 of 3'); + expect(table).toContain('| rule | 10 | 9 of 10 | 8 of 10, every red qualifying | 7 of 10 (or 8 that does not qualify): 10 more on identical inputs, then 18 of 20, or 16 of 20 qualified |'); + expect(table).toContain('| behavior | 12 (4 panels of 3) | 11 of 12 | 10 of 12, every red qualifying | 9 of 12 (or 10 that does not qualify): 12 more on identical inputs, then 22 of 24, or 20 of 24 qualified |'); + expect(table).toContain('| judge | 10 outputs, each its median 3-sample panel | 9 of 10 outputs |'); expect(table).toContain('$25 total; asks above $2/trial'); expect(table).toContain('Repair rounds: at most 3'); expect(table).toContain('never changes a recorded verdict'); + expect(table).toMatch(/more than 30%[^\n]*void[^\n]*redispatched once/); }); test('config keys override defaults; an empty value is the default; an invalid value throws', () => { const values: Record = { ship_measure_rule_trials: '20', ship_measure_budget_usd: '7.5', ship_rerun_backend: 'ubicloud', ship_measure_max_rounds: '' }; const config = readMeasureConfig(key => values[key]); expect(config).toEqual({ ...MEASURE_DEFAULTS, ruleTrials: 20, budgetUsd: 7.5, rerunBackend: 'ubicloud' }); - expect(kindPlan('rule', config).trialTarget).toBe(18); + expect(kindPlan('rule', config).bar.strict).toBe(18); + expect(readMeasureConfig(key => ({ ship_measure_sweep_cases: '2', ship_measure_sweep_budget_usd: '12.5' } as Record)[key])).toMatchObject({ sweepCases: 2, sweepBudgetUsd: 12.5 }); + expect(() => readMeasureConfig(key => key === 'ship_measure_sweep_budget_usd' ? '-1' : undefined)).toThrow('ship_measure_sweep_budget_usd'); expect(() => readMeasureConfig(key => key === 'ship_measure_behavior_panels' ? '0' : undefined)).toThrow('ship_measure_behavior_panels'); expect(() => readMeasureConfig(key => key === 'ship_rerun_backend' ? 'cloud' : undefined)).toThrow('not local or ubicloud'); }); @@ -76,60 +80,56 @@ describe('per-kind plans and the printed table (CEO-9, DX-7)', () => { const config = (...args: string[]) => spawnSync('bash', [path.join(ROOT, 'bin/gstack-config'), ...args], { encoding: 'utf8', timeout: 30_000, env }); const defaults = config('defaults').stdout; for (const [key, value] of [['ship_measure_rule_trials', '10'], ['ship_measure_behavior_panels', '4'], ['ship_measure_judge_outputs', '10'], - ['ship_measure_ask_per_trial_usd', '2'], ['ship_measure_budget_usd', '25'], ['ship_measure_max_rounds', '3'], ['ship_rerun_backend', 'local']]) { + ['ship_measure_ask_per_trial_usd', '2'], ['ship_measure_budget_usd', '25'], ['ship_measure_max_rounds', '3'], ['ship_rerun_backend', 'local'], + ['ship_measure_sweep_cases', '5'], ['ship_measure_sweep_budget_usd', '150']]) { expect(defaults).toMatch(new RegExp(`${key}:\\s+${value}\\n`)); expect(config('get', key!).stdout.trim()).toBe(value!); } expect(config('set', 'ship_measure_budget_usd', '0').status).toBe(1); expect(config('set', 'ship_measure_rule_trials', '2.5').status).toBe(1); expect(config('set', 'ship_rerun_backend', 'cloud').status).toBe(1); + expect(config('set', 'ship_measure_sweep_budget_usd', '0').status).toBe(1); + expect(config('set', 'ship_measure_sweep_cases', 'two').status).toBe(1); expect(config('set', 'ship_measure_budget_usd', '12.5').status).toBe(0); expect(readMeasureConfig(key => config('get', key).stdout.trim()).budgetUsd).toBe(12.5); expect(config('list').stdout).toMatch(/ship_measure_budget_usd:\s+12\.5 \(set/); }); }); -describe('aggregation per kind with fake runners (ENG-12)', () => { - test('rule: 8 of 10 is below target, 9 of 10 meets; every trial gets its own diagnostic directory', async () => { +describe('per-kind batches decided by the measurement bar, with fake runners (ENG-12)', () => { + test('rule: 8 of 10 needs its reds classified, 9 of 10 MEETS; every trial gets its own diagnostic directory', async () => { const red = fake(trial => ({ passed: ![3, 6].includes(trial), costUsd: 0.5, ...(trial === 3 ? { failureCause: 'assertion', failureDetail: 'unsorted list' } : {}) })); const m = await measure('rule', red); - expect(m).toMatchObject({ status: 'measured', label: 'diagnostic', verdict: null, unitPasses: 8, meets: false }); + expect(m).toMatchObject({ status: 'measured', label: 'diagnostic', verdict: null, passes: 8, counted: 10, decision: 'needs-classify', next: 'classify' }); expect(red.calls).toHaveLength(10); const dirs = m.trials.map(t => t.dir); expect(new Set(dirs).size).toBe(10); for (const dir of dirs) { expect(path.basename(path.dirname(dir))).toBe('baseline'); - expect(JSON.parse(fs.readFileSync(path.join(dir, 'trial.json'), 'utf8'))).toMatchObject({ label: 'diagnostic', verdict: null }); + expect(JSON.parse(fs.readFileSync(path.join(dir, 'trial.json'), 'utf8'))).toMatchObject({ label: 'diagnostic', verdict: null, set: 'initial' }); } const saved = JSON.parse(fs.readFileSync(path.join(path.dirname(dirs[0]!), 'measurement.json'), 'utf8')); - expect(saved).toMatchObject({ label: 'diagnostic', verdict: null, round: 'baseline', unitPasses: 8, meets: false }); + expect(saved).toMatchObject({ label: 'diagnostic', verdict: null, round: 'baseline', passes: 8, decision: 'needs-classify', identity: { tree: 'fixed' } }); expect(saved.trials.find((t: TrialRecord) => t.trial === 3)).toMatchObject({ failureCause: 'assertion', failureDetail: 'unsorted list' }); - expect((await measure('rule', fake(trial => ({ passed: trial !== 4, costUsd: 0.5 })))).meets).toBe(true); + expect((await measure('rule', fake(trial => ({ passed: trial !== 4, costUsd: 0.5 })))).decision).toBe('MEETS'); }); - test('behavior: each panel of 3 needs its own 2 of 3, and 11 of 12 trials overall', async () => { - // Trials 1 and 2 fail: 10 of 12 pass, which a single 2-of-12 panel would PASS; panel 1 is 1 of 3. - const panelRed = await measure('behavior', fake(trial => ({ passed: trial > 2 }))); - expect(panelRed).toMatchObject({ unitPasses: 3, trialPasses: 10, meets: false }); - // One failure in each of two panels: every panel passes at 2 of 3, but 10 of 12 is under 11. - const tooMany = await measure('behavior', fake(trial => ({ passed: trial !== 1 && trial !== 4 }))); - expect(tooMany).toMatchObject({ unitPasses: 4, trialPasses: 10, meets: false }); + test('behavior: 12 trials in panels of 3, decided on the trial count (11 MEETS, 10 needs classifying); a contract violation is a fix round', async () => { const oneMiss = await measure('behavior', fake(trial => ({ passed: trial !== 7 }))); - expect(oneMiss).toMatchObject({ unitPasses: 4, trialPasses: 11, meets: true }); + expect(oneMiss).toMatchObject({ passes: 11, counted: 12, decision: 'MEETS' }); expect(oneMiss.trials.map(t => t.unit)).toEqual([1, 1, 1, 2, 2, 2, 3, 3, 3, 4, 4, 4]); - // A contract violation fails its panel even when the trial's assertions passed. + expect((await measure('behavior', fake(trial => ({ passed: trial > 2 })))).decision).toBe('needs-classify'); const contract = await measure('behavior', fake(trial => ({ passed: true, ...(trial === 5 ? { contract: true } : {}) }))); - expect(contract).toMatchObject({ unitPasses: 3, trialPasses: 11, meets: false }); + expect(contract).toMatchObject({ passes: 11, decision: 'BELOW', next: 'fix' }); }); - test('judge: an output passes at 2 of its 3 samples, whatever the runner claimed; 9 of 10 outputs meets', async () => { + test('judge: an output passes at 2 of its 3 samples, whatever the runner claimed; 9 of 10 outputs MEETS', async () => { const judged = await measure('judge', fake(trial => ({ passed: true, samples: trial === 2 ? [true, false, false] : [true, false, true] }))); - expect(judged).toMatchObject({ unitPasses: 9, meets: true }); + expect(judged).toMatchObject({ passes: 9, decision: 'MEETS' }); const red = await measure('judge', fake(trial => ({ passed: true, samples: trial <= 2 ? [false, false, true] : [true, true, true] }))); - expect(red).toMatchObject({ unitPasses: 8, meets: false }); - const wrongCount = aggregate(kindPlan('judge', { ...MEASURE_DEFAULTS, judgeOutputs: 1 }), - [{ trial: 1, unit: 1, batch: 1, dir: '/x', passed: true, samples: [true, true] }]); - expect(wrongCount.meets).toBe(false); + expect(red).toMatchObject({ passes: 8, decision: 'needs-classify' }); + const wrongCount = await measure('judge', fake(() => ({ passed: true, samples: [true, true] }))); + expect(wrongCount).toMatchObject({ passes: 0, decision: 'BELOW' }); }); test('standalone judge ids resolve to the one paid file that names them', () => { @@ -147,7 +147,7 @@ describe('spend, approval, rounds and artifacts', () => { expect(baseline).toMatchObject({ status: 'budget_exhausted', estimatedUsd: 24, actualUsd: 24 }); expect(baseline.trials.map(t => t.batch)).toEqual([1, 1, 1, 1, 2, 2, 2, 2]); expect(run.maxConcurrent).toBeLessThanOrEqual(4); - expect(baseline.reason).toContain('8 of 10 trials ran'); + expect(baseline.reason).toContain('8 of 10 initial trials ran'); // The budget is per red case: a repair round starts from what the baseline spent. const round = await measure('rule', fake(() => ({ passed: true, costUsd: 3 })), { outDir, round: 'round-1', costPerTrialUsd: 3 }); expect(round.status).toBe('budget_exhausted'); @@ -158,7 +158,7 @@ describe('spend, approval, rounds and artifacts', () => { const m = await measure('rule', fake(() => ({ passed: true, costUsd: 1 })), { costPerTrialUsd: 2.5, approved: true, config: { ...MEASURE_DEFAULTS, budgetUsd: 12 } }); // Batch 1 reserves 4 x 2.5 = 10 but spends 4; batch 2 then fits 3 (8 / 2.5); batch 3 fits 2 (5 / 2.5); batch 4 one more. expect(m.trials.map(t => t.batch)).toEqual([1, 1, 1, 1, 2, 2, 2, 3, 3, 4]); - expect(m).toMatchObject({ status: 'measured', actualUsd: 10, estimatedUsd: 25, meets: true }); + expect(m).toMatchObject({ status: 'measured', actualUsd: 10, estimatedUsd: 25, decision: 'MEETS' }); }); test('asks once: no estimate or a per-trial estimate above the threshold runs nothing until approved', async () => { @@ -205,9 +205,9 @@ describe('spend, approval, rounds and artifacts', () => { recordUnmeasured(outDir, 'browse-basic', 'provider outage before the first turn'); expect(() => recordUnmeasured(outDir, 'other', ' ')).toThrow('--reason'); const report = formatReport(outDir); - expect(report).toContain('| seeded-case | rule | baseline | observed 7/10 | 9/10 | $5.00 | $2.50 | below target |'); - expect(report).toContain('| seeded-case | rule | round-1: after fix at sort the shuffled list in src/order.js | observed 10/10 | 9/10 | $5.00 | $2.50 | at or above target |'); - expect(report).toContain('| browse-basic | — | — | — | — | — | — | unmeasured (provider outage before the first turn); not a pass |'); + expect(report).toContain('| seeded-case | rule | baseline | observed 7/10 | 9/10 strict, 8/10 qualified | needs-classify | $5.00 | $2.50 |'); + expect(report).toContain('| seeded-case | rule | round-1: after fix at sort the shuffled list in src/order.js | observed 10/10 | 9/10 strict, 8/10 qualified | MEETS | $5.00 | $2.50 |'); + expect(report).toContain('| browse-basic | — | — | — | — | unmeasured (provider outage before the first turn); not a pass | — | — |'); expect(report).toContain('never change a recorded verdict'); }); }); @@ -226,8 +226,8 @@ if [ "$GSTACK_SHIP_MEASURE_TRIAL" = 2 ]; then echo "failure_cause: assertion"; e }); const out = path.join(project, '.context', 'ship-measure'); const red = cli('measure', '--case', 'flaky-sort', '--command', './evals.sh case {case}', '--cost-per-trial', '0.1', '--jobs', '3'); - expect(red.stdout).toContain('| rule | 10 | at least 9 of 10 trials |'); - expect(red.stdout).toContain('DIAGNOSTIC flaky-sort baseline: observed 9/10 trials (baseline); at or above target'); + expect(red.stdout).toContain('| rule | 10 | 9 of 10 |'); + expect(red.stdout).toContain('DIAGNOSTIC flaky-sort baseline: observed 9/10 trials (baseline); MEETS'); expect(red.status).toBe(0); const trial2 = JSON.parse(fs.readFileSync(path.join(out, 'flaky-sort', 'baseline', 't02', 'trial.json'), 'utf8')); expect(trial2).toMatchObject({ passed: false, costUsd: 0.1, failureCause: 'assertion', failureDetail: 'order.js:3 unsorted', label: 'diagnostic' });