From 7dc6cd2c00e272dd1a3b61d23cb6b88b03778ffe Mon Sep 17 00:00:00 2001 From: garrytan Date: Tue, 6 Oct 2026 20:43:39 +0000 Subject: [PATCH] feat(ship): ship-measure runner for the measure-then-fix loop (A6) scripts/ship-measure.ts owns /ship's diagnostic measurement of a red eval case: per-kind aggregation (rule trials; behavior panels of 3, each at its own 2 of 3 plus 90% of trials; judge outputs scored by 3-sample panels, standalone judge ids accepted), one artifact directory per baseline, repair round and trial, a diagnostic label with verdict null that refuses the eval history dir, an estimated admission budget that reserves every concurrent trial and reconciles actual cost, the ask-once rule, the repair-round limit, the unmeasured skip and the per-case PR table. Free-suite flakes (CEO-1): 'free' reruns one shard's exact file list N times, rerun 1 alone as the baseline and the rest in parallel at max(1, floor(cores / shard concurrency)), each with its own HOME, state root and flake ledger, GSTACK_FREE_RETRY_FLAKY=0, under a wall cap that reports a partial count. test:ubicloud --diagnostic runs the same on one VM with the retry off. bin/gstack-config gains the DX-7 keys with their CEO-9 defaults and rejecting validators. Free tests drive every kind, the budget stop, the unmeasured skip and the free reruns with fake runners. --- bin/gstack-config | 51 ++- scripts/ship-measure.ts | 631 ++++++++++++++++++++++++++++++++++ scripts/ubicloud/test-free.sh | 13 + test/ship-measure.test.ts | 323 +++++++++++++++++ 4 files changed, 1016 insertions(+), 2 deletions(-) create mode 100644 scripts/ship-measure.ts create mode 100644 test/ship-measure.test.ts diff --git a/bin/gstack-config b/bin/gstack-config index 37ea5b071..f1aebac24 100755 --- a/bin/gstack-config +++ b/bin/gstack-config @@ -185,6 +185,22 @@ CONFIG_HEADER='# gstack configuration — edit freely, changes take effect on ne # # or project .env can change it. An invalid value # # is REJECTED and the stored value kept. # +# ─── /ship measure-then-fix loop ───────────────────────────────────── +# A red eval case is measured alone before the full gate reruns +# (scripts/ship-measure.ts; docs/TESTING_INTERNALS.md#ship-measure). +# ship_measure_rule_trials: 10 # rule cases: trials; pass at 90% (9 of 10) +# ship_measure_behavior_panels: 4 # behavior cases: panels of 3; every panel +# # at 2 of 3 and 90% of trials (11 of 12) +# ship_measure_judge_outputs: 10 # judge cases: outputs, each scored by a +# # 3-sample panel; pass at 90% of outputs +# ship_measure_ask_per_trial_usd: 2 # ask first above this estimated cost per trial +# ship_measure_budget_usd: 25 # estimated admission budget per red case +# # (baseline, repair rounds and judge scoring) +# ship_measure_max_rounds: 3 # repair rounds before a named-red stop +# ship_rerun_backend: local # local | ubicloud — where free-suite flake reruns +# # run; ubicloud also needs UBICLOUD_API_KEY. +# # Invalid values are REJECTED (value kept). +# # ─── Workspace-aware ship ──────────────────────────────────────────── # workspace_root: $HOME/conductor/workspaces # Where /ship looks for sibling # # Conductor worktrees when picking a VERSION slot. @@ -248,6 +264,14 @@ lookup_default() { # a synthesized "unknown" default turns the classifier into dead code. # Empty + exit 0 = "no override set, go classify". repo_mode) echo "" ;; + # /ship measure-then-fix loop (scripts/ship-measure.ts reads these). + ship_measure_rule_trials) echo "10" ;; + ship_measure_behavior_panels) echo "4" ;; + ship_measure_judge_outputs) echo "10" ;; + ship_measure_ask_per_trial_usd) echo "2" ;; + ship_measure_budget_usd) echo "25" ;; + ship_measure_max_rounds) echo "3" ;; + ship_rerun_backend) echo "local" ;; # local | ubicloud (Ubicloud spend is opt-in) # Unknown key: exit non-zero instead of printing "". The fallback pattern # the preambles use, # VAR=$(gstack-config get 2>/dev/null || echo "") @@ -648,6 +672,25 @@ case "${1:-}" in echo "Error: codex_reviews '$VALUE' not recognized. Valid values: enabled, disabled. Existing value left unchanged." >&2 exit 1 fi + # The measure-loop controls bound paid spend: reject a typo instead of + # coercing it, so a bad value never changes how much /ship may spend. + case "$KEY" in + ship_measure_rule_trials|ship_measure_behavior_panels|ship_measure_judge_outputs|ship_measure_max_rounds) + if ! printf '%s' "$VALUE" | LC_ALL=C grep -qE '^[1-9][0-9]*$'; then + echo "Error: $KEY '$VALUE' is not a positive integer. Existing value left unchanged." >&2 + exit 1 + fi ;; + ship_measure_ask_per_trial_usd|ship_measure_budget_usd) + if ! printf '%s' "$VALUE" | LC_ALL=C grep -qE '^([0-9]+(\.[0-9]+)?|\.[0-9]+)$' || ! printf '%s' "$VALUE" | LC_ALL=C grep -qE '[1-9]'; then + echo "Error: $KEY '$VALUE' is not a positive amount in USD (e.g. 25 or 2.50). Existing value left unchanged." >&2 + exit 1 + fi ;; + ship_rerun_backend) + if [ "$VALUE" != "local" ] && [ "$VALUE" != "ubicloud" ]; then + echo "Error: ship_rerun_backend '$VALUE' not recognized. Valid values: local, ubicloud. Existing value left unchanged." >&2 + exit 1 + fi ;; + esac # design_detector_install_prompted records "never ask again" for the one-time # engine install offer. Rejecting a typo keeps the offer from silently coming # back (or never coming back) because of a mistyped value. @@ -831,7 +874,9 @@ case "${1:-}" in codex_reviews gstack_contributor skip_eng_review workspace_root \ artifacts_sync_mode artifacts_sync_mode_prompted plan_tune_hooks \ timeline_stop_hook design_detector design_detector_install_prompted memorable_recall \ - browse_extension_id; do + browse_extension_id ship_measure_rule_trials ship_measure_behavior_panels \ + ship_measure_judge_outputs ship_measure_ask_per_trial_usd ship_measure_budget_usd \ + ship_measure_max_rounds ship_rerun_backend; do gstack_config_select "$KEY" VALUE="$_gstack_cfg_value" SOURCE="default" @@ -852,7 +897,9 @@ case "${1:-}" in codex_reviews gstack_contributor skip_eng_review workspace_root \ artifacts_sync_mode artifacts_sync_mode_prompted plan_tune_hooks \ timeline_stop_hook design_detector design_detector_install_prompted memorable_recall \ - browse_extension_id; do + browse_extension_id ship_measure_rule_trials ship_measure_behavior_panels \ + ship_measure_judge_outputs ship_measure_ask_per_trial_usd ship_measure_budget_usd \ + ship_measure_max_rounds ship_rerun_backend; do printf ' %-24s %s\n' "$KEY:" "$(lookup_default "$KEY")" done ;; diff --git a/scripts/ship-measure.ts b/scripts/ship-measure.ts new file mode 100644 index 000000000..1d9ed150a --- /dev/null +++ b/scripts/ship-measure.ts @@ -0,0 +1,631 @@ +#!/usr/bin/env bun +/** + * ship-measure — /ship's measure-then-fix runner for a red eval case or a red + * free-suite shard. Every trial it runs is DIAGNOSTIC: it never changes a + * recorded verdict, its artifacts are labeled `diagnostic` with `verdict: + * null`, and they live under `.context/ship-measure/`, never in the project + * eval dir that pass-rate history and CI uploads read. The lane verdict still + * comes from the one full gate run /ship makes after the case measures at or + * above target (docs/TESTING_INTERNALS.md#ship-measure). + * + * Per-kind aggregation (config keys in bin/gstack-config, CEO-9 defaults): + * rule N trials (ship_measure_rule_trials, 10); meets at >= 90% of N. + * behavior P panels of 3 (ship_measure_behavior_panels, 4); every panel + * passes at its own 2 of 3 with no contract violation, and >= 90% + * of all 3P trials pass (11 of 12). + * judge O outputs (ship_measure_judge_outputs, 10), each scored by its + * 3-sample panel (passes at 2 of 3); meets at >= 90% of O. + * + * Spend: ship_measure_budget_usd is an estimated ADMISSION budget per red case + * across the baseline and every repair round. Before each batch the runner + * reserves the estimated cost of every concurrent trial and admits only what + * fits beside what was already spent; actual costs are reconciled after the + * batch. With no per-trial estimate it asks once ("estimate unknown"), then + * runs one calibration trial alone. ship_measure_ask_per_trial_usd asks first + * above that per-trial estimate. ship_measure_max_rounds caps repair rounds. + * + * Usage: + * bun run scripts/ship-measure.ts table + * bun run scripts/ship-measure.ts measure --case ID --round baseline|round-N + * [--kind rule|behavior|judge] [--command 'CMD {case}'] [--cost-per-trial USD] + * [--approved] [--fix TEXT] [--jobs N] [--out DIR] + * bun run scripts/ship-measure.ts skip --case ID --reason TEXT [--out DIR] + * bun run scripts/ship-measure.ts report [--out DIR] + * bun run scripts/ship-measure.ts free (--files a,b,... | --shard I) [--reruns N] + * [--concurrency C] [--backend local|ubicloud] [--wall-cap SECS] [--out DIR] + * + * measure exits 0 at or above target, 1 below target, 2 when it needs the + * user's approval (nothing ran), 3 on a named-red stop (budget exhausted or + * round limit), 4 on a usage or setup error. free exits 0 when every + * completed rerun passed, else 1. + */ +import * as fs from 'node:fs'; +import * as os from 'node:os'; +import * as path from 'node:path'; +import { spawn, spawnSync } from 'node:child_process'; +import { EVAL_POLICY } from '../test/helpers/periodic-exclude-data'; +import { E2E_KINDS, E2E_TIERS, LLM_JUDGE_TOUCHFILES } from '../test/helpers/touchfiles-data'; +import { getProjectEvalDir, isFinalizedEvalResultFile } from '../test/helpers/eval-store'; +import { parseCliFlags } from './lib/shard-engine'; +import { collectPaidTestFiles, paidSelectionEnv } from './lib/paid-select'; +import { DEFAULT_JOBS } from './lib/paid-types'; +import { privateFreeHome } from './lib/free-home-guard'; +import { + TREE_MUTATING, assignFilesToShards, collectFreeTestFiles, fullSuiteJobs, loadFreeTestDurations, packShardsByDuration, + runFreeShard, wallTimeoutForShard, type FreeShardOutcome, +} from './test-free-shards'; + +const ROOT = path.resolve(import.meta.dir, '..'); + +export type MeasureKind = 'rule' | 'behavior' | 'judge'; +export type RerunBackend = 'local' | 'ubicloud'; +export const PANEL = EVAL_POLICY.panel; +export const JUDGE_SAMPLES = EVAL_POLICY.judge.samples; + +export interface MeasureConfig { + ruleTrials: number; behaviorPanels: number; judgeOutputs: number; + askPerTrialUsd: number; budgetUsd: number; maxRounds: number; rerunBackend: RerunBackend; +} + +export const MEASURE_DEFAULTS: MeasureConfig = { + ruleTrials: 10, behaviorPanels: 4, judgeOutputs: 10, askPerTrialUsd: 2, budgetUsd: 25, maxRounds: 3, rerunBackend: 'local', +}; + +function gstackConfigGet(key: string): string | undefined { + const r = spawnSync(path.join(ROOT, 'bin', 'gstack-config'), ['get', key], { encoding: 'utf8', timeout: 10_000 }); + return r.status === 0 ? r.stdout.trim() : undefined; +} + +/** The measure controls from gstack config; an empty value takes the default, an invalid one throws. */ +export function readMeasureConfig(configGet: (key: string) => string | undefined = gstackConfigGet): MeasureConfig { + const raw: Array<[keyof MeasureConfig, string, string | undefined]> = [ + ['ruleTrials', 'ship_measure_rule_trials', configGet('ship_measure_rule_trials')], + ['behaviorPanels', 'ship_measure_behavior_panels', configGet('ship_measure_behavior_panels')], + ['judgeOutputs', 'ship_measure_judge_outputs', configGet('ship_measure_judge_outputs')], + ['askPerTrialUsd', 'ship_measure_ask_per_trial_usd', configGet('ship_measure_ask_per_trial_usd')], + ['budgetUsd', 'ship_measure_budget_usd', configGet('ship_measure_budget_usd')], + ['maxRounds', 'ship_measure_max_rounds', configGet('ship_measure_max_rounds')], + ['rerunBackend', 'ship_rerun_backend', configGet('ship_rerun_backend')], + ]; + const config = { ...MEASURE_DEFAULTS }; + for (const [field, key, rawValue] of raw) { + const value = rawValue?.trim(); + if (!value) continue; + if (field === 'rerunBackend') { + if (value !== 'local' && value !== 'ubicloud') throw new Error(`${key} '${value}' is not local or ubicloud. Fix: gstack-config set ${key} local`); + config.rerunBackend = value; + continue; + } + const usd = field === 'askPerTrialUsd' || field === 'budgetUsd'; + if (!(usd ? Number.isFinite(Number(value)) && Number(value) > 0 : /^[1-9][0-9]*$/.test(value))) { + throw new Error(`${key} '${value}' is not a ${usd ? 'positive amount in USD' : 'positive integer'}. Fix: gstack-config set ${key} ${MEASURE_DEFAULTS[field]}`); + } + (config as Record)[field] = Number(value); + } + return config; +} + +const ninetyPercent = (n: number) => Math.ceil((n * 9) / 10); + +export interface KindPlan { + kind: MeasureKind; + /** Scored units: trials (rule), panels (behavior), outputs (judge). */ + units: number; + trialsPerUnit: number; + trials: number; + /** Units that must pass. */ + unitTarget: number; + /** Trials (outputs for judge) that must pass. */ + trialTarget: number; +} + +export function kindPlan(kind: MeasureKind, config: MeasureConfig): KindPlan { + if (kind === 'behavior') { + const trials = config.behaviorPanels * PANEL.n; + return { kind, units: config.behaviorPanels, trialsPerUnit: PANEL.n, trials, unitTarget: config.behaviorPanels, trialTarget: ninetyPercent(trials) }; + } + const units = kind === 'judge' ? config.judgeOutputs : config.ruleTrials; + return { kind, units, trialsPerUnit: 1, trials: units, unitTarget: ninetyPercent(units), trialTarget: ninetyPercent(units) }; +} + +/** The per-kind table /ship prints before it runs anything. */ +export function formatKindTable(config: MeasureConfig): string { + const rule = kindPlan('rule', config); + const behavior = kindPlan('behavior', config); + const judge = kindPlan('judge', config); + const budget = `$${config.budgetUsd} total; asks above $${config.askPerTrialUsd}/trial or with no estimate`; + return [ + '| Kind | Trials | Pass bar | Panels | Budget per red case |', + '|---|---|---|---|---|', + `| rule | ${rule.trials} | at least ${rule.trialTarget} of ${rule.trials} trials | none | ${budget} |`, + `| behavior | ${behavior.trials} | every panel at ${PANEL.k} of ${PANEL.n}, no contract violation, at least ${behavior.trialTarget} of ${behavior.trials} trials | ${behavior.units} of ${PANEL.n} trials | ${budget} |`, + `| judge | ${judge.trials} outputs | at least ${judge.unitTarget} of ${judge.trials} outputs | ${JUDGE_SAMPLES} samples per output, passes at 2 of ${JUDGE_SAMPLES} | ${budget} |`, + '', + `Repair rounds: at most ${config.maxRounds}. Free-suite rerun backend: ${config.rerunBackend}. Every trial is diagnostic and never changes a recorded verdict.`, + ].join('\n'); +} + +// ─── Trials and aggregation ──────────────────────────────────────────────── + +export interface TrialResult { + passed: boolean; + /** A contract violation fails its unit at any count. */ + contract?: boolean; + /** Billed cost; absent means unknown. */ + costUsd?: number; + /** Judge outputs: each sample's pass. */ + samples?: boolean[]; + failureCause?: string; + failureDetail?: string; +} + +export interface TrialRequest { caseId: string; kind: MeasureKind; round: string; trial: number; dir: string } +export type TrialRunner = (request: TrialRequest) => Promise; + +export interface TrialRecord extends TrialResult { trial: number; unit: number; batch: number; dir: string } + +export interface Aggregate { unitPasses: number; trialPasses: number; complete: boolean; meets: boolean; unitResults: boolean[] } + +function unitPassed(kind: MeasureKind, trials: TrialRecord[]): boolean { + if (trials.some(t => t.contract)) return false; + if (kind === 'behavior') return trials.length === PANEL.n && trials.filter(t => t.passed).length >= PANEL.k; + const [only] = trials; + if (!only) return false; + if (kind === 'judge' && only.samples) return only.samples.length === JUDGE_SAMPLES && only.samples.filter(Boolean).length >= 2; + return only.passed; +} + +export function aggregate(plan: KindPlan, trials: TrialRecord[]): Aggregate { + const unitResults = Array.from({ length: plan.units }, (_, i) => unitPassed(plan.kind, trials.filter(t => t.unit === i + 1))); + const unitPasses = unitResults.filter(Boolean).length; + const trialPasses = plan.kind === 'behavior' ? trials.filter(t => t.passed && !t.contract).length : unitPasses; + const complete = trials.length === plan.trials; + return { unitPasses, trialPasses, complete, unitResults, meets: complete && unitPasses >= plan.unitTarget && trialPasses >= plan.trialTarget }; +} + +// ─── One measurement: baseline or a repair round ─────────────────────────── + +export type MeasureStatus = 'measured' | 'needs_approval' | 'budget_exhausted' | 'round_limit'; + +export interface Measurement { + schema: 'gstack-ship-measure/1'; + label: 'diagnostic'; + verdict: null; + case: string; + kind: MeasureKind; + round: string; + fix?: string; + status: MeasureStatus; + reason?: string; + plan: KindPlan; + trials: TrialRecord[]; + unitPasses: number; + trialPasses: number; + meets: boolean; + /** Reserved before admission; null when a batch ran without an estimate. */ + estimatedUsd: number | null; + actualUsd: number; + costUnknownTrials: number; + parallel: number; +} + +interface Ledger { case: string; kind: MeasureKind; approved: boolean; spentUsd: number; rounds: string[] } + +export interface MeasureOptions { + caseId: string; + kind: MeasureKind; + round: string; + config: MeasureConfig; + runner: TrialRunner; + outDir: string; + parallel: number; + costPerTrialUsd: number | null; + approved: boolean; + fix?: string; + /** Pass-rate history directory diagnostic artifacts must stay out of (default: the project eval dir). */ + evalDir?: string; + log?: (line: string) => void; +} + +export const caseSlug = (id: string) => id.replace(/[^A-Za-z0-9._-]+/g, '_'); +const readJson = (file: string): T | null => { try { return JSON.parse(fs.readFileSync(file, 'utf8')) as T; } catch { return null; } }; +const writeJson = (file: string, value: unknown) => { fs.mkdirSync(path.dirname(file), { recursive: true }); fs.writeFileSync(file, `${JSON.stringify(value, null, 2)}\n`); }; +const inside = (child: string, parent: string) => { const rel = path.relative(path.resolve(parent), path.resolve(child)); return rel === '' || (!rel.startsWith('..') && !path.isAbsolute(rel)); }; + +/** Throws when diagnostic artifacts would land where verdict history is read. */ +export function assertDiagnosticDir(outDir: string, evalDir: string): void { + if (inside(outDir, evalDir)) throw new Error(`ship-measure: ${outDir} is inside the eval history dir ${evalDir}; diagnostic trials are never recorded as verdicts. Use the default .context/ship-measure.`); +} + +function roundNumber(round: string): number { + if (round === 'baseline') return 0; + const match = /^round-([1-9][0-9]*)$/.exec(round); + if (!match) throw new Error(`--round must be baseline or round-N. Received: ${round}`); + return Number(match[1]); +} + +export async function measureCase(o: MeasureOptions): Promise { + const log = o.log ?? ((line: string) => console.log(line)); + const plan = kindPlan(o.kind, o.config); + assertDiagnosticDir(o.outDir, o.evalDir ?? getProjectEvalDir()); + const caseDir = path.join(o.outDir, caseSlug(o.caseId)); + const ledgerPath = path.join(caseDir, 'ledger.json'); + const ledger: Ledger = readJson(ledgerPath) ?? { case: o.caseId, kind: o.kind, approved: false, spentUsd: 0, rounds: [] }; + const base: Measurement = { + schema: 'gstack-ship-measure/1', label: 'diagnostic', verdict: null, case: o.caseId, kind: o.kind, round: o.round, + ...(o.fix ? { fix: o.fix } : {}), status: 'measured', plan, trials: [], unitPasses: 0, trialPasses: 0, meets: false, + estimatedUsd: 0, actualUsd: 0, costUnknownTrials: 0, parallel: Math.max(1, o.parallel), + }; + if (roundNumber(o.round) > o.config.maxRounds) { + return { ...base, status: 'round_limit', reason: `repair-round limit ${o.config.maxRounds} reached; stop with the named-red report` }; + } + const estimate = o.costPerTrialUsd; + const ask = estimate === null ? 'estimate unknown' : estimate > o.config.askPerTrialUsd + ? `estimated $${estimate.toFixed(2)}/trial is above $${o.config.askPerTrialUsd}/trial` : null; + if (ask && !o.approved && !ledger.approved) { + return { ...base, status: 'needs_approval', reason: `${ask}; ${plan.trials} trials, budget $${o.config.budgetUsd} per red case. Ask once, then rerun with --approved.` }; + } + if (ask) ledger.approved = true; + const roundDir = path.join(caseDir, o.round); + if (fs.existsSync(roundDir)) throw new Error(`ship-measure: ${roundDir} already exists; measurements are never overwritten. Use the next round.`); + fs.mkdirSync(roundDir, { recursive: true }); + const result = await runBatches(o, plan, roundDir, ledger, estimate, log); + const totals = aggregate(plan, result.trials); + const measurement: Measurement = { ...base, ...result, unitPasses: totals.unitPasses, trialPasses: totals.trialPasses, meets: totals.meets }; + ledger.rounds.push(o.round); + writeJson(path.join(roundDir, 'measurement.json'), measurement); + writeJson(ledgerPath, ledger); + log(formatMeasurementLine(measurement)); + return measurement; +} + +async function runBatches(o: MeasureOptions, plan: KindPlan, roundDir: string, ledger: Ledger, initialEstimate: number | null, + log: (line: string) => void): Promise> { + const trials: TrialRecord[] = []; + let estimate = initialEstimate; + let estimatedUsd: number | null = 0; + let actualUsd = 0; + let costUnknownTrials = 0; + let knownCosts = 0; + let batch = 0; + const parallel = Math.max(1, o.parallel); + while (trials.length < plan.trials) { + const remaining = plan.trials - trials.length; + const headroom = o.config.budgetUsd - ledger.spentUsd; + let size = Math.min(parallel, remaining); + if (estimate === null) size = knownCosts === 0 && trials.length === 0 ? 1 : size; + else size = Math.min(size, Math.floor((headroom + 1e-9) / estimate)); + if (size < 1 || headroom <= 0) { + return { trials, status: 'budget_exhausted', estimatedUsd, actualUsd, costUnknownTrials, parallel, + reason: `budget $${o.config.budgetUsd} per red case: spent $${ledger.spentUsd.toFixed(2)}, next trial reserves $${(estimate ?? 0).toFixed(2)}; ${trials.length} of ${plan.trials} trials ran` }; + } + batch += 1; + estimatedUsd = estimate === null || estimatedUsd === null ? null : estimatedUsd + size * estimate; + log(`[ship-measure] ${o.caseId} ${o.round}: batch ${batch} admits ${size} trial(s)${estimate === null ? ' (calibration, no estimate)' : `, reserves $${(size * estimate).toFixed(2)}`}; spent $${ledger.spentUsd.toFixed(2)} of $${o.config.budgetUsd}`); + const start = trials.length; + const results = await Promise.all(Array.from({ length: size }, async (_, i) => { + const trial = start + i + 1; + const dir = path.join(roundDir, `t${String(trial).padStart(2, '0')}`); + fs.mkdirSync(dir, { recursive: false }); + let result: TrialResult; + try { result = await o.runner({ caseId: o.caseId, kind: o.kind, round: o.round, trial, dir }); } + catch (error) { result = { passed: false, failureCause: 'unknown', failureDetail: `runner error: ${(error as Error).message}` }; } + return { ...result, trial, unit: Math.ceil(trial / plan.trialsPerUnit), batch, dir } satisfies TrialRecord; + })); + for (const record of results) { + writeJson(path.join(record.dir, 'trial.json'), { label: 'diagnostic', verdict: null, ...record }); + trials.push(record); + if (typeof record.costUsd === 'number' && Number.isFinite(record.costUsd)) { + actualUsd += record.costUsd; ledger.spentUsd += record.costUsd; knownCosts += 1; + } else { + costUnknownTrials += 1; + if (estimate !== null) ledger.spentUsd += estimate; + } + } + if (initialEstimate === null && knownCosts > 0) estimate = actualUsd / knownCosts; + } + return { trials, status: 'measured', estimatedUsd, actualUsd, costUnknownTrials, parallel }; +} + +export function formatMeasurementLine(m: Measurement): string { + const observed = m.kind === 'behavior' + ? `${m.trialPasses}/${m.plan.trials} trials, ${m.unitPasses}/${m.plan.units} panels` + : `${m.unitPasses}/${m.plan.units} ${m.kind === 'judge' ? 'outputs' : 'trials'}`; + const when = m.fix ? ` after fix at ${m.fix}` : m.round === 'baseline' ? ' (baseline)' : ''; + const verdict = m.status === 'measured' ? (m.meets ? 'at or above target' : 'below target') : m.status.replace('_', ' '); + return `[ship-measure] DIAGNOSTIC ${m.case} ${m.round}: observed ${observed}${when}; ${verdict}; est ${usd(m.estimatedUsd)}, actual ${usd(m.actualUsd)}${m.costUnknownTrials ? ` (+${m.costUnknownTrials} trial(s) cost unknown)` : ''}`; +} + +const usd = (value: number | null) => value === null ? 'unknown' : `$${value.toFixed(2)}`; + +// ─── Unmeasured skip and the PR-body report ──────────────────────────────── + +export function recordUnmeasured(outDir: string, caseId: string, reason: string): string { + if (!reason.trim()) throw new Error('skip needs --reason: why this red is infrastructure and is not measured'); + const file = path.join(outDir, caseSlug(caseId), 'unmeasured.json'); + writeJson(file, { schema: 'gstack-ship-measure/1', label: 'diagnostic', verdict: null, case: caseId, status: 'unmeasured', reason: reason.trim() }); + return file; +} + +/** The per-case table for the PR body: every measurement, estimated and actual spend, never a pass for an unmeasured case. */ +export function formatReport(outDir: string): string { + const rows = ['| Case | Kind | Measurement | Observed | Target | Est. spend | Actual spend | Status |', '|---|---|---|---|---|---|---|---|']; + const cases = fs.existsSync(outDir) ? fs.readdirSync(outDir).filter(name => fs.statSync(path.join(outDir, name)).isDirectory()).sort() : []; + for (const name of cases) { + const dir = path.join(outDir, name); + const skip = readJson<{ case: string; reason: string }>(path.join(dir, 'unmeasured.json')); + if (skip) { rows.push(`| ${skip.case} | — | — | — | — | — | — | unmeasured (${skip.reason.replace(/\|/g, '/')}); not a pass |`); continue; } + const ledger = readJson(path.join(dir, 'ledger.json')); + for (const round of ledger?.rounds ?? []) { + const m = readJson(path.join(dir, round, 'measurement.json')); + if (!m) continue; + const observed = m.kind === 'behavior' ? `${m.trialPasses}/${m.trials.length} trials, ${m.unitPasses}/${m.plan.units} panels` : `${m.unitPasses}/${m.trials.length}`; + const target = m.kind === 'behavior' ? `all ${m.plan.units} panels, ${m.plan.trialTarget}/${m.plan.trials}` : `${m.plan.unitTarget}/${m.plan.units}`; + const what = m.fix ? `${m.round}: after fix at ${m.fix.replace(/\|/g, '/')}` : m.round; + const status = m.status === 'measured' ? (m.meets ? 'at or above target' : 'below target') : m.status.replace('_', ' '); + rows.push(`| ${m.case} | ${m.kind} | ${what} | observed ${observed} | ${target} | ${usd(m.estimatedUsd)} | ${usd(m.actualUsd)}${m.costUnknownTrials ? ` (${m.costUnknownTrials} unknown)` : ''} | ${status} |`); + } + } + return [...rows, '', 'Diagnostic measurements: they never change a recorded verdict; the lane verdict is the full gate run.'].join('\n'); +} + +// ─── Trial runners ───────────────────────────────────────────────────────── + +const TRIAL_TIMEOUT_MS = 60 * 60_000; + +/** Cost, failure fields and judge records a trial's eval dir holds (any finalized eval-store result under it). */ +export function readTrialRecords(evalDir: string): Array> { + if (!fs.existsSync(evalDir)) return []; + return (fs.readdirSync(evalDir, { recursive: true }) as string[]).filter(isFinalizedEvalResultFile) + .flatMap(name => readJson<{ tests?: unknown[] }>(path.join(evalDir, name))?.tests ?? []) + .filter((t): t is Record => !!t && typeof t === 'object'); +} + +function costOf(records: Array>): number | undefined { + if (!records.length || records.some(r => r.cost_known === false || typeof r.cost_usd !== 'number')) return undefined; + return records.reduce((sum, r) => sum + r.cost_usd, 0); +} + +/** Runs argv as one trial in its own process group, logging to /output.log; returns exit code and text. */ +async function runTrialProcess(argv: string[], cwd: string, env: NodeJS.ProcessEnv, dir: string, timeoutMs = TRIAL_TIMEOUT_MS): Promise<{ code: number | null; output: string }> { + const logFile = fs.createWriteStream(path.join(dir, 'output.log')); + const chunks: string[] = []; + const child = spawn(argv[0]!, argv.slice(1), { cwd, env, stdio: ['ignore', 'pipe', 'pipe'], detached: process.platform !== 'win32' }); + for (const stream of [child.stdout, child.stderr]) stream?.on('data', (chunk: Buffer) => { chunks.push(chunk.toString('utf8')); logFile.write(chunk); }); + const timer = setTimeout(() => { try { process.kill(-(child.pid ?? 0), 'SIGKILL'); } catch { child.kill('SIGKILL'); } }, timeoutMs); + const code = await new Promise(resolve => { child.on('error', () => resolve(null)); child.on('close', resolve); }); + clearTimeout(timer); + await new Promise(resolve => logFile.end(resolve)); + return { code, output: chunks.join('') }; +} + +function parseTrialOutput(output: string): Partial { + const field = (name: string) => new RegExp(`^\\s*${name}\\s*[:=]\\s*(.+?)\\s*$`, 'm').exec(output)?.[1]; + const cost = field('cost_usd'); + const samples = field('samples'); + return { + ...(field('failure_cause') ? { failureCause: field('failure_cause') } : {}), + ...(field('failure_detail') ? { failureDetail: field('failure_detail') } : {}), + ...(cost !== undefined && Number.isFinite(Number(cost)) ? { costUsd: Number(cost) } : {}), + ...(samples ? { samples: samples.split(/[,\s]+/).filter(Boolean).map(s => s === '1' || s === 'pass' || s === 'true') } : {}), + ...(/^\s*contract_violation\b/m.test(output) ? { contract: true } : {}), + }; +} + +/** + * The project's documented single-case command, split on whitespace (no + * shell), with {case} replaced by the case id. Exit 0 is a pass; the trial's + * own lines `cost_usd:`, `failure_cause:`, `failure_detail:`, `samples:` and + * `contract_violation` are read, and so are eval-store records it writes to + * GSTACK_EVAL_DIR. + */ +export function commandRunner(template: string, cwd = process.cwd()): TrialRunner { + const words = template.trim().split(/\s+/).filter(Boolean); + if (!words.length) throw new Error('--command is empty'); + return async ({ caseId, trial, dir }) => { + const env = { ...process.env, GSTACK_EVAL_DIR: path.join(dir, 'eval'), GSTACK_SHIP_MEASURE_DIR: dir, GSTACK_SHIP_MEASURE_LABEL: 'diagnostic', GSTACK_SHIP_MEASURE_TRIAL: String(trial) }; + const { code, output } = await runTrialProcess(words.map(w => w.replaceAll('{case}', caseId)), cwd, env, dir); + const parsed = parseTrialOutput(output); + const records = readTrialRecords(path.join(dir, 'eval')); + const failed = records.find(r => r.passed !== true); + return { + passed: code === 0, ...parsed, + ...(parsed.costUsd === undefined && costOf(records) !== undefined ? { costUsd: costOf(records) } : {}), + ...(code !== 0 && !parsed.failureCause && failed?.failure_cause ? { failureCause: String(failed.failure_cause) } : {}), + ...(code !== 0 && !parsed.failureDetail && failed?.failure_detail ? { failureDetail: JSON.stringify(failed.failure_detail).slice(0, 300) } : {}), + }; + }; +} + +/** The one paid file that names a standalone judge id, else a thrown reason. */ +export function judgeFile(id: string, rootDir = ROOT): string { + const owners = collectPaidTestFiles(rootDir).filter(file => { + const source = fs.readFileSync(path.join(rootDir, file), 'utf8'); + return source.includes(`'${id}'`) || source.includes(`"${id}"`); + }); + if (owners.length !== 1) throw new Error(`judge ${id}: ${owners.length ? `named by ${owners.join(', ')}` : 'no paid file names it'}; it needs exactly one`); + return owners[0]!; +} + +/** + * gstack's default runner. An E2E case runs one trial through + * `scripts/test-paid-shards.ts --case --trials 1` (the CI panel runner, + * whose exit code is that trial's verdict); a standalone judge runs its file + * with the judge selected alone and passes only on its own passing record. + */ +export function gstackRunner(rootDir = ROOT): TrialRunner { + return async ({ caseId, dir }) => { + const evalDir = path.join(dir, 'eval'); + const isJudge = !Object.hasOwn(E2E_TIERS, caseId); + if (isJudge && !Object.hasOwn(LLM_JUDGE_TOUCHFILES, caseId)) throw new Error(`${caseId} is neither an E2E case nor a standalone judge`); + const argv = isJudge + ? [process.execPath, 'test', path.join(rootDir, judgeFile(caseId, rootDir))] + : [process.execPath, 'run', path.join(rootDir, 'scripts/test-paid-shards.ts'), '--tier', E2E_TIERS[caseId]!, '--case', caseId, '--trials', '1']; + const env = { + ...process.env, GSTACK_EVAL_DIR: evalDir, GSTACK_SHIP_MEASURE_LABEL: 'diagnostic', EVALS_JOBS: '1', + ...(isJudge ? { EVALS: '1', EVALS_TIER: 'gate', EVALS_ALL: '1', ...paidSelectionEnv('full', { e2e: [], judges: [caseId] }, 'ship-measure judge') } : {}), + }; + const { code, output } = await runTrialProcess(argv, rootDir, env, dir); + const records = readTrialRecords(evalDir); + const own = isJudge ? records.filter(r => r.name === caseId || r.case_id === caseId) : records; + const failed = own.find(r => r.passed !== true); + const passed = code === 0 && (!isJudge || (own.length > 0 && !failed)); + const cost = costOf(own); + return { + passed, ...(cost !== undefined ? { costUsd: cost } : {}), + ...(own.some(r => r.failure_class === 'contract') ? { contract: true } : {}), + ...(passed ? {} : { + failureCause: failed?.failure_cause ? String(failed.failure_cause) : own.length ? 'assertion' : 'unknown', + failureDetail: failed?.failure_detail ? JSON.stringify(failed.failure_detail).slice(0, 300) + : own.length ? String(failed?.error ?? '').slice(0, 300) : `no trial record; exit ${code}; ${output.trim().split('\n').slice(-1)[0] ?? ''}`.slice(0, 300), + }), + }; + }; +} + +// ─── Free-suite shard reruns (CEO-1) ─────────────────────────────────────── + +export interface FreeRerunOutcome { rerun: number; mode: 'baseline' | 'parallel'; status: string; completed: boolean; failingFiles: string[]; elapsedMs: number; log: string } + +export interface FreeMeasurement { + schema: 'gstack-ship-measure/1'; label: 'diagnostic'; verdict: null; + files: string[]; reruns: number; completed: number; passed: number; parallel: number; concurrency: number; + backend: 'local'; wallCapMs: number; capHit: boolean; outcomes: FreeRerunOutcome[]; +} + +export interface FreeRerunOptions { + files: string[]; + reruns: number; + /** Within-shard concurrency of the original run (the free lane runs each shard at 1). */ + concurrency: number; + wallCapMs: number; + outDir: string; + cores?: number; + runShard?: typeof runFreeShard; + log?: (line: string) => void; +} + +/** Local parallelism for free reruns: max(1, floor(cores / shard concurrency)). */ +export const freeParallelism = (cores: number, concurrency: number) => Math.max(1, Math.floor(cores / Math.max(1, concurrency))); + +/** + * Rerun one shard's exact file list N times with the flaky retry off: rerun 1 + * alone as the baseline, the rest in parallel up to the cap, each with its own + * HOME, state root and flake ledger. No rerun starts after the wall cap, and + * one the cap cuts short is not counted as completed. + */ +export async function measureFreeShard(o: FreeRerunOptions): Promise { + const log = o.log ?? ((line: string) => console.log(line)); + const runShard = o.runShard ?? runFreeShard; + const parallel = freeParallelism(o.cores ?? os.availableParallelism(), o.concurrency); + fs.mkdirSync(path.join(o.outDir, 'free'), { recursive: true }); + const dir = fs.mkdtempSync(path.join(o.outDir, 'free', 'rerun-')); + const deadline = Date.now() + o.wallCapMs; + const shardWall = wallTimeoutForShard(o.files.length); + const outcomes: FreeRerunOutcome[] = []; + const runOne = async (rerun: number, mode: FreeRerunOutcome['mode']) => { + const runDir = path.join(dir, `r${String(rerun).padStart(2, '0')}`); + const state = path.join(runDir, 'state'); + fs.mkdirSync(state, { recursive: true }); + const remaining = deadline - Date.now(); + const wallTimeoutMs = Math.max(1, Math.min(shardWall, remaining)); + const env = { ...process.env, GSTACK_HOME: state, GSTACK_STATE_ROOT: state, GSTACK_FREE_RETRY_FLAKY: '0', GSTACK_FLAKE_LEDGER: path.join(runDir, 'flake-ledger.jsonl') }; + const logPath = path.join(runDir, 'shard.log'); + const outcome: FreeShardOutcome = await runShard(o.files, rerun, o.reruns, { env, wallTimeoutMs, quiet: true, log: () => {}, logFilePath: logPath, homeGuard: privateFreeHome }); + const cut = outcome.status === 'timed-out' && wallTimeoutMs < shardWall; + outcomes.push({ rerun, mode, status: cut ? 'cut by wall cap' : outcome.status, completed: !cut, failingFiles: outcome.failingFiles, elapsedMs: outcome.elapsedMs, log: logPath }); + }; + if (o.reruns > 0) await runOne(1, 'baseline'); + let next = 2; + const worker = async () => { + while (next <= o.reruns && Date.now() < deadline) await runOne(next++, 'parallel'); + }; + await Promise.all(Array.from({ length: Math.min(parallel, Math.max(0, o.reruns - 1)) }, worker)); + outcomes.sort((a, b) => a.rerun - b.rerun); + const done = outcomes.filter(r => r.completed); + const measurement: FreeMeasurement = { + schema: 'gstack-ship-measure/1', label: 'diagnostic', verdict: null, files: o.files, reruns: o.reruns, + completed: done.length, passed: done.filter(r => r.status === 'passed').length, parallel, concurrency: o.concurrency, + backend: 'local', wallCapMs: o.wallCapMs, capHit: done.length < o.reruns, outcomes, + }; + writeJson(path.join(dir, 'free-measurement.json'), measurement); + log(`[ship-measure] DIAGNOSTIC free shard (${o.files.length} files): observed ${measurement.passed}/${measurement.completed} passed; ${measurement.completed} of ${o.reruns} completed${measurement.capHit ? ` (wall cap ${Math.round(o.wallCapMs / 1000)}s)` : ''}; rerun 1 alone as baseline, the rest at parallelism ${parallel}; flaky retry off. Captures: ${dir}`); + return measurement; +} + +/** The exact file list of local free-suite shard I, as `bun run test` packs it. */ +export function localFreeShardFiles(index: number, rootDir = ROOT): string[] { + const files = collectFreeTestFiles(rootDir); + const exclusive = files.filter(f => f in TREE_MUTATING); + const readers = files.filter(f => !(f in TREE_MUTATING)); + const jobs = fullSuiteJobs(); + const durations = loadFreeTestDurations(rootDir); + const shards = durations ? packShardsByDuration(readers, jobs, durations).shards : assignFilesToShards(readers, jobs); + if (exclusive.length) shards.push(exclusive); + const shard = shards[index - 1]; + if (!shard) throw new Error(`--shard ${index}: the local free suite has ${shards.length} shards`); + return shard; +} + +// ─── CLI ─────────────────────────────────────────────────────────────────── + +function positive(raw: string | undefined, flag: string, integer = true): number { + const value = Number(raw); + if (raw === undefined || !(integer ? /^[1-9][0-9]*$/.test(raw) : Number.isFinite(value) && value >= 0)) throw new Error(`${flag} needs a ${integer ? 'positive integer' : 'non-negative number'}. Received: ${raw}`); + return value; +} + +export async function main(argv: string[], env: NodeJS.ProcessEnv = process.env): Promise { + const [command, ...rest] = argv; + const flags: Record = {}; + const value = (name: string) => (next: () => string | undefined) => { const v = next(); if (v === undefined) throw new Error(`${name} needs a value`); flags[name] = v; }; + const handlers = Object.fromEntries(['--case', '--round', '--kind', '--command', '--cost-per-trial', '--fix', '--jobs', '--out', '--reason', + '--files', '--shard', '--reruns', '--concurrency', '--backend', '--wall-cap'].map(name => [name, value(name)])); + parseCliFlags(rest, { ...handlers, '--approved': () => { flags['--approved'] = '1'; } }, 'Usage: see the header of scripts/ship-measure.ts'); + const config = readMeasureConfig(); + const outDir = path.resolve(flags['--out'] ?? path.join(process.cwd(), '.context', 'ship-measure')); + if (command === 'table') { console.log(formatKindTable(config)); return 0; } + if (command === 'report') { console.log(formatReport(outDir)); return 0; } + if (command === 'skip') { console.log(`[ship-measure] ${flags['--case']} labeled unmeasured: ${recordUnmeasured(outDir, flags['--case'] ?? '', flags['--reason'] ?? '')}`); return 0; } + if (command === 'free') return runFreeCli(flags, config, outDir, env); + if (command !== 'measure') throw new Error(`Unknown command: ${command ?? '(none)'}. Use table, measure, skip, report or free.`); + const caseId = flags['--case']; + if (!caseId) throw new Error('measure needs --case'); + const kind = (flags['--kind'] ?? E2E_KINDS[caseId] ?? 'rule') as MeasureKind; + if (!['rule', 'behavior', 'judge'].includes(kind)) throw new Error(`--kind must be rule, behavior or judge. Received: ${kind}`); + const isGstack = path.resolve(process.cwd()) === ROOT; + if (!flags['--command'] && !isGstack) throw new Error('No single-case eval command: pass --command \' {case}\' (ask the user once and record the answer).'); + console.log(formatKindTable(config)); + const m = await measureCase({ + caseId, kind, round: flags['--round'] ?? 'baseline', config, outDir, fix: flags['--fix'], + runner: flags['--command'] ? commandRunner(flags['--command']) : gstackRunner(), + parallel: flags['--jobs'] ? positive(flags['--jobs'], '--jobs') : Number(env.EVALS_JOBS) || DEFAULT_JOBS, + costPerTrialUsd: flags['--cost-per-trial'] !== undefined ? positive(flags['--cost-per-trial'], '--cost-per-trial', false) : null, + approved: flags['--approved'] === '1', + }); + if (m.status === 'needs_approval') { console.log(`[ship-measure] NEEDS APPROVAL ${caseId}: ${m.reason}`); return 2; } + if (m.status !== 'measured') { console.log(`[ship-measure] NAMED RED ${caseId}: ${m.reason}`); return 3; } + return m.meets ? 0 : 1; +} + +async function runFreeCli(flags: Record, config: MeasureConfig, outDir: string, env: NodeJS.ProcessEnv): Promise { + const files = flags['--files'] ? flags['--files'].split(',').map(f => f.trim()).filter(Boolean) + : flags['--shard'] ? localFreeShardFiles(positive(flags['--shard'], '--shard')) : []; + if (!files.length) throw new Error('free needs --files a,b,... (the failing shard\'s exact file list) or --shard I'); + const reruns = flags['--reruns'] ? positive(flags['--reruns'], '--reruns') : config.ruleTrials; + const backend = (flags['--backend'] ?? config.rerunBackend) as RerunBackend; + if (backend === 'ubicloud') { + if (!env.UBICLOUD_API_KEY) throw new Error('ship_rerun_backend is ubicloud but UBICLOUD_API_KEY is not set; set it or use --backend local'); + const r = spawnSync('bash', [path.join(ROOT, 'scripts/ubicloud/test-free.sh'), '--diagnostic', '--files', files.join(','), '--reruns', String(reruns)], + { stdio: 'inherit', env, timeout: 30 * 60_000 }); + return r.status ?? 1; + } + if (backend !== 'local') throw new Error(`--backend must be local or ubicloud. Received: ${backend}`); + const m = await measureFreeShard({ files, reruns, outDir, concurrency: flags['--concurrency'] ? positive(flags['--concurrency'], '--concurrency') : 1, + wallCapMs: (flags['--wall-cap'] ? positive(flags['--wall-cap'], '--wall-cap') : 600) * 1000 }); + return m.completed > 0 && m.passed === m.completed ? 0 : 1; +} + +if (import.meta.main) { + main(process.argv.slice(2)).then(code => process.exit(code), (error: Error) => { console.error(`[ship-measure] ${error.message}`); process.exit(4); }); +} diff --git a/scripts/ubicloud/test-free.sh b/scripts/ubicloud/test-free.sh index 233cbe65d..307f8b75e 100755 --- a/scripts/ubicloud/test-free.sh +++ b/scripts/ubicloud/test-free.sh @@ -4,12 +4,25 @@ # lane's environment. Shard logs land in .context/ubicloud//. # With --record-durations, the refreshed scripts/free-test-durations.json is # copied back into this checkout. +# +# --diagnostic [ship-measure free args]: /ship's free-suite flake measurement +# (scripts/ship-measure.ts free) on one VM, rerunning one shard's exact file +# list N times in parallel with the flaky retry OFF, so a retry never inflates +# the measured rate. Its captures land in .context/ubicloud//. set -euo pipefail here=$(cd "$(dirname "$0")" && pwd) root=$(cd "$here/../.." && pwd) logs="$root/.context/ubicloud/$(date +%Y%m%d-%H%M%S)" +if [ "${1:-}" = --diagnostic ]; then + shift + exec "$here/ubi-runner.sh" run --src "$root" --setup "$here/setup-free-suite.sh" \ + --env GSTACK_EXPECT_BINARIES=1 --env GSTACK_FREE_RETRY_FLAKY=0 \ + --pull "work/$(basename "$root")/.context/ship-measure:$logs" \ + -- "xvfb-run -a bun run scripts/ship-measure.ts free --backend local $*" +fi + args=(--src "$root" --setup "$here/setup-free-suite.sh" --env GSTACK_EXPECT_BINARIES=1 --env GSTACK_FREE_RETRY_FLAKY=1 --env GSTACK_FLAKE_LEDGER=/tmp/gstack-free-test-flake-ledger.jsonl diff --git a/test/ship-measure.test.ts b/test/ship-measure.test.ts new file mode 100644 index 000000000..9d14de89f --- /dev/null +++ b/test/ship-measure.test.ts @@ -0,0 +1,323 @@ +/** + * Free contracts for scripts/ship-measure.ts, /ship's measure-then-fix runner: + * per-kind aggregation (rule trials, behavior panels of 3 at their own 2 of 3, + * judge outputs scored by 3-sample panels), unique artifact directories, the + * diagnostic label, the estimated admission budget, the ask-once rule, the + * repair-round limit, the unmeasured skip, the PR table and the free-suite + * reruns with the flaky retry off. Every runner here is a fake; nothing paid. + */ +import { afterAll, describe, expect, test } from 'bun:test'; +import * as fs from 'node:fs'; +import * as os from 'node:os'; +import * as path from 'node:path'; +import { spawnSync } from 'node:child_process'; +import { + MEASURE_DEFAULTS, aggregate, assertDiagnosticDir, formatKindTable, formatReport, freeParallelism, judgeFile, kindPlan, + measureCase, measureFreeShard, readMeasureConfig, recordUnmeasured, + type MeasureConfig, type MeasureKind, type TrialRecord, type TrialRequest, type TrialResult, +} from '../scripts/ship-measure'; +import type { FreeShardOutcome, RunFreeShardOptions } from '../scripts/test-free-shards'; + +const ROOT = path.resolve(import.meta.dir, '..'); +const temps: string[] = []; +const tmp = (prefix: string) => { const dir = fs.mkdtempSync(path.join(os.tmpdir(), prefix)); temps.push(dir); return dir; }; +afterAll(() => { for (const dir of temps) fs.rmSync(dir, { recursive: true, force: true }); }); + +interface FakeRun { calls: TrialRequest[]; maxConcurrent: number; runner: (r: TrialRequest) => Promise } +/** A fake trial runner: `outcome(trial)` decides each result; it records calls and peak concurrency. */ +function fake(outcome: (trial: number) => TrialResult): FakeRun { + const run: FakeRun = { calls: [], maxConcurrent: 0, runner: async () => ({ passed: true }) }; + let active = 0; + run.runner = async (request) => { + run.calls.push(request); + active += 1; + run.maxConcurrent = Math.max(run.maxConcurrent, active); + await Bun.sleep(2); + active -= 1; + return outcome(request.trial); + }; + return run; +} + +const measure = (kind: MeasureKind, run: FakeRun, extra: Partial[0]> = {}) => { + const outDir = extra.outDir ?? tmp('ship-measure-out-'); + return measureCase({ + caseId: 'seeded-case', kind, round: 'baseline', config: MEASURE_DEFAULTS, runner: run.runner, outDir, parallel: 4, + costPerTrialUsd: 0.5, approved: false, evalDir: tmp('ship-measure-history-'), log: () => {}, ...extra, + }); +}; + +describe('per-kind plans and the printed table (CEO-9, DX-7)', () => { + test('defaults: rule 10 trials at 9, behavior 4 panels of 3 at 11 of 12, judge 10 outputs at 9', () => { + expect(kindPlan('rule', MEASURE_DEFAULTS)).toEqual({ kind: 'rule', units: 10, trialsPerUnit: 1, trials: 10, unitTarget: 9, trialTarget: 9 }); + expect(kindPlan('behavior', MEASURE_DEFAULTS)).toEqual({ kind: 'behavior', units: 4, trialsPerUnit: 3, trials: 12, unitTarget: 4, trialTarget: 11 }); + expect(kindPlan('judge', MEASURE_DEFAULTS)).toEqual({ kind: 'judge', units: 10, trialsPerUnit: 1, trials: 10, unitTarget: 9, trialTarget: 9 }); + const table = formatKindTable(MEASURE_DEFAULTS); + expect(table).toContain('| rule | 10 | at least 9 of 10 trials |'); + expect(table).toContain('every panel at 2 of 3, no contract violation, at least 11 of 12 trials | 4 of 3 trials'); + expect(table).toContain('| judge | 10 outputs | at least 9 of 10 outputs | 3 samples per output, passes at 2 of 3'); + expect(table).toContain('$25 total; asks above $2/trial'); + expect(table).toContain('Repair rounds: at most 3'); + expect(table).toContain('never changes a recorded verdict'); + }); + + test('config keys override defaults; an empty value is the default; an invalid value throws', () => { + const values: Record = { ship_measure_rule_trials: '20', ship_measure_budget_usd: '7.5', ship_rerun_backend: 'ubicloud', ship_measure_max_rounds: '' }; + const config = readMeasureConfig(key => values[key]); + expect(config).toEqual({ ...MEASURE_DEFAULTS, ruleTrials: 20, budgetUsd: 7.5, rerunBackend: 'ubicloud' }); + expect(kindPlan('rule', config).trialTarget).toBe(18); + expect(() => readMeasureConfig(key => key === 'ship_measure_behavior_panels' ? '0' : undefined)).toThrow('ship_measure_behavior_panels'); + expect(() => readMeasureConfig(key => key === 'ship_rerun_backend' ? 'cloud' : undefined)).toThrow('not local or ubicloud'); + }); + + test('bin/gstack-config knows every key with its default and rejects a value that would change spend', () => { + const state = tmp('ship-measure-config-'); + const env = { PATH: process.env.PATH, HOME: state, GSTACK_STATE_ROOT: state }; + const config = (...args: string[]) => spawnSync('bash', [path.join(ROOT, 'bin/gstack-config'), ...args], { encoding: 'utf8', timeout: 30_000, env }); + const defaults = config('defaults').stdout; + for (const [key, value] of [['ship_measure_rule_trials', '10'], ['ship_measure_behavior_panels', '4'], ['ship_measure_judge_outputs', '10'], + ['ship_measure_ask_per_trial_usd', '2'], ['ship_measure_budget_usd', '25'], ['ship_measure_max_rounds', '3'], ['ship_rerun_backend', 'local']]) { + expect(defaults).toMatch(new RegExp(`${key}:\\s+${value}\\n`)); + expect(config('get', key!).stdout.trim()).toBe(value!); + } + expect(config('set', 'ship_measure_budget_usd', '0').status).toBe(1); + expect(config('set', 'ship_measure_rule_trials', '2.5').status).toBe(1); + expect(config('set', 'ship_rerun_backend', 'cloud').status).toBe(1); + expect(config('set', 'ship_measure_budget_usd', '12.5').status).toBe(0); + expect(readMeasureConfig(key => config('get', key).stdout.trim()).budgetUsd).toBe(12.5); + expect(config('list').stdout).toMatch(/ship_measure_budget_usd:\s+12\.5 \(set/); + }); +}); + +describe('aggregation per kind with fake runners (ENG-12)', () => { + test('rule: 8 of 10 is below target, 9 of 10 meets; every trial gets its own diagnostic directory', async () => { + const red = fake(trial => ({ passed: ![3, 6].includes(trial), costUsd: 0.5, ...(trial === 3 ? { failureCause: 'assertion', failureDetail: 'unsorted list' } : {}) })); + const m = await measure('rule', red); + expect(m).toMatchObject({ status: 'measured', label: 'diagnostic', verdict: null, unitPasses: 8, meets: false }); + expect(red.calls).toHaveLength(10); + const dirs = m.trials.map(t => t.dir); + expect(new Set(dirs).size).toBe(10); + for (const dir of dirs) { + expect(path.basename(path.dirname(dir))).toBe('baseline'); + expect(JSON.parse(fs.readFileSync(path.join(dir, 'trial.json'), 'utf8'))).toMatchObject({ label: 'diagnostic', verdict: null }); + } + const saved = JSON.parse(fs.readFileSync(path.join(path.dirname(dirs[0]!), 'measurement.json'), 'utf8')); + expect(saved).toMatchObject({ label: 'diagnostic', verdict: null, round: 'baseline', unitPasses: 8, meets: false }); + expect(saved.trials.find((t: TrialRecord) => t.trial === 3)).toMatchObject({ failureCause: 'assertion', failureDetail: 'unsorted list' }); + expect((await measure('rule', fake(trial => ({ passed: trial !== 4, costUsd: 0.5 })))).meets).toBe(true); + }); + + test('behavior: each panel of 3 needs its own 2 of 3, and 11 of 12 trials overall', async () => { + // Trials 1 and 2 fail: 10 of 12 pass, which a single 2-of-12 panel would PASS; panel 1 is 1 of 3. + const panelRed = await measure('behavior', fake(trial => ({ passed: trial > 2 }))); + expect(panelRed).toMatchObject({ unitPasses: 3, trialPasses: 10, meets: false }); + // One failure in each of two panels: every panel passes at 2 of 3, but 10 of 12 is under 11. + const tooMany = await measure('behavior', fake(trial => ({ passed: trial !== 1 && trial !== 4 }))); + expect(tooMany).toMatchObject({ unitPasses: 4, trialPasses: 10, meets: false }); + const oneMiss = await measure('behavior', fake(trial => ({ passed: trial !== 7 }))); + expect(oneMiss).toMatchObject({ unitPasses: 4, trialPasses: 11, meets: true }); + expect(oneMiss.trials.map(t => t.unit)).toEqual([1, 1, 1, 2, 2, 2, 3, 3, 3, 4, 4, 4]); + // A contract violation fails its panel even when the trial's assertions passed. + const contract = await measure('behavior', fake(trial => ({ passed: true, ...(trial === 5 ? { contract: true } : {}) }))); + expect(contract).toMatchObject({ unitPasses: 3, trialPasses: 11, meets: false }); + }); + + test('judge: an output passes at 2 of its 3 samples, whatever the runner claimed; 9 of 10 outputs meets', async () => { + const judged = await measure('judge', fake(trial => ({ passed: true, samples: trial === 2 ? [true, false, false] : [true, false, true] }))); + expect(judged).toMatchObject({ unitPasses: 9, meets: true }); + const red = await measure('judge', fake(trial => ({ passed: true, samples: trial <= 2 ? [false, false, true] : [true, true, true] }))); + expect(red).toMatchObject({ unitPasses: 8, meets: false }); + const wrongCount = aggregate(kindPlan('judge', { ...MEASURE_DEFAULTS, judgeOutputs: 1 }), + [{ trial: 1, unit: 1, batch: 1, dir: '/x', passed: true, samples: [true, true] }]); + expect(wrongCount.meets).toBe(false); + }); + + test('standalone judge ids resolve to the one paid file that names them', () => { + expect(judgeFile('review/SKILL.md workflow')).toBe('test/skill-llm-eval.test.ts'); + expect(() => judgeFile('no-such-judge-id')).toThrow('no paid file names it'); + }); +}); + +describe('spend, approval, rounds and artifacts', () => { + test('admission budget: each batch reserves every concurrent trial and stops before reserved plus spent passes the budget', async () => { + const run = fake(() => ({ passed: false, costUsd: 3 })); + const outDir = tmp('ship-measure-budget-'); + const baseline = await measure('rule', run, { outDir, costPerTrialUsd: 3, approved: true }); + // $25 at $3/trial with 4 parallel: 4 (12) + 4 (24), then $1 of headroom admits nothing. + expect(baseline).toMatchObject({ status: 'budget_exhausted', estimatedUsd: 24, actualUsd: 24 }); + expect(baseline.trials.map(t => t.batch)).toEqual([1, 1, 1, 1, 2, 2, 2, 2]); + expect(run.maxConcurrent).toBeLessThanOrEqual(4); + expect(baseline.reason).toContain('8 of 10 trials ran'); + // The budget is per red case: a repair round starts from what the baseline spent. + const round = await measure('rule', fake(() => ({ passed: true, costUsd: 3 })), { outDir, round: 'round-1', costPerTrialUsd: 3 }); + expect(round.status).toBe('budget_exhausted'); + expect(round.trials).toHaveLength(0); + }); + + test('actual costs are reconciled after each batch, so cheaper trials admit more', async () => { + const m = await measure('rule', fake(() => ({ passed: true, costUsd: 1 })), { costPerTrialUsd: 2.5, approved: true, config: { ...MEASURE_DEFAULTS, budgetUsd: 12 } }); + // Batch 1 reserves 4 x 2.5 = 10 but spends 4; batch 2 then fits 3 (8 / 2.5); batch 3 fits 2 (5 / 2.5); batch 4 one more. + expect(m.trials.map(t => t.batch)).toEqual([1, 1, 1, 1, 2, 2, 2, 3, 3, 4]); + expect(m).toMatchObject({ status: 'measured', actualUsd: 10, estimatedUsd: 25, meets: true }); + }); + + test('asks once: no estimate or a per-trial estimate above the threshold runs nothing until approved', async () => { + const outDir = tmp('ship-measure-ask-'); + const run = fake(() => ({ passed: true, costUsd: 0.75 })); + const unknown = await measure('rule', run, { outDir, costPerTrialUsd: null }); + expect(unknown).toMatchObject({ status: 'needs_approval' }); + expect(unknown.reason).toContain('estimate unknown'); + expect(run.calls).toHaveLength(0); + expect(fs.existsSync(path.join(outDir, 'seeded-case', 'baseline'))).toBe(false); + const approved = await measure('rule', run, { outDir, costPerTrialUsd: null, approved: true }); + // No estimate: one calibration trial alone, then batches at its actual cost. + expect(approved.trials.map(t => t.batch)).toEqual([1, 2, 2, 2, 2, 3, 3, 3, 3, 4]); + expect(approved).toMatchObject({ status: 'measured', estimatedUsd: null, actualUsd: 7.5 }); + // The answer is recorded: the next round does not ask again. + const next = await measure('rule', run, { outDir, round: 'round-1', costPerTrialUsd: null }); + expect(next.status).toBe('measured'); + const pricey = await measure('rule', fake(() => ({ passed: true })), { costPerTrialUsd: 2.01 }); + expect(pricey.status).toBe('needs_approval'); + expect(pricey.reason).toContain('above $2/trial'); + }); + + test('the repair-round limit stops with a named red and never runs a trial', async () => { + const run = fake(() => ({ passed: true })); + const m = await measure('rule', run, { round: 'round-4' }); + expect(m).toMatchObject({ status: 'round_limit' }); + expect(run.calls).toHaveLength(0); + await expect(measure('rule', run, { round: 'round-x' })).rejects.toThrow('baseline or round-N'); + }); + + test('measurements are never overwritten and never land in the eval history dir', async () => { + const outDir = tmp('ship-measure-unique-'); + await measure('rule', fake(() => ({ passed: true })), { outDir }); + await expect(measure('rule', fake(() => ({ passed: true })), { outDir })).rejects.toThrow('never overwritten'); + const history = tmp('ship-measure-evals-'); + expect(() => assertDiagnosticDir(path.join(history, 'ship-measure'), history)).toThrow('never recorded as verdicts'); + await expect(measure('rule', fake(() => ({ passed: true })), { outDir: path.join(history, 'x'), evalDir: history })).rejects.toThrow('eval history dir'); + }); + + test('unmeasured skip is labeled in the report and never counts as a pass; the table shows estimated and actual spend', async () => { + const outDir = tmp('ship-measure-report-'); + await measure('rule', fake(trial => ({ passed: ![2, 5, 9].includes(trial), costUsd: 0.25 })), { outDir }); + await measure('rule', fake(() => ({ passed: true, costUsd: 0.25 })), { outDir, round: 'round-1', fix: 'sort the shuffled list in src/order.js' }); + recordUnmeasured(outDir, 'browse-basic', 'provider outage before the first turn'); + expect(() => recordUnmeasured(outDir, 'other', ' ')).toThrow('--reason'); + const report = formatReport(outDir); + expect(report).toContain('| seeded-case | rule | baseline | observed 7/10 | 9/10 | $5.00 | $2.50 | below target |'); + expect(report).toContain('| seeded-case | rule | round-1: after fix at sort the shuffled list in src/order.js | observed 10/10 | 9/10 | $5.00 | $2.50 | at or above target |'); + expect(report).toContain('| browse-basic | — | — | — | — | — | — | unmeasured (provider outage before the first turn); not a pass |'); + expect(report).toContain('never change a recorded verdict'); + }); +}); + +describe('ship-measure CLI with a documented single-case command', () => { + test('runs the command per trial (no shell), reads its cost and cause lines, and exits by target', () => { + const project = tmp('ship-measure-cli-'); + const state = tmp('ship-measure-cli-state-'); + fs.writeFileSync(path.join(project, 'evals.sh'), `#!/usr/bin/env bash +echo "case=$2 trial=$GSTACK_SHIP_MEASURE_TRIAL label=$GSTACK_SHIP_MEASURE_LABEL" +echo "cost_usd: 0.10" +if [ "$GSTACK_SHIP_MEASURE_TRIAL" = 2 ]; then echo "failure_cause: assertion"; echo "failure_detail: order.js:3 unsorted"; exit 1; fi +`, { mode: 0o755 }); + const cli = (...args: string[]) => spawnSync(process.execPath, ['run', path.join(ROOT, 'scripts/ship-measure.ts'), ...args], { + cwd: project, encoding: 'utf8', timeout: 60_000, env: { ...process.env, GSTACK_HOME: state, GSTACK_STATE_ROOT: state }, + }); + const out = path.join(project, '.context', 'ship-measure'); + const red = cli('measure', '--case', 'flaky-sort', '--command', './evals.sh case {case}', '--cost-per-trial', '0.1', '--jobs', '3'); + expect(red.stdout).toContain('| rule | 10 | at least 9 of 10 trials |'); + expect(red.stdout).toContain('DIAGNOSTIC flaky-sort baseline: observed 9/10 trials (baseline); at or above target'); + expect(red.status).toBe(0); + const trial2 = JSON.parse(fs.readFileSync(path.join(out, 'flaky-sort', 'baseline', 't02', 'trial.json'), 'utf8')); + expect(trial2).toMatchObject({ passed: false, costUsd: 0.1, failureCause: 'assertion', failureDetail: 'order.js:3 unsorted', label: 'diagnostic' }); + expect(fs.readFileSync(path.join(out, 'flaky-sort', 'baseline', 't02', 'output.log'), 'utf8')).toContain('case=flaky-sort trial=2 label=diagnostic'); + const ask = cli('measure', '--case', 'flaky-sort', '--round', 'round-1', '--command', './evals.sh case {case}', '--cost-per-trial', '5', '--out', path.join(project, 'other')); + expect(ask.status).toBe(2); + expect(ask.stdout).toContain('NEEDS APPROVAL'); + const limit = cli('measure', '--case', 'flaky-sort', '--round', 'round-4', '--command', './evals.sh case {case}', '--cost-per-trial', '0.1'); + expect(limit.status).toBe(3); + expect(limit.stdout).toContain('NAMED RED'); + const missing = cli('measure', '--case', 'flaky-sort'); + expect(missing.status).toBe(4); + expect(missing.stderr).toContain('No single-case eval command'); + expect(cli('skip', '--case', 'browse-basic', '--reason', 'runner lost').status).toBe(0); + expect(cli('report').stdout).toContain('unmeasured (runner lost); not a pass'); + }); +}); + +describe('free-suite shard reruns (CEO-1)', () => { + const outcome = (files: string[], status: FreeShardOutcome['status'], failing: string[] = []): FreeShardOutcome => ({ + shard: 1, files, status, exitCode: status === 'passed' ? 0 : 1, elapsedMs: 5, groupPid: null, failingFiles: failing, unattributedFailures: 0, + } as FreeShardOutcome); + + test('reruns the exact file list with the flaky retry off, fresh state per rerun, baseline alone, and records every outcome', async () => { + const files = ['test/a.test.ts', 'test/b.test.ts']; + const seen: Array<{ files: string[]; shard: number; options: RunFreeShardOptions; concurrentAtStart: number }> = []; + let active = 0; + const runShard = async (shardFiles: string[], shard: number, _total: number, options: RunFreeShardOptions = {}) => { + seen.push({ files: shardFiles, shard, options, concurrentAtStart: ++active }); + await Bun.sleep(5); + active -= 1; + return outcome(shardFiles, [3, 7].includes(shard) ? 'failed' : 'passed', [3, 7].includes(shard) ? ['test/b.test.ts'] : []); + }; + const m = await measureFreeShard({ files, reruns: 10, concurrency: 1, cores: 3, wallCapMs: 60_000, outDir: tmp('ship-measure-free-'), runShard, log: () => {} }); + expect(m).toMatchObject({ label: 'diagnostic', verdict: null, reruns: 10, completed: 10, passed: 8, parallel: 3, capHit: false }); + expect(m.outcomes.map(o => o.status)).toEqual(['passed', 'passed', 'failed', 'passed', 'passed', 'passed', 'failed', 'passed', 'passed', 'passed']); + expect(m.outcomes[2]!.failingFiles).toEqual(['test/b.test.ts']); + expect(m.outcomes[0]!.mode).toBe('baseline'); + expect(m.outcomes.slice(1).every(o => o.mode === 'parallel')).toBe(true); + const first = seen.find(s => s.shard === 1)!; + expect(first.concurrentAtStart).toBe(1); + expect(Math.max(...seen.map(s => s.concurrentAtStart))).toBeLessThanOrEqual(3); + for (const s of seen) { + expect(s.files).toEqual(files); + expect(s.options.env?.GSTACK_FREE_RETRY_FLAKY).toBe('0'); + expect(s.options.homeGuard?.name).toBe('privateFreeHome'); + } + const roots = new Set(seen.map(s => s.options.env?.GSTACK_STATE_ROOT)); + expect(roots.size).toBe(10); + expect(new Set(seen.map(s => s.options.logFilePath)).size).toBe(10); + const saved = JSON.parse(fs.readFileSync(path.join(path.dirname(path.dirname(m.outcomes[0]!.log)), 'free-measurement.json'), 'utf8')); + expect(saved).toMatchObject({ label: 'diagnostic', verdict: null, passed: 8, completed: 10 }); + }); + + test('the wall cap reports a partial count and never starts a rerun after it', async () => { + const runShard = async (files: string[], _shard: number, _total: number, options: RunFreeShardOptions = {}) => { + const wait = Math.min(options.wallTimeoutMs ?? 40, 40); + await Bun.sleep(wait); + return outcome(files, wait < 40 ? 'timed-out' : 'passed'); + }; + const lines: string[] = []; + const m = await measureFreeShard({ files: ['test/a.test.ts'], reruns: 10, concurrency: 1, cores: 1, wallCapMs: 150, outDir: tmp('ship-measure-cap-'), runShard, log: line => lines.push(line) }); + expect(m.capHit).toBe(true); + expect(m.completed).toBeGreaterThan(0); + expect(m.completed).toBeLessThan(10); + expect(m.outcomes.length).toBeLessThan(10); + expect(lines.join('\n')).toContain(`${m.completed} of 10 completed (wall cap 0s)`); + }); + + test('parallelism is max(1, floor(cores / shard concurrency))', () => { + expect(freeParallelism(16, 1)).toBe(16); + expect(freeParallelism(16, 3)).toBe(5); + expect(freeParallelism(2, 4)).toBe(1); + }); + + test('the Ubicloud diagnostic launcher turns the flaky retry off and runs ship-measure free; the normal lane keeps it on', () => { + const dir = tmp('ship-measure-ubi-'); + fs.mkdirSync(path.join(dir, 'scripts', 'ubicloud'), { recursive: true }); + fs.copyFileSync(path.join(ROOT, 'scripts/ubicloud/test-free.sh'), path.join(dir, 'scripts/ubicloud/test-free.sh')); + fs.writeFileSync(path.join(dir, 'scripts/ubicloud/ubi-runner.sh'), '#!/usr/bin/env bash\nprintf "%s\\n" "$@"\n', { mode: 0o755 }); + const launch = (...args: string[]) => spawnSync('bash', [path.join(dir, 'scripts/ubicloud/test-free.sh'), ...args], { encoding: 'utf8', timeout: 10_000 }); + const diagnostic = launch('--diagnostic', '--files', 'test/a.test.ts,test/b.test.ts', '--reruns', '10'); + expect(diagnostic.status).toBe(0); + const args = diagnostic.stdout.split('\n'); + expect(args).toContain('GSTACK_FREE_RETRY_FLAKY=0'); + expect(args).not.toContain('GSTACK_FREE_RETRY_FLAKY=1'); + expect(args).toContain('xvfb-run -a bun run scripts/ship-measure.ts free --backend local --files test/a.test.ts,test/b.test.ts --reruns 10'); + expect(args.some(a => a.startsWith('work/') && a.includes('/.context/ship-measure:'))).toBe(true); + const normal = launch().stdout.split('\n'); + expect(normal).toContain('GSTACK_FREE_RETRY_FLAKY=1'); + expect(normal).toContain('xvfb-run -a bun run test:free '); + }); +});