/** * Periodic-lane exclusions — LITERALS ONLY (own file, deliberately NOT in * touchfiles-data.ts: that file is evaluated standalone by map-diff against * old git versions, and its contract must not grow unrelated exports). * * The weekly periodic CI lane runs EVERY periodic-tier file (EVALS_ALL=1) so * tests can't rot invisibly — the coverage contract. A file lands here only * when running it weekly is KNOWN waste (documented-red or requires manual * hardware), and every entry must carry a tracking pointer with a re-entry * condition, so an exclusion is a decision with an owner, not a place tests * go to die. Pinned by test/periodic-exclude-policy.test.ts: entries must * name real files and carry non-empty reason + tracking. * * Removing an entry re-activates the file on the next weekly run — that IS * the re-entry mechanism. */ export const PERIODIC_CI_EXCLUDE: Record = { 'test/skill-e2e-aside.test.ts': { reason: 'needs macOS with the Aside app open (asideAvailable()); CI runners are Linux, so every case self-skips', tracking: 'TODOS.md "CI-unrunnable paid evals" (re-entry: the CLI/device is available in the CI image; review by 2026-12-28)', }, 'test/skill-e2e-ios-device.test.ts': { reason: 'needs a physical iPhone over USB/devicectl — manual hardware, not a CI runner capability', tracking: 'TODOS.md "CI-unrunnable paid evals" (re-entry: the CLI/device is available in the CI image; review by 2026-12-28)', }, }; /** * Case-level exclusions for case-sharded files (`#`), same * contract as above: a case lands here only when a CI runner cannot execute it * (it self-skips), with reason + tracking. The planner records each as an * excluded manifest entry instead of an empty case shard, so the exact * one-case check stays strict for every planned case. Pinned by * test/periodic-exclude-policy.test.ts. */ export const CASE_CI_EXCLUDE: Record = { 'test/skill-e2e-design.test.ts#design-review-fix': { reason: '/design-review drives the Aside browser; CI runners are Linux without Aside, so the case registers test.skip("needs Aside")', tracking: 'TODOS.md "CI-unrunnable paid evals" (re-entry: the CLI/device is available in the CI image; review by 2026-12-28)', }, }; /** * Paid-eval verdict policy, pre-registered (v1 approved 2026-09-29; v2 * approved 2026-10-04, D1 of docs/test-audit-2026-10.md; v3 approved * 2026-10-06, the judge median gate). Frozen before the * census: any change after seeing census results needs Garry's re-approval * and a fresh census, and bumps `version` (every trial record carries it as * policy_version; readers score only their own version, so pass-rate history * segments at the change). * v2 - series identity = the bytes a case owns (paid test file, * fixtures, its skills' prompt files) + HARNESS_VERSION * (scripts/harness-version.json), keyed with model and CLI * version; weekly history = scheduled main runs plus main * dispatches; only current-version trials count toward * quarantine entry. * v3 - the judge gate changes from the panel mean to the * per-dimension median of 3 samples; v2's identity and * history rules are unchanged, and v2 trials stop scoring. * panel - behavior cases and quarantined cases run n independent * trials; a behavior panel PASSES at >= k passing trials with * no contract violation. Rule and judge cases run one trial. * quarantine - entry below `entry.rate` per trial over >= `entry.minTrials` * new-policy trials; exit at >= `exit.rate` over >= * `exit.minTrials`; at most `capFraction` of each tier's * blocking cases; an entry expires after `expiryWeeklyRuns`. * judge - a judge case draws exactly `samples` (3) independent samples * of one prompt concurrently; a numeric dimension passes when * its median (at least 2 of 3 samples) meets the unchanged * threshold, and the mean is reported only; booleans gate on a * strict majority; any other sample count or an erroring sample * fails the panel, never resampled. * drift - one-sided Fisher exact alarm between input-identity series * (Holm-controlled across the cases tested in one report). * infraRedispatch - a census whose every red verdict is machine-classified * INFRA or INCOMPLETE may be re-dispatched this many times as * a new run; both runs are reported. * flaky rerun - (CREW rule 16, Garry 2026-10-19: "you can just run the * failed test again and note that it is flaky") the gate reruns * a red case verdict once at the same head with the same trials * and threshold; a pass is a flaky-annotated PASS, a second * failure is red. Trial records and this version are unchanged; * a rerun's trials stay in its own artifacts and flake record * (scripts/lib/paid-rerun.ts, docs/TESTING_INTERNALS.md#flaky-reruns). */ export const EVAL_POLICY = { version: 3, panel: { n: 3, k: 2 }, quarantine: { entry: { rate: 0.95, minTrials: 10 }, exit: { rate: 0.97, minTrials: 10 }, capFraction: 0.10, expiryWeeklyRuns: 8, }, judge: { samples: 3 }, drift: { fisherAlpha: 0.05, fisherMinPerSide: 6 }, infraRedispatch: 1, } as const; /** * Quarantined paid cases, keyed by registry id (an E2E_TIERS key). A * quarantined case still runs its full panel and reports in every lane, but * its failed verdict cannot fail the lane unless the panel is a hard break * (0 of n) or a trial violated a contract; it never counts as passing * coverage. An entry needs the entry rule met on the current input identity, * a written diagnosis that the failures are detector, harness or model-latency * failures (a product defect is never quarantined), and unchanged case * touchfiles in the change that adds it. Pinned by * test/periodic-exclude-policy.test.ts. * reason - the written diagnosis, with the pass-rate evidence * failureClass - what the diagnosis found; a product defect has no class here * tracking - issue or TODOS pointer * owner - who removes it * enteredAt - YYYY-MM-DD the entry landed (expiry counts weekly runs from here) * exit - the measurable exit condition * At most EVAL_POLICY.quarantine.capFraction of a tier's cases (gate and * periodic are the blocking tiers) may be quarantined at once. */ export const CASE_QUARANTINE: Record = {};