/** * Periodic-lane exclusions — LITERALS ONLY (own file, deliberately NOT in * touchfiles-data.ts: that file is evaluated standalone by map-diff against * old git versions, and its contract must not grow unrelated exports). * * The weekly periodic CI lane runs EVERY periodic-tier file (EVALS_ALL=1) so * tests can't rot invisibly — the coverage contract. A file lands here only * when running it weekly is KNOWN waste (documented-red or requires manual * hardware), and every entry must carry a tracking pointer with a re-entry * condition, so an exclusion is a decision with an owner, not a place tests * go to die. Pinned by test/periodic-exclude-policy.test.ts: entries must * name real files and carry non-empty reason + tracking. * * Removing an entry re-activates the file on the next weekly run — that IS * the re-entry mechanism. */ export const PERIODIC_CI_EXCLUDE: Record = { 'test/codex-e2e.test.ts': { reason: 'the codex CLI is not installed in the CI image (Dockerfile.ci ships only claude-code); every case self-skips', tracking: 'TODOS.md "CI-unrunnable paid evals" (re-entry: the CLI/device is available in the CI image; review by 2026-12-28)', }, 'test/codex-e2e-sol-scope.test.ts': { reason: 'the codex CLI is not installed in the CI image (Dockerfile.ci ships only claude-code); every case self-skips', tracking: 'TODOS.md "CI-unrunnable paid evals" (re-entry: the CLI/device is available in the CI image; review by 2026-12-28)', }, 'test/codex-e2e-shared-libs.test.ts': { reason: 'the codex CLI is not installed in the CI image (Dockerfile.ci ships only claude-code); every case self-skips', tracking: 'TODOS.md "CI-unrunnable paid evals" (re-entry: the CLI/device is available in the CI image; review by 2026-12-28)', }, 'test/codex-e2e-recommendation-substance.test.ts': { reason: 'the codex CLI is not installed in the CI image (Dockerfile.ci ships only claude-code); every case self-skips', tracking: 'TODOS.md "CI-unrunnable paid evals" (re-entry: the CLI/device is available in the CI image; review by 2026-12-28)', }, 'test/skill-e2e-outside-voice.test.ts': { reason: 'needs both the claude and codex CLIs; codex is not in the CI image, so every case self-skips', tracking: 'TODOS.md "CI-unrunnable paid evals" (re-entry: the CLI/device is available in the CI image; review by 2026-12-28)', }, 'test/skill-e2e-aside.test.ts': { reason: 'needs macOS with the Aside app open (asideAvailable()); CI runners are Linux, so every case self-skips', tracking: 'TODOS.md "CI-unrunnable paid evals" (re-entry: the CLI/device is available in the CI image; review by 2026-12-28)', }, 'test/skill-e2e-ios-device.test.ts': { reason: 'needs a physical iPhone over USB/devicectl — manual hardware, not a CI runner capability', tracking: 'TODOS.md "CI-unrunnable paid evals" (re-entry: the CLI/device is available in the CI image; review by 2026-12-28)', }, }; /** * Case-level exclusions for case-sharded files (`#`), same * contract as above: a case lands here only when a CI runner cannot execute it * (it self-skips), with reason + tracking. The planner records each as an * excluded manifest entry instead of an empty case shard, so the exact * one-case check stays strict for every planned case. Pinned by * test/periodic-exclude-policy.test.ts. */ export const CASE_CI_EXCLUDE: Record = { 'test/skill-e2e-design.test.ts#design-review-fix': { reason: '/design-review drives the Aside browser; CI runners are Linux without Aside, so the case registers test.skip("needs Aside")', tracking: 'TODOS.md "CI-unrunnable paid evals" (re-entry: the CLI/device is available in the CI image; review by 2026-12-28)', }, }; /** * Paid-eval verdict policy, pre-registered (approved 2026-09-29). Frozen before * the census: any change after seeing census results needs Garry's * re-approval and a fresh census, and bumps `version` (every trial record * carries it as policy_version, so pass-rate history segments at the change). * panel - behavior cases and quarantined cases run n independent * trials; a behavior panel PASSES at >= k passing trials with * no contract violation. Rule and judge cases run one trial. * quarantine - entry below `entry.rate` per trial over >= `entry.minTrials` * new-policy trials; exit at >= `exit.rate` over >= * `exit.minTrials`; at most `capFraction` of each tier's * blocking cases; an entry expires after `expiryWeeklyRuns`. * drift - one-sided Fisher exact alarm between input-identity series. * infraRedispatch - a census whose every red verdict is machine-classified * INFRA or INCOMPLETE may be re-dispatched this many times as * a new run; both runs are reported. */ export const EVAL_POLICY = { version: 1, panel: { n: 3, k: 2 }, quarantine: { entry: { rate: 0.95, minTrials: 10 }, exit: { rate: 0.97, minTrials: 10 }, capFraction: 0.10, expiryWeeklyRuns: 8, }, drift: { fisherAlpha: 0.05, fisherMinPerSide: 6 }, infraRedispatch: 1, } as const; /** * Quarantined paid cases, keyed by registry id (an E2E_TIERS key). A * quarantined case still runs its full panel and reports in every lane, but * its failed verdict cannot fail the lane unless the panel is a hard break * (0 of n) or a trial violated a contract; it never counts as passing * coverage. An entry needs the entry rule met on the current input identity, * a written diagnosis that the failures are detector, harness or model-latency * failures (a product defect is never quarantined), and unchanged case * touchfiles in the change that adds it. Pinned by * test/periodic-exclude-policy.test.ts. * reason - the written diagnosis * tracking - issue or TODOS pointer * owner - who removes it * enteredAt - ISO date the entry landed (expiry counts weekly runs from here) * exit - the measurable exit condition */ export const CASE_QUARANTINE: Record = {};