mirror of
https://github.com/garrytan/gstack.git
synced 2026-10-02 17:40:02 +02:00
garrytan
d6b3559b78
scripts/eval-flake-rank.ts becomes eval:pass-rates (eval:flake-rank stays an alias, and the legacy aggregate stays exported). It reads eval-store's trial-outcomes JSONL from the last N completed evals-periodic runs on this branch and main (gh, downloading only the trial-outcomes artifact, cached and size-capped, parsed as data), plus local eval dirs, and prints per-case per-trial pass rates with 95% Wilson intervals. A series is a case's own touchfiles minus GLOBAL_TOUCHFILES (caseSeriesIdentities, for the report job to stamp), per model, CLI version and policy version. Labels: INCONCLUSIVE, BROKEN, FLAKY, FAILING, PASSING. --backfill imports legacy slice artifacts as pre-policy trials (first attempt only, attributed by registry id, never guessed) for display only. --gate fails with ACTION REQUIRED on post-policy evidence only: drift below the quarantine entry rule, a rule case behaving like behavior, a one-sided Fisher drop against the previous identity (Holm-controlled), and quarantine entries that met their exit rule, expired after 8 weekly runs, broke the 10% tier cap or are invalid. CASE_QUARANTINE entries now carry a failureClass (detector, harness or model-latency); a product defect has no class and is never quarantined. The policy test pins EVAL_POLICY's approved constants.
126 lines
6.9 KiB
TypeScript
126 lines
6.9 KiB
TypeScript
/**
|
|
* Periodic-lane exclusions — LITERALS ONLY (own file, deliberately NOT in
|
|
* touchfiles-data.ts: that file is evaluated standalone by map-diff against
|
|
* old git versions, and its contract must not grow unrelated exports).
|
|
*
|
|
* The weekly periodic CI lane runs EVERY periodic-tier file (EVALS_ALL=1) so
|
|
* tests can't rot invisibly — the coverage contract. A file lands here only
|
|
* when running it weekly is KNOWN waste (documented-red or requires manual
|
|
* hardware), and every entry must carry a tracking pointer with a re-entry
|
|
* condition, so an exclusion is a decision with an owner, not a place tests
|
|
* go to die. Pinned by test/periodic-exclude-policy.test.ts: entries must
|
|
* name real files and carry non-empty reason + tracking.
|
|
*
|
|
* Removing an entry re-activates the file on the next weekly run — that IS
|
|
* the re-entry mechanism.
|
|
*/
|
|
export const PERIODIC_CI_EXCLUDE: Record<string, { reason: string; tracking: string }> = {
|
|
'test/codex-e2e.test.ts': {
|
|
reason: 'the codex CLI is not installed in the CI image (Dockerfile.ci ships only claude-code); every case self-skips',
|
|
tracking: 'TODOS.md "CI-unrunnable paid evals" (re-entry: the CLI/device is available in the CI image; review by 2026-12-28)',
|
|
},
|
|
'test/codex-e2e-sol-scope.test.ts': {
|
|
reason: 'the codex CLI is not installed in the CI image (Dockerfile.ci ships only claude-code); every case self-skips',
|
|
tracking: 'TODOS.md "CI-unrunnable paid evals" (re-entry: the CLI/device is available in the CI image; review by 2026-12-28)',
|
|
},
|
|
'test/codex-e2e-shared-libs.test.ts': {
|
|
reason: 'the codex CLI is not installed in the CI image (Dockerfile.ci ships only claude-code); every case self-skips',
|
|
tracking: 'TODOS.md "CI-unrunnable paid evals" (re-entry: the CLI/device is available in the CI image; review by 2026-12-28)',
|
|
},
|
|
'test/codex-e2e-recommendation-substance.test.ts': {
|
|
reason: 'the codex CLI is not installed in the CI image (Dockerfile.ci ships only claude-code); every case self-skips',
|
|
tracking: 'TODOS.md "CI-unrunnable paid evals" (re-entry: the CLI/device is available in the CI image; review by 2026-12-28)',
|
|
},
|
|
'test/skill-e2e-outside-voice.test.ts': {
|
|
reason: 'needs both the claude and codex CLIs; codex is not in the CI image, so every case self-skips',
|
|
tracking: 'TODOS.md "CI-unrunnable paid evals" (re-entry: the CLI/device is available in the CI image; review by 2026-12-28)',
|
|
},
|
|
'test/skill-e2e-aside.test.ts': {
|
|
reason: 'needs macOS with the Aside app open (asideAvailable()); CI runners are Linux, so every case self-skips',
|
|
tracking: 'TODOS.md "CI-unrunnable paid evals" (re-entry: the CLI/device is available in the CI image; review by 2026-12-28)',
|
|
},
|
|
'test/skill-e2e-ios-device.test.ts': {
|
|
reason: 'needs a physical iPhone over USB/devicectl — manual hardware, not a CI runner capability',
|
|
tracking: 'TODOS.md "CI-unrunnable paid evals" (re-entry: the CLI/device is available in the CI image; review by 2026-12-28)',
|
|
},
|
|
};
|
|
|
|
/**
|
|
* Case-level exclusions for case-sharded files (`<file>#<case id>`), same
|
|
* contract as above: a case lands here only when a CI runner cannot execute it
|
|
* (it self-skips), with reason + tracking. The planner records each as an
|
|
* excluded manifest entry instead of an empty case shard, so the exact
|
|
* one-case check stays strict for every planned case. Pinned by
|
|
* test/periodic-exclude-policy.test.ts.
|
|
*/
|
|
export const CASE_CI_EXCLUDE: Record<string, { reason: string; tracking: string }> = {
|
|
'test/skill-e2e-design.test.ts#design-review-fix': {
|
|
reason: '/design-review drives the Aside browser; CI runners are Linux without Aside, so the case registers test.skip("needs Aside")',
|
|
tracking: 'TODOS.md "CI-unrunnable paid evals" (re-entry: the CLI/device is available in the CI image; review by 2026-12-28)',
|
|
},
|
|
};
|
|
|
|
/**
|
|
* Paid-eval verdict policy, pre-registered (approved 2026-09-29). Frozen before
|
|
* the census: any change after seeing census results needs Garry's
|
|
* re-approval and a fresh census, and bumps `version` (every trial record
|
|
* carries it as policy_version, so pass-rate history segments at the change).
|
|
* panel - behavior cases and quarantined cases run n independent
|
|
* trials; a behavior panel PASSES at >= k passing trials with
|
|
* no contract violation. Rule and judge cases run one trial.
|
|
* quarantine - entry below `entry.rate` per trial over >= `entry.minTrials`
|
|
* new-policy trials; exit at >= `exit.rate` over >=
|
|
* `exit.minTrials`; at most `capFraction` of each tier's
|
|
* blocking cases; an entry expires after `expiryWeeklyRuns`.
|
|
* judge - a judge case draws `samples` independent samples of one
|
|
* prompt concurrently; numeric dimensions gate on the panel
|
|
* mean against the unchanged threshold, booleans on a strict
|
|
* majority; an erroring sample fails the panel, never resampled.
|
|
* drift - one-sided Fisher exact alarm between input-identity series
|
|
* (Holm-controlled across the cases tested in one report).
|
|
* infraRedispatch - a census whose every red verdict is machine-classified
|
|
* INFRA or INCOMPLETE may be re-dispatched this many times as
|
|
* a new run; both runs are reported.
|
|
*/
|
|
export const EVAL_POLICY = {
|
|
version: 1,
|
|
panel: { n: 3, k: 2 },
|
|
quarantine: {
|
|
entry: { rate: 0.95, minTrials: 10 },
|
|
exit: { rate: 0.97, minTrials: 10 },
|
|
capFraction: 0.10,
|
|
expiryWeeklyRuns: 8,
|
|
},
|
|
judge: { samples: 3 },
|
|
drift: { fisherAlpha: 0.05, fisherMinPerSide: 6 },
|
|
infraRedispatch: 1,
|
|
} as const;
|
|
|
|
/**
|
|
* Quarantined paid cases, keyed by registry id (an E2E_TIERS key). A
|
|
* quarantined case still runs its full panel and reports in every lane, but
|
|
* its failed verdict cannot fail the lane unless the panel is a hard break
|
|
* (0 of n) or a trial violated a contract; it never counts as passing
|
|
* coverage. An entry needs the entry rule met on the current input identity,
|
|
* a written diagnosis that the failures are detector, harness or model-latency
|
|
* failures (a product defect is never quarantined), and unchanged case
|
|
* touchfiles in the change that adds it. Pinned by
|
|
* test/periodic-exclude-policy.test.ts.
|
|
* reason - the written diagnosis, with the pass-rate evidence
|
|
* failureClass - what the diagnosis found; a product defect has no class here
|
|
* tracking - issue or TODOS pointer
|
|
* owner - who removes it
|
|
* enteredAt - YYYY-MM-DD the entry landed (expiry counts weekly runs from here)
|
|
* exit - the measurable exit condition
|
|
* At most EVAL_POLICY.quarantine.capFraction of a tier's cases (gate and
|
|
* periodic are the blocking tiers) may be quarantined at once.
|
|
*/
|
|
export const CASE_QUARANTINE: Record<string, {
|
|
reason: string;
|
|
failureClass: 'detector' | 'harness' | 'model-latency';
|
|
tracking: string;
|
|
owner: string;
|
|
enteredAt: string;
|
|
exit: string;
|
|
}> = {};
|