mirror of
https://github.com/garrytan/gstack.git
synced 2026-10-02 17:40:02 +02:00
test(evals): add E2E_KINDS, BEHAVIOR_WHY, EVAL_POLICY and CASE_QUARANTINE skeletons
Every E2E_TIERS and LLM_JUDGE_TOUCHFILES key starts as 'rule'; BEHAVIOR_WHY and CASE_QUARANTINE start empty. EVAL_POLICY pre-registers the approved panel (3, majority 2), quarantine entry 0.95/10 and exit 0.97/10, 10% cap, 8-weekly-run expiry, Fisher drift alarm and one INFRA re-dispatch.
This commit is contained in:
garrytan committed
1 parent
4c5fc0e2ce
commit
3c441fe311
2 files changed
+312
No files matched your search
@@ -59,3 +59,57 @@ export const CASE_CI_EXCLUDE: Record<string, { reason: string; tracking: string
|
||||
tracking: 'TODOS.md "CI-unrunnable paid evals" (re-entry: the CLI/device is available in the CI image; review by 2026-12-28)',
|
||||
},
|
||||
};
|
||||
|
||||
/**
|
||||
* Paid-eval verdict policy, pre-registered (approved 2026-09-29). Frozen before
|
||||
* the census: any change after seeing census results needs Garry's
|
||||
* re-approval and a fresh census, and bumps `version` (every trial record
|
||||
* carries it as policy_version, so pass-rate history segments at the change).
|
||||
* panel - behavior cases and quarantined cases run n independent
|
||||
* trials; a behavior panel PASSES at >= k passing trials with
|
||||
* no contract violation. Rule and judge cases run one trial.
|
||||
* quarantine - entry below `entry.rate` per trial over >= `entry.minTrials`
|
||||
* new-policy trials; exit at >= `exit.rate` over >=
|
||||
* `exit.minTrials`; at most `capFraction` of each tier's
|
||||
* blocking cases; an entry expires after `expiryWeeklyRuns`.
|
||||
* drift - one-sided Fisher exact alarm between input-identity series.
|
||||
* infraRedispatch - a census whose every red verdict is machine-classified
|
||||
* INFRA or INCOMPLETE may be re-dispatched this many times as
|
||||
* a new run; both runs are reported.
|
||||
*/
|
||||
export const EVAL_POLICY = {
|
||||
version: 1,
|
||||
panel: { n: 3, k: 2 },
|
||||
quarantine: {
|
||||
entry: { rate: 0.95, minTrials: 10 },
|
||||
exit: { rate: 0.97, minTrials: 10 },
|
||||
capFraction: 0.10,
|
||||
expiryWeeklyRuns: 8,
|
||||
},
|
||||
drift: { fisherAlpha: 0.05, fisherMinPerSide: 6 },
|
||||
infraRedispatch: 1,
|
||||
} as const;
|
||||
|
||||
/**
|
||||
* Quarantined paid cases, keyed by registry id (an E2E_TIERS key). A
|
||||
* quarantined case still runs its full panel and reports in every lane, but
|
||||
* its failed verdict cannot fail the lane unless the panel is a hard break
|
||||
* (0 of n) or a trial violated a contract; it never counts as passing
|
||||
* coverage. An entry needs the entry rule met on the current input identity,
|
||||
* a written diagnosis that the failures are detector, harness or model-latency
|
||||
* failures (a product defect is never quarantined), and unchanged case
|
||||
* touchfiles in the change that adds it. Pinned by
|
||||
* test/periodic-exclude-policy.test.ts.
|
||||
* reason - the written diagnosis
|
||||
* tracking - issue or TODOS pointer
|
||||
* owner - who removes it
|
||||
* enteredAt - ISO date the entry landed (expiry counts weekly runs from here)
|
||||
* exit - the measurable exit condition
|
||||
*/
|
||||
export const CASE_QUARANTINE: Record<string, {
|
||||
reason: string;
|
||||
tracking: string;
|
||||
owner: string;
|
||||
enteredAt: string;
|
||||
exit: string;
|
||||
}> = {};
|
||||
@@ -1573,3 +1573,261 @@ export const GLOBAL_TOUCHFILES = [
|
||||
// diffed per key, so a data-only edit runs just the affected tests.
|
||||
// Map-diff fails CLOSED — any error on that path still runs everything.
|
||||
];
|
||||
|
||||
/**
|
||||
* Eval kind per live case (every E2E_TIERS and LLM_JUDGE_TOUCHFILES key).
|
||||
* The kind fixes the trial policy before the run (EVAL_POLICY in
|
||||
* periodic-exclude-data.ts):
|
||||
* rule - one trial; any failed assertion fails the verdict. The default.
|
||||
* behavior - a panel of independent trials, PASS at the policy majority;
|
||||
* needs a BEHAVIOR_WHY entry naming the tolerated deviation.
|
||||
* judge - an LLM-judge score of a static input, sampled as a panel.
|
||||
* Reclassification is a reviewed diff, never a runtime switch.
|
||||
*/
|
||||
export const E2E_KINDS: Record<string, 'rule' | 'behavior' | 'judge'> = {
|
||||
'ship-skipped-queued-finding': 'rule',
|
||||
'investigate-owned-completion': 'rule',
|
||||
'investigate-owned-abort': 'rule',
|
||||
'investigate-owned-ending-error': 'rule',
|
||||
'shared-libs-review-path-eligibility': 'rule',
|
||||
'shared-libs-review-index-flags': 'rule',
|
||||
'shared-libs-review-prior-coverage': 'rule',
|
||||
'shared-libs-codex-read-only': 'rule',
|
||||
'shared-libs-read-only': 'rule',
|
||||
'shared-libs-unsupported-git': 'rule',
|
||||
'shared-libs-review-lifecycle': 'rule',
|
||||
'shared-libs-review-revalidation': 'rule',
|
||||
'shared-libs-opportunity-judgment': 'rule',
|
||||
'shared-libs-pr-coverage': 'rule',
|
||||
'shared-libs-plan-callers': 'rule',
|
||||
'browse-basic': 'rule',
|
||||
'browse-snapshot': 'rule',
|
||||
'aside-browse-basic': 'rule',
|
||||
'aside-browse-flow': 'rule',
|
||||
'aside-qa-quick': 'rule',
|
||||
'aside-scrape-json': 'rule',
|
||||
'aside-canary-quick': 'rule',
|
||||
'hermetic-canary': 'rule',
|
||||
'hermetic-sentinel': 'rule',
|
||||
'skillmd-setup-discovery': 'rule',
|
||||
'skillmd-no-local-binary': 'rule',
|
||||
'skillmd-outside-git': 'rule',
|
||||
'session-awareness': 'rule',
|
||||
'operational-learning': 'rule',
|
||||
'first-task-scaffold': 'rule',
|
||||
'qa-quick': 'rule',
|
||||
'qa-b6-static': 'rule',
|
||||
'qa-b7-spa': 'rule',
|
||||
'qa-b8-checkout': 'rule',
|
||||
'qa-only-no-fix': 'rule',
|
||||
'qa-fix-loop': 'rule',
|
||||
'qa-bootstrap': 'rule',
|
||||
'review-exploratory-small-cli': 'rule',
|
||||
'ship-exploratory-small-cli': 'rule',
|
||||
'ship-exploratory-unavailable': 'rule',
|
||||
'ship-exploratory-plan-checks': 'rule',
|
||||
'ship-exploratory-late-input': 'rule',
|
||||
'qa-functional-cli-report': 'rule',
|
||||
'qa-functional-webhook-report': 'rule',
|
||||
'qa-functional-cli-fix': 'rule',
|
||||
'qa-functional-webhook-fix': 'rule',
|
||||
'review-sql-injection': 'rule',
|
||||
'review-enum-completeness': 'rule',
|
||||
'review-base-branch': 'rule',
|
||||
'review-design-lite': 'rule',
|
||||
'review-coverage-audit': 'rule',
|
||||
'review-dashboard-via': 'rule',
|
||||
'review-army-migration-safety': 'rule',
|
||||
'review-army-perf-n-plus-one': 'rule',
|
||||
'review-army-delivery-audit': 'rule',
|
||||
'review-army-quality-score': 'rule',
|
||||
'review-army-json-findings': 'rule',
|
||||
'review-army-red-team': 'rule',
|
||||
'review-army-consensus': 'rule',
|
||||
'review-army-simplification': 'rule',
|
||||
'review-army-simplification-precision': 'rule',
|
||||
'office-hours-spec-review': 'rule',
|
||||
'office-hours-brain-writeback': 'rule',
|
||||
'gbrain-roundtrip-local': 'rule',
|
||||
'sync-gbrain-read-ready': 'rule',
|
||||
'sync-gbrain-read-unknown': 'rule',
|
||||
'office-hours-forcing-energy': 'rule',
|
||||
'office-hours-builder-wildness': 'rule',
|
||||
'plan-ceo-review': 'rule',
|
||||
'plan-ceo-review-selective': 'rule',
|
||||
'plan-ceo-review-benefits': 'rule',
|
||||
'plan-ceo-review-expansion-energy': 'rule',
|
||||
'plan-eng-review': 'rule',
|
||||
'plan-eng-review-artifact': 'rule',
|
||||
'plan-eng-coverage-audit': 'rule',
|
||||
'plan-review-report': 'rule',
|
||||
'plan-ceo-review-plan-mode': 'rule',
|
||||
'plan-eng-review-plan-mode': 'rule',
|
||||
'plan-design-review-plan-mode': 'rule',
|
||||
'plan-devex-review-plan-mode': 'rule',
|
||||
'plan-mode-no-op': 'rule',
|
||||
'office-hours-auto-mode': 'rule',
|
||||
'auto-decide-preserved': 'rule',
|
||||
'auq-format-gate': 'rule',
|
||||
'plan-ceo-mode-routing': 'rule',
|
||||
'plan-design-with-ui-scope': 'rule',
|
||||
'tpa-present': 'rule',
|
||||
'tpa-absent-linux': 'rule',
|
||||
'tpa-broken': 'rule',
|
||||
'tpa-absent-darwin': 'rule',
|
||||
'tpa-apple-ban': 'rule',
|
||||
'ship-section-loading': 'rule',
|
||||
'plan-ceo-section-loading': 'rule',
|
||||
'carve-section-loading': 'rule',
|
||||
'plan-eng-finding-floor': 'rule',
|
||||
'plan-ceo-finding-floor': 'rule',
|
||||
'plan-design-finding-floor': 'rule',
|
||||
'plan-devex-finding-floor': 'rule',
|
||||
'plan-eng-multi-finding-batching': 'rule',
|
||||
'plan-ceo-split-overflow': 'rule',
|
||||
'setup-gbrain-remote': 'rule',
|
||||
'setup-gbrain-bad-token': 'rule',
|
||||
'setup-gbrain-path4-local-pglite': 'rule',
|
||||
'plan-ceo-review-format-mode': 'rule',
|
||||
'plan-ceo-review-format-approach': 'rule',
|
||||
'plan-eng-review-format-coverage': 'rule',
|
||||
'plan-eng-review-format-kind': 'rule',
|
||||
'office-hours-phase4-fork': 'rule',
|
||||
'llm-judge-recommendation': 'rule',
|
||||
'plan-ceo-review-prosons-cadence': 'rule',
|
||||
'plan-review-prosons-format': 'rule',
|
||||
'plan-review-prosons-hardstop-neg': 'rule',
|
||||
'plan-review-prosons-neutral-neg': 'rule',
|
||||
'plan-tune-inspect': 'rule',
|
||||
'codex-offered-office-hours': 'rule',
|
||||
'codex-offered-ceo-review': 'rule',
|
||||
'codex-offered-design-review': 'rule',
|
||||
'codex-offered-eng-review': 'rule',
|
||||
'timeline-event-flow': 'rule',
|
||||
'context-recovery-artifacts': 'rule',
|
||||
'context-save-writes-file': 'rule',
|
||||
'context-restore-loads-latest': 'rule',
|
||||
'context-save-routing': 'rule',
|
||||
'context-save-then-restore-roundtrip': 'rule',
|
||||
'context-restore-fragment-match': 'rule',
|
||||
'context-restore-empty-state': 'rule',
|
||||
'context-restore-list-delegates': 'rule',
|
||||
'context-restore-legacy-compat': 'rule',
|
||||
'context-save-list-current-branch': 'rule',
|
||||
'context-save-list-all-branches': 'rule',
|
||||
'ship-base-branch': 'rule',
|
||||
'ship-local-workflow': 'rule',
|
||||
'ship-managed-hook-refresh': 'rule',
|
||||
'ship-unmanaged-hook-consent': 'rule',
|
||||
'ship-local-hook-preservation': 'rule',
|
||||
'ship-coverage-audit': 'rule',
|
||||
'ship-triage': 'rule',
|
||||
'ship-docsync-missing-marker': 'rule',
|
||||
'ship-docsync-missing-asset': 'rule',
|
||||
'ship-docsync-launch-failure': 'rule',
|
||||
'ship-docsync-timeout-unsettled': 'rule',
|
||||
'ship-docsync-late-result': 'rule',
|
||||
'ship-docsync-stale-before': 'rule',
|
||||
'ship-docsync-stale-after': 'rule',
|
||||
'ship-docsync-recovery': 'rule',
|
||||
'ship-docsync-completion': 'rule',
|
||||
'ship-docsync-current': 'rule',
|
||||
'ship-docsync-failure': 'rule',
|
||||
'ship-docsync-store': 'rule',
|
||||
'docsync-spawned': 'rule',
|
||||
'retro': 'rule',
|
||||
'retro-base-branch': 'rule',
|
||||
'cso-full-audit': 'rule',
|
||||
'cso-diff-mode': 'rule',
|
||||
'cso-infra-scope': 'rule',
|
||||
'learnings-show': 'rule',
|
||||
'document-release': 'rule',
|
||||
'codex-review': 'rule',
|
||||
'codex-discover-skill': 'rule',
|
||||
'codex-review-findings': 'rule',
|
||||
'outside-voice-codex-to-claude-code': 'rule',
|
||||
'outside-voice-claude-code-to-codex': 'rule',
|
||||
'outside-plan-disabled-no-fallback': 'rule',
|
||||
'codex-sol-scope-termination': 'rule',
|
||||
'design-consultation-core': 'rule',
|
||||
'design-consultation-existing': 'rule',
|
||||
'design-consultation-research': 'rule',
|
||||
'design-consultation-preview': 'rule',
|
||||
'plan-design-review-no-ui-scope': 'rule',
|
||||
'design-review-fix': 'rule',
|
||||
'design-review-detector-shim': 'rule',
|
||||
'design-review-detector-shim-dom': 'rule',
|
||||
'design-review-plugin-handoff': 'rule',
|
||||
'design-html-slop-gate': 'rule',
|
||||
'diagram-triplet': 'rule',
|
||||
'diagram-authoring-quality': 'rule',
|
||||
'gstack-upgrade-happy-path': 'rule',
|
||||
'land-and-deploy-workflow': 'rule',
|
||||
'land-and-deploy-first-run': 'rule',
|
||||
'land-and-deploy-review-gate': 'rule',
|
||||
'canary-workflow': 'rule',
|
||||
'benchmark-workflow': 'rule',
|
||||
'setup-deploy-workflow': 'rule',
|
||||
'autoplan-dual-voice': 'rule',
|
||||
'benchmark-providers-live': 'rule',
|
||||
'scrape-match-path': 'rule',
|
||||
'scrape-prototype-path': 'rule',
|
||||
'skillify-happy-path': 'rule',
|
||||
'skillify-provenance-refusal': 'rule',
|
||||
'skillify-approval-reject': 'rule',
|
||||
'journey-ideation': 'rule',
|
||||
'journey-plan-eng': 'rule',
|
||||
'journey-debug': 'rule',
|
||||
'journey-qa': 'rule',
|
||||
'journey-code-review': 'rule',
|
||||
'journey-ship': 'rule',
|
||||
'journey-docs': 'rule',
|
||||
'journey-retro': 'rule',
|
||||
'journey-design-system': 'rule',
|
||||
'journey-visual-qa': 'rule',
|
||||
'ios-qa-device': 'rule',
|
||||
'arm-benchmark-native-overbuild': 'rule',
|
||||
'arm-benchmark-crud-endpoint': 'rule',
|
||||
'arm-benchmark-bugfix-decoys': 'rule',
|
||||
'office-hours-section-loading': 'rule',
|
||||
'office-hours-design-draft': 'rule',
|
||||
'plan-decision-classification': 'rule',
|
||||
'plan-devex-peer-comparison-classification': 'rule',
|
||||
'health-reporting': 'rule',
|
||||
'overlay-harness-claude-dedicated-tools-vs-bash': 'rule',
|
||||
'overlay-harness-opus-4-7-effort-match-trivial': 'rule',
|
||||
'overlay-harness-opus-4-7-literal-interpretation': 'rule',
|
||||
'overlay-harness-claude-dedicated-tools-vs-bash-sonnet': 'rule',
|
||||
'journey-negatives': 'rule',
|
||||
'review/SKILL.md workflow': 'rule',
|
||||
'setup-browser-cookies/SKILL.md workflow': 'rule',
|
||||
'browse/SKILL.md reference': 'rule',
|
||||
'setup block': 'rule',
|
||||
'qa/SKILL.md workflow': 'rule',
|
||||
'qa/SKILL.md health rubric': 'rule',
|
||||
'qa/SKILL.md anti-refusal': 'rule',
|
||||
'cross-skill greptile consistency': 'rule',
|
||||
'ship/SKILL.md workflow': 'rule',
|
||||
'document-release/SKILL.md workflow': 'rule',
|
||||
'plan-ceo-review/SKILL.md modes': 'rule',
|
||||
'plan-eng-review/SKILL.md sections': 'rule',
|
||||
'plan-design-review/SKILL.md passes': 'rule',
|
||||
'design-review/SKILL.md fix loop': 'rule',
|
||||
'design-consultation/SKILL.md research': 'rule',
|
||||
'land-and-deploy/SKILL.md workflow': 'rule',
|
||||
'canary/SKILL.md monitoring loop': 'rule',
|
||||
'benchmark/SKILL.md perf collection': 'rule',
|
||||
'setup-deploy/SKILL.md platform setup': 'rule',
|
||||
'retro/SKILL.md instructions': 'rule',
|
||||
'qa-only/SKILL.md workflow': 'rule',
|
||||
'gstack-upgrade/SKILL.md upgrade flow': 'rule',
|
||||
'sync-gbrain/SKILL.md read-only readiness': 'rule',
|
||||
'voice directive tone': 'rule',
|
||||
};
|
||||
|
||||
/**
|
||||
* One-line tolerance for every behavior-kind case: why an occasional
|
||||
* deviation is acceptable product behavior. Keys equal the behavior ids of
|
||||
* E2E_KINDS; values are non-empty.
|
||||
*/
|
||||
export const BEHAVIOR_WHY: Record<string, string> = {};
|
||||
Reference in new issue
Block a user