diff --git a/test/helpers/periodic-exclude-data.ts b/test/helpers/periodic-exclude-data.ts index 7538b4ccc..5959f44d7 100644 --- a/test/helpers/periodic-exclude-data.ts +++ b/test/helpers/periodic-exclude-data.ts @@ -59,3 +59,57 @@ export const CASE_CI_EXCLUDE: Record= k passing trials with + * no contract violation. Rule and judge cases run one trial. + * quarantine - entry below `entry.rate` per trial over >= `entry.minTrials` + * new-policy trials; exit at >= `exit.rate` over >= + * `exit.minTrials`; at most `capFraction` of each tier's + * blocking cases; an entry expires after `expiryWeeklyRuns`. + * drift - one-sided Fisher exact alarm between input-identity series. + * infraRedispatch - a census whose every red verdict is machine-classified + * INFRA or INCOMPLETE may be re-dispatched this many times as + * a new run; both runs are reported. + */ +export const EVAL_POLICY = { + version: 1, + panel: { n: 3, k: 2 }, + quarantine: { + entry: { rate: 0.95, minTrials: 10 }, + exit: { rate: 0.97, minTrials: 10 }, + capFraction: 0.10, + expiryWeeklyRuns: 8, + }, + drift: { fisherAlpha: 0.05, fisherMinPerSide: 6 }, + infraRedispatch: 1, +} as const; + +/** + * Quarantined paid cases, keyed by registry id (an E2E_TIERS key). A + * quarantined case still runs its full panel and reports in every lane, but + * its failed verdict cannot fail the lane unless the panel is a hard break + * (0 of n) or a trial violated a contract; it never counts as passing + * coverage. An entry needs the entry rule met on the current input identity, + * a written diagnosis that the failures are detector, harness or model-latency + * failures (a product defect is never quarantined), and unchanged case + * touchfiles in the change that adds it. Pinned by + * test/periodic-exclude-policy.test.ts. + * reason - the written diagnosis + * tracking - issue or TODOS pointer + * owner - who removes it + * enteredAt - ISO date the entry landed (expiry counts weekly runs from here) + * exit - the measurable exit condition + */ +export const CASE_QUARANTINE: Record = {}; diff --git a/test/helpers/touchfiles-data.ts b/test/helpers/touchfiles-data.ts index 5d9711d2c..18fc17f77 100644 --- a/test/helpers/touchfiles-data.ts +++ b/test/helpers/touchfiles-data.ts @@ -1573,3 +1573,261 @@ export const GLOBAL_TOUCHFILES = [ // diffed per key, so a data-only edit runs just the affected tests. // Map-diff fails CLOSED — any error on that path still runs everything. ]; + +/** + * Eval kind per live case (every E2E_TIERS and LLM_JUDGE_TOUCHFILES key). + * The kind fixes the trial policy before the run (EVAL_POLICY in + * periodic-exclude-data.ts): + * rule - one trial; any failed assertion fails the verdict. The default. + * behavior - a panel of independent trials, PASS at the policy majority; + * needs a BEHAVIOR_WHY entry naming the tolerated deviation. + * judge - an LLM-judge score of a static input, sampled as a panel. + * Reclassification is a reviewed diff, never a runtime switch. + */ +export const E2E_KINDS: Record = { + 'ship-skipped-queued-finding': 'rule', + 'investigate-owned-completion': 'rule', + 'investigate-owned-abort': 'rule', + 'investigate-owned-ending-error': 'rule', + 'shared-libs-review-path-eligibility': 'rule', + 'shared-libs-review-index-flags': 'rule', + 'shared-libs-review-prior-coverage': 'rule', + 'shared-libs-codex-read-only': 'rule', + 'shared-libs-read-only': 'rule', + 'shared-libs-unsupported-git': 'rule', + 'shared-libs-review-lifecycle': 'rule', + 'shared-libs-review-revalidation': 'rule', + 'shared-libs-opportunity-judgment': 'rule', + 'shared-libs-pr-coverage': 'rule', + 'shared-libs-plan-callers': 'rule', + 'browse-basic': 'rule', + 'browse-snapshot': 'rule', + 'aside-browse-basic': 'rule', + 'aside-browse-flow': 'rule', + 'aside-qa-quick': 'rule', + 'aside-scrape-json': 'rule', + 'aside-canary-quick': 'rule', + 'hermetic-canary': 'rule', + 'hermetic-sentinel': 'rule', + 'skillmd-setup-discovery': 'rule', + 'skillmd-no-local-binary': 'rule', + 'skillmd-outside-git': 'rule', + 'session-awareness': 'rule', + 'operational-learning': 'rule', + 'first-task-scaffold': 'rule', + 'qa-quick': 'rule', + 'qa-b6-static': 'rule', + 'qa-b7-spa': 'rule', + 'qa-b8-checkout': 'rule', + 'qa-only-no-fix': 'rule', + 'qa-fix-loop': 'rule', + 'qa-bootstrap': 'rule', + 'review-exploratory-small-cli': 'rule', + 'ship-exploratory-small-cli': 'rule', + 'ship-exploratory-unavailable': 'rule', + 'ship-exploratory-plan-checks': 'rule', + 'ship-exploratory-late-input': 'rule', + 'qa-functional-cli-report': 'rule', + 'qa-functional-webhook-report': 'rule', + 'qa-functional-cli-fix': 'rule', + 'qa-functional-webhook-fix': 'rule', + 'review-sql-injection': 'rule', + 'review-enum-completeness': 'rule', + 'review-base-branch': 'rule', + 'review-design-lite': 'rule', + 'review-coverage-audit': 'rule', + 'review-dashboard-via': 'rule', + 'review-army-migration-safety': 'rule', + 'review-army-perf-n-plus-one': 'rule', + 'review-army-delivery-audit': 'rule', + 'review-army-quality-score': 'rule', + 'review-army-json-findings': 'rule', + 'review-army-red-team': 'rule', + 'review-army-consensus': 'rule', + 'review-army-simplification': 'rule', + 'review-army-simplification-precision': 'rule', + 'office-hours-spec-review': 'rule', + 'office-hours-brain-writeback': 'rule', + 'gbrain-roundtrip-local': 'rule', + 'sync-gbrain-read-ready': 'rule', + 'sync-gbrain-read-unknown': 'rule', + 'office-hours-forcing-energy': 'rule', + 'office-hours-builder-wildness': 'rule', + 'plan-ceo-review': 'rule', + 'plan-ceo-review-selective': 'rule', + 'plan-ceo-review-benefits': 'rule', + 'plan-ceo-review-expansion-energy': 'rule', + 'plan-eng-review': 'rule', + 'plan-eng-review-artifact': 'rule', + 'plan-eng-coverage-audit': 'rule', + 'plan-review-report': 'rule', + 'plan-ceo-review-plan-mode': 'rule', + 'plan-eng-review-plan-mode': 'rule', + 'plan-design-review-plan-mode': 'rule', + 'plan-devex-review-plan-mode': 'rule', + 'plan-mode-no-op': 'rule', + 'office-hours-auto-mode': 'rule', + 'auto-decide-preserved': 'rule', + 'auq-format-gate': 'rule', + 'plan-ceo-mode-routing': 'rule', + 'plan-design-with-ui-scope': 'rule', + 'tpa-present': 'rule', + 'tpa-absent-linux': 'rule', + 'tpa-broken': 'rule', + 'tpa-absent-darwin': 'rule', + 'tpa-apple-ban': 'rule', + 'ship-section-loading': 'rule', + 'plan-ceo-section-loading': 'rule', + 'carve-section-loading': 'rule', + 'plan-eng-finding-floor': 'rule', + 'plan-ceo-finding-floor': 'rule', + 'plan-design-finding-floor': 'rule', + 'plan-devex-finding-floor': 'rule', + 'plan-eng-multi-finding-batching': 'rule', + 'plan-ceo-split-overflow': 'rule', + 'setup-gbrain-remote': 'rule', + 'setup-gbrain-bad-token': 'rule', + 'setup-gbrain-path4-local-pglite': 'rule', + 'plan-ceo-review-format-mode': 'rule', + 'plan-ceo-review-format-approach': 'rule', + 'plan-eng-review-format-coverage': 'rule', + 'plan-eng-review-format-kind': 'rule', + 'office-hours-phase4-fork': 'rule', + 'llm-judge-recommendation': 'rule', + 'plan-ceo-review-prosons-cadence': 'rule', + 'plan-review-prosons-format': 'rule', + 'plan-review-prosons-hardstop-neg': 'rule', + 'plan-review-prosons-neutral-neg': 'rule', + 'plan-tune-inspect': 'rule', + 'codex-offered-office-hours': 'rule', + 'codex-offered-ceo-review': 'rule', + 'codex-offered-design-review': 'rule', + 'codex-offered-eng-review': 'rule', + 'timeline-event-flow': 'rule', + 'context-recovery-artifacts': 'rule', + 'context-save-writes-file': 'rule', + 'context-restore-loads-latest': 'rule', + 'context-save-routing': 'rule', + 'context-save-then-restore-roundtrip': 'rule', + 'context-restore-fragment-match': 'rule', + 'context-restore-empty-state': 'rule', + 'context-restore-list-delegates': 'rule', + 'context-restore-legacy-compat': 'rule', + 'context-save-list-current-branch': 'rule', + 'context-save-list-all-branches': 'rule', + 'ship-base-branch': 'rule', + 'ship-local-workflow': 'rule', + 'ship-managed-hook-refresh': 'rule', + 'ship-unmanaged-hook-consent': 'rule', + 'ship-local-hook-preservation': 'rule', + 'ship-coverage-audit': 'rule', + 'ship-triage': 'rule', + 'ship-docsync-missing-marker': 'rule', + 'ship-docsync-missing-asset': 'rule', + 'ship-docsync-launch-failure': 'rule', + 'ship-docsync-timeout-unsettled': 'rule', + 'ship-docsync-late-result': 'rule', + 'ship-docsync-stale-before': 'rule', + 'ship-docsync-stale-after': 'rule', + 'ship-docsync-recovery': 'rule', + 'ship-docsync-completion': 'rule', + 'ship-docsync-current': 'rule', + 'ship-docsync-failure': 'rule', + 'ship-docsync-store': 'rule', + 'docsync-spawned': 'rule', + 'retro': 'rule', + 'retro-base-branch': 'rule', + 'cso-full-audit': 'rule', + 'cso-diff-mode': 'rule', + 'cso-infra-scope': 'rule', + 'learnings-show': 'rule', + 'document-release': 'rule', + 'codex-review': 'rule', + 'codex-discover-skill': 'rule', + 'codex-review-findings': 'rule', + 'outside-voice-codex-to-claude-code': 'rule', + 'outside-voice-claude-code-to-codex': 'rule', + 'outside-plan-disabled-no-fallback': 'rule', + 'codex-sol-scope-termination': 'rule', + 'design-consultation-core': 'rule', + 'design-consultation-existing': 'rule', + 'design-consultation-research': 'rule', + 'design-consultation-preview': 'rule', + 'plan-design-review-no-ui-scope': 'rule', + 'design-review-fix': 'rule', + 'design-review-detector-shim': 'rule', + 'design-review-detector-shim-dom': 'rule', + 'design-review-plugin-handoff': 'rule', + 'design-html-slop-gate': 'rule', + 'diagram-triplet': 'rule', + 'diagram-authoring-quality': 'rule', + 'gstack-upgrade-happy-path': 'rule', + 'land-and-deploy-workflow': 'rule', + 'land-and-deploy-first-run': 'rule', + 'land-and-deploy-review-gate': 'rule', + 'canary-workflow': 'rule', + 'benchmark-workflow': 'rule', + 'setup-deploy-workflow': 'rule', + 'autoplan-dual-voice': 'rule', + 'benchmark-providers-live': 'rule', + 'scrape-match-path': 'rule', + 'scrape-prototype-path': 'rule', + 'skillify-happy-path': 'rule', + 'skillify-provenance-refusal': 'rule', + 'skillify-approval-reject': 'rule', + 'journey-ideation': 'rule', + 'journey-plan-eng': 'rule', + 'journey-debug': 'rule', + 'journey-qa': 'rule', + 'journey-code-review': 'rule', + 'journey-ship': 'rule', + 'journey-docs': 'rule', + 'journey-retro': 'rule', + 'journey-design-system': 'rule', + 'journey-visual-qa': 'rule', + 'ios-qa-device': 'rule', + 'arm-benchmark-native-overbuild': 'rule', + 'arm-benchmark-crud-endpoint': 'rule', + 'arm-benchmark-bugfix-decoys': 'rule', + 'office-hours-section-loading': 'rule', + 'office-hours-design-draft': 'rule', + 'plan-decision-classification': 'rule', + 'plan-devex-peer-comparison-classification': 'rule', + 'health-reporting': 'rule', + 'overlay-harness-claude-dedicated-tools-vs-bash': 'rule', + 'overlay-harness-opus-4-7-effort-match-trivial': 'rule', + 'overlay-harness-opus-4-7-literal-interpretation': 'rule', + 'overlay-harness-claude-dedicated-tools-vs-bash-sonnet': 'rule', + 'journey-negatives': 'rule', + 'review/SKILL.md workflow': 'rule', + 'setup-browser-cookies/SKILL.md workflow': 'rule', + 'browse/SKILL.md reference': 'rule', + 'setup block': 'rule', + 'qa/SKILL.md workflow': 'rule', + 'qa/SKILL.md health rubric': 'rule', + 'qa/SKILL.md anti-refusal': 'rule', + 'cross-skill greptile consistency': 'rule', + 'ship/SKILL.md workflow': 'rule', + 'document-release/SKILL.md workflow': 'rule', + 'plan-ceo-review/SKILL.md modes': 'rule', + 'plan-eng-review/SKILL.md sections': 'rule', + 'plan-design-review/SKILL.md passes': 'rule', + 'design-review/SKILL.md fix loop': 'rule', + 'design-consultation/SKILL.md research': 'rule', + 'land-and-deploy/SKILL.md workflow': 'rule', + 'canary/SKILL.md monitoring loop': 'rule', + 'benchmark/SKILL.md perf collection': 'rule', + 'setup-deploy/SKILL.md platform setup': 'rule', + 'retro/SKILL.md instructions': 'rule', + 'qa-only/SKILL.md workflow': 'rule', + 'gstack-upgrade/SKILL.md upgrade flow': 'rule', + 'sync-gbrain/SKILL.md read-only readiness': 'rule', + 'voice directive tone': 'rule', +}; + +/** + * One-line tolerance for every behavior-kind case: why an occasional + * deviation is acceptable product behavior. Keys equal the behavior ids of + * E2E_KINDS; values are non-empty. + */ +export const BEHAVIOR_WHY: Record = {};