mirror of
https://github.com/garrytan/gstack.git
synced 2026-10-31 22:14:25 +01:00
garrytan
80fe5de7bd
One pure failureCauseOf() beside failureClassOf() sets failure_cause (contract > pre_turn_infra > api_error > refusal > provider_stall > session_timeout > observer_timeout > assertion > unknown; no machine-set 'detector') with one bounded evidence line, and failureDetailOf() keeps a Bun matcher's Expected/Received or a judge's failing dimensions in failure_detail. Both record builders (classifyTrialShard, junitCensus) call them and attach the trial's ledger sessions. failure_class, error and TRIAL_ERROR_MAX are computed exactly as before; the new fields are optional, bounded and validated only when present, and a frozen copy of the v1 reader accepts every new record. A provider stall needs partial messages and STALL_WINDOW_MS (120 s, eval-budgets.ts beside HEADROOM_WARN/FAIL) of in-request silence with nothing outstanding; it stays a failure under EVAL_POLICY v1.
150 lines
7.2 KiB
TypeScript
150 lines
7.2 KiB
TypeScript
/**
|
||
* Timeout policy for paid tests — five tiers instead of hand-tuned sprawl.
|
||
*
|
||
* Before this module the paid suite carried 46×300s, 46×120s, 44×360s,
|
||
* 44×180s, 27×240s, 19×150s, 13×420s, 12×600s, 7×700s… hand-ratcheted
|
||
* per test, several inflated to paper over the old 40-way in-shard
|
||
* concurrency (session startup queued behind 39 siblings and ate the
|
||
* budget before turn one — dead with the sharded runner's 1-file-per-shard
|
||
* model). Pick the tier that matches the test's SHAPE; escape-hatch raw
|
||
* literals stay legal with a justification comment (count-ratcheted by
|
||
* test/eval-budgets-policy.test.ts).
|
||
*
|
||
* Every tier must fit inside the lane walls — pinned by the fit test in
|
||
* test/eval-budgets-policy.test.ts against the sharded runner's
|
||
* DEFAULT_SHARD_TIMEOUT_MS. Budget above the wall is fiction, not headroom.
|
||
*/
|
||
|
||
/** LLM-judge call over an existing capture (no agent session). */
|
||
export const JUDGE_MS = 120_000;
|
||
|
||
/** One bounded capture: SDK execution or the first displayed native question. */
|
||
export const CAPTURE_MS = 300_000;
|
||
|
||
/** Multi-capture or long multi-turn `claude -p` flows. */
|
||
export const CAPTURE_LONG_MS = 600_000;
|
||
|
||
/** Interactive real-PTY flow (spawn + skill + a few interactions). */
|
||
export const PTY_MS = 900_000;
|
||
|
||
/**
|
||
* Chained/judged PTY observation — the ceiling tier. 1200s leaves the
|
||
* 1800s shard wall real overhead; anything that genuinely needs more
|
||
* should be split or use an explicitly registered workflow exception with
|
||
* corresponding runner and CI walls; never inflate an ordinary tier.
|
||
*/
|
||
export const PTY_LONG_MS = 1_200_000;
|
||
|
||
export const ALL_TIERS = {
|
||
JUDGE_MS,
|
||
CAPTURE_MS,
|
||
CAPTURE_LONG_MS,
|
||
PTY_MS,
|
||
PTY_LONG_MS,
|
||
} as const;
|
||
|
||
/** Supervision reserve added to every registered whole-file wall. */
|
||
export const SHARD_RESERVE_MS = 2 * 60_000;
|
||
|
||
/**
|
||
* Retry policy (approved 2026-09-29, eval reliability wave): paid evals never
|
||
* retry. Each case's kind (E2E_KINDS) fixes its trials before the run: `rule`
|
||
* one trial, `behavior` a panel of EVAL_POLICY.panel independent trials, and
|
||
* `judge` one case that samples its judge panel internally. A failed verdict
|
||
* is final for that run; a manual re-run adds trials under a new run attempt
|
||
* and never replaces the original verdict. Rows below keep only wall
|
||
* supervision; per-case budgets never change with this rule.
|
||
*/
|
||
|
||
/** Whole-file supervision for one run of every case.
|
||
* These fixtures allow 25 minutes per case.
|
||
* Reserve the sequential upper bound even when Bun runs sibling cases together.
|
||
*/
|
||
export const FINDING_RETRY_BUDGETS = [
|
||
{ file: 'test/skill-e2e-plan-ceo-split-overflow.test.ts', cases: 1 },
|
||
{ file: 'test/skill-e2e-plan-eng-multi-finding-batching.test.ts', cases: 1 },
|
||
].map(({ file, cases }) => ({
|
||
file, cases,
|
||
id: `${file.slice('test/skill-e2e-'.length, -'.test.ts'.length)}-existing-retry-v1`,
|
||
testMs: 1_500_000,
|
||
caseMs: 1_500_000,
|
||
shardReserveMs: SHARD_RESERVE_MS,
|
||
shardMs: cases * 1_500_000 + SHARD_RESERVE_MS,
|
||
}));
|
||
|
||
/** Three existing captures in one 16-minute case. */
|
||
export const AUQ_CONSISTENCY_RETRY_BUDGET = {
|
||
file: 'test/skill-e2e-auq-consistency.test.ts',
|
||
id: 'auq-consistency-existing-retry-v1',
|
||
cases: 1,
|
||
testMs: 3 * CAPTURE_MS + 60_000,
|
||
caseMs: 3 * CAPTURE_MS + 60_000,
|
||
shardReserveMs: SHARD_RESERVE_MS,
|
||
shardMs: 3 * CAPTURE_MS + 60_000 + SHARD_RESERVE_MS,
|
||
} as const;
|
||
|
||
/** These fixtures have a fixed case count in every supported tier. */
|
||
export const STRICT_RETRY_CASE_BUDGETS = [...FINDING_RETRY_BUDGETS, AUQ_CONSISTENCY_RETRY_BUDGET];
|
||
|
||
/** Whole-file walls cover all existing cases, even if Bun runs them
|
||
* sequentially. Mixed-tier files reserve their larger complete tier, never a
|
||
* currently selected subset. caseMs is the longest single case budget, the
|
||
* wall of one isolated case shard. These rows add no case-count or model-work
|
||
* policy. The 10-second terms preserve the existing Codex/recording
|
||
* finalization grace.
|
||
*/
|
||
export const FILE_RETRY_BUDGETS = [
|
||
...STRICT_RETRY_CASE_BUDGETS,
|
||
...[
|
||
{ file: 'test/skill-e2e-qa-callers.test.ts', attemptMs: 5 * (CAPTURE_MS + 15_000), caseMs: CAPTURE_MS + 15_000 },
|
||
{ file: 'test/skill-e2e-shared-libs-paths.test.ts', attemptMs: 3 * CAPTURE_LONG_MS, caseMs: CAPTURE_LONG_MS },
|
||
{ file: 'test/skill-e2e-ship-docsync.test.ts', attemptMs: 4 * CAPTURE_LONG_MS + 8 * CAPTURE_MS, caseMs: CAPTURE_LONG_MS },
|
||
// Seventeen workflow judges include their 10s recording grace; the other
|
||
// seven judges retain 120s. Supervise all 24.
|
||
{ file: 'test/skill-llm-eval.test.ts', attemptMs: 17 * (JUDGE_MS + 10_000) + 7 * JUDGE_MS, caseMs: JUDGE_MS + 10_000 },
|
||
{ file: 'test/skill-e2e-auq-matrix.test.ts', attemptMs: 6 * CAPTURE_MS, caseMs: CAPTURE_MS },
|
||
{ file: 'test/skill-e2e-plan-format.test.ts', attemptMs: 4 * (CAPTURE_MS + 10_000), caseMs: CAPTURE_MS + 10_000 },
|
||
{ file: 'test/skill-e2e-auto-decide-preserved.test.ts', attemptMs: PTY_MS, caseMs: PTY_MS },
|
||
{ file: 'test/skill-e2e-plan-ceo-finding-floor.test.ts', attemptMs: PTY_MS, caseMs: PTY_MS },
|
||
{ file: 'test/skill-e2e-plan-eng-finding-floor.test.ts', attemptMs: PTY_MS, caseMs: PTY_MS },
|
||
{ file: 'test/skill-e2e-plan-design-finding-floor.test.ts', attemptMs: PTY_MS, caseMs: PTY_MS },
|
||
{ file: 'test/skill-e2e-plan-devex-finding-floor.test.ts', attemptMs: PTY_MS, caseMs: PTY_MS },
|
||
{ file: 'test/skill-e2e-plan-mode-no-op.test.ts', attemptMs: 6 * CAPTURE_LONG_MS, caseMs: CAPTURE_LONG_MS },
|
||
{ file: 'test/skill-e2e-plan-ceo-mode-routing.test.ts', attemptMs: 2 * CAPTURE_LONG_MS, caseMs: CAPTURE_LONG_MS },
|
||
{ file: 'test/skill-e2e-plan-eng-plan-mode.test.ts', attemptMs: 2 * CAPTURE_LONG_MS, caseMs: CAPTURE_LONG_MS },
|
||
{ file: 'test/skill-e2e-plan-prosons.test.ts', attemptMs: 4 * (CAPTURE_MS + 10_000), caseMs: CAPTURE_MS + 10_000 },
|
||
// Gate: six 300s cases + one 610s case; periodic: two 900s + two 600s
|
||
// (plan-eng-review-artifact runs from its own PTY file).
|
||
{ file: 'test/skill-e2e-plan.test.ts', attemptMs: Math.max(6 * CAPTURE_MS + CAPTURE_LONG_MS + 10_000, 2 * PTY_MS + 2 * CAPTURE_LONG_MS), caseMs: PTY_MS },
|
||
].map(({ file, attemptMs, caseMs }) => ({
|
||
file, attemptMs, caseMs,
|
||
id: `${file.slice('test/'.length, -'.test.ts'.length)}-existing-retry-v1`,
|
||
shardReserveMs: SHARD_RESERVE_MS,
|
||
shardMs: attemptMs + SHARD_RESERVE_MS,
|
||
})),
|
||
];
|
||
|
||
/** No paid test may exceed the ordinary tiers; arbitrary per-file escapes fail. */
|
||
export function assertPaidTestBudget(file: string, ms: number): void {
|
||
if (!Number.isSafeInteger(ms) || ms <= 0 || ms > PTY_LONG_MS * 1.25) {
|
||
throw new Error(`Unregistered paid test budget: ${file}: ${ms}`);
|
||
}
|
||
}
|
||
|
||
/**
|
||
* Diagnostic thresholds, outside EVAL_POLICY: v1 verdicts never read them.
|
||
* A session above HEADROOM_WARN of the timeout it armed is flagged by
|
||
* `eval:pass-rates --headroom`; above HEADROOM_FAIL the weekly `--gate` alarms.
|
||
* Cut work to get under them; budgets are never raised.
|
||
*/
|
||
export const HEADROOM_WARN = 0.75;
|
||
export const HEADROOM_FAIL = 0.85;
|
||
|
||
/**
|
||
* In-request stream silence that labels a provider stall (failure_cause
|
||
* provider_stall, still a failure under EVAL_POLICY v1). Pre-registered from
|
||
* census 37198445662 on Claude Code 2.1.284: the longest gap inside one model
|
||
* message across 50 messages was 9.6 s, thinking deltas included.
|
||
*/
|
||
export const STALL_WINDOW_MS = 120_000;
|