From f9e3fc8ab4281f31783f101e6360ad45f59d3188 Mon Sep 17 00:00:00 2001
From: Garry Tan <garrytan@gmail.com>
Date: Sat, 15 Aug 2026 08:11:53 -0700
Subject: [PATCH] evals: enforce detach-timeout floor from the live shard
 census
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

New free tripwire: eval:bg:gate / eval:bg:periodic --timeout must cover
ceil(shards/jobs) x shard-timeout x 1.05, recomputed from the actual paid
test census every run. Hand-derived numbers go stale every time a paid
file lands — the review split just proved it: periodic's 28800s dropped
BELOW its new 32130s worst case (raised to 32400s here). An undersized
watchdog kills healthy runs and the tail reports never-started.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
---
 package.json                           |  2 +-
 test/eval-detach-timeout-floor.test.ts | 68 ++++++++++++++++++++++++++
 2 files changed, 69 insertions(+), 1 deletion(-)
 create mode 100644 test/eval-detach-timeout-floor.test.ts

diff --git a/package.json b/package.json
index 4681a8552..8d613f978 100644
--- a/package.json
+++ b/package.json
@@ -39,7 +39,7 @@
     "eval:bg": "bin/gstack-detach --label evals --lock gstack-evals --timeout 5400 -- bun run test:evals",
     "eval:bg:all": "bin/gstack-detach --label evals-all --lock gstack-evals --timeout 7200 -- bun run test:evals:all",
     "eval:bg:gate": "bin/gstack-detach --label evals-gate --lock gstack-evals --timeout 25200 -- bun run test:gate:sharded",
-    "eval:bg:periodic": "bin/gstack-detach --label evals-periodic --lock gstack-evals --timeout 28800 -- bun run test:periodic:sharded",
+    "eval:bg:periodic": "bin/gstack-detach --label evals-periodic --lock gstack-evals --timeout 32400 -- bun run test:periodic:sharded",
     "eval:list": "bun run scripts/eval-list.ts",
     "eval:compare": "bun run scripts/eval-compare.ts",
     "eval:summary": "bun run scripts/eval-summary.ts",
diff --git a/test/eval-detach-timeout-floor.test.ts b/test/eval-detach-timeout-floor.test.ts
new file mode 100644
index 000000000..17f3961da
--- /dev/null
+++ b/test/eval-detach-timeout-floor.test.ts
@@ -0,0 +1,68 @@
+/**
+ * Detach-timeout floor — free, gate-tier tripwire.
+ *
+ * The eval:bg:gate / eval:bg:periodic scripts wrap the sharded paid runner in
+ * bin/gstack-detach with a hard --timeout. If that number dips below the
+ * runner's worst-case wall clock — ceil(shards / jobs) × shard timeout — the
+ * watchdog kills a healthy run mid-flight and the tail shards report
+ * never-started: paid truncation by configuration. That nearly shipped once
+ * (a review pass proposed 10800s against a 19,800s gate worst case), so the
+ * bound is enforced here against the LIVE shard census instead of a comment
+ * snapshot that goes stale every time a paid test file is added.
+ *
+ * If this test fails you have two honest options: raise the --timeout in the
+ * package.json script it names, or reduce the tier's worst case (split fewer
+ * files per shard, raise DEFAULT_JOBS after verifying API rate headroom).
+ */
+
+import { describe, test, expect } from 'bun:test';
+import * as fs from 'fs';
+import * as path from 'path';
+import {
+  collectPaidTestFiles,
+  selectPaidTestFiles,
+  DEFAULT_JOBS,
+  DEFAULT_SHARD_TIMEOUT_MS,
+  type PaidTier,
+} from '../scripts/test-paid-shards';
+
+const ROOT = path.resolve(import.meta.dir, '..');
+// 5% margin over the theoretical bound: detach setup, lock wait, aggregation.
+const MARGIN = 1.05;
+
+function detachTimeoutSeconds(scriptName: string): number {
+  const pkg = JSON.parse(fs.readFileSync(path.join(ROOT, 'package.json'), 'utf-8'));
+  const script: string | undefined = pkg.scripts?.[scriptName];
+  expect(script, `package.json is missing the "${scriptName}" script`).toBeTruthy();
+  const m = script!.match(/--timeout\s+(\d+)/);
+  expect(m, `"${scriptName}" has no gstack-detach --timeout flag`).toBeTruthy();
+  return parseInt(m![1], 10);
+}
+
+function worstCaseSeconds(tier: PaidTier): number {
+  const shards = selectPaidTestFiles(collectPaidTestFiles(), tier).selected.length;
+  expect(shards).toBeGreaterThan(0);
+  return Math.ceil(shards / DEFAULT_JOBS) * (DEFAULT_SHARD_TIMEOUT_MS / 1000);
+}
+
+describe('eval:bg detach timeouts cover the sharded runner worst case', () => {
+  for (const [tier, script] of [
+    ['gate', 'eval:bg:gate'],
+    ['periodic', 'eval:bg:periodic'],
+  ] as Array<[PaidTier, string]>) {
+    test(`${script} >= ceil(${tier} shards / jobs) x shard timeout x ${MARGIN}`, () => {
+      const floor = Math.ceil(worstCaseSeconds(tier) * MARGIN);
+      const configured = detachTimeoutSeconds(script);
+      if (configured < floor) {
+        throw new Error(
+          `${script} --timeout ${configured}s is below the ${tier} tier's worst-case ` +
+          `wall clock of ${floor}s (ceil(shards/${DEFAULT_JOBS} jobs) x ` +
+          `${DEFAULT_SHARD_TIMEOUT_MS / 1000}s shard timeout x ${MARGIN} margin). ` +
+          `An undersized detach watchdog kills healthy runs mid-flight and the tail ` +
+          `shards report never-started. Raise the --timeout in package.json or reduce ` +
+          `the tier's worst case.`,
+        );
+      }
+    });
+  }
+});