v1.91.6.0 feat: balance PR eval slices by recorded duration without weakening timeout coverage (#2991)

CHANGELOG.md

@@ -1,5 +1,13 @@
 # Changelog
 
+## [1.91.6.0] - 2026-09-28
+
+PR eval slices are balanced by how long each eval actually takes, so the slowest slice no longer carries most of the run.
+
+### Changed
+- The paid eval planner re-packs slices using recorded per-file wall times from real CI runs (`scripts/paid-test-durations.json`). It starts from the existing supervised allocation and only moves or swaps a file out of the heaviest slice when no slice's worst-case wall, for 1–4 workers, rises above that allocation's maximum, so CI timeout coverage never gets weaker. Estimated from the recorded times with two workers per slice, the heaviest slice for a typical PR run drops from about 15 minutes to 12 (the length of the single longest eval), and for the full gate census from about 17 minutes to 13.
+- `bun run scripts/test-paid-shards.ts --report <dir> --write-durations` merges a report's executed single-file shard times into the seed. Skipped-only and sub-second shards are ignored.
+
 ## [1.91.5.0] - 2026-09-28
 
 The free suite now finishes in about half the time on a 16-core Linux machine, `bun run test:ubicloud` runs it on a fresh 16-vCPU Ubicloud VM from any dev box, container, or cloud sandbox, and re-pushing a PR no longer waits behind the previous commit's eval run.

VERSION

@@ -1 +1 @@
-1.91.5.0
+1.91.6.0

agents-digest/gstack-AGENTS.md

@@ -1,4 +1,4 @@
-# gstack digest v1.91.5.0 — regenerate/re-copy after upgrading gstack
+# gstack digest v1.91.6.0 — regenerate/re-copy after upgrading gstack
 
 Behavioral rules from gstack (https://github.com/garrytan/gstack), compressed
 for agent hosts without a full skill install. The full skills add workflows,

docs/TESTING_INTERNALS.md

@@ -270,7 +270,13 @@ selection + the slice plan ONCE (killing per-slice selector divergence);
 `--plan <path> --slice i` executors consume the manifest and write
 slice-result artifacts; `--report <dir>` reconciles them FAIL-CLOSED (a slice
 whose artifact never landed, or a planned shard nobody reported, is a
-failure). Under `EVALS_ALL` the hollow-shard guard marks exit-0 shards with
+failure). Slices start from the supervision baseline (registered long files
+spread by budget, the rest round-robin), then are re-packed by the recorded
+wall times in `scripts/paid-test-durations.json`: a file moves or swaps out of
+the heaviest slice only if no slice's worst-case wall (`paidShardWallUpperBoundMs`
+for 1–4 workers) rises above the baseline's maximum, so CI timeout coverage is
+never weakened. Refresh the seed from a downloaded report directory with
+`--report <dir> --write-durations`. Under `EVALS_ALL` the hollow-shard guard marks exit-0 shards with
 ZERO executed tests `passed-empty` (a failure) — census-health, not just
 test runs. evals.yml runs the sliced gate lane per PR — the ONLY paid lane
 since the legacy 17-row matrix (22.6 min/$21 per PR serialized ahead of the

package.json

@@ -1,6 +1,6 @@
 {
   "name": "gstack",
-  "version": "1.91.5",
+  "version": "1.91.6",
   "description": "Garry's Stack — Claude Code skills + fast headless browser. One repo, one install, entire AI engineering workflow.",
   "license": "MIT",
   "type": "module",

scripts/paid-test-durations.json

@@ -0,0 +1,46 @@
+{
+  "version": 1,
+  "recordedAt": "2026-09-28T23:00:00Z",
+  "durations": {
+    "test/skill-e2e-ask-user-question-format-compliance.test.ts": 67000,
+    "test/skill-e2e-bws.test.ts": 99000,
+    "test/skill-e2e-coverage-audit.test.ts": 60000,
+    "test/skill-e2e-cso.test.ts": 220000,
+    "test/skill-e2e-deploy.test.ts": 496000,
+    "test/skill-e2e-design.test.ts": 228000,
+    "test/skill-e2e-diagram.test.ts": 25000,
+    "test/skill-e2e-docsync-spawned.test.ts": 49000,
+    "test/skill-e2e-hermetic-canary.test.ts": 5000,
+    "test/skill-e2e-investigate-owned-completion.test.ts": 50000,
+    "test/skill-e2e-investigate-owned-termination.test.ts": 49000,
+    "test/skill-e2e-learnings.test.ts": 32000,
+    "test/skill-e2e-office-hours-auto-mode.test.ts": 61000,
+    "test/skill-e2e-opus-47.test.ts": 20000,
+    "test/skill-e2e-plan-ceo-finding-floor.test.ts": 233000,
+    "test/skill-e2e-plan-ceo-plan-mode.test.ts": 35000,
+    "test/skill-e2e-plan-design-with-ui.test.ts": 435000,
+    "test/skill-e2e-plan-devex-finding-floor.test.ts": 187000,
+    "test/skill-e2e-plan-devex-plan-mode.test.ts": 103000,
+    "test/skill-e2e-plan-mode-no-op.test.ts": 206000,
+    "test/skill-e2e-plan-tune-cathedral.test.ts": 1000,
+    "test/skill-e2e-plan-tune.test.ts": 58000,
+    "test/skill-e2e-plan.test.ts": 312000,
+    "test/skill-e2e-qa-workflow.test.ts": 437000,
+    "test/skill-e2e-retro.test.ts": 139000,
+    "test/skill-e2e-review-army.test.ts": 568000,
+    "test/skill-e2e-review-attribution.test.ts": 80000,
+    "test/skill-e2e-review.test.ts": 135000,
+    "test/skill-e2e-session-intelligence.test.ts": 47000,
+    "test/skill-e2e-shared-libs-paths.test.ts": 713000,
+    "test/skill-e2e-shared-libs.test.ts": 662000,
+    "test/skill-e2e-ship-docsync.test.ts": 148000,
+    "test/skill-e2e-ship-hook-consent.test.ts": 72000,
+    "test/skill-e2e-ship-hook-refresh.test.ts": 66000,
+    "test/skill-e2e-skillify.test.ts": 156000,
+    "test/skill-e2e-third-party-actions.test.ts": 75000,
+    "test/skill-e2e-triage.test.ts": 49000,
+    "test/skill-e2e-workflow.test.ts": 234000,
+    "test/skill-llm-eval.test.ts": 103000,
+    "test/skill-routing-e2e.test.ts": 1000
+  }
+}

scripts/test-paid-shards.ts

@@ -1017,7 +1017,47 @@ export function retriesForFiles(files: string[]): number {
   return Math.max(1, ...files.map((f) => RETRY_OVERRIDES[normalizeRelativePath(f)] ?? 1));
 }
 
-/** Round-robin the RUNNABLE (sorted) shard plan across K slices — deterministic. */
+export const PAID_TEST_DURATIONS_FILE = 'scripts/paid-test-durations.json';
+
+/**
+ * Recorded per-file paid-shard wall times (ms) from real CI slice reports,
+ * refreshed with `--report <dir> --write-durations`. A packing hint only: a
+ * missing or corrupt seed keeps the supervision-budget allocation.
+ */
+export function loadPaidTestDurations(rootDir = ROOT): Record<string, number> {
+  try {
+    const parsed = JSON.parse(fs.readFileSync(path.join(rootDir, PAID_TEST_DURATIONS_FILE), 'utf8')) as { durations?: Record<string, unknown> };
+    return Object.fromEntries(Object.entries(parsed.durations ?? {})
+      .filter((entry): entry is [string, number] => typeof entry[1] === 'number' && Number.isFinite(entry[1]) && entry[1] > 0));
+  } catch {
+    return {};
+  }
+}
+
+/** Merge a report's executed single-file outcomes into the seed; all-skipped shards carry no cost signal. */
+export function mergePaidTestDurations(seed: Record<string, number>, results: SliceResult[]): Record<string, number> {
+  const merged = { ...seed };
+  for (const result of results) {
+    for (const outcome of result.outcomes) {
+      if (outcome.files.length !== 1 || outcome.elapsedMs < 1_000 || isAllSkippedPass(outcome)) continue;
+      merged[normalizeRelativePath(outcome.files[0])] = outcome.elapsedMs;
+    }
+  }
+  return Object.fromEntries(Object.entries(merged).sort(([a], [b]) => (a < b ? -1 : 1)));
+}
+
+/** Worker counts whose worst-case slice wall duration packing may never worsen. */
+export const SUPERVISED_WORKER_COUNTS = [1, 2, 3, 4] as const;
+
+/**
+ * Allocate the RUNNABLE shard plan across K slices — deterministic. Registered
+ * long files are spread by supervision budget and the rest round-robin; that
+ * baseline fixes each slice's worst-case wall. With a duration seed, files are
+ * then re-packed longest-recorded-first onto the lightest slice, accepting a
+ * placement only if no slice's worst-case wall exceeds the baseline's maximum
+ * for any supervised worker count. If any file cannot be placed, the baseline
+ * stands.
+ */
 export function buildRunManifest(opts: {
   tier: PaidTier;
   profile?: PaidProfile;
@@ -1029,6 +1069,8 @@ export function buildRunManifest(opts: {
   env?: NodeJS.ProcessEnv;
   rootDir?: string;
   changedFiles?: string[];
+  /** Recorded per-file durations; defaults to the committed seed under rootDir. */
+  durations?: Record<string, number>;
 }): PaidRunManifest {
   if (!Number.isInteger(opts.sliceCount) || opts.sliceCount <= 0) {
     throw new Error(`--slices needs a positive integer. Received: ${opts.sliceCount}`);
@@ -1081,11 +1123,60 @@ export function buildRunManifest(opts: {
     }
   }
   let ordinaryIndex = 0;
+  for (const files of ordinary) {
+    if (!allocations.has(files[0])) allocations.set(files[0], (ordinaryIndex++ % ordinarySlices) + 1);
+  }
+  const packed = packByRecordedDuration();
+  function packByRecordedDuration(): Map<string, number> | null {
+    const recorded = opts.durations ?? loadPaidTestDurations(rootDir);
+    if (ordinarySlices < 2 || ordinary.length === 0 || Object.keys(recorded).length === 0) return null;
+    const bound = (files: string[], jobs: number) => paidShardWallUpperBoundMs([...files].sort(), jobs, opts.timeoutMs);
+    const lanes = Array.from({ length: ordinarySlices }, (_, lane) =>
+      ordinary.filter(files => allocations.get(files[0]) === lane + 1).map(files => files[0]));
+    const caps = SUPERVISED_WORKER_COUNTS.map(jobs => Math.max(...lanes.map(files => bound(files, jobs))));
+    const fits = (files: string[]) => SUPERVISED_WORKER_COUNTS.every((jobs, k) => bound(files, jobs) <= caps[k]);
+    const known = ordinary.map(files => recorded[normalizeRelativePath(files[0])])
+      .filter((ms): ms is number => ms !== undefined).sort((a, b) => a - b);
+    const fallback = known.length ? known[Math.min(known.length - 1, Math.floor(known.length * 0.75))] : 1;
+    const weight = (file: string) => recorded[normalizeRelativePath(file)] ?? fallback;
+    const load = (files: string[]) => files.reduce((sum, file) => sum + weight(file), 0);
+    const registeredFiles = new Set(registered.map(files => files[0]));
+    // Local search from the supervised baseline: move or swap a file out of
+    // the heaviest slice whenever that lowers its recorded load without
+    // making the other slice the new maximum or breaching any worst-case cap.
+    // Registered files only trade places with registered files, so the long
+    // lanes keep their ownership.
+    for (let step = 0; step < 10 * ordinary.length; step++) {
+      const loads = lanes.map(load);
+      const heavy = loads.indexOf(Math.max(...loads));
+      let best: { gain: number; apply: () => void } | null = null;
+      for (let other = 0; other < lanes.length; other++) {
+        if (other === heavy) continue;
+        for (const a of lanes[heavy]) {
+          const moves: Array<string | null> = registeredFiles.has(a) ? lanes[other].filter(b => registeredFiles.has(b)) : [null, ...lanes[other].filter(b => !registeredFiles.has(b))];
+          for (const b of moves) {
+            const delta = weight(a) - (b === null ? 0 : weight(b));
+            if (delta <= 0 || loads[other] + delta >= loads[heavy]) continue;
+            const gain = Math.min(delta, loads[heavy] - loads[other] - delta);
+            if (best && gain <= best.gain) continue;
+            const heavyAfter = lanes[heavy].filter(file => file !== a).concat(b === null ? [] : [b]);
+            const otherAfter = lanes[other].filter(file => file !== b).concat([a]);
+            if (!fits(heavyAfter) || !fits(otherAfter)) continue;
+            const [h, o] = [heavy, other];
+            best = { gain, apply: () => { lanes[h] = heavyAfter; lanes[o] = otherAfter; } };
+          }
+        }
+      }
+      if (!best) break;
+      best.apply();
+    }
+    return new Map(lanes.flatMap((files, lane) => files.map(file => [file, lane + 1] as const)));
+  }
   runnable.forEach((files) => {
     const autoplan = files[0] === AUTOPLAN_CHAIN_BUDGET.file;
     const slice = opts.dedicatedAutoplanSlice && autoplan ? opts.sliceCount
       : files.some(isOverlayTestFile) ? overlaySlice
-        : allocations.get(files[0]) ?? (ordinaryIndex++ % ordinarySlices) + 1;
+        : (packed ?? allocations).get(files[0])!;
     entries.push({ file: files[0], slice, status: 'planned',
       ...(autoplan || FILE_RETRY_BUDGETS.some(budget => budget.file === files[0])
         ? { budget: resolvePaidShardBudget(files, opts.timeoutMs) } : {}) });
@@ -1355,6 +1446,8 @@ type CliOptions = {
   sliceIndex: number | null;
   /** Report mode: reconcile manifest.json + slice-*.json under this dir. */
   reportDir: string | null;
+  /** Report mode: merge executed shard wall times into the duration seed. */
+  writeDurations: boolean;
 };
 
 function parsePositiveInt(value: string | undefined, flag: string): number {
@@ -1406,6 +1499,7 @@ export function parseCliOptions(argv: string[], env: NodeJS.ProcessEnv = process
     planPath: null,
     sliceIndex: null,
     reportDir: null,
+    writeDurations: false,
   };
 
   for (let index = 0; index < argv.length; index += 1) {
@@ -1443,8 +1537,10 @@ export function parseCliOptions(argv: string[], env: NodeJS.ProcessEnv = process
       if (!value) throw new Error('--report needs a directory');
       options.reportDir = value; continue;
     }
+    if (arg === '--write-durations') { options.writeDurations = true; continue; }
     throw new Error(`Unknown argument: ${arg}`);
   }
+  if (options.writeDurations && !options.reportDir) throw new Error('--write-durations requires --report');
   if (options.dedicatedAutoplanSlice && !options.emitPlanPath) throw new Error('--autoplan-slice requires --emit-plan');
   if (options.profile === 'pr' && options.tier !== 'gate') throw new Error('PR profile requires gate tier');
   if (options.profile === 'pr' && options.maxFilesPerShard !== 1) throw new Error('PR profile requires one file per shard to preserve case accounting');
@@ -1496,6 +1592,14 @@ async function main(): Promise<number> {
         console.log(`  slice ${result.sliceIndex}  ${outcome.status.padEnd(15)} ${String(Math.round(outcome.elapsedMs / 1000)).padStart(5)}s  ${outcome.files.join(' ')}`);
       }
     }
+    if (options.writeDurations) {
+      const durations = mergePaidTestDurations(loadPaidTestDurations(), results);
+      const target = path.join(ROOT, PAID_TEST_DURATIONS_FILE);
+      const temporary = `${target}.tmp-${process.pid}`;
+      fs.writeFileSync(temporary, `${JSON.stringify({ version: 1, recordedAt: new Date().toISOString(), durations }, null, 2)}\n`);
+      fs.renameSync(temporary, target);
+      console.log(`[test:paid] wrote ${Object.keys(durations).length} durations to ${PAID_TEST_DURATIONS_FILE}`);
+    }
     // Historical flaky_retries includes every case with multiple attempts,
     // whether its final result passed or failed. Report attempts separately
     // from the shard verdict; reconciliation above still controls gating.

test/paid-run-manifest.test.ts

@@ -19,7 +19,12 @@ import {
   applyHollowShardGuard,
   buildPaidShardArgs,
   buildRunManifest,
+  loadPaidTestDurations,
+  mergePaidTestDurations,
+  paidShardWallUpperBoundMs,
+  parseCliOptions,
   parseRunManifest,
+  SUPERVISED_WORKER_COUNTS,
   retriesForFiles,
   RETRY_OVERRIDES,
   summarize,
@@ -99,6 +104,72 @@ describe('run manifest (planner)', () => {
   });
 });
 
+describe('recorded-duration slice packing', () => {
+  const recorded = loadPaidTestDurations();
+  const lanes = (manifest: PaidRunManifest) => {
+    const planned = manifest.entries.filter(e => e.status === 'planned');
+    return Array.from({ length: manifest.sliceCount }, (_, i) => planned.filter(e => e.slice === i + 1).map(e => e.file));
+  };
+  const plans = [
+    { tier: 'gate' as const, sliceCount: 6 },
+    { tier: 'gate' as const, sliceCount: 7 },
+    { tier: 'periodic' as const, sliceCount: 8, dedicatedAutoplanSlice: true },
+  ];
+
+  test('the committed seed records real wall times for the fast PR profile', () => {
+    expect(Object.keys(recorded).length).toBeGreaterThanOrEqual(30);
+    expect(Object.values(recorded).every(ms => Number.isInteger(ms) && ms >= 1_000)).toBe(true);
+  });
+
+  for (const plan of plans) {
+    test(`${plan.tier} x${plan.sliceCount}: no slice's worst-case wall exceeds the supervised baseline's for any worker count`, () => {
+      const env = { EVALS_ALL: '1' };
+      const packed = lanes(buildRunManifest({ ...plan, evalsAll: true, env }));
+      const baseline = lanes(buildRunManifest({ ...plan, evalsAll: true, env, durations: {} }));
+      expect(packed.flat().sort()).toEqual(baseline.flat().sort());
+      for (const jobs of SUPERVISED_WORKER_COUNTS) {
+        const bound = (files: string[]) => paidShardWallUpperBoundMs([...files].sort(), jobs);
+        expect(Math.max(...packed.map(bound))).toBeLessThanOrEqual(Math.max(...baseline.map(bound)));
+      }
+      // Same estimate the planner packs by: recorded time, else the 75th percentile of recorded files.
+      const known = baseline.flat().map(file => recorded[file]).filter(ms => ms !== undefined).sort((x, y) => x - y);
+      const fallback = known[Math.min(known.length - 1, Math.floor(known.length * 0.75))];
+      const load = (files: string[]) => files.reduce((sum, file) => sum + (recorded[file] ?? fallback), 0);
+      expect(Math.max(...packed.map(load))).toBeLessThanOrEqual(Math.max(...baseline.map(load)));
+    });
+  }
+
+  test('the PR-profile file set spreads recorded time instead of stacking it', () => {
+    const env = { EVALS_ALL: '1' };
+    const discovered = Object.keys(recorded);
+    const load = (files: string[]) => files.reduce((sum, file) => sum + recorded[file], 0);
+    const packed = lanes(buildRunManifest({ tier: 'gate', sliceCount: 6, evalsAll: true, env, discovered })).map(load);
+    const baseline = lanes(buildRunManifest({ tier: 'gate', sliceCount: 6, evalsAll: true, env, discovered, durations: {} })).map(load);
+    expect(Math.max(...packed)).toBeLessThan(Math.max(...baseline));
+    expect(buildRunManifest({ tier: 'gate', sliceCount: 6, evalsAll: true, env, discovered: [...discovered].reverse() }).entries)
+      .toEqual(buildRunManifest({ tier: 'gate', sliceCount: 6, evalsAll: true, env, discovered }).entries);
+  });
+
+  test('report durations merge only executed single-file outcomes', () => {
+    const outcome = (files: string[], elapsedMs: number, over: Partial<ShardOutcome> = {}) =>
+      ({ files, status: 'passed', exitCode: 0, elapsedMs, executedTests: 2, skippedTests: 0, ...over }) as SliceResult['outcomes'][number];
+    const merged = mergePaidTestDurations({ 'test/b.test.ts': 5_000, 'test/a.test.ts': 9_000 }, [{
+      sliceIndex: 1,
+      outcomes: [
+        outcome(['test/a.test.ts'], 42_000),
+        outcome(['test/c.test.ts'], 500),
+        outcome(['test/d.test.ts', 'test/e.test.ts'], 60_000),
+        outcome(['test/f.test.ts'], 30_000, { executedTests: 2, skippedTests: 2 }),
+        outcome(['test/g.test.ts'], 70_000, { status: 'failed', exitCode: 1 }),
+      ],
+    } as SliceResult]);
+    expect(merged).toEqual({ 'test/a.test.ts': 42_000, 'test/b.test.ts': 5_000, 'test/g.test.ts': 70_000 });
+    expect(Object.keys(merged)).toEqual(['test/a.test.ts', 'test/b.test.ts', 'test/g.test.ts']);
+    expect(() => parseCliOptions(['--write-durations'])).toThrow('--write-durations requires --report');
+    expect(parseCliOptions(['--report', '/tmp/r', '--write-durations']).writeDurations).toBe(true);
+  });
+});
+
 describe('manifest executor scope', () => {
   test('a conflicting inherited carve scope cannot suppress a planned case; direct Bun stays scoped', () => {
     const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'paid-manifest-scope-'));