test: docsync fault replays and the runShadow deadline test fit loaded remote VMs (GST-1103)

Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>

changelog.d/capy-gst1103-docsync-timing.md

@@ -0,0 +1,5 @@
+**The docsync fault-callback replays and the runShadow deadline test stop failing on loaded remote pregate VMs.**
+
+### Fixed
+- **Docsync fault-callback replays** (`test/helpers/docsync-fault-callbacks.ts` and its three wrappers): each replay in the generated child now has a 30 s limit instead of bun's 5 s default, and each wrapper's limit is that per-replay limit times its replay count plus one. A replay spawns 30 to 40 real actor, hook and git processes. That takes about 1.3 s on CI and up to 4.9 s on an idle 4-core machine, and it ran past 5 s on GSTA-234's remote pregate for #3892. With 8 busy loops on 4 cores, main fails all 12 legacy-control replays at 5 s, and this branch passes all 12 (172 s for the whole wrapper).
+- **`test/autoplan-shadow.test.ts` "one deadline covers the call and its retry"**: the first call now has 1 s of room inside the deadline instead of 300 ms. The time limit allows 2.4 s past the deadline before it would match an unshared retry, up from 0.9 s; the remote failure ran 1.06 s past. The test still requires the retry to be cut (p1 malformed) and the call to end before an unshared retry could finish.

test/autoplan-shadow.test.ts

@@ -271,15 +271,15 @@ describe('runShadow: retry, truncation, failures and the deadline', async () =>
     expect(readLedger(path.join(r.out, 'spend.json')).settled.filter(s => s.label === 'shadow').map(s => s.usd)).toEqual([0]);
   });
   test('one deadline covers the call and its retry: a slow malformed reply leaves the retry no time', async () => {
-    const r = await shadowed('deadline-retry', [{ mode: 'missing', delay_ms: 1200 }, { mode: 'valid', delay_ms: 1200 }], undefined, 1500);
-    expect(r.ms).toBeLessThan(2400);
+    const r = await shadowed('deadline-retry', [{ mode: 'missing', delay_ms: 2500 }, { mode: 'valid', delay_ms: 3400 }], undefined, 3500);
+    expect(r.ms).toBeLessThan(5900);
     expect(r.outcome.status).toBe('recorded');
     expect(ids(r.out).map(x => x.status)).toEqual(['recorded', 'recorded', 'recorded', 'recorded', 'recorded', 'recorded', 'malformed']);
-    const slow = await shadowed('deadline-call', [{ mode: 'valid', delay_ms: 3000 }], undefined, 800);
+    const slow = await shadowed('deadline-call', [{ mode: 'valid', delay_ms: 6000 }], undefined, 800);
     expect(slow.outcome.line).toBe('SHADOW: failed timeout');
-    expect(slow.ms).toBeLessThan(2500);
+    expect(slow.ms).toBeLessThan(5000);
     expect(readLedger(path.join(slow.out, 'spend.json')).settled.filter(s => s.label === 'shadow').map(s => s.usd)).toEqual(['unknown']);
-  });
+  }, 15_000);
   test('a publish that throws journals shadow_failed and drops the lease, so a later claim pays no second call', async () => {
     const f = await fake('publish-throws', [{ mode: 'valid' }]);
     const out = gateRun();

test/docsync-fault-interface-faults.test.ts

@@ -3,7 +3,7 @@ import { expect, test } from 'bun:test';
 import * as fs from 'node:fs';
 import * as os from 'node:os';
 import * as path from 'node:path';
-import { docsFaultCallbackScript } from './helpers/docsync-fault-callbacks';
+import { callbackBudgetMs, docsFaultCallbackScript } from './helpers/docsync-fault-callbacks';
 
 const CALLBACK_SPLITS = [
   ['fault controls', '^ship-docsync-(?!failure )[a-z-]+ rejects ', 14],
@@ -21,7 +21,7 @@ for (const [split, pattern, count] of CALLBACK_SPLITS) test(`registered fault ca
       env: { ...process.env, EVALS: '', EVALS_TIER: '', EVALS_ALL: '', EVALS_RUN_ID: 'free-docsync-faults',
         HOME: home, USERPROFILE: home,
         GSTACK_EVAL_DIR: path.join(dir, 'evidence'), GSTACK_HOME: path.join(dir, 'state') },
-      stdout: 'pipe', stderr: 'pipe', timeout: 120_000,
+      stdout: 'pipe', stderr: 'pipe', timeout: callbackBudgetMs(count),
     });
     const output = result.stdout.toString() + result.stderr.toString();
     expect(result.exitCode, output).toBe(0);
@@ -29,4 +29,4 @@ for (const [split, pattern, count] of CALLBACK_SPLITS) test(`registered fault ca
     expect(output).toContain(`${36 - count} filtered out`);
     expect(output).toContain('0 fail');
   } finally { fs.rmSync(dir, { recursive: true, force: true }); }
-}, 120_000);
+}, callbackBudgetMs(count) + 10_000);

test/docsync-fault-interface-legacy.test.ts

@@ -3,7 +3,7 @@ import { expect, test } from 'bun:test';
 import * as fs from 'node:fs';
 import * as os from 'node:os';
 import * as path from 'node:path';
-import { docsFaultCallbackScript } from './helpers/docsync-fault-callbacks';
+import { callbackBudgetMs, docsFaultCallbackScript } from './helpers/docsync-fault-callbacks';
 
 const CALLBACK_SPLITS = [
   ['legacy controls', '^ship-docsync-failure rejects ', 12],
@@ -21,7 +21,7 @@ for (const [split, pattern, count] of CALLBACK_SPLITS) test(`registered fault ca
       env: { ...process.env, EVALS: '', EVALS_TIER: '', EVALS_ALL: '', EVALS_RUN_ID: 'free-docsync-faults',
         HOME: home, USERPROFILE: home,
         GSTACK_EVAL_DIR: path.join(dir, 'evidence'), GSTACK_HOME: path.join(dir, 'state') },
-      stdout: 'pipe', stderr: 'pipe', timeout: 120_000,
+      stdout: 'pipe', stderr: 'pipe', timeout: callbackBudgetMs(count),
     });
     const output = result.stdout.toString() + result.stderr.toString();
     expect(result.exitCode, output).toBe(0);
@@ -29,4 +29,4 @@ for (const [split, pattern, count] of CALLBACK_SPLITS) test(`registered fault ca
     expect(output).toContain(`${36 - count} filtered out`);
     expect(output).toContain('0 fail');
   } finally { fs.rmSync(dir, { recursive: true, force: true }); }
-}, 120_000);
+}, callbackBudgetMs(count) + 10_000);

test/docsync-fault-interface.test.ts

@@ -8,7 +8,7 @@ import { DOC_PATH, docsSnapshotArgv, fixtureDocs, gitAt, gitBlobId, repoSnapshot
 import { DOCS_CHECKPOINT_MARKER, DOCS_SEEDED_AUDIT_ID, docsActorCommand, docsActorHook, docsActorSeeded, installDocsActor, seedDocsFirstAttempt, type DocsActorState } from './helpers/docsync-fault-actor';
 import { docsActorVerdict } from './helpers/docsync-fault-eval';
 import { extractDocsDispatch, parseDocsCompletion } from './helpers/docsync-contract';
-import { docsFaultCallbackScript } from './helpers/docsync-fault-callbacks';
+import { callbackBudgetMs, docsFaultCallbackScript } from './helpers/docsync-fault-callbacks';
 import { docsNativeInterface, docsWriteFailures, observeDocsWrites } from './helpers/docsync-observer';
 
 test.skipIf(process.platform !== 'linux')('actor CLI validates owned state before creating even transient locks', async () => {
@@ -485,7 +485,7 @@ for (const [split, pattern, count] of CALLBACK_SPLITS) test(`registered fault ca
       env: { ...process.env, EVALS: '', EVALS_TIER: '', EVALS_ALL: '', EVALS_RUN_ID: 'free-docsync-faults',
         HOME: home, USERPROFILE: home,
         GSTACK_EVAL_DIR: path.join(dir, 'evidence'), GSTACK_HOME: path.join(dir, 'state') },
-      stdout: 'pipe', stderr: 'pipe', timeout: 120_000,
+      stdout: 'pipe', stderr: 'pipe', timeout: callbackBudgetMs(count),
     });
     const output = result.stdout.toString() + result.stderr.toString();
     expect(result.exitCode, output).toBe(0);
@@ -493,4 +493,4 @@ for (const [split, pattern, count] of CALLBACK_SPLITS) test(`registered fault ca
     expect(output).toContain(`${36 - count} filtered out`);
     expect(output).toContain('0 fail');
   } finally { fs.rmSync(dir, { recursive: true, force: true }); }
-}, 120_000);
+}, callbackBudgetMs(count) + 10_000);

test/helpers/docsync-fault-callbacks.ts

@@ -6,6 +6,9 @@
 import * as path from 'node:path';
 import { DOCS_CHECKPOINT_MARKER } from './docsync-fault-actor';
 
+export const CALLBACK_TEST_MS = 30_000;
+export const callbackBudgetMs = (count: number): number => (count + 1) * CALLBACK_TEST_MS;
+
 export function docsFaultCallbackScript(root: string): string {
   return `
 import { expect, mock, test } from 'bun:test';
@@ -20,6 +23,7 @@ const observers = { ...await import(observerModule) };
 const { CAPTURE_MS, CAPTURE_LONG_MS } = await import(path.join(root, 'test/helpers/eval-budgets.ts'));
 const { parseDocsCompletion } = await import(path.join(root, 'test/helpers/docsync-contract.ts'));
 const { docsActorSeeded, DOCS_SEEDED_AUDIT_ID } = await import(path.join(root, 'test/helpers/docsync-fault-actor.ts'));
+const CALLBACK_TEST_MS = ${CALLBACK_TEST_MS};
 const callbacks = new Map();
 let fixture, control = '', launches = 0, recorded, legacyReaudit = false;
 let returnedResult: SkillTestResult | undefined;
@@ -342,7 +346,7 @@ for (const name of names) test(name + ' consumes actual adapter results', async
   expect(recorded).toEqual({ passed: true });
   expect(consumers).toEqual(['logCost', 'recordE2E']);
   expect(fs.existsSync(fixture.home)).toBe(false);
-});
+}, CALLBACK_TEST_MS);
 test('ship-docsync-failure accepts its remaining changed-input audit and preserves raw output in Markdown', async () => {
   control = '';
   legacyReaudit = true;
@@ -353,7 +357,7 @@ test('ship-docsync-failure accepts its remaining changed-input audit and preserv
   expect(recorded).toEqual({ passed: true });
   expect(consumers).toEqual(['logCost', 'recordE2E']);
   expect(fs.existsSync(fixture.home)).toBe(false);
-});
+}, CALLBACK_TEST_MS);
 for (const [name, mutation] of [
   ['ship-docsync-failure', 'false-current'], ['ship-docsync-failure', 'invalid-publication'],
   ['ship-docsync-failure', 'restore-partial'], ['ship-docsync-failure', 'user-content'],
@@ -381,6 +385,6 @@ for (const [name, mutation] of [
   expect(recorded).toEqual({ passed: false });
   expect(consumers).toEqual(['logCost', 'recordE2E']);
   expect(fs.existsSync(fixture.home)).toBe(false);
-});
+}, CALLBACK_TEST_MS);
 `;
 }