fix(gbrain-sync): configurable timeouts + resume from gbrain checkpoint (#1611)

The memory and code stages hardcoded a 35-min spawn timeout. On brains with
~2000+ staged files, /sync-gbrain --full reliably SIGTERM'd the child at
exactly 35 minutes with exit 143. gbrain left ~/.gbrain/import-checkpoint.json
pointing at the staging dir, but gstack-memory-ingest's SIGTERM handler
unconditionally cleaned the dir up — so the next run found a checkpoint
pointing at nothing and restaged from scratch, repeating the SIGTERM forever.

Three changes:

1. Configurable timeouts via env (bounds 60_000ms - 86_400_000ms, default
   2_100_000ms = 35min unchanged):
     GSTACK_SYNC_MEMORY_TIMEOUT_MS
     GSTACK_SYNC_CODE_TIMEOUT_MS
   Out-of-range or non-numeric values warn and fall back to the default.

2. SIGTERM in gstack-memory-ingest no longer always cleans up the staging
   dir. If gbrain has written ~/.gbrain/import-checkpoint.json pointing at
   the active staging dir, the dir is PRESERVED for next-run resume.
   Otherwise (no checkpoint pointing here, crash before gbrain ever
   touched it) it's cleaned up as before.

3. Next /sync-gbrain run detects gbrain's checkpoint via decideResume() in
   gstack-gbrain-sync.ts:
     - no checkpoint               → fresh ingest pass
     - checkpoint + staging ok     → set GSTACK_INGEST_RESUME_DIR; child
                                      reuses staging dir and skips
                                      writeStaged; gbrain import resumes
                                      from processedIndex+1
     - checkpoint + staging gone   → warn "previous checkpoint stale
                                      (staging dir gone), restaging from
                                      scratch" and proceed

Reuses gbrain's own checkpoint as the source of truth (D1 — no double-store
state). Detect-then-fallback semantics per C1.

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>

bin/gstack-gbrain-sync.ts

@@ -80,6 +80,111 @@ const STATE_PATH = join(GSTACK_HOME, ".gbrain-sync-state.json");
 const LOCK_PATH = join(GSTACK_HOME, ".sync-gbrain.lock");
 const STALE_LOCK_MS = 5 * 60 * 1000;
 
+// Default 35-minute timeout for code-walk + memory-ingest stages. Override via
+// GSTACK_SYNC_CODE_TIMEOUT_MS / GSTACK_SYNC_MEMORY_TIMEOUT_MS. Bounds-checked
+// in resolveStageTimeoutMs below so wildly-low values don't make resume
+// useless and wildly-high values don't mask config typos. See #1611.
+const DEFAULT_STAGE_TIMEOUT_MS = 35 * 60 * 1000; // 2_100_000ms = 35min
+const MIN_STAGE_TIMEOUT_MS = 60_000;             // 1 minute floor
+const MAX_STAGE_TIMEOUT_MS = 86_400_000;         // 24 hour ceiling
+
+/**
+ * Parse a stage-timeout env value with bounds validation. Returns the bounded
+ * value or the default with a stderr warning if the env was malformed or
+ * out-of-range. Exported for the regression test.
+ */
+export function resolveStageTimeoutMs(
+  envValue: string | undefined,
+  envName: string,
+): number {
+  if (envValue === undefined || envValue === "") return DEFAULT_STAGE_TIMEOUT_MS;
+  const n = Number.parseInt(envValue, 10);
+  if (!Number.isFinite(n) || Number.isNaN(n) || n <= 0) {
+    console.warn(
+      `[sync] ${envName}="${envValue}" is not a positive integer; falling back to ${DEFAULT_STAGE_TIMEOUT_MS}ms`,
+    );
+    return DEFAULT_STAGE_TIMEOUT_MS;
+  }
+  if (n < MIN_STAGE_TIMEOUT_MS) {
+    console.warn(
+      `[sync] ${envName}=${n} is below the ${MIN_STAGE_TIMEOUT_MS}ms (1min) floor; falling back to ${DEFAULT_STAGE_TIMEOUT_MS}ms`,
+    );
+    return DEFAULT_STAGE_TIMEOUT_MS;
+  }
+  if (n > MAX_STAGE_TIMEOUT_MS) {
+    console.warn(
+      `[sync] ${envName}=${n} is above the ${MAX_STAGE_TIMEOUT_MS}ms (24h) ceiling; falling back to ${DEFAULT_STAGE_TIMEOUT_MS}ms`,
+    );
+    return DEFAULT_STAGE_TIMEOUT_MS;
+  }
+  return n;
+}
+
+/**
+ * gbrain writes ~/.gbrain/import-checkpoint.json on every import run. If a
+ * previous /sync-gbrain hit the timeout (SIGTERM = exit 143), the checkpoint
+ * + its staging dir survive on disk. Detect both and let gbrain resume from
+ * processedIndex+1 on the next run. If the staging dir is missing/empty/
+ * unreadable, fall through to a fresh restage with a one-line warning so the
+ * user sees we noticed. See #1611 + plan D1/C1.
+ */
+interface GbrainCheckpoint {
+  dir?: string;
+  totalFiles?: number;
+  processedIndex?: number;
+  completedFiles?: number;
+  timestamp?: string;
+}
+
+export function readGbrainCheckpoint(): GbrainCheckpoint | null {
+  const cpPath = join(homedir(), ".gbrain", "import-checkpoint.json");
+  if (!existsSync(cpPath)) return null;
+  try {
+    const raw = readFileSync(cpPath, "utf-8");
+    const parsed = JSON.parse(raw);
+    if (!parsed || typeof parsed !== "object") return null;
+    return parsed as GbrainCheckpoint;
+  } catch {
+    // Corrupt JSON — treat as no checkpoint and fall through to fresh restage.
+    return null;
+  }
+}
+
+export type ResumeVerdict =
+  | { kind: "no-checkpoint" }
+  | { kind: "resume"; stagingDir: string; processedIndex: number; totalFiles: number }
+  | { kind: "stale-staging-missing"; stagingDir: string };
+
+/**
+ * Decide whether the next memory-ingest run should resume from gbrain's
+ * checkpoint or restage from scratch.
+ *   - no checkpoint              → run a fresh ingest pass
+ *   - checkpoint + staging ok    → resume (gbrain picks up at processedIndex+1)
+ *   - checkpoint + staging gone  → warn, fall through to fresh restage
+ */
+export function decideResume(): ResumeVerdict {
+  const cp = readGbrainCheckpoint();
+  if (!cp || !cp.dir) return { kind: "no-checkpoint" };
+  const stagingDir = cp.dir;
+  if (!existsSync(stagingDir)) {
+    return { kind: "stale-staging-missing", stagingDir };
+  }
+  // Treat "non-empty" as the safe-to-resume signal. statSync on a missing
+  // file throws; we already handled missing above so this is dir-level shape.
+  try {
+    const st = statSync(stagingDir);
+    if (!st.isDirectory()) return { kind: "stale-staging-missing", stagingDir };
+  } catch {
+    return { kind: "stale-staging-missing", stagingDir };
+  }
+  return {
+    kind: "resume",
+    stagingDir,
+    processedIndex: cp.processedIndex ?? 0,
+    totalFiles: cp.totalFiles ?? 0,
+  };
+}
+
 // ── CLI ────────────────────────────────────────────────────────────────────
 
 function printUsage(): void {
@@ -607,9 +712,13 @@ async function runCodeImport(args: CliArgs): Promise<StageResult> {
   // code`, so --full must run it FIRST, then reindex-code, to honor the
   // documented "full walk + reindex" contract for both fresh and populated
   // sources.
+  const codeTimeoutMs = resolveStageTimeoutMs(
+    process.env.GSTACK_SYNC_CODE_TIMEOUT_MS,
+    "GSTACK_SYNC_CODE_TIMEOUT_MS",
+  );
   const walkResult = spawnGbrain(["sync", "--strategy", "code", "--source", sourceId], {
     stdio: args.quiet ? ["ignore", "ignore", "ignore"] : ["ignore", "inherit", "inherit"],
-    timeout: 35 * 60 * 1000,
+    timeout: codeTimeoutMs,
     baseEnv: gbrainEnv,
   });
 
@@ -627,7 +736,7 @@ async function runCodeImport(args: CliArgs): Promise<StageResult> {
   if (args.mode === "full") {
     const reindexResult = spawnGbrain(["reindex-code", "--source", sourceId, "--yes"], {
       stdio: args.quiet ? ["ignore", "ignore", "ignore"] : ["ignore", "inherit", "inherit"],
-      timeout: 35 * 60 * 1000,
+      timeout: codeTimeoutMs,
       baseEnv: gbrainEnv,
     });
 
@@ -770,6 +879,25 @@ function runMemoryIngest(args: CliArgs): StageResult {
     return skipStageForLocalStatus("memory", localStatus, t0);
   }
 
+  // Resume detection (#1611 / plan D1 + C1). If a previous run hit the
+  // timeout and gbrain left ~/.gbrain/import-checkpoint.json plus its staging
+  // dir on disk, signal the grandchild via env so it skips the prepare phase
+  // and lets `gbrain import` resume from processedIndex+1 against the same
+  // staging dir. If the staging dir is gone (disk pressure cleanup, OS
+  // reboot, user manual cleanup), warn and fall through to a fresh restage.
+  const resume = decideResume();
+  const childEnv = buildGbrainEnv({ announce: false });
+  if (resume.kind === "resume") {
+    console.error(
+      `[sync:memory] resuming from gbrain checkpoint (${resume.processedIndex}/${resume.totalFiles} files staged at ${resume.stagingDir})`,
+    );
+    childEnv.GSTACK_INGEST_RESUME_DIR = resume.stagingDir;
+  } else if (resume.kind === "stale-staging-missing") {
+    console.error(
+      `[sync:memory] previous checkpoint stale (staging dir ${resume.stagingDir} gone), restaging from scratch`,
+    );
+  }
+
   const ingestPath = join(import.meta.dir, "gstack-memory-ingest.ts");
   const ingestArgs = ["run", ingestPath];
   if (args.mode === "full") ingestArgs.push("--bulk");
@@ -780,10 +908,14 @@ function runMemoryIngest(args: CliArgs): StageResult {
   // .env.local footgun affects gstack-memory-ingest.ts too, not just the
   // direct gbrain spawns in this file). The grandchild calls gbrain import
   // internally and must see the DATABASE_URL from gbrain's own config.
+  const memoryTimeoutMs = resolveStageTimeoutMs(
+    process.env.GSTACK_SYNC_MEMORY_TIMEOUT_MS,
+    "GSTACK_SYNC_MEMORY_TIMEOUT_MS",
+  );
   const result = spawnSync("bun", ingestArgs, {
     encoding: "utf-8",
-    timeout: 35 * 60 * 1000,
-    env: buildGbrainEnv({ announce: false }),
+    timeout: memoryTimeoutMs,
+    env: childEnv,
   });
 
   // D6: parse [memory-ingest] lines from the child's stderr. ERR-prefixed