mirror of
https://github.com/garrytan/gstack.git
synced 2026-06-17 15:20:11 +02:00
feat(gbrain-sync): --dream call-graph stage with lock-free gate + honest outcome guard
Adds a source-scoped `gbrain dream --source <id>` stage that builds this worktree's call graph (code-callers/code-callees). Runs lock-free after the sync lock releases so it never blocks sibling worktrees; a .dream-in-progress marker dedupes concurrent dreams. --full auto-runs it only when the cycle was never built; explicit --dream always forces; --no-dream opts out. The stage parses the cycle's own output and reports the truth, not a flat "built": a WARN when the schema pack can't extract code symbols, when the embed phase failed for a missing key, or when 0 edges resolved; OK with the resolved-edge count otherwise. gbrain exits 0 even when it skips on a held cycle lock (e.g. autopilot), so that case reports SKIP, not success. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
Garry Tan
+411
-24
@@ -37,7 +37,7 @@ import { createHash } from "crypto";
|
||||
|
||||
import "../lib/conductor-env-shim";
|
||||
import { detectEngineTier, withErrorContext, canonicalizeRemote } from "../lib/gstack-memory-helpers";
|
||||
import { ensureSourceRegistered, sourcePageCount } from "../lib/gbrain-sources";
|
||||
import { ensureSourceRegistered, sourcePageCount, cycleCompleted, type CycleStatus } from "../lib/gbrain-sources";
|
||||
import { localEngineStatus, type LocalEngineStatus } from "../lib/gbrain-local-status";
|
||||
import { buildGbrainEnv, spawnGbrain, execGbrainJson } from "../lib/gbrain-exec";
|
||||
|
||||
@@ -45,13 +45,17 @@ import { buildGbrainEnv, spawnGbrain, execGbrainJson } from "../lib/gbrain-exec"
|
||||
|
||||
type Mode = "incremental" | "full" | "dry-run";
|
||||
|
||||
interface CliArgs {
|
||||
export interface CliArgs {
|
||||
mode: Mode;
|
||||
quiet: boolean;
|
||||
noCode: boolean;
|
||||
noMemory: boolean;
|
||||
noBrainSync: boolean;
|
||||
codeOnly: boolean;
|
||||
/** Force the brain-global dream cycle (builds the call graph). Always runs. */
|
||||
dream: boolean;
|
||||
/** Opt out of the dream cycle that `--full` would otherwise auto-run. */
|
||||
noDream: boolean;
|
||||
}
|
||||
|
||||
interface CodeStageDetail {
|
||||
@@ -68,6 +72,13 @@ interface StageResult {
|
||||
ok: boolean;
|
||||
duration_ms: number;
|
||||
summary: string;
|
||||
/**
|
||||
* Stage ran and did not error, but the outcome is a degraded no-op the user
|
||||
* should know about (e.g. dream completed but the schema pack can't extract
|
||||
* code symbols, so the call graph stays empty). Rendered as WARN, counts as
|
||||
* ok for the exit code — it's not a failure, just not the happy path.
|
||||
*/
|
||||
warn?: boolean;
|
||||
/** Stage-specific structured detail. Code stage carries source_id + page_count. */
|
||||
detail?: CodeStageDetail;
|
||||
}
|
||||
@@ -80,6 +91,24 @@ const STATE_PATH = join(GSTACK_HOME, ".gbrain-sync-state.json");
|
||||
const LOCK_PATH = join(GSTACK_HOME, ".sync-gbrain.lock");
|
||||
const STALE_LOCK_MS = 5 * 60 * 1000;
|
||||
|
||||
// Dream (call-graph build) is brain-global and runs LOCK-FREE after the sync
|
||||
// lock releases, so it can't use the sync lock to dedupe across worktrees. A
|
||||
// dedicated short-TTL marker prevents two worktrees from launching duplicate
|
||||
// ~35-min global jobs. TTL matches the dream timeout default so a crashed run
|
||||
// can't wedge the marker longer than one cycle.
|
||||
const DEFAULT_DREAM_TIMEOUT_MS = 45 * 60 * 1000; // 45min — dream is the slow stage
|
||||
const DREAM_MARKER_STALE_MS = DEFAULT_DREAM_TIMEOUT_MS;
|
||||
|
||||
/**
|
||||
* Marker path computed fresh per call (not a module const) so tests can mutate
|
||||
* GSTACK_HOME at runtime — same pattern as cacheFilePath() in
|
||||
* lib/gbrain-local-status.ts. Avoids the ESM static-import hoist trap where a
|
||||
* module-load-time const captures the real ~/.gstack before a test can redirect.
|
||||
*/
|
||||
export function dreamMarkerPath(): string {
|
||||
return join(process.env.GSTACK_HOME || join(homedir(), ".gstack"), ".dream-in-progress");
|
||||
}
|
||||
|
||||
// Default 35-minute timeout for code-walk + memory-ingest stages. Override via
|
||||
// GSTACK_SYNC_CODE_TIMEOUT_MS / GSTACK_SYNC_MEMORY_TIMEOUT_MS. Bounds-checked
|
||||
// in resolveStageTimeoutMs below so wildly-low values don't make resume
|
||||
@@ -96,26 +125,27 @@ const MAX_STAGE_TIMEOUT_MS = 86_400_000; // 24 hour ceiling
|
||||
export function resolveStageTimeoutMs(
|
||||
envValue: string | undefined,
|
||||
envName: string,
|
||||
defaultMs: number = DEFAULT_STAGE_TIMEOUT_MS,
|
||||
): number {
|
||||
if (envValue === undefined || envValue === "") return DEFAULT_STAGE_TIMEOUT_MS;
|
||||
if (envValue === undefined || envValue === "") return defaultMs;
|
||||
const n = Number.parseInt(envValue, 10);
|
||||
if (!Number.isFinite(n) || Number.isNaN(n) || n <= 0) {
|
||||
console.warn(
|
||||
`[sync] ${envName}="${envValue}" is not a positive integer; falling back to ${DEFAULT_STAGE_TIMEOUT_MS}ms`,
|
||||
`[sync] ${envName}="${envValue}" is not a positive integer; falling back to ${defaultMs}ms`,
|
||||
);
|
||||
return DEFAULT_STAGE_TIMEOUT_MS;
|
||||
return defaultMs;
|
||||
}
|
||||
if (n < MIN_STAGE_TIMEOUT_MS) {
|
||||
console.warn(
|
||||
`[sync] ${envName}=${n} is below the ${MIN_STAGE_TIMEOUT_MS}ms (1min) floor; falling back to ${DEFAULT_STAGE_TIMEOUT_MS}ms`,
|
||||
`[sync] ${envName}=${n} is below the ${MIN_STAGE_TIMEOUT_MS}ms (1min) floor; falling back to ${defaultMs}ms`,
|
||||
);
|
||||
return DEFAULT_STAGE_TIMEOUT_MS;
|
||||
return defaultMs;
|
||||
}
|
||||
if (n > MAX_STAGE_TIMEOUT_MS) {
|
||||
console.warn(
|
||||
`[sync] ${envName}=${n} is above the ${MAX_STAGE_TIMEOUT_MS}ms (24h) ceiling; falling back to ${DEFAULT_STAGE_TIMEOUT_MS}ms`,
|
||||
`[sync] ${envName}=${n} is above the ${MAX_STAGE_TIMEOUT_MS}ms (24h) ceiling; falling back to ${defaultMs}ms`,
|
||||
);
|
||||
return DEFAULT_STAGE_TIMEOUT_MS;
|
||||
return defaultMs;
|
||||
}
|
||||
return n;
|
||||
}
|
||||
@@ -205,10 +235,17 @@ Options:
|
||||
--no-memory Skip the gstack-memory-ingest stage (transcripts + artifacts).
|
||||
--no-brain-sync Skip the gstack-brain-sync git pipeline stage.
|
||||
--code-only Only run the code-import stage (alias for --no-memory --no-brain-sync).
|
||||
--dream Force the brain-global dream cycle that builds the call
|
||||
graph (gbrain code-callers/code-callees). Runs lock-free
|
||||
AFTER the sync stages. ~minutes. Default timeout 45min,
|
||||
override GSTACK_SYNC_DREAM_TIMEOUT_MS.
|
||||
--no-dream Opt out of the dream cycle that --full would auto-run.
|
||||
--help This text.
|
||||
|
||||
Stages run in order: code → memory ingest → curated git push.
|
||||
Each stage failure is non-fatal; subsequent stages still run.
|
||||
Stages run in order: code → memory ingest → curated git push, then (lock-free)
|
||||
the optional dream call-graph build. --full auto-runs dream ONLY when the call
|
||||
graph was never built; --dream always forces it. Each stage failure is
|
||||
non-fatal; subsequent stages still run.
|
||||
`);
|
||||
}
|
||||
|
||||
@@ -220,6 +257,8 @@ function parseArgs(): CliArgs {
|
||||
let noMemory = false;
|
||||
let noBrainSync = false;
|
||||
let codeOnly = false;
|
||||
let dream = false;
|
||||
let noDream = false;
|
||||
|
||||
for (let i = 0; i < args.length; i++) {
|
||||
const a = args[i];
|
||||
@@ -236,6 +275,10 @@ function parseArgs(): CliArgs {
|
||||
noMemory = true;
|
||||
noBrainSync = true;
|
||||
break;
|
||||
// --dream forces the cycle; --full only chains it at the call site (so
|
||||
// --no-dream can override) — do NOT set dream from --full here.
|
||||
case "--dream": dream = true; break;
|
||||
case "--no-dream": noDream = true; break;
|
||||
case "--help":
|
||||
case "-h":
|
||||
printUsage();
|
||||
@@ -247,7 +290,7 @@ function parseArgs(): CliArgs {
|
||||
}
|
||||
}
|
||||
|
||||
return { mode, quiet, noCode, noMemory, noBrainSync, codeOnly };
|
||||
return { mode, quiet, noCode, noMemory, noBrainSync, codeOnly, dream, noDream };
|
||||
}
|
||||
|
||||
// ── Helpers ────────────────────────────────────────────────────────────────
|
||||
@@ -575,6 +618,58 @@ function releaseLock(): void {
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Acquire the dream marker (`~/.gstack/.dream-in-progress`). Returns false when
|
||||
* a FRESH marker already exists (another worktree is mid-dream) — the caller
|
||||
* then SKIPs rather than launching a duplicate ~35-min global job. A stale
|
||||
* marker (older than DREAM_MARKER_STALE_MS, i.e. a crashed run) is taken over.
|
||||
* Mirrors acquireLock but with the dream TTL and its own path.
|
||||
*/
|
||||
export function acquireDreamMarker(): boolean {
|
||||
const path = dreamMarkerPath();
|
||||
mkdirSync(dirname(path), { recursive: true });
|
||||
if (existsSync(path)) {
|
||||
try {
|
||||
const stat = statSync(path);
|
||||
if (Date.now() - stat.mtimeMs > DREAM_MARKER_STALE_MS) {
|
||||
unlinkSync(path);
|
||||
} else {
|
||||
return false;
|
||||
}
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
const info: LockInfo = { pid: process.pid, started_at: new Date().toISOString() };
|
||||
try {
|
||||
writeFileSync(path, JSON.stringify(info), { encoding: "utf-8", flag: "wx" });
|
||||
return true;
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
export function releaseDreamMarker(): void {
|
||||
try {
|
||||
const path = dreamMarkerPath();
|
||||
if (!existsSync(path)) return;
|
||||
const info = JSON.parse(readFileSync(path, "utf-8")) as LockInfo;
|
||||
if (info.pid === process.pid) unlinkSync(path);
|
||||
} catch {
|
||||
// Best-effort cleanup.
|
||||
}
|
||||
}
|
||||
|
||||
/** Read the pid recorded in a fresh dream marker, for the "already running" message. */
|
||||
function dreamMarkerPid(): number | null {
|
||||
try {
|
||||
const info = JSON.parse(readFileSync(dreamMarkerPath(), "utf-8")) as LockInfo;
|
||||
return typeof info.pid === "number" ? info.pid : null;
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
// ── Stage runners ──────────────────────────────────────────────────────────
|
||||
|
||||
/**
|
||||
@@ -589,7 +684,7 @@ function releaseLock(): void {
|
||||
* broken-db → "config points at unreachable DB; see /setup-gbrain Step 1.5"
|
||||
*/
|
||||
function skipStageForLocalStatus(
|
||||
stage: "code" | "memory",
|
||||
stage: "code" | "memory" | "dream",
|
||||
status: LocalEngineStatus,
|
||||
t0: number,
|
||||
): StageResult {
|
||||
@@ -979,6 +1074,240 @@ function runBrainSyncPush(args: CliArgs): StageResult {
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Decide whether the dream (call-graph build) cycle should run. PURE so the
|
||||
* gate matrix is unit-testable without spawning a real ~35-min dream.
|
||||
*
|
||||
* - explicit --dream → always run (force), regardless of cycle state / --no-code.
|
||||
* - --full → run ONLY when the call graph was never built (cycle === "never"),
|
||||
* and only when not opted out via --no-dream / --no-code. "completed" skips
|
||||
* (edges already built); "unknown" skips (a flaky doctor must not trigger a
|
||||
* surprise 35-min cycle — see gbrain-doctor-overstrict).
|
||||
* - everything else → skip.
|
||||
*
|
||||
* `cycle` is only consulted on the --full auto path; pass null when forcing.
|
||||
*/
|
||||
export function shouldRunDream(args: CliArgs, cycle: CycleStatus | null): boolean {
|
||||
if (args.dream) return true;
|
||||
if (args.mode === "full" && !args.noDream && !args.noCode) {
|
||||
return cycle === "never";
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
/**
|
||||
* Run `gbrain dream` — the brain-global maintenance cycle whose
|
||||
* resolve_symbol_edges phase builds the call graph. Runs LOCK-FREE (called
|
||||
* after the sync lock releases) so it never freezes sibling worktrees; the
|
||||
* `.dream-in-progress` marker dedupes concurrent dreams instead.
|
||||
*
|
||||
* Returns a StageResult (never throws). SKIP (ran:false, ok:true) for: dry-run
|
||||
* preview, local engine not ok, or a fresh marker present. ERR (ran:true,
|
||||
* ok:false) for: non-zero/timeout exit, or a spawn-setup failure (missing
|
||||
* binary / malformed env) — a broken install must be visible, not disguised as
|
||||
* optional maintenance.
|
||||
*/
|
||||
export async function runDream(args: CliArgs): Promise<StageResult> {
|
||||
const t0 = Date.now();
|
||||
|
||||
if (args.mode === "dry-run") {
|
||||
const root = repoRoot();
|
||||
const sourceId = root ? deriveCodeSourceId(root) : null;
|
||||
return {
|
||||
name: "dream",
|
||||
ran: false,
|
||||
ok: true,
|
||||
duration_ms: 0,
|
||||
summary: sourceId
|
||||
? `would: gbrain dream --source ${sourceId} (build this source's call graph)`
|
||||
: "would: gbrain dream (call-graph build)",
|
||||
};
|
||||
}
|
||||
|
||||
const localStatus = localEngineStatus({ noCache: false });
|
||||
if (localStatus !== "ok") {
|
||||
return skipStageForLocalStatus("dream", localStatus, t0);
|
||||
}
|
||||
|
||||
// Dedupe concurrent dreams across worktrees (lock-free path).
|
||||
if (!acquireDreamMarker()) {
|
||||
const pid = dreamMarkerPid();
|
||||
return {
|
||||
name: "dream",
|
||||
ran: false,
|
||||
ok: true,
|
||||
duration_ms: Date.now() - t0,
|
||||
summary: `dream already running${pid !== null ? ` (pid ${pid})` : ""} — skipped`,
|
||||
};
|
||||
}
|
||||
|
||||
try {
|
||||
const dreamTimeoutMs = resolveStageTimeoutMs(
|
||||
process.env.GSTACK_SYNC_DREAM_TIMEOUT_MS,
|
||||
"GSTACK_SYNC_DREAM_TIMEOUT_MS",
|
||||
DEFAULT_DREAM_TIMEOUT_MS,
|
||||
);
|
||||
|
||||
// Scope the cycle to THIS worktree's code source: `gbrain dream --source <id>`.
|
||||
// Verified empirically (not just from `gbrain --help`): plain `gbrain dream`
|
||||
// cycles the brain's default source and never runs the source-scoped `extract`
|
||||
// phase for our code source, so the call graph for the pinned source stays
|
||||
// empty. `gbrain dream --source <id>` runs the per-source cycle (the form
|
||||
// `gbrain doctor` recommends for stale sources) and is what actually populates
|
||||
// code-callers/code-callees for this worktree. Falls back to plain `dream`
|
||||
// only when we can't derive the source id (not in a git repo).
|
||||
const root = repoRoot();
|
||||
const sourceId = root ? deriveCodeSourceId(root) : null;
|
||||
const dreamArgs = sourceId ? ["dream", "--source", sourceId] : ["dream"];
|
||||
|
||||
// spawnGbrain seeds DATABASE_URL from gbrain's config via buildGbrainEnv.
|
||||
//
|
||||
// We CAPTURE output (pipe) rather than inherit because `gbrain dream` exits 0
|
||||
// even when it SKIPS the cycle — when another cycle already holds gbrain's own
|
||||
// DB lock (e.g. a running `gbrain autopilot`), it prints "Skipped: another
|
||||
// cycle is already running. (locked)" and exits 0. Trusting the exit code
|
||||
// alone would falsely report "call graph built". Trade-off: no live streaming
|
||||
// for a long cycle; we echo the captured output afterward instead.
|
||||
if (!args.quiet) {
|
||||
process.stderr.write("[dream] running gbrain cycle (call-graph build; this can take a few minutes)...\n");
|
||||
}
|
||||
let result: ReturnType<typeof spawnGbrain>;
|
||||
try {
|
||||
result = spawnGbrain(dreamArgs, {
|
||||
stdio: ["ignore", "pipe", "pipe"],
|
||||
timeout: dreamTimeoutMs,
|
||||
baseEnv: process.env,
|
||||
announce: !args.quiet,
|
||||
});
|
||||
} catch (err) {
|
||||
// Spawn-setup failure (missing binary, bad env): ERR, not a benign skip.
|
||||
return {
|
||||
name: "dream",
|
||||
ran: true,
|
||||
ok: false,
|
||||
duration_ms: Date.now() - t0,
|
||||
summary: `gbrain dream failed to start: ${(err as Error).message}`,
|
||||
};
|
||||
}
|
||||
|
||||
if (result.error) {
|
||||
const e = result.error as NodeJS.ErrnoException;
|
||||
const why = e.code === "ENOENT" ? "gbrain not on PATH" : e.message;
|
||||
return {
|
||||
name: "dream",
|
||||
ran: true,
|
||||
ok: false,
|
||||
duration_ms: Date.now() - t0,
|
||||
summary: `gbrain dream failed to start: ${why}`,
|
||||
};
|
||||
}
|
||||
|
||||
const out = `${result.stdout || ""}${result.stderr || ""}`;
|
||||
if (!args.quiet && out.trim()) {
|
||||
process.stderr.write(out.endsWith("\n") ? out : `${out}\n`);
|
||||
}
|
||||
|
||||
if (result.status !== 0) {
|
||||
return {
|
||||
name: "dream",
|
||||
ran: true,
|
||||
ok: false,
|
||||
duration_ms: Date.now() - t0,
|
||||
summary: `gbrain dream exited ${result.status === null ? "null (killed by signal / timeout)" : result.status}`,
|
||||
};
|
||||
}
|
||||
|
||||
// Exit 0 but the cycle was SKIPPED because gbrain's own lock is held by
|
||||
// another cycle (typically `gbrain autopilot`). Report SKIP, not "built" —
|
||||
// the graph builds on that other cycle, not this invocation.
|
||||
if (/already running|\block(?:ed)?\b|Skipped:/i.test(out)) {
|
||||
return {
|
||||
name: "dream",
|
||||
ran: false,
|
||||
ok: true,
|
||||
duration_ms: Date.now() - t0,
|
||||
summary: "skipped — a gbrain cycle is already running (e.g. autopilot); the call graph builds on that cycle",
|
||||
};
|
||||
}
|
||||
|
||||
// Exit 0 and the cycle actually ran. Parse the cycle's OWN output to report
|
||||
// the truth, not a flat "built": `gbrain dream` exits 0 even when the call
|
||||
// graph could not be built, and a misleading "built" turns a multi-minute
|
||||
// no-op into a silent dead end. gbrain only surfaces these conditions in the
|
||||
// cycle log (there is no pre-flight pack-capability query as of 0.41.x), so
|
||||
// string-matching the log is the available signal; an unrecognized log
|
||||
// degrades to the generic success summary below.
|
||||
const dreamWarn = classifyDreamOutcome(out);
|
||||
if (dreamWarn) {
|
||||
return {
|
||||
name: "dream",
|
||||
ran: true,
|
||||
ok: true,
|
||||
warn: true,
|
||||
duration_ms: Date.now() - t0,
|
||||
summary: dreamWarn,
|
||||
};
|
||||
}
|
||||
|
||||
const edges = parseResolvedEdges(out);
|
||||
return {
|
||||
name: "dream",
|
||||
ran: true,
|
||||
ok: true,
|
||||
duration_ms: Date.now() - t0,
|
||||
summary:
|
||||
edges !== null
|
||||
? `call graph built (${edges} edge${edges === 1 ? "" : "s"} resolved)`
|
||||
: "call graph built (resolve_symbol_edges complete)",
|
||||
};
|
||||
} finally {
|
||||
releaseDreamMarker();
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Parse `<n>` from a `resolve_symbol_edges ... resolved <n>` cycle-log line.
|
||||
* Returns null when the line is absent (older gbrain / different pack). The
|
||||
* `[^\n]*?` is newline-bounded so it matches the `✓ resolve_symbol_edges ...`
|
||||
* summary line, not the bracketed `[cycle.resolve_symbol_edges] start` markers.
|
||||
*/
|
||||
export function parseResolvedEdges(out: string): number | null {
|
||||
const m = out.match(/resolve_symbol_edges\b[^\n]*?\bresolved\s+(\d+)/i);
|
||||
return m ? parseInt(m[1], 10) : null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Inspect a completed (exit-0) `gbrain dream` log and return a WARN summary when
|
||||
* the cycle ran but could not actually build the call graph. Returns null on the
|
||||
* happy path (caller emits the normal "call graph built" summary). Order matters:
|
||||
* the pack-capability gap is the most actionable, so it wins over a 0-edge count
|
||||
* (both appear together when the pack lacks the code-symbol phase).
|
||||
*/
|
||||
export function classifyDreamOutcome(out: string): string | null {
|
||||
// The active schema pack doesn't declare the code-symbol extraction phase, so
|
||||
// no symbols are extracted and resolve_symbol_edges has nothing to match.
|
||||
if (/does not declare this phase/i.test(out)) {
|
||||
return (
|
||||
"dream ran, but this source's schema pack does not extract code symbols, " +
|
||||
"so the call graph stays empty. Switch this source to a code-aware schema " +
|
||||
"pack (`gbrain schema use <pack>`) to enable code-callers/code-callees."
|
||||
);
|
||||
}
|
||||
// The embed phase failed for a missing key; symbols can't index without it.
|
||||
if (/embed phase failed/i.test(out) || /requires\s+\S*_API_KEY/i.test(out)) {
|
||||
return (
|
||||
"dream ran, but the embed phase failed (missing embedding API key), so " +
|
||||
"symbols won't index. Ensure the embedding provider's key is set for the " +
|
||||
"gbrain process, then re-run /sync-gbrain --dream."
|
||||
);
|
||||
}
|
||||
// Cycle ran and embedded fine, but matched zero call-graph edges.
|
||||
if (parseResolvedEdges(out) === 0) {
|
||||
return "dream ran but resolved 0 call-graph edges (no code symbols matched for this source yet).";
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
// ── State file ─────────────────────────────────────────────────────────────
|
||||
|
||||
interface SyncState {
|
||||
@@ -1017,10 +1346,28 @@ function saveSyncState(state: SyncState): void {
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Persist the dream stage result with read-modify-write semantics.
|
||||
*
|
||||
* Dream runs AFTER the sync lock releases, so a sibling worktree may have
|
||||
* written newer state in the meantime. Overwriting the whole file with our
|
||||
* pre-dream snapshot + dream result would clobber that sibling's sync. Instead
|
||||
* re-read the CURRENT state, replace only the `dream` entry in last_stages, and
|
||||
* atomic-rename. (Atomic rename alone isn't race-safe; the re-read + targeted
|
||||
* merge is what prevents the clobber.)
|
||||
*/
|
||||
function mergeDreamIntoState(dream: StageResult): void {
|
||||
const fresh = loadSyncState();
|
||||
const others = (fresh.last_stages || []).filter((s) => s.name !== "dream");
|
||||
fresh.last_stages = [...others, dream];
|
||||
fresh.last_sync = new Date().toISOString();
|
||||
saveSyncState(fresh);
|
||||
}
|
||||
|
||||
// ── Output ─────────────────────────────────────────────────────────────────
|
||||
|
||||
function formatStage(s: StageResult): string {
|
||||
const status = !s.ran ? "SKIP" : s.ok ? "OK" : "ERR";
|
||||
export function formatStage(s: StageResult): string {
|
||||
const status = !s.ran ? "SKIP" : !s.ok ? "ERR" : s.warn ? "WARN" : "OK";
|
||||
const dur = s.duration_ms > 0 ? ` (${(s.duration_ms / 1000).toFixed(1)}s)` : "";
|
||||
return ` ${status.padEnd(5)} ${s.name.padEnd(12)} ${s.summary}${dur}`;
|
||||
}
|
||||
@@ -1056,9 +1403,9 @@ async function main(): Promise<void> {
|
||||
process.on("SIGTERM", () => { cleanup(); process.exit(143); });
|
||||
|
||||
let exitCode = 0;
|
||||
const stages: StageResult[] = [];
|
||||
try {
|
||||
const state = loadSyncState();
|
||||
const stages: StageResult[] = [];
|
||||
|
||||
if (!args.noCode) {
|
||||
stages.push(await withErrorContext("sync:code", () => runCodeImport(args), "gstack-gbrain-sync"));
|
||||
@@ -1077,20 +1424,60 @@ async function main(): Promise<void> {
|
||||
saveSyncState(state);
|
||||
}
|
||||
|
||||
if (!args.quiet || args.mode === "dry-run") {
|
||||
console.log(`\ngstack-gbrain-sync (${args.mode}):`);
|
||||
for (const s of stages) console.log(formatStage(s));
|
||||
const okCount = stages.filter((s) => s.ok).length;
|
||||
const errCount = stages.filter((s) => !s.ok && s.ran).length;
|
||||
console.log(`\n ${okCount} ok, ${errCount} error, ${stages.length - okCount - errCount} skipped`);
|
||||
}
|
||||
|
||||
const anyError = stages.some((s) => s.ran && !s.ok);
|
||||
exitCode = anyError ? 1 : 0;
|
||||
} finally {
|
||||
// Release the sync lock BEFORE the dream cycle. Dream is brain-global and
|
||||
// can run ~35 min; holding the machine-wide lock that long would freeze
|
||||
// every other worktree's /sync-gbrain. Dream is guarded by its own marker.
|
||||
cleanup();
|
||||
}
|
||||
|
||||
// ── Dream (call-graph build) — LOCK-FREE, after the sync lock releases ─────
|
||||
let dreamStage: StageResult | null = null;
|
||||
if (args.mode === "dry-run") {
|
||||
// Preview only; never probes doctor or spawns. `--dry-run` and `--full` are
|
||||
// mutually exclusive modes (last one wins in parseArgs), so the only dream
|
||||
// preview that applies to a dry-run is the explicit --dream force.
|
||||
if (args.dream) {
|
||||
dreamStage = await runDream(args);
|
||||
}
|
||||
} else {
|
||||
// Resolve cycle state only on the --full auto path (perf: the steady-state
|
||||
// incremental sync never pays a doctor subprocess). Explicit --dream forces.
|
||||
let cycle: CycleStatus | null = null;
|
||||
if (!args.dream && args.mode === "full" && !args.noDream && !args.noCode) {
|
||||
const root = repoRoot();
|
||||
cycle = root ? cycleCompleted(deriveCodeSourceId(root), process.env) : "unknown";
|
||||
}
|
||||
if (shouldRunDream(args, cycle)) {
|
||||
dreamStage = await runDream(args);
|
||||
mergeDreamIntoState(dreamStage);
|
||||
if (dreamStage.ran && !dreamStage.ok) exitCode = 1;
|
||||
} else if (cycle === "unknown") {
|
||||
// --full wanted to auto-build but doctor couldn't confirm the graph state.
|
||||
// Surface a WARN-style SKIP so the user knows to run --dream if needed,
|
||||
// rather than silently doing nothing (a flaky doctor must not trigger a
|
||||
// surprise 35-min run — gbrain-doctor-overstrict).
|
||||
dreamStage = {
|
||||
name: "dream",
|
||||
ran: false,
|
||||
ok: true,
|
||||
duration_ms: 0,
|
||||
summary: "call-graph state unknown (doctor unavailable) — run /sync-gbrain --dream if code-callers returns 0",
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
if (!args.quiet || args.mode === "dry-run") {
|
||||
const allStages = dreamStage ? [...stages, dreamStage] : stages;
|
||||
console.log(`\ngstack-gbrain-sync (${args.mode}):`);
|
||||
for (const s of allStages) console.log(formatStage(s));
|
||||
const okCount = allStages.filter((s) => s.ok).length;
|
||||
const errCount = allStages.filter((s) => !s.ok && s.ran).length;
|
||||
console.log(`\n ${okCount} ok, ${errCount} error, ${allStages.length - okCount - errCount} skipped`);
|
||||
}
|
||||
|
||||
process.exit(exitCode);
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,250 @@
|
||||
/**
|
||||
* Tests for the dream (call-graph build) stage of bin/gstack-gbrain-sync.ts.
|
||||
*
|
||||
* We deliberately do NOT exercise the real `gbrain dream` spawn here — that's a
|
||||
* ~35-min brain-global job and must never run in CI. Instead we cover:
|
||||
* 1. shouldRunDream() — the pure gate matrix (issues 1/2/4). Highest-risk logic.
|
||||
* 2. runDream() dry-run — returns a preview before any engine probe / spawn.
|
||||
* 3. Dream marker (acquire/release/stale-takeover) — the concurrency guard.
|
||||
* 4. CLI gate wiring via --dry-run subprocess (safe: dry-run never spawns dream).
|
||||
*
|
||||
* The live spawn + lock-free ordering + serialization are covered by the manual
|
||||
* E2E verification in the plan (running the orchestrator against a real brain),
|
||||
* not by a unit test that could launch a real dream.
|
||||
*/
|
||||
|
||||
import { describe, it, expect, afterEach } from "bun:test";
|
||||
import { mkdtempSync, existsSync, writeFileSync, utimesSync, rmSync } from "fs";
|
||||
import { tmpdir } from "os";
|
||||
import { join } from "path";
|
||||
import { spawnSync } from "child_process";
|
||||
|
||||
import {
|
||||
shouldRunDream,
|
||||
runDream,
|
||||
acquireDreamMarker,
|
||||
releaseDreamMarker,
|
||||
dreamMarkerPath,
|
||||
classifyDreamOutcome,
|
||||
parseResolvedEdges,
|
||||
formatStage,
|
||||
type CliArgs,
|
||||
} from "../bin/gstack-gbrain-sync";
|
||||
|
||||
const SCRIPT = join(import.meta.dir, "..", "bin", "gstack-gbrain-sync.ts");
|
||||
|
||||
/** Build a CliArgs with all flags off, overriding only what a case needs. */
|
||||
function args(overrides: Partial<CliArgs> = {}): CliArgs {
|
||||
return {
|
||||
mode: "incremental",
|
||||
quiet: false,
|
||||
noCode: false,
|
||||
noMemory: false,
|
||||
noBrainSync: false,
|
||||
codeOnly: false,
|
||||
dream: false,
|
||||
noDream: false,
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
describe("shouldRunDream — gate matrix", () => {
|
||||
it("explicit --dream always runs (cycle irrelevant)", () => {
|
||||
expect(shouldRunDream(args({ dream: true }), null)).toBe(true);
|
||||
expect(shouldRunDream(args({ dream: true }), "completed")).toBe(true);
|
||||
expect(shouldRunDream(args({ dream: true }), "never")).toBe(true);
|
||||
expect(shouldRunDream(args({ dream: true }), "unknown")).toBe(true);
|
||||
});
|
||||
|
||||
it("explicit --dream runs even with --code-only / --no-code (force)", () => {
|
||||
expect(shouldRunDream(args({ dream: true, codeOnly: true, noMemory: true, noBrainSync: true }), null)).toBe(true);
|
||||
expect(shouldRunDream(args({ dream: true, noCode: true }), null)).toBe(true);
|
||||
});
|
||||
|
||||
it("--full auto-runs ONLY when the cycle was never built", () => {
|
||||
expect(shouldRunDream(args({ mode: "full" }), "never")).toBe(true);
|
||||
expect(shouldRunDream(args({ mode: "full" }), "completed")).toBe(false);
|
||||
expect(shouldRunDream(args({ mode: "full" }), "unknown")).toBe(false);
|
||||
expect(shouldRunDream(args({ mode: "full" }), null)).toBe(false);
|
||||
});
|
||||
|
||||
it("--full + --no-dream never auto-runs", () => {
|
||||
expect(shouldRunDream(args({ mode: "full", noDream: true }), "never")).toBe(false);
|
||||
});
|
||||
|
||||
it("--full + --no-code never auto-runs", () => {
|
||||
expect(shouldRunDream(args({ mode: "full", noCode: true }), "never")).toBe(false);
|
||||
});
|
||||
|
||||
it("plain incremental never runs (no flag, no full)", () => {
|
||||
expect(shouldRunDream(args(), "never")).toBe(false);
|
||||
expect(shouldRunDream(args(), null)).toBe(false);
|
||||
});
|
||||
});
|
||||
|
||||
describe("runDream — dry-run preview", () => {
|
||||
it("returns a 'would' preview without spawning (ran=false, ok=true)", async () => {
|
||||
const r = await runDream(args({ mode: "dry-run", dream: true }));
|
||||
expect(r.name).toBe("dream");
|
||||
expect(r.ran).toBe(false);
|
||||
expect(r.ok).toBe(true);
|
||||
expect(r.summary).toContain("would: gbrain dream");
|
||||
});
|
||||
});
|
||||
|
||||
describe("dream marker — concurrency guard", () => {
|
||||
const saved = process.env.GSTACK_HOME;
|
||||
let tmp: string;
|
||||
|
||||
afterEach(() => {
|
||||
if (tmp) rmSync(tmp, { recursive: true, force: true });
|
||||
if (saved === undefined) delete process.env.GSTACK_HOME;
|
||||
else process.env.GSTACK_HOME = saved;
|
||||
});
|
||||
|
||||
function redirectHome(): void {
|
||||
tmp = mkdtempSync(join(tmpdir(), "gbrain-dream-marker-"));
|
||||
process.env.GSTACK_HOME = tmp;
|
||||
}
|
||||
|
||||
it("acquire creates the marker; a second acquire on a fresh marker fails", () => {
|
||||
redirectHome();
|
||||
expect(acquireDreamMarker()).toBe(true);
|
||||
expect(existsSync(dreamMarkerPath())).toBe(true);
|
||||
// Fresh marker present → a concurrent worktree must NOT launch a duplicate.
|
||||
expect(acquireDreamMarker()).toBe(false);
|
||||
});
|
||||
|
||||
it("release removes the marker (same pid)", () => {
|
||||
redirectHome();
|
||||
expect(acquireDreamMarker()).toBe(true);
|
||||
releaseDreamMarker();
|
||||
expect(existsSync(dreamMarkerPath())).toBe(false);
|
||||
});
|
||||
|
||||
it("a stale marker (older than TTL) is taken over", () => {
|
||||
redirectHome();
|
||||
// Plant a marker with an mtime ~46 min in the past (TTL is 45 min).
|
||||
const path = dreamMarkerPath();
|
||||
writeFileSync(path, JSON.stringify({ pid: 999999, started_at: "old" }));
|
||||
const old = new Date(Date.now() - 46 * 60 * 1000);
|
||||
utimesSync(path, old, old);
|
||||
expect(acquireDreamMarker()).toBe(true); // takeover
|
||||
expect(existsSync(path)).toBe(true);
|
||||
});
|
||||
});
|
||||
|
||||
describe("CLI gate wiring (dry-run subprocess — never spawns a real dream)", () => {
|
||||
// NOTE: we only pass --dry-run (optionally + --dream). We must NOT pass
|
||||
// --full here: parseArgs is last-mode-wins, so `--dry-run --full` resolves to
|
||||
// mode=full and would run a REAL ~minutes full sync + reindex. The --full
|
||||
// auto-chain gate is covered purely by the shouldRunDream matrix above.
|
||||
function run(extra: string[]): string {
|
||||
const r = spawnSync("bun", [SCRIPT, "--dry-run", ...extra], {
|
||||
encoding: "utf-8",
|
||||
timeout: 60000,
|
||||
env: { ...process.env },
|
||||
});
|
||||
return (r.stdout || "") + (r.stderr || "");
|
||||
}
|
||||
|
||||
it("--dry-run --dream shows the dream preview row", () => {
|
||||
expect(run(["--dream"])).toContain("would: gbrain dream");
|
||||
});
|
||||
|
||||
it("plain --dry-run (incremental) omits the dream row", () => {
|
||||
expect(run([])).not.toContain("would: gbrain dream");
|
||||
});
|
||||
});
|
||||
|
||||
// Canned `gbrain dream` cycle logs (verbatim shapes observed against a real
|
||||
// 0.41.x brain). These let us test the post-flight guard WITHOUT a real cycle.
|
||||
const LOG = {
|
||||
// Pack lacks the code-symbol phase: extract_atoms is undeclared AND the edge
|
||||
// resolver matches nothing. Both signals present — pack message must win.
|
||||
notCodeAware:
|
||||
"[cycle.extract] done\n" +
|
||||
" - extract_atoms extract_atoms: active pack does not declare this phase\n" +
|
||||
"[cycle.resolve_symbol_edges] start\n" +
|
||||
"[cycle.resolve_symbol_edges] done\n" +
|
||||
" ✓ resolve_symbol_edges 3864 chunk(s) walked; resolved 0, ambiguous 0, unmatched 0\n" +
|
||||
" totals: extracted=0 embedded=1\n",
|
||||
// Embed phase failed for a missing key (isolated: no pack-capability line).
|
||||
embedFailed:
|
||||
"[cycle.embed] start\n" +
|
||||
"[cycle.embed] done\n" +
|
||||
" ✗ embed embed phase failed\n" +
|
||||
' [LLMError/UNKNOWN] Embedding model "openai:text-embedding-3-large" requires OPENAI_API_KEY.\n' +
|
||||
" totals: extracted=0 embedded=0\n",
|
||||
// Cycle ran clean but matched zero edges (no other failure signal).
|
||||
zeroEdges:
|
||||
" ✓ resolve_symbol_edges 120 chunk(s) walked; resolved 0, ambiguous 0, unmatched 0\n",
|
||||
// Happy path: edges resolved.
|
||||
builtEdges:
|
||||
" ✓ resolve_symbol_edges 500 chunk(s) walked; resolved 42, ambiguous 3, unmatched 1\n",
|
||||
// Old gbrain / different pack: no resolve_symbol_edges summary line at all.
|
||||
noEdgeLine: "[cycle.lint] done\n[cycle.sync] done\n totals: lint=53\n",
|
||||
};
|
||||
|
||||
describe("parseResolvedEdges", () => {
|
||||
it("reads the resolved count from the ✓ summary line", () => {
|
||||
expect(parseResolvedEdges(LOG.builtEdges)).toBe(42);
|
||||
expect(parseResolvedEdges(LOG.zeroEdges)).toBe(0);
|
||||
});
|
||||
it("returns null when there is no resolve_symbol_edges summary", () => {
|
||||
expect(parseResolvedEdges(LOG.noEdgeLine)).toBeNull();
|
||||
});
|
||||
it("does not match the bracketed [cycle.resolve_symbol_edges] marker lines", () => {
|
||||
// Markers have no 'resolved N' on the same line, so they must not match.
|
||||
const markersOnly = "[cycle.resolve_symbol_edges] start\n[cycle.resolve_symbol_edges] done\n";
|
||||
expect(parseResolvedEdges(markersOnly)).toBeNull();
|
||||
});
|
||||
});
|
||||
|
||||
describe("classifyDreamOutcome — post-flight truth guard", () => {
|
||||
it("flags a non-code-aware schema pack (wins over the 0-edge signal)", () => {
|
||||
const w = classifyDreamOutcome(LOG.notCodeAware);
|
||||
expect(w).not.toBeNull();
|
||||
expect(w).toContain("schema pack");
|
||||
expect(w).toContain("code-aware");
|
||||
});
|
||||
|
||||
it("flags a failed embed phase / missing embedding key", () => {
|
||||
const w = classifyDreamOutcome(LOG.embedFailed);
|
||||
expect(w).not.toBeNull();
|
||||
expect(w).toContain("embed");
|
||||
expect(w!.toLowerCase()).toContain("key");
|
||||
});
|
||||
|
||||
it("flags a clean cycle that resolved 0 edges", () => {
|
||||
const w = classifyDreamOutcome(LOG.zeroEdges);
|
||||
expect(w).not.toBeNull();
|
||||
expect(w).toContain("0 call-graph edges");
|
||||
});
|
||||
|
||||
it("returns null on the happy path (edges resolved)", () => {
|
||||
expect(classifyDreamOutcome(LOG.builtEdges)).toBeNull();
|
||||
});
|
||||
|
||||
it("returns null when no recognizable signal is present (degrade to success)", () => {
|
||||
expect(classifyDreamOutcome(LOG.noEdgeLine)).toBeNull();
|
||||
});
|
||||
});
|
||||
|
||||
describe("formatStage — WARN render", () => {
|
||||
const base = { name: "dream", duration_ms: 0, summary: "x" };
|
||||
it("renders WARN for a ran+ok+warn stage (degraded no-op)", () => {
|
||||
expect(formatStage({ ...base, ran: true, ok: true, warn: true })).toContain("WARN");
|
||||
});
|
||||
it("renders OK for a ran+ok stage without warn", () => {
|
||||
const s = formatStage({ ...base, ran: true, ok: true });
|
||||
expect(s).toContain("OK");
|
||||
expect(s).not.toContain("WARN");
|
||||
});
|
||||
it("renders ERR for a ran+!ok stage even if warn is set", () => {
|
||||
expect(formatStage({ ...base, ran: true, ok: false, warn: true })).toContain("ERR");
|
||||
});
|
||||
it("renders SKIP for a !ran stage", () => {
|
||||
expect(formatStage({ ...base, ran: false, ok: true })).toContain("SKIP");
|
||||
});
|
||||
});
|
||||
Reference in New Issue
Block a user