CalvinBackup/gstack
1
0
Fork 0
mirror of https://github.com/garrytan/gstack.git synced 2026-06-27 20:20:03 +02:00
Code Issues Packages Projects Releases Wiki Activity

feat(gbrain): orchestrator SKIP when local engine not ok + remote-http transcripts via artifacts pipeline

Browse Source 
Two changes in the sync orchestrator, both per plan D11/D12:

1. bin/gstack-gbrain-sync.ts: runCodeImport + runMemoryIngest call
   localEngineStatus() (shared classifier from lib/gbrain-local-status.ts).
   When status is not 'ok', return a SKIP stage result with a clear reason
   instead of crashing with "source registration failed: gbrain not
   configured". Brain-sync stage runs regardless — it doesn't depend on
   local engine. dry-run preview path is gated above the check so it
   continues to show would-do steps even when the engine is broken.

2. bin/gstack-memory-ingest.ts: when gbrain MCP is registered as
   remote-http (Path 4), persist staged transcripts to
   ~/.gstack/transcripts/run-<pid>-<ts>/ instead of the ephemeral
   ~/.gstack/.staging-ingest-<pid>-<ts>/ tmp dir, and SKIP the local
   `gbrain import` call entirely. The artifacts pipeline (gstack-brain-sync
   push to git, brain admin pulls and indexes) handles routing to the
   remote brain. Local PGLite (when present via Step 4.5) stays code-only.

State recording still happens — prepared pages get their mtime+sha256
stamped under remote-http mode so the next /sync-gbrain doesn't
re-stage them. Cleanup is skipped intentionally so the persisted dir
survives until gstack-brain-sync moves it.

Adds test/gbrain-sync-skip.test.ts covering 5 SKIP scenarios (broken-db,
broken-config, no-cli, missing-config, ok pass-through). All 25
sync-related unit tests pass.

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
Garry Tan
2026-05-13 11:30:19 -07:00
parent 3cf2885bae
commit f35afb7b1f
3 changed files with 370 additions and 5 deletions
Download Patch File Download Diff File Expand all files Collapse all files
bin/gstack-gbrain-sync.ts
+60
View File
@@ -37,6 +37,7 @@ import { createHash } from "crypto";
import { detectEngineTier, withErrorContext, canonicalizeRemote } from "../lib/gstack-memory-helpers";
import { ensureSourceRegistered, sourcePageCount } from "../lib/gbrain-sources";
import { localEngineStatus, type LocalEngineStatus } from "../lib/gbrain-local-status";
// ── Types ──────────────────────────────────────────────────────────────────
@@ -290,6 +291,42 @@ function releaseLock(): void {
// ── Stage runners ──────────────────────────────────────────────────────────
/**
* Build a SKIP result for the code/memory stage when the local engine is
* not in 'ok' state (per plan D12). Surface the status verbatim so the
* verdict block tells the user exactly what's wrong without re-probing.
*
* Reasons mapped to user-actionable summaries:
* no-cli → "gbrain CLI not on PATH; install via /setup-gbrain"
* missing-config → "no local engine; run /setup-gbrain to add local PGLite"
* broken-config → "config file at ~/.gbrain/config.json is malformed; see /setup-gbrain Step 1.5"
* broken-db → "config points at unreachable DB; see /setup-gbrain Step 1.5"
*/
function skipStageForLocalStatus(
stage: "code" | "memory",
status: LocalEngineStatus,
t0: number,
): StageResult {
const reasons: Record<Exclude<LocalEngineStatus, "ok">, string> = {
"no-cli": "gbrain CLI not on PATH; install via /setup-gbrain",
"missing-config":
"no local engine; run /setup-gbrain to add local PGLite for code search",
"broken-config":
"config at ~/.gbrain/config.json is malformed; see /setup-gbrain Step 1.5",
"broken-db":
"config points at unreachable DB; see /setup-gbrain Step 1.5",
};
const reason = reasons[status as Exclude<LocalEngineStatus, "ok">];
return {
name: stage,
ran: false,
ok: true, // SKIP (per D12) — not a stage failure, just an unsatisfied prerequisite
duration_ms: Date.now() - t0,
summary: `skipped — local engine ${status} — ${reason}`,
};
}
async function runCodeImport(args: CliArgs): Promise<StageResult> {
const t0 = Date.now();
const root = repoRoot();
@@ -302,6 +339,9 @@ async function runCodeImport(args: CliArgs): Promise<StageResult> {
const sourceId = deriveCodeSourceId(root);
// dry-run preview always shows the would-do steps, regardless of local
// engine state. Useful for "what would /sync-gbrain do" without probing
// the engine.
if (args.mode === "dry-run") {
return {
name: "code",
@@ -313,6 +353,17 @@ async function runCodeImport(args: CliArgs): Promise<StageResult> {
};
}
// Split-engine pre-flight (per plan D12): when local engine is not ok, SKIP
// code stage cleanly. Brain-sync stage still runs because it doesn't depend
// on local engine. The /sync-gbrain Step 1.5 pre-flight surfaces the user
// remediation message; this skip just keeps the orchestrator from crashing
// when the local DB is dead. Skipped on --dry-run (above) since dry-run
// never actually probes anything.
const localStatus = localEngineStatus({ noCache: false });
if (localStatus !== "ok") {
return skipStageForLocalStatus("code", localStatus, t0);
}
// Step 0: Best-effort cleanup of pre-pathhash legacy source.
// Earlier /sync-gbrain versions registered `gstack-code-<slug>` (no path
// suffix). On a multi-worktree repo, those collapsed onto a single id
@@ -431,6 +482,15 @@ function runMemoryIngest(args: CliArgs): StageResult {
return { name: "memory", ran: false, ok: true, duration_ms: 0, summary: "would: gstack-memory-ingest --probe" };
}
// Split-engine pre-flight (per plan D12). gstack-memory-ingest shells out
// to `gbrain import` which targets the LOCAL engine. When that engine is
// not ok, SKIP cleanly so brain-sync (the only stage that doesn't depend
// on local engine) still runs.
const localStatus = localEngineStatus({ noCache: false });
if (localStatus !== "ok") {
return skipStageForLocalStatus("memory", localStatus, t0);
}
const ingestPath = join(import.meta.dir, "gstack-memory-ingest.ts");
const ingestArgs = ["run", ingestPath];
if (args.mode === "full") ingestArgs.push("--bulk");
bin/gstack-memory-ingest.ts
+119 -5
View File
@@ -1202,6 +1202,57 @@ function makeStagingDir(): string {
return dir;
}
/**
* Persistent staging dir used in remote-http MCP mode (split-engine D11).
*
* Instead of staging to ~/.gstack/.staging-ingest-<pid>-<ts>/ and cleaning up
* after `gbrain import`, remote-http users get a stable path that survives.
* gstack-brain-sync's allowlist pushes ~/.gstack/transcripts/** to the
* artifacts repo; the brain admin's pull job indexes them into the remote
* brain. Local PGLite (if present) stays code-only.
*
* Path: ~/.gstack/transcripts/<run-id>/ (run-id pid+ts so concurrent passes
* stay separate; brain-sync push doesn't care about subdir naming).
*/
function makePersistentTranscriptDir(): string {
const dir = join(
GSTACK_HOME,
"transcripts",
`run-${process.pid}-${Date.now()}`,
);
mkdirSync(dir, { recursive: true });
return dir;
}
/**
* Detect whether the gbrain MCP is remote-http (Path 4) — and therefore we
* should NOT call `gbrain import` because we don't want the local PGLite
* polluted with transcripts (per plan D11).
*
* Reads ~/.claude.json directly (same fallback chain as gstack-gbrain-detect
* Tier 3). Cheap: one fs read, no fork-exec.
*/
function isRemoteHttpMcpMode(): boolean {
const home = process.env.HOME || homedir();
const claudeJsonPath = join(home, ".claude.json");
if (!existsSync(claudeJsonPath)) return false;
try {
const parsed = JSON.parse(readFileSync(claudeJsonPath, "utf-8")) as {
mcpServers?: {
gbrain?: { type?: string; transport?: string; url?: string };
};
};
const entry = parsed.mcpServers?.gbrain;
if (!entry) return false;
const mtype = entry.type || entry.transport || "";
if (mtype === "url" || mtype === "http" || mtype === "sse") return true;
if (entry.url) return true;
return false;
} catch {
return false;
}
}
/**
* Best-effort recursive cleanup. Failures swallowed — at worst we leak a
* staging dir to disk; the next run uses a new one and they age out via
@@ -1387,12 +1438,24 @@ async function ingestPass(args: CliArgs): Promise<BulkResult> {
};
}
// Phase 2: stage to a per-run dir + invoke gbrain import.
const stagingDir = makeStagingDir();
// Phase 2: stage + (optionally) invoke gbrain import.
//
// Split-engine branch per plan D11: in remote-http MCP mode, we stage to a
// PERSISTENT dir under ~/.gstack/transcripts/ and SKIP `gbrain import`
// entirely. gstack-brain-sync push will pick the dir up via its allowlist
// and the brain admin's pull job will index transcripts into the remote
// brain. Local PGLite (if any) stays code-only.
const remoteHttpMode = isRemoteHttpMcpMode();
const stagingDir = remoteHttpMode
? makePersistentTranscriptDir()
: makeStagingDir();
// Register staging dir with the signal forwarder so SIGTERM/SIGINT can
// synchronously clean it up before process.exit (the async finally block
// below does NOT run after a signal-handler exit).
_activeStagingDir = stagingDir;
// below does NOT run after a signal-handler exit). In remote-http mode we
// skip registration — the dir is meant to persist.
if (!remoteHttpMode) {
_activeStagingDir = stagingDir;
}
try {
const staging = writeStaged(prep.prepared, stagingDir);
failed += staging.errors.length;
@@ -1415,11 +1478,62 @@ async function ingestPass(args: CliArgs): Promise<BulkResult> {
}
if (!args.quiet) {
const action = remoteHttpMode
? "persisting to artifacts pipeline (skipping local gbrain import — remote-http mode)"
: "running gbrain import";
console.error(
`[memory-ingest] staged ${staging.written} pages → ${stagingDir}; running gbrain import...`,
`[memory-ingest] staged ${staging.written} pages → ${stagingDir}; ${action}...`,
);
}
// Remote-http branch (split-engine D11): no local gbrain import. The
// staged markdown lives under ~/.gstack/transcripts/<run-id>/ and the
// next gstack-brain-sync push will move it to the artifacts repo. From
// there the brain admin's pull job indexes into the remote brain.
//
// We treat ALL prepared pages as "written" since the import didn't run
// and we have no per-page failures from gbrain to filter on. The
// brain admin's pull pipeline is the authoritative gate; from this
// machine's perspective, the act of staging IS the write.
if (remoteHttpMode) {
const nowIso = new Date().toISOString();
for (const p of prep.prepared) {
try {
state.sessions[p.source_path] = {
mtime_ns: Math.floor(statSync(p.source_path).mtimeMs * 1e6),
sha256: fileSha256(p.source_path),
ingested_at: nowIso,
page_slug: p.page_slug,
partial: p.partial,
};
written++;
} catch (err) {
console.error(
`[state-record] ${p.source_path}: ${(err as Error).message}`,
);
}
}
state.last_full_walk = nowIso;
state.last_writer = "gstack-memory-ingest (remote-http mode)";
saveState(state);
if (!args.quiet) {
console.error(
`[memory-ingest] persisted ${written} pages to ${stagingDir} (brain admin will index on next pull)`,
);
}
// Skip the gbrain-import error handling + cleanupStagingDir paths
// below by short-circuiting the function.
return {
written,
skipped_secret: prep.skippedSecret,
skipped_dedup: prep.skippedDedup,
skipped_unattributed: prep.skippedUnattributed,
failed,
duration_ms: Date.now() - t0,
partial_pages: prep.partialPages,
};
}
// D6: single batch import. `--no-embed` matches the prior per-file
// behavior (we never enabled embedding); embeddings happen on-demand
// via gbrain's own pipelines. `--json` gives us structured counts.