fix(memory-ingest): stop silently ingesting 0 pages — include gitignored staging, reconcile counts

Pages stage into ~/.gstack/.staging-ingest-*/ inside a repo whose .gitignore
is `*`, and gbrain import honours .gitignore — so it collected 0 files,
imported nothing, and the ingest still reported "written: N" from the STAGED
count while advancing state, meaning no future run ever retried. Three
layers now: (1) pass --include-gitignored (root cause); (2) if the installed
gbrain predates the flag, retry without it (subcommand --help is generic, so
the attempt is the only probe) with an upgrade pointer; (3) reconcile
gbrain's imported+unchanged accounting against the staged count and REFUSE
to advance state on a shortfall, naming the gitignore collision.

Fixes #2144, #2104.

Contributed by @gawievanblerk (PR #2560) and @Charles-Grant (PR #2486).

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
Garry Tan
2026-08-14 20:20:52 -07:00
co-authored by Claude Fable 5
parent 75cd221850
commit 7847fcf9ba
3 changed files with 224 additions and 3 deletions
+53 -1
View File
@@ -330,7 +330,7 @@ describe("gstack-memory-ingest --limit", () => {
*/
function installFakeGbrain(
home: string,
opts: { failingPaths?: string[] } = {},
opts: { failingPaths?: string[]; collectNothing?: boolean } = {},
): { binDir: string; logFile: string; argsFile: string; stagingListFile: string } {
const binDir = join(home, "fake-bin");
mkdirSync(binDir, { recursive: true });
@@ -392,6 +392,13 @@ EOF
else
TOTAL=0
fi
# collectNothing: simulate gbrain walking the staging dir and finding
# nothing — the real-world shape when .gitignore hides every staged file
# from collect_files. Crucially this writes NO sync-failures.jsonl entry,
# because there is no per-file failure: gbrain never saw the files.
if [ "${opts.collectNothing ? "1" : "0"}" = "1" ]; then
TOTAL=0
fi
ERRORS=0
if [ -n "\$FAILING_LIST" ]; then
ERRORS=\$(echo "\$FAILING_LIST" | tr '|' '\\n' | wc -l | tr -d ' ')
@@ -470,6 +477,51 @@ describe("gstack-memory-ingest writer (gbrain v0.20+ batch `import` interface)",
expect(stagedList).toMatch(/^\.\/transcripts\/claude-code\/.+\.md$/m);
});
// Silent-data-loss regression: gbrain accepts the import call, exits 0, and
// reports imported=0 because collect_files found nothing in the staging dir
// (real-world cause: gstack-artifacts-init writes `.gitignore = "*"` into
// $GSTACK_HOME, and `gbrain import` honours .gitignore, so every file staged
// under $GSTACK_HOME is invisible to it).
//
// No per-file failure is written to sync-failures.jsonl — gbrain never SAW
// the files — so readNewFailures returns empty. Before the reconciliation
// check, that made a total loss indistinguishable from success: every
// prepared file got state-recorded as ingested and the pass reported
// "N written". State then said "done", so no later run ever retried.
it("refuses to advance state when gbrain imports fewer pages than were staged", () => {
const home = makeTestHome();
const gstackHome = join(home, ".gstack");
mkdirSync(gstackHome, { recursive: true });
const { binDir, logFile } = installFakeGbrain(home, { collectNothing: true });
const session =
`{"type":"user","message":{"role":"user","content":"hi"},"timestamp":"2026-05-01T00:00:00Z","cwd":"/tmp/foo"}\n` +
`{"type":"assistant","message":{"role":"assistant","content":"hello"},"timestamp":"2026-05-01T00:00:01Z"}\n`;
writeClaudeCodeSession(home, "tmp-foo", "abc123", session);
const r = runScript(["--bulk", "--include-unattributed", "--quiet"], {
HOME: home,
GSTACK_HOME: gstackHome,
PATH: `${binDir}:${process.env.PATH || ""}`,
});
// gbrain WAS called — this is not a "gbrain missing" path.
expect(existsSync(logFile)).toBe(true);
// The pass must not claim success.
expect(r.stderr).toMatch(/\[memory-ingest\] ERR:.*accounted for 0 of 1 staged page/);
expect(r.stderr).toMatch(/Refusing to advance state/);
expect(r.stdout).not.toMatch(/written:\s+1/);
// The critical assertion: state must NOT mark the session ingested, or the
// next run skips it forever and the transcript is lost silently.
const statePath = join(gstackHome, ".transcript-ingest-state.json");
if (existsSync(statePath)) {
const state = JSON.parse(readFileSync(statePath, "utf-8"));
expect(Object.keys(state.sessions || {}).length).toBe(0);
}
});
// Originally landed in v1.32.0.0 (PR #1411) on the per-file `gbrain put`
// path. Postgres rejects 0x00 in UTF-8 text columns. Some Claude Code
// transcripts contain NUL inside user-pasted content or tool output. The