CalvinBackup/gstack
1
0
Fork 0
mirror of https://github.com/garrytan/gstack.git synced 2026-09-09 14:38:59 +02:00
Code Issues Packages Projects Releases Wiki Activity

fix(memory-ingest): slug assignments persist across runs via the state consult

Browse Source 
First-occurrence-keeps-bare was walk-order-dependent ACROSS runs: a source
that got the suffixed slug once could take the bare slug the next run (its
collider aged out or was skipped as unchanged), leaving gbrain holding the
same transcript under two slugs — and a NEW collider could claim a bare
slug that state shows belongs to an unchanged source, silently overwriting
that page. disambiguateSlugs now consults state.sessions: a recorded slug
stays owned by its source_path, re-ingested sources keep their slug
verbatim, fresh assignments never take another source's slug, and legacy
duplicate records (pre-#2724 overwrites) resolve first-owner-wins and
self-heal on the next state write. Stateless behavior is unchanged.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
Garry Tan
2026-09-01 16:28:41 +00:00
co-authored by Claude Fable 5
parent 8712a1f692
commit 9eab3ca336
2 changed files with 107 additions and 20 deletions
Download Patch File Download Diff File Expand all files Collapse all files
bin/gstack-memory-ingest.ts
+52 -20
View File
@@ -1304,7 +1304,8 @@ async function probeMode(args: CliArgs): Promise<ProbeReport> {
}
/**
* Disambiguate colliding page slugs before staging (#2724).
* Disambiguate colliding page slugs before staging (#2724), consulting the
* ingest state so an assignment is stable across RUNS, not just within one.
*
* Two distinct source files can map to one transcript slug
* (transcripts/<agent>/<repo>/<date>-<session_id[:12]>): a session resumed
@@ -1314,27 +1315,57 @@ async function probeMode(args: CliArgs): Promise<ProbeReport> {
* N-1 of N, and the staged-vs-collected reconciliation guard (correctly) fails
* the whole batch. It repeats every run until the inputs age out of the window.
*
* Fix: keep the first occurrence's slug; give each later collider a stable
* `-<sha8(source_path)>` suffix. Deterministic (same source path → same slug
* across runs, so incremental dedup and gbrain's session_id dedup still line
* up) and it mutates slug + page_slug together so every downstream consumer
* (writeStaged, readNewFailures mapping, state recording) computes the same key.
* Within a run: keep the first occurrence's slug; give each later collider a
* stable `-<sha8(source_path)>` suffix, mutating slug + page_slug together so
* every downstream consumer (writeStaged, readNewFailures mapping, state
* recording) computes the same key.
*
* Across runs (the state consult): "first occurrence" is walk-order-dependent,
* so without memory a source that got the suffixed slug in one run could take
* the bare slug in the next (its old collider aged out or was skipped as
* unchanged) — gbrain then holds the SAME transcript under two slugs. Worse,
* a NEW collider could claim a bare slug that state shows belongs to an
* unchanged (not-restaged) source, silently overwriting that page in gbrain.
* So: a slug recorded in state stays owned by its source_path — a re-ingested
* source keeps its recorded slug verbatim, and a fresh assignment never takes
* a slug owned by a DIFFERENT source. Legacy states that recorded the same
* slug for two sources (pre-#2724 overwrites) resolve first-owner-wins and
* self-heal on the next state write.
*/
export function disambiguateSlugs(pages: PreparedPage[]): void {
const seen = new Set<string>();
export function disambiguateSlugs(
pages: PreparedPage[],
state?: { sessions: Record<string, { page_slug: string }> },
): void {
// slug → owning source_path, from prior runs. First writer wins on legacy
// duplicate records; state key order is stable (re-read from the same file).
const ownedBy = new Map<string, string>();
for (const [src, rec] of Object.entries(state?.sessions ?? {})) {
if (rec?.page_slug && !ownedBy.has(rec.page_slug)) ownedBy.set(rec.page_slug, src);
}
const claimed = new Set<string>();
const available = (slug: string, src: string) =>
!claimed.has(slug) && (!ownedBy.has(slug) || ownedBy.get(slug) === src);
for (const p of pages) {
if (!seen.has(p.slug)) {
seen.add(p.slug);
const recorded = state?.sessions[p.source_path]?.page_slug;
if (recorded && !claimed.has(recorded) && ownedBy.get(recorded) === p.source_path) {
claimed.add(recorded);
p.slug = recorded;
p.page_slug = recorded;
continue;
}
const suffix = createHash("sha256").update(p.source_path).digest("hex").slice(0, 8);
let candidate = `${p.slug}-${suffix}`;
// Guarantee uniqueness even if a prior page already took the suffixed slug
// (two colliders sharing a source_path-hash prefix is astronomically
// unlikely, but a stuck source is not the place to trust luck).
let n = 1;
while (seen.has(candidate)) candidate = `${p.slug}-${suffix}-${n++}`;
seen.add(candidate);
let candidate = p.slug;
if (!available(candidate, p.source_path)) {
const suffix = createHash("sha256").update(p.source_path).digest("hex").slice(0, 8);
candidate = `${p.slug}-${suffix}`;
// Guarantee uniqueness even if a prior page already took the suffixed
// slug (two colliders sharing a source_path-hash prefix is
// astronomically unlikely, but a stuck source is not the place to
// trust luck).
let n = 1;
while (!available(candidate, p.source_path)) candidate = `${p.slug}-${suffix}-${n++}`;
}
claimed.add(candidate);
p.slug = candidate;
p.page_slug = candidate;
}
@@ -1527,8 +1558,9 @@ function preparePages(
// Colliding path-derived slugs would overwrite in the staging dir, so two
// source files land as one page and the staged-vs-collected guard fails the
// whole batch every run (#2724: 887 staged → 0 ingested). Disambiguate before staging.
disambiguateSlugs(finalPrepared);
// whole batch every run (#2724: 887 staged → 0 ingested). Disambiguate
// before staging, consulting state so assignments hold across runs.
disambiguateSlugs(finalPrepared, state);
// Derived from the FINAL set: partial counts must describe pages that are
// actually eligible and within the limit, not the whole scanned corpus.
test/regression-transcript-slug-collision.test.ts
+55
View File
@@ -65,6 +65,61 @@ describe("regression: disambiguateSlugs resolves colliding staged slugs", () =>
expect(pages.map((p) => p.slug)).toEqual(before);
});
it("state consult: a source keeps its recorded suffixed slug when its old collider is absent", () => {
// Run 1 assigned A the bare slug and B the suffix; run 2 sees only B
// (A unchanged or gone). Without the consult B would flip to bare and
// gbrain would hold the same transcript under two slugs.
const slug = "transcripts/claude-code/repo/2026-08-25-abc123def456";
const state = {
sessions: {
"/a.jsonl": { page_slug: slug },
"/b.jsonl": { page_slug: `${slug}-cafe0123` },
},
};
const pages = [mk(slug, "/b.jsonl")];
disambiguateSlugs(pages, state);
expect(pages[0].slug).toBe(`${slug}-cafe0123`);
expect(pages[0].page_slug).toBe(pages[0].slug);
});
it("state consult: a new source never takes a bare slug owned by an unchanged source", () => {
// A owns the bare slug from a prior run but is NOT restaged this run;
// new collider C must suffix, not silently overwrite A's page in gbrain.
const slug = "transcripts/claude-code/repo/2026-08-25-abc123def456";
const state = { sessions: { "/a.jsonl": { page_slug: slug } } };
const pages = [mk(slug, "/c.jsonl")];
disambiguateSlugs(pages, state);
expect(pages[0].slug).not.toBe(slug);
expect(pages[0].slug.startsWith(slug + "-")).toBe(true);
});
it("state consult: legacy duplicate records resolve first-owner-wins and self-heal", () => {
// Pre-#2724 states could record the SAME bare slug for two sources.
// The first owner in state order keeps it; the other gets a stable
// suffix — after this run the state records distinct slugs.
const slug = "transcripts/codex/repo/2026-08-25-deadbeefcafe";
const state = {
sessions: {
"/first.jsonl": { page_slug: slug },
"/second.jsonl": { page_slug: slug },
},
};
const pages = [mk(slug, "/first.jsonl"), mk(slug, "/second.jsonl")];
disambiguateSlugs(pages, state);
expect(pages[0].slug).toBe(slug);
expect(pages[1].slug).not.toBe(slug);
expect(pages[1].slug.startsWith(slug + "-")).toBe(true);
});
it("state consult: no state (or empty sessions) behaves exactly like the stateless algorithm", () => {
const slug = "transcripts/claude-code/repo/2026-08-25-abc123def456";
const a = [mk(slug, "/a.jsonl"), mk(slug, "/b.jsonl")];
const b = [mk(slug, "/a.jsonl"), mk(slug, "/b.jsonl")];
disambiguateSlugs(a);
disambiguateSlugs(b, { sessions: {} });
expect(b.map((p) => p.slug)).toEqual(a.map((p) => p.slug));
});
it("call-site wiring: the prepare/stage flow actually invokes disambiguateSlugs (source pin)", () => {
// The unit tests above prove the function works; nothing else proves the
// flow CALLS it — a refactor could drop the invocation and every test