mirror of
https://github.com/garrytan/gstack.git
synced 2026-09-09 14:38:59 +02:00
fix(memory-ingest): stop two silent transcript-ingest failures
Two independent bugs made transcript pages silently fail to reach the brain.
1. Frontmatter fence gluing. buildTranscriptPage() built the closing "---"
with no trailing newline, and session bodies always start with "## ", so
the rendered page ended "...---## User". gbrain's frontmatter matcher
(/^---\r?\n([\s\S]*?)\r?\n---(\r?\n|$)/ in src/core/markdown.ts) requires
the closing "---" to end its own line, so it skipped the glued fence,
latched onto the next standalone "---" in the transcript body, parsed the
prose between as YAML, and dropped the page with "Invalid YAML frontmatter".
Transcripts with no later "---" fell back to body-only, silently losing
their frontmatter. Fix: emit the fence on its own line with a blank
separator, matching renderPageBody()'s artifact branch.
2. Slug collisions. Two source files can map to one path-derived slug (a
session resumed under the same id on one day, or two ids sharing a 12-char
prefix). writeStaged() names each file "${slug}.md", so the second
overwrote the first; gbrain collected N-1 of N staged files and the
reconciliation guard failed the whole batch every run. Fix:
disambiguateSlugs() keeps the first occurrence and gives each later collider
a stable "-<sha8(source_path)>" suffix (deterministic, and slug + page_slug
move together so writeStaged, the failure mapping, and state recording agree).
Exports buildTranscriptPage, renderPageBody, and disambiguateSlugs for tests.
Adds regression tests for both failures.
Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
Wave-amended: contributor's local-workaround docblock note removed; issue refs retargeted #2653 (closed by its author) -> #2724 (the live 887-staged-to-0-ingested report)
This commit is contained in:
committed by
Garry Tan
co-authored by
Claude Opus 4.8
parent
9f4e8eef48
commit
96ac9bbfef
@@ -0,0 +1,65 @@
|
||||
/**
|
||||
* Regression: transcript frontmatter fence must terminate its own line.
|
||||
*
|
||||
* buildTranscriptPage() emitted a closing `---` with no trailing newline, and
|
||||
* session bodies always start with "## ", so the rendered page ended
|
||||
* `...---## User`. gbrain's frontmatter matcher
|
||||
* (`/^---\r?\n([\s\S]*?)\r?\n---(\r?\n|$)/`, src/core/markdown.ts) requires the
|
||||
* closing `---` to end its own line, so it skipped the glued fence, latched onto
|
||||
* the next standalone `---` in the transcript body, parsed the prose between as
|
||||
* YAML, and dropped the whole page with "Invalid YAML frontmatter". Transcripts
|
||||
* with no later `---` fell back to body-only (frontmatter silently lost).
|
||||
*/
|
||||
import { describe, it, expect } from "bun:test";
|
||||
import { mkdtempSync, writeFileSync, rmSync } from "fs";
|
||||
import { tmpdir } from "os";
|
||||
import { join } from "path";
|
||||
import {
|
||||
parseTranscriptJsonl,
|
||||
buildTranscriptPage,
|
||||
renderPageBody,
|
||||
} from "../bin/gstack-memory-ingest";
|
||||
|
||||
// The exact fence matcher gbrain uses (src/core/markdown.ts). Kept here so the
|
||||
// test fails if the rendered fence ever regresses.
|
||||
const GBRAIN_FENCE_RE = /^---\r?\n([\s\S]*?)\r?\n---(\r?\n|$)/;
|
||||
|
||||
describe("regression: transcript frontmatter fence terminates its own line", () => {
|
||||
it("keeps a body-embedded `---` out of the frontmatter block", () => {
|
||||
const dir = mkdtempSync(join(tmpdir(), "gstack-fence-"));
|
||||
const file = join(dir, "sess.jsonl");
|
||||
// User content carries a markdown horizontal rule (`---`) followed by a
|
||||
// colon-bearing line — exactly what the old glued fence swept into YAML.
|
||||
const content =
|
||||
`{"type":"user","message":{"role":"user","content":"before rule\\n\\n---\\n\\nnot: valid: yaml: here"},` +
|
||||
`"timestamp":"2026-05-01T00:00:00Z","cwd":"${dir}"}\n` +
|
||||
`{"type":"assistant","message":{"role":"assistant","content":"ok"},"timestamp":"2026-05-01T00:00:01Z"}\n`;
|
||||
writeFileSync(file, content, "utf-8");
|
||||
|
||||
const session = parseTranscriptJsonl(file);
|
||||
expect(session).not.toBeNull();
|
||||
const page = buildTranscriptPage(file, session!);
|
||||
const staged = renderPageBody(page);
|
||||
|
||||
// The fence is never glued onto the body.
|
||||
expect(staged).not.toContain("---##");
|
||||
|
||||
// gbrain's matcher closes the frontmatter at the real fence, not at the
|
||||
// horizontal rule deep in the transcript body.
|
||||
const m = staged.match(GBRAIN_FENCE_RE);
|
||||
expect(m).not.toBeNull();
|
||||
const frontmatter = m![1];
|
||||
expect(frontmatter).toContain("session_id:");
|
||||
expect(frontmatter).toContain("title:");
|
||||
// The body (headings, the HR, the colon-trap line) must NOT bleed into YAML.
|
||||
expect(frontmatter).not.toContain("## User");
|
||||
expect(frontmatter).not.toContain("not: valid: yaml: here");
|
||||
|
||||
// And the parsed body (after the fence's trailing blank line) is the
|
||||
// session content, not YAML-absorbed prose.
|
||||
const body = staged.slice(m![0].length);
|
||||
expect(body.trimStart().startsWith("## User")).toBe(true);
|
||||
|
||||
rmSync(dir, { recursive: true, force: true });
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,65 @@
|
||||
/**
|
||||
* Regression: disambiguateSlugs resolves colliding staged slugs.
|
||||
*
|
||||
* Two source files can map to one path-derived transcript slug (a session
|
||||
* resumed under the same id on one day, or two session ids sharing a 12-char
|
||||
* prefix). writeStaged() names each file `${slug}.md`, so the second overwrote
|
||||
* the first; gbrain then collected N-1 of N staged files and the
|
||||
* staged-vs-collected reconciliation guard failed the whole batch every run
|
||||
* ("accounted for N-1 of N staged ... Refusing to advance state").
|
||||
*/
|
||||
import { describe, it, expect } from "bun:test";
|
||||
import { disambiguateSlugs } from "../bin/gstack-memory-ingest";
|
||||
|
||||
const mk = (slug: string, source_path: string) => ({
|
||||
slug,
|
||||
source_path,
|
||||
rendered_body: "---\ntitle: x\n---\n\nbody",
|
||||
page_slug: slug,
|
||||
partial: false,
|
||||
type: "transcript" as const,
|
||||
git_remote: undefined,
|
||||
});
|
||||
|
||||
describe("regression: disambiguateSlugs resolves colliding staged slugs", () => {
|
||||
it("keeps the first occurrence and suffixes later colliders deterministically", () => {
|
||||
const slug = "transcripts/claude-code/repo/2026-08-25-abc123def456";
|
||||
const run = () => {
|
||||
const pages = [mk(slug, "/a.jsonl"), mk(slug, "/b.jsonl")];
|
||||
disambiguateSlugs(pages);
|
||||
return pages;
|
||||
};
|
||||
const pages = run();
|
||||
// First keeps the clean slug; second is disambiguated.
|
||||
expect(pages[0].slug).toBe(slug);
|
||||
expect(pages[1].slug).not.toBe(slug);
|
||||
expect(pages[1].slug.startsWith(slug + "-")).toBe(true);
|
||||
// slug and page_slug move together (downstream consumers must agree).
|
||||
expect(pages[1].page_slug).toBe(pages[1].slug);
|
||||
// Deterministic across runs (same source path → same suffix).
|
||||
expect(run()[1].slug).toBe(pages[1].slug);
|
||||
});
|
||||
|
||||
it("gives every member of a 3-way collision a distinct slug", () => {
|
||||
const slug = "transcripts/codex/repo/2026-08-25-deadbeefcafe";
|
||||
const pages = [
|
||||
mk(slug, "/one.jsonl"),
|
||||
mk(slug, "/two.jsonl"),
|
||||
mk(slug, "/three.jsonl"),
|
||||
];
|
||||
disambiguateSlugs(pages);
|
||||
const slugs = new Set(pages.map((p) => p.slug));
|
||||
expect(slugs.size).toBe(3);
|
||||
expect(pages[0].slug).toBe(slug);
|
||||
});
|
||||
|
||||
it("leaves non-colliding slugs untouched", () => {
|
||||
const pages = [
|
||||
mk("transcripts/a/repo/2026-08-25-1111", "/x.jsonl"),
|
||||
mk("transcripts/b/repo/2026-08-25-2222", "/y.jsonl"),
|
||||
];
|
||||
const before = pages.map((p) => p.slug);
|
||||
disambiguateSlugs(pages);
|
||||
expect(pages.map((p) => p.slug)).toEqual(before);
|
||||
});
|
||||
});
|
||||
Reference in New Issue
Block a user