feat(memory-ingest): honor the per-remote deny/read-only trust policy (#2392)

Transcript ingest now respects the same trust store as code import — the gate existed only in gstack-gbrain-sync's runCodeImport, so memory-ingest happily ingested transcripts from deny-listed repos. preparePages filters prepared transcript pages through ONE batch policy lookup (new 'get --batch' verb on bin/gstack-gbrain-repo-policy — the script owns URL normalization; the client adds repoPolicyTierBatch, one spawn for all distinct remotes, so large corpora never pay a 10s-timeout subprocess per remote). Outcomes match code-import semantics: read-only → clean skip (skipped_policy_readonly), deny → counted refusal (skipped_policy_deny), corrupted/unreadable store → HARD ERROR before any write (state, staging, egress receipt, and import all untouched) with the recovery command named — policy corruption must never read as successful ingestion. Artifacts are never policy-filtered (their git_remote is a project slug, not a remote). Fixes #2392. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
2026-09-09 22:48:57 +02:00 · 2026-08-17 10:39:24 -07:00
parent 40e4a53f74
commit 2494276742
6 changed files with 632 additions and 2 deletions
@@ -0,0 +1,152 @@
+/**
+ * lib/gbrain-repo-policy-client — batch trust-tier lookup (#2392).
+ *
+ * Covers the `get --batch` verb of bin/gstack-gbrain-repo-policy (bash level:
+ * multiple urls in, per-line verdicts out, input order preserved) and the
+ * TypeScript client `repoPolicyTierBatch` (ONE spawn, dedup, fast paths, and
+ * the whole-batch `unreadable` classification on a corrupt store).
+ *
+ * Each test uses a temp GSTACK_HOME so nothing leaks into the user's real
+ * ~/.gstack. The client is exercised against the REAL bash script — the
+ * script owns URL normalization, so stub stores are seeded through its own
+ * `set` verb.
+ */
+
+import { describe, test, expect, beforeEach, afterEach } from "bun:test";
+import * as fs from "fs";
+import * as path from "path";
+import * as os from "os";
+import { spawnSync } from "child_process";
+
+import { repoPolicyTierBatch } from "../lib/gbrain-repo-policy-client";
+
+const ROOT = path.resolve(import.meta.dir, "..");
+const BIN = path.join(ROOT, "bin", "gstack-gbrain-repo-policy");
+
+let tmpHome: string;
+
+function env(): NodeJS.ProcessEnv {
+  return { ...process.env, GSTACK_HOME: tmpHome };
+}
+
+function run(args: string[], input?: string) {
+  const res = spawnSync(BIN, args, { env: env(), encoding: "utf-8", input });
+  return {
+    stdout: res.stdout || "",
+    stderr: res.stderr || "",
+    status: res.status ?? -1,
+  };
+}
+
+function policyFile(): string {
+  return path.join(tmpHome, "gbrain-repo-policy.json");
+}
+
+beforeEach(() => {
+  tmpHome = fs.mkdtempSync(path.join(os.tmpdir(), "gbrain-policy-client-"));
+});
+
+afterEach(() => {
+  fs.rmSync(tmpHome, { recursive: true, force: true });
+});
+
+describe("bin/gstack-gbrain-repo-policy get --batch (bash level)", () => {
+  test("multiple urls in, per-line verdicts out, input order preserved", () => {
+    expect(run(["set", "https://github.com/foo/bar.git", "deny"]).status).toBe(0);
+    expect(run(["set", "git@github.com:baz/qux.git", "read-only"]).status).toBe(0);
+    expect(run(["set", "https://github.com/rw/repo", "read-write"]).status).toBe(0);
+
+    const r = run(
+      ["get", "--batch"],
+      // Mixed URL forms — the script's normalize() collapses them to the
+      // stored keys. `nope/never` has no entry → none.
+      "git@github.com:foo/bar.git\nhttps://github.com/nope/never\nhttps://github.com/baz/qux\nhttps://github.com/rw/repo.git\n",
+    );
+    expect(r.status).toBe(0);
+    expect(r.stdout).toBe("deny\nnone\nread-only\nread-write\n");
+  });
+
+  test("no store on disk: every line is none, and no file is created", () => {
+    const r = run(["get", "--batch"], "https://github.com/a/a\nhttps://github.com/b/b\n");
+    expect(r.status).toBe(0);
+    expect(r.stdout).toBe("none\nnone\n");
+    expect(fs.existsSync(policyFile())).toBe(false);
+  });
+
+  test("corrupt store: hard error (exit 2), NOT quarantined, names recovery", () => {
+    fs.writeFileSync(policyFile(), "not valid json{", { mode: 0o600 });
+    const r = run(["get", "--batch"], "https://github.com/foo/bar\n");
+    expect(r.status).toBe(2);
+    expect(r.stderr).toContain("corrupt");
+    expect(r.stderr).toContain("gstack-gbrain-repo-policy list");
+    // Unlike interactive `get`, batch must never quarantine-and-proceed —
+    // that would bypass a set deny policy on an unattended ingest run.
+    expect(fs.readFileSync(policyFile(), "utf-8")).toBe("not valid json{");
+    expect(
+      fs.readdirSync(tmpHome).find((f) => f.includes(".corrupt-")),
+    ).toBeUndefined();
+  });
+
+  test("legacy allow entries migrate to read-write on batch read", () => {
+    fs.writeFileSync(
+      policyFile(),
+      JSON.stringify({ "github.com/foo/bar": "allow" }),
+      { mode: 0o600 },
+    );
+    const r = run(["get", "--batch"], "https://github.com/foo/bar\n");
+    expect(r.status).toBe(0);
+    expect(r.stdout).toBe("read-write\n");
+  });
+});
+
+describe("repoPolicyTierBatch (TypeScript client)", () => {
+  test("maps each input url to its verdict, dedup included", () => {
+    expect(run(["set", "https://github.com/foo/bar", "deny"]).status).toBe(0);
+    expect(run(["set", "https://github.com/baz/qux", "read-only"]).status).toBe(0);
+
+    const verdicts = repoPolicyTierBatch(
+      [
+        "github.com/foo/bar", // canonical form, as memory-ingest passes it
+        "github.com/baz/qux",
+        "github.com/nope/never",
+        "github.com/foo/bar", // duplicate — dedup keeps ONE map entry
+      ],
+      env(),
+    );
+    expect(verdicts.size).toBe(3);
+    expect(verdicts.get("github.com/foo/bar")).toEqual({ tier: "deny" });
+    expect(verdicts.get("github.com/baz/qux")).toEqual({ tier: "read-only" });
+    expect(verdicts.get("github.com/nope/never")).toEqual({ tier: "none" });
+  });
+
+  test("no store on disk: every url is tier none with no error (fast path)", () => {
+    const verdicts = repoPolicyTierBatch(["github.com/a/a", "github.com/b/b"], env());
+    expect(verdicts.get("github.com/a/a")).toEqual({ tier: "none" });
+    expect(verdicts.get("github.com/b/b")).toEqual({ tier: "none" });
+    expect(fs.existsSync(policyFile())).toBe(false);
+  });
+
+  test("empty url list returns an empty map without spawning", () => {
+    const verdicts = repoPolicyTierBatch([], env());
+    expect(verdicts.size).toBe(0);
+  });
+
+  test("corrupt store: EVERY url maps to { tier: none, error: unreadable }", () => {
+    fs.writeFileSync(policyFile(), "not valid json{", { mode: 0o600 });
+    const verdicts = repoPolicyTierBatch(["github.com/foo/bar", "github.com/baz/qux"], env());
+    expect(verdicts.get("github.com/foo/bar")).toEqual({ tier: "none", error: "unreadable" });
+    expect(verdicts.get("github.com/baz/qux")).toEqual({ tier: "none", error: "unreadable" });
+  });
+
+  test("store unreadable on disk (chmod 000): whole batch classified unreadable", () => {
+    if (process.platform === "win32" || process.getuid?.() === 0) return; // chmod semantics differ
+    expect(run(["set", "https://github.com/foo/bar", "deny"]).status).toBe(0);
+    fs.chmodSync(policyFile(), 0o000);
+    try {
+      const verdicts = repoPolicyTierBatch(["github.com/foo/bar"], env());
+      expect(verdicts.get("github.com/foo/bar")).toEqual({ tier: "none", error: "unreadable" });
+    } finally {
+      fs.chmodSync(policyFile(), 0o600);
+    }
+  });
+});
@@ -939,3 +939,213 @@ describe("#2394: probe applies the same attribution gate as prepare", () => {
    rmSync(home, { recursive: true, force: true });
  });
 });
+
+// ── #2392: transcript ingest honors the per-remote trust policy ─────────────
+//
+// The same store the code-import gate honors (bin/gstack-gbrain-sync.ts):
+// tier `deny` and `read-only` transcripts are skipped with their own counters;
+// a store that EXISTS but can't be read is a hard error before any writes
+// (never a silent bypass of a set policy); no store at all = zero policy work.
+// The policy store is seeded through the REAL bin/gstack-gbrain-repo-policy
+// script (its `set` verb owns the file schema + URL normalization).
+
+describe("#2392: transcript ingest honors per-remote trust policy", () => {
+  const POLICY_BIN = join(import.meta.dir, "..", "bin", "gstack-gbrain-repo-policy");
+
+  /** Attributable temp git repo whose origin points at `remoteUrl`. */
+  function makeRepoWithRemote(home: string, name: string, remoteUrl: string): string {
+    const repo = join(home, "work", name);
+    mkdirSync(repo, { recursive: true });
+    spawnSync("git", ["-C", repo, "init", "-q"], { encoding: "utf-8" });
+    spawnSync("git", ["-C", repo, "remote", "add", "origin", remoteUrl], { encoding: "utf-8" });
+    return repo;
+  }
+
+  function writeSessionForRepo(home: string, projectName: string, sessionId: string, cwd: string): void {
+    const record = JSON.stringify({
+      type: "user",
+      message: { role: "user", content: `hello from ${sessionId}` },
+      timestamp: new Date().toISOString(),
+      cwd,
+    });
+    writeClaudeCodeSession(home, projectName, sessionId, record + "\n");
+  }
+
+  function setPolicy(gstackHome: string, url: string, tier: string): void {
+    const r = spawnSync(POLICY_BIN, ["set", url, tier], {
+      encoding: "utf-8",
+      env: { ...process.env, GSTACK_HOME: gstackHome },
+    });
+    expect(r.status).toBe(0);
+  }
+
+  function stateSessions(gstackHome: string): string[] {
+    const statePath = join(gstackHome, ".transcript-ingest-state.json");
+    if (!existsSync(statePath)) return [];
+    return Object.keys(JSON.parse(readFileSync(statePath, "utf-8")).sessions || {});
+  }
+
+  it("(a) deny remote's transcript is skipped and counted as skipped_policy_deny", () => {
+    const home = makeTestHome();
+    const gstackHome = join(home, ".gstack");
+    mkdirSync(gstackHome, { recursive: true });
+    const { binDir, logFile } = installFakeGbrain(home);
+
+    const denyCwd = makeRepoWithRemote(home, "denied", "https://github.com/denyme/denied.git");
+    const okCwd = makeRepoWithRemote(home, "allowed", "https://github.com/okorg/okrepo.git");
+    writeSessionForRepo(home, "work-denied", "denysess1", denyCwd);
+    writeSessionForRepo(home, "work-allowed", "oksess1", okCwd);
+    setPolicy(gstackHome, "https://github.com/denyme/denied.git", "deny");
+
+    const r = runScript(["--bulk", "--quiet"], {
+      HOME: home,
+      GSTACK_HOME: gstackHome,
+      PATH: `${binDir}:${process.env.PATH || ""}`,
+    });
+
+    expect(r.exitCode).toBe(0);
+    expect(r.stdout).toMatch(/written:\s+1/);
+    expect(r.stdout).toMatch(/skipped \(policy deny\):\s+1/);
+    expect(r.stdout).not.toMatch(/skipped \(policy read-only\)/);
+
+    // Only the allowed session was imported + state-recorded.
+    expect(existsSync(logFile)).toBe(true);
+    const sessions = stateSessions(gstackHome);
+    expect(sessions.length).toBe(1);
+    expect(sessions[0]).toContain("oksess1");
+
+    rmSync(home, { recursive: true, force: true });
+  });
+
+  it("(b) read-only remote's transcript is skipped and counted as skipped_policy_readonly", () => {
+    const home = makeTestHome();
+    const gstackHome = join(home, ".gstack");
+    mkdirSync(gstackHome, { recursive: true });
+    const { binDir } = installFakeGbrain(home);
+
+    const roCwd = makeRepoWithRemote(home, "readonly", "https://github.com/roorg/rorepo.git");
+    const okCwd = makeRepoWithRemote(home, "allowed", "https://github.com/okorg/okrepo.git");
+    writeSessionForRepo(home, "work-readonly", "rosess1", roCwd);
+    writeSessionForRepo(home, "work-allowed", "oksess1", okCwd);
+    setPolicy(gstackHome, "https://github.com/roorg/rorepo.git", "read-only");
+
+    const r = runScript(["--bulk", "--quiet"], {
+      HOME: home,
+      GSTACK_HOME: gstackHome,
+      PATH: `${binDir}:${process.env.PATH || ""}`,
+    });
+
+    expect(r.exitCode).toBe(0);
+    expect(r.stdout).toMatch(/written:\s+1/);
+    expect(r.stdout).toMatch(/skipped \(policy read-only\):\s+1/);
+    const sessions = stateSessions(gstackHome);
+    expect(sessions.length).toBe(1);
+    expect(sessions[0]).toContain("oksess1");
+
+    rmSync(home, { recursive: true, force: true });
+  });
+
+  it("(c) read-write remote's transcript is ingested (reaches gbrain import)", () => {
+    const home = makeTestHome();
+    const gstackHome = join(home, ".gstack");
+    mkdirSync(gstackHome, { recursive: true });
+    const { binDir, logFile } = installFakeGbrain(home);
+
+    const rwCwd = makeRepoWithRemote(home, "readwrite", "https://github.com/rworg/rwrepo.git");
+    writeSessionForRepo(home, "work-readwrite", "rwsess1", rwCwd);
+    setPolicy(gstackHome, "https://github.com/rworg/rwrepo.git", "read-write");
+
+    const r = runScript(["--bulk", "--quiet"], {
+      HOME: home,
+      GSTACK_HOME: gstackHome,
+      PATH: `${binDir}:${process.env.PATH || ""}`,
+    });
+
+    expect(r.exitCode).toBe(0);
+    expect(r.stdout).toMatch(/written:\s+1/);
+    expect(r.stdout).not.toMatch(/skipped \(policy/);
+    // gbrain import ran exactly once — the page reached the import stage.
+    const calls = readFileSync(logFile, "utf-8").trim().split("\n").filter(Boolean);
+    expect(calls.length).toBe(1);
+    expect(stateSessions(gstackHome).length).toBe(1);
+
+    rmSync(home, { recursive: true, force: true });
+  });
+
+  it("(d) corrupted store: hard error before any writes, message names recovery", () => {
+    const home = makeTestHome();
+    const gstackHome = join(home, ".gstack");
+    mkdirSync(gstackHome, { recursive: true });
+    const { binDir, logFile } = installFakeGbrain(home);
+
+    const cwd = makeRepoWithRemote(home, "somerepo", "https://github.com/some/repo.git");
+    writeSessionForRepo(home, "work-somerepo", "somesess1", cwd);
+    // Corrupt store — the batch verb refuses (exit 2), the client classifies
+    // `unreadable`, and ingest must abort rather than bypass a set policy.
+    writeFileSync(join(gstackHome, "gbrain-repo-policy.json"), "not valid json{", "utf-8");
+
+    const r = runScript(["--bulk", "--quiet"], {
+      HOME: home,
+      GSTACK_HOME: gstackHome,
+      PATH: `${binDir}:${process.env.PATH || ""}`,
+    });
+
+    expect(r.exitCode).toBe(1);
+    expect(r.stderr).toMatch(/\[memory-ingest\] ERR:.*repo policy store exists/);
+    expect(r.stderr).toContain("gstack-gbrain-repo-policy list");
+    expect(r.stderr).toContain("/setup-gbrain");
+    // Nothing written: no gbrain import call, no state file, store untouched.
+    expect(existsSync(logFile)).toBe(false);
+    expect(stateSessions(gstackHome).length).toBe(0);
+    expect(readFileSync(join(gstackHome, "gbrain-repo-policy.json"), "utf-8")).toBe("not valid json{");
+
+    rmSync(home, { recursive: true, force: true });
+  });
+
+  it("(e) no store at all: no policy filtering, transcript ingests normally", () => {
+    const home = makeTestHome();
+    const gstackHome = join(home, ".gstack");
+    mkdirSync(gstackHome, { recursive: true });
+    const { binDir } = installFakeGbrain(home);
+
+    const cwd = makeRepoWithRemote(home, "freerepo", "https://github.com/free/repo.git");
+    writeSessionForRepo(home, "work-freerepo", "freesess1", cwd);
+
+    const r = runScript(["--bulk", "--quiet"], {
+      HOME: home,
+      GSTACK_HOME: gstackHome,
+      PATH: `${binDir}:${process.env.PATH || ""}`,
+    });
+
+    expect(r.exitCode).toBe(0);
+    expect(r.stdout).toMatch(/written:\s+1/);
+    expect(r.stdout).not.toMatch(/skipped \(policy/);
+    expect(stateSessions(gstackHome).length).toBe(1);
+
+    rmSync(home, { recursive: true, force: true });
+  });
+
+  it("artifacts are never policy-filtered, even when their project's remote is denied", () => {
+    const home = makeTestHome();
+    const gstackHome = join(home, ".gstack");
+    mkdirSync(join(gstackHome, "projects", "denyme-denied"), { recursive: true });
+    const { binDir } = installFakeGbrain(home);
+
+    // A learning artifact under a project slug matching a denied remote —
+    // the policy is keyed by git remote, which artifacts don't have.
+    writeFileSync(join(gstackHome, "projects", "denyme-denied", "learnings.jsonl"), '{"key":"a","insight":"b"}\n');
+    setPolicy(gstackHome, "https://github.com/denyme/denied.git", "deny");
+
+    const r = runScript(["--bulk", "--quiet"], {
+      HOME: home,
+      GSTACK_HOME: gstackHome,
+      PATH: `${binDir}:${process.env.PATH || ""}`,
+    });
+
+    expect(r.exitCode).toBe(0);
+    expect(r.stdout).toMatch(/written:\s+1/);
+    expect(r.stdout).not.toMatch(/skipped \(policy/);
+
+    rmSync(home, { recursive: true, force: true });
+  });
+});