mirror of
https://github.com/garrytan/gstack.git
synced 2026-11-02 06:54:50 +01:00
W4: transcript ingest needs affirmative consent (recent|all)
normalizeTranscriptConsent() in gstack-memory-ingest reads transcript_ingest_mode once per process. Only recent (90 days) and all (all history) are consent; an absent key, off, legacy gate letters, incremental and unknown values skip transcripts while other memory still syncs. The ingest's default source set drops transcript without consent; an explicit --sources list naming transcript overrides. gstack-gbrain-sync reads consent once at sync start and pins a non-consented run to an explicit source list, so a mid-run config change cannot add transcripts. It prints the pinned skip notice on every such run (even --quiet), records the normalized mode and skip reason in the memory stage, restages an interrupted import when the consent policy changed, passes --all-history on --full for mode all, and documents the default in --help. Tests: new gstack-gbrain-sync-transcript-mode.test.ts; memory-ingest fixtures seed consent; regression-2922's every-type case seeds consent. Findings: s09 H16 (enforce the consent choice in code).
This commit is contained in:
1 parent
f30b7b788a
commit
7da92a7741
5 files changed
+634
-11
No files matched your search
+105
-10
@@ -45,6 +45,7 @@ import { buildGbrainEnv, spawnGbrain, spawnGbrainAsync, execGbrainJson, NEEDS_SH
|
||||
import { repoPolicyTier as sharedRepoPolicyTier } from "../lib/gbrain-repo-policy-client";
|
||||
import { checkOwnedStagingDir } from "../lib/staging-guard";
|
||||
import { resolveStateRoot } from "../lib/state-root";
|
||||
import { normalizeTranscriptConsent, type TranscriptConsent } from "./gstack-memory-ingest";
|
||||
|
||||
// ── Types ──────────────────────────────────────────────────────────────────
|
||||
|
||||
@@ -101,6 +102,16 @@ interface StageResult {
|
||||
detail?: CodeStageDetail;
|
||||
/** Memory stage: the type selection its ingest staged (resume must match it). */
|
||||
memory_sources?: string[];
|
||||
/** Memory stage: transcript consent read at sync start (resume must match it). */
|
||||
transcript_consent?: TranscriptConsentRecord;
|
||||
}
|
||||
|
||||
interface TranscriptConsentRecord {
|
||||
/** Normalized transcript_ingest_mode: recent | all | off | not-set | legacy | unrecognized. */
|
||||
mode: string;
|
||||
window: "recent" | "all" | null;
|
||||
/** Why transcripts were not ingested; absent when they were. */
|
||||
skip_reason?: string;
|
||||
}
|
||||
|
||||
// ── Constants ──────────────────────────────────────────────────────────────
|
||||
@@ -253,6 +264,59 @@ export function selectMemorySources(
|
||||
};
|
||||
}
|
||||
|
||||
export interface ConsentedSelection {
|
||||
/** Value for the ingest's --sources flag; null = let the ingest walk its default. */
|
||||
sources: string[] | null;
|
||||
/** Types this run stages (recorded for resume). */
|
||||
selected: string[];
|
||||
/** Transcripts are ingested only because an explicit list names them. */
|
||||
override: boolean;
|
||||
why: string;
|
||||
}
|
||||
|
||||
/**
|
||||
* Apply transcript consent to a memory source selection. Without consent
|
||||
* (anything but `recent` or `all`), transcript is removed and the remaining
|
||||
* types are passed as an explicit list, so a config change made while the
|
||||
* sync runs cannot add transcripts to this run. Only an explicit --sources or
|
||||
* GSTACK_MEMORY_INGEST_SOURCES list that names transcript overrides that;
|
||||
* `all`, empty and invalid-only values resolve to null and do not.
|
||||
*/
|
||||
export function applyTranscriptConsent(selection: MemorySourceSelection, consent: TranscriptConsent): ConsentedSelection {
|
||||
const explicit = selection.why === "--sources" || selection.why === "GSTACK_MEMORY_INGEST_SOURCES";
|
||||
const override = !consent.affirmative && explicit && !!selection.sources?.includes("transcript");
|
||||
if (consent.affirmative || override) {
|
||||
return { sources: selection.sources, selected: selection.sources ?? [...MEMORY_INGEST_TYPES], override, why: selection.why };
|
||||
}
|
||||
const selected = (selection.sources ?? [...MEMORY_INGEST_TYPES]).filter((t) => t !== "transcript");
|
||||
return { sources: selected, selected, override: false, why: `${selection.why}; no transcript consent (${consent.reason})` };
|
||||
}
|
||||
|
||||
export const TRANSCRIPT_CHOICE_HINT =
|
||||
"To choose: run /sync-gbrain, or gstack-config set transcript_ingest_mode recent|all|off. Details: setup-gbrain/memory.md#transcripts";
|
||||
|
||||
/**
|
||||
* The stderr notice for a run that does not ingest transcripts by consent.
|
||||
* Returns null when nothing should print: consent given, or a stored `off`
|
||||
* under --quiet.
|
||||
*/
|
||||
export function transcriptConsentNotice(consent: TranscriptConsent, quiet: boolean, override: string | null = null): string | null {
|
||||
if (consent.affirmative) return null;
|
||||
const value = consent.value ?? "not set";
|
||||
if (override) {
|
||||
return `gbrain-sync: transcripts ingested because ${override} names transcript (transcript_ingest_mode=${value}). ${TRANSCRIPT_CHOICE_HINT}`;
|
||||
}
|
||||
if (consent.reason === "off") return quiet ? null : "gbrain-sync: transcripts off (your choice)";
|
||||
return `gbrain-sync: transcripts skipped (transcript_ingest_mode=${value}). Other memory still syncs. ${TRANSCRIPT_CHOICE_HINT}`;
|
||||
}
|
||||
|
||||
/** The consent record a memory stage stores; resume requires mode and window to match. */
|
||||
export function transcriptConsentRecord(consent: TranscriptConsent, override: boolean): TranscriptConsentRecord {
|
||||
const record: TranscriptConsentRecord = { mode: consent.reason, window: consent.window };
|
||||
if (!consent.affirmative && !override) record.skip_reason = consent.reason;
|
||||
return record;
|
||||
}
|
||||
|
||||
/**
|
||||
* Parse a stage-timeout env value with bounds validation. Returns the bounded
|
||||
* value or the default with a stderr warning if the env was malformed or
|
||||
@@ -382,7 +446,13 @@ Options:
|
||||
${MEMORY_INGEST_TYPES.join(",")}.
|
||||
Env: GSTACK_MEMORY_INGEST_SOURCES. Default: every type
|
||||
except ${FEDERATED_CURATED_TYPES.join(",")} when a federated
|
||||
gstack source already indexes them (#2922).
|
||||
gstack source already indexes them, and except
|
||||
transcript unless transcript_ingest_mode is recent
|
||||
(last 90 days) or all (all history; --full passes
|
||||
--all-history). Without that consent each run prints a
|
||||
notice, even with --quiet. A list naming transcript
|
||||
overrides the mode for that run. See
|
||||
setup-gbrain/memory.md#transcripts.
|
||||
--help This text.
|
||||
|
||||
Stages run in order: code → memory ingest → curated git push, then (lock-free)
|
||||
@@ -1400,11 +1470,18 @@ export function ensureGbrainSourceGitignored(root: string): void {
|
||||
}
|
||||
}
|
||||
|
||||
function runMemoryIngest(args: CliArgs, previous: SyncState): StageResult {
|
||||
function printTranscriptNotice(args: CliArgs, consent: TranscriptConsent, selection: ConsentedSelection, why: string): void {
|
||||
const notice = transcriptConsentNotice(consent, args.quiet, selection.override ? why : null);
|
||||
if (notice) console.error(notice);
|
||||
}
|
||||
|
||||
function runMemoryIngest(args: CliArgs, previous: SyncState, consent: TranscriptConsent): StageResult {
|
||||
const t0 = Date.now();
|
||||
|
||||
if (args.mode === "dry-run") {
|
||||
const preview = selectMemorySources(args.memorySources, process.env, false);
|
||||
const raw = selectMemorySources(args.memorySources, process.env, false);
|
||||
const preview = applyTranscriptConsent(raw, consent);
|
||||
printTranscriptNotice(args, consent, preview, raw.why);
|
||||
const flag = preview.sources ? ` --sources ${preview.sources.join(",")}` : "";
|
||||
return { name: "memory", ran: false, ok: true, duration_ms: 0, summary: `would: gstack-memory-ingest --probe${flag} (${preview.why})` };
|
||||
}
|
||||
@@ -1427,20 +1504,33 @@ function runMemoryIngest(args: CliArgs, previous: SyncState): StageResult {
|
||||
// staging dir. If the staging dir is gone (disk pressure cleanup, OS
|
||||
// reboot, user manual cleanup), warn and fall through to a fresh restage.
|
||||
const childEnv = buildGbrainEnv({ announce: false });
|
||||
const selection = selectMemorySources(args.memorySources, childEnv, true);
|
||||
const selected = selection.sources ?? [...MEMORY_INGEST_TYPES];
|
||||
const raw = selectMemorySources(args.memorySources, childEnv, true);
|
||||
const selection = applyTranscriptConsent(raw, consent);
|
||||
const selected = selection.selected;
|
||||
const consentRecord = transcriptConsentRecord(consent, selection.override);
|
||||
printTranscriptNotice(args, consent, selection, raw.why);
|
||||
let resume = decideResume();
|
||||
if (resume.kind === "resume") {
|
||||
// The staging dir holds whatever the checkpointed run selected; resuming
|
||||
// it under a different selection would import types the user excluded.
|
||||
// Runs recorded before #2922 walked every type.
|
||||
const staged = previous.last_stages?.find((st) => st.name === "memory")?.memory_sources ?? [...MEMORY_INGEST_TYPES];
|
||||
// The staging dir holds whatever the checkpointed run selected under its
|
||||
// transcript consent; resuming it under a different selection or consent
|
||||
// would import pages the user excluded. Runs recorded before #2922 walked
|
||||
// every type; runs recorded before the consent record walked the 90-day
|
||||
// transcript window.
|
||||
const prevStage = previous.last_stages?.find((st) => st.name === "memory");
|
||||
const staged = prevStage?.memory_sources ?? [...MEMORY_INGEST_TYPES];
|
||||
const prevConsent = prevStage?.transcript_consent ?? { mode: "recent", window: "recent" };
|
||||
if ([...staged].sort().join(",") !== [...selected].sort().join(",")) {
|
||||
console.error(
|
||||
`[sync:memory] memory source selection changed since the checkpointed run (${staged.join(",")} → ${selected.join(",")}); restaging from scratch.`,
|
||||
);
|
||||
resume = { kind: "no-checkpoint" };
|
||||
}
|
||||
if (prevConsent.mode !== consentRecord.mode || prevConsent.window !== consentRecord.window) {
|
||||
console.error(
|
||||
`gbrain-sync: transcript consent changed since the interrupted import (${prevConsent.mode} → ${consentRecord.mode}); restaging memory from scratch once.`,
|
||||
);
|
||||
resume = { kind: "no-checkpoint" };
|
||||
}
|
||||
}
|
||||
if (resume.kind === "resume") {
|
||||
console.error(
|
||||
@@ -1464,6 +1554,7 @@ function runMemoryIngest(args: CliArgs, previous: SyncState): StageResult {
|
||||
const ingestArgs = ["run", ingestPath];
|
||||
if (args.mode === "full") ingestArgs.push("--bulk");
|
||||
else ingestArgs.push("--incremental");
|
||||
if (args.mode === "full" && consent.window === "all") ingestArgs.push("--all-history");
|
||||
if (args.quiet) ingestArgs.push("--quiet");
|
||||
if (selection.sources) ingestArgs.push("--sources", selection.sources.join(","));
|
||||
if (!args.quiet) console.error(`[sync:memory] sources: ${selected.join(",")} (${selection.why})`);
|
||||
@@ -1507,6 +1598,7 @@ function runMemoryIngest(args: CliArgs, previous: SyncState): StageResult {
|
||||
? summary
|
||||
: `${summary}${result.status === null ? " (killed by signal / timeout)" : ` (exit ${result.status})`}`,
|
||||
memory_sources: selected,
|
||||
transcript_consent: consentRecord,
|
||||
};
|
||||
}
|
||||
|
||||
@@ -1869,6 +1961,9 @@ export function formatStage(s: StageResult): string {
|
||||
|
||||
async function main(): Promise<void> {
|
||||
const args = parseArgs();
|
||||
// Read once at sync start: a config change made while the sync runs never
|
||||
// adds transcripts to this run.
|
||||
const consent = args.noMemory ? null : normalizeTranscriptConsent();
|
||||
|
||||
if (!args.quiet) {
|
||||
const engine = detectEngineTier();
|
||||
@@ -1904,7 +1999,7 @@ async function main(): Promise<void> {
|
||||
stages.push(await withErrorContext("sync:code", () => runCodeImport(args), "gstack-gbrain-sync"));
|
||||
}
|
||||
if (!args.noMemory) {
|
||||
stages.push(await withErrorContext("sync:memory", () => runMemoryIngest(args, state), "gstack-gbrain-sync"));
|
||||
stages.push(await withErrorContext("sync:memory", () => runMemoryIngest(args, state, consent!), "gstack-gbrain-sync"));
|
||||
}
|
||||
if (!args.noBrainSync) {
|
||||
stages.push(await withErrorContext("sync:brain-sync", () => runBrainSyncPush(args), "gstack-gbrain-sync"));
|
||||
|
||||
@@ -72,7 +72,7 @@ import { execGbrainText, spawnGbrainAsync } from "../lib/gbrain-exec";
|
||||
import { writeReceipt } from "../lib/egress-receipt";
|
||||
import { checkOwnedStagingDir, STAGING_MARKER } from "../lib/staging-guard";
|
||||
import { hasRepoPolicyStore, repoPolicyTierBatch } from "../lib/gbrain-repo-policy-client";
|
||||
import { resolveStateRoot } from "../lib/state-root";
|
||||
import { resolveStateRoot, readConfigKeyWithRoot, type StateRootEnv } from "../lib/state-root";
|
||||
|
||||
// ── Types ──────────────────────────────────────────────────────────────────
|
||||
|
||||
@@ -202,6 +202,45 @@ const ALL_TYPES: MemoryType[] = [
|
||||
"builder-profile-entry",
|
||||
];
|
||||
|
||||
// ── Transcript consent ─────────────────────────────────────────────────────
|
||||
|
||||
export interface TranscriptConsent {
|
||||
/** True only for a stored `recent` or `all`. */
|
||||
affirmative: boolean;
|
||||
/** Transcript walk window when affirmative: `recent` = last 90 days, `all` = all history. */
|
||||
window: "recent" | "all" | null;
|
||||
/**
|
||||
* Normalized mode: the window when affirmative; otherwise why transcripts
|
||||
* are skipped. `legacy` covers the old gate letters (A-E) and `incremental`,
|
||||
* which predate the consent rule and need a new choice.
|
||||
*/
|
||||
reason: "recent" | "all" | "off" | "not-set" | "legacy" | "unrecognized";
|
||||
/** The stored value as read (null when the key is absent). */
|
||||
value: string | null;
|
||||
}
|
||||
|
||||
let cachedConsent: TranscriptConsent | null = null;
|
||||
|
||||
/**
|
||||
* Read `transcript_ingest_mode` once per process (same parse as
|
||||
* `gstack-config has` + `get`) and decide whether transcripts may be ingested.
|
||||
* Only `recent` and `all` are consent. An absent key, `off`, a legacy value
|
||||
* and anything unrecognized are not. Pass `env` to bypass the per-process cache.
|
||||
*/
|
||||
export function normalizeTranscriptConsent(env?: StateRootEnv): TranscriptConsent {
|
||||
if (!env && cachedConsent) return cachedConsent;
|
||||
const value = readConfigKeyWithRoot("transcript_ingest_mode", env ?? process.env).value;
|
||||
const v = (value ?? "").trim().toLowerCase();
|
||||
let consent: TranscriptConsent;
|
||||
if (value === null) consent = { affirmative: false, window: null, reason: "not-set", value };
|
||||
else if (v === "recent" || v === "all") consent = { affirmative: true, window: v, reason: v, value };
|
||||
else if (v === "off") consent = { affirmative: false, window: null, reason: "off", value };
|
||||
else if (/^[a-e]$/.test(v) || v === "incremental") consent = { affirmative: false, window: null, reason: "legacy", value };
|
||||
else consent = { affirmative: false, window: null, reason: "unrecognized", value };
|
||||
if (!env) cachedConsent = consent;
|
||||
return consent;
|
||||
}
|
||||
|
||||
// ── CLI ────────────────────────────────────────────────────────────────────
|
||||
|
||||
function printUsage(): void {
|
||||
@@ -218,6 +257,9 @@ Options:
|
||||
--include-unattributed Ingest sessions with no resolvable git remote.
|
||||
--all-history Walk transcripts older than 90 days too.
|
||||
--sources <list> Comma-separated subset: ${ALL_TYPES.join(",")}
|
||||
Default: every type, minus transcript unless
|
||||
transcript_ingest_mode is recent (90 days) or all
|
||||
(all history). A list naming transcript overrides it.
|
||||
--limit <N> Stop after N pages written (smoke testing).
|
||||
--no-write Skip gbrain put calls (still updates state file).
|
||||
Used by tests + dry runs without actual ingest.
|
||||
@@ -237,6 +279,7 @@ function parseArgs(): CliArgs {
|
||||
let allHistory = false;
|
||||
let limit: number | null = null;
|
||||
let sources: Set<MemoryType> = new Set(ALL_TYPES);
|
||||
let sourcesExplicit = false;
|
||||
let noWrite = process.env.GSTACK_MEMORY_INGEST_NO_WRITE === "1";
|
||||
let scanSecrets = process.env.GSTACK_MEMORY_INGEST_SCAN_SECRETS === "1";
|
||||
|
||||
@@ -266,6 +309,7 @@ function parseArgs(): CliArgs {
|
||||
console.error(`--sources must include at least one of: ${ALL_TYPES.join(",")}`);
|
||||
process.exit(1);
|
||||
}
|
||||
sourcesExplicit = true;
|
||||
break;
|
||||
}
|
||||
case "--help":
|
||||
@@ -279,6 +323,15 @@ function parseArgs(): CliArgs {
|
||||
}
|
||||
}
|
||||
|
||||
const consent = normalizeTranscriptConsent();
|
||||
if (!sourcesExplicit && !consent.affirmative) {
|
||||
sources.delete("transcript");
|
||||
if (!quiet) {
|
||||
console.error(`gstack-memory-ingest: transcripts skipped (transcript_ingest_mode=${consent.value ?? "not set"}); set it to recent or all, or pass --sources transcript.`);
|
||||
}
|
||||
}
|
||||
if (consent.window === "all") allHistory = true;
|
||||
|
||||
return { mode, quiet, benchmark, includeUnattributed, allHistory, sources, limit, noWrite, scanSecrets };
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,458 @@
|
||||
/**
|
||||
* Transcript consent (transcript_ingest_mode) in the memory stage.
|
||||
*
|
||||
* Transcripts ingest only for a stored `recent` (last 90 days) or `all` (all
|
||||
* history). An absent key, `off`, legacy gate values and anything else skip
|
||||
* transcripts while the other memory types still sync, and the sync prints a
|
||||
* notice naming how to choose. Runs the real orchestrator and memory ingest
|
||||
* against a stub gbrain in a temp HOME / GSTACK_HOME.
|
||||
*/
|
||||
import { describe, expect, test } from "bun:test";
|
||||
import { chmodSync, existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, utimesSync, writeFileSync } from "fs";
|
||||
import { tmpdir } from "os";
|
||||
import { join } from "path";
|
||||
import { spawn, spawnSync } from "child_process";
|
||||
|
||||
import {
|
||||
MEMORY_INGEST_TYPES,
|
||||
applyTranscriptConsent,
|
||||
transcriptConsentNotice,
|
||||
} from "../bin/gstack-gbrain-sync";
|
||||
import { normalizeTranscriptConsent } from "../bin/gstack-memory-ingest";
|
||||
|
||||
const ROOT = join(import.meta.dir, "..");
|
||||
const SYNC = join(ROOT, "bin", "gstack-gbrain-sync.ts");
|
||||
const INGEST = join(ROOT, "bin", "gstack-memory-ingest.ts");
|
||||
const HINT =
|
||||
"To choose: run /sync-gbrain, or gstack-config set transcript_ingest_mode recent|all|off. Details: setup-gbrain/memory.md#transcripts";
|
||||
const notice = (value: string) =>
|
||||
`gbrain-sync: transcripts skipped (transcript_ingest_mode=${value}). Other memory still syncs. ${HINT}`;
|
||||
|
||||
describe("normalizeTranscriptConsent", () => {
|
||||
const consent = (yaml: string | null) => {
|
||||
const dir = mkdtempSync(join(tmpdir(), "gstack-consent-"));
|
||||
try {
|
||||
if (yaml !== null) writeFileSync(join(dir, "config.yaml"), yaml);
|
||||
return normalizeTranscriptConsent({ GSTACK_STATE_ROOT: dir, HOME: dir });
|
||||
} finally {
|
||||
rmSync(dir, { recursive: true, force: true });
|
||||
}
|
||||
};
|
||||
|
||||
test("only recent and all are consent", () => {
|
||||
expect(consent("transcript_ingest_mode: recent\n")).toEqual({ affirmative: true, window: "recent", reason: "recent", value: "recent" });
|
||||
expect(consent("transcript_ingest_mode: all\n")).toEqual({ affirmative: true, window: "all", reason: "all", value: "all" });
|
||||
expect(consent("transcript_ingest_mode: off\n")).toMatchObject({ affirmative: false, window: null, reason: "off" });
|
||||
expect(consent(null)).toEqual({ affirmative: false, window: null, reason: "not-set", value: null });
|
||||
expect(consent("telemetry: off\n")).toMatchObject({ affirmative: false, reason: "not-set" });
|
||||
});
|
||||
|
||||
test("legacy gate values, any case and padding, need a new choice", () => {
|
||||
for (const v of [" A ", "b", "C", "D", "e", "incremental", "Incremental"]) {
|
||||
expect(consent(`transcript_ingest_mode: ${v}\n`)).toMatchObject({ affirmative: false, window: null, reason: "legacy" });
|
||||
}
|
||||
});
|
||||
|
||||
test("anything else is unrecognized, not consent", () => {
|
||||
for (const v of ["yes", "new-only", "90d", "true"]) {
|
||||
expect(consent(`transcript_ingest_mode: ${v}\n`)).toMatchObject({ affirmative: false, reason: "unrecognized", value: v });
|
||||
}
|
||||
});
|
||||
|
||||
test("the last line wins, like gstack-config get", () => {
|
||||
expect(consent("transcript_ingest_mode: all\ntranscript_ingest_mode: off\n")).toMatchObject({ reason: "off" });
|
||||
});
|
||||
});
|
||||
|
||||
describe("applyTranscriptConsent", () => {
|
||||
const yes = { affirmative: true, window: "recent" as const, reason: "recent" as const, value: "recent" };
|
||||
const no = { affirmative: false, window: null, reason: "not-set" as const, value: null };
|
||||
|
||||
test("without consent, transcript leaves the default and the rest is pinned as an explicit list", () => {
|
||||
const r = applyTranscriptConsent({ sources: null, why: "default: no federated gstack source registered" }, no);
|
||||
expect(r.sources).toEqual(MEMORY_INGEST_TYPES.filter((t) => t !== "transcript"));
|
||||
expect(r.selected).toEqual(r.sources!);
|
||||
expect(r.override).toBe(false);
|
||||
});
|
||||
|
||||
test("with consent, the default stays null so the ingest walks its own default", () => {
|
||||
const r = applyTranscriptConsent({ sources: null, why: "default: no federated gstack source registered" }, yes);
|
||||
expect(r.sources).toBeNull();
|
||||
expect(r.selected).toEqual([...MEMORY_INGEST_TYPES]);
|
||||
});
|
||||
|
||||
test("an explicit list naming transcript overrides; a derived list does not", () => {
|
||||
expect(applyTranscriptConsent({ sources: ["transcript", "learning"], why: "--sources" }, no)).toMatchObject({ override: true, sources: ["transcript", "learning"] });
|
||||
expect(applyTranscriptConsent({ sources: ["transcript"], why: "GSTACK_MEMORY_INGEST_SOURCES" }, no)).toMatchObject({ override: true });
|
||||
const federated = applyTranscriptConsent({ sources: ["transcript", "learning"], why: "default: ceo-plan,design-doc already indexed" }, no);
|
||||
expect(federated).toMatchObject({ override: false, sources: ["learning"] });
|
||||
});
|
||||
|
||||
test("notice copy", () => {
|
||||
expect(transcriptConsentNotice(no, true)).toBe(notice("not set"));
|
||||
expect(transcriptConsentNotice({ ...no, reason: "legacy", value: "A" }, false)).toBe(notice("A"));
|
||||
expect(transcriptConsentNotice({ ...no, reason: "off", value: "off" }, false)).toBe("gbrain-sync: transcripts off (your choice)");
|
||||
expect(transcriptConsentNotice({ ...no, reason: "off", value: "off" }, true)).toBeNull();
|
||||
expect(transcriptConsentNotice(yes, false)).toBeNull();
|
||||
});
|
||||
});
|
||||
|
||||
interface Sandbox {
|
||||
home: string;
|
||||
gstackHome: string;
|
||||
stagedList: string;
|
||||
calls: string;
|
||||
env: Record<string, string>;
|
||||
cleanup: () => void;
|
||||
}
|
||||
|
||||
const DAY_MS = 24 * 60 * 60 * 1000;
|
||||
|
||||
function sandbox(opts: { federated?: boolean; mode?: string } = {}): Sandbox {
|
||||
const home = mkdtempSync(join(tmpdir(), "gstack-tmode-"));
|
||||
const gstackHome = join(home, ".gstack");
|
||||
const worktree = join(home, ".gstack-brain-worktree");
|
||||
const bindir = join(home, "bin");
|
||||
const stagedList = join(home, "staged.list");
|
||||
const calls = join(home, "gbrain.calls");
|
||||
for (const d of [gstackHome, worktree, bindir, join(home, ".gbrain")]) mkdirSync(d, { recursive: true });
|
||||
writeFileSync(join(home, ".gbrain", "config.json"), JSON.stringify({ engine: "pglite", database_url: "pglite:///test" }));
|
||||
if (opts.mode !== undefined) writeFileSync(join(gstackHome, "config.yaml"), `transcript_ingest_mode: ${opts.mode}\n`);
|
||||
|
||||
// Curated memory: one learning and one CEO plan.
|
||||
const proj = join(gstackHome, "projects", "acme-app");
|
||||
mkdirSync(join(proj, "ceo-plans"), { recursive: true });
|
||||
writeFileSync(join(proj, "ceo-plans", "2026-09-01-brand.md"), "# Brand plan\n\nShip the brand refresh.\n");
|
||||
writeFileSync(
|
||||
join(proj, "learnings.jsonl"),
|
||||
JSON.stringify({ ts: "2026-09-01T00:00:00Z", skill: "review", type: "pitfall", key: "k", insight: "Check the lock first.", confidence: 8 }) + "\n",
|
||||
);
|
||||
|
||||
// Two attributable Claude Code sessions: one recent, one 200 days old.
|
||||
const repo = join(home, "work", "acme");
|
||||
mkdirSync(repo, { recursive: true });
|
||||
const git = (...args: string[]) => spawnSync("git", ["-C", repo, ...args], { encoding: "utf-8", timeout: 30_000 });
|
||||
git("init", "-q");
|
||||
git("remote", "add", "origin", "https://github.com/acme/app.git");
|
||||
git("config", "--local", "url.https://github.com/acme/app.git.insteadOf", "https://github.com/acme/app.git");
|
||||
const sessionDir = join(home, ".claude", "projects", "work-acme");
|
||||
mkdirSync(sessionDir, { recursive: true });
|
||||
const session = (id: string, text: string) =>
|
||||
`{"type":"user","message":{"role":"user","content":"${text}"},"timestamp":"2026-09-01T00:00:00Z","cwd":"${repo}","sessionId":"${id}"}\n` +
|
||||
`{"type":"assistant","message":{"role":"assistant","content":"ok"},"timestamp":"2026-09-01T00:00:01Z"}\n`;
|
||||
writeFileSync(join(sessionDir, "recent0001.jsonl"), session("recent0001", "recent session"));
|
||||
const old = join(sessionDir, "old000000001.jsonl");
|
||||
writeFileSync(old, session("old000000001", "old session"));
|
||||
const oldTime = new Date(Date.now() - 200 * DAY_MS);
|
||||
utimesSync(old, oldTime, oldTime);
|
||||
|
||||
// Stub gbrain. With GBRAIN_PAUSE_DIR set, `sources list` (the first gbrain
|
||||
// call of the memory stage) waits until the test releases it.
|
||||
const sources = opts.federated
|
||||
? `{"sources":[{"id":"gstack-artifacts-acme","local_path":"${worktree}","federated":true,"page_count":12}]}`
|
||||
: `{"sources":[]}`;
|
||||
writeFileSync(join(bindir, "gbrain"), `#!/bin/sh
|
||||
printf '%s\\n' "$*" >> "${calls}"
|
||||
case "$1" in
|
||||
--version) echo "gbrain 0.60.28.0" ;;
|
||||
--help) printf 'Commands:\\n import <dir> Import markdown directory\\n' ;;
|
||||
sources)
|
||||
if [ -n "$GBRAIN_PAUSE_DIR" ]; then
|
||||
touch "$GBRAIN_PAUSE_DIR/paused"
|
||||
i=0
|
||||
while [ ! -e "$GBRAIN_PAUSE_DIR/release" ] && [ $i -lt 300 ]; do sleep 0.1; i=$((i+1)); done
|
||||
fi
|
||||
echo '${sources}' ;;
|
||||
import)
|
||||
( cd "$2" && find . -name '*.md' -type f | sort ) > "${stagedList}"
|
||||
n=$(wc -l < "${stagedList}" | tr -d ' ')
|
||||
echo "{\\"status\\":\\"success\\",\\"imported\\":$n,\\"skipped\\":0,\\"errors\\":0,\\"total_files\\":$n}"
|
||||
;;
|
||||
*) echo "unexpected gbrain $*" >&2; exit 1 ;;
|
||||
esac
|
||||
`);
|
||||
chmodSync(join(bindir, "gbrain"), 0o755);
|
||||
return {
|
||||
home,
|
||||
gstackHome,
|
||||
stagedList,
|
||||
calls,
|
||||
env: {
|
||||
HOME: home,
|
||||
GSTACK_HOME: gstackHome,
|
||||
GBRAIN_HOME: "",
|
||||
GSTACK_MEMORY_INGEST_SOURCES: "",
|
||||
GIT_CONFIG_NOSYSTEM: "1",
|
||||
GIT_CONFIG_GLOBAL: join(home, ".gitconfig"),
|
||||
PATH: `${bindir}:${process.env.PATH || ""}`,
|
||||
},
|
||||
cleanup: () => rmSync(home, { recursive: true, force: true }),
|
||||
};
|
||||
}
|
||||
|
||||
function runSync(sb: Sandbox, args: string[], env: Record<string, string> = {}) {
|
||||
return spawnSync("bun", [SYNC, "--no-code", "--no-brain-sync", ...args], {
|
||||
encoding: "utf-8",
|
||||
timeout: 60_000,
|
||||
cwd: sb.home,
|
||||
env: { ...process.env, ...sb.env, ...env },
|
||||
});
|
||||
}
|
||||
|
||||
function staged(sb: Sandbox): string {
|
||||
return existsSync(sb.stagedList) ? readFileSync(sb.stagedList, "utf-8") : "";
|
||||
}
|
||||
|
||||
function memoryStage(sb: Sandbox): { ok: boolean; memory_sources?: string[]; transcript_consent?: Record<string, unknown> } {
|
||||
const state = JSON.parse(readFileSync(join(sb.gstackHome, ".gbrain-sync-state.json"), "utf-8"));
|
||||
return state.last_stages.find((s: { name: string }) => s.name === "memory");
|
||||
}
|
||||
|
||||
const transcripts = (sb: Sandbox) => staged(sb).split("\n").filter((l) => l.includes("transcripts/"));
|
||||
|
||||
describe.skipIf(process.platform === "win32")("memory stage transcript consent", () => {
|
||||
test("absent key: transcripts skipped, other memory syncs, notice prints under --quiet", () => {
|
||||
const sb = sandbox();
|
||||
try {
|
||||
const r = runSync(sb, ["--incremental", "--quiet"]);
|
||||
expect(r.status).toBe(0);
|
||||
expect(transcripts(sb)).toEqual([]);
|
||||
expect(staged(sb)).toMatch(/learning/);
|
||||
expect(staged(sb)).toMatch(/ceo-plan/);
|
||||
expect(r.stderr).toContain(notice("not set"));
|
||||
const stage = memoryStage(sb);
|
||||
expect(stage.memory_sources).not.toContain("transcript");
|
||||
expect(stage.transcript_consent).toEqual({ mode: "not-set", window: null, skip_reason: "not-set" });
|
||||
} finally {
|
||||
sb.cleanup();
|
||||
}
|
||||
});
|
||||
|
||||
test("stored off: skipped; neutral line only without --quiet", () => {
|
||||
const sb = sandbox({ mode: "off" });
|
||||
try {
|
||||
const loud = runSync(sb, ["--incremental"]);
|
||||
expect(loud.status).toBe(0);
|
||||
expect(transcripts(sb)).toEqual([]);
|
||||
expect(loud.stderr).toContain("gbrain-sync: transcripts off (your choice)");
|
||||
expect(loud.stderr).not.toContain("transcripts skipped");
|
||||
const quiet = runSync(sb, ["--incremental", "--quiet"]);
|
||||
expect(quiet.status).toBe(0);
|
||||
expect(quiet.stderr).not.toContain("gbrain-sync: transcripts");
|
||||
expect(memoryStage(sb).transcript_consent).toEqual({ mode: "off", window: null, skip_reason: "off" });
|
||||
} finally {
|
||||
sb.cleanup();
|
||||
}
|
||||
});
|
||||
|
||||
test("legacy values are not consent", () => {
|
||||
for (const mode of [" A ", "b", "D", "incremental"]) {
|
||||
const sb = sandbox({ mode });
|
||||
try {
|
||||
const r = runSync(sb, ["--full", "--no-dream", "--quiet"]);
|
||||
expect(r.status).toBe(0);
|
||||
expect(transcripts(sb)).toEqual([]);
|
||||
expect(staged(sb)).toMatch(/learning/);
|
||||
expect(r.stderr).toContain(notice(mode.trim()));
|
||||
expect(memoryStage(sb).transcript_consent).toMatchObject({ mode: "legacy", skip_reason: "legacy" });
|
||||
} finally {
|
||||
sb.cleanup();
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
test("recent: the 90-day window, on --full too", () => {
|
||||
const sb = sandbox({ mode: "recent" });
|
||||
try {
|
||||
const r = runSync(sb, ["--full", "--no-dream", "--quiet"]);
|
||||
expect(r.status).toBe(0);
|
||||
expect(transcripts(sb).join("\n")).toMatch(/recent0001/);
|
||||
expect(transcripts(sb).join("\n")).not.toMatch(/old00000000/);
|
||||
expect(r.stderr).not.toContain("gbrain-sync: transcripts");
|
||||
expect(memoryStage(sb).transcript_consent).toEqual({ mode: "recent", window: "recent" });
|
||||
} finally {
|
||||
sb.cleanup();
|
||||
}
|
||||
});
|
||||
|
||||
test("all: --full walks all history", () => {
|
||||
const sb = sandbox({ mode: "all" });
|
||||
try {
|
||||
const r = runSync(sb, ["--full", "--no-dream", "--quiet"]);
|
||||
expect(r.status).toBe(0);
|
||||
expect(transcripts(sb).join("\n")).toMatch(/recent0001/);
|
||||
expect(transcripts(sb).join("\n")).toMatch(/old00000000/);
|
||||
expect(memoryStage(sb).transcript_consent).toEqual({ mode: "all", window: "all" });
|
||||
} finally {
|
||||
sb.cleanup();
|
||||
}
|
||||
});
|
||||
|
||||
test("an explicit list naming transcript overrides an absent key; the notice still prints", () => {
|
||||
for (const [args, env, why] of [
|
||||
[["--sources", "transcript,learning"], {}, "--sources"],
|
||||
[[], { GSTACK_MEMORY_INGEST_SOURCES: "transcript" }, "GSTACK_MEMORY_INGEST_SOURCES"],
|
||||
] as const) {
|
||||
const sb = sandbox();
|
||||
try {
|
||||
const r = runSync(sb, ["--incremental", "--quiet", ...args], { ...env });
|
||||
expect(r.status).toBe(0);
|
||||
expect(transcripts(sb).join("\n")).toMatch(/recent0001/);
|
||||
expect(r.stderr).toContain(`gbrain-sync: transcripts ingested because ${why} names transcript (transcript_ingest_mode=not set). ${HINT}`);
|
||||
expect(memoryStage(sb).transcript_consent).toEqual({ mode: "not-set", window: null });
|
||||
} finally {
|
||||
sb.cleanup();
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
test("--sources all, an empty env list and an invalid-only list do not override", () => {
|
||||
for (const [args, env] of [
|
||||
[["--sources", "all"], {}],
|
||||
[[], { GSTACK_MEMORY_INGEST_SOURCES: "" }],
|
||||
[["--sources", "bogus"], {}],
|
||||
[[], { GSTACK_MEMORY_INGEST_SOURCES: "nope" }],
|
||||
] as const) {
|
||||
const sb = sandbox();
|
||||
try {
|
||||
const r = runSync(sb, ["--incremental", "--quiet", ...args], { ...env });
|
||||
expect(r.status).toBe(0);
|
||||
expect(transcripts(sb)).toEqual([]);
|
||||
expect(staged(sb)).toMatch(/learning/);
|
||||
expect(r.stderr).toContain(notice("not set"));
|
||||
} finally {
|
||||
sb.cleanup();
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
test("federated source registered + absent key: no curated pages, no transcripts", () => {
|
||||
const sb = sandbox({ federated: true });
|
||||
try {
|
||||
const r = runSync(sb, ["--incremental", "--quiet"]);
|
||||
expect(r.status).toBe(0);
|
||||
expect(staged(sb)).toMatch(/learning/);
|
||||
expect(staged(sb)).not.toMatch(/ceo-plan/);
|
||||
expect(transcripts(sb)).toEqual([]);
|
||||
expect(memoryStage(sb).memory_sources).toEqual(
|
||||
MEMORY_INGEST_TYPES.filter((t) => !["transcript", "ceo-plan", "design-doc"].includes(t)),
|
||||
);
|
||||
} finally {
|
||||
sb.cleanup();
|
||||
}
|
||||
});
|
||||
|
||||
test("a config change during the run does not add transcripts to it", async () => {
|
||||
const sb = sandbox();
|
||||
const pause = join(sb.home, "pause");
|
||||
mkdirSync(pause);
|
||||
try {
|
||||
const child = spawn("bun", [SYNC, "--no-code", "--no-brain-sync", "--incremental", "--quiet"], {
|
||||
cwd: sb.home,
|
||||
env: { ...process.env, ...sb.env, GBRAIN_PAUSE_DIR: pause },
|
||||
stdio: ["ignore", "pipe", "pipe"],
|
||||
});
|
||||
let stderr = "";
|
||||
child.stderr.on("data", (d) => (stderr += d));
|
||||
child.stdout.resume();
|
||||
const exited = new Promise<number | null>((resolve) => child.on("exit", resolve));
|
||||
const deadline = Date.now() + 30_000;
|
||||
while (!existsSync(join(pause, "paused")) && Date.now() < deadline) await Bun.sleep(50);
|
||||
expect(existsSync(join(pause, "paused"))).toBe(true);
|
||||
writeFileSync(join(sb.gstackHome, "config.yaml"), "transcript_ingest_mode: all\n");
|
||||
writeFileSync(join(pause, "release"), "");
|
||||
expect(await exited).toBe(0);
|
||||
expect(transcripts(sb)).toEqual([]);
|
||||
expect(staged(sb)).toMatch(/learning/);
|
||||
expect(stderr).toContain(notice("not set"));
|
||||
expect(memoryStage(sb).transcript_consent).toMatchObject({ mode: "not-set" });
|
||||
} finally {
|
||||
sb.cleanup();
|
||||
}
|
||||
}, 60_000);
|
||||
|
||||
describe("checkpoint resume requires the same consent policy", () => {
|
||||
function checkpoint(sb: Sandbox, previous: { memory_sources: string[]; transcript_consent?: Record<string, unknown> }): void {
|
||||
const stagingDir = join(sb.gstackHome, ".staging-ingest-1-1");
|
||||
mkdirSync(join(stagingDir, "projects"), { recursive: true });
|
||||
writeFileSync(join(stagingDir, ".gstack-staging"), "");
|
||||
writeFileSync(join(stagingDir, "projects", "stale-page.md"), "# stale\n");
|
||||
writeFileSync(join(sb.home, ".gbrain", "import-checkpoint.json"), JSON.stringify({ dir: stagingDir, processedIndex: 0, totalFiles: 1 }));
|
||||
writeFileSync(join(sb.gstackHome, ".gbrain-sync-state.json"), JSON.stringify({
|
||||
schema_version: 1,
|
||||
last_writer: "gstack-gbrain-sync",
|
||||
last_stages: [{ name: "memory", ran: true, ok: false, duration_ms: 1, summary: "timeout", ...previous }],
|
||||
}));
|
||||
}
|
||||
|
||||
test("same types and policy resume the checkpoint", () => {
|
||||
const sb = sandbox({ mode: "recent" });
|
||||
try {
|
||||
checkpoint(sb, { memory_sources: [...MEMORY_INGEST_TYPES], transcript_consent: { mode: "recent", window: "recent" } });
|
||||
// The fixture's staging dir is not a real interrupted import, so the
|
||||
// import's page accounting fails afterwards; the decision is what counts.
|
||||
const r = runSync(sb, ["--incremental", "--quiet"]);
|
||||
expect(r.stderr).toContain("resuming from gbrain checkpoint");
|
||||
expect(r.stderr).not.toContain("transcript consent changed");
|
||||
expect(staged(sb)).toContain("stale-page");
|
||||
} finally {
|
||||
sb.cleanup();
|
||||
}
|
||||
});
|
||||
|
||||
test("recent → all restages once and says so", () => {
|
||||
const sb = sandbox({ mode: "all" });
|
||||
try {
|
||||
checkpoint(sb, { memory_sources: [...MEMORY_INGEST_TYPES], transcript_consent: { mode: "recent", window: "recent" } });
|
||||
const r = runSync(sb, ["--incremental", "--quiet"]);
|
||||
expect(r.status).toBe(0);
|
||||
expect(r.stderr).toContain("gbrain-sync: transcript consent changed since the interrupted import (recent → all)");
|
||||
expect(r.stderr).not.toContain("resuming from gbrain checkpoint");
|
||||
expect(staged(sb)).not.toContain("stale-page");
|
||||
} finally {
|
||||
sb.cleanup();
|
||||
}
|
||||
});
|
||||
|
||||
test("all → off restages, with the skip notice first", () => {
|
||||
const sb = sandbox({ mode: "off" });
|
||||
try {
|
||||
checkpoint(sb, { memory_sources: [...MEMORY_INGEST_TYPES], transcript_consent: { mode: "all", window: "all" } });
|
||||
const r = runSync(sb, ["--incremental"]);
|
||||
expect(r.status).toBe(0);
|
||||
const offAt = r.stderr.indexOf("gbrain-sync: transcripts off (your choice)");
|
||||
const restageAt = r.stderr.indexOf("gbrain-sync: transcript consent changed since the interrupted import (all → off)");
|
||||
expect(offAt).toBeGreaterThanOrEqual(0);
|
||||
expect(restageAt).toBeGreaterThan(offAt);
|
||||
expect(staged(sb)).not.toContain("stale-page");
|
||||
expect(transcripts(sb)).toEqual([]);
|
||||
} finally {
|
||||
sb.cleanup();
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
test("gstack-memory-ingest --bulk with the key absent ingests no transcripts", () => {
|
||||
const sb = sandbox();
|
||||
try {
|
||||
const r = spawnSync("bun", [INGEST, "--bulk"], { encoding: "utf-8", timeout: 60_000, cwd: sb.home, env: { ...process.env, ...sb.env } });
|
||||
expect(r.status).toBe(0);
|
||||
expect(transcripts(sb)).toEqual([]);
|
||||
expect(staged(sb)).toMatch(/learning/);
|
||||
expect(r.stderr).toContain("gstack-memory-ingest: transcripts skipped (transcript_ingest_mode=not set)");
|
||||
} finally {
|
||||
sb.cleanup();
|
||||
}
|
||||
});
|
||||
|
||||
test("gstack-memory-ingest --bulk with all walks all history", () => {
|
||||
const sb = sandbox({ mode: "all" });
|
||||
try {
|
||||
const r = spawnSync("bun", [INGEST, "--bulk", "--quiet"], { encoding: "utf-8", timeout: 60_000, cwd: sb.home, env: { ...process.env, ...sb.env } });
|
||||
expect(r.status).toBe(0);
|
||||
expect(transcripts(sb).join("\n")).toMatch(/old00000000/);
|
||||
} finally {
|
||||
sb.cleanup();
|
||||
}
|
||||
});
|
||||
});
|
||||
@@ -791,7 +791,22 @@ function isolateGitRemote(repo: string, url: string): void {
|
||||
expect(git("remote", "get-url", "origin")).toBe(url);
|
||||
}
|
||||
|
||||
/**
|
||||
* These cases exercise transcript ingest after the user consented, so the
|
||||
* isolated state root gets `transcript_ingest_mode: recent` unless the case
|
||||
* set the key itself. The no-consent paths live in
|
||||
* gstack-gbrain-sync-transcript-mode.test.ts.
|
||||
*/
|
||||
function seedTranscriptConsent(gstackHome: string | undefined): void {
|
||||
if (!gstackHome) return;
|
||||
mkdirSync(gstackHome, { recursive: true });
|
||||
const config = join(gstackHome, "config.yaml");
|
||||
const current = existsSync(config) ? readFileSync(config, "utf-8") : "";
|
||||
if (!/^transcript_ingest_mode:/m.test(current)) writeFileSync(config, `${current}transcript_ingest_mode: recent\n`);
|
||||
}
|
||||
|
||||
function runScript(args: string[], env: Record<string, string> = {}): { stdout: string; stderr: string; exitCode: number } {
|
||||
seedTranscriptConsent(env.GSTACK_HOME);
|
||||
const result = spawnSync("bun", [SCRIPT, ...args], {
|
||||
encoding: "utf-8",
|
||||
timeout: 30000,
|
||||
|
||||
@@ -194,6 +194,8 @@ describe.skipIf(process.platform === "win32")("memory stage source selection (#2
|
||||
test("without a federated source, the default still ingests every type", () => {
|
||||
const sb = sandbox({ federated: false });
|
||||
try {
|
||||
// Transcripts join the default selection only with consent.
|
||||
writeFileSync(join(sb.gstackHome, "config.yaml"), "transcript_ingest_mode: recent\n");
|
||||
const r = runSync(sb, ["--incremental"]);
|
||||
expect(r.status).toBe(0);
|
||||
expect(ingestedTypes(sb)).toEqual(["ceo-plan", "design-doc", "learning"]);
|
||||
|
||||
Reference in new issue
Block a user