W6 T1, T4 (eng), T3 (DX): scoped consent in ingest and sync

Both parsers apply a new@ cutoff to each session's first record timestamp
(prefix read, never mtime; recent keeps its mtime window exactly as
before), the allowlist applies after the attribution gate, and an explicit
--sources/GSTACK_MEMORY_INGEST_SOURCES override never widens either.
Summaries count exclusions by reason, a new@ run with no sessions says so,
pages carry session_started_at, scoped tightening purges staged pages and
their fingerprints, and the stage consent record carries the cutoff and
allowlist hash so a scope change restages.
This commit is contained in:
garrytan committed 2026-10-03 21:14:47 +00:00
1 parent a95f5fa209
commit bc233f33e8
3 files changed
+447 -65

No files matched your search

+46 -16
View File
@@ -45,7 +45,14 @@ import { buildGbrainEnv, spawnGbrain, spawnGbrainAsync, execGbrainJson, NEEDS_SH
import { repoPolicyTier as sharedRepoPolicyTier } from "../lib/gbrain-repo-policy-client";
import { checkOwnedStagingDir } from "../lib/staging-guard";
import { resolveStateRoot } from "../lib/state-root";
import { normalizeTranscriptConsent, type TranscriptConsent } from "./gstack-memory-ingest";
import type { TranscriptConsent } from "./gstack-memory-ingest";
import {
describeTranscriptPolicy,
isScoped,
readTranscriptConsent,
reposHash,
type TranscriptPolicy,
} from "../lib/transcript-consent";
// ── Types ──────────────────────────────────────────────────────────────────
@@ -107,11 +114,15 @@ interface StageResult {
}
interface TranscriptConsentRecord {
/** Normalized transcript_ingest_mode: recent | all | off | not-set | legacy | unrecognized. */
/** Normalized transcript_ingest_mode: recent | all | new | off | not-set | legacy | unrecognized | repos-unreadable. */
mode: string;
window: "recent" | "all" | null;
window: "recent" | "all" | "new" | null;
/** Why transcripts were not ingested; absent when they were. */
skip_reason?: string;
/** The new@ cutoff in force; a changed cutoff restages. */
cutoff?: string;
/** Hash of the transcript_repos allowlist in force; a scope change restages. */
repos_hash?: string;
}
// ── Constants ──────────────────────────────────────────────────────────────
@@ -304,16 +315,28 @@ export function transcriptConsentNotice(consent: TranscriptConsent, quiet: boole
if (consent.affirmative) return null;
const value = consent.value ?? "not set";
if (override) {
return `gbrain-sync: transcripts ingested because ${override} names transcript (transcript_ingest_mode=${value}). ${TRANSCRIPT_CHOICE_HINT}`;
const scope = isScoped(consent) ? " The new@ cutoff and transcript_repos allowlist still apply." : "";
return `gbrain-sync: transcripts ingested because ${override} names transcript (transcript_ingest_mode=${value}).${scope} ${TRANSCRIPT_CHOICE_HINT}`;
}
if (consent.reason === "off") return quiet ? null : "gbrain-sync: transcripts off (your choice)";
if (consent.reason === "repos-unreadable") {
return `gbrain-sync: transcripts skipped (transcript_ingest_mode=${value}): its +repos marker needs a transcript_repos allowlist, which is missing or empty. Other memory still syncs. ${TRANSCRIPT_CHOICE_HINT}`;
}
return `gbrain-sync: transcripts skipped (transcript_ingest_mode=${value}). Other memory still syncs. ${TRANSCRIPT_CHOICE_HINT}`;
}
/** The consent record a memory stage stores; resume requires mode and window to match. */
/** The consent in words for a consenting run (window or cutoff, repos, config roots). */
export function transcriptConsentSummary(consent: TranscriptPolicy, quiet: boolean): string | null {
if (!consent.affirmative || quiet) return null;
return `gbrain-sync: transcripts: ${describeTranscriptPolicy(consent)}`;
}
/** The consent record a memory stage stores; resume requires every field to match. */
export function transcriptConsentRecord(consent: TranscriptConsent, override: boolean): TranscriptConsentRecord {
const record: TranscriptConsentRecord = { mode: consent.reason, window: consent.window };
if (!consent.affirmative && !override) record.skip_reason = consent.reason;
if (consent.cutoff !== undefined) record.cutoff = consent.cutoff;
if (consent.repos !== undefined) record.repos_hash = reposHash(consent.repos);
return record;
}
@@ -448,10 +471,12 @@ Options:
except ${FEDERATED_CURATED_TYPES.join(",")} when a federated
gstack source already indexes them, and except
transcript unless transcript_ingest_mode is recent
(last 90 days) or all (all history; --full passes
--all-history). Without that consent each run prints a
notice, even with --quiet. A list naming transcript
overrides the mode for that run. See
(last 90 days), all (all history; --full passes
--all-history) or new@<UTC> (sessions started after
it). Without that consent each run prints a notice,
even with --quiet. A list naming transcript overrides
the mode for that run, but a new@ cutoff and the
transcript_repos allowlist still apply. See
setup-gbrain/memory.md#transcripts.
--help This text.
@@ -1470,12 +1495,12 @@ export function ensureGbrainSourceGitignored(root: string): void {
}
}
function printTranscriptNotice(args: CliArgs, consent: TranscriptConsent, selection: ConsentedSelection, why: string): void {
const notice = transcriptConsentNotice(consent, args.quiet, selection.override ? why : null);
function printTranscriptNotice(args: CliArgs, consent: TranscriptPolicy, selection: ConsentedSelection, why: string): void {
const notice = transcriptConsentNotice(consent, args.quiet, selection.override ? why : null) ?? transcriptConsentSummary(consent, args.quiet);
if (notice) console.error(notice);
}
function runMemoryIngest(args: CliArgs, previous: SyncState, consent: TranscriptConsent): StageResult {
function runMemoryIngest(args: CliArgs, previous: SyncState, consent: TranscriptPolicy): StageResult {
const t0 = Date.now();
if (args.mode === "dry-run") {
@@ -1518,14 +1543,19 @@ function runMemoryIngest(args: CliArgs, previous: SyncState, consent: Transcript
// transcript window.
const prevStage = previous.last_stages?.find((st) => st.name === "memory");
const staged = prevStage?.memory_sources ?? [...MEMORY_INGEST_TYPES];
const prevConsent = prevStage?.transcript_consent ?? { mode: "recent", window: "recent" };
const prevConsent: TranscriptConsentRecord = prevStage?.transcript_consent ?? { mode: "recent", window: "recent" };
if ([...staged].sort().join(",") !== [...selected].sort().join(",")) {
console.error(
`[sync:memory] memory source selection changed since the checkpointed run (${staged.join(",")} → ${selected.join(",")}); restaging from scratch.`,
);
resume = { kind: "no-checkpoint" };
}
if (prevConsent.mode !== consentRecord.mode || prevConsent.window !== consentRecord.window) {
if (
prevConsent.mode !== consentRecord.mode ||
prevConsent.window !== consentRecord.window ||
prevConsent.cutoff !== consentRecord.cutoff ||
prevConsent.repos_hash !== consentRecord.repos_hash
) {
console.error(
`gbrain-sync: transcript consent changed since the interrupted import (${prevConsent.mode} → ${consentRecord.mode}); restaging memory from scratch once.`,
);
@@ -1554,7 +1584,7 @@ function runMemoryIngest(args: CliArgs, previous: SyncState, consent: Transcript
const ingestArgs = ["run", ingestPath];
if (args.mode === "full") ingestArgs.push("--bulk");
else ingestArgs.push("--incremental");
if (args.mode === "full" && consent.window === "all") ingestArgs.push("--all-history");
if (args.mode === "full" && consent.affirmative && consent.window !== "recent") ingestArgs.push("--all-history");
if (args.quiet) ingestArgs.push("--quiet");
if (selection.sources) ingestArgs.push("--sources", selection.sources.join(","));
if (!args.quiet) console.error(`[sync:memory] sources: ${selected.join(",")} (${selection.why})`);
@@ -1963,7 +1993,7 @@ async function main(): Promise<void> {
const args = parseArgs();
// Read once at sync start: a config change made while the sync runs never
// adds transcripts to this run.
const consent = args.noMemory ? null : normalizeTranscriptConsent();
const consent = args.noMemory ? null : readTranscriptConsent();
if (!args.quiet) {
const engine = detectEngineTier();
+143 -49
View File
@@ -72,7 +72,15 @@ import { execGbrainText, spawnGbrainAsync } from "../lib/gbrain-exec";
import { writeReceipt } from "../lib/egress-receipt";
import { checkOwnedStagingDir, STAGING_MARKER } from "../lib/staging-guard";
import { hasRepoPolicyStore, repoPolicyTierBatch } from "../lib/gbrain-repo-policy-client";
import { resolveStateRoot, readConfigKeyWithRoot, type StateRootEnv } from "../lib/state-root";
import { resolveStateRoot, mergedStateRoots, type StateRootEnv } from "../lib/state-root";
import {
cutoffExclusion,
purgeStagedTranscripts,
readTranscriptConsent,
repoExclusion,
type TranscriptConsent,
type TranscriptPolicy,
} from "../lib/transcript-consent";
// ── Types ──────────────────────────────────────────────────────────────────
@@ -155,6 +163,8 @@ interface ProbeReport {
*/
skipped_policy_deny: number;
skipped_policy_readonly: number;
/** Sessions a scoped consent (new@ cutoff, transcript_repos) kept out. */
scope: TranscriptScopeCounts;
estimate_minutes: number;
}
@@ -174,6 +184,8 @@ interface BulkResult {
failed: number;
duration_ms: number;
partial_pages: number;
/** Sessions a scoped consent kept out, and transcript pages selected. */
scope?: TranscriptScopeCounts;
/**
* D6: when set, indicates a process-level failure (gbrain CLI missing
* or `gbrain import` crashed). Per-file errors (FILE_TOO_LARGE etc.)
@@ -204,43 +216,39 @@ const ALL_TYPES: MemoryType[] = [
// ── Transcript consent ─────────────────────────────────────────────────────
export interface TranscriptConsent {
/** True only for a stored `recent` or `all`. */
affirmative: boolean;
/** Transcript walk window when affirmative: `recent` = last 90 days, `all` = all history. */
window: "recent" | "all" | null;
/**
* Normalized mode: the window when affirmative; otherwise why transcripts
* are skipped. `legacy` covers the old gate letters (A-E) and `incremental`,
* which predate the consent rule and need a new choice.
*/
reason: "recent" | "all" | "off" | "not-set" | "legacy" | "unrecognized";
/** The stored value as read (null when the key is absent). */
value: string | null;
}
export type { TranscriptConsent } from "../lib/transcript-consent";
let cachedConsent: TranscriptConsent | null = null;
let cachedConsent: TranscriptPolicy | null = null;
/**
* Read `transcript_ingest_mode` once per process (same parse as
* `gstack-config has` + `get`) and decide whether transcripts may be ingested.
* Only `recent` and `all` are consent. An absent key, `off`, a legacy value
* and anything unrecognized are not. Pass `env` to bypass the per-process cache.
* Read transcript consent once per process across every merged state root
* (lib/transcript-consent.ts). `recent`, `all` and `new@<time>` are consent;
* an absent key, `off`, a legacy value (A-E, incremental) and anything
* unrecognized are not. A cutoff or allowlist applies even under an explicit
* `--sources transcript`. Pass `env` to bypass the per-process cache.
*/
export function normalizeTranscriptConsent(env?: StateRootEnv): TranscriptConsent {
function readIngestConsent(env?: StateRootEnv): TranscriptPolicy {
if (!env && cachedConsent) return cachedConsent;
const value = readConfigKeyWithRoot("transcript_ingest_mode", env ?? process.env).value;
const v = (value ?? "").trim().toLowerCase();
let consent: TranscriptConsent;
if (value === null) consent = { affirmative: false, window: null, reason: "not-set", value };
else if (v === "recent" || v === "all") consent = { affirmative: true, window: v, reason: v, value };
else if (v === "off") consent = { affirmative: false, window: null, reason: "off", value };
else if (/^[a-e]$/.test(v) || v === "incremental") consent = { affirmative: false, window: null, reason: "legacy", value };
else consent = { affirmative: false, window: null, reason: "unrecognized", value };
if (!env) cachedConsent = consent;
const policy = readTranscriptConsent(mergedStateRoots(env ?? process.env));
if (!env) cachedConsent = policy;
return policy;
}
/** The consent fields without the contributing roots. */
export function normalizeTranscriptConsent(env?: StateRootEnv): TranscriptConsent {
const { roots: _roots, ...consent } = readIngestConsent(env);
return consent;
}
/** Sessions a scoped consent kept out of this run, by reason. */
export interface TranscriptScopeCounts {
pre_cutoff: number;
missing_start: number;
not_allowlisted: number;
/** Transcript pages this run selected for staging. */
sessions_selected: number;
}
// ── CLI ────────────────────────────────────────────────────────────────────
function printUsage(): void {
@@ -258,8 +266,10 @@ Options:
--all-history Walk transcripts older than 90 days too.
--sources <list> Comma-separated subset: ${ALL_TYPES.join(",")}
Default: every type, minus transcript unless
transcript_ingest_mode is recent (90 days) or all
(all history). A list naming transcript overrides it.
transcript_ingest_mode is recent (90 days), all (all
history) or new@<UTC> (sessions started after it).
A list naming transcript overrides it; a new@ cutoff
and transcript_repos still apply.
--limit <N> Stop after N pages written (smoke testing).
--no-write Skip gbrain put calls (still updates state file).
Used by tests + dry runs without actual ingest.
@@ -323,14 +333,15 @@ function parseArgs(): CliArgs {
}
}
const consent = normalizeTranscriptConsent();
const consent = readIngestConsent();
if (!sourcesExplicit && !consent.affirmative) {
sources.delete("transcript");
if (!quiet) {
console.error(`gstack-memory-ingest: transcripts skipped (transcript_ingest_mode=${consent.value ?? "not set"}); set it to recent or all, or pass --sources transcript.`);
const why = consent.reason === "repos-unreadable" ? "; its +repos marker needs a transcript_repos allowlist" : "";
console.error(`gstack-memory-ingest: transcripts skipped (transcript_ingest_mode=${consent.value ?? "not set"}${why}); set it to recent, all or new@<UTC time>, or pass --sources transcript.`);
}
}
if (consent.window === "all") allHistory = true;
if (consent.affirmative && consent.window !== "recent") allHistory = true;
return { mode, quiet, benchmark, includeUnattributed, allHistory, sources, limit, noWrite, scanSecrets };
}
@@ -425,17 +436,45 @@ interface WalkContext {
args: CliArgs;
state: IngestState;
windowStartMs: number; // ignore files older than this unless --all-history
consent: TranscriptConsent;
scope: TranscriptScopeCounts;
}
function makeWalkContext(args: CliArgs, state: IngestState): WalkContext {
export function emptyScopeCounts(): TranscriptScopeCounts {
return { pre_cutoff: 0, missing_start: 0, not_allowlisted: 0, sessions_selected: 0 };
}
function makeWalkContext(args: CliArgs, state: IngestState, scope: TranscriptScopeCounts = emptyScopeCounts()): WalkContext {
const ninetyDaysAgoMs = Date.now() - 90 * 24 * 60 * 60 * 1000;
return {
args,
state,
windowStartMs: args.allHistory ? 0 : ninetyDaysAgoMs,
consent: readIngestConsent(),
scope,
};
}
/**
* The `new@` cutoff, applied in both parsers' walks on each session's first
* record timestamp (read from the file prefix, never mtime): a session that
* started before the cutoff and was appended later stays out, and a session
* with no start timestamp is excluded (fail-closed).
*/
function sessionPassesCutoff(ctx: WalkContext, path: string): boolean {
const why = cutoffExclusion(ctx.consent, ctx.consent.cutoff === undefined ? null : transcriptPrefix(path).startedAt);
if (why === "pre-cutoff") ctx.scope.pre_cutoff++;
else if (why === "missing-start") ctx.scope.missing_start++;
return why === null;
}
/** The transcript_repos allowlist, applied after the attribution gate. */
function sessionRepoAllowed(ctx: WalkContext, remote: string | undefined): boolean {
if (repoExclusion(ctx.consent, remote) === null) return true;
ctx.scope.not_allowlisted++;
return false;
}
function* walkClaudeCodeProjects(ctx: WalkContext): Generator<{ path: string; type: MemoryType }> {
const root = join(HOME, ".claude", "projects");
if (!existsSync(root)) return;
@@ -462,6 +501,7 @@ function* walkClaudeCodeProjects(ctx: WalkContext): Generator<{ path: string; ty
} catch {
continue;
}
if (!sessionPassesCutoff(ctx, fullPath)) continue;
yield { path: fullPath, type: "transcript" };
}
}
@@ -495,6 +535,7 @@ function* walkCodexSessions(ctx: WalkContext): Generator<{ path: string; type: M
}
}
for (const path of recurse(root, 0)) {
if (!sessionPassesCutoff(ctx, path)) continue;
yield { path, type: "transcript" };
}
}
@@ -801,6 +842,12 @@ function dateOnly(ts: string | undefined): string {
}
}
/** A timestamp as ISO-8601 UTC, or "" when absent or unparseable. */
function isoOrEmpty(ts: string | undefined): string {
const ms = ts ? Date.parse(ts) : NaN;
return Number.isFinite(ms) ? new Date(ms).toISOString() : "";
}
export function buildTranscriptPage(path: string, session: ParsedSession): PageRecord {
const remote = resolveGitRemote(session.cwd);
const slug_repo = repoSlug(remote);
@@ -826,6 +873,7 @@ export function buildTranscriptPage(path: string, session: ParsedSession): PageR
`cwd: ${session.cwd || ""}`,
`git_remote: ${remote || "_unattributed"}`,
`start_time: ${session.start_time || ""}`,
`session_started_at: ${isoOrEmpty(session.start_time)}`,
`end_time: ${session.end_time || ""}`,
`message_count: ${session.message_count}`,
`tool_calls: ${session.tool_calls}`,
@@ -1182,7 +1230,7 @@ export function readNewFailures(
/**
* The ONE attribution gate (#2394): a transcript is attributable iff its cwd
* resolves to a git remote. Both probeMode (via transcriptCwdFromPrefix +
* resolves to a git remote. Both probeMode (via transcriptPrefix +
* resolveGitRemote — the same memoized resolver) and preparePages route
* through THIS logic, so the two stages' post-attribution counts are
* structurally identical — the parity the probe report promises.
@@ -1218,10 +1266,15 @@ const TRANSCRIPT_PROBE_MAX_BYTES = 256 * 1024;
* - Claude Code cwd comes from the first record that carries one;
* - unparseable lines are skipped (the truncated-tail case included).
*
* The session start (the `new@` cutoff input) follows the same rules: Codex
* takes the first record's timestamp (or payload.timestamp), Claude Code the
* first record that carries a timestamp. A start beyond the prefix reads as
* missing, which a cutoff treats as excluded.
*
* Non-transcript types (artifacts) always pass — the attribution filter in
* preparePages only applies to transcripts (#2394).
*/
function transcriptCwdFromPrefix(path: string): string {
export function transcriptPrefix(path: string): { cwd: string; startedAt: string } {
// Chunked read until the prefix contains at least one COMPLETE record
// (newline), up to the hard cap — a first record larger than one chunk
// (giant pasted prompt) must not truncate mid-JSON and mis-classify a
@@ -1246,12 +1299,12 @@ function transcriptCwdFromPrefix(path: string): string {
closeSync(fd);
}
} catch {
return "";
return { cwd: "", startedAt: "" };
}
const lines = raw.split("\n").filter((l) => l.trim().length > 0);
if (lines.length === 0) return "";
let cwd = "";
let startedAt = "";
let sawFirstParseable = false;
for (const line of lines) {
let rec: any;
@@ -1265,18 +1318,18 @@ function transcriptCwdFromPrefix(path: string): string {
// Format detection mirrors parseTranscriptJsonl's `first` record check.
const isCodex = rec?.type === "session_meta" || rec?.payload?.id != null;
if (isCodex) {
// Codex: cwd comes from the session_meta FIRST record only.
// Codex: cwd and start come from the session_meta FIRST record only.
cwd = rec.payload?.cwd || rec.cwd || "";
startedAt = rec.timestamp || rec.payload?.timestamp || "";
break;
}
}
// Claude Code: first record with a cwd wins (the first record included).
if (rec?.cwd) {
cwd = rec.cwd;
break;
}
// Claude Code: the first record with a cwd and the first with a timestamp.
if (!cwd && rec?.cwd) cwd = rec.cwd;
if (!startedAt && rec?.timestamp) startedAt = rec.timestamp;
if (cwd && startedAt) break;
}
return cwd;
return { cwd, startedAt: typeof startedAt === "string" ? startedAt : "" };
}
async function probeMode(args: CliArgs): Promise<ProbeReport> {
@@ -1314,12 +1367,13 @@ async function probeMode(args: CliArgs): Promise<ProbeReport> {
// skip transcripts with no resolvable git remote unless --include-unattributed.
let remote = "";
if (type === "transcript") {
const cwd = transcriptCwdFromPrefix(path);
const { cwd } = transcriptPrefix(path);
remote = cwd ? resolveGitRemote(cwd) : "";
if (!args.includeUnattributed && remote === "") {
skippedUnattributed++;
continue;
}
if (!sessionRepoAllowed(ctx, remote || "_unattributed")) continue;
}
candidates.push({ path, type, remote });
}
@@ -1380,6 +1434,7 @@ async function probeMode(args: CliArgs): Promise<ProbeReport> {
skipped_unattributed: skippedUnattributed,
skipped_policy_deny: skippedPolicyDeny,
skipped_policy_readonly: skippedPolicyReadonly,
scope: ctx.scope,
estimate_minutes: estimateMinutes,
};
}
@@ -1544,6 +1599,7 @@ function preparePages(
continue;
}
page = buildTranscriptPage(path, session);
if (!sessionRepoAllowed(ctx, page.git_remote)) continue;
} else {
page = buildArtifactPage(path, type, raw);
}
@@ -1663,6 +1719,7 @@ function preparePages(
// whole batch every run (#2724: 887 staged → 0 ingested). Disambiguate
// before staging, consulting state so assignments hold across runs.
disambiguateSlugs(finalPrepared, state);
ctx.scope.sessions_selected = finalPrepared.filter((p) => p.type === "transcript").length;
// Derived from the FINAL set: partial counts must describe pages that are
// actually eligible and within the limit, not the whole scanned corpus.
@@ -2015,9 +2072,20 @@ function runGbrainImportOnce(
}
async function ingestPass(args: CliArgs): Promise<BulkResult> {
// Tightening a scoped consent removes unpublished staged pages outside it and
// their fingerprints, BEFORE the state loads, so a later widening re-stages.
if (!args.noWrite) {
const purged = purgeStagedTranscripts(GSTACK_HOME, readIngestConsent());
if (purged > 0) console.error(`[memory-ingest] removed ${purged} staged transcript pages outside the new scope`);
}
const scope = emptyScopeCounts();
return { ...(await ingestPassScoped(args, scope)), scope };
}
async function ingestPassScoped(args: CliArgs, scope: TranscriptScopeCounts): Promise<BulkResult> {
const t0 = Date.now();
const state = loadState();
const ctx = makeWalkContext(args, state);
const ctx = makeWalkContext(args, state, scope);
const remoteHttpMode = isRemoteHttpMcpMode();
const resumeDir = process.env.GSTACK_INGEST_RESUME_DIR;
const resuming = !args.noWrite && !remoteHttpMode
@@ -2651,6 +2719,7 @@ function printProbeReport(r: ProbeReport, json: boolean): void {
if (r.skipped_policy_readonly > 0) {
console.log(`Skipped (policy read-only): ${r.skipped_policy_readonly} (remote tier is read-only; transcript ingest writes pages)`);
}
for (const line of scopeLines(r.scope)) console.log(line);
console.log("By type:");
for (const [t, v] of Object.entries(r.by_type)) {
if (v.count > 0) {
@@ -2660,6 +2729,26 @@ function printProbeReport(r: ProbeReport, json: boolean): void {
console.log(`\nEstimate: ~${r.estimate_minutes} min for full --bulk pass.`);
}
/** Exclusion counts by reason; nothing when consent has no cutoff or allowlist. */
function scopeLines(scope: TranscriptScopeCounts | undefined, indent = ""): string[] {
if (!scope) return [];
const lines: string[] = [];
if (scope.pre_cutoff > 0) lines.push(`${indent}Excluded (pre-cutoff): ${scope.pre_cutoff} (session started before the new@ cutoff)`);
if (scope.missing_start > 0) lines.push(`${indent}Excluded (missing start): ${scope.missing_start} (no start timestamp; excluded under new@)`);
if (scope.not_allowlisted > 0) lines.push(`${indent}Excluded (not allowlisted): ${scope.not_allowlisted} (repo not in transcript_repos)`);
return lines;
}
/**
* Under a new@ cutoff, a run that selects no session says so, so "nothing
* happened" reads as expected rather than broken.
*/
function zeroSessionsNotice(r: BulkResult, args: CliArgs): string | null {
const consent = readIngestConsent();
if (!consent.cutoff || !args.sources.has("transcript") || !r.scope || r.scope.sessions_selected > 0) return null;
return `[memory-ingest] 0 transcript sessions ingested under new@${consent.cutoff}: ${r.scope.pre_cutoff} started before the cutoff; new sessions will appear on the next sync`;
}
function printBulkResult(r: BulkResult, args: CliArgs): void {
console.log(`\nIngest pass complete (${args.mode}):`);
console.log(` written: ${r.written}`);
@@ -2673,6 +2762,7 @@ function printBulkResult(r: BulkResult, args: CliArgs): void {
if (r.skipped_policy_deny > 0) {
console.log(` skipped (policy deny): ${r.skipped_policy_deny} (change with: gstack-gbrain-repo-policy set <remote> read-write)`);
}
for (const line of scopeLines(r.scope, " ")) console.log(line);
console.log(` failed: ${r.failed}`);
console.log(` duration: ${(r.duration_ms / 1000).toFixed(1)}s`);
if (args.benchmark) {
@@ -2706,6 +2796,8 @@ async function main(): Promise<void> {
if (result.written > 0 || result.failed > 0) {
console.error(`[memory-ingest] ${result.written} written, ${result.failed} failed in ${dt}ms`);
}
const zero = zeroSessionsNotice(result, args);
if (zero) console.error(zero);
// D6: system_error → process-level failure; orchestrator sees ERR.
// Per-file errors do NOT exit non-zero.
if (result.system_error) process.exit(1);
@@ -2714,6 +2806,8 @@ async function main(): Promise<void> {
const result = await ingestPass(args);
printBulkResult(result, args);
const zero = zeroSessionsNotice(result, args);
if (zero) console.error(zero);
if (result.system_error) process.exit(1);
}
+258
View File
@@ -0,0 +1,258 @@
/**
* Scoped transcript consent in the memory ingest and the sync orchestrator:
* the new@ cutoff on each session's first record (both parsers), the
* transcript_repos allowlist after the attribution gate, exclusion counts by
* reason, explicit overrides that never widen a scoped consent, the consent
* record that restages on a scope change, and the regression contract that
* `recent` selects exactly the sessions it did before (file mtime, 90 days).
* Runs the real ingest and orchestrator against a stub gbrain.
*/
import { describe, expect, test } from 'bun:test';
import { chmodSync, existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, utimesSync, writeFileSync } from 'fs';
import { tmpdir } from 'os';
import { join } from 'path';
import { spawnSync } from 'child_process';
import { transcriptConsentRecord, transcriptConsentSummary } from '../bin/gstack-gbrain-sync';
import { reposHash } from '../lib/transcript-consent';
const ROOT = join(import.meta.dir, '..');
const INGEST = join(ROOT, 'bin', 'gstack-memory-ingest.ts');
const SYNC = join(ROOT, 'bin', 'gstack-gbrain-sync.ts');
const DAY = 24 * 60 * 60 * 1000;
const iso = (ms: number) => new Date(ms).toISOString().replace(/\.\d{3}Z$/, 'Z');
interface Box {
home: string;
gstackHome: string;
env: Record<string, string>;
repo: (slug: string) => string;
claude: (id: string, opts: { cwd: string; start?: string; mtimeMs?: number; appendNow?: boolean }) => string;
codex: (id: string, opts: { cwd: string; start: string }) => string;
staged: () => string[];
cleanup: () => void;
}
function box(config: string): Box {
const home = mkdtempSync(join(tmpdir(), 'gstack-tscope-'));
const gstackHome = join(home, '.gstack');
const bindir = join(home, 'bin');
const stagedList = join(home, 'staged.list');
for (const d of [gstackHome, bindir, join(home, '.gbrain'), join(home, '.claude', 'projects', 'p')]) mkdirSync(d, { recursive: true });
writeFileSync(join(gstackHome, 'config.yaml'), config);
writeFileSync(join(home, '.gbrain', 'config.json'), JSON.stringify({ engine: 'pglite', database_url: 'pglite:///test' }));
writeFileSync(join(bindir, 'gbrain'), `#!/bin/sh
case "$1" in
--version) echo "gbrain 0.60.28.0" ;;
--help) printf 'Commands:\\n import <dir> Import markdown directory\\n' ;;
sources) echo '{"sources":[]}' ;;
import)
( cd "$2" && find . -name '*.md' -type f | sort ) > "${stagedList}"
n=$(wc -l < "${stagedList}" | tr -d ' ')
echo "{\\"status\\":\\"success\\",\\"imported\\":$n,\\"skipped\\":0,\\"errors\\":0,\\"total_files\\":$n}" ;;
*) echo "unexpected gbrain $*" >&2; exit 1 ;;
esac
`);
chmodSync(join(bindir, 'gbrain'), 0o755);
const env = {
HOME: home,
GSTACK_HOME: gstackHome,
GBRAIN_HOME: '',
GSTACK_MEMORY_INGEST_SOURCES: '',
GIT_CONFIG_NOSYSTEM: '1',
GIT_CONFIG_GLOBAL: join(home, '.gitconfig'),
PATH: `${bindir}:${process.env.PATH || ''}`,
};
const line = (rec: Record<string, unknown>) => JSON.stringify(rec) + '\n';
return {
home,
gstackHome,
env,
repo: (slug) => {
const dir = join(home, 'work', slug.replace('/', '-'));
mkdirSync(dir, { recursive: true });
spawnSync('git', ['-C', dir, 'init', '-q'], { timeout: 30_000 });
spawnSync('git', ['-C', dir, 'remote', 'add', 'origin', `https://github.com/${slug}.git`], { timeout: 30_000 });
return dir;
},
claude: (id, { cwd, start, mtimeMs, appendNow }) => {
const path = join(home, '.claude', 'projects', 'p', `${id}.jsonl`);
const ts = start ? { timestamp: start } : {};
writeFileSync(path, line({ type: 'user', message: { role: 'user', content: `hello ${id}` }, cwd, ...ts }) + line({ type: 'assistant', message: { role: 'assistant', content: 'ok' }, ...ts }));
if (appendNow) writeFileSync(path, readFileSync(path, 'utf-8') + line({ type: 'user', message: { role: 'user', content: 'later' }, timestamp: iso(Date.now()) }));
if (mtimeMs !== undefined) utimesSync(path, new Date(mtimeMs), new Date(mtimeMs));
return path;
},
codex: (id, { cwd, start }) => {
const dir = join(home, '.codex', 'sessions', '2026', '10', '01');
mkdirSync(dir, { recursive: true });
const path = join(dir, `rollout-${id}.jsonl`);
writeFileSync(path, line({ type: 'session_meta', timestamp: start, payload: { id, cwd } }) + line({ type: 'response_item', timestamp: start, payload: { type: 'message', role: 'user', content: [{ type: 'input_text', text: `hi ${id}` }] } }));
return path;
},
staged: () => (existsSync(stagedList) ? readFileSync(stagedList, 'utf-8').split('\n').filter((l) => l.includes('transcripts/')) : []),
cleanup: () => rmSync(home, { recursive: true, force: true }),
};
}
const ingest = (b: Box, args: string[], env: Record<string, string> = {}) =>
spawnSync('bun', [INGEST, ...args], { encoding: 'utf-8', timeout: 60_000, cwd: b.home, env: { ...process.env, ...b.env, ...env } });
const sync = (b: Box, args: string[], env: Record<string, string> = {}) =>
spawnSync('bun', [SYNC, '--no-code', '--no-brain-sync', ...args], { encoding: 'utf-8', timeout: 60_000, cwd: b.home, env: { ...process.env, ...b.env, ...env } });
describe.skipIf(process.platform === 'win32')('new@ cutoff', () => {
test('both parsers filter on the first record timestamp; appended pre-cutoff and start-less sessions stay out', () => {
const now = Date.now();
const b = box(`transcript_ingest_mode: new@${iso(now - DAY)}\n`);
try {
const app = b.repo('acme/app');
b.claude('preappend0001', { cwd: app, start: iso(now - 2 * DAY), appendNow: true });
b.claude('postcutoff001', { cwd: app, start: iso(now - 60_000) });
b.claude('nostart000001', { cwd: app });
b.codex('codexold00001', { cwd: app, start: iso(now - 3 * DAY) });
b.codex('codexnew00001', { cwd: app, start: iso(now - 60_000) });
const probe = ingest(b, ['--probe']);
expect(probe.status).toBe(0);
expect(probe.stdout).toContain('Total files in window: 2');
expect(probe.stdout).toMatch(/Excluded \(pre-cutoff\):\s+2/);
expect(probe.stdout).toMatch(/Excluded \(missing start\):\s+1/);
const bulk = ingest(b, ['--bulk', '--quiet']);
expect(bulk.status).toBe(0);
const staged = b.staged().join('\n');
expect(staged).toMatch(/postcutoff00/);
expect(staged).toMatch(/codexnew0000/);
expect(staged).not.toMatch(/preappend000|nostart00000|codexold0000/);
const page = readFileSync(join(b.gstackHome, '.transcript-ingest-state.json'), 'utf-8');
expect(page).not.toContain('preappend0001');
} finally {
b.cleanup();
}
});
test('a run that ingests no session says so, with the pre-cutoff count', () => {
const now = Date.now();
const b = box(`transcript_ingest_mode: new@${iso(now - DAY)}\n`);
try {
b.claude('preappend0001', { cwd: b.repo('acme/app'), start: iso(now - 2 * DAY), appendNow: true });
const r = ingest(b, ['--bulk', '--quiet']);
expect(r.status).toBe(0);
expect(r.stderr).toContain(`[memory-ingest] 0 transcript sessions ingested under new@${iso(now - DAY)}: 1 started before the cutoff; new sessions will appear on the next sync`);
} finally {
b.cleanup();
}
});
});
describe.skipIf(process.platform === 'win32')('transcript_repos allowlist', () => {
test('only allowlisted repos ingest, after the attribution gate, with the exclusion counted', () => {
const b = box('transcript_ingest_mode: all+repos\ntranscript_repos: github.com/acme/app\n');
try {
b.claude('appsession001', { cwd: b.repo('acme/app'), start: iso(Date.now()) });
b.claude('websession001', { cwd: b.repo('acme/web'), start: iso(Date.now()) });
b.codex('codexweb00001', { cwd: join(b.home, 'work', 'acme-web'), start: iso(Date.now()) });
const probe = ingest(b, ['--probe']);
expect(probe.stdout).toContain('Total files in window: 1');
expect(probe.stdout).toMatch(/Excluded \(not allowlisted\):\s+2/);
ingest(b, ['--bulk', '--quiet']);
expect(b.staged().join('\n')).toMatch(/appsession00/);
expect(b.staged().join('\n')).not.toMatch(/websession00|codexweb0000/);
} finally {
b.cleanup();
}
});
});
describe.skipIf(process.platform === 'win32')('explicit overrides never widen a scoped consent', () => {
for (const [args, env, why] of [
[['--sources', 'transcript,learning'], {}, '--sources'],
[[], { GSTACK_MEMORY_INGEST_SOURCES: 'transcript' }, 'GSTACK_MEMORY_INGEST_SOURCES'],
] as const) {
test(`${why}: the cutoff still applies under consent, and a broken allowlist still denies`, () => {
const now = Date.now();
const b = box(`transcript_ingest_mode: new@${iso(now - DAY)}\n`);
try {
const app = b.repo('acme/app');
b.claude('preappend0001', { cwd: app, start: iso(now - 2 * DAY), appendNow: true });
b.claude('postcutoff001', { cwd: app, start: iso(now - 60_000) });
const scoped = sync(b, ['--incremental', '--quiet', ...args], { ...env });
expect(scoped.status).toBe(0);
expect(b.staged().join('\n')).toMatch(/postcutoff00/);
expect(b.staged().join('\n')).not.toMatch(/preappend000/);
writeFileSync(join(b.gstackHome, 'config.yaml'), 'transcript_ingest_mode: recent+repos\n');
rmSync(join(b.gstackHome, '.transcript-ingest-state.json'), { force: true });
rmSync(join(b.home, 'staged.list'), { force: true });
const denied = sync(b, ['--incremental', '--quiet', ...args], { ...env });
expect(denied.status).toBe(0);
expect(b.staged()).toEqual([]);
expect(denied.stderr).toContain(`gbrain-sync: transcripts ingested because ${why} names transcript (transcript_ingest_mode=recent+repos). The new@ cutoff and transcript_repos allowlist still apply.`);
} finally {
b.cleanup();
}
});
}
});
describe.skipIf(process.platform === 'win32')('consent record and notice', () => {
test('the stage record carries the cutoff and allowlist hash; unscoped records are unchanged', () => {
const cutoff = '2026-10-03T17:00:00Z';
expect(transcriptConsentRecord({ affirmative: true, window: 'recent', reason: 'recent', value: 'recent' }, false)).toEqual({ mode: 'recent', window: 'recent' });
expect(transcriptConsentRecord({ affirmative: true, window: 'new', reason: 'new', value: `new@${cutoff}+repos`, cutoff, repos: ['github.com/a/b'] }, false))
.toEqual({ mode: 'new', window: 'new', cutoff, repos_hash: reposHash(['github.com/a/b']) });
});
test('the consent notice prints the effective consent in words, unless --quiet', () => {
const p = { affirmative: true, window: 'recent' as const, reason: 'recent' as const, value: 'recent+repos', repos: ['github.com/a/b'], roots: ['/s'] };
expect(transcriptConsentSummary(p, false)).toBe('gbrain-sync: transcripts: sessions from the last 90 days; repo github.com/a/b; config: /s');
expect(transcriptConsentSummary(p, true)).toBeNull();
expect(transcriptConsentSummary({ ...p, affirmative: false, reason: 'off' }, false)).toBeNull();
});
test('a changed cutoff restages an interrupted import', () => {
const now = Date.now();
const b = box(`transcript_ingest_mode: new@${iso(now - DAY)}\n`);
try {
const stagingDir = join(b.gstackHome, '.staging-ingest-1-1');
mkdirSync(join(stagingDir, 'projects'), { recursive: true });
writeFileSync(join(stagingDir, '.gstack-staging'), '');
writeFileSync(join(stagingDir, 'projects', 'stale-page.md'), '# stale\n');
writeFileSync(join(b.home, '.gbrain', 'import-checkpoint.json'), JSON.stringify({ dir: stagingDir, processedIndex: 0, totalFiles: 1 }));
const { MEMORY_INGEST_TYPES } = require('../bin/gstack-gbrain-sync');
writeFileSync(join(b.gstackHome, '.gbrain-sync-state.json'), JSON.stringify({
schema_version: 1,
last_writer: 'gstack-gbrain-sync',
last_stages: [{ name: 'memory', ran: true, ok: false, duration_ms: 1, summary: 'timeout', memory_sources: [...MEMORY_INGEST_TYPES], transcript_consent: { mode: 'new', window: 'new', cutoff: iso(now - 2 * DAY) } }],
}));
const r = sync(b, ['--incremental', '--quiet']);
expect(r.stderr).toContain('gbrain-sync: transcript consent changed since the interrupted import');
expect(r.stderr).not.toContain('resuming from gbrain checkpoint');
} finally {
b.cleanup();
}
});
});
describe.skipIf(process.platform === 'win32')('regression contract: recent selects the same sessions as before', () => {
test('recent keeps the 90-day file-mtime window, whatever the session start', () => {
const now = Date.now();
const b = box('transcript_ingest_mode: recent\n');
try {
const app = b.repo('acme/app');
b.claude('mtimenowold1', { cwd: app, start: iso(now - 200 * DAY) });
b.claude('mtimeoldnew1', { cwd: app, start: iso(now), mtimeMs: now - 200 * DAY });
b.claude('bothrecent01', { cwd: app, start: iso(now - DAY) });
b.claude('botholdxxxx1', { cwd: app, start: iso(now - 200 * DAY), mtimeMs: now - 200 * DAY });
b.claude('nostampsxxx1', { cwd: app });
const r = ingest(b, ['--bulk', '--quiet']);
expect(r.status).toBe(0);
const ids = b.staged().map((l) => l.replace(/^.*-([a-z0-9]{12})\.md$/, '$1')).sort();
expect(ids).toEqual(['bothrecent01', 'mtimenowold1', 'nostampsxxx1']);
const probe = ingest(b, ['--probe']);
expect(probe.stdout).not.toContain('Excluded');
} finally {
b.cleanup();
}
});
});