fix(memory-ingest): --probe counts post-attribution, through the same gate --bulk uses

probeMode previously stat'd every walked file, so setup-gbrain gated its
silent bulk ingest on pre-filter counts that the write path would never
ingest (#2394). The attribution decision now lives in ONE shared gate
(sessionIsAttributable — cheap-parse: cwd extraction + memoized
resolveGitRemote, never a full page build) used by BOTH probeMode and
preparePages, so the two stages' post-attribution counts are structurally
identical. ProbeReport gains skipped_unattributed; the probe prints what it
excluded and --include-unattributed restores raw counts. The parity is
pinned at the prepare stage (probe post-attribution == transcripts reaching
import), deliberately NOT == final written.

Re-derived from PR #2612 under the generated-file screening rule; the
shared-gate design and the remote memo are additions from the plan review.

Fixes #2394.
Contributed by @Lockyer228

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
Garry Tan
2026-08-17 10:15:53 -07:00
co-authored by Claude Fable 5
parent f9f0e84add
commit 0762fab809
2 changed files with 169 additions and 9 deletions
+87 -5
View File
@@ -141,6 +141,7 @@ interface ProbeReport {
new_count: number;
updated_count: number;
unchanged_count: number;
skipped_unattributed: number;
estimate_minutes: number;
}
@@ -677,8 +678,21 @@ function extractContentText(rec: any): string {
return "";
}
// Memo: probe and prepare both resolve remotes per-transcript, and transcripts
// share a small set of cwds — without this an 11.7K-file probe would spawn git
// 11.7K times instead of once per distinct cwd.
const REMOTE_MEMO = new Map<string, string>();
function resolveGitRemote(cwd: string): string {
if (!cwd) return "";
const memo = REMOTE_MEMO.get(cwd);
if (memo !== undefined) return memo;
const resolved = resolveGitRemoteUncached(cwd);
REMOTE_MEMO.set(cwd, resolved);
return resolved;
}
function resolveGitRemoteUncached(cwd: string): string {
try {
// execFileSync (no shell) so `cwd` cannot trigger command substitution.
// Transcript JSONL records are an untrusted surface (a poisoned `.cwd`
@@ -1046,6 +1060,60 @@ export function readNewFailures(
// ── Main ingest passes ─────────────────────────────────────────────────────
/**
* Lightweight attribution check: does a transcript have a resolvable git
* remote for its cwd? Extracts the cwd from the first JSONL line that has
* one (mirroring the logic in parseTranscriptJsonl) and calls
* resolveGitRemote. Avoids the full parse (body rendering, message counting)
* because probe only needs the yes/no answer.
*
* Non-transcript types (artifacts) always pass — the attribution filter in
* preparePages only applies to transcripts (#2394).
*/
/**
* The ONE attribution gate (#2394): a transcript is attributable iff its cwd
* resolves to a git remote. Both probeMode (via transcriptIsAttributable) and
* preparePages route through THIS function, so the two stages' post-attribution
* counts are structurally identical — the parity the probe report promises.
*/
function sessionIsAttributable(cwd: string | undefined | null): boolean {
if (!cwd) return false;
return resolveGitRemote(cwd) !== "";
}
function transcriptIsAttributable(path: string): boolean {
let raw: string;
try {
raw = readFileSync(path, "utf-8");
} catch {
return false;
}
const lines = raw.split("\n").filter((l) => l.trim().length > 0);
if (lines.length === 0) return false;
// Detect format: Codex first line has type=session_meta, Claude Code
// has cwd on a user/assistant record.
let cwd = "";
for (const line of lines) {
try {
const rec = JSON.parse(line);
if (rec?.type === "session_meta") {
cwd = rec.payload?.cwd || rec.cwd || "";
break;
}
if (rec?.cwd) {
cwd = rec.cwd;
break;
}
} catch {
continue;
}
}
if (!cwd) return false;
return sessionIsAttributable(cwd);
}
async function probeMode(args: CliArgs): Promise<ProbeReport> {
const state = loadState();
const ctx = makeWalkContext(args, state);
@@ -1066,8 +1134,18 @@ async function probeMode(args: CliArgs): Promise<ProbeReport> {
let newCount = 0;
let updatedCount = 0;
let unchangedCount = 0;
let skippedUnattributed = 0;
for (const { path, type } of walkAllSources(ctx)) {
// Apply the same attribution filter preparePages uses (#2394):
// skip transcripts with no resolvable git remote unless --include-unattributed.
if (type === "transcript" && !args.includeUnattributed) {
if (!transcriptIsAttributable(path)) {
skippedUnattributed++;
continue;
}
}
totalFiles++;
let size = 0;
try {
@@ -1096,6 +1174,7 @@ async function probeMode(args: CliArgs): Promise<ProbeReport> {
new_count: newCount,
updated_count: updatedCount,
unchanged_count: unchangedCount,
skipped_unattributed: skippedUnattributed,
estimate_minutes: estimateMinutes,
};
}
@@ -1176,15 +1255,15 @@ function preparePages(
parseFailed++;
continue;
}
if (!args.includeUnattributed && !session.cwd) {
// The SAME gate probeMode uses (#2394) — routing both through
// sessionIsAttributable is what makes probe counts trustworthy.
// (Semantically identical to the old two-step check: no cwd, or a cwd
// whose remote resolves empty, both rendered git_remote "_unattributed".)
if (!args.includeUnattributed && !sessionIsAttributable(session.cwd)) {
skippedUnattributed++;
continue;
}
page = buildTranscriptPage(path, session);
if (!args.includeUnattributed && page.git_remote === "_unattributed") {
skippedUnattributed++;
continue;
}
if (page.partial) partialPages++;
} else {
page = buildArtifactPage(path, type);
@@ -2042,6 +2121,9 @@ function printProbeReport(r: ProbeReport, json: boolean): void {
console.log(`New (never ingested): ${r.new_count}`);
console.log(`Updated (mtime/hash): ${r.updated_count}`);
console.log(`Unchanged: ${r.unchanged_count}`);
if (r.skipped_unattributed > 0) {
console.log(`Skipped (unattributed): ${r.skipped_unattributed} (no git remote; use --include-unattributed to include)`);
}
console.log("By type:");
for (const [t, v] of Object.entries(r.by_type)) {
if (v.count > 0) {