diff --git a/bin/gstack-brain-sync b/bin/gstack-brain-sync index 3c64634c0..c701f08c6 100755 --- a/bin/gstack-brain-sync +++ b/bin/gstack-brain-sync @@ -393,15 +393,38 @@ for p in final: PYEOF } -# Transcript consent: transcript_ingest_mode is recent or all (trimmed, any -# case). An absent key, off, a legacy value or anything else is not consent. -# Same rule as normalizeTranscriptConsent in gstack-memory-ingest.ts. +# Transcript consent comes from the one shared policy reader +# (lib/transcript-consent.ts): every merged state root, the new@ cutoff and the +# transcript_repos allowlist. Without bun nothing transcript-related can be +# checked, so transcript pages are held (fail-closed). +CONSENT_TS="$SCRIPT_DIR/../lib/transcript-consent.ts" +transcript_policy() { + command -v bun >/dev/null 2>&1 || return 2 + GSTACK_STATE_ROOT="$GSTACK_HOME" bun run "$CONSENT_TS" "$@" +} + +# 0 when staged transcript pages may sync at all. transcripts_consented() { - "$CONFIG_BIN" has transcript_ingest_mode 2>/dev/null || return 1 - local v - v=$("$CONFIG_BIN" get transcript_ingest_mode 2>/dev/null || true) - v=$(printf '%s' "$v" | tr -d '[:space:]' | tr '[:upper:]' '[:lower:]') - [ "$v" = "recent" ] || [ "$v" = "all" ] + transcript_policy --check 2>/dev/null +} + +# Before every push: scan the unpublished range for transcript pages outside +# the current consent; when some exist, the helper rewrites gstack-brain-sync's +# own unpublished commits without them, or refuses (exit 3) when it cannot. +# Returns 0 when the push may go ahead. A range that touches no transcript +# page needs no scan (and no bun). +guard_unpublished_range() { + local branch range="HEAD" rc=0 + branch=$(git -C "$GSTACK_HOME" rev-parse --abbrev-ref HEAD 2>/dev/null || echo "") + if [ -n "$branch" ] && [ "$branch" != "HEAD" ] && git -C "$GSTACK_HOME" rev-parse --verify --quiet "origin/$branch" >/dev/null 2>&1; then + range="origin/$branch..HEAD" + fi + [ -n "$(git -C "$GSTACK_HOME" log --format= --name-only "$range" -- transcripts 2>/dev/null | head -1)" ] || return 0 + transcript_policy --guard-unpublished || rc=$? + if [ "$rc" -eq 2 ]; then + echo "BRAIN_SYNC: push blocked: unpushed commits hold transcript pages and bun is not on PATH to check them against your transcript consent" >&2 + fi + [ "$rc" -eq 0 ] } # Finalize the drain: delete exactly the spool record files this drain @@ -637,7 +660,7 @@ subcmd_once() { det_now=$(date +%s) det_last=$(cat "$GSTACK_HOME/.brain-last-push-attempt" 2>/dev/null || echo 0) case "$det_last" in ''|*[!0-9]*) det_last=0 ;; esac - if [ "$det_unpushed" -gt 0 ] && [ "$det_unpushed" -eq "$det_total" ] && [ $(( det_now - det_last )) -ge 600 ]; then + if [ "$det_unpushed" -gt 0 ] && [ "$det_unpushed" -eq "$det_total" ] && [ $(( det_now - det_last )) -ge 600 ] && guard_unpublished_range; then echo "$det_now" > "$GSTACK_HOME/.brain-last-push-attempt" 2>/dev/null || true local det_host det_host=$(remote_host) @@ -676,6 +699,9 @@ subcmd_once() { # snapshot manifest but no classification. Finalizing on that state is what # used to wipe the queue — so on a nonzero exit, warn loudly, do NOT call # finalize_queue, and leave everything queued for the next drain. + # Tightened consent (new@ cutoff, narrower transcript_repos): remove the + # unpublished staged pages it now excludes before anything is staged. + [ -d "$GSTACK_HOME/transcripts" ] && { transcript_policy --purge || true; } local transcripts_ok=0 transcripts_consented && transcripts_ok=1 if ! compute_paths_to_stage "$mode" "$class_file" "$snapshot_file" "$transcripts_ok" > "$paths_file"; then @@ -743,6 +769,8 @@ subcmd_once() { exit 0 } + # Nothing outside the transcript consent is pushed; a refused rewrite keeps the commit local. + guard_unpublished_range || { write_status "blocked" "push blocked: unpushed commits hold transcript pages outside your transcript consent; commit retained locally"; finalize_queue "$snapshot_file" "$class_file" "$paths_file"; exit 0; } # Push. On reject, fetch + merge (merge driver handles JSONL) + retry once. local push_err push_err=$(git -C "$GSTACK_HOME" push origin HEAD 2>&1 >/dev/null) || { @@ -766,7 +794,7 @@ subcmd_once() { bash -c 'git -C "$1" fetch origin 2>/dev/null' _ "$GSTACK_HOME"; then local branch branch=$(git -C "$GSTACK_HOME" rev-parse --abbrev-ref HEAD 2>/dev/null || echo main) - if git -C "$GSTACK_HOME" merge --no-edit "origin/$branch" >/dev/null 2>&1; then + if git -C "$GSTACK_HOME" merge --no-edit "origin/$branch" >/dev/null 2>&1 && guard_unpublished_range; then if GSTACK_HOME="$GSTACK_HOME" _receipted_git closed brain-sync "$push_host" curated-memory-git-push "artifacts_sync_mode!=off" \ bash -c 'git -C "$1" push origin HEAD 2>/dev/null' _ "$GSTACK_HOME"; then finalize_queue "$snapshot_file" "$class_file" "$paths_file" diff --git a/bin/gstack-config b/bin/gstack-config index a34bf5e26..7b2706890 100755 --- a/bin/gstack-config +++ b/bin/gstack-config @@ -7,6 +7,7 @@ # config file (get returns DEFAULTS for absent keys, # so callers that need provenance use this instead) # gstack-config set — write a config value +# gstack-config unset — remove a key from the config file # gstack-config list — show all config (values + defaults) # gstack-config defaults — show just the defaults table # @@ -92,17 +93,24 @@ CONFIG_HEADER='# gstack configuration — edit freely, changes take effect on ne # # Flip back to false to be re-prompted. # # ─── Transcript ingest (gbrain memory) ─────────────────────────────── -# transcript_ingest_mode: off # recent | all | off — consent for /sync-gbrain -# # to ingest Claude Code and Codex session -# # transcripts (every project repo policy allows) -# # into your brain (local or Supabase). +# transcript_ingest_mode: off # recent | all | off | new@ — consent for +# # /sync-gbrain to ingest Claude Code and Codex +# # session transcripts (every project repo policy +# # allows) into your brain (local or Supabase). # # recent — last 90 days # # all — all history +# # new@2026-10-03T17:00:00Z — only sessions that +# # start after that UTC time # # off — never (other memory still syncs) +# # A +repos suffix is kept automatically while +# # transcript_repos is set. # # Absent, legacy (A-E, incremental) or unknown # # values are not consent: transcripts are skipped # # and /sync-gbrain asks once. An invalid value is # # REJECTED and the stored value kept. +# transcript_repos: # comma-separated git remotes; only transcripts +# # from these repos are ingested. Clear it with +# # `gstack-config unset transcript_repos`. # # Docs: setup-gbrain/memory.md#transcripts # # ─── Timeline Stop hook ────────────────────────────────────────────── @@ -220,7 +228,8 @@ lookup_default() { # the callers already assume in their own `|| echo ""` fallback. question_tuning) echo "false" ;; team_mode) echo "false" ;; - transcript_ingest_mode) echo "off" ;; # recent | all | off — transcript consent; absent = not consented (use `has`) + transcript_ingest_mode) echo "off" ;; # recent | all | off | new@ [+repos] — transcript consent; absent = not consented (use `has`) + transcript_repos) echo "" ;; # empty = every repo; comma-separated canonical remotes narrow transcript ingest # repo_mode: EMPTY is load-bearing — gstack-repo-mode treats any non-empty # answer as a user override and skips its own classification entirely, so # a synthesized "unknown" default turns the classifier into dead code. @@ -348,14 +357,16 @@ resolve_user_slug() { _slug="anonymous-$(sha8_of "$(hostname 2>/dev/null || echo unknown)")" fi - # Persist via direct file write (avoid recursion into gstack-config set) - mkdir -p "$STATE_DIR" - if [ ! -f "$CONFIG_FILE" ]; then - printf '%s' "$CONFIG_HEADER" > "$CONFIG_FILE" - fi - if ! grep -qE "^user_slug_at_${_hash}:" "$CONFIG_FILE" 2>/dev/null; then - echo "user_slug_at_${_hash}: ${_slug}" >> "$CONFIG_FILE" + # Persist via the locked read-modify-write (no recursion into `set`). + config_lock + config_edit_begin + if [ -z "$(_cfg_get "user_slug_at_${_hash}")" ]; then + _cfg_put "user_slug_at_${_hash}" "$_slug" + config_edit_commit + else + rm -f "$_cfg_tmp" fi + config_unlock printf '%s' "$_slug" } @@ -366,6 +377,142 @@ read_config_value() { gstack_read_config_key "$1" } +# ── Config writes: one lock, one read-modify-write rename ────────────────── +# Every mutation of config.yaml (set, unset, the transcript_repos pair, the +# user-slug writer) holds an atomic mkdir lock and replaces the file with one +# rename, so concurrent writers never lose each other's keys and an +# interrupted write leaves the old file. The pid file holds " "; +# a lock whose holder is dead or older than 10 s is taken over. No flock(1): +# stock macOS does not ship it. +CONFIG_LOCK="$STATE_DIR/.config-yaml.lock" +_config_locked="" + +config_lock() { + mkdir -p "$STATE_DIR" + local waited=0 held pid ts stale + while ! mkdir "$CONFIG_LOCK" 2>/dev/null; do + if [ ! -w "$STATE_DIR" ]; then + echo "Error: cannot write to $STATE_DIR (its config lock could not be created). Existing value left unchanged." >&2 + exit 1 + fi + waited=$(( waited + 1 )) + if [ "$waited" -gt 400 ]; then + echo "Error: $CONFIG_FILE is locked by another gstack-config ($(cat "$CONFIG_LOCK/pid" 2>/dev/null || echo 'unknown holder')). Existing value left unchanged. fix: retry, or remove $CONFIG_LOCK if no gstack-config is running." >&2 + exit 1 + fi + held=$(cat "$CONFIG_LOCK/pid" 2>/dev/null || true) + pid=""; ts=""; stale="" + case "$held" in *" "*) pid="${held%% *}"; ts="${held#* }"; ts="${ts%%[!0-9]*}" ;; esac + if [ -n "$pid" ]; then + if ! kill -0 "$pid" 2>/dev/null; then stale=1 + elif [ -n "$ts" ] && [ $(( $(date +%s) - ts )) -gt 10 ]; then stale=1; fi + elif [ "$waited" -gt 200 ]; then + stale=1 + fi + # Take over only the lock we judged: a holder that changed meanwhile is live. + if [ -n "$stale" ] && [ "$(cat "$CONFIG_LOCK/pid" 2>/dev/null || true)" = "$held" ]; then + rm -rf "$CONFIG_LOCK" 2>/dev/null || true + continue + fi + sleep 0.05 + done + printf '%s %s\n' "$$" "$(date +%s)" > "$CONFIG_LOCK/pid" + _config_locked=1 + trap 'config_unlock' EXIT +} + +config_unlock() { + [ -n "$_config_locked" ] || return 0 + rm -rf "$CONFIG_LOCK" 2>/dev/null || true + _config_locked="" +} + +# config_edit_begin — copy config.yaml (or the annotated header for a new +# file) to a temp file beside it; edits go there until config_edit_commit. +config_edit_begin() { + _cfg_tmp="$(mktemp "$STATE_DIR/.config.yaml.XXXXXX")" + if [ -f "$CONFIG_FILE" ]; then cat "$CONFIG_FILE" > "$_cfg_tmp"; else printf '%s' "$CONFIG_HEADER" > "$_cfg_tmp"; fi +} + +config_edit_commit() { + mv "$_cfg_tmp" "$CONFIG_FILE" +} + +# _cfg_get KEY — last value of KEY in the temp file ("" when absent). +_cfg_get() { + awk -v k="$1" 'BEGIN { n = length(k) + 1 } substr($0, 1, n) == k ":" { v = substr($0, n + 1) } END { sub(/^[ \t]+/, "", v); sub(/[ \t\r]+$/, "", v); print v }' "$_cfg_tmp" +} + +# _cfg_put KEY VALUE — replace every KEY line in the temp file, or append one. +# The value travels through ENVIRON so awk applies no escape processing. +_cfg_put() { + _CFG_V="$2" awk -v k="$1" 'BEGIN { n = length(k) + 1; v = ENVIRON["_CFG_V"] } substr($0, 1, n) == k ":" { print k ": " v; done = 1; next } { print } END { if (!done) print k ": " v }' "$_cfg_tmp" > "$_cfg_tmp.n" + mv "$_cfg_tmp.n" "$_cfg_tmp" +} + +_cfg_del() { + awk -v k="$1" 'BEGIN { n = length(k) + 1 } substr($0, 1, n) != k ":"' "$_cfg_tmp" > "$_cfg_tmp.n" + mv "$_cfg_tmp.n" "$_cfg_tmp" +} + +# ── Transcript consent grammar ───────────────────────────────────────────── +# transcript_ingest_mode = [@][+repos]; base is +# recent, all, off or new. `@` is required for new and invalid otherwise; off +# never carries +repos. `set` takes the canonical lowercase form (readers in +# lib/transcript-consent.ts are case-insensitive). +valid_utc_second() { + [[ "$1" =~ ^([0-9]{4})-([0-9]{2})-([0-9]{2})T([0-9]{2}):([0-9]{2}):([0-9]{2})Z$ ]] || return 1 + local y=$((10#${BASH_REMATCH[1]})) m=$((10#${BASH_REMATCH[2]})) d=$((10#${BASH_REMATCH[3]})) + local hh=$((10#${BASH_REMATCH[4]})) mm=$((10#${BASH_REMATCH[5]})) ss=$((10#${BASH_REMATCH[6]})) dim=31 + [ "$m" -ge 1 ] && [ "$m" -le 12 ] && [ "$hh" -le 23 ] && [ "$mm" -le 59 ] && [ "$ss" -le 59 ] || return 1 + case "$m" in + 4|6|9|11) dim=30 ;; + 2) dim=28; if [ $((y % 4)) -eq 0 ] && { [ $((y % 100)) -ne 0 ] || [ $((y % 400)) -eq 0 ]; }; then dim=29; fi ;; + esac + [ "$d" -ge 1 ] && [ "$d" -le "$dim" ] +} + +# valid_transcript_mode VALUE — canonical grammar check (exit 0 = valid). +valid_transcript_mode() { + local v="$1" base + base="${v%+repos}" + case "$base" in + recent|all) ;; + off) [ "$base" = "$v" ] || return 1 ;; + new@*) valid_utc_second "${base#new@}" || return 1 ;; + *) return 1 ;; + esac +} + +# canonical_remote URL — same rules as canonicalizeRemote in +# lib/gstack-memory-helpers.ts (test/transcript-consent.test.ts keeps them equal). +canonical_remote() { + local s="$1" + s="${s#"${s%%[![:space:]]*}"}"; s="${s%"${s##*[![:space:]]}"}" + s="${s#[\"\']}"; s="${s%[\"\']}" + if [[ "$s" =~ ^[^@[:space:]]+@([^:]+):(.+)$ ]]; then + s="${BASH_REMATCH[1]}/${BASH_REMATCH[2]}" + else + s=$(printf '%s' "$s" | sed -E 's#^[a-zA-Z][a-zA-Z0-9+.-]*://##; s#^[^@/]+@##') + fi + printf '%s' "$s" | sed -E 's#/+$##; s#\.[gG][iI][tT]$##; s#/+$##; s#/{2,}#/#g' | tr '[:upper:]' '[:lower:]' +} + +# transcript_marker_sync — keep the +repos marker on transcript_ingest_mode in +# step with transcript_repos inside the temp file: present while an allowlist +# exists and the base consents, absent otherwise. Unrecognized values are +# left untouched (they are not consent either way). +transcript_marker_sync() { + local mode list base + mode=$(_cfg_get transcript_ingest_mode) + [ -n "$mode" ] || return 0 + valid_transcript_mode "$mode" || return 0 + list=$(_cfg_get transcript_repos) + base="${mode%+repos}" + if [ -n "$list" ] && [ "$base" != "off" ]; then _cfg_put transcript_ingest_mode "$base+repos" + else _cfg_put transcript_ingest_mode "$base"; fi +} + _STATE_DOC="https://github.com/garrytan/gstack/blob/main/docs/state-root.md" _SELF="$(cd "$(dirname "$0")" && pwd)/gstack-config" @@ -509,12 +656,31 @@ case "${1:-}" in echo "Error: cross_project_learnings '$VALUE' not recognized. Valid values: true, false. Existing value left unchanged." >&2 exit 1 fi - # transcript_ingest_mode is a CONSENT key: recent/all let /sync-gbrain + # transcript_ingest_mode is a CONSENT key: recent/all/new@ let /sync-gbrain # ingest coding-agent transcripts. Reject a typo; never coerce or store it. - if [ "$KEY" = "transcript_ingest_mode" ] && [ "$VALUE" != "recent" ] && [ "$VALUE" != "all" ] && [ "$VALUE" != "off" ]; then + if [ "$KEY" = "transcript_ingest_mode" ] && ! valid_transcript_mode "$VALUE"; then echo "Error: transcript_ingest_mode '$VALUE' not recognized. Valid values: recent, all, off. Existing value left unchanged." >&2 + echo "Also valid: new@ (only sessions that start after that UTC time, e.g. new@$(date -u +%Y-%m-%dT%H:%M:%SZ)). The +repos suffix follows transcript_repos; off never carries it. Docs: setup-gbrain/memory.md#transcripts" >&2 exit 1 fi + # transcript_repos narrows transcript consent to these repos. Each entry is + # stored as its canonical remote (host/owner/repo); an empty list would + # deny every repo, so it is rejected: `unset` clears the allowlist. + if [ "$KEY" = "transcript_repos" ]; then + _clean="" + _ifs="$IFS"; IFS=',' + for _x in $VALUE; do + _x="$(canonical_remote "$_x")" + [ -n "$_x" ] || continue + case ",$_clean," in *",$_x,"*) ;; *) _clean="${_clean:+$_clean,}$_x" ;; esac + done + IFS="$_ifs" + if [ -z "$_clean" ]; then + echo "Error: transcript_repos '$VALUE' names no repo. Valid values: comma-separated git remotes (e.g. github.com/owner/repo). Existing value left unchanged. To allow every repo: gstack-config unset transcript_repos" >&2 + exit 1 + fi + VALUE="$_clean" + fi # memorable_recall is a CONSENT key: `on` lets a Claude Code hook hand every # prompt to a third-party binary. Reject like codex_reviews -- a typo must # never flip consent in either direction, so nothing is coerced or stored. @@ -581,21 +747,23 @@ case "${1:-}" in done done fi - mkdir -p "$STATE_DIR" - # Write annotated header on first creation - if [ ! -f "$CONFIG_FILE" ]; then - printf '%s' "$CONFIG_HEADER" > "$CONFIG_FILE" + # Locked read-modify-write; the annotated header is written on first creation. + config_lock + config_edit_begin + if [ "$KEY" = "transcript_ingest_mode" ] && [ "${VALUE%+repos}" != "$VALUE" ] && [ -z "$(_cfg_get transcript_repos)" ]; then + rm -f "$_cfg_tmp" + echo "Error: transcript_ingest_mode '$VALUE' needs a repo allowlist, but transcript_repos is not set. Existing value left unchanged. fix: gstack-config set transcript_repos , then set transcript_ingest_mode ${VALUE%+repos}" >&2 + exit 1 fi - # Escape sed replacement metacharacters (newlines were rejected above). + _cfg_put "$KEY" "$VALUE" + # The allowlist and its +repos marker change together, in this one rename. + if [ "$KEY" = "transcript_ingest_mode" ] || [ "$KEY" = "transcript_repos" ]; then + transcript_marker_sync + if [ "$KEY" = "transcript_ingest_mode" ]; then VALUE="$(_cfg_get transcript_ingest_mode)"; fi + fi + config_edit_commit + config_unlock SAFE_VALUE="$VALUE" - ESC_VALUE="$(printf '%s' "$SAFE_VALUE" | sed 's/[&/\]/\\&/g')" - if grep -qE "^${KEY}:" "$CONFIG_FILE" 2>/dev/null; then - # Portable in-place edit (BSD sed uses -i '', GNU sed uses -i without arg) - _tmpfile="$(mktemp "${CONFIG_FILE}.XXXXXX")" - sed "/^${KEY}:/s/.*/${KEY}: ${ESC_VALUE}/" "$CONFIG_FILE" > "$_tmpfile" && mv "$_tmpfile" "$CONFIG_FILE" - else - echo "${KEY}: ${SAFE_VALUE}" >> "$CONFIG_FILE" - fi # Report only when another root still overrides the value just written. gstack_config_select "$KEY" if [ -n "$_gstack_cfg_root" ] && [ "$_gstack_cfg_root" != "$STATE_DIR" ] && [ "$_gstack_cfg_value" != "$SAFE_VALUE" ]; then @@ -616,6 +784,21 @@ case "${1:-}" in fi fi ;; + unset) + KEY="${2:?Usage: gstack-config unset }" + if ! printf '%s' "$KEY" | LC_ALL=C grep -qE '^[a-zA-Z0-9_]+(@[a-zA-Z0-9]+)?$'; then + echo "Error: key must contain only alphanumeric characters, underscores, and an optional @ suffix" >&2 + exit 1 + fi + [ -f "$CONFIG_FILE" ] || exit 0 + config_lock + config_edit_begin + _cfg_del "$KEY" + # Clearing the allowlist drops the +repos marker in the same write. + if [ "$KEY" = "transcript_repos" ]; then transcript_marker_sync; fi + config_edit_commit + config_unlock + ;; list) if [ -f "$CONFIG_FILE" ]; then cat "$CONFIG_FILE" @@ -787,7 +970,7 @@ case "${1:-}" in esac ;; *) - echo "Usage: gstack-config {get|set|list|defaults|endpoint-hash|resolve-user-slug|gbrain-refresh} [key] [value]" + echo "Usage: gstack-config {get|has|set|unset|list|defaults|endpoint-hash|resolve-user-slug|gbrain-refresh} [key] [value]" exit 1 ;; esac diff --git a/bin/gstack-gbrain-sync.ts b/bin/gstack-gbrain-sync.ts index 1f627b118..d6634ce60 100644 --- a/bin/gstack-gbrain-sync.ts +++ b/bin/gstack-gbrain-sync.ts @@ -45,7 +45,14 @@ import { buildGbrainEnv, spawnGbrain, spawnGbrainAsync, execGbrainJson, NEEDS_SH import { repoPolicyTier as sharedRepoPolicyTier } from "../lib/gbrain-repo-policy-client"; import { checkOwnedStagingDir } from "../lib/staging-guard"; import { resolveStateRoot } from "../lib/state-root"; -import { normalizeTranscriptConsent, type TranscriptConsent } from "./gstack-memory-ingest"; +import type { TranscriptConsent } from "./gstack-memory-ingest"; +import { + describeTranscriptPolicy, + isScoped, + readTranscriptConsent, + reposHash, + type TranscriptPolicy, +} from "../lib/transcript-consent"; // ── Types ────────────────────────────────────────────────────────────────── @@ -107,11 +114,15 @@ interface StageResult { } interface TranscriptConsentRecord { - /** Normalized transcript_ingest_mode: recent | all | off | not-set | legacy | unrecognized. */ + /** Normalized transcript_ingest_mode: recent | all | new | off | not-set | legacy | unrecognized | repos-unreadable. */ mode: string; - window: "recent" | "all" | null; + window: "recent" | "all" | "new" | null; /** Why transcripts were not ingested; absent when they were. */ skip_reason?: string; + /** The new@ cutoff in force; a changed cutoff restages. */ + cutoff?: string; + /** Hash of the transcript_repos allowlist in force; a scope change restages. */ + repos_hash?: string; } // ── Constants ────────────────────────────────────────────────────────────── @@ -304,16 +315,28 @@ export function transcriptConsentNotice(consent: TranscriptConsent, quiet: boole if (consent.affirmative) return null; const value = consent.value ?? "not set"; if (override) { - return `gbrain-sync: transcripts ingested because ${override} names transcript (transcript_ingest_mode=${value}). ${TRANSCRIPT_CHOICE_HINT}`; + const scope = isScoped(consent) ? " The new@ cutoff and transcript_repos allowlist still apply." : ""; + return `gbrain-sync: transcripts ingested because ${override} names transcript (transcript_ingest_mode=${value}).${scope} ${TRANSCRIPT_CHOICE_HINT}`; } if (consent.reason === "off") return quiet ? null : "gbrain-sync: transcripts off (your choice)"; + if (consent.reason === "repos-unreadable") { + return `gbrain-sync: transcripts skipped (transcript_ingest_mode=${value}): its +repos marker needs a transcript_repos allowlist, which is missing or empty. Other memory still syncs. ${TRANSCRIPT_CHOICE_HINT}`; + } return `gbrain-sync: transcripts skipped (transcript_ingest_mode=${value}). Other memory still syncs. ${TRANSCRIPT_CHOICE_HINT}`; } -/** The consent record a memory stage stores; resume requires mode and window to match. */ +/** The consent in words for a consenting run (window or cutoff, repos, config roots). */ +export function transcriptConsentSummary(consent: TranscriptPolicy, quiet: boolean): string | null { + if (!consent.affirmative || quiet) return null; + return `gbrain-sync: transcripts: ${describeTranscriptPolicy(consent)}`; +} + +/** The consent record a memory stage stores; resume requires every field to match. */ export function transcriptConsentRecord(consent: TranscriptConsent, override: boolean): TranscriptConsentRecord { const record: TranscriptConsentRecord = { mode: consent.reason, window: consent.window }; if (!consent.affirmative && !override) record.skip_reason = consent.reason; + if (consent.cutoff !== undefined) record.cutoff = consent.cutoff; + if (consent.repos !== undefined) record.repos_hash = reposHash(consent.repos); return record; } @@ -448,10 +471,12 @@ Options: except ${FEDERATED_CURATED_TYPES.join(",")} when a federated gstack source already indexes them, and except transcript unless transcript_ingest_mode is recent - (last 90 days) or all (all history; --full passes - --all-history). Without that consent each run prints a - notice, even with --quiet. A list naming transcript - overrides the mode for that run. See + (last 90 days), all (all history; --full passes + --all-history) or new@ (sessions started after + it). Without that consent each run prints a notice, + even with --quiet. A list naming transcript overrides + the mode for that run, but a new@ cutoff and the + transcript_repos allowlist still apply. See setup-gbrain/memory.md#transcripts. --help This text. @@ -1470,12 +1495,12 @@ export function ensureGbrainSourceGitignored(root: string): void { } } -function printTranscriptNotice(args: CliArgs, consent: TranscriptConsent, selection: ConsentedSelection, why: string): void { - const notice = transcriptConsentNotice(consent, args.quiet, selection.override ? why : null); +function printTranscriptNotice(args: CliArgs, consent: TranscriptPolicy, selection: ConsentedSelection, why: string): void { + const notice = transcriptConsentNotice(consent, args.quiet, selection.override ? why : null) ?? transcriptConsentSummary(consent, args.quiet); if (notice) console.error(notice); } -function runMemoryIngest(args: CliArgs, previous: SyncState, consent: TranscriptConsent): StageResult { +function runMemoryIngest(args: CliArgs, previous: SyncState, consent: TranscriptPolicy): StageResult { const t0 = Date.now(); if (args.mode === "dry-run") { @@ -1518,14 +1543,19 @@ function runMemoryIngest(args: CliArgs, previous: SyncState, consent: Transcript // transcript window. const prevStage = previous.last_stages?.find((st) => st.name === "memory"); const staged = prevStage?.memory_sources ?? [...MEMORY_INGEST_TYPES]; - const prevConsent = prevStage?.transcript_consent ?? { mode: "recent", window: "recent" }; + const prevConsent: TranscriptConsentRecord = prevStage?.transcript_consent ?? { mode: "recent", window: "recent" }; if ([...staged].sort().join(",") !== [...selected].sort().join(",")) { console.error( `[sync:memory] memory source selection changed since the checkpointed run (${staged.join(",")} → ${selected.join(",")}); restaging from scratch.`, ); resume = { kind: "no-checkpoint" }; } - if (prevConsent.mode !== consentRecord.mode || prevConsent.window !== consentRecord.window) { + if ( + prevConsent.mode !== consentRecord.mode || + prevConsent.window !== consentRecord.window || + prevConsent.cutoff !== consentRecord.cutoff || + prevConsent.repos_hash !== consentRecord.repos_hash + ) { console.error( `gbrain-sync: transcript consent changed since the interrupted import (${prevConsent.mode} → ${consentRecord.mode}); restaging memory from scratch once.`, ); @@ -1554,7 +1584,7 @@ function runMemoryIngest(args: CliArgs, previous: SyncState, consent: Transcript const ingestArgs = ["run", ingestPath]; if (args.mode === "full") ingestArgs.push("--bulk"); else ingestArgs.push("--incremental"); - if (args.mode === "full" && consent.window === "all") ingestArgs.push("--all-history"); + if (args.mode === "full" && consent.affirmative && consent.window !== "recent") ingestArgs.push("--all-history"); if (args.quiet) ingestArgs.push("--quiet"); if (selection.sources) ingestArgs.push("--sources", selection.sources.join(",")); if (!args.quiet) console.error(`[sync:memory] sources: ${selected.join(",")} (${selection.why})`); @@ -1963,7 +1993,7 @@ async function main(): Promise { const args = parseArgs(); // Read once at sync start: a config change made while the sync runs never // adds transcripts to this run. - const consent = args.noMemory ? null : normalizeTranscriptConsent(); + const consent = args.noMemory ? null : readTranscriptConsent(); if (!args.quiet) { const engine = detectEngineTier(); diff --git a/bin/gstack-memory-ingest.ts b/bin/gstack-memory-ingest.ts index 22489b756..80f813545 100644 --- a/bin/gstack-memory-ingest.ts +++ b/bin/gstack-memory-ingest.ts @@ -72,7 +72,15 @@ import { execGbrainText, spawnGbrainAsync } from "../lib/gbrain-exec"; import { writeReceipt } from "../lib/egress-receipt"; import { checkOwnedStagingDir, STAGING_MARKER } from "../lib/staging-guard"; import { hasRepoPolicyStore, repoPolicyTierBatch } from "../lib/gbrain-repo-policy-client"; -import { resolveStateRoot, readConfigKeyWithRoot, type StateRootEnv } from "../lib/state-root"; +import { resolveStateRoot, mergedStateRoots, type StateRootEnv } from "../lib/state-root"; +import { + cutoffExclusion, + purgeStagedTranscripts, + readTranscriptConsent, + repoExclusion, + type TranscriptConsent, + type TranscriptPolicy, +} from "../lib/transcript-consent"; // ── Types ────────────────────────────────────────────────────────────────── @@ -155,6 +163,8 @@ interface ProbeReport { */ skipped_policy_deny: number; skipped_policy_readonly: number; + /** Sessions a scoped consent (new@ cutoff, transcript_repos) kept out. */ + scope: TranscriptScopeCounts; estimate_minutes: number; } @@ -174,6 +184,8 @@ interface BulkResult { failed: number; duration_ms: number; partial_pages: number; + /** Sessions a scoped consent kept out, and transcript pages selected. */ + scope?: TranscriptScopeCounts; /** * D6: when set, indicates a process-level failure (gbrain CLI missing * or `gbrain import` crashed). Per-file errors (FILE_TOO_LARGE etc.) @@ -204,43 +216,39 @@ const ALL_TYPES: MemoryType[] = [ // ── Transcript consent ───────────────────────────────────────────────────── -export interface TranscriptConsent { - /** True only for a stored `recent` or `all`. */ - affirmative: boolean; - /** Transcript walk window when affirmative: `recent` = last 90 days, `all` = all history. */ - window: "recent" | "all" | null; - /** - * Normalized mode: the window when affirmative; otherwise why transcripts - * are skipped. `legacy` covers the old gate letters (A-E) and `incremental`, - * which predate the consent rule and need a new choice. - */ - reason: "recent" | "all" | "off" | "not-set" | "legacy" | "unrecognized"; - /** The stored value as read (null when the key is absent). */ - value: string | null; -} +export type { TranscriptConsent } from "../lib/transcript-consent"; -let cachedConsent: TranscriptConsent | null = null; +let cachedConsent: TranscriptPolicy | null = null; /** - * Read `transcript_ingest_mode` once per process (same parse as - * `gstack-config has` + `get`) and decide whether transcripts may be ingested. - * Only `recent` and `all` are consent. An absent key, `off`, a legacy value - * and anything unrecognized are not. Pass `env` to bypass the per-process cache. + * Read transcript consent once per process across every merged state root + * (lib/transcript-consent.ts). `recent`, `all` and `new@