mirror of
https://github.com/garrytan/gstack.git
synced 2026-11-06 08:54:04 +01:00
Merge remote-tracking branch 'origin/garrytan/opus55fu-wip-H1' into garrytan/opus55-followups
This commit is contained in:
garrytan committed
commit
f6707237a3
15 files changed
+1926
-137
No files matched your search
+38
-10
@@ -393,15 +393,38 @@ for p in final:
|
||||
PYEOF
|
||||
}
|
||||
|
||||
# Transcript consent: transcript_ingest_mode is recent or all (trimmed, any
|
||||
# case). An absent key, off, a legacy value or anything else is not consent.
|
||||
# Same rule as normalizeTranscriptConsent in gstack-memory-ingest.ts.
|
||||
# Transcript consent comes from the one shared policy reader
|
||||
# (lib/transcript-consent.ts): every merged state root, the new@ cutoff and the
|
||||
# transcript_repos allowlist. Without bun nothing transcript-related can be
|
||||
# checked, so transcript pages are held (fail-closed).
|
||||
CONSENT_TS="$SCRIPT_DIR/../lib/transcript-consent.ts"
|
||||
transcript_policy() {
|
||||
command -v bun >/dev/null 2>&1 || return 2
|
||||
GSTACK_STATE_ROOT="$GSTACK_HOME" bun run "$CONSENT_TS" "$@"
|
||||
}
|
||||
|
||||
# 0 when staged transcript pages may sync at all.
|
||||
transcripts_consented() {
|
||||
"$CONFIG_BIN" has transcript_ingest_mode 2>/dev/null || return 1
|
||||
local v
|
||||
v=$("$CONFIG_BIN" get transcript_ingest_mode 2>/dev/null || true)
|
||||
v=$(printf '%s' "$v" | tr -d '[:space:]' | tr '[:upper:]' '[:lower:]')
|
||||
[ "$v" = "recent" ] || [ "$v" = "all" ]
|
||||
transcript_policy --check 2>/dev/null
|
||||
}
|
||||
|
||||
# Before every push: scan the unpublished range for transcript pages outside
|
||||
# the current consent; when some exist, the helper rewrites gstack-brain-sync's
|
||||
# own unpublished commits without them, or refuses (exit 3) when it cannot.
|
||||
# Returns 0 when the push may go ahead. A range that touches no transcript
|
||||
# page needs no scan (and no bun).
|
||||
guard_unpublished_range() {
|
||||
local branch range="HEAD" rc=0
|
||||
branch=$(git -C "$GSTACK_HOME" rev-parse --abbrev-ref HEAD 2>/dev/null || echo "")
|
||||
if [ -n "$branch" ] && [ "$branch" != "HEAD" ] && git -C "$GSTACK_HOME" rev-parse --verify --quiet "origin/$branch" >/dev/null 2>&1; then
|
||||
range="origin/$branch..HEAD"
|
||||
fi
|
||||
[ -n "$(git -C "$GSTACK_HOME" log --format= --name-only "$range" -- transcripts 2>/dev/null | head -1)" ] || return 0
|
||||
transcript_policy --guard-unpublished || rc=$?
|
||||
if [ "$rc" -eq 2 ]; then
|
||||
echo "BRAIN_SYNC: push blocked: unpushed commits hold transcript pages and bun is not on PATH to check them against your transcript consent" >&2
|
||||
fi
|
||||
[ "$rc" -eq 0 ]
|
||||
}
|
||||
|
||||
# Finalize the drain: delete exactly the spool record files this drain
|
||||
@@ -637,7 +660,7 @@ subcmd_once() {
|
||||
det_now=$(date +%s)
|
||||
det_last=$(cat "$GSTACK_HOME/.brain-last-push-attempt" 2>/dev/null || echo 0)
|
||||
case "$det_last" in ''|*[!0-9]*) det_last=0 ;; esac
|
||||
if [ "$det_unpushed" -gt 0 ] && [ "$det_unpushed" -eq "$det_total" ] && [ $(( det_now - det_last )) -ge 600 ]; then
|
||||
if [ "$det_unpushed" -gt 0 ] && [ "$det_unpushed" -eq "$det_total" ] && [ $(( det_now - det_last )) -ge 600 ] && guard_unpublished_range; then
|
||||
echo "$det_now" > "$GSTACK_HOME/.brain-last-push-attempt" 2>/dev/null || true
|
||||
local det_host
|
||||
det_host=$(remote_host)
|
||||
@@ -676,6 +699,9 @@ subcmd_once() {
|
||||
# snapshot manifest but no classification. Finalizing on that state is what
|
||||
# used to wipe the queue — so on a nonzero exit, warn loudly, do NOT call
|
||||
# finalize_queue, and leave everything queued for the next drain.
|
||||
# Tightened consent (new@ cutoff, narrower transcript_repos): remove the
|
||||
# unpublished staged pages it now excludes before anything is staged.
|
||||
[ -d "$GSTACK_HOME/transcripts" ] && { transcript_policy --purge || true; }
|
||||
local transcripts_ok=0
|
||||
transcripts_consented && transcripts_ok=1
|
||||
if ! compute_paths_to_stage "$mode" "$class_file" "$snapshot_file" "$transcripts_ok" > "$paths_file"; then
|
||||
@@ -743,6 +769,8 @@ subcmd_once() {
|
||||
exit 0
|
||||
}
|
||||
|
||||
# Nothing outside the transcript consent is pushed; a refused rewrite keeps the commit local.
|
||||
guard_unpublished_range || { write_status "blocked" "push blocked: unpushed commits hold transcript pages outside your transcript consent; commit retained locally"; finalize_queue "$snapshot_file" "$class_file" "$paths_file"; exit 0; }
|
||||
# Push. On reject, fetch + merge (merge driver handles JSONL) + retry once.
|
||||
local push_err
|
||||
push_err=$(git -C "$GSTACK_HOME" push origin HEAD 2>&1 >/dev/null) || {
|
||||
@@ -766,7 +794,7 @@ subcmd_once() {
|
||||
bash -c 'git -C "$1" fetch origin 2>/dev/null' _ "$GSTACK_HOME"; then
|
||||
local branch
|
||||
branch=$(git -C "$GSTACK_HOME" rev-parse --abbrev-ref HEAD 2>/dev/null || echo main)
|
||||
if git -C "$GSTACK_HOME" merge --no-edit "origin/$branch" >/dev/null 2>&1; then
|
||||
if git -C "$GSTACK_HOME" merge --no-edit "origin/$branch" >/dev/null 2>&1 && guard_unpublished_range; then
|
||||
if GSTACK_HOME="$GSTACK_HOME" _receipted_git closed brain-sync "$push_host" curated-memory-git-push "artifacts_sync_mode!=off" \
|
||||
bash -c 'git -C "$1" push origin HEAD 2>/dev/null' _ "$GSTACK_HOME"; then
|
||||
finalize_queue "$snapshot_file" "$class_file" "$paths_file"
|
||||
|
||||
+211
-28
@@ -7,6 +7,7 @@
|
||||
# config file (get returns DEFAULTS for absent keys,
|
||||
# so callers that need provenance use this instead)
|
||||
# gstack-config set <key> <value> — write a config value
|
||||
# gstack-config unset <key> — remove a key from the config file
|
||||
# gstack-config list — show all config (values + defaults)
|
||||
# gstack-config defaults — show just the defaults table
|
||||
#
|
||||
@@ -92,17 +93,24 @@ CONFIG_HEADER='# gstack configuration — edit freely, changes take effect on ne
|
||||
# # Flip back to false to be re-prompted.
|
||||
#
|
||||
# ─── Transcript ingest (gbrain memory) ───────────────────────────────
|
||||
# transcript_ingest_mode: off # recent | all | off — consent for /sync-gbrain
|
||||
# # to ingest Claude Code and Codex session
|
||||
# # transcripts (every project repo policy allows)
|
||||
# # into your brain (local or Supabase).
|
||||
# transcript_ingest_mode: off # recent | all | off | new@<UTC> — consent for
|
||||
# # /sync-gbrain to ingest Claude Code and Codex
|
||||
# # session transcripts (every project repo policy
|
||||
# # allows) into your brain (local or Supabase).
|
||||
# # recent — last 90 days
|
||||
# # all — all history
|
||||
# # new@2026-10-03T17:00:00Z — only sessions that
|
||||
# # start after that UTC time
|
||||
# # off — never (other memory still syncs)
|
||||
# # A +repos suffix is kept automatically while
|
||||
# # transcript_repos is set.
|
||||
# # Absent, legacy (A-E, incremental) or unknown
|
||||
# # values are not consent: transcripts are skipped
|
||||
# # and /sync-gbrain asks once. An invalid value is
|
||||
# # REJECTED and the stored value kept.
|
||||
# transcript_repos: # comma-separated git remotes; only transcripts
|
||||
# # from these repos are ingested. Clear it with
|
||||
# # `gstack-config unset transcript_repos`.
|
||||
# # Docs: setup-gbrain/memory.md#transcripts
|
||||
#
|
||||
# ─── Timeline Stop hook ──────────────────────────────────────────────
|
||||
@@ -220,7 +228,8 @@ lookup_default() {
|
||||
# the callers already assume in their own `|| echo "<default>"` fallback.
|
||||
question_tuning) echo "false" ;;
|
||||
team_mode) echo "false" ;;
|
||||
transcript_ingest_mode) echo "off" ;; # recent | all | off — transcript consent; absent = not consented (use `has`)
|
||||
transcript_ingest_mode) echo "off" ;; # recent | all | off | new@<UTC> [+repos] — transcript consent; absent = not consented (use `has`)
|
||||
transcript_repos) echo "" ;; # empty = every repo; comma-separated canonical remotes narrow transcript ingest
|
||||
# repo_mode: EMPTY is load-bearing — gstack-repo-mode treats any non-empty
|
||||
# answer as a user override and skips its own classification entirely, so
|
||||
# a synthesized "unknown" default turns the classifier into dead code.
|
||||
@@ -348,14 +357,16 @@ resolve_user_slug() {
|
||||
_slug="anonymous-$(sha8_of "$(hostname 2>/dev/null || echo unknown)")"
|
||||
fi
|
||||
|
||||
# Persist via direct file write (avoid recursion into gstack-config set)
|
||||
mkdir -p "$STATE_DIR"
|
||||
if [ ! -f "$CONFIG_FILE" ]; then
|
||||
printf '%s' "$CONFIG_HEADER" > "$CONFIG_FILE"
|
||||
fi
|
||||
if ! grep -qE "^user_slug_at_${_hash}:" "$CONFIG_FILE" 2>/dev/null; then
|
||||
echo "user_slug_at_${_hash}: ${_slug}" >> "$CONFIG_FILE"
|
||||
# Persist via the locked read-modify-write (no recursion into `set`).
|
||||
config_lock
|
||||
config_edit_begin
|
||||
if [ -z "$(_cfg_get "user_slug_at_${_hash}")" ]; then
|
||||
_cfg_put "user_slug_at_${_hash}" "$_slug"
|
||||
config_edit_commit
|
||||
else
|
||||
rm -f "$_cfg_tmp"
|
||||
fi
|
||||
config_unlock
|
||||
|
||||
printf '%s' "$_slug"
|
||||
}
|
||||
@@ -366,6 +377,142 @@ read_config_value() {
|
||||
gstack_read_config_key "$1"
|
||||
}
|
||||
|
||||
# ── Config writes: one lock, one read-modify-write rename ──────────────────
|
||||
# Every mutation of config.yaml (set, unset, the transcript_repos pair, the
|
||||
# user-slug writer) holds an atomic mkdir lock and replaces the file with one
|
||||
# rename, so concurrent writers never lose each other's keys and an
|
||||
# interrupted write leaves the old file. The pid file holds "<pid> <epoch>";
|
||||
# a lock whose holder is dead or older than 10 s is taken over. No flock(1):
|
||||
# stock macOS does not ship it.
|
||||
CONFIG_LOCK="$STATE_DIR/.config-yaml.lock"
|
||||
_config_locked=""
|
||||
|
||||
config_lock() {
|
||||
mkdir -p "$STATE_DIR"
|
||||
local waited=0 held pid ts stale
|
||||
while ! mkdir "$CONFIG_LOCK" 2>/dev/null; do
|
||||
if [ ! -w "$STATE_DIR" ]; then
|
||||
echo "Error: cannot write to $STATE_DIR (its config lock could not be created). Existing value left unchanged." >&2
|
||||
exit 1
|
||||
fi
|
||||
waited=$(( waited + 1 ))
|
||||
if [ "$waited" -gt 400 ]; then
|
||||
echo "Error: $CONFIG_FILE is locked by another gstack-config ($(cat "$CONFIG_LOCK/pid" 2>/dev/null || echo 'unknown holder')). Existing value left unchanged. fix: retry, or remove $CONFIG_LOCK if no gstack-config is running." >&2
|
||||
exit 1
|
||||
fi
|
||||
held=$(cat "$CONFIG_LOCK/pid" 2>/dev/null || true)
|
||||
pid=""; ts=""; stale=""
|
||||
case "$held" in *" "*) pid="${held%% *}"; ts="${held#* }"; ts="${ts%%[!0-9]*}" ;; esac
|
||||
if [ -n "$pid" ]; then
|
||||
if ! kill -0 "$pid" 2>/dev/null; then stale=1
|
||||
elif [ -n "$ts" ] && [ $(( $(date +%s) - ts )) -gt 10 ]; then stale=1; fi
|
||||
elif [ "$waited" -gt 200 ]; then
|
||||
stale=1
|
||||
fi
|
||||
# Take over only the lock we judged: a holder that changed meanwhile is live.
|
||||
if [ -n "$stale" ] && [ "$(cat "$CONFIG_LOCK/pid" 2>/dev/null || true)" = "$held" ]; then
|
||||
rm -rf "$CONFIG_LOCK" 2>/dev/null || true
|
||||
continue
|
||||
fi
|
||||
sleep 0.05
|
||||
done
|
||||
printf '%s %s\n' "$$" "$(date +%s)" > "$CONFIG_LOCK/pid"
|
||||
_config_locked=1
|
||||
trap 'config_unlock' EXIT
|
||||
}
|
||||
|
||||
config_unlock() {
|
||||
[ -n "$_config_locked" ] || return 0
|
||||
rm -rf "$CONFIG_LOCK" 2>/dev/null || true
|
||||
_config_locked=""
|
||||
}
|
||||
|
||||
# config_edit_begin — copy config.yaml (or the annotated header for a new
|
||||
# file) to a temp file beside it; edits go there until config_edit_commit.
|
||||
config_edit_begin() {
|
||||
_cfg_tmp="$(mktemp "$STATE_DIR/.config.yaml.XXXXXX")"
|
||||
if [ -f "$CONFIG_FILE" ]; then cat "$CONFIG_FILE" > "$_cfg_tmp"; else printf '%s' "$CONFIG_HEADER" > "$_cfg_tmp"; fi
|
||||
}
|
||||
|
||||
config_edit_commit() {
|
||||
mv "$_cfg_tmp" "$CONFIG_FILE"
|
||||
}
|
||||
|
||||
# _cfg_get KEY — last value of KEY in the temp file ("" when absent).
|
||||
_cfg_get() {
|
||||
awk -v k="$1" 'BEGIN { n = length(k) + 1 } substr($0, 1, n) == k ":" { v = substr($0, n + 1) } END { sub(/^[ \t]+/, "", v); sub(/[ \t\r]+$/, "", v); print v }' "$_cfg_tmp"
|
||||
}
|
||||
|
||||
# _cfg_put KEY VALUE — replace every KEY line in the temp file, or append one.
|
||||
# The value travels through ENVIRON so awk applies no escape processing.
|
||||
_cfg_put() {
|
||||
_CFG_V="$2" awk -v k="$1" 'BEGIN { n = length(k) + 1; v = ENVIRON["_CFG_V"] } substr($0, 1, n) == k ":" { print k ": " v; done = 1; next } { print } END { if (!done) print k ": " v }' "$_cfg_tmp" > "$_cfg_tmp.n"
|
||||
mv "$_cfg_tmp.n" "$_cfg_tmp"
|
||||
}
|
||||
|
||||
_cfg_del() {
|
||||
awk -v k="$1" 'BEGIN { n = length(k) + 1 } substr($0, 1, n) != k ":"' "$_cfg_tmp" > "$_cfg_tmp.n"
|
||||
mv "$_cfg_tmp.n" "$_cfg_tmp"
|
||||
}
|
||||
|
||||
# ── Transcript consent grammar ─────────────────────────────────────────────
|
||||
# transcript_ingest_mode = <base>[@<YYYY-MM-DDTHH:MM:SSZ>][+repos]; base is
|
||||
# recent, all, off or new. `@` is required for new and invalid otherwise; off
|
||||
# never carries +repos. `set` takes the canonical lowercase form (readers in
|
||||
# lib/transcript-consent.ts are case-insensitive).
|
||||
valid_utc_second() {
|
||||
[[ "$1" =~ ^([0-9]{4})-([0-9]{2})-([0-9]{2})T([0-9]{2}):([0-9]{2}):([0-9]{2})Z$ ]] || return 1
|
||||
local y=$((10#${BASH_REMATCH[1]})) m=$((10#${BASH_REMATCH[2]})) d=$((10#${BASH_REMATCH[3]}))
|
||||
local hh=$((10#${BASH_REMATCH[4]})) mm=$((10#${BASH_REMATCH[5]})) ss=$((10#${BASH_REMATCH[6]})) dim=31
|
||||
[ "$m" -ge 1 ] && [ "$m" -le 12 ] && [ "$hh" -le 23 ] && [ "$mm" -le 59 ] && [ "$ss" -le 59 ] || return 1
|
||||
case "$m" in
|
||||
4|6|9|11) dim=30 ;;
|
||||
2) dim=28; if [ $((y % 4)) -eq 0 ] && { [ $((y % 100)) -ne 0 ] || [ $((y % 400)) -eq 0 ]; }; then dim=29; fi ;;
|
||||
esac
|
||||
[ "$d" -ge 1 ] && [ "$d" -le "$dim" ]
|
||||
}
|
||||
|
||||
# valid_transcript_mode VALUE — canonical grammar check (exit 0 = valid).
|
||||
valid_transcript_mode() {
|
||||
local v="$1" base
|
||||
base="${v%+repos}"
|
||||
case "$base" in
|
||||
recent|all) ;;
|
||||
off) [ "$base" = "$v" ] || return 1 ;;
|
||||
new@*) valid_utc_second "${base#new@}" || return 1 ;;
|
||||
*) return 1 ;;
|
||||
esac
|
||||
}
|
||||
|
||||
# canonical_remote URL — same rules as canonicalizeRemote in
|
||||
# lib/gstack-memory-helpers.ts (test/transcript-consent.test.ts keeps them equal).
|
||||
canonical_remote() {
|
||||
local s="$1"
|
||||
s="${s#"${s%%[![:space:]]*}"}"; s="${s%"${s##*[![:space:]]}"}"
|
||||
s="${s#[\"\']}"; s="${s%[\"\']}"
|
||||
if [[ "$s" =~ ^[^@[:space:]]+@([^:]+):(.+)$ ]]; then
|
||||
s="${BASH_REMATCH[1]}/${BASH_REMATCH[2]}"
|
||||
else
|
||||
s=$(printf '%s' "$s" | sed -E 's#^[a-zA-Z][a-zA-Z0-9+.-]*://##; s#^[^@/]+@##')
|
||||
fi
|
||||
printf '%s' "$s" | sed -E 's#/+$##; s#\.[gG][iI][tT]$##; s#/+$##; s#/{2,}#/#g' | tr '[:upper:]' '[:lower:]'
|
||||
}
|
||||
|
||||
# transcript_marker_sync — keep the +repos marker on transcript_ingest_mode in
|
||||
# step with transcript_repos inside the temp file: present while an allowlist
|
||||
# exists and the base consents, absent otherwise. Unrecognized values are
|
||||
# left untouched (they are not consent either way).
|
||||
transcript_marker_sync() {
|
||||
local mode list base
|
||||
mode=$(_cfg_get transcript_ingest_mode)
|
||||
[ -n "$mode" ] || return 0
|
||||
valid_transcript_mode "$mode" || return 0
|
||||
list=$(_cfg_get transcript_repos)
|
||||
base="${mode%+repos}"
|
||||
if [ -n "$list" ] && [ "$base" != "off" ]; then _cfg_put transcript_ingest_mode "$base+repos"
|
||||
else _cfg_put transcript_ingest_mode "$base"; fi
|
||||
}
|
||||
|
||||
_STATE_DOC="https://github.com/garrytan/gstack/blob/main/docs/state-root.md"
|
||||
_SELF="$(cd "$(dirname "$0")" && pwd)/gstack-config"
|
||||
|
||||
@@ -509,12 +656,31 @@ case "${1:-}" in
|
||||
echo "Error: cross_project_learnings '$VALUE' not recognized. Valid values: true, false. Existing value left unchanged." >&2
|
||||
exit 1
|
||||
fi
|
||||
# transcript_ingest_mode is a CONSENT key: recent/all let /sync-gbrain
|
||||
# transcript_ingest_mode is a CONSENT key: recent/all/new@ let /sync-gbrain
|
||||
# ingest coding-agent transcripts. Reject a typo; never coerce or store it.
|
||||
if [ "$KEY" = "transcript_ingest_mode" ] && [ "$VALUE" != "recent" ] && [ "$VALUE" != "all" ] && [ "$VALUE" != "off" ]; then
|
||||
if [ "$KEY" = "transcript_ingest_mode" ] && ! valid_transcript_mode "$VALUE"; then
|
||||
echo "Error: transcript_ingest_mode '$VALUE' not recognized. Valid values: recent, all, off. Existing value left unchanged." >&2
|
||||
echo "Also valid: new@<YYYY-MM-DDTHH:MM:SSZ> (only sessions that start after that UTC time, e.g. new@$(date -u +%Y-%m-%dT%H:%M:%SZ)). The +repos suffix follows transcript_repos; off never carries it. Docs: setup-gbrain/memory.md#transcripts" >&2
|
||||
exit 1
|
||||
fi
|
||||
# transcript_repos narrows transcript consent to these repos. Each entry is
|
||||
# stored as its canonical remote (host/owner/repo); an empty list would
|
||||
# deny every repo, so it is rejected: `unset` clears the allowlist.
|
||||
if [ "$KEY" = "transcript_repos" ]; then
|
||||
_clean=""
|
||||
_ifs="$IFS"; IFS=','
|
||||
for _x in $VALUE; do
|
||||
_x="$(canonical_remote "$_x")"
|
||||
[ -n "$_x" ] || continue
|
||||
case ",$_clean," in *",$_x,"*) ;; *) _clean="${_clean:+$_clean,}$_x" ;; esac
|
||||
done
|
||||
IFS="$_ifs"
|
||||
if [ -z "$_clean" ]; then
|
||||
echo "Error: transcript_repos '$VALUE' names no repo. Valid values: comma-separated git remotes (e.g. github.com/owner/repo). Existing value left unchanged. To allow every repo: gstack-config unset transcript_repos" >&2
|
||||
exit 1
|
||||
fi
|
||||
VALUE="$_clean"
|
||||
fi
|
||||
# memorable_recall is a CONSENT key: `on` lets a Claude Code hook hand every
|
||||
# prompt to a third-party binary. Reject like codex_reviews -- a typo must
|
||||
# never flip consent in either direction, so nothing is coerced or stored.
|
||||
@@ -581,21 +747,23 @@ case "${1:-}" in
|
||||
done
|
||||
done
|
||||
fi
|
||||
mkdir -p "$STATE_DIR"
|
||||
# Write annotated header on first creation
|
||||
if [ ! -f "$CONFIG_FILE" ]; then
|
||||
printf '%s' "$CONFIG_HEADER" > "$CONFIG_FILE"
|
||||
# Locked read-modify-write; the annotated header is written on first creation.
|
||||
config_lock
|
||||
config_edit_begin
|
||||
if [ "$KEY" = "transcript_ingest_mode" ] && [ "${VALUE%+repos}" != "$VALUE" ] && [ -z "$(_cfg_get transcript_repos)" ]; then
|
||||
rm -f "$_cfg_tmp"
|
||||
echo "Error: transcript_ingest_mode '$VALUE' needs a repo allowlist, but transcript_repos is not set. Existing value left unchanged. fix: gstack-config set transcript_repos <remotes>, then set transcript_ingest_mode ${VALUE%+repos}" >&2
|
||||
exit 1
|
||||
fi
|
||||
# Escape sed replacement metacharacters (newlines were rejected above).
|
||||
_cfg_put "$KEY" "$VALUE"
|
||||
# The allowlist and its +repos marker change together, in this one rename.
|
||||
if [ "$KEY" = "transcript_ingest_mode" ] || [ "$KEY" = "transcript_repos" ]; then
|
||||
transcript_marker_sync
|
||||
if [ "$KEY" = "transcript_ingest_mode" ]; then VALUE="$(_cfg_get transcript_ingest_mode)"; fi
|
||||
fi
|
||||
config_edit_commit
|
||||
config_unlock
|
||||
SAFE_VALUE="$VALUE"
|
||||
ESC_VALUE="$(printf '%s' "$SAFE_VALUE" | sed 's/[&/\]/\\&/g')"
|
||||
if grep -qE "^${KEY}:" "$CONFIG_FILE" 2>/dev/null; then
|
||||
# Portable in-place edit (BSD sed uses -i '', GNU sed uses -i without arg)
|
||||
_tmpfile="$(mktemp "${CONFIG_FILE}.XXXXXX")"
|
||||
sed "/^${KEY}:/s/.*/${KEY}: ${ESC_VALUE}/" "$CONFIG_FILE" > "$_tmpfile" && mv "$_tmpfile" "$CONFIG_FILE"
|
||||
else
|
||||
echo "${KEY}: ${SAFE_VALUE}" >> "$CONFIG_FILE"
|
||||
fi
|
||||
# Report only when another root still overrides the value just written.
|
||||
gstack_config_select "$KEY"
|
||||
if [ -n "$_gstack_cfg_root" ] && [ "$_gstack_cfg_root" != "$STATE_DIR" ] && [ "$_gstack_cfg_value" != "$SAFE_VALUE" ]; then
|
||||
@@ -616,6 +784,21 @@ case "${1:-}" in
|
||||
fi
|
||||
fi
|
||||
;;
|
||||
unset)
|
||||
KEY="${2:?Usage: gstack-config unset <key>}"
|
||||
if ! printf '%s' "$KEY" | LC_ALL=C grep -qE '^[a-zA-Z0-9_]+(@[a-zA-Z0-9]+)?$'; then
|
||||
echo "Error: key must contain only alphanumeric characters, underscores, and an optional @<endpoint-id> suffix" >&2
|
||||
exit 1
|
||||
fi
|
||||
[ -f "$CONFIG_FILE" ] || exit 0
|
||||
config_lock
|
||||
config_edit_begin
|
||||
_cfg_del "$KEY"
|
||||
# Clearing the allowlist drops the +repos marker in the same write.
|
||||
if [ "$KEY" = "transcript_repos" ]; then transcript_marker_sync; fi
|
||||
config_edit_commit
|
||||
config_unlock
|
||||
;;
|
||||
list)
|
||||
if [ -f "$CONFIG_FILE" ]; then
|
||||
cat "$CONFIG_FILE"
|
||||
@@ -787,7 +970,7 @@ case "${1:-}" in
|
||||
esac
|
||||
;;
|
||||
*)
|
||||
echo "Usage: gstack-config {get|set|list|defaults|endpoint-hash|resolve-user-slug|gbrain-refresh} [key] [value]"
|
||||
echo "Usage: gstack-config {get|has|set|unset|list|defaults|endpoint-hash|resolve-user-slug|gbrain-refresh} [key] [value]"
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
+46
-16
@@ -45,7 +45,14 @@ import { buildGbrainEnv, spawnGbrain, spawnGbrainAsync, execGbrainJson, NEEDS_SH
|
||||
import { repoPolicyTier as sharedRepoPolicyTier } from "../lib/gbrain-repo-policy-client";
|
||||
import { checkOwnedStagingDir } from "../lib/staging-guard";
|
||||
import { resolveStateRoot } from "../lib/state-root";
|
||||
import { normalizeTranscriptConsent, type TranscriptConsent } from "./gstack-memory-ingest";
|
||||
import type { TranscriptConsent } from "./gstack-memory-ingest";
|
||||
import {
|
||||
describeTranscriptPolicy,
|
||||
isScoped,
|
||||
readTranscriptConsent,
|
||||
reposHash,
|
||||
type TranscriptPolicy,
|
||||
} from "../lib/transcript-consent";
|
||||
|
||||
// ── Types ──────────────────────────────────────────────────────────────────
|
||||
|
||||
@@ -107,11 +114,15 @@ interface StageResult {
|
||||
}
|
||||
|
||||
interface TranscriptConsentRecord {
|
||||
/** Normalized transcript_ingest_mode: recent | all | off | not-set | legacy | unrecognized. */
|
||||
/** Normalized transcript_ingest_mode: recent | all | new | off | not-set | legacy | unrecognized | repos-unreadable. */
|
||||
mode: string;
|
||||
window: "recent" | "all" | null;
|
||||
window: "recent" | "all" | "new" | null;
|
||||
/** Why transcripts were not ingested; absent when they were. */
|
||||
skip_reason?: string;
|
||||
/** The new@ cutoff in force; a changed cutoff restages. */
|
||||
cutoff?: string;
|
||||
/** Hash of the transcript_repos allowlist in force; a scope change restages. */
|
||||
repos_hash?: string;
|
||||
}
|
||||
|
||||
// ── Constants ──────────────────────────────────────────────────────────────
|
||||
@@ -304,16 +315,28 @@ export function transcriptConsentNotice(consent: TranscriptConsent, quiet: boole
|
||||
if (consent.affirmative) return null;
|
||||
const value = consent.value ?? "not set";
|
||||
if (override) {
|
||||
return `gbrain-sync: transcripts ingested because ${override} names transcript (transcript_ingest_mode=${value}). ${TRANSCRIPT_CHOICE_HINT}`;
|
||||
const scope = isScoped(consent) ? " The new@ cutoff and transcript_repos allowlist still apply." : "";
|
||||
return `gbrain-sync: transcripts ingested because ${override} names transcript (transcript_ingest_mode=${value}).${scope} ${TRANSCRIPT_CHOICE_HINT}`;
|
||||
}
|
||||
if (consent.reason === "off") return quiet ? null : "gbrain-sync: transcripts off (your choice)";
|
||||
if (consent.reason === "repos-unreadable") {
|
||||
return `gbrain-sync: transcripts skipped (transcript_ingest_mode=${value}): its +repos marker needs a transcript_repos allowlist, which is missing or empty. Other memory still syncs. ${TRANSCRIPT_CHOICE_HINT}`;
|
||||
}
|
||||
return `gbrain-sync: transcripts skipped (transcript_ingest_mode=${value}). Other memory still syncs. ${TRANSCRIPT_CHOICE_HINT}`;
|
||||
}
|
||||
|
||||
/** The consent record a memory stage stores; resume requires mode and window to match. */
|
||||
/** The consent in words for a consenting run (window or cutoff, repos, config roots). */
|
||||
export function transcriptConsentSummary(consent: TranscriptPolicy, quiet: boolean): string | null {
|
||||
if (!consent.affirmative || quiet) return null;
|
||||
return `gbrain-sync: transcripts: ${describeTranscriptPolicy(consent)}`;
|
||||
}
|
||||
|
||||
/** The consent record a memory stage stores; resume requires every field to match. */
|
||||
export function transcriptConsentRecord(consent: TranscriptConsent, override: boolean): TranscriptConsentRecord {
|
||||
const record: TranscriptConsentRecord = { mode: consent.reason, window: consent.window };
|
||||
if (!consent.affirmative && !override) record.skip_reason = consent.reason;
|
||||
if (consent.cutoff !== undefined) record.cutoff = consent.cutoff;
|
||||
if (consent.repos !== undefined) record.repos_hash = reposHash(consent.repos);
|
||||
return record;
|
||||
}
|
||||
|
||||
@@ -448,10 +471,12 @@ Options:
|
||||
except ${FEDERATED_CURATED_TYPES.join(",")} when a federated
|
||||
gstack source already indexes them, and except
|
||||
transcript unless transcript_ingest_mode is recent
|
||||
(last 90 days) or all (all history; --full passes
|
||||
--all-history). Without that consent each run prints a
|
||||
notice, even with --quiet. A list naming transcript
|
||||
overrides the mode for that run. See
|
||||
(last 90 days), all (all history; --full passes
|
||||
--all-history) or new@<UTC> (sessions started after
|
||||
it). Without that consent each run prints a notice,
|
||||
even with --quiet. A list naming transcript overrides
|
||||
the mode for that run, but a new@ cutoff and the
|
||||
transcript_repos allowlist still apply. See
|
||||
setup-gbrain/memory.md#transcripts.
|
||||
--help This text.
|
||||
|
||||
@@ -1470,12 +1495,12 @@ export function ensureGbrainSourceGitignored(root: string): void {
|
||||
}
|
||||
}
|
||||
|
||||
function printTranscriptNotice(args: CliArgs, consent: TranscriptConsent, selection: ConsentedSelection, why: string): void {
|
||||
const notice = transcriptConsentNotice(consent, args.quiet, selection.override ? why : null);
|
||||
function printTranscriptNotice(args: CliArgs, consent: TranscriptPolicy, selection: ConsentedSelection, why: string): void {
|
||||
const notice = transcriptConsentNotice(consent, args.quiet, selection.override ? why : null) ?? transcriptConsentSummary(consent, args.quiet);
|
||||
if (notice) console.error(notice);
|
||||
}
|
||||
|
||||
function runMemoryIngest(args: CliArgs, previous: SyncState, consent: TranscriptConsent): StageResult {
|
||||
function runMemoryIngest(args: CliArgs, previous: SyncState, consent: TranscriptPolicy): StageResult {
|
||||
const t0 = Date.now();
|
||||
|
||||
if (args.mode === "dry-run") {
|
||||
@@ -1518,14 +1543,19 @@ function runMemoryIngest(args: CliArgs, previous: SyncState, consent: Transcript
|
||||
// transcript window.
|
||||
const prevStage = previous.last_stages?.find((st) => st.name === "memory");
|
||||
const staged = prevStage?.memory_sources ?? [...MEMORY_INGEST_TYPES];
|
||||
const prevConsent = prevStage?.transcript_consent ?? { mode: "recent", window: "recent" };
|
||||
const prevConsent: TranscriptConsentRecord = prevStage?.transcript_consent ?? { mode: "recent", window: "recent" };
|
||||
if ([...staged].sort().join(",") !== [...selected].sort().join(",")) {
|
||||
console.error(
|
||||
`[sync:memory] memory source selection changed since the checkpointed run (${staged.join(",")} → ${selected.join(",")}); restaging from scratch.`,
|
||||
);
|
||||
resume = { kind: "no-checkpoint" };
|
||||
}
|
||||
if (prevConsent.mode !== consentRecord.mode || prevConsent.window !== consentRecord.window) {
|
||||
if (
|
||||
prevConsent.mode !== consentRecord.mode ||
|
||||
prevConsent.window !== consentRecord.window ||
|
||||
prevConsent.cutoff !== consentRecord.cutoff ||
|
||||
prevConsent.repos_hash !== consentRecord.repos_hash
|
||||
) {
|
||||
console.error(
|
||||
`gbrain-sync: transcript consent changed since the interrupted import (${prevConsent.mode} → ${consentRecord.mode}); restaging memory from scratch once.`,
|
||||
);
|
||||
@@ -1554,7 +1584,7 @@ function runMemoryIngest(args: CliArgs, previous: SyncState, consent: Transcript
|
||||
const ingestArgs = ["run", ingestPath];
|
||||
if (args.mode === "full") ingestArgs.push("--bulk");
|
||||
else ingestArgs.push("--incremental");
|
||||
if (args.mode === "full" && consent.window === "all") ingestArgs.push("--all-history");
|
||||
if (args.mode === "full" && consent.affirmative && consent.window !== "recent") ingestArgs.push("--all-history");
|
||||
if (args.quiet) ingestArgs.push("--quiet");
|
||||
if (selection.sources) ingestArgs.push("--sources", selection.sources.join(","));
|
||||
if (!args.quiet) console.error(`[sync:memory] sources: ${selected.join(",")} (${selection.why})`);
|
||||
@@ -1963,7 +1993,7 @@ async function main(): Promise<void> {
|
||||
const args = parseArgs();
|
||||
// Read once at sync start: a config change made while the sync runs never
|
||||
// adds transcripts to this run.
|
||||
const consent = args.noMemory ? null : normalizeTranscriptConsent();
|
||||
const consent = args.noMemory ? null : readTranscriptConsent();
|
||||
|
||||
if (!args.quiet) {
|
||||
const engine = detectEngineTier();
|
||||
|
||||
+143
-49
@@ -72,7 +72,15 @@ import { execGbrainText, spawnGbrainAsync } from "../lib/gbrain-exec";
|
||||
import { writeReceipt } from "../lib/egress-receipt";
|
||||
import { checkOwnedStagingDir, STAGING_MARKER } from "../lib/staging-guard";
|
||||
import { hasRepoPolicyStore, repoPolicyTierBatch } from "../lib/gbrain-repo-policy-client";
|
||||
import { resolveStateRoot, readConfigKeyWithRoot, type StateRootEnv } from "../lib/state-root";
|
||||
import { resolveStateRoot, mergedStateRoots, type StateRootEnv } from "../lib/state-root";
|
||||
import {
|
||||
cutoffExclusion,
|
||||
purgeStagedTranscripts,
|
||||
readTranscriptConsent,
|
||||
repoExclusion,
|
||||
type TranscriptConsent,
|
||||
type TranscriptPolicy,
|
||||
} from "../lib/transcript-consent";
|
||||
|
||||
// ── Types ──────────────────────────────────────────────────────────────────
|
||||
|
||||
@@ -155,6 +163,8 @@ interface ProbeReport {
|
||||
*/
|
||||
skipped_policy_deny: number;
|
||||
skipped_policy_readonly: number;
|
||||
/** Sessions a scoped consent (new@ cutoff, transcript_repos) kept out. */
|
||||
scope: TranscriptScopeCounts;
|
||||
estimate_minutes: number;
|
||||
}
|
||||
|
||||
@@ -174,6 +184,8 @@ interface BulkResult {
|
||||
failed: number;
|
||||
duration_ms: number;
|
||||
partial_pages: number;
|
||||
/** Sessions a scoped consent kept out, and transcript pages selected. */
|
||||
scope?: TranscriptScopeCounts;
|
||||
/**
|
||||
* D6: when set, indicates a process-level failure (gbrain CLI missing
|
||||
* or `gbrain import` crashed). Per-file errors (FILE_TOO_LARGE etc.)
|
||||
@@ -204,43 +216,39 @@ const ALL_TYPES: MemoryType[] = [
|
||||
|
||||
// ── Transcript consent ─────────────────────────────────────────────────────
|
||||
|
||||
export interface TranscriptConsent {
|
||||
/** True only for a stored `recent` or `all`. */
|
||||
affirmative: boolean;
|
||||
/** Transcript walk window when affirmative: `recent` = last 90 days, `all` = all history. */
|
||||
window: "recent" | "all" | null;
|
||||
/**
|
||||
* Normalized mode: the window when affirmative; otherwise why transcripts
|
||||
* are skipped. `legacy` covers the old gate letters (A-E) and `incremental`,
|
||||
* which predate the consent rule and need a new choice.
|
||||
*/
|
||||
reason: "recent" | "all" | "off" | "not-set" | "legacy" | "unrecognized";
|
||||
/** The stored value as read (null when the key is absent). */
|
||||
value: string | null;
|
||||
}
|
||||
export type { TranscriptConsent } from "../lib/transcript-consent";
|
||||
|
||||
let cachedConsent: TranscriptConsent | null = null;
|
||||
let cachedConsent: TranscriptPolicy | null = null;
|
||||
|
||||
/**
|
||||
* Read `transcript_ingest_mode` once per process (same parse as
|
||||
* `gstack-config has` + `get`) and decide whether transcripts may be ingested.
|
||||
* Only `recent` and `all` are consent. An absent key, `off`, a legacy value
|
||||
* and anything unrecognized are not. Pass `env` to bypass the per-process cache.
|
||||
* Read transcript consent once per process across every merged state root
|
||||
* (lib/transcript-consent.ts). `recent`, `all` and `new@<time>` are consent;
|
||||
* an absent key, `off`, a legacy value (A-E, incremental) and anything
|
||||
* unrecognized are not. A cutoff or allowlist applies even under an explicit
|
||||
* `--sources transcript`. Pass `env` to bypass the per-process cache.
|
||||
*/
|
||||
export function normalizeTranscriptConsent(env?: StateRootEnv): TranscriptConsent {
|
||||
function readIngestConsent(env?: StateRootEnv): TranscriptPolicy {
|
||||
if (!env && cachedConsent) return cachedConsent;
|
||||
const value = readConfigKeyWithRoot("transcript_ingest_mode", env ?? process.env).value;
|
||||
const v = (value ?? "").trim().toLowerCase();
|
||||
let consent: TranscriptConsent;
|
||||
if (value === null) consent = { affirmative: false, window: null, reason: "not-set", value };
|
||||
else if (v === "recent" || v === "all") consent = { affirmative: true, window: v, reason: v, value };
|
||||
else if (v === "off") consent = { affirmative: false, window: null, reason: "off", value };
|
||||
else if (/^[a-e]$/.test(v) || v === "incremental") consent = { affirmative: false, window: null, reason: "legacy", value };
|
||||
else consent = { affirmative: false, window: null, reason: "unrecognized", value };
|
||||
if (!env) cachedConsent = consent;
|
||||
const policy = readTranscriptConsent(mergedStateRoots(env ?? process.env));
|
||||
if (!env) cachedConsent = policy;
|
||||
return policy;
|
||||
}
|
||||
|
||||
/** The consent fields without the contributing roots. */
|
||||
export function normalizeTranscriptConsent(env?: StateRootEnv): TranscriptConsent {
|
||||
const { roots: _roots, ...consent } = readIngestConsent(env);
|
||||
return consent;
|
||||
}
|
||||
|
||||
/** Sessions a scoped consent kept out of this run, by reason. */
|
||||
export interface TranscriptScopeCounts {
|
||||
pre_cutoff: number;
|
||||
missing_start: number;
|
||||
not_allowlisted: number;
|
||||
/** Transcript pages this run selected for staging. */
|
||||
sessions_selected: number;
|
||||
}
|
||||
|
||||
// ── CLI ────────────────────────────────────────────────────────────────────
|
||||
|
||||
function printUsage(): void {
|
||||
@@ -258,8 +266,10 @@ Options:
|
||||
--all-history Walk transcripts older than 90 days too.
|
||||
--sources <list> Comma-separated subset: ${ALL_TYPES.join(",")}
|
||||
Default: every type, minus transcript unless
|
||||
transcript_ingest_mode is recent (90 days) or all
|
||||
(all history). A list naming transcript overrides it.
|
||||
transcript_ingest_mode is recent (90 days), all (all
|
||||
history) or new@<UTC> (sessions started after it).
|
||||
A list naming transcript overrides it; a new@ cutoff
|
||||
and transcript_repos still apply.
|
||||
--limit <N> Stop after N pages written (smoke testing).
|
||||
--no-write Skip gbrain put calls (still updates state file).
|
||||
Used by tests + dry runs without actual ingest.
|
||||
@@ -323,14 +333,15 @@ function parseArgs(): CliArgs {
|
||||
}
|
||||
}
|
||||
|
||||
const consent = normalizeTranscriptConsent();
|
||||
const consent = readIngestConsent();
|
||||
if (!sourcesExplicit && !consent.affirmative) {
|
||||
sources.delete("transcript");
|
||||
if (!quiet) {
|
||||
console.error(`gstack-memory-ingest: transcripts skipped (transcript_ingest_mode=${consent.value ?? "not set"}); set it to recent or all, or pass --sources transcript.`);
|
||||
const why = consent.reason === "repos-unreadable" ? "; its +repos marker needs a transcript_repos allowlist" : "";
|
||||
console.error(`gstack-memory-ingest: transcripts skipped (transcript_ingest_mode=${consent.value ?? "not set"}${why}); set it to recent, all or new@<UTC time>, or pass --sources transcript.`);
|
||||
}
|
||||
}
|
||||
if (consent.window === "all") allHistory = true;
|
||||
if (consent.affirmative && consent.window !== "recent") allHistory = true;
|
||||
|
||||
return { mode, quiet, benchmark, includeUnattributed, allHistory, sources, limit, noWrite, scanSecrets };
|
||||
}
|
||||
@@ -425,17 +436,45 @@ interface WalkContext {
|
||||
args: CliArgs;
|
||||
state: IngestState;
|
||||
windowStartMs: number; // ignore files older than this unless --all-history
|
||||
consent: TranscriptConsent;
|
||||
scope: TranscriptScopeCounts;
|
||||
}
|
||||
|
||||
function makeWalkContext(args: CliArgs, state: IngestState): WalkContext {
|
||||
export function emptyScopeCounts(): TranscriptScopeCounts {
|
||||
return { pre_cutoff: 0, missing_start: 0, not_allowlisted: 0, sessions_selected: 0 };
|
||||
}
|
||||
|
||||
function makeWalkContext(args: CliArgs, state: IngestState, scope: TranscriptScopeCounts = emptyScopeCounts()): WalkContext {
|
||||
const ninetyDaysAgoMs = Date.now() - 90 * 24 * 60 * 60 * 1000;
|
||||
return {
|
||||
args,
|
||||
state,
|
||||
windowStartMs: args.allHistory ? 0 : ninetyDaysAgoMs,
|
||||
consent: readIngestConsent(),
|
||||
scope,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* The `new@` cutoff, applied in both parsers' walks on each session's first
|
||||
* record timestamp (read from the file prefix, never mtime): a session that
|
||||
* started before the cutoff and was appended later stays out, and a session
|
||||
* with no start timestamp is excluded (fail-closed).
|
||||
*/
|
||||
function sessionPassesCutoff(ctx: WalkContext, path: string): boolean {
|
||||
const why = cutoffExclusion(ctx.consent, ctx.consent.cutoff === undefined ? null : transcriptPrefix(path).startedAt);
|
||||
if (why === "pre-cutoff") ctx.scope.pre_cutoff++;
|
||||
else if (why === "missing-start") ctx.scope.missing_start++;
|
||||
return why === null;
|
||||
}
|
||||
|
||||
/** The transcript_repos allowlist, applied after the attribution gate. */
|
||||
function sessionRepoAllowed(ctx: WalkContext, remote: string | undefined): boolean {
|
||||
if (repoExclusion(ctx.consent, remote) === null) return true;
|
||||
ctx.scope.not_allowlisted++;
|
||||
return false;
|
||||
}
|
||||
|
||||
function* walkClaudeCodeProjects(ctx: WalkContext): Generator<{ path: string; type: MemoryType }> {
|
||||
const root = join(HOME, ".claude", "projects");
|
||||
if (!existsSync(root)) return;
|
||||
@@ -462,6 +501,7 @@ function* walkClaudeCodeProjects(ctx: WalkContext): Generator<{ path: string; ty
|
||||
} catch {
|
||||
continue;
|
||||
}
|
||||
if (!sessionPassesCutoff(ctx, fullPath)) continue;
|
||||
yield { path: fullPath, type: "transcript" };
|
||||
}
|
||||
}
|
||||
@@ -495,6 +535,7 @@ function* walkCodexSessions(ctx: WalkContext): Generator<{ path: string; type: M
|
||||
}
|
||||
}
|
||||
for (const path of recurse(root, 0)) {
|
||||
if (!sessionPassesCutoff(ctx, path)) continue;
|
||||
yield { path, type: "transcript" };
|
||||
}
|
||||
}
|
||||
@@ -801,6 +842,12 @@ function dateOnly(ts: string | undefined): string {
|
||||
}
|
||||
}
|
||||
|
||||
/** A timestamp as ISO-8601 UTC, or "" when absent or unparseable. */
|
||||
function isoOrEmpty(ts: string | undefined): string {
|
||||
const ms = ts ? Date.parse(ts) : NaN;
|
||||
return Number.isFinite(ms) ? new Date(ms).toISOString() : "";
|
||||
}
|
||||
|
||||
export function buildTranscriptPage(path: string, session: ParsedSession): PageRecord {
|
||||
const remote = resolveGitRemote(session.cwd);
|
||||
const slug_repo = repoSlug(remote);
|
||||
@@ -826,6 +873,7 @@ export function buildTranscriptPage(path: string, session: ParsedSession): PageR
|
||||
`cwd: ${session.cwd || ""}`,
|
||||
`git_remote: ${remote || "_unattributed"}`,
|
||||
`start_time: ${session.start_time || ""}`,
|
||||
`session_started_at: ${isoOrEmpty(session.start_time)}`,
|
||||
`end_time: ${session.end_time || ""}`,
|
||||
`message_count: ${session.message_count}`,
|
||||
`tool_calls: ${session.tool_calls}`,
|
||||
@@ -1182,7 +1230,7 @@ export function readNewFailures(
|
||||
|
||||
/**
|
||||
* The ONE attribution gate (#2394): a transcript is attributable iff its cwd
|
||||
* resolves to a git remote. Both probeMode (via transcriptCwdFromPrefix +
|
||||
* resolves to a git remote. Both probeMode (via transcriptPrefix +
|
||||
* resolveGitRemote — the same memoized resolver) and preparePages route
|
||||
* through THIS logic, so the two stages' post-attribution counts are
|
||||
* structurally identical — the parity the probe report promises.
|
||||
@@ -1218,10 +1266,15 @@ const TRANSCRIPT_PROBE_MAX_BYTES = 256 * 1024;
|
||||
* - Claude Code cwd comes from the first record that carries one;
|
||||
* - unparseable lines are skipped (the truncated-tail case included).
|
||||
*
|
||||
* The session start (the `new@` cutoff input) follows the same rules: Codex
|
||||
* takes the first record's timestamp (or payload.timestamp), Claude Code the
|
||||
* first record that carries a timestamp. A start beyond the prefix reads as
|
||||
* missing, which a cutoff treats as excluded.
|
||||
*
|
||||
* Non-transcript types (artifacts) always pass — the attribution filter in
|
||||
* preparePages only applies to transcripts (#2394).
|
||||
*/
|
||||
function transcriptCwdFromPrefix(path: string): string {
|
||||
export function transcriptPrefix(path: string): { cwd: string; startedAt: string } {
|
||||
// Chunked read until the prefix contains at least one COMPLETE record
|
||||
// (newline), up to the hard cap — a first record larger than one chunk
|
||||
// (giant pasted prompt) must not truncate mid-JSON and mis-classify a
|
||||
@@ -1246,12 +1299,12 @@ function transcriptCwdFromPrefix(path: string): string {
|
||||
closeSync(fd);
|
||||
}
|
||||
} catch {
|
||||
return "";
|
||||
return { cwd: "", startedAt: "" };
|
||||
}
|
||||
const lines = raw.split("\n").filter((l) => l.trim().length > 0);
|
||||
if (lines.length === 0) return "";
|
||||
|
||||
let cwd = "";
|
||||
let startedAt = "";
|
||||
let sawFirstParseable = false;
|
||||
for (const line of lines) {
|
||||
let rec: any;
|
||||
@@ -1265,18 +1318,18 @@ function transcriptCwdFromPrefix(path: string): string {
|
||||
// Format detection mirrors parseTranscriptJsonl's `first` record check.
|
||||
const isCodex = rec?.type === "session_meta" || rec?.payload?.id != null;
|
||||
if (isCodex) {
|
||||
// Codex: cwd comes from the session_meta FIRST record only.
|
||||
// Codex: cwd and start come from the session_meta FIRST record only.
|
||||
cwd = rec.payload?.cwd || rec.cwd || "";
|
||||
startedAt = rec.timestamp || rec.payload?.timestamp || "";
|
||||
break;
|
||||
}
|
||||
}
|
||||
// Claude Code: first record with a cwd wins (the first record included).
|
||||
if (rec?.cwd) {
|
||||
cwd = rec.cwd;
|
||||
break;
|
||||
}
|
||||
// Claude Code: the first record with a cwd and the first with a timestamp.
|
||||
if (!cwd && rec?.cwd) cwd = rec.cwd;
|
||||
if (!startedAt && rec?.timestamp) startedAt = rec.timestamp;
|
||||
if (cwd && startedAt) break;
|
||||
}
|
||||
return cwd;
|
||||
return { cwd, startedAt: typeof startedAt === "string" ? startedAt : "" };
|
||||
}
|
||||
|
||||
async function probeMode(args: CliArgs): Promise<ProbeReport> {
|
||||
@@ -1314,12 +1367,13 @@ async function probeMode(args: CliArgs): Promise<ProbeReport> {
|
||||
// skip transcripts with no resolvable git remote unless --include-unattributed.
|
||||
let remote = "";
|
||||
if (type === "transcript") {
|
||||
const cwd = transcriptCwdFromPrefix(path);
|
||||
const { cwd } = transcriptPrefix(path);
|
||||
remote = cwd ? resolveGitRemote(cwd) : "";
|
||||
if (!args.includeUnattributed && remote === "") {
|
||||
skippedUnattributed++;
|
||||
continue;
|
||||
}
|
||||
if (!sessionRepoAllowed(ctx, remote || "_unattributed")) continue;
|
||||
}
|
||||
candidates.push({ path, type, remote });
|
||||
}
|
||||
@@ -1380,6 +1434,7 @@ async function probeMode(args: CliArgs): Promise<ProbeReport> {
|
||||
skipped_unattributed: skippedUnattributed,
|
||||
skipped_policy_deny: skippedPolicyDeny,
|
||||
skipped_policy_readonly: skippedPolicyReadonly,
|
||||
scope: ctx.scope,
|
||||
estimate_minutes: estimateMinutes,
|
||||
};
|
||||
}
|
||||
@@ -1544,6 +1599,7 @@ function preparePages(
|
||||
continue;
|
||||
}
|
||||
page = buildTranscriptPage(path, session);
|
||||
if (!sessionRepoAllowed(ctx, page.git_remote)) continue;
|
||||
} else {
|
||||
page = buildArtifactPage(path, type, raw);
|
||||
}
|
||||
@@ -1663,6 +1719,7 @@ function preparePages(
|
||||
// whole batch every run (#2724: 887 staged → 0 ingested). Disambiguate
|
||||
// before staging, consulting state so assignments hold across runs.
|
||||
disambiguateSlugs(finalPrepared, state);
|
||||
ctx.scope.sessions_selected = finalPrepared.filter((p) => p.type === "transcript").length;
|
||||
|
||||
// Derived from the FINAL set: partial counts must describe pages that are
|
||||
// actually eligible and within the limit, not the whole scanned corpus.
|
||||
@@ -2015,9 +2072,20 @@ function runGbrainImportOnce(
|
||||
}
|
||||
|
||||
async function ingestPass(args: CliArgs): Promise<BulkResult> {
|
||||
// Tightening a scoped consent removes unpublished staged pages outside it and
|
||||
// their fingerprints, BEFORE the state loads, so a later widening re-stages.
|
||||
if (!args.noWrite) {
|
||||
const purged = purgeStagedTranscripts(GSTACK_HOME, readIngestConsent());
|
||||
if (purged > 0) console.error(`[memory-ingest] removed ${purged} staged transcript pages outside the new scope`);
|
||||
}
|
||||
const scope = emptyScopeCounts();
|
||||
return { ...(await ingestPassScoped(args, scope)), scope };
|
||||
}
|
||||
|
||||
async function ingestPassScoped(args: CliArgs, scope: TranscriptScopeCounts): Promise<BulkResult> {
|
||||
const t0 = Date.now();
|
||||
const state = loadState();
|
||||
const ctx = makeWalkContext(args, state);
|
||||
const ctx = makeWalkContext(args, state, scope);
|
||||
const remoteHttpMode = isRemoteHttpMcpMode();
|
||||
const resumeDir = process.env.GSTACK_INGEST_RESUME_DIR;
|
||||
const resuming = !args.noWrite && !remoteHttpMode
|
||||
@@ -2651,6 +2719,7 @@ function printProbeReport(r: ProbeReport, json: boolean): void {
|
||||
if (r.skipped_policy_readonly > 0) {
|
||||
console.log(`Skipped (policy read-only): ${r.skipped_policy_readonly} (remote tier is read-only; transcript ingest writes pages)`);
|
||||
}
|
||||
for (const line of scopeLines(r.scope)) console.log(line);
|
||||
console.log("By type:");
|
||||
for (const [t, v] of Object.entries(r.by_type)) {
|
||||
if (v.count > 0) {
|
||||
@@ -2660,6 +2729,26 @@ function printProbeReport(r: ProbeReport, json: boolean): void {
|
||||
console.log(`\nEstimate: ~${r.estimate_minutes} min for full --bulk pass.`);
|
||||
}
|
||||
|
||||
/** Exclusion counts by reason; nothing when consent has no cutoff or allowlist. */
|
||||
function scopeLines(scope: TranscriptScopeCounts | undefined, indent = ""): string[] {
|
||||
if (!scope) return [];
|
||||
const lines: string[] = [];
|
||||
if (scope.pre_cutoff > 0) lines.push(`${indent}Excluded (pre-cutoff): ${scope.pre_cutoff} (session started before the new@ cutoff)`);
|
||||
if (scope.missing_start > 0) lines.push(`${indent}Excluded (missing start): ${scope.missing_start} (no start timestamp; excluded under new@)`);
|
||||
if (scope.not_allowlisted > 0) lines.push(`${indent}Excluded (not allowlisted): ${scope.not_allowlisted} (repo not in transcript_repos)`);
|
||||
return lines;
|
||||
}
|
||||
|
||||
/**
|
||||
* Under a new@ cutoff, a run that selects no session says so, so "nothing
|
||||
* happened" reads as expected rather than broken.
|
||||
*/
|
||||
function zeroSessionsNotice(r: BulkResult, args: CliArgs): string | null {
|
||||
const consent = readIngestConsent();
|
||||
if (!consent.cutoff || !args.sources.has("transcript") || !r.scope || r.scope.sessions_selected > 0) return null;
|
||||
return `[memory-ingest] 0 transcript sessions ingested under new@${consent.cutoff}: ${r.scope.pre_cutoff} started before the cutoff; new sessions will appear on the next sync`;
|
||||
}
|
||||
|
||||
function printBulkResult(r: BulkResult, args: CliArgs): void {
|
||||
console.log(`\nIngest pass complete (${args.mode}):`);
|
||||
console.log(` written: ${r.written}`);
|
||||
@@ -2673,6 +2762,7 @@ function printBulkResult(r: BulkResult, args: CliArgs): void {
|
||||
if (r.skipped_policy_deny > 0) {
|
||||
console.log(` skipped (policy deny): ${r.skipped_policy_deny} (change with: gstack-gbrain-repo-policy set <remote> read-write)`);
|
||||
}
|
||||
for (const line of scopeLines(r.scope, " ")) console.log(line);
|
||||
console.log(` failed: ${r.failed}`);
|
||||
console.log(` duration: ${(r.duration_ms / 1000).toFixed(1)}s`);
|
||||
if (args.benchmark) {
|
||||
@@ -2706,6 +2796,8 @@ async function main(): Promise<void> {
|
||||
if (result.written > 0 || result.failed > 0) {
|
||||
console.error(`[memory-ingest] ${result.written} written, ${result.failed} failed in ${dt}ms`);
|
||||
}
|
||||
const zero = zeroSessionsNotice(result, args);
|
||||
if (zero) console.error(zero);
|
||||
// D6: system_error → process-level failure; orchestrator sees ERR.
|
||||
// Per-file errors do NOT exit non-zero.
|
||||
if (result.system_error) process.exit(1);
|
||||
@@ -2714,6 +2806,8 @@ async function main(): Promise<void> {
|
||||
|
||||
const result = await ingestPass(args);
|
||||
printBulkResult(result, args);
|
||||
const zero = zeroSessionsNotice(result, args);
|
||||
if (zero) console.error(zero);
|
||||
if (result.system_error) process.exit(1);
|
||||
}
|
||||
|
||||
|
||||
+1
-1
@@ -55,7 +55,7 @@ export const MERGED_CONFIG_KEYS: Record<string, readonly string[]> = {
|
||||
};
|
||||
|
||||
/** Same parse as `gstack-config get`: last `^key:` line wins, value trimmed, empty = unset. */
|
||||
function readKeyFromRoot(root: string, key: string): string | null {
|
||||
export function readKeyFromRoot(root: string, key: string): string | null {
|
||||
let yaml: string;
|
||||
try {
|
||||
yaml = fs.readFileSync(`${root}/config.yaml`, 'utf-8');
|
||||
|
||||
@@ -0,0 +1,425 @@
|
||||
/**
|
||||
* transcript-consent — the one reader of transcript consent. Reads
|
||||
* `transcript_ingest_mode` and `transcript_repos` together from every root in
|
||||
* mergedStateRoots() and returns one policy that gstack-memory-ingest,
|
||||
* gstack-gbrain-sync and gstack-brain-sync (through this file's CLI) share.
|
||||
*
|
||||
* Value grammar: `<base>[@<YYYY-MM-DDTHH:MM:SSZ>][+repos]`, base recent | all |
|
||||
* off | new. `@<time>` is required for `new` and invalid otherwise; `off+repos`
|
||||
* is invalid. Readers parse case-insensitively. `+repos` marks a value whose
|
||||
* scope lives in `transcript_repos`, so a gstack that predates the allowlist
|
||||
* reads it as unrecognized and ingests nothing.
|
||||
*
|
||||
* Across roots: a non-consenting value in any root wins (off, legacy,
|
||||
* unrecognized). The resolved root must hold the consent itself. Time
|
||||
* constraints stack (a 90-day window from any `recent`, the latest `new@`
|
||||
* cutoff), and the repo set is the intersection of every root's allowlist.
|
||||
*
|
||||
* CLI (bin/gstack-brain-sync):
|
||||
* --check exit 0 iff transcript pages may sync at all
|
||||
* --describe print the effective consent in words
|
||||
* --purge remove unpublished staged pages outside a scoped consent
|
||||
* --guard-unpublished drop excluded pages from unpublished sync commits;
|
||||
* exit 3 when the push must not happen
|
||||
* Docs: setup-gbrain/memory.md#transcripts.
|
||||
*/
|
||||
import * as fs from 'node:fs';
|
||||
import * as path from 'node:path';
|
||||
import { spawnSync } from 'node:child_process';
|
||||
import { createHash } from 'node:crypto';
|
||||
import { canonicalizeRemote } from './gstack-memory-helpers';
|
||||
import { mergedStateRoots, readKeyFromRoot, resolveStateRoot } from './state-root';
|
||||
|
||||
export const MODE_KEY = 'transcript_ingest_mode';
|
||||
export const REPOS_KEY = 'transcript_repos';
|
||||
const BRAIN_SYNC_AUTHOR = 'gstack-brain-sync';
|
||||
const MODE_RE = /^(recent|all|off|new)(?:@(\d{4}-\d{2}-\d{2}t\d{2}:\d{2}:\d{2}z))?(\+repos)?$/i;
|
||||
const ROLLING_DAYS = 90;
|
||||
|
||||
export type TranscriptBase = 'recent' | 'all' | 'off' | 'new';
|
||||
|
||||
export interface TranscriptModeValue {
|
||||
base: TranscriptBase;
|
||||
/** ISO-8601 UTC cutoff (`new` only). */
|
||||
cutoff: string | null;
|
||||
/** The value carries the `+repos` allowlist marker. */
|
||||
repos: boolean;
|
||||
}
|
||||
|
||||
/** True when `s` is exactly `YYYY-MM-DDTHH:MM:SSZ` and names a real instant. */
|
||||
export function isUtcSecond(s: string): boolean {
|
||||
const ms = Date.parse(s);
|
||||
return Number.isFinite(ms) && new Date(ms).toISOString().replace('.000Z', 'Z') === s;
|
||||
}
|
||||
|
||||
/** Parse one stored value (case-insensitive). null = not valid grammar. */
|
||||
export function parseTranscriptMode(raw: string): TranscriptModeValue | null {
|
||||
const m = MODE_RE.exec(raw.trim());
|
||||
if (!m) return null;
|
||||
const base = m[1].toLowerCase() as TranscriptBase;
|
||||
const cutoff = m[2] ? m[2].toUpperCase() : null;
|
||||
if ((base === 'new') !== (cutoff !== null)) return null;
|
||||
if (cutoff !== null && !isUtcSecond(cutoff)) return null;
|
||||
if (base === 'off' && m[3]) return null;
|
||||
return { base, cutoff, repos: m[3] !== undefined };
|
||||
}
|
||||
|
||||
export type TranscriptConsentReason =
|
||||
| 'recent'
|
||||
| 'all'
|
||||
| 'new'
|
||||
| 'off'
|
||||
| 'not-set'
|
||||
| 'legacy'
|
||||
| 'unrecognized'
|
||||
| 'repos-unreadable';
|
||||
|
||||
export interface TranscriptConsent {
|
||||
/** Every root consents; transcripts may be walked without an override. */
|
||||
affirmative: boolean;
|
||||
/** recent = rolling 90 days (mtime, as before), new = cutoff only, all = no time limit. */
|
||||
window: 'recent' | 'all' | 'new' | null;
|
||||
reason: TranscriptConsentReason;
|
||||
/** The resolved root's stored value (null when absent). */
|
||||
value: string | null;
|
||||
/** Latest `new@` cutoff across roots. Applies even under an explicit override. */
|
||||
cutoff?: string;
|
||||
/** Canonical remotes allowed across roots ([] = none). Applies even under an override. */
|
||||
repos?: string[];
|
||||
}
|
||||
|
||||
export interface TranscriptPolicy extends TranscriptConsent {
|
||||
/** Roots whose config.yaml sets either key. */
|
||||
roots: string[];
|
||||
}
|
||||
|
||||
export type TranscriptExclusion = 'pre-cutoff' | 'missing-start' | 'not-allowlisted';
|
||||
|
||||
export function parseRepoList(raw: string): string[] {
|
||||
const out: string[] = [];
|
||||
for (const part of raw.split(',')) {
|
||||
const remote = canonicalizeRemote(part);
|
||||
if (remote && !out.includes(remote)) out.push(remote);
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
/** Read the effective transcript consent across `roots` (resolved root first). */
|
||||
export function readTranscriptConsent(roots: string[] = mergedStateRoots()): TranscriptPolicy {
|
||||
const resolvedValue = roots.length > 0 ? readKeyFromRoot(roots[0], MODE_KEY) : null;
|
||||
let reason: TranscriptConsentReason | null = resolvedValue === null ? 'not-set' : null;
|
||||
let rolling = false;
|
||||
let cutoff: string | undefined;
|
||||
let repos: string[] | undefined;
|
||||
const contributing: string[] = [];
|
||||
for (const root of roots) {
|
||||
const raw = readKeyFromRoot(root, MODE_KEY);
|
||||
const list = readKeyFromRoot(root, REPOS_KEY);
|
||||
if (raw === null && list === null) continue;
|
||||
contributing.push(root);
|
||||
let marker = false;
|
||||
if (raw !== null) {
|
||||
const v = raw.trim().toLowerCase();
|
||||
const parsed = parseTranscriptMode(v);
|
||||
let rootReason: TranscriptConsentReason | null = null;
|
||||
if (!parsed) rootReason = /^[a-e]$/.test(v) || v === 'incremental' ? 'legacy' : 'unrecognized';
|
||||
else if (parsed.base === 'off') rootReason = 'off';
|
||||
else {
|
||||
marker = parsed.repos;
|
||||
if (parsed.base === 'recent') rolling = true;
|
||||
if (parsed.cutoff && (!cutoff || parsed.cutoff > cutoff)) cutoff = parsed.cutoff;
|
||||
}
|
||||
reason ??= rootReason;
|
||||
}
|
||||
const entries = list === null ? [] : parseRepoList(list);
|
||||
if (entries.length > 0) repos = repos ? repos.filter((r) => entries.includes(r)) : entries;
|
||||
else if (marker) {
|
||||
repos = [];
|
||||
reason ??= 'repos-unreadable';
|
||||
}
|
||||
}
|
||||
const affirmative = reason === null;
|
||||
const window = !affirmative ? null : rolling ? 'recent' : cutoff ? 'new' : 'all';
|
||||
const policy: TranscriptPolicy = {
|
||||
affirmative,
|
||||
window,
|
||||
reason: reason ?? (window as TranscriptConsentReason),
|
||||
value: resolvedValue,
|
||||
roots: contributing,
|
||||
};
|
||||
if (cutoff !== undefined) policy.cutoff = cutoff;
|
||||
if (repos !== undefined) policy.repos = repos;
|
||||
return policy;
|
||||
}
|
||||
|
||||
/** A cutoff or an allowlist narrows consent to some sessions. */
|
||||
export function isScoped(c: TranscriptConsent): boolean {
|
||||
return c.cutoff !== undefined || c.repos !== undefined;
|
||||
}
|
||||
|
||||
/** Transcript pages may sync at all: consent, and a repo set that is not empty. */
|
||||
export function allowsTranscripts(c: TranscriptConsent): boolean {
|
||||
return c.affirmative && !(c.repos !== undefined && c.repos.length === 0);
|
||||
}
|
||||
|
||||
/** Short stable id of the allowlist, recorded with a memory stage so a scope change restages. */
|
||||
export function reposHash(repos: string[]): string {
|
||||
return createHash('sha256').update([...repos].sort().join('\n')).digest('hex').slice(0, 12);
|
||||
}
|
||||
|
||||
/** The `new@` cutoff check on a session's first record timestamp (never mtime). */
|
||||
export function cutoffExclusion(c: TranscriptConsent, startedAt: string | null | undefined): 'pre-cutoff' | 'missing-start' | null {
|
||||
if (c.cutoff === undefined) return null;
|
||||
const ms = startedAt ? Date.parse(startedAt) : NaN;
|
||||
if (!Number.isFinite(ms)) return 'missing-start';
|
||||
return ms < Date.parse(c.cutoff) ? 'pre-cutoff' : null;
|
||||
}
|
||||
|
||||
/** The allowlist check on a session's canonical git remote. */
|
||||
export function repoExclusion(c: TranscriptConsent, remote: string | null | undefined): 'not-allowlisted' | null {
|
||||
if (c.repos === undefined) return null;
|
||||
const canonical = canonicalizeRemote(remote ?? '');
|
||||
return canonical && c.repos.includes(canonical) ? null : 'not-allowlisted';
|
||||
}
|
||||
|
||||
/** Why one session is outside the scope, or null when it is inside. Consent itself is checked by callers. */
|
||||
export function sessionExclusion(
|
||||
c: TranscriptConsent,
|
||||
session: { startedAt?: string | null; remote?: string | null },
|
||||
): TranscriptExclusion | null {
|
||||
return cutoffExclusion(c, session.startedAt) ?? repoExclusion(c, session.remote);
|
||||
}
|
||||
|
||||
function formatCutoff(cutoff: string): string {
|
||||
return `${cutoff.slice(0, 10)} ${cutoff.slice(11, 19)} UTC`;
|
||||
}
|
||||
|
||||
/** The effective consent in words: window or cutoff, up to 3 repos, contributing roots. */
|
||||
export function describeTranscriptPolicy(p: TranscriptPolicy): string {
|
||||
let time: string;
|
||||
if (p.window === 'recent') time = `sessions from the last ${ROLLING_DAYS} days${p.cutoff ? ` that started after ${formatCutoff(p.cutoff)}` : ''}`;
|
||||
else if (p.cutoff) time = `sessions that started after ${formatCutoff(p.cutoff)}`;
|
||||
else time = 'all history';
|
||||
let scope: string;
|
||||
if (p.repos === undefined) scope = 'every repo your repo policy allows';
|
||||
else if (p.repos.length === 0) scope = 'no repo (the repo allowlists share no repo, or one is unreadable)';
|
||||
else {
|
||||
const shown = p.repos.slice(0, 3).join(', ');
|
||||
scope = `${p.repos.length === 1 ? 'repo' : 'repos'} ${shown}${p.repos.length > 3 ? ` +${p.repos.length - 3} more` : ''}`;
|
||||
}
|
||||
const roots = p.roots.length > 0 ? `; config: ${p.roots.join(', ')}` : '';
|
||||
if (!p.affirmative) return `not consented (${p.reason}); scope if overridden: ${time}; ${scope}${roots}`;
|
||||
return `${time}; ${scope}${roots}`;
|
||||
}
|
||||
|
||||
// ── Staged pages ────────────────────────────────────────────────────────────
|
||||
|
||||
/** Frontmatter fields of a staged page (flat `key: value` lines). */
|
||||
export function pageFrontmatter(text: string): Record<string, string> {
|
||||
const out: Record<string, string> = {};
|
||||
if (!text.startsWith('---\n')) return out;
|
||||
const end = text.indexOf('\n---', 4);
|
||||
if (end < 0) return out;
|
||||
for (const line of text.slice(4, end).split('\n')) {
|
||||
const m = /^([a-z_]+):\s?(.*)$/.exec(line);
|
||||
if (m) out[m[1]] = m[2].trim();
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
/**
|
||||
* Why a staged page under transcripts/ must not sync, or null. Pages typed as
|
||||
* another memory type (artifacts staged in a remote-http run dir) are never
|
||||
* excluded here. Without consent every transcript page is excluded.
|
||||
*/
|
||||
export function pageExclusion(c: TranscriptConsent, text: string): TranscriptExclusion | 'not-consented' | null {
|
||||
const fm = pageFrontmatter(text);
|
||||
if (fm.type && fm.type !== 'transcript') return null;
|
||||
if (!c.affirmative) return 'not-consented';
|
||||
return sessionExclusion(c, { startedAt: fm.session_started_at || fm.start_time, remote: fm.git_remote });
|
||||
}
|
||||
|
||||
function git(home: string, args: string[]): { ok: boolean; out: string } {
|
||||
const r = spawnSync('git', ['-C', home, ...args], { encoding: 'utf-8', timeout: 30_000, maxBuffer: 64 * 1024 * 1024 });
|
||||
return { ok: r.status === 0, out: r.stdout || '' };
|
||||
}
|
||||
|
||||
/** origin/<branch> for the checked-out branch, or null (no repo, detached, never pushed). */
|
||||
function upstreamRef(home: string): string | null {
|
||||
if (!fs.existsSync(path.join(home, '.git'))) return null;
|
||||
const branch = git(home, ['rev-parse', '--abbrev-ref', 'HEAD']).out.trim();
|
||||
if (!branch || branch === 'HEAD') return null;
|
||||
const ref = `origin/${branch}`;
|
||||
return git(home, ['rev-parse', '--verify', '--quiet', ref]).ok ? ref : null;
|
||||
}
|
||||
|
||||
function walkMarkdown(dir: string, out: string[] = []): string[] {
|
||||
let entries: fs.Dirent[];
|
||||
try {
|
||||
entries = fs.readdirSync(dir, { withFileTypes: true });
|
||||
} catch {
|
||||
return out;
|
||||
}
|
||||
for (const e of entries) {
|
||||
const full = path.join(dir, e.name);
|
||||
if (e.isDirectory()) walkMarkdown(full, out);
|
||||
else if (e.isFile() && e.name.endsWith('.md')) out.push(full);
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
/** Drop ingest fingerprints so a later widening re-stages these sources. */
|
||||
function clearFingerprints(home: string, sourcePaths: string[]): void {
|
||||
if (sourcePaths.length === 0) return;
|
||||
const statePath = path.join(home, '.transcript-ingest-state.json');
|
||||
let state: { sessions?: Record<string, unknown> };
|
||||
try {
|
||||
state = JSON.parse(fs.readFileSync(statePath, 'utf-8'));
|
||||
} catch {
|
||||
return;
|
||||
}
|
||||
if (!state.sessions) return;
|
||||
let changed = false;
|
||||
for (const p of sourcePaths) {
|
||||
if (p in state.sessions) {
|
||||
delete state.sessions[p];
|
||||
changed = true;
|
||||
}
|
||||
}
|
||||
if (!changed) return;
|
||||
const tmp = `${statePath}.tmp.${process.pid}`;
|
||||
fs.writeFileSync(tmp, JSON.stringify(state, null, 2), 'utf-8');
|
||||
fs.renameSync(tmp, statePath);
|
||||
}
|
||||
|
||||
/**
|
||||
* Remove staged transcript pages that are not yet published and fall outside
|
||||
* a scoped consent (cutoff or allowlist). Without consent pages are kept, as
|
||||
* before: they wait in the queue until the user consents. Published pages are
|
||||
* left alone; removing them from the brain is a separate operation.
|
||||
*/
|
||||
export function purgeStagedTranscripts(home: string, c: TranscriptConsent): number {
|
||||
if (!c.affirmative || !isScoped(c)) return 0;
|
||||
const dir = path.join(home, 'transcripts');
|
||||
if (!fs.existsSync(dir)) return 0;
|
||||
const ref = upstreamRef(home);
|
||||
const published = new Set(
|
||||
ref ? git(home, ['ls-tree', '-r', '--name-only', ref, '--', 'transcripts']).out.split('\n').filter(Boolean) : [],
|
||||
);
|
||||
const sources: string[] = [];
|
||||
let removed = 0;
|
||||
for (const file of walkMarkdown(dir)) {
|
||||
const rel = path.relative(home, file).split(path.sep).join('/');
|
||||
if (published.has(rel)) continue;
|
||||
let text: string;
|
||||
try {
|
||||
text = fs.readFileSync(file, 'utf-8');
|
||||
} catch {
|
||||
continue;
|
||||
}
|
||||
if (pageExclusion(c, text) === null) continue;
|
||||
fs.rmSync(file, { force: true });
|
||||
removed++;
|
||||
const source = pageFrontmatter(text).source_path;
|
||||
if (source) sources.push(source);
|
||||
}
|
||||
clearFingerprints(home, sources);
|
||||
return removed;
|
||||
}
|
||||
|
||||
/** Queue a path again (same record shape as bin/gstack-brain-enqueue). */
|
||||
function enqueue(home: string, rel: string, seq: number): void {
|
||||
const spool = path.join(home, '.brain-queue.d');
|
||||
fs.mkdirSync(spool, { recursive: true });
|
||||
const tmp = path.join(spool, `.tmp-${process.pid}-g${seq}`);
|
||||
fs.writeFileSync(tmp, JSON.stringify({ file: rel, ts: new Date().toISOString().replace(/\.\d{3}Z$/, 'Z') }) + '\n');
|
||||
fs.renameSync(tmp, path.join(spool, `${Math.floor(Date.now() / 1000)}-${process.pid}-g${seq}.json`));
|
||||
}
|
||||
|
||||
export interface GuardResult {
|
||||
status: 'clean' | 'rewritten' | 'blocked';
|
||||
removed: string[];
|
||||
message?: string;
|
||||
}
|
||||
|
||||
/**
|
||||
* Before a push: scan the unpublished range for transcript pages outside the
|
||||
* current consent. When some exist and every unpublished commit is
|
||||
* gstack-brain-sync's own, soft-reset to the upstream, unstage the excluded
|
||||
* pages and recommit, so nothing excluded is ever pushed. Otherwise block.
|
||||
* Under a scoped consent the excluded pages are also purged from disk (with
|
||||
* their fingerprints). Pages removed while consent is absent stay on disk and
|
||||
* are queued again, so they wait for consent like never-committed pages.
|
||||
*/
|
||||
export function guardUnpublished(home: string, c: TranscriptConsent): GuardResult {
|
||||
if (!fs.existsSync(path.join(home, '.git'))) return { status: 'clean', removed: [] };
|
||||
const ref = upstreamRef(home);
|
||||
const range = ref ? `${ref}..HEAD` : 'HEAD';
|
||||
const touched = [...new Set(git(home, ['log', '--format=', '--name-only', range, '--', 'transcripts']).out.split('\n').filter(Boolean))];
|
||||
if (touched.length === 0) return { status: 'clean', removed: [] };
|
||||
purgeStagedTranscripts(home, c);
|
||||
const inHead = new Set(git(home, ['ls-tree', '-r', '--name-only', 'HEAD', '--', 'transcripts']).out.split('\n').filter(Boolean));
|
||||
const excluded: string[] = [];
|
||||
for (const rel of touched) {
|
||||
let text: string | null = null;
|
||||
if (inHead.has(rel)) text = git(home, ['show', `HEAD:${rel}`]).out;
|
||||
else {
|
||||
const sha = git(home, ['log', '-n1', '--format=%H', '--diff-filter=AM', range, '--', rel]).out.trim();
|
||||
if (sha) text = git(home, ['show', `${sha}:${rel}`]).out;
|
||||
}
|
||||
if (text !== null && pageExclusion(c, text) !== null) excluded.push(rel);
|
||||
}
|
||||
if (excluded.length === 0) return { status: 'clean', removed: [] };
|
||||
const authors = new Set(git(home, ['log', '--no-merges', '--format=%an', range]).out.split('\n').filter(Boolean));
|
||||
if (!ref) return { status: 'blocked', removed: [], message: `push blocked: ${excluded.length} transcript page(s) outside your transcript consent are in unpushed commits and there is no upstream branch to rewrite against` };
|
||||
if ([...authors].some((a) => a !== BRAIN_SYNC_AUTHOR)) {
|
||||
return { status: 'blocked', removed: [], message: `push blocked: ${excluded.length} transcript page(s) outside your transcript consent are in unpushed commits that include your own commits; remove them (git rm --cached) and amend, then sync again` };
|
||||
}
|
||||
if (!git(home, ['diff', '--cached', '--quiet']).ok) {
|
||||
return { status: 'blocked', removed: [], message: 'push blocked: excluded transcript pages are in unpushed commits and the index has other staged changes' };
|
||||
}
|
||||
if (!git(home, ['reset', '-q', '--soft', ref]).ok) return { status: 'blocked', removed: [], message: 'push blocked: could not rewrite unpushed sync commits' };
|
||||
const headExcluded = excluded.filter((rel) => inHead.has(rel));
|
||||
const published = new Set(git(home, ['ls-tree', '-r', '--name-only', ref, '--', ...headExcluded]).out.split('\n').filter(Boolean));
|
||||
const restore = headExcluded.filter((rel) => published.has(rel));
|
||||
const drop = headExcluded.filter((rel) => !published.has(rel));
|
||||
if (restore.length > 0) git(home, ['reset', '-q', ref, '--', ...restore]);
|
||||
if (drop.length > 0) git(home, ['rm', '-q', '--cached', '--ignore-unmatch', '--', ...drop]);
|
||||
if (!git(home, ['diff', '--cached', '--quiet']).ok) {
|
||||
const ts = new Date().toISOString().replace(/\.\d{3}Z$/, 'Z');
|
||||
const commit = spawnSync('git', ['-C', home, '-c', 'user.email=gstack@localhost', '-c', `user.name=${BRAIN_SYNC_AUTHOR}`, 'commit', '-q', '-m', `sync: rewritten without ${excluded.length} excluded transcript page(s) | ${ts}`], { encoding: 'utf-8', timeout: 30_000 });
|
||||
if (commit.status !== 0) return { status: 'blocked', removed: [], message: 'push blocked: could not recommit unpushed sync changes' };
|
||||
}
|
||||
if (!c.affirmative) {
|
||||
let seq = 0;
|
||||
for (const rel of drop) if (fs.existsSync(path.join(home, rel))) enqueue(home, rel, ++seq);
|
||||
}
|
||||
return { status: 'rewritten', removed: excluded };
|
||||
}
|
||||
|
||||
if (import.meta.main) {
|
||||
const cmd = process.argv[2];
|
||||
const home = resolveStateRoot();
|
||||
const policy = readTranscriptConsent();
|
||||
if (cmd === '--check') process.exit(allowsTranscripts(policy) ? 0 : 1);
|
||||
if (cmd === '--describe') {
|
||||
console.log(describeTranscriptPolicy(policy));
|
||||
process.exit(0);
|
||||
}
|
||||
if (cmd === '--purge') {
|
||||
const n = purgeStagedTranscripts(home, policy);
|
||||
if (n > 0) console.error(`BRAIN_SYNC: removed ${n} staged transcript pages outside the new scope`);
|
||||
process.exit(0);
|
||||
}
|
||||
if (cmd === '--guard-unpublished') {
|
||||
const r = guardUnpublished(home, policy);
|
||||
if (r.status === 'rewritten') console.error(`BRAIN_SYNC: removed ${r.removed.length} excluded transcript pages from unpublished sync commits`);
|
||||
if (r.status === 'blocked') {
|
||||
console.error(`BRAIN_SYNC: ${r.message}`);
|
||||
process.exit(3);
|
||||
}
|
||||
process.exit(0);
|
||||
}
|
||||
console.error('Usage: bun run lib/transcript-consent.ts --check | --describe | --purge | --guard-unpublished');
|
||||
process.exit(2);
|
||||
}
|
||||
+58
-15
@@ -25,13 +25,15 @@ answer does.
|
||||
Curated memory (learnings, timeline, plans, designs, retros, eureka, builder
|
||||
profile) syncs on every `/sync-gbrain`. Session transcripts sync only after
|
||||
you choose to share them. The choice is the `transcript_ingest_mode` config
|
||||
key:
|
||||
key, optionally narrowed by the `transcript_repos` allowlist:
|
||||
|
||||
| Value | What `/sync-gbrain` ingests |
|
||||
|---|---|
|
||||
| `recent` | Sessions from the last 90 days |
|
||||
| `all` | All history (`--full` walks every session file) |
|
||||
| `new@2026-10-03T17:00:00Z` | Only sessions whose first record is at or after that UTC time. A session that started earlier stays out even when it is appended to later. A session with no start timestamp stays out. |
|
||||
| `off` | No transcripts; other memory still syncs |
|
||||
| `recent+repos`, `all+repos`, `new@…+repos` | The same window, only for the repos in `transcript_repos`. gstack adds and removes `+repos` itself. |
|
||||
|
||||
Both sources are covered: Claude Code sessions (`~/.claude/projects/`) and
|
||||
Codex sessions (`~/.codex/sessions/`), from every project on this machine
|
||||
@@ -43,38 +45,79 @@ Set or change it:
|
||||
|
||||
```bash
|
||||
gstack-config set transcript_ingest_mode recent # or all, or off
|
||||
gstack-config set transcript_ingest_mode "new@$(date -u +%Y-%m-%dT%H:%M:%SZ)" # only sessions from now on
|
||||
gstack-config set transcript_repos github.com/acme/app,github.com/acme/api # only these repos
|
||||
gstack-config unset transcript_repos # every repo again
|
||||
gstack-config has transcript_ingest_mode && gstack-config get transcript_ingest_mode
|
||||
bun run lib/transcript-consent.ts --describe # the effective consent in words
|
||||
```
|
||||
|
||||
`gstack-config set` rejects any other value and keeps the stored one.
|
||||
`gstack-config set` rejects any other value and keeps the stored one. The
|
||||
cutoff takes only the `Z` (UTC) form, to the second. `transcript_repos`
|
||||
accepts any git remote spelling (`https://…`, `git@…:…`, with or without
|
||||
`.git`) and stores its canonical form; an empty list is rejected. Setting
|
||||
or clearing the allowlist updates the `+repos` marker in the same write.
|
||||
`gstack-config set transcript_ingest_mode <base>` keeps the marker while an
|
||||
allowlist exists, and `off` never carries it.
|
||||
|
||||
**Several config roots.** When gstack reads more than one state root (for
|
||||
example `GSTACK_HOME` plus `~/.gstack`), the most restrictive answer wins:
|
||||
`off` in any root means off, the latest cutoff and any 90-day window both
|
||||
apply, and only repos in every root's allowlist count. The resolved root
|
||||
must hold the consent itself.
|
||||
|
||||
**Not set.** Until you choose, transcripts are skipped. Each sync prints one
|
||||
line saying so, even with `--quiet`, and the next interactive `/sync-gbrain`
|
||||
asks once. Older values from previous gate versions (`A`-`E`, `incremental`)
|
||||
also count as not chosen and get the question again. A stored `off` prints
|
||||
`transcripts off (your choice)` without `--quiet` and is never asked again.
|
||||
and values this version does not recognize also count as not chosen and get
|
||||
the question again. A stored `off` prints `transcripts off (your choice)`
|
||||
without `--quiet` and is never asked again.
|
||||
|
||||
**One-run override.** A `--sources` list (or `GSTACK_MEMORY_INGEST_SOURCES`)
|
||||
that names `transcript` ingests transcripts for that run whatever the mode;
|
||||
the sync still prints the mode notice. `--sources all`, an empty list, or a
|
||||
list with no valid types does not override.
|
||||
the sync still prints the mode notice. An override never widens a scoped
|
||||
consent: a `new@` cutoff and the `transcript_repos` allowlist still apply.
|
||||
`--sources all`, an empty list, or a list with no valid types does not
|
||||
override.
|
||||
|
||||
```bash
|
||||
bun run bin/gstack-gbrain-sync.ts --incremental --sources transcript
|
||||
```
|
||||
|
||||
**What each sync tells you.** Without `--quiet` the sync prints the consent
|
||||
in words (window or cutoff, up to three repos, the config roots it came
|
||||
from). The ingest summary counts sessions left out by reason: before the
|
||||
cutoff, no start time, not in the allowlist, and repo policy deny. A sync
|
||||
under `new@` that ingests nothing says so and says new sessions will appear
|
||||
on the next sync.
|
||||
|
||||
**Interrupted imports.** If a sync was interrupted mid-import and your
|
||||
transcript choice changed before the next run, the next sync restages memory
|
||||
from scratch once instead of resuming, and prints a line saying so.
|
||||
transcript choice changed before the next run (the mode, the cutoff or the
|
||||
allowlist), the next sync restages memory from scratch once instead of
|
||||
resuming, and prints a line saying so.
|
||||
|
||||
**Staged pages wait.** Transcript pages already staged under
|
||||
`~/.gstack/transcripts/` (remote-http mode) stay on disk and are not pushed
|
||||
by `gstack-brain-sync` until the mode is `recent` or `all`. Other artifacts
|
||||
keep pushing.
|
||||
**Staged pages.** Transcript pages already staged under
|
||||
`~/.gstack/transcripts/` (remote-http mode) wait while the mode is not
|
||||
`recent`, `all` or `new@…`: they stay on disk and are not pushed by
|
||||
`gstack-brain-sync`. Other artifacts keep pushing. When you narrow a scoped
|
||||
consent (a later cutoff, a smaller allowlist), the next sync removes the
|
||||
staged pages that are now out of scope and prints `removed N staged
|
||||
transcript pages outside the new scope`; widening again re-stages them from
|
||||
the original sessions. Before every push, `gstack-brain-sync` also checks
|
||||
its own unpushed commits and rewrites them without any page outside your
|
||||
consent (`removed N excluded transcript pages from unpublished sync
|
||||
commits`). The removal covers staged and unpushed pages only.
|
||||
|
||||
**Already-ingested transcripts stay.** Switching to `off` stops new ingests;
|
||||
it does not remove pages already in the brain. Delete them there (see
|
||||
"Delete a page" below).
|
||||
**Already-ingested transcripts stay.** Switching to `off` or narrowing the
|
||||
scope stops new ingests; it does not remove pages already in the brain or
|
||||
already pushed. Delete them there (see "Delete a page" below).
|
||||
|
||||
**Downgrading gstack.** A gstack that predates `new@` and `transcript_repos`
|
||||
reads those values as unrecognized and ingests no transcripts
|
||||
automatically. It does honour an explicit transcript override without any
|
||||
scope, so before downgrading: remove `--sources transcript` and
|
||||
`GSTACK_MEMORY_INGEST_SOURCES=transcript` from scripts, and let pending sync
|
||||
publish, or run `gstack-config set transcript_ingest_mode off`.
|
||||
|
||||
## What stays local
|
||||
|
||||
|
||||
@@ -3,24 +3,27 @@ config (Step 8), ask whether this machine's coding-agent session
|
||||
transcripts may go into gbrain. Curated `~/.gstack/` artifacts (learnings,
|
||||
timeline, plans, designs, retros) sync whatever the answer; this question
|
||||
covers transcripts only. Transcripts are ingested only after the user picks
|
||||
`recent` or `all`.
|
||||
`recent`, `all` or `new` (only sessions that start from now on).
|
||||
|
||||
Check whether the user already chose. The gate keys on `has`, because `get`
|
||||
prints the `off` default for a key that was never set:
|
||||
```bash
|
||||
_TIM=$(~/.claude/skills/gstack/bin/gstack-config get transcript_ingest_mode 2>/dev/null || true)
|
||||
if ~/.claude/skills/gstack/bin/gstack-config has transcript_ingest_mode; then
|
||||
_T='[0-9][0-9][0-9][0-9]-[0-9][0-9]-[0-9][0-9]t[0-9][0-9]:[0-9][0-9]:[0-9][0-9]z'
|
||||
case "$(printf '%s' "$_TIM" | tr -d '[:space:]' | tr '[:upper:]' '[:lower:]')" in
|
||||
recent|all|off) echo "TRANSCRIPT_MODE: $_TIM" ;;
|
||||
*) echo "TRANSCRIPT_MODE: ask (stored value '$_TIM' is from an older version)" ;;
|
||||
recent|all|off|recent+repos|all+repos|new@$_T|new@$_T+repos)
|
||||
echo "TRANSCRIPT_MODE: $_TIM (repos: $(~/.claude/skills/gstack/bin/gstack-config get transcript_repos 2>/dev/null || true))" ;;
|
||||
*) echo "TRANSCRIPT_MODE: ask (stored value '$_TIM' is not recognized by this version)" ;;
|
||||
esac
|
||||
else
|
||||
echo "TRANSCRIPT_MODE: ask (not set)"
|
||||
fi
|
||||
```
|
||||
|
||||
- `TRANSCRIPT_MODE: recent`, `all` or `off`: the user already chose. Do not
|
||||
ask again; continue to Step 8.
|
||||
- `TRANSCRIPT_MODE:` with a value (`recent`, `all`, `off`, `new@<time>`,
|
||||
any of them with `+repos`): the user already chose. Do not ask again;
|
||||
continue to Step 8.
|
||||
- `TRANSCRIPT_MODE: ask`: ask once, below. With `SESSION_KIND: spawned` or
|
||||
`headless`, do not ask and do not store a value. Consent is never
|
||||
auto-chosen; leave the key unset (transcripts stay skipped) and report
|
||||
@@ -38,7 +41,7 @@ Read `Total files in window`, `Total bytes` and the estimate from each.
|
||||
|
||||
> "Ingest coding-agent sessions as they appear?"
|
||||
|
||||
Yes stores `recent`; No stores `off`.
|
||||
Yes means `recent` (then ask the scope question below); No stores `off`.
|
||||
|
||||
**Otherwise**, AskUserQuestion with the counts. Name both session sources
|
||||
and the destination brain from Step 4: the user's Supabase brain (Paths 1,
|
||||
@@ -68,14 +71,24 @@ and the destination brain from Step 4: the user's Supabase brain (Paths 1,
|
||||
Options:
|
||||
- A) Yes, last 90 days (`recent`)
|
||||
- B) Yes, all history (`all`)
|
||||
- C) Yes, only new sessions starting now (`new`)
|
||||
- E) No, never ingest transcripts (`off`)
|
||||
|
||||
Store the mode value, never the letter:
|
||||
On any yes, ask a second, separate question: "Which repos' sessions?"
|
||||
A) every project repo your repo policy allows, or B) only this repo (offer
|
||||
B only when `git remote get-url origin` succeeds here). Store nothing until
|
||||
both answers are known; if the user cancels in between, store nothing.
|
||||
Then store the scope, then the mode value (never the letter; `new` stores
|
||||
`new@` plus the current UTC time):
|
||||
```bash
|
||||
~/.claude/skills/gstack/bin/gstack-config set transcript_ingest_mode <recent|all|off>
|
||||
~/.claude/skills/gstack/bin/gstack-config set transcript_repos "$(git remote get-url origin)" # only this repo
|
||||
~/.claude/skills/gstack/bin/gstack-config unset transcript_repos # every repo
|
||||
~/.claude/skills/gstack/bin/gstack-config set transcript_ingest_mode <recent|all|off|new@$(date -u +%Y-%m-%dT%H:%M:%SZ)>
|
||||
bun run ~/.claude/skills/gstack/lib/transcript-consent.ts --describe
|
||||
```
|
||||
Tell the user the `--describe` line: what will be ingested, in words.
|
||||
|
||||
For `recent` or `all`, run the first sync now:
|
||||
For `recent`, `all` or `new@…`, run the first sync now:
|
||||
```bash
|
||||
bun run ~/.claude/skills/gstack/bin/gstack-gbrain-sync.ts --full --no-brain-sync
|
||||
```
|
||||
|
||||
@@ -190,7 +190,9 @@ gets the actionable remediation message.
|
||||
## Step 1.6: Transcript consent
|
||||
|
||||
Claude Code and Codex session transcripts are ingested only when
|
||||
`transcript_ingest_mode` is `recent` (last 90 days) or `all` (all history).
|
||||
`transcript_ingest_mode` is `recent` (last 90 days), `all` (all history) or
|
||||
`new@<UTC time>` (only sessions that start after it), optionally narrowed to
|
||||
the repos in `transcript_repos` (the value then ends in `+repos`).
|
||||
Skip this step for `--code-only`, `--no-memory`, `--dry-run`,
|
||||
`--refresh-cache` and `--audit`. Otherwise check whether the user chose;
|
||||
`has` tells an absent key from the `off` default that `get` prints:
|
||||
@@ -198,16 +200,19 @@ Skip this step for `--code-only`, `--no-memory`, `--dry-run`,
|
||||
```bash
|
||||
_TIM=$(~/.claude/skills/gstack/bin/gstack-config get transcript_ingest_mode 2>/dev/null || true)
|
||||
if ~/.claude/skills/gstack/bin/gstack-config has transcript_ingest_mode; then
|
||||
_T='[0-9][0-9][0-9][0-9]-[0-9][0-9]-[0-9][0-9]t[0-9][0-9]:[0-9][0-9]:[0-9][0-9]z'
|
||||
case "$(printf '%s' "$_TIM" | tr -d '[:space:]' | tr '[:upper:]' '[:lower:]')" in
|
||||
recent|all|off) echo "TRANSCRIPT_MODE: $_TIM" ;;
|
||||
*) echo "TRANSCRIPT_MODE: ask (stored value '$_TIM' is from an older version)" ;;
|
||||
recent|all|off|recent+repos|all+repos|new@$_T|new@$_T+repos)
|
||||
echo "TRANSCRIPT_MODE: $_TIM (repos: $(~/.claude/skills/gstack/bin/gstack-config get transcript_repos 2>/dev/null || true))" ;;
|
||||
*) echo "TRANSCRIPT_MODE: ask (stored value '$_TIM' is not recognized by this version)" ;;
|
||||
esac
|
||||
else
|
||||
echo "TRANSCRIPT_MODE: ask (not set)"
|
||||
fi
|
||||
```
|
||||
|
||||
`recent`, `all` or `off`: continue to Step 2 without asking. On `ask`, with
|
||||
A stored value (`recent`, `all`, `off`, `new@<time>`, any with `+repos`):
|
||||
continue to Step 2 without asking. On `ask`, with
|
||||
`SESSION_KIND: spawned` or `headless`, do not ask and store nothing: the sync
|
||||
skips transcripts and prints how to choose. In an interactive session, ask
|
||||
once. Count first (`--sources transcript` counts sessions that are not
|
||||
@@ -219,7 +224,7 @@ bun run ~/.claude/skills/gstack/bin/gstack-memory-ingest.ts --probe --sources tr
|
||||
```
|
||||
|
||||
If both report `Total files in window: 0`, ask yes/no: "Ingest coding-agent
|
||||
sessions as they appear?" Yes stores `recent`, No stores `off`. Otherwise
|
||||
sessions as they appear?" Yes means `recent`, No stores `off`. Otherwise
|
||||
AskUserQuestion: name both sources (Claude Code and Codex sessions from every
|
||||
project on this machine that repo policy allows), the counts from the two
|
||||
probes, and the destination: the local brain (`gbrain_engine` from Step 1:
|
||||
@@ -229,14 +234,24 @@ and skills stay usable meanwhile. Options:
|
||||
|
||||
- A) Yes, last 90 days (`recent`)
|
||||
- B) Yes, all history (`all`)
|
||||
- C) Yes, only new sessions starting now (`new`)
|
||||
- E) No, never ingest transcripts (`off`)
|
||||
|
||||
Store the value, never the letter, then continue to Step 2:
|
||||
On any yes (the yes/no form included), ask a second, separate question:
|
||||
"Which repos' sessions?" A) every project repo your repo policy allows, or
|
||||
B) only this repo (offer B only when `git remote get-url origin` succeeds).
|
||||
Store nothing until both answers are known; a cancel in between stores
|
||||
nothing. Then store the scope, then the value (never the letter; `new`
|
||||
stores `new@` plus the current UTC time), and continue to Step 2:
|
||||
|
||||
```bash
|
||||
~/.claude/skills/gstack/bin/gstack-config set transcript_ingest_mode <recent|all|off>
|
||||
~/.claude/skills/gstack/bin/gstack-config set transcript_repos "$(git remote get-url origin)" # only this repo
|
||||
~/.claude/skills/gstack/bin/gstack-config unset transcript_repos # every repo
|
||||
~/.claude/skills/gstack/bin/gstack-config set transcript_ingest_mode <recent|all|off|new@$(date -u +%Y-%m-%dT%H:%M:%SZ)>
|
||||
```
|
||||
|
||||
The sync prints what it will ingest, in words; repeat that line to the user.
|
||||
|
||||
Other memory types sync whatever the answer. Details:
|
||||
`setup-gbrain/memory.md#transcripts`.
|
||||
|
||||
|
||||
@@ -47,13 +47,13 @@ afterEach(() => {
|
||||
});
|
||||
|
||||
describe("gstack-config key validation is locale-independent", () => {
|
||||
test("get, has, and set all validate ASCII ranges under the C locale", () => {
|
||||
test("get, has, set and unset all validate ASCII ranges under the C locale", () => {
|
||||
const source = fs.readFileSync(CONFIG, "utf8");
|
||||
const guardedValidators = source.match(
|
||||
/LC_ALL=C grep -qE '\^\[a-zA-Z0-9_\]\+\(@\[a-zA-Z0-9\]\+\)\?\$'/g,
|
||||
);
|
||||
|
||||
expect(guardedValidators).toHaveLength(3);
|
||||
expect(guardedValidators).toHaveLength(4);
|
||||
});
|
||||
|
||||
test("round-trips existing keys that contain i", () => {
|
||||
|
||||
@@ -0,0 +1,161 @@
|
||||
/**
|
||||
* bin/gstack-config writes: the transcript_repos allowlist and its +repos
|
||||
* marker change together in one locked rename, `unset` removes a key, and
|
||||
* every mutation goes through one mkdir lock with a read-modify-write rename,
|
||||
* so concurrent writers lose nothing and an interrupted write leaves the old
|
||||
* file.
|
||||
*/
|
||||
import { afterEach, beforeEach, describe, expect, test } from 'bun:test';
|
||||
import { spawn, spawnSync } from 'child_process';
|
||||
import { chmodSync, existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'fs';
|
||||
import { tmpdir } from 'os';
|
||||
import { join } from 'path';
|
||||
|
||||
const CONFIG_BIN = join(import.meta.dir, '..', 'bin', 'gstack-config');
|
||||
let root: string;
|
||||
const env = () => ({ ...process.env, GSTACK_STATE_ROOT: root });
|
||||
const cfg = (...args: string[]) => {
|
||||
const r = spawnSync('bash', [CONFIG_BIN, ...args], { encoding: 'utf-8', timeout: 30_000, env: env() });
|
||||
return { status: r.status, out: r.stdout ?? '', err: r.stderr ?? '' };
|
||||
};
|
||||
const file = () => join(root, 'config.yaml');
|
||||
const snapshot = () => (existsSync(file()) ? readFileSync(file(), 'utf-8') : null);
|
||||
const keyLines = (key: string) => (snapshot() ?? '').split('\n').filter((l) => l.startsWith(`${key}:`));
|
||||
|
||||
beforeEach(() => {
|
||||
root = mkdtempSync(join(tmpdir(), 'gstack-config-scope-'));
|
||||
});
|
||||
afterEach(() => {
|
||||
rmSync(root, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
describe('transcript_repos and the +repos marker', () => {
|
||||
test('setting the allowlist stores canonical remotes and marks the mode in the same write', () => {
|
||||
expect(cfg('set', 'transcript_ingest_mode', 'recent').status).toBe(0);
|
||||
expect(cfg('set', 'transcript_repos', 'https://github.com/Acme/App.git, git@github.com:acme/api.git,github.com/acme/app').status).toBe(0);
|
||||
expect(cfg('get', 'transcript_repos').out).toBe('github.com/acme/app,github.com/acme/api');
|
||||
expect(cfg('get', 'transcript_ingest_mode').out).toBe('recent+repos');
|
||||
});
|
||||
|
||||
test('a mode change keeps the marker while the allowlist exists; off never carries it', () => {
|
||||
cfg('set', 'transcript_repos', 'github.com/acme/app');
|
||||
expect(cfg('get', 'transcript_ingest_mode').status).toBe(0);
|
||||
expect(keyLines('transcript_ingest_mode')).toEqual([]);
|
||||
cfg('set', 'transcript_ingest_mode', 'all');
|
||||
expect(cfg('get', 'transcript_ingest_mode').out).toBe('all+repos');
|
||||
cfg('set', 'transcript_ingest_mode', 'new@2026-10-03T17:00:00Z');
|
||||
expect(cfg('get', 'transcript_ingest_mode').out).toBe('new@2026-10-03T17:00:00Z+repos');
|
||||
cfg('set', 'transcript_ingest_mode', 'off');
|
||||
expect(cfg('get', 'transcript_ingest_mode').out).toBe('off');
|
||||
cfg('set', 'transcript_ingest_mode', 'recent+repos');
|
||||
expect(cfg('get', 'transcript_ingest_mode').out).toBe('recent+repos');
|
||||
});
|
||||
|
||||
test('unset transcript_repos drops the marker in the same write; unset of an absent key is a no-op', () => {
|
||||
cfg('set', 'transcript_ingest_mode', 'recent');
|
||||
cfg('set', 'transcript_repos', 'github.com/acme/app');
|
||||
expect(cfg('unset', 'transcript_repos').status).toBe(0);
|
||||
expect(cfg('has', 'transcript_repos').status).toBe(1);
|
||||
expect(cfg('get', 'transcript_ingest_mode').out).toBe('recent');
|
||||
const before = snapshot();
|
||||
expect(cfg('unset', 'transcript_repos').status).toBe(0);
|
||||
expect(snapshot()).toBe(before);
|
||||
expect(cfg('unset', 'bad key!').status).toBe(1);
|
||||
});
|
||||
|
||||
test('unset removes only the named key', () => {
|
||||
cfg('set', 'proactive', 'false');
|
||||
cfg('set', 'skill_prefix', 'true');
|
||||
expect(cfg('unset', 'proactive').status).toBe(0);
|
||||
expect(cfg('has', 'proactive').status).toBe(1);
|
||||
expect(cfg('get', 'skill_prefix').out).toBe('true');
|
||||
});
|
||||
|
||||
test('a +repos value without an allowlist and an empty allowlist are rejected with the file unchanged', () => {
|
||||
cfg('set', 'transcript_ingest_mode', 'recent');
|
||||
const before = snapshot();
|
||||
const marked = cfg('set', 'transcript_ingest_mode', 'all+repos');
|
||||
expect(marked.status).toBe(1);
|
||||
expect(marked.err).toContain('Existing value left unchanged');
|
||||
for (const value of [' , ', ',']) {
|
||||
const r = cfg('set', 'transcript_repos', value);
|
||||
expect(r.status).toBe(1);
|
||||
expect(r.err).toContain('Existing value left unchanged');
|
||||
expect(r.err).toContain('gstack-config unset transcript_repos');
|
||||
}
|
||||
expect(snapshot()).toBe(before);
|
||||
});
|
||||
});
|
||||
|
||||
describe('one lock and one rename for every mutation', () => {
|
||||
test('concurrent unrelated sets lose nothing', async () => {
|
||||
cfg('set', 'proactive', 'true');
|
||||
const keys = Array.from({ length: 12 }, (_, i) => `scope_test_key_${i}`);
|
||||
const codes = await Promise.all(
|
||||
keys.map(
|
||||
(key, i) =>
|
||||
new Promise<number | null>((resolve) => {
|
||||
const child = spawn('bash', [CONFIG_BIN, 'set', key, `value-${i}`], { env: env(), stdio: 'ignore' });
|
||||
child.on('exit', resolve);
|
||||
}),
|
||||
),
|
||||
);
|
||||
expect(codes).toEqual(keys.map(() => 0));
|
||||
for (const [i, key] of keys.entries()) expect(keyLines(key)).toEqual([`${key}: value-${i}`]);
|
||||
expect(keyLines('proactive')).toEqual(['proactive: true']);
|
||||
expect(existsSync(join(root, '.config-yaml.lock'))).toBe(false);
|
||||
}, 60_000);
|
||||
|
||||
test('an interrupted write leaves the old file, and the next writer takes over the dead lock', () => {
|
||||
cfg('set', 'transcript_ingest_mode', 'recent');
|
||||
cfg('set', 'transcript_repos', 'github.com/acme/app');
|
||||
const before = snapshot();
|
||||
// A mv that kills gstack-config right before the rename over config.yaml.
|
||||
const shim = join(root, 'shim');
|
||||
mkdirSync(shim);
|
||||
writeFileSync(join(shim, 'mv'), '#!/bin/sh\nfor last; do :; done\ncase "$last" in */config.yaml) kill -9 $PPID; exit 1 ;; esac\nexec /bin/mv "$@"\n');
|
||||
chmodSync(join(shim, 'mv'), 0o755);
|
||||
const crashed = spawnSync('bash', [CONFIG_BIN, 'unset', 'transcript_repos'], {
|
||||
encoding: 'utf-8',
|
||||
timeout: 30_000,
|
||||
env: { ...env(), PATH: `${shim}:${process.env.PATH}` },
|
||||
});
|
||||
expect(crashed.status).not.toBe(0);
|
||||
expect(snapshot()).toBe(before);
|
||||
expect(existsSync(join(root, '.config-yaml.lock'))).toBe(true);
|
||||
const t0 = Date.now();
|
||||
expect(cfg('unset', 'transcript_repos').status).toBe(0);
|
||||
expect(Date.now() - t0).toBeLessThan(10_000);
|
||||
expect(cfg('get', 'transcript_ingest_mode').out).toBe('recent');
|
||||
}, 60_000);
|
||||
|
||||
test('a lock held longer than 10 s is stale even when its pid is alive', () => {
|
||||
mkdirSync(join(root, '.config-yaml.lock'));
|
||||
writeFileSync(join(root, '.config-yaml.lock', 'pid'), `${process.pid} ${Math.floor(Date.now() / 1000) - 20}\n`);
|
||||
expect(cfg('set', 'proactive', 'false').status).toBe(0);
|
||||
expect(cfg('get', 'proactive').out).toBe('false');
|
||||
});
|
||||
|
||||
test('a live, fresh lock makes the writer wait instead of writing alongside it', async () => {
|
||||
mkdirSync(join(root, '.config-yaml.lock'));
|
||||
writeFileSync(join(root, '.config-yaml.lock', 'pid'), `${process.pid} ${Math.floor(Date.now() / 1000)}\n`);
|
||||
const child = spawn('bash', [CONFIG_BIN, 'set', 'proactive', 'false'], { env: env(), stdio: 'ignore' });
|
||||
const exited = new Promise<number | null>((resolve) => child.on('exit', resolve));
|
||||
await Bun.sleep(500);
|
||||
expect(snapshot()).toBeNull();
|
||||
rmSync(join(root, '.config-yaml.lock'), { recursive: true, force: true });
|
||||
expect(await exited).toBe(0);
|
||||
expect(keyLines('proactive')).toEqual(['proactive: false']);
|
||||
}, 30_000);
|
||||
|
||||
test('the user-slug writer goes through the same lock', () => {
|
||||
const r = spawnSync('bash', [CONFIG_BIN, 'resolve-user-slug'], {
|
||||
encoding: 'utf-8',
|
||||
timeout: 30_000,
|
||||
env: { ...env(), HOME: root, USER: 'scope-tester' },
|
||||
});
|
||||
expect(r.stdout.length).toBeGreaterThan(0);
|
||||
expect(keyLines('user_slug_at_local')).toEqual([`user_slug_at_local: ${r.stdout}`]);
|
||||
expect(existsSync(join(root, '.config-yaml.lock'))).toBe(false);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,285 @@
|
||||
/**
|
||||
* lib/transcript-consent.ts: the value grammar, the multi-root policy, the
|
||||
* per-session and per-page scope checks, and the parity of the bash validator
|
||||
* and remote canonicalizer in bin/gstack-config with their TS twins.
|
||||
*/
|
||||
import { describe, expect, test } from 'bun:test';
|
||||
import { spawnSync } from 'child_process';
|
||||
import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from 'fs';
|
||||
import { tmpdir } from 'os';
|
||||
import { join } from 'path';
|
||||
|
||||
import { canonicalizeRemote } from '../lib/gstack-memory-helpers';
|
||||
import {
|
||||
describeTranscriptPolicy,
|
||||
pageExclusion,
|
||||
parseTranscriptMode,
|
||||
readTranscriptConsent,
|
||||
reposHash,
|
||||
sessionExclusion,
|
||||
type TranscriptConsent,
|
||||
} from '../lib/transcript-consent';
|
||||
|
||||
const CONFIG_BIN = join(import.meta.dir, '..', 'bin', 'gstack-config');
|
||||
const TS = '2026-10-03T17:00:00Z';
|
||||
|
||||
/** Every grammar combination: base x (cutoff | none) x (+repos | none). */
|
||||
function combinations(): Array<{ value: string; valid: boolean }> {
|
||||
const out: Array<{ value: string; valid: boolean }> = [];
|
||||
for (const base of ['recent', 'all', 'off', 'new']) {
|
||||
for (const at of ['', `@${TS}`]) {
|
||||
for (const marker of ['', '+repos']) {
|
||||
const valid = (base === 'new') === (at !== '') && !(base === 'off' && marker !== '');
|
||||
out.push({ value: `${base}${at}${marker}`, valid });
|
||||
}
|
||||
}
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
const INVALID_FORMS = [
|
||||
`new@2026-10-03T17:00:00+00:00`, // offsets are rejected; only Z
|
||||
`new@2026-10-03T17:00:00.000Z`, // milliseconds are not part of the grammar
|
||||
`new@2026-10-03 17:00:00Z`,
|
||||
`new@2026-10-03`,
|
||||
`new@2026-02-30T00:00:00Z`, // not a real day
|
||||
`new@2026-13-01T00:00:00Z`,
|
||||
`new@2026-10-03T24:00:00Z`,
|
||||
`new+repos@${TS}`, // @ precedes +repos
|
||||
`new@`,
|
||||
'recent+repo',
|
||||
'yes',
|
||||
'A',
|
||||
'incremental',
|
||||
'',
|
||||
];
|
||||
|
||||
describe('grammar', () => {
|
||||
test('every base x cutoff x marker combination', () => {
|
||||
for (const { value, valid } of combinations()) {
|
||||
const parsed = parseTranscriptMode(value);
|
||||
expect({ value, valid: parsed !== null }).toEqual({ value, valid });
|
||||
if (parsed) {
|
||||
expect(parsed.base).toBe(value.split(/[@+]/)[0] as never);
|
||||
expect(parsed.cutoff).toBe(value.includes('@') ? TS : null);
|
||||
expect(parsed.repos).toBe(value.endsWith('+repos'));
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
test('readers are case-insensitive; offsets, bad dates and misplaced parts are rejected', () => {
|
||||
expect(parseTranscriptMode(`NEW@2026-10-03t17:00:00z+REPOS`)).toEqual({ base: 'new', cutoff: TS, repos: true });
|
||||
expect(parseTranscriptMode(' Recent ')).toEqual({ base: 'recent', cutoff: null, repos: false });
|
||||
expect(parseTranscriptMode('new@2028-02-29T00:00:00Z')).not.toBeNull();
|
||||
for (const value of INVALID_FORMS) expect({ value, parsed: parseTranscriptMode(value) }).toEqual({ value, parsed: null });
|
||||
});
|
||||
|
||||
test('gstack-config set accepts exactly the canonical valid forms and rejects the rest with the file unchanged', () => {
|
||||
const root = mkdtempSync(join(tmpdir(), 'gstack-grammar-'));
|
||||
try {
|
||||
const env = { ...process.env, GSTACK_STATE_ROOT: root };
|
||||
const run = (...args: string[]) => spawnSync('bash', [CONFIG_BIN, ...args], { encoding: 'utf-8', timeout: 30_000, env });
|
||||
expect(run('set', 'transcript_repos', 'github.com/acme/app').status).toBe(0);
|
||||
for (const { value, valid } of combinations()) {
|
||||
const r = run('set', 'transcript_ingest_mode', value);
|
||||
expect({ value, status: r.status }).toEqual({ value, status: valid ? 0 : 1 });
|
||||
}
|
||||
run('set', 'transcript_ingest_mode', 'recent');
|
||||
for (const value of [...INVALID_FORMS.filter(Boolean), 'Recent', `new@2026-10-03t17:00:00z`]) {
|
||||
const r = run('set', 'transcript_ingest_mode', value);
|
||||
expect({ value, status: r.status }).toEqual({ value, status: 1 });
|
||||
expect(r.stderr).toContain(`Error: transcript_ingest_mode '${value}' not recognized. Valid values: recent, all, off. Existing value left unchanged.`);
|
||||
expect(r.stderr).toContain('new@<YYYY-MM-DDTHH:MM:SSZ>');
|
||||
}
|
||||
expect(run('get', 'transcript_ingest_mode').stdout).toBe('recent+repos');
|
||||
} finally {
|
||||
rmSync(root, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
test('the bash remote canonicalizer matches canonicalizeRemote', () => {
|
||||
const inputs = [
|
||||
'https://github.com/Acme/App.git',
|
||||
'git@github.com:acme/app.git',
|
||||
'ssh://git@github.com/acme/app',
|
||||
'http://user@gitlab.example.com/group/sub/repo.git/',
|
||||
' "github.com/acme/app/" ',
|
||||
'github.com//acme///app',
|
||||
'/srv/git/repo/.git',
|
||||
'file:///srv/git/repo',
|
||||
];
|
||||
const script = `. /dev/null; source <(sed -n '/^canonical_remote() {/,/^}/p' "${CONFIG_BIN}"); for x in "$@"; do canonical_remote "$x"; echo; done`;
|
||||
const r = spawnSync('bash', ['-c', script, 'bash', ...inputs], { encoding: 'utf-8', timeout: 30_000 });
|
||||
expect(r.status).toBe(0);
|
||||
expect(r.stdout.split('\n').slice(0, inputs.length)).toEqual(inputs.map((i) => canonicalizeRemote(i)));
|
||||
});
|
||||
});
|
||||
|
||||
/** The normalizer shipped before this change (bin/gstack-memory-ingest.ts at v1.91.16.0), verbatim in behavior. */
|
||||
function olderNormalizerConsents(value: string | null): boolean {
|
||||
const v = (value ?? '').trim().toLowerCase();
|
||||
return value !== null && (v === 'recent' || v === 'all');
|
||||
}
|
||||
|
||||
describe('fail-closed encoding', () => {
|
||||
test('a gstack without new@ or allowlist support reads every new value form as no consent', () => {
|
||||
for (const value of [`new@${TS}`, `new@${TS}+repos`, 'recent+repos', 'all+repos']) {
|
||||
expect({ value, consents: olderNormalizerConsents(value) }).toEqual({ value, consents: false });
|
||||
}
|
||||
});
|
||||
|
||||
test('the current normalizer treats any unrecognized value as no consent', () => {
|
||||
const root = mkdtempSync(join(tmpdir(), 'gstack-consent-'));
|
||||
try {
|
||||
for (const value of ['yes', 'new', `recent@${TS}`, 'off+repos', 'new@tomorrow', 'recent+repos+repos']) {
|
||||
writeFileSync(join(root, 'config.yaml'), `transcript_ingest_mode: ${value}\n`);
|
||||
expect(readTranscriptConsent([root])).toMatchObject({ affirmative: false, reason: 'unrecognized', value });
|
||||
}
|
||||
} finally {
|
||||
rmSync(root, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
describe('effective consent across state roots', () => {
|
||||
const roots: string[] = [];
|
||||
const root = (yaml: string | null) => {
|
||||
const dir = mkdtempSync(join(tmpdir(), 'gstack-root-'));
|
||||
roots.push(dir);
|
||||
if (yaml !== null) writeFileSync(join(dir, 'config.yaml'), yaml);
|
||||
return dir;
|
||||
};
|
||||
const cleanup = () => roots.splice(0).forEach((d) => rmSync(d, { recursive: true, force: true }));
|
||||
|
||||
test('off in one root plus new@ in another gives no consent', () => {
|
||||
try {
|
||||
const p = readTranscriptConsent([root(`transcript_ingest_mode: new@${TS}\n`), root('transcript_ingest_mode: off\n')]);
|
||||
expect(p).toMatchObject({ affirmative: false, reason: 'off', cutoff: TS });
|
||||
const q = readTranscriptConsent([root('transcript_ingest_mode: off\n'), root(`transcript_ingest_mode: new@${TS}\n`)]);
|
||||
expect(q).toMatchObject({ affirmative: false, reason: 'off' });
|
||||
} finally {
|
||||
cleanup();
|
||||
}
|
||||
});
|
||||
|
||||
test('time constraints stack: the latest cutoff and any 90-day window', () => {
|
||||
try {
|
||||
const later = '2026-11-01T00:00:00Z';
|
||||
expect(readTranscriptConsent([root(`transcript_ingest_mode: new@${TS}\n`), root(`transcript_ingest_mode: new@${later}\n`)]))
|
||||
.toMatchObject({ affirmative: true, window: 'new', reason: 'new', cutoff: later });
|
||||
expect(readTranscriptConsent([root('transcript_ingest_mode: all\n'), root(`transcript_ingest_mode: new@${TS}\n`)]))
|
||||
.toMatchObject({ affirmative: true, window: 'new', cutoff: TS });
|
||||
expect(readTranscriptConsent([root(`transcript_ingest_mode: new@${TS}\n`), root('transcript_ingest_mode: recent\n')]))
|
||||
.toMatchObject({ affirmative: true, window: 'recent', cutoff: TS });
|
||||
const all = readTranscriptConsent([root('transcript_ingest_mode: all\n'), root(null)]);
|
||||
expect(all).toMatchObject({ affirmative: true, window: 'all' });
|
||||
expect(all.cutoff).toBeUndefined();
|
||||
} finally {
|
||||
cleanup();
|
||||
}
|
||||
});
|
||||
|
||||
test('allowlists intersect; disjoint lists give EMPTY and nothing is ingested', () => {
|
||||
try {
|
||||
const a = root('transcript_ingest_mode: recent+repos\ntranscript_repos: github.com/acme/app,github.com/acme/api\n');
|
||||
const b = root('transcript_repos: https://github.com/acme/api.git,github.com/acme/web\n');
|
||||
expect(readTranscriptConsent([a, b])).toMatchObject({ affirmative: true, repos: ['github.com/acme/api'], roots: [a, b] });
|
||||
const c = root('transcript_ingest_mode: all+repos\ntranscript_repos: github.com/other/x\n');
|
||||
const empty = readTranscriptConsent([a, c]);
|
||||
expect(empty.repos).toEqual([]);
|
||||
for (const remote of ['github.com/acme/app', 'github.com/other/x', '']) {
|
||||
expect(sessionExclusion(empty, { startedAt: TS, remote })).toBe('not-allowlisted');
|
||||
}
|
||||
} finally {
|
||||
cleanup();
|
||||
}
|
||||
});
|
||||
|
||||
test('a +repos marker without a readable allowlist denies', () => {
|
||||
try {
|
||||
for (const yaml of ['transcript_ingest_mode: recent+repos\n', 'transcript_ingest_mode: recent+repos\ntranscript_repos: \n', 'transcript_ingest_mode: all+repos\ntranscript_repos: ,\n']) {
|
||||
expect(readTranscriptConsent([root(yaml)])).toMatchObject({ affirmative: false, reason: 'repos-unreadable', repos: [] });
|
||||
}
|
||||
} finally {
|
||||
cleanup();
|
||||
}
|
||||
});
|
||||
|
||||
test('the resolved root must hold the consent; other roots only restrict', () => {
|
||||
try {
|
||||
expect(readTranscriptConsent([root(null), root('transcript_ingest_mode: all\n')])).toMatchObject({ affirmative: false, reason: 'not-set' });
|
||||
expect(readTranscriptConsent([root('transcript_ingest_mode: recent\n'), root('transcript_ingest_mode: A\n')])).toMatchObject({ affirmative: false, reason: 'legacy' });
|
||||
} finally {
|
||||
cleanup();
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
describe('scope checks', () => {
|
||||
const scoped: TranscriptConsent = { affirmative: true, window: 'new', reason: 'new', value: `new@${TS}+repos`, cutoff: TS, repos: ['github.com/acme/app'] };
|
||||
|
||||
test('sessions: the cutoff uses the start time; the allowlist uses the canonical remote', () => {
|
||||
expect(sessionExclusion(scoped, { startedAt: '2026-10-03T16:59:59Z', remote: 'github.com/acme/app' })).toBe('pre-cutoff');
|
||||
expect(sessionExclusion(scoped, { startedAt: TS, remote: 'git@github.com:Acme/App.git' })).toBeNull();
|
||||
expect(sessionExclusion(scoped, { startedAt: '', remote: 'github.com/acme/app' })).toBe('missing-start');
|
||||
expect(sessionExclusion(scoped, { startedAt: 'not a date', remote: 'github.com/acme/app' })).toBe('missing-start');
|
||||
expect(sessionExclusion(scoped, { startedAt: '2026-10-04T00:00:00Z', remote: 'github.com/acme/web' })).toBe('not-allowlisted');
|
||||
expect(sessionExclusion({ affirmative: true, window: 'recent', reason: 'recent', value: 'recent' }, { startedAt: null, remote: null })).toBeNull();
|
||||
});
|
||||
|
||||
test('pages: frontmatter decides; artifact pages are never excluded; no consent excludes every transcript page', () => {
|
||||
const page = (fm: string) => `---\n${fm}\n---\n\nbody\n`;
|
||||
expect(pageExclusion(scoped, page(`type: transcript\nsession_started_at: 2026-10-04T00:00:00.000Z\ngit_remote: github.com/acme/app`))).toBeNull();
|
||||
expect(pageExclusion(scoped, page(`type: transcript\nstart_time: 2026-09-01T00:00:00Z\ngit_remote: github.com/acme/app`))).toBe('pre-cutoff');
|
||||
expect(pageExclusion(scoped, page('type: transcript\ngit_remote: github.com/acme/app'))).toBe('missing-start');
|
||||
expect(pageExclusion(scoped, page('type: learning'))).toBeNull();
|
||||
expect(pageExclusion(scoped, '# no frontmatter\n')).toBe('missing-start');
|
||||
const recent: TranscriptConsent = { affirmative: true, window: 'recent', reason: 'recent', value: 'recent' };
|
||||
expect(pageExclusion(recent, '# no frontmatter\n')).toBeNull();
|
||||
const off: TranscriptConsent = { affirmative: false, window: null, reason: 'off', value: 'off' };
|
||||
expect(pageExclusion(off, page('type: transcript'))).toBe('not-consented');
|
||||
});
|
||||
|
||||
test('the allowlist hash is order-independent', () => {
|
||||
expect(reposHash(['b', 'a'])).toBe(reposHash(['a', 'b']));
|
||||
expect(reposHash(['a'])).not.toBe(reposHash(['a', 'b']));
|
||||
});
|
||||
});
|
||||
|
||||
describe('consent in words', () => {
|
||||
test('window or cutoff, up to three repos then +N more, contributing roots', () => {
|
||||
const base = { affirmative: true, reason: 'new' as const, value: 'x', roots: ['/r1', '/r2'] };
|
||||
expect(describeTranscriptPolicy({ ...base, window: 'recent', reason: 'recent' })).toBe('sessions from the last 90 days; every repo your repo policy allows; config: /r1, /r2');
|
||||
expect(describeTranscriptPolicy({ ...base, window: 'new', cutoff: TS, repos: ['a/b/c', 'd/e/f', 'g/h/i', 'j/k/l', 'm/n/o'] }))
|
||||
.toBe('sessions that started after 2026-10-03 17:00:00 UTC; repos a/b/c, d/e/f, g/h/i +2 more; config: /r1, /r2');
|
||||
expect(describeTranscriptPolicy({ ...base, window: 'all', reason: 'all', repos: ['a/b/c'], roots: [] })).toBe('all history; repo a/b/c');
|
||||
});
|
||||
});
|
||||
|
||||
describe('CLI', () => {
|
||||
test('--check exits 0 only when transcript pages may sync', () => {
|
||||
const root = mkdtempSync(join(tmpdir(), 'gstack-check-'));
|
||||
mkdirSync(join(root, 'legacy'));
|
||||
try {
|
||||
const check = () =>
|
||||
spawnSync('bun', ['run', join(import.meta.dir, '..', 'lib', 'transcript-consent.ts'), '--check'], {
|
||||
encoding: 'utf-8',
|
||||
timeout: 30_000,
|
||||
env: { ...process.env, GSTACK_STATE_ROOT: root, GSTACK_TEST_LEGACY_ROOT: join(root, 'legacy') },
|
||||
}).status;
|
||||
expect(check()).toBe(1);
|
||||
for (const [yaml, status] of [
|
||||
['transcript_ingest_mode: recent\n', 0],
|
||||
[`transcript_ingest_mode: new@${TS}\n`, 0],
|
||||
['transcript_ingest_mode: off\n', 1],
|
||||
['transcript_ingest_mode: recent+repos\n', 1],
|
||||
['transcript_ingest_mode: recent+repos\ntranscript_repos: github.com/a/b\n', 0],
|
||||
] as const) {
|
||||
writeFileSync(join(root, 'config.yaml'), yaml);
|
||||
expect({ yaml, status: check() }).toEqual({ yaml, status });
|
||||
}
|
||||
} finally {
|
||||
rmSync(root, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,74 @@
|
||||
/**
|
||||
* Both transcript consent gates (setup-gbrain's transcript-gate section and
|
||||
* /sync-gbrain Step 1.6) run their rendered check block against every stored
|
||||
* value form: each valid form is recognized so a stored choice is never asked
|
||||
* again, and anything else is asked as "not recognized by this version".
|
||||
*/
|
||||
import { describe, expect, test } from 'bun:test';
|
||||
import { spawnSync } from 'child_process';
|
||||
import { mkdtempSync, readFileSync, rmSync, writeFileSync } from 'fs';
|
||||
import { tmpdir } from 'os';
|
||||
import { join } from 'path';
|
||||
|
||||
const ROOT = join(import.meta.dir, '..');
|
||||
const CONFIG_BIN = join(ROOT, 'bin', 'gstack-config');
|
||||
const GATES = {
|
||||
'setup-gbrain transcript gate': join(ROOT, 'setup-gbrain', 'sections', 'transcript-gate.md'),
|
||||
'sync-gbrain Step 1.6': join(ROOT, 'sync-gbrain', 'SKILL.md'),
|
||||
};
|
||||
const TS = '2026-10-03T17:00:00Z';
|
||||
const RECOGNIZED = ['recent', 'all', 'off', 'recent+repos', 'all+repos', `new@${TS}`, `new@${TS}+repos`, ' Recent ', `NEW@${TS.toLowerCase()}+REPOS`];
|
||||
const UNRECOGNIZED = ['A', 'incremental', 'new', 'off+repos', `recent@${TS}`, 'new@2026-10-03', `new@${TS}x`, 'yes'];
|
||||
|
||||
function gateBlock(file: string): string {
|
||||
const text = readFileSync(file, 'utf-8');
|
||||
const block = text.split('```bash\n').map((b) => b.split('\n```')[0]).find((b) => b.includes('has transcript_ingest_mode'));
|
||||
if (!block) throw new Error(`no transcript gate check block in ${file}`);
|
||||
return block.replaceAll('~/.claude/skills/gstack/bin/gstack-config', CONFIG_BIN);
|
||||
}
|
||||
|
||||
function runGate(block: string, yaml: string | null): string {
|
||||
const root = mkdtempSync(join(tmpdir(), 'gstack-gate-'));
|
||||
try {
|
||||
if (yaml !== null) writeFileSync(join(root, 'config.yaml'), yaml);
|
||||
const r = spawnSync('bash', ['-c', block], { encoding: 'utf-8', timeout: 30_000, env: { ...process.env, GSTACK_STATE_ROOT: root } });
|
||||
expect(r.status).toBe(0);
|
||||
return r.stdout.trim();
|
||||
} finally {
|
||||
rmSync(root, { recursive: true, force: true });
|
||||
}
|
||||
}
|
||||
|
||||
describe('transcript consent gates recognize every value form', () => {
|
||||
for (const [name, file] of Object.entries(GATES)) {
|
||||
const block = gateBlock(file);
|
||||
|
||||
test(`${name}: every stored grammar form is a choice, never re-asked`, () => {
|
||||
for (const value of RECOGNIZED) {
|
||||
const out = runGate(block, `transcript_ingest_mode: ${value}\ntranscript_repos: github.com/acme/app\n`);
|
||||
expect({ value, out }).toEqual({ value, out: `TRANSCRIPT_MODE: ${value.trim()} (repos: github.com/acme/app)` });
|
||||
}
|
||||
});
|
||||
|
||||
test(`${name}: anything else is asked as not recognized by this version; absent is asked as not set`, () => {
|
||||
for (const value of UNRECOGNIZED) {
|
||||
const out = runGate(block, `transcript_ingest_mode: ${value}\n`);
|
||||
expect({ value, out }).toEqual({ value, out: `TRANSCRIPT_MODE: ask (stored value '${value}' is not recognized by this version)` });
|
||||
}
|
||||
expect(runGate(block, null)).toBe('TRANSCRIPT_MODE: ask (not set)');
|
||||
});
|
||||
|
||||
test(`${name}: two questions, nothing stored until both are answered, scope before mode`, () => {
|
||||
const text = readFileSync(file, 'utf-8');
|
||||
expect(text).toContain('`new`');
|
||||
expect(text).toMatch(/second, separate question/i);
|
||||
expect(text).toMatch(/only this repo/i);
|
||||
expect(text).toMatch(/store nothing until\s+both answers are known/i);
|
||||
const scopeAt = text.indexOf('gstack-config set transcript_repos');
|
||||
const modeAt = text.indexOf('gstack-config set transcript_ingest_mode', scopeAt);
|
||||
expect(scopeAt).toBeGreaterThan(0);
|
||||
expect(modeAt).toBeGreaterThan(scopeAt);
|
||||
expect(text).toContain('gstack-config unset transcript_repos');
|
||||
});
|
||||
}
|
||||
});
|
||||
@@ -0,0 +1,180 @@
|
||||
/**
|
||||
* Scoped transcript consent across the publication path: the ingest stages
|
||||
* transcript pages under ~/.gstack/transcripts/ (remote-http mode), and
|
||||
* gstack-brain-sync purges out-of-scope staged pages and scans its unpushed
|
||||
* commits before every push, so a page outside the current consent is never
|
||||
* published. Local bare repo as the remote; stub gbrain; no network.
|
||||
*/
|
||||
import { afterEach, beforeEach, describe, expect, test } from 'bun:test';
|
||||
import { spawnSync } from 'child_process';
|
||||
import { chmodSync, existsSync, mkdirSync, mkdtempSync, readdirSync, readFileSync, rmSync, writeFileSync } from 'fs';
|
||||
import { tmpdir } from 'os';
|
||||
import { join, relative } from 'path';
|
||||
|
||||
const ROOT = join(import.meta.dir, '..');
|
||||
const BIN = join(ROOT, 'bin');
|
||||
const DAY = 24 * 60 * 60 * 1000;
|
||||
const iso = (ms: number) => new Date(ms).toISOString().replace(/\.\d{3}Z$/, 'Z');
|
||||
|
||||
let home: string;
|
||||
let gstackHome: string;
|
||||
let remote: string;
|
||||
|
||||
beforeEach(() => {
|
||||
home = mkdtempSync(join(tmpdir(), 'gstack-bscope-'));
|
||||
gstackHome = join(home, '.gstack');
|
||||
remote = join(home, 'remote.git');
|
||||
mkdirSync(gstackHome, { recursive: true });
|
||||
spawnSync('git', ['init', '--bare', '-q', '-b', 'main', remote], { timeout: 30_000 });
|
||||
const bindir = join(home, 'bin');
|
||||
mkdirSync(bindir);
|
||||
writeFileSync(join(bindir, 'gbrain'), `#!/bin/sh\ncase "$1" in\n --help) printf 'Commands:\\n import <dir> Import markdown directory\\n' ;;\n *) echo '{}' ;;\nesac\n`);
|
||||
chmodSync(join(bindir, 'gbrain'), 0o755);
|
||||
// Remote-http MCP mode: the ingest stages to ~/.gstack/transcripts/ for brain-sync to push.
|
||||
writeFileSync(join(home, '.claude.json'), JSON.stringify({ mcpServers: { gbrain: { type: 'url', url: 'https://brain.example/mcp' } } }));
|
||||
});
|
||||
afterEach(() => {
|
||||
rmSync(home, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
const env = () => ({
|
||||
...process.env,
|
||||
HOME: home,
|
||||
GSTACK_HOME: gstackHome,
|
||||
GIT_CONFIG_NOSYSTEM: '1',
|
||||
GIT_CONFIG_GLOBAL: join(home, '.gitconfig'),
|
||||
PATH: `${join(home, 'bin')}:${process.env.PATH || ''}`,
|
||||
});
|
||||
const run = (cmd: string, ...args: string[]) => {
|
||||
const r = spawnSync(join(BIN, cmd), args, { encoding: 'utf-8', timeout: 60_000, cwd: home, env: env() });
|
||||
return { status: r.status, out: r.stdout ?? '', err: r.stderr ?? '' };
|
||||
};
|
||||
const ingest = () => {
|
||||
const r = spawnSync('bun', [join(BIN, 'gstack-memory-ingest.ts'), '--bulk', '--quiet', '--sources', 'transcript'], { encoding: 'utf-8', timeout: 60_000, cwd: home, env: env() });
|
||||
return { status: r.status, err: r.stderr ?? '' };
|
||||
};
|
||||
const remoteFiles = () =>
|
||||
spawnSync('git', ['--git-dir=' + remote, 'ls-tree', '-r', '--name-only', 'main'], { encoding: 'utf-8', timeout: 30_000 }).stdout.split('\n').filter((l) => l.startsWith('transcripts/'));
|
||||
const stagedOnDisk = (): string[] => {
|
||||
const out: string[] = [];
|
||||
const walk = (dir: string) => {
|
||||
if (!existsSync(dir)) return;
|
||||
for (const e of readdirSync(dir, { withFileTypes: true })) {
|
||||
if (e.isDirectory()) walk(join(dir, e.name));
|
||||
else if (e.name.endsWith('.md')) out.push(relative(gstackHome, join(dir, e.name)));
|
||||
}
|
||||
};
|
||||
walk(join(gstackHome, 'transcripts'));
|
||||
return out;
|
||||
};
|
||||
|
||||
function repo(slug: string): string {
|
||||
const dir = join(home, 'work', slug.replace('/', '-'));
|
||||
mkdirSync(dir, { recursive: true });
|
||||
spawnSync('git', ['-C', dir, 'init', '-q'], { timeout: 30_000 });
|
||||
spawnSync('git', ['-C', dir, 'remote', 'add', 'origin', `https://github.com/${slug}.git`], { timeout: 30_000 });
|
||||
return dir;
|
||||
}
|
||||
|
||||
function session(id: string, cwd: string, start: string, appendNow = false): void {
|
||||
const dir = join(home, '.claude', 'projects', 'p');
|
||||
mkdirSync(dir, { recursive: true });
|
||||
let body = JSON.stringify({ type: 'user', message: { role: 'user', content: `hello ${id}` }, cwd, timestamp: start }) + '\n';
|
||||
if (appendNow) body += JSON.stringify({ type: 'user', message: { role: 'user', content: 'later' }, timestamp: iso(Date.now()) }) + '\n';
|
||||
writeFileSync(join(dir, `${id}.jsonl`), body);
|
||||
}
|
||||
|
||||
function initArtifacts(): void {
|
||||
expect(run('gstack-artifacts-init', '--remote', remote).status).toBe(0);
|
||||
expect(run('gstack-config', 'set', 'artifacts_sync_mode', 'full').status).toBe(0);
|
||||
}
|
||||
|
||||
/** A staged transcript page as the ingest renders it. */
|
||||
function stagePage(rel: string, remoteName: string, start: string): void {
|
||||
mkdirSync(join(gstackHome, rel, '..'), { recursive: true });
|
||||
writeFileSync(join(gstackHome, rel), `---\nagent: claude-code\nsession_id: ${rel}\ngit_remote: ${remoteName}\nstart_time: ${start}\nsession_started_at: ${start}\nsource_path: /src/${rel}\ntitle: "x"\ntype: transcript\n---\n\nbody\n`);
|
||||
}
|
||||
|
||||
describe.skipIf(process.platform === 'win32')('scoped consent end to end', () => {
|
||||
test('new@: a pre-cutoff session appended later is never staged or pushed; a new one is', () => {
|
||||
initArtifacts();
|
||||
const now = Date.now();
|
||||
expect(run('gstack-config', 'set', 'transcript_ingest_mode', `new@${iso(now - DAY)}`).status).toBe(0);
|
||||
const app = repo('acme/app');
|
||||
session('preappend0001', app, iso(now - 2 * DAY), true);
|
||||
session('postcutoff001', app, iso(now - 60_000));
|
||||
expect(ingest().status).toBe(0);
|
||||
expect(stagedOnDisk().join('\n')).not.toMatch(/preappend000/);
|
||||
expect(stagedOnDisk().join('\n')).toMatch(/postcutoff00/);
|
||||
expect(run('gstack-brain-sync', '--discover-new').status).toBe(0);
|
||||
expect(run('gstack-brain-sync', '--once').status).toBe(0);
|
||||
expect(remoteFiles().join('\n')).toMatch(/postcutoff00/);
|
||||
expect(remoteFiles().join('\n')).not.toMatch(/preappend000/);
|
||||
});
|
||||
|
||||
test('allow, stage, revoke, allow: the purge removes the page and its fingerprint, so an unchanged source re-stages', () => {
|
||||
const app = repo('acme/app');
|
||||
repo('acme/web');
|
||||
session('appsession001', app, iso(Date.now()));
|
||||
expect(run('gstack-config', 'set', 'transcript_ingest_mode', 'all').status).toBe(0);
|
||||
expect(run('gstack-config', 'set', 'transcript_repos', 'github.com/acme/app').status).toBe(0);
|
||||
expect(ingest().status).toBe(0);
|
||||
expect(stagedOnDisk()).toHaveLength(1);
|
||||
const statePath = join(gstackHome, '.transcript-ingest-state.json');
|
||||
expect(Object.keys(JSON.parse(readFileSync(statePath, 'utf-8')).sessions)).toHaveLength(1);
|
||||
|
||||
expect(run('gstack-config', 'set', 'transcript_repos', 'github.com/acme/web').status).toBe(0);
|
||||
const revoked = ingest();
|
||||
expect(revoked.err).toContain('[memory-ingest] removed 1 staged transcript pages outside the new scope');
|
||||
expect(stagedOnDisk()).toEqual([]);
|
||||
expect(Object.keys(JSON.parse(readFileSync(statePath, 'utf-8')).sessions)).toEqual([]);
|
||||
|
||||
expect(run('gstack-config', 'set', 'transcript_repos', 'github.com/acme/app').status).toBe(0);
|
||||
expect(ingest().status).toBe(0);
|
||||
expect(stagedOnDisk()).toHaveLength(1);
|
||||
});
|
||||
});
|
||||
|
||||
describe.skipIf(process.platform === 'win32')('the unpublished range is scanned before every push', () => {
|
||||
function strandCommit(): void {
|
||||
initArtifacts();
|
||||
expect(run('gstack-config', 'set', 'transcript_ingest_mode', 'all').status).toBe(0);
|
||||
stagePage('transcripts/run-1-1/app.md', 'github.com/acme/app', iso(Date.now()));
|
||||
stagePage('transcripts/run-1-1/web.md', 'github.com/acme/web', iso(Date.now()));
|
||||
run('gstack-brain-enqueue', 'transcripts/run-1-1/app.md');
|
||||
run('gstack-brain-enqueue', 'transcripts/run-1-1/web.md');
|
||||
const hook = join(remote, 'hooks', 'pre-receive');
|
||||
writeFileSync(hook, '#!/bin/sh\nexit 1\n');
|
||||
chmodSync(hook, 0o755);
|
||||
expect(run('gstack-brain-sync', '--once').status).toBe(0);
|
||||
expect(remoteFiles()).toEqual([]);
|
||||
rmSync(hook);
|
||||
writeFileSync(join(gstackHome, '.brain-last-push-attempt'), '0');
|
||||
}
|
||||
|
||||
test('failed push, narrower allowlist, retry: the excluded page is rewritten out and purged; the rest pushes', () => {
|
||||
strandCommit();
|
||||
expect(run('gstack-config', 'set', 'transcript_repos', 'github.com/acme/app').status).toBe(0);
|
||||
const retry = run('gstack-brain-sync', '--once');
|
||||
expect(retry.status).toBe(0);
|
||||
expect(retry.err).toContain('removed 1 excluded transcript pages from unpublished sync commits');
|
||||
expect(remoteFiles()).toEqual(['transcripts/run-1-1/app.md']);
|
||||
expect(existsSync(join(gstackHome, 'transcripts/run-1-1/web.md'))).toBe(false);
|
||||
const log = spawnSync('git', ['--git-dir=' + remote, 'log', '--format=%an %s', 'main'], { encoding: 'utf-8', timeout: 30_000 }).stdout;
|
||||
expect(log).toContain('rewritten without 1 excluded transcript page(s)');
|
||||
});
|
||||
|
||||
test('failed push, then off: nothing transcript is pushed; pages wait on disk and in the queue until consent returns', () => {
|
||||
strandCommit();
|
||||
expect(run('gstack-config', 'set', 'transcript_ingest_mode', 'off').status).toBe(0);
|
||||
const retry = run('gstack-brain-sync', '--once');
|
||||
expect(retry.status).toBe(0);
|
||||
expect(retry.err).toContain('removed 2 excluded transcript pages from unpublished sync commits');
|
||||
expect(remoteFiles()).toEqual([]);
|
||||
expect(stagedOnDisk().sort()).toEqual(['transcripts/run-1-1/app.md', 'transcripts/run-1-1/web.md']);
|
||||
|
||||
expect(run('gstack-config', 'set', 'transcript_ingest_mode', 'all').status).toBe(0);
|
||||
expect(run('gstack-brain-sync', '--once').status).toBe(0);
|
||||
expect(remoteFiles().sort()).toEqual(['transcripts/run-1-1/app.md', 'transcripts/run-1-1/web.md']);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,258 @@
|
||||
/**
|
||||
* Scoped transcript consent in the memory ingest and the sync orchestrator:
|
||||
* the new@ cutoff on each session's first record (both parsers), the
|
||||
* transcript_repos allowlist after the attribution gate, exclusion counts by
|
||||
* reason, explicit overrides that never widen a scoped consent, the consent
|
||||
* record that restages on a scope change, and the regression contract that
|
||||
* `recent` selects exactly the sessions it did before (file mtime, 90 days).
|
||||
* Runs the real ingest and orchestrator against a stub gbrain.
|
||||
*/
|
||||
import { describe, expect, test } from 'bun:test';
|
||||
import { chmodSync, existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, utimesSync, writeFileSync } from 'fs';
|
||||
import { tmpdir } from 'os';
|
||||
import { join } from 'path';
|
||||
import { spawnSync } from 'child_process';
|
||||
|
||||
import { transcriptConsentRecord, transcriptConsentSummary } from '../bin/gstack-gbrain-sync';
|
||||
import { reposHash } from '../lib/transcript-consent';
|
||||
|
||||
const ROOT = join(import.meta.dir, '..');
|
||||
const INGEST = join(ROOT, 'bin', 'gstack-memory-ingest.ts');
|
||||
const SYNC = join(ROOT, 'bin', 'gstack-gbrain-sync.ts');
|
||||
const DAY = 24 * 60 * 60 * 1000;
|
||||
const iso = (ms: number) => new Date(ms).toISOString().replace(/\.\d{3}Z$/, 'Z');
|
||||
|
||||
interface Box {
|
||||
home: string;
|
||||
gstackHome: string;
|
||||
env: Record<string, string>;
|
||||
repo: (slug: string) => string;
|
||||
claude: (id: string, opts: { cwd: string; start?: string; mtimeMs?: number; appendNow?: boolean }) => string;
|
||||
codex: (id: string, opts: { cwd: string; start: string }) => string;
|
||||
staged: () => string[];
|
||||
cleanup: () => void;
|
||||
}
|
||||
|
||||
function box(config: string): Box {
|
||||
const home = mkdtempSync(join(tmpdir(), 'gstack-tscope-'));
|
||||
const gstackHome = join(home, '.gstack');
|
||||
const bindir = join(home, 'bin');
|
||||
const stagedList = join(home, 'staged.list');
|
||||
for (const d of [gstackHome, bindir, join(home, '.gbrain'), join(home, '.claude', 'projects', 'p')]) mkdirSync(d, { recursive: true });
|
||||
writeFileSync(join(gstackHome, 'config.yaml'), config);
|
||||
writeFileSync(join(home, '.gbrain', 'config.json'), JSON.stringify({ engine: 'pglite', database_url: 'pglite:///test' }));
|
||||
writeFileSync(join(bindir, 'gbrain'), `#!/bin/sh
|
||||
case "$1" in
|
||||
--version) echo "gbrain 0.60.28.0" ;;
|
||||
--help) printf 'Commands:\\n import <dir> Import markdown directory\\n' ;;
|
||||
sources) echo '{"sources":[]}' ;;
|
||||
import)
|
||||
( cd "$2" && find . -name '*.md' -type f | sort ) > "${stagedList}"
|
||||
n=$(wc -l < "${stagedList}" | tr -d ' ')
|
||||
echo "{\\"status\\":\\"success\\",\\"imported\\":$n,\\"skipped\\":0,\\"errors\\":0,\\"total_files\\":$n}" ;;
|
||||
*) echo "unexpected gbrain $*" >&2; exit 1 ;;
|
||||
esac
|
||||
`);
|
||||
chmodSync(join(bindir, 'gbrain'), 0o755);
|
||||
const env = {
|
||||
HOME: home,
|
||||
GSTACK_HOME: gstackHome,
|
||||
GBRAIN_HOME: '',
|
||||
GSTACK_MEMORY_INGEST_SOURCES: '',
|
||||
GIT_CONFIG_NOSYSTEM: '1',
|
||||
GIT_CONFIG_GLOBAL: join(home, '.gitconfig'),
|
||||
PATH: `${bindir}:${process.env.PATH || ''}`,
|
||||
};
|
||||
const line = (rec: Record<string, unknown>) => JSON.stringify(rec) + '\n';
|
||||
return {
|
||||
home,
|
||||
gstackHome,
|
||||
env,
|
||||
repo: (slug) => {
|
||||
const dir = join(home, 'work', slug.replace('/', '-'));
|
||||
mkdirSync(dir, { recursive: true });
|
||||
spawnSync('git', ['-C', dir, 'init', '-q'], { timeout: 30_000 });
|
||||
spawnSync('git', ['-C', dir, 'remote', 'add', 'origin', `https://github.com/${slug}.git`], { timeout: 30_000 });
|
||||
return dir;
|
||||
},
|
||||
claude: (id, { cwd, start, mtimeMs, appendNow }) => {
|
||||
const path = join(home, '.claude', 'projects', 'p', `${id}.jsonl`);
|
||||
const ts = start ? { timestamp: start } : {};
|
||||
writeFileSync(path, line({ type: 'user', message: { role: 'user', content: `hello ${id}` }, cwd, ...ts }) + line({ type: 'assistant', message: { role: 'assistant', content: 'ok' }, ...ts }));
|
||||
if (appendNow) writeFileSync(path, readFileSync(path, 'utf-8') + line({ type: 'user', message: { role: 'user', content: 'later' }, timestamp: iso(Date.now()) }));
|
||||
if (mtimeMs !== undefined) utimesSync(path, new Date(mtimeMs), new Date(mtimeMs));
|
||||
return path;
|
||||
},
|
||||
codex: (id, { cwd, start }) => {
|
||||
const dir = join(home, '.codex', 'sessions', '2026', '10', '01');
|
||||
mkdirSync(dir, { recursive: true });
|
||||
const path = join(dir, `rollout-${id}.jsonl`);
|
||||
writeFileSync(path, line({ type: 'session_meta', timestamp: start, payload: { id, cwd } }) + line({ type: 'response_item', timestamp: start, payload: { type: 'message', role: 'user', content: [{ type: 'input_text', text: `hi ${id}` }] } }));
|
||||
return path;
|
||||
},
|
||||
staged: () => (existsSync(stagedList) ? readFileSync(stagedList, 'utf-8').split('\n').filter((l) => l.includes('transcripts/')) : []),
|
||||
cleanup: () => rmSync(home, { recursive: true, force: true }),
|
||||
};
|
||||
}
|
||||
|
||||
const ingest = (b: Box, args: string[], env: Record<string, string> = {}) =>
|
||||
spawnSync('bun', [INGEST, ...args], { encoding: 'utf-8', timeout: 60_000, cwd: b.home, env: { ...process.env, ...b.env, ...env } });
|
||||
const sync = (b: Box, args: string[], env: Record<string, string> = {}) =>
|
||||
spawnSync('bun', [SYNC, '--no-code', '--no-brain-sync', ...args], { encoding: 'utf-8', timeout: 60_000, cwd: b.home, env: { ...process.env, ...b.env, ...env } });
|
||||
|
||||
describe.skipIf(process.platform === 'win32')('new@ cutoff', () => {
|
||||
test('both parsers filter on the first record timestamp; appended pre-cutoff and start-less sessions stay out', () => {
|
||||
const now = Date.now();
|
||||
const b = box(`transcript_ingest_mode: new@${iso(now - DAY)}\n`);
|
||||
try {
|
||||
const app = b.repo('acme/app');
|
||||
b.claude('preappend0001', { cwd: app, start: iso(now - 2 * DAY), appendNow: true });
|
||||
b.claude('postcutoff001', { cwd: app, start: iso(now - 60_000) });
|
||||
b.claude('nostart000001', { cwd: app });
|
||||
b.codex('codexold00001', { cwd: app, start: iso(now - 3 * DAY) });
|
||||
b.codex('codexnew00001', { cwd: app, start: iso(now - 60_000) });
|
||||
|
||||
const probe = ingest(b, ['--probe']);
|
||||
expect(probe.status).toBe(0);
|
||||
expect(probe.stdout).toContain('Total files in window: 2');
|
||||
expect(probe.stdout).toMatch(/Excluded \(pre-cutoff\):\s+2/);
|
||||
expect(probe.stdout).toMatch(/Excluded \(missing start\):\s+1/);
|
||||
|
||||
const bulk = ingest(b, ['--bulk', '--quiet']);
|
||||
expect(bulk.status).toBe(0);
|
||||
const staged = b.staged().join('\n');
|
||||
expect(staged).toMatch(/postcutoff00/);
|
||||
expect(staged).toMatch(/codexnew0000/);
|
||||
expect(staged).not.toMatch(/preappend000|nostart00000|codexold0000/);
|
||||
const page = readFileSync(join(b.gstackHome, '.transcript-ingest-state.json'), 'utf-8');
|
||||
expect(page).not.toContain('preappend0001');
|
||||
} finally {
|
||||
b.cleanup();
|
||||
}
|
||||
});
|
||||
|
||||
test('a run that ingests no session says so, with the pre-cutoff count', () => {
|
||||
const now = Date.now();
|
||||
const b = box(`transcript_ingest_mode: new@${iso(now - DAY)}\n`);
|
||||
try {
|
||||
b.claude('preappend0001', { cwd: b.repo('acme/app'), start: iso(now - 2 * DAY), appendNow: true });
|
||||
const r = ingest(b, ['--bulk', '--quiet']);
|
||||
expect(r.status).toBe(0);
|
||||
expect(r.stderr).toContain(`[memory-ingest] 0 transcript sessions ingested under new@${iso(now - DAY)}: 1 started before the cutoff; new sessions will appear on the next sync`);
|
||||
} finally {
|
||||
b.cleanup();
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
describe.skipIf(process.platform === 'win32')('transcript_repos allowlist', () => {
|
||||
test('only allowlisted repos ingest, after the attribution gate, with the exclusion counted', () => {
|
||||
const b = box('transcript_ingest_mode: all+repos\ntranscript_repos: github.com/acme/app\n');
|
||||
try {
|
||||
b.claude('appsession001', { cwd: b.repo('acme/app'), start: iso(Date.now()) });
|
||||
b.claude('websession001', { cwd: b.repo('acme/web'), start: iso(Date.now()) });
|
||||
b.codex('codexweb00001', { cwd: join(b.home, 'work', 'acme-web'), start: iso(Date.now()) });
|
||||
const probe = ingest(b, ['--probe']);
|
||||
expect(probe.stdout).toContain('Total files in window: 1');
|
||||
expect(probe.stdout).toMatch(/Excluded \(not allowlisted\):\s+2/);
|
||||
ingest(b, ['--bulk', '--quiet']);
|
||||
expect(b.staged().join('\n')).toMatch(/appsession00/);
|
||||
expect(b.staged().join('\n')).not.toMatch(/websession00|codexweb0000/);
|
||||
} finally {
|
||||
b.cleanup();
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
describe.skipIf(process.platform === 'win32')('explicit overrides never widen a scoped consent', () => {
|
||||
for (const [args, env, why] of [
|
||||
[['--sources', 'transcript,learning'], {}, '--sources'],
|
||||
[[], { GSTACK_MEMORY_INGEST_SOURCES: 'transcript' }, 'GSTACK_MEMORY_INGEST_SOURCES'],
|
||||
] as const) {
|
||||
test(`${why}: the cutoff still applies under consent, and a broken allowlist still denies`, () => {
|
||||
const now = Date.now();
|
||||
const b = box(`transcript_ingest_mode: new@${iso(now - DAY)}\n`);
|
||||
try {
|
||||
const app = b.repo('acme/app');
|
||||
b.claude('preappend0001', { cwd: app, start: iso(now - 2 * DAY), appendNow: true });
|
||||
b.claude('postcutoff001', { cwd: app, start: iso(now - 60_000) });
|
||||
const scoped = sync(b, ['--incremental', '--quiet', ...args], { ...env });
|
||||
expect(scoped.status).toBe(0);
|
||||
expect(b.staged().join('\n')).toMatch(/postcutoff00/);
|
||||
expect(b.staged().join('\n')).not.toMatch(/preappend000/);
|
||||
|
||||
writeFileSync(join(b.gstackHome, 'config.yaml'), 'transcript_ingest_mode: recent+repos\n');
|
||||
rmSync(join(b.gstackHome, '.transcript-ingest-state.json'), { force: true });
|
||||
rmSync(join(b.home, 'staged.list'), { force: true });
|
||||
const denied = sync(b, ['--incremental', '--quiet', ...args], { ...env });
|
||||
expect(denied.status).toBe(0);
|
||||
expect(b.staged()).toEqual([]);
|
||||
expect(denied.stderr).toContain(`gbrain-sync: transcripts ingested because ${why} names transcript (transcript_ingest_mode=recent+repos). The new@ cutoff and transcript_repos allowlist still apply.`);
|
||||
} finally {
|
||||
b.cleanup();
|
||||
}
|
||||
});
|
||||
}
|
||||
});
|
||||
|
||||
describe.skipIf(process.platform === 'win32')('consent record and notice', () => {
|
||||
test('the stage record carries the cutoff and allowlist hash; unscoped records are unchanged', () => {
|
||||
const cutoff = '2026-10-03T17:00:00Z';
|
||||
expect(transcriptConsentRecord({ affirmative: true, window: 'recent', reason: 'recent', value: 'recent' }, false)).toEqual({ mode: 'recent', window: 'recent' });
|
||||
expect(transcriptConsentRecord({ affirmative: true, window: 'new', reason: 'new', value: `new@${cutoff}+repos`, cutoff, repos: ['github.com/a/b'] }, false))
|
||||
.toEqual({ mode: 'new', window: 'new', cutoff, repos_hash: reposHash(['github.com/a/b']) });
|
||||
});
|
||||
|
||||
test('the consent notice prints the effective consent in words, unless --quiet', () => {
|
||||
const p = { affirmative: true, window: 'recent' as const, reason: 'recent' as const, value: 'recent+repos', repos: ['github.com/a/b'], roots: ['/s'] };
|
||||
expect(transcriptConsentSummary(p, false)).toBe('gbrain-sync: transcripts: sessions from the last 90 days; repo github.com/a/b; config: /s');
|
||||
expect(transcriptConsentSummary(p, true)).toBeNull();
|
||||
expect(transcriptConsentSummary({ ...p, affirmative: false, reason: 'off' }, false)).toBeNull();
|
||||
});
|
||||
|
||||
test('a changed cutoff restages an interrupted import', () => {
|
||||
const now = Date.now();
|
||||
const b = box(`transcript_ingest_mode: new@${iso(now - DAY)}\n`);
|
||||
try {
|
||||
const stagingDir = join(b.gstackHome, '.staging-ingest-1-1');
|
||||
mkdirSync(join(stagingDir, 'projects'), { recursive: true });
|
||||
writeFileSync(join(stagingDir, '.gstack-staging'), '');
|
||||
writeFileSync(join(stagingDir, 'projects', 'stale-page.md'), '# stale\n');
|
||||
writeFileSync(join(b.home, '.gbrain', 'import-checkpoint.json'), JSON.stringify({ dir: stagingDir, processedIndex: 0, totalFiles: 1 }));
|
||||
const { MEMORY_INGEST_TYPES } = require('../bin/gstack-gbrain-sync');
|
||||
writeFileSync(join(b.gstackHome, '.gbrain-sync-state.json'), JSON.stringify({
|
||||
schema_version: 1,
|
||||
last_writer: 'gstack-gbrain-sync',
|
||||
last_stages: [{ name: 'memory', ran: true, ok: false, duration_ms: 1, summary: 'timeout', memory_sources: [...MEMORY_INGEST_TYPES], transcript_consent: { mode: 'new', window: 'new', cutoff: iso(now - 2 * DAY) } }],
|
||||
}));
|
||||
const r = sync(b, ['--incremental', '--quiet']);
|
||||
expect(r.stderr).toContain('gbrain-sync: transcript consent changed since the interrupted import');
|
||||
expect(r.stderr).not.toContain('resuming from gbrain checkpoint');
|
||||
} finally {
|
||||
b.cleanup();
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
describe.skipIf(process.platform === 'win32')('regression contract: recent selects the same sessions as before', () => {
|
||||
test('recent keeps the 90-day file-mtime window, whatever the session start', () => {
|
||||
const now = Date.now();
|
||||
const b = box('transcript_ingest_mode: recent\n');
|
||||
try {
|
||||
const app = b.repo('acme/app');
|
||||
b.claude('mtimenowold1', { cwd: app, start: iso(now - 200 * DAY) });
|
||||
b.claude('mtimeoldnew1', { cwd: app, start: iso(now), mtimeMs: now - 200 * DAY });
|
||||
b.claude('bothrecent01', { cwd: app, start: iso(now - DAY) });
|
||||
b.claude('botholdxxxx1', { cwd: app, start: iso(now - 200 * DAY), mtimeMs: now - 200 * DAY });
|
||||
b.claude('nostampsxxx1', { cwd: app });
|
||||
const r = ingest(b, ['--bulk', '--quiet']);
|
||||
expect(r.status).toBe(0);
|
||||
const ids = b.staged().map((l) => l.replace(/^.*-([a-z0-9]{12})\.md$/, '$1')).sort();
|
||||
expect(ids).toEqual(['bothrecent01', 'mtimenowold1', 'nostampsxxx1']);
|
||||
const probe = ingest(b, ['--probe']);
|
||||
expect(probe.stdout).not.toContain('Excluded');
|
||||
} finally {
|
||||
b.cleanup();
|
||||
}
|
||||
});
|
||||
});
|
||||
Reference in new issue
Block a user