diff --git a/bin/gstack-brain-sync b/bin/gstack-brain-sync index bc7ac76d5..093fc2693 100755 --- a/bin/gstack-brain-sync +++ b/bin/gstack-brain-sync @@ -10,6 +10,9 @@ # gstack-brain-sync --drop-queue --yes clear queue without committing # gstack-brain-sync --discover-new scan allowlist dirs, enqueue changed files # gstack-brain-sync --attention print skill start's attention lines +# gstack-brain-sync --publish-removals [--yes] +# list (or, with --yes, commit and push) +# the removal of synced files gone from disk # # Invoked by the preamble at skill START and END boundaries. No persistent # daemon. Typical run <1s when queue empty; ~200-800ms with network push. @@ -18,6 +21,14 @@ # with every other gstack writer to ~/.gstack's git state. A drain that finds # it held skips; the next skill boundary catches up. # +# Removals: with artifacts_sync_removals=on, each drain also publishes the +# removal of synced files that are gone from disk (a deleted design doc, the +# old directory after a project slug migration), so the remote and the brain +# stop serving stale copies. A drain that would remove more than +# REMOVAL_VALVE files at once publishes none of them and says so: that many +# at once looks like a loss, not a cleanup. --publish-removals --yes publishes +# them on demand, whatever the setting. +# # A file the secret scan flags is held back on its own (with the rest of its # coupled group) and stays queued; everything else syncs. `--status` lists # each held file, the scanner rule that matched (never the matched text) and @@ -98,6 +109,8 @@ SELF_BIN="$SCRIPT_DIR/gstack-brain-sync" # independent. COUPLED_GROUPS='[{"name": "decisions", "members": ["projects/*/decisions.jsonl", "projects/*/decisions.active.json"], "check": "decisions-active"}]' +REMOVAL_VALVE=20 + # Fields every status write of this drain carries (see write_status). ST_HELD='[]' ST_DRAINABLE=0 @@ -911,6 +924,104 @@ hold_flagged_paths() { return 1 } +# Write to $1 the tracked files gone from disk that the sync may remove, one +# per line: allowlisted, not transcript pages (their consent guard owns those), +# and a coupled group only when every member is gone. +removal_candidates() { + git -C "$GSTACK_HOME" ls-files -d -z > "$1.gone" 2>/dev/null || { rm -f "$1.gone"; return 1; } + python3 - "$GSTACK_HOME" "$ALLOWLIST" "$COUPLED_GROUPS" "$1" <<'PYEOF' +import fnmatch, json, os, re, sys +home, allowlist_path, coupled, out = sys.argv[1:5] +try: + with open(allowlist_path) as f: + globs = [l.strip() for l in f if l.strip() and not l.lstrip().startswith("#")] +except FileNotFoundError: + globs = [] +with open(out + ".gone", "rb") as f: + gone = {p for p in f.read().decode("utf-8", "surrogateescape").split("\0") if p} +os.remove(out + ".gone") +def removable(p): + if p.split("/")[0] == "transcripts" or not any(fnmatch.fnmatchcase(p, g) for g in globs): + return False + for spec in json.loads(coupled): + for member in spec["members"]: + m = re.fullmatch(re.escape(member).replace(r"\*", "([^/]+)"), p) + if m: + return all(not os.path.exists(os.path.join(home, x.replace("*", m.group(1), 1))) for x in spec["members"]) + return True +with open(out, "w", newline="\n", encoding="utf-8", errors="surrogateescape") as f: + for p in sorted(gone): + if removable(p): + f.write(p + "\n") +PYEOF +} + +# Commit the removal of the paths listed in $1 and push it, receipted like +# every push. Returns 0 pushed, 1 commit failed, 2 committed but not pushed +# (the run-start detector retries it). +publish_removals() { + local list="$1" n + n=$(grep -c . "$list" 2>/dev/null || true) + git --literal-pathspecs -C "$GSTACK_HOME" -c user.email="gstack@localhost" -c user.name="gstack-brain-sync" \ + commit -q -m "sync: remove $n file(s) | $(date -u +%Y-%m-%dT%H:%M:%SZ)" --pathspec-from-file="$list" >/dev/null 2>&1 || return 1 + guard_unpublished_range || return 2 + GSTACK_HOME="$GSTACK_HOME" _receipted_git closed brain-sync "$(remote_host)" curated-memory-git-push "artifacts_sync_mode!=off" \ + bash -c 'GIT_TERMINAL_PROMPT=0 git -c http.lowSpeedLimit=1024 -c http.lowSpeedTime=30 -C "$1" push origin HEAD 2>/dev/null' _ "$GSTACK_HOME" || return 2 + date -u +%Y-%m-%dT%H:%M:%SZ > "$LAST_PUSH_FILE" +} + +# The drain's removal step (artifacts_sync_removals=on). Records the outcome +# in ST_NOTE; sets REMOVALS_PUSHED when the removals reached the remote. +sync_removals() { + local list n rc=0 + REMOVALS_PUSHED=0 + list=$(mktemp "${TMPDIR:-/tmp}/brain-sync-removals.XXXXXX") || return 0 + if ! removal_candidates "$list"; then + rm -f "$list" "$list.gone" + ST_NOTE="$ST_NOTE; could not list removed files; none published" + return 0 + fi + n=$(grep -c . "$list" 2>/dev/null || true) + if [ "$n" -gt "$REMOVAL_VALVE" ]; then + ST_NOTE="$ST_NOTE; $n synced file(s) are gone from disk, more than $REMOVAL_VALVE at once, so none of the removals were published (review: $SELF_BIN --publish-removals; publish: $SELF_BIN --publish-removals --yes)" + echo "BRAIN_SYNC: holding $n removal(s), more than $REMOVAL_VALVE at once; review with $SELF_BIN --publish-removals" >&2 + elif [ "$n" -gt 0 ]; then + publish_removals "$list" || rc=$? + case "$rc" in + 0) REMOVALS_PUSHED=1; ST_NOTE="$ST_NOTE; published $n removal(s)" ;; + 1) ST_NOTE="$ST_NOTE; committing $n removal(s) failed; the next sync retries" ;; + *) ST_NOTE="$ST_NOTE; $n removal(s) committed locally but not pushed" ;; + esac + fi + rm -f "$list" +} + +subcmd_publish_removals() { + local confirm="${1:-}" list n rc=0 + sync_active || { echo "BRAIN_SYNC: artifacts sync is not active; nothing to publish" >&2; exit 1; } + gstack_brain_lock_acquire "$GSTACK_HOME" try || { echo "BRAIN_SYNC: another sync is running; try again" >&2; exit 1; } + list=$(mktemp "${TMPDIR:-/tmp}/brain-sync-removals.XXXXXX") || exit 1 + trap 'rm -f "$list" "$list.gone"; gstack_brain_lock_release' EXIT INT TERM + removal_candidates "$list" || { echo "BRAIN_SYNC: could not list removed files" >&2; exit 1; } + n=$(grep -c . "$list" 2>/dev/null || true) + if [ "$n" -eq 0 ]; then + echo "no removals to publish" + exit 0 + fi + if [ "$confirm" != "--yes" ]; then + echo "$n synced file(s) are gone from disk:" + sed 's/^/ /' "$list" + echo "publish their removal: $SELF_BIN --publish-removals --yes" + exit 0 + fi + publish_removals "$list" || rc=$? + case "$rc" in + 0) echo "published $n removal(s)" ;; + 1) echo "BRAIN_SYNC: error: committing the removals failed" >&2; exit 1 ;; + *) echo "BRAIN_SYNC: $n removal(s) committed locally; the push failed and retries at skill start" >&2; exit 1 ;; + esac +} + subcmd_once() { if ! sync_active; then # Silent no-op when feature not initialized / disabled. @@ -962,6 +1073,11 @@ subcmd_once() { fi fi + # Removals go first, so a removal commit whose push fails is one more + # unpushed commit for the detector below. + REMOVALS_PUSHED=0 + [ "$("$CONFIG_BIN" get artifacts_sync_removals 2>/dev/null || echo off)" = "on" ] && sync_removals + # #2549 unpushed-commit detector: a prior drain may have COMMITTED but # failed to push (auth blip, offline). The data was never lost — it sits in # a local commit — but nothing re-pushed it until NEW changes arrived. @@ -1030,7 +1146,7 @@ subcmd_once() { write_status "push_failed" "$det_total local commit(s) are not pushed yet; the push retries at a skill start at most every 10 minutes (commits made outside gstack-brain-sync are never auto-pushed: push them with git -C $GSTACK_HOME push origin HEAD)" exit 0 fi - write_status "idle" "queue empty" + if [ "$REMOVALS_PUSHED" = 1 ]; then write_status "ok" "queue empty"; else write_status "idle" "queue empty"; fi exit 0 fi @@ -1472,6 +1588,7 @@ case "${1:-}" in --drop-queue) shift; subcmd_drop_queue "${1:-}" ;; --discover-new) subcmd_discover_new ;; --attention) subcmd_attention ;; + --publish-removals) shift; subcmd_publish_removals "${1:-}" ;; --help|-h) awk 'NR > 1 && /^# Heredoc delivery guard/ { exit } NR > 1' "$0" | sed 's/^# \{0,1\}//' ;; diff --git a/bin/gstack-config b/bin/gstack-config index 84c46c57f..bb7067d67 100755 --- a/bin/gstack-config +++ b/bin/gstack-config @@ -92,6 +92,11 @@ CONFIG_HEADER='# gstack configuration — edit freely, changes take effect on ne # # Set to true once the privacy gate has asked the user. # # Flip back to false to be re-prompted. # +# artifacts_sync_removals: off # on | off — on: each sync also publishes the +# # removal of synced files gone from disk; more than +# # 20 at once is held for gstack-brain-sync +# # --publish-removals. See docs/gbrain-sync.md. +# # ─── Transcript ingest (gbrain memory) ─────────────────────────────── # transcript_ingest_mode: off # recent | all | off | new@ — consent for # # /sync-gbrain to ingest Claude Code and Codex @@ -252,6 +257,7 @@ lookup_default() { cross_project_learnings) echo "" ;; # intentionally empty → unset triggers first-time prompt artifacts_sync_mode) echo "off" ;; artifacts_sync_mode_prompted) echo "false" ;; + artifacts_sync_removals) echo "off" ;; # on | off — publish removals of synced files gone from disk plan_tune_hooks) echo "prompt" ;; # prompt | yes | no — controls ./setup plan-tune hook install timeline_stop_hook) echo "yes" ;; # yes | no — controls ./setup timeline Stop hook registration (#2677) @@ -665,6 +671,10 @@ case "${1:-}" in echo "Warning: artifacts_sync_mode '$VALUE' not recognized. Valid values: off, artifacts-only, full. Using off." >&2 VALUE="off" fi + if [ "$KEY" = "artifacts_sync_removals" ] && [ "$VALUE" != "on" ] && [ "$VALUE" != "off" ]; then + echo "Warning: artifacts_sync_removals '$VALUE' not recognized. Valid values: on, off. Using off." >&2 + VALUE="off" + fi # redact_repo_visibility: a LOCAL override for repos gh/glab can't read (e.g. # self-hosted GitLab). It lives in ~/.gstack/config.yaml (never committed), so # it can't be used to weaken the gate repo-wide for other contributors. @@ -925,7 +935,7 @@ case "${1:-}" in for KEY in proactive routing_declined telemetry auto_upgrade update_check \ skill_prefix explain_level \ codex_reviews gstack_contributor skip_eng_review workspace_root \ - artifacts_sync_mode artifacts_sync_mode_prompted plan_tune_hooks \ + artifacts_sync_mode artifacts_sync_mode_prompted artifacts_sync_removals plan_tune_hooks \ timeline_stop_hook design_detector design_detector_install_prompted memorable_recall \ browse_extension_id ship_measure_rule_trials ship_measure_behavior_panels \ ship_measure_judge_outputs ship_measure_ask_per_trial_usd ship_measure_budget_usd \ @@ -950,7 +960,7 @@ case "${1:-}" in for KEY in proactive routing_declined telemetry auto_upgrade update_check \ skill_prefix explain_level \ codex_reviews gstack_contributor skip_eng_review workspace_root \ - artifacts_sync_mode artifacts_sync_mode_prompted plan_tune_hooks \ + artifacts_sync_mode artifacts_sync_mode_prompted artifacts_sync_removals plan_tune_hooks \ timeline_stop_hook design_detector design_detector_install_prompted memorable_recall \ browse_extension_id ship_measure_rule_trials ship_measure_behavior_panels \ ship_measure_judge_outputs ship_measure_ask_per_trial_usd ship_measure_budget_usd \ diff --git a/docs/gbrain-sync.md b/docs/gbrain-sync.md index 6e308d89e..3e4348784 100644 --- a/docs/gbrain-sync.md +++ b/docs/gbrain-sync.md @@ -114,6 +114,28 @@ gstack-config set artifacts_sync_mode full gstack-config set artifacts_sync_mode off ``` +## Removed and moved files + +By default the sync never publishes a removal: delete a synced design doc, +or move a project directory (a project slug migration does), and the old +path stays on the remote, where the brain keeps indexing it next to the new +copy. To publish removals too: + +```bash +gstack-config set artifacts_sync_removals on +``` + +Each sync then commits the removal of synced files that are gone from disk. +Transcript pages are left alone, and a decision log and its active snapshot +go only together. A sync that would remove more than 20 files at once +publishes none of them: that looks like a loss, not a cleanup. Review and +publish those yourself, whatever the setting: + +```bash +gstack-brain-sync --publish-removals # list them +gstack-brain-sync --publish-removals --yes # commit and push their removal +``` + ## Secret protection Every commit is scanned for credential-shaped content before it leaves diff --git a/test/brain-sync-removals.test.ts b/test/brain-sync-removals.test.ts new file mode 100644 index 000000000..66b457375 --- /dev/null +++ b/test/brain-sync-removals.test.ts @@ -0,0 +1,153 @@ +/** + * gstack-brain-sync removals (artifacts_sync_removals): off by default, a + * synced file gone from disk stays published; on, the drain publishes its + * removal, and a drain that would remove more than 20 files at once publishes + * none of them until `--publish-removals --yes`. Temp GSTACK_HOME, local bare + * remote; no network. + */ +import { describe, test, expect, beforeEach, afterEach } from 'bun:test'; +import * as fs from 'fs'; +import * as os from 'os'; +import * as path from 'path'; +import { spawnSync } from 'child_process'; + +const ROOT = path.resolve(import.meta.dir, '..'); + +let home: string; +let remote: string; + +function run(argv: string[]) { + const r = spawnSync(path.join(ROOT, 'bin', argv[0]), argv.slice(1), { + env: { ...process.env, HOME: home, GSTACK_HOME: home }, + encoding: 'utf-8', + timeout: 60_000, + }); + return { stdout: r.stdout || '', stderr: r.stderr || '', status: r.status ?? -1 }; +} + +const remoteFiles = () => + spawnSync('git', ['--git-dir=' + remote, 'ls-tree', '-r', '--name-only', 'main'], { encoding: 'utf-8', timeout: 30_000 }) + .stdout.split('\n').filter((p) => p.startsWith('projects/') || p.startsWith('transcripts/')); +const status = () => JSON.parse(fs.readFileSync(path.join(home, '.brain-sync-status.json'), 'utf-8')); + +function write(p: string, body = '# doc\n') { + fs.mkdirSync(path.dirname(path.join(home, p)), { recursive: true }); + fs.writeFileSync(path.join(home, p), body); +} + +function syncAll(files: string[]) { + for (const f of files) write(f); + for (const f of files) run(['gstack-brain-enqueue', f]); + const r = run(['gstack-brain-sync', '--once']); + expect(r.status).toBe(0); + expect(remoteFiles()).toEqual(expect.arrayContaining(files)); +} + +const designs = (dir: string, n: number) => Array.from({ length: n }, (_, i) => `projects/${dir}/designs/d${i}.md`); + +beforeEach(() => { + home = fs.mkdtempSync(path.join(os.tmpdir(), 'brain-removals-home-')); + remote = fs.mkdtempSync(path.join(os.tmpdir(), 'brain-removals-remote-')); + spawnSync('git', ['init', '--bare', '-q', '-b', 'main', remote], { timeout: 30_000 }); + expect(run(['gstack-artifacts-init', '--remote', remote]).status).toBe(0); + run(['gstack-config', 'set', 'artifacts_sync_mode', 'full']); +}); + +afterEach(() => { + fs.rmSync(home, { recursive: true, force: true }); + fs.rmSync(remote, { recursive: true, force: true }); +}); + +describe('artifacts_sync_removals', () => { + test('off by default: a deleted synced file stays published', () => { + expect(run(['gstack-config', 'get', 'artifacts_sync_removals']).stdout.trim()).toBe('off'); + syncAll(['projects/p/designs/a.md', 'projects/p/designs/b.md']); + fs.rmSync(path.join(home, 'projects/p/designs/a.md')); + run(['gstack-brain-sync', '--once']); + expect(remoteFiles()).toContain('projects/p/designs/a.md'); + }); + + test('on: a slug migration removes the old copies with the queue empty or not', () => { + run(['gstack-config', 'set', 'artifacts_sync_removals', 'on']); + syncAll(['projects/acme/designs/x.md', 'projects/acme/designs/y.md']); + fs.renameSync(path.join(home, 'projects/acme'), path.join(home, 'projects/owner-acme')); + run(['gstack-brain-sync', '--discover-new']); + expect(run(['gstack-brain-sync', '--once']).status).toBe(0); + expect(remoteFiles().sort()).toEqual(['projects/owner-acme/designs/x.md', 'projects/owner-acme/designs/y.md']); + expect(status().message).toContain('published 2 removal(s)'); + + fs.rmSync(path.join(home, 'projects/owner-acme/designs/y.md')); + expect(run(['gstack-brain-sync', '--once']).status).toBe(0); + expect(remoteFiles()).toEqual(['projects/owner-acme/designs/x.md']); + expect(status().status).toBe('ok'); + }); + + test('on: tracked files outside the allowlist and transcript pages are never removed', () => { + run(['gstack-config', 'set', 'artifacts_sync_removals', 'on']); + syncAll(['projects/p/designs/a.md']); + write('projects/p/notes.txt'); + write('transcripts/run-1/s1.md'); + spawnSync('git', ['-C', home, 'add', '-f', 'projects/p/notes.txt', 'transcripts/run-1/s1.md'], { timeout: 30_000 }); + spawnSync('git', ['-C', home, '-c', 'user.email=u@example.com', '-c', 'user.name=u', 'commit', '-qm', 'manual'], { timeout: 30_000 }); + spawnSync('git', ['-C', home, 'push', '-q', 'origin', 'HEAD'], { timeout: 30_000 }); + for (const p of ['projects/p/notes.txt', 'transcripts/run-1/s1.md', 'projects/p/designs/a.md']) fs.rmSync(path.join(home, p)); + run(['gstack-brain-sync', '--once']); + expect(remoteFiles().sort()).toEqual(['projects/p/notes.txt', 'transcripts/run-1/s1.md']); + }); + + test('on: a decision log goes only together with its active snapshot', () => { + run(['gstack-config', 'set', 'artifacts_sync_removals', 'on']); + const LOG = 'projects/p/decisions.jsonl'; + const SNAP = 'projects/p/decisions.active.json'; + write(SNAP, JSON.stringify([{ id: 'a', kind: 'decide', note: '' }])); + write(LOG, JSON.stringify({ id: 'a', kind: 'decide', decision: 'd a', scope: 'repo', date: '2026-10-01', source: 'user' }) + '\n'); + run(['gstack-brain-enqueue', LOG]); + run(['gstack-brain-sync', '--once']); + expect(remoteFiles()).toEqual(expect.arrayContaining([LOG, SNAP])); + fs.rmSync(path.join(home, LOG)); + run(['gstack-brain-sync', '--once']); + expect(remoteFiles()).toEqual(expect.arrayContaining([LOG, SNAP])); + fs.rmSync(path.join(home, SNAP)); + run(['gstack-brain-sync', '--once']); + expect(remoteFiles()).toEqual([]); + }); + + test('on: more than 20 removals at once publishes none until --publish-removals --yes', () => { + run(['gstack-config', 'set', 'artifacts_sync_removals', 'on']); + const files = designs('big', 21); + syncAll(files); + fs.rmSync(path.join(home, 'projects/big'), { recursive: true }); + const drain = run(['gstack-brain-sync', '--once']); + expect(drain.stderr).toContain('holding 21 removal(s), more than 20 at once'); + expect(remoteFiles().sort()).toEqual([...files].sort()); + expect(status().message).toContain('none of the removals were published'); + + const listed = run(['gstack-brain-sync', '--publish-removals']); + expect(listed.stdout).toContain('21 synced file(s) are gone from disk:'); + expect(listed.stdout).toContain(' projects/big/designs/d0.md'); + expect(remoteFiles()).toHaveLength(21); + + const published = run(['gstack-brain-sync', '--publish-removals', '--yes']); + expect(published.status).toBe(0); + expect(published.stdout).toContain('published 21 removal(s)'); + expect(remoteFiles()).toEqual([]); + }); + + test('--publish-removals --yes works with the setting off and only touches removals', () => { + syncAll(['projects/p/designs/a.md', 'projects/p/designs/b.md']); + fs.rmSync(path.join(home, 'projects/p/designs/a.md')); + write('projects/p/designs/b.md', '# changed, not yet synced\n'); + const r = run(['gstack-brain-sync', '--publish-removals', '--yes']); + expect(r.status).toBe(0); + expect(remoteFiles()).toEqual(['projects/p/designs/b.md']); + const remoteB = spawnSync('git', ['--git-dir=' + remote, 'show', 'main:projects/p/designs/b.md'], { encoding: 'utf-8', timeout: 30_000 }); + expect(remoteB.stdout).toBe('# doc\n'); + expect(run(['gstack-brain-sync', '--publish-removals', '--yes']).stdout).toContain('no removals to publish'); + }); + + test('config rejects values other than on and off', () => { + const r = run(['gstack-config', 'set', 'artifacts_sync_removals', 'yes']); + expect(r.stderr).toContain("artifacts_sync_removals 'yes' not recognized"); + expect(run(['gstack-config', 'get', 'artifacts_sync_removals']).stdout.trim()).toBe('off'); + }); +});