diff --git a/README.md b/README.md index 4d387b6bb..cee9ea2ce 100644 --- a/README.md +++ b/README.md @@ -712,6 +712,8 @@ types into that element; bare `browse type ` types into whatever has focus **Windows users:** gstack works on Windows 11 via Git Bash or WSL. Aside is macOS-only, so on Windows (and Linux) the browser skills, `/make-pdf`, and `/diagram` always use gstack's bundled browser. Node.js is required in addition to Bun — Bun has a known bug with Playwright's pipe transport on Windows ([bun#4253](https://github.com/oven-sh/bun/issues/4253)). The browse server automatically falls back to Node.js. Make sure both `bun` and `node` are on your PATH. Native `/cso` additionally requires Windows PowerShell and Visual Studio 2022 Build Tools with the Desktop development with C++ workload; setup leaves that skill explicitly unavailable when they are absent. +From PowerShell, `.\setup.ps1` (same arguments as `./setup`) checks that Git for Windows, Bun and Node.js are on PATH, prints the `winget` command for any that are missing, and otherwise runs `./setup` in Git Bash. + On Windows without Developer Mode (MSYS2 / Git Bash), `setup` falls back to file copies instead of symlinks because `ln -snf` produces frozen copies that don't refresh on `git pull`. **Re-run `cd ~/.claude/skills/gstack && ./setup` after every `git pull`** so your skill files match the repo. `setup` prints a one-line note reminding you. Unix and WSL keep symlinks and don't need the re-run. **Chromium install failed or hung during `./setup`?** The bundled browser is diff --git a/bin/gstack-brain-sync b/bin/gstack-brain-sync index bc7ac76d5..093fc2693 100755 --- a/bin/gstack-brain-sync +++ b/bin/gstack-brain-sync @@ -10,6 +10,9 @@ # gstack-brain-sync --drop-queue --yes clear queue without committing # gstack-brain-sync --discover-new scan allowlist dirs, enqueue changed files # gstack-brain-sync --attention print skill start's attention lines +# gstack-brain-sync --publish-removals [--yes] +# list (or, with --yes, commit and push) +# the removal of synced files gone from disk # # Invoked by the preamble at skill START and END boundaries. No persistent # daemon. Typical run <1s when queue empty; ~200-800ms with network push. @@ -18,6 +21,14 @@ # with every other gstack writer to ~/.gstack's git state. A drain that finds # it held skips; the next skill boundary catches up. # +# Removals: with artifacts_sync_removals=on, each drain also publishes the +# removal of synced files that are gone from disk (a deleted design doc, the +# old directory after a project slug migration), so the remote and the brain +# stop serving stale copies. A drain that would remove more than +# REMOVAL_VALVE files at once publishes none of them and says so: that many +# at once looks like a loss, not a cleanup. --publish-removals --yes publishes +# them on demand, whatever the setting. +# # A file the secret scan flags is held back on its own (with the rest of its # coupled group) and stays queued; everything else syncs. `--status` lists # each held file, the scanner rule that matched (never the matched text) and @@ -98,6 +109,8 @@ SELF_BIN="$SCRIPT_DIR/gstack-brain-sync" # independent. COUPLED_GROUPS='[{"name": "decisions", "members": ["projects/*/decisions.jsonl", "projects/*/decisions.active.json"], "check": "decisions-active"}]' +REMOVAL_VALVE=20 + # Fields every status write of this drain carries (see write_status). ST_HELD='[]' ST_DRAINABLE=0 @@ -911,6 +924,104 @@ hold_flagged_paths() { return 1 } +# Write to $1 the tracked files gone from disk that the sync may remove, one +# per line: allowlisted, not transcript pages (their consent guard owns those), +# and a coupled group only when every member is gone. +removal_candidates() { + git -C "$GSTACK_HOME" ls-files -d -z > "$1.gone" 2>/dev/null || { rm -f "$1.gone"; return 1; } + python3 - "$GSTACK_HOME" "$ALLOWLIST" "$COUPLED_GROUPS" "$1" <<'PYEOF' +import fnmatch, json, os, re, sys +home, allowlist_path, coupled, out = sys.argv[1:5] +try: + with open(allowlist_path) as f: + globs = [l.strip() for l in f if l.strip() and not l.lstrip().startswith("#")] +except FileNotFoundError: + globs = [] +with open(out + ".gone", "rb") as f: + gone = {p for p in f.read().decode("utf-8", "surrogateescape").split("\0") if p} +os.remove(out + ".gone") +def removable(p): + if p.split("/")[0] == "transcripts" or not any(fnmatch.fnmatchcase(p, g) for g in globs): + return False + for spec in json.loads(coupled): + for member in spec["members"]: + m = re.fullmatch(re.escape(member).replace(r"\*", "([^/]+)"), p) + if m: + return all(not os.path.exists(os.path.join(home, x.replace("*", m.group(1), 1))) for x in spec["members"]) + return True +with open(out, "w", newline="\n", encoding="utf-8", errors="surrogateescape") as f: + for p in sorted(gone): + if removable(p): + f.write(p + "\n") +PYEOF +} + +# Commit the removal of the paths listed in $1 and push it, receipted like +# every push. Returns 0 pushed, 1 commit failed, 2 committed but not pushed +# (the run-start detector retries it). +publish_removals() { + local list="$1" n + n=$(grep -c . "$list" 2>/dev/null || true) + git --literal-pathspecs -C "$GSTACK_HOME" -c user.email="gstack@localhost" -c user.name="gstack-brain-sync" \ + commit -q -m "sync: remove $n file(s) | $(date -u +%Y-%m-%dT%H:%M:%SZ)" --pathspec-from-file="$list" >/dev/null 2>&1 || return 1 + guard_unpublished_range || return 2 + GSTACK_HOME="$GSTACK_HOME" _receipted_git closed brain-sync "$(remote_host)" curated-memory-git-push "artifacts_sync_mode!=off" \ + bash -c 'GIT_TERMINAL_PROMPT=0 git -c http.lowSpeedLimit=1024 -c http.lowSpeedTime=30 -C "$1" push origin HEAD 2>/dev/null' _ "$GSTACK_HOME" || return 2 + date -u +%Y-%m-%dT%H:%M:%SZ > "$LAST_PUSH_FILE" +} + +# The drain's removal step (artifacts_sync_removals=on). Records the outcome +# in ST_NOTE; sets REMOVALS_PUSHED when the removals reached the remote. +sync_removals() { + local list n rc=0 + REMOVALS_PUSHED=0 + list=$(mktemp "${TMPDIR:-/tmp}/brain-sync-removals.XXXXXX") || return 0 + if ! removal_candidates "$list"; then + rm -f "$list" "$list.gone" + ST_NOTE="$ST_NOTE; could not list removed files; none published" + return 0 + fi + n=$(grep -c . "$list" 2>/dev/null || true) + if [ "$n" -gt "$REMOVAL_VALVE" ]; then + ST_NOTE="$ST_NOTE; $n synced file(s) are gone from disk, more than $REMOVAL_VALVE at once, so none of the removals were published (review: $SELF_BIN --publish-removals; publish: $SELF_BIN --publish-removals --yes)" + echo "BRAIN_SYNC: holding $n removal(s), more than $REMOVAL_VALVE at once; review with $SELF_BIN --publish-removals" >&2 + elif [ "$n" -gt 0 ]; then + publish_removals "$list" || rc=$? + case "$rc" in + 0) REMOVALS_PUSHED=1; ST_NOTE="$ST_NOTE; published $n removal(s)" ;; + 1) ST_NOTE="$ST_NOTE; committing $n removal(s) failed; the next sync retries" ;; + *) ST_NOTE="$ST_NOTE; $n removal(s) committed locally but not pushed" ;; + esac + fi + rm -f "$list" +} + +subcmd_publish_removals() { + local confirm="${1:-}" list n rc=0 + sync_active || { echo "BRAIN_SYNC: artifacts sync is not active; nothing to publish" >&2; exit 1; } + gstack_brain_lock_acquire "$GSTACK_HOME" try || { echo "BRAIN_SYNC: another sync is running; try again" >&2; exit 1; } + list=$(mktemp "${TMPDIR:-/tmp}/brain-sync-removals.XXXXXX") || exit 1 + trap 'rm -f "$list" "$list.gone"; gstack_brain_lock_release' EXIT INT TERM + removal_candidates "$list" || { echo "BRAIN_SYNC: could not list removed files" >&2; exit 1; } + n=$(grep -c . "$list" 2>/dev/null || true) + if [ "$n" -eq 0 ]; then + echo "no removals to publish" + exit 0 + fi + if [ "$confirm" != "--yes" ]; then + echo "$n synced file(s) are gone from disk:" + sed 's/^/ /' "$list" + echo "publish their removal: $SELF_BIN --publish-removals --yes" + exit 0 + fi + publish_removals "$list" || rc=$? + case "$rc" in + 0) echo "published $n removal(s)" ;; + 1) echo "BRAIN_SYNC: error: committing the removals failed" >&2; exit 1 ;; + *) echo "BRAIN_SYNC: $n removal(s) committed locally; the push failed and retries at skill start" >&2; exit 1 ;; + esac +} + subcmd_once() { if ! sync_active; then # Silent no-op when feature not initialized / disabled. @@ -962,6 +1073,11 @@ subcmd_once() { fi fi + # Removals go first, so a removal commit whose push fails is one more + # unpushed commit for the detector below. + REMOVALS_PUSHED=0 + [ "$("$CONFIG_BIN" get artifacts_sync_removals 2>/dev/null || echo off)" = "on" ] && sync_removals + # #2549 unpushed-commit detector: a prior drain may have COMMITTED but # failed to push (auth blip, offline). The data was never lost — it sits in # a local commit — but nothing re-pushed it until NEW changes arrived. @@ -1030,7 +1146,7 @@ subcmd_once() { write_status "push_failed" "$det_total local commit(s) are not pushed yet; the push retries at a skill start at most every 10 minutes (commits made outside gstack-brain-sync are never auto-pushed: push them with git -C $GSTACK_HOME push origin HEAD)" exit 0 fi - write_status "idle" "queue empty" + if [ "$REMOVALS_PUSHED" = 1 ]; then write_status "ok" "queue empty"; else write_status "idle" "queue empty"; fi exit 0 fi @@ -1472,6 +1588,7 @@ case "${1:-}" in --drop-queue) shift; subcmd_drop_queue "${1:-}" ;; --discover-new) subcmd_discover_new ;; --attention) subcmd_attention ;; + --publish-removals) shift; subcmd_publish_removals "${1:-}" ;; --help|-h) awk 'NR > 1 && /^# Heredoc delivery guard/ { exit } NR > 1' "$0" | sed 's/^# \{0,1\}//' ;; diff --git a/bin/gstack-config b/bin/gstack-config index 84c46c57f..bb7067d67 100755 --- a/bin/gstack-config +++ b/bin/gstack-config @@ -92,6 +92,11 @@ CONFIG_HEADER='# gstack configuration — edit freely, changes take effect on ne # # Set to true once the privacy gate has asked the user. # # Flip back to false to be re-prompted. # +# artifacts_sync_removals: off # on | off — on: each sync also publishes the +# # removal of synced files gone from disk; more than +# # 20 at once is held for gstack-brain-sync +# # --publish-removals. See docs/gbrain-sync.md. +# # ─── Transcript ingest (gbrain memory) ─────────────────────────────── # transcript_ingest_mode: off # recent | all | off | new@ — consent for # # /sync-gbrain to ingest Claude Code and Codex @@ -252,6 +257,7 @@ lookup_default() { cross_project_learnings) echo "" ;; # intentionally empty → unset triggers first-time prompt artifacts_sync_mode) echo "off" ;; artifacts_sync_mode_prompted) echo "false" ;; + artifacts_sync_removals) echo "off" ;; # on | off — publish removals of synced files gone from disk plan_tune_hooks) echo "prompt" ;; # prompt | yes | no — controls ./setup plan-tune hook install timeline_stop_hook) echo "yes" ;; # yes | no — controls ./setup timeline Stop hook registration (#2677) @@ -665,6 +671,10 @@ case "${1:-}" in echo "Warning: artifacts_sync_mode '$VALUE' not recognized. Valid values: off, artifacts-only, full. Using off." >&2 VALUE="off" fi + if [ "$KEY" = "artifacts_sync_removals" ] && [ "$VALUE" != "on" ] && [ "$VALUE" != "off" ]; then + echo "Warning: artifacts_sync_removals '$VALUE' not recognized. Valid values: on, off. Using off." >&2 + VALUE="off" + fi # redact_repo_visibility: a LOCAL override for repos gh/glab can't read (e.g. # self-hosted GitLab). It lives in ~/.gstack/config.yaml (never committed), so # it can't be used to weaken the gate repo-wide for other contributors. @@ -925,7 +935,7 @@ case "${1:-}" in for KEY in proactive routing_declined telemetry auto_upgrade update_check \ skill_prefix explain_level \ codex_reviews gstack_contributor skip_eng_review workspace_root \ - artifacts_sync_mode artifacts_sync_mode_prompted plan_tune_hooks \ + artifacts_sync_mode artifacts_sync_mode_prompted artifacts_sync_removals plan_tune_hooks \ timeline_stop_hook design_detector design_detector_install_prompted memorable_recall \ browse_extension_id ship_measure_rule_trials ship_measure_behavior_panels \ ship_measure_judge_outputs ship_measure_ask_per_trial_usd ship_measure_budget_usd \ @@ -950,7 +960,7 @@ case "${1:-}" in for KEY in proactive routing_declined telemetry auto_upgrade update_check \ skill_prefix explain_level \ codex_reviews gstack_contributor skip_eng_review workspace_root \ - artifacts_sync_mode artifacts_sync_mode_prompted plan_tune_hooks \ + artifacts_sync_mode artifacts_sync_mode_prompted artifacts_sync_removals plan_tune_hooks \ timeline_stop_hook design_detector design_detector_install_prompted memorable_recall \ browse_extension_id ship_measure_rule_trials ship_measure_behavior_panels \ ship_measure_judge_outputs ship_measure_ask_per_trial_usd ship_measure_budget_usd \ diff --git a/bin/gstack-redact-prepush b/bin/gstack-redact-prepush index 98e9b931c..31a87c97e 100755 --- a/bin/gstack-redact-prepush +++ b/bin/gstack-redact-prepush @@ -24,6 +24,10 @@ * - `pii.email` skips the pusher's own address, `gstack.redact.allowEmail` * entries, and addresses already in the destination's commit metadata * (see emailScanOptions). + * - `.gstack-redact-allowlist`, read from each pushed commit (never the + * working tree), lists exact matched spans reviewed as benign, one per + * line, `#` comments. Only a finding whose matched text equals an entry is + * suppressed; every push that carries entries lists what they suppressed. * - GSTACK_REDACT_PREPUSH=skip → log + exit 0 (escape valve). * * Installed/uninstalled via `gstack-redact install-prepush-hook` (see the @@ -549,11 +553,40 @@ function emailScanOptions(historyRevs: string[], trackingRemote: string | null): }; } +const ALLOWLIST_PATH = ".gstack-redact-allowlist"; +const ALLOWLIST_MAX_BYTES = 64 * 1024; + +/** + * Entries of the allowlist committed in `sha`. Read from the pushed commit so + * a suppression exists only as a reviewable commit. Absent, unreadable or + * oversized means no entries: the file can record a reviewed exception, never + * loosen the scan by accident. + */ +function committedAllowlist(sha: string): string[] { + const spec = `${sha}:${ALLOWLIST_PATH}`; + const size = Number(git(["cat-file", "-s", spec]).trim()); + if (!size) return []; + if (size > ALLOWLIST_MAX_BYTES) { + process.stderr.write( + `gstack-redact-prepush: ${ALLOWLIST_PATH} in ${sha.slice(0, 12)} is over ${ALLOWLIST_MAX_BYTES} bytes; ignoring it.\n`, + ); + return []; + } + return git(["cat-file", "blob", spec]).split("\n") + .map((l) => l.trim()) + .filter((l) => l && !l.startsWith("#")); +} + +/** File line (and commit) of a finding. Never the matched value. */ +function findingWhere(f: Finding, part: AddedFile): string { + const line = part.lines[f.line - 1]; + return `${part.file || "(unknown file)"}${line ? `:${line}` : ""}` + + (part.commit ? ` in commit ${part.commit.slice(0, 12)}` : ""); +} + /** One MEDIUM report line: rule id, file line (and commit), and the fix. Never the matched value. */ function mediumLine(f: Finding, part: AddedFile): string { - const line = part.lines[f.line - 1]; - const where = `${part.file || "(unknown file)"}${line ? `:${line}` : ""}` + - (part.commit ? ` in commit ${part.commit.slice(0, 12)}` : ""); + const where = findingWhere(f, part); const fix = f.id === "pii.email" ? "remove the address, or allow it: git config --add gstack.redact.allowEmail
" : "remove the value before it becomes public"; @@ -574,7 +607,8 @@ function main() { .filter(Boolean) .map((l) => l.split(/\s+/)); - const parts: AddedFile[] = []; + const parts: Array<{ part: AddedFile; allowlist: string[] }> = []; + const allowEntries = new Set(); const historyRevs = new Set(); let trackingRemote: string | null = null; @@ -610,7 +644,9 @@ function main() { ); process.exit(1); } - parts.push(...added.filter((part) => part.text.trim())); + const allowlist = committedAllowlist(localSha); + for (const entry of allowlist) allowEntries.add(entry); + for (const part of added) if (part.text.trim()) parts.push({ part, allowlist }); historyRevs.add(localSha); if (!ZERO.test(remoteSha) && objectExists(remoteSha)) historyRevs.add(remoteSha); else { @@ -628,9 +664,21 @@ function main() { // Sliced (see scanAddedLines) so a large-but-legitimate diff is actually // scanned rather than blocked unscanned on the engine's size cap. // Scanned per file so a pattern can use the file's type (#3048). - for (const part of parts) { + // An allowlisted span is found by scanning with and without the entries, + // so the report names exactly what they suppressed. + const allowed: string[] = []; + const findingKey = (f: Finding) => `${f.id}:${f.line}:${f.col}`; + for (const { part, allowlist } of parts) { const opts = { repoVisibility: "private" as const, ...emails.options, ...(part.file ? { sourcePath: part.file } : {}) }; - for (const f of scanAddedLines(part.text, opts)) { + const findings = scanAddedLines(part.text, opts); + const kept = allowlist.length + ? new Set(scanAddedLines(part.text, { ...opts, allowlist }).map(findingKey)) + : null; + for (const f of findings) { + if (kept && !kept.has(findingKey(f))) { + if (f.severity === "HIGH" || f.severity === "MEDIUM") allowed.push(` ${f.severity} ${f.id} ${findingWhere(f, part)}\n`); + continue; + } if (f.severity === "HIGH") allHigh.push(f); else if (f.severity === "MEDIUM") { mediums.push(mediumLine(f, part)); @@ -639,6 +687,14 @@ function main() { } } + if (allowEntries.size > 0) { + process.stderr.write( + `gstack-redact-prepush: ${ALLOWLIST_PATH} (${allowEntries.size} entr${allowEntries.size === 1 ? "y" : "ies"}) ` + + `suppressed ${allowed.length} finding(s) in this push${allowed.length ? ":" : "."}\n`, + ); + for (const line of allowed) process.stderr.write(line); + } + if (mediums.length > 0) { process.stderr.write( `gstack-redact-prepush: ${mediums.length} MEDIUM finding(s) in pushed diff (PII/internal). ` + @@ -673,7 +729,9 @@ function main() { process.stderr.write(` HIGH ${f.id} ${f.preview}\n`); } process.stderr.write( - "\nRotate the credential (a pushed secret is compromised) and remove it from the diff.\n", + "\nRotate the credential (a pushed secret is compromised) and remove it from the diff.\n" + + `If the value is a known-benign sample, commit its exact matched text as a line of ${ALLOWLIST_PATH}\n` + + "at the repo root instead of reshaping code to dodge the pattern; every push lists what that file suppresses.\n", ); } diff --git a/docs/CONTRIBUTOR_REFERENCE.md b/docs/CONTRIBUTOR_REFERENCE.md index 45a05a3f8..b6b89d996 100644 --- a/docs/CONTRIBUTOR_REFERENCE.md +++ b/docs/CONTRIBUTOR_REFERENCE.md @@ -134,6 +134,9 @@ determined leaker (a CHANGELOG line that does would fail a hostile screenshotter - **Config keys:** `redact_repo_visibility` (public|private|unknown, local-only override for repos gh/glab can't read), `redact_prepush_hook` (true|false). There is intentionally NO key to disable HIGH blocking. +- **Committed allowlist:** `.gstack-redact-allowlist` at the root of the pushed commit lists + exact matched spans reviewed as benign; the pre-push hook suppresses only a + finding whose span equals an entry and lists every suppression on each push. - **Audit:** the /spec semantic pass appends a content-free record (categories + body sha256, no spec text) to `~/.gstack/security/semantic-reviews.jsonl` (0600). diff --git a/docs/gbrain-sync.md b/docs/gbrain-sync.md index 6e308d89e..3e4348784 100644 --- a/docs/gbrain-sync.md +++ b/docs/gbrain-sync.md @@ -114,6 +114,28 @@ gstack-config set artifacts_sync_mode full gstack-config set artifacts_sync_mode off ``` +## Removed and moved files + +By default the sync never publishes a removal: delete a synced design doc, +or move a project directory (a project slug migration does), and the old +path stays on the remote, where the brain keeps indexing it next to the new +copy. To publish removals too: + +```bash +gstack-config set artifacts_sync_removals on +``` + +Each sync then commits the removal of synced files that are gone from disk. +Transcript pages are left alone, and a decision log and its active snapshot +go only together. A sync that would remove more than 20 files at once +publishes none of them: that looks like a loss, not a cleanup. Review and +publish those yourself, whatever the setting: + +```bash +gstack-brain-sync --publish-removals # list them +gstack-brain-sync --publish-removals --yes # commit and push their removal +``` + ## Secret protection Every commit is scanned for credential-shaped content before it leaves diff --git a/docs/troubleshooting.md b/docs/troubleshooting.md index f71008f53..64c78c1bf 100644 --- a/docs/troubleshooting.md +++ b/docs/troubleshooting.md @@ -1120,6 +1120,24 @@ up deployed, so it still blocks. **Fix.** Use `postgres://postgres:postgres@localhost:5432/...` in local and CI config, or read the URL from an env var. If the credential is real, rotate it. Bypass once: `GSTACK_REDACT_PREPUSH=skip git push`. +If the URL is a reviewed, public dev-only value, list it in +[`.gstack-redact-allowlist`](#redact-allowlist). + + +### `.gstack-redact-allowlist ( entries) suppressed finding(s) in this push` + +**Meaning.** The pushed commit carries `.gstack-redact-allowlist` at the repo +root. Each line (after trimming; `#` starts a comment) is one exact matched +span: the whole `postgres://USER:PASSWORD@host:port` URL for +`db.url_with_password`, the key for a key pattern, the address for `pii.email`. A finding is suppressed only when its +matched text equals an entry, so a password alone, a substring, or a different +key still reports and still blocks. The hook reads the file from each pushed +commit, never the working tree, and ignores a file over 64 KiB. Every push that +carries entries prints this line and each suppressed finding's file and line. +It works alongside `gstack.redact.allowEmail`, which stays a local setting. + +**Fix.** Nothing, if every listed suppression is the reviewed value. Remove an +entry that no longer applies; rotate a credential that is real. ### `pii.ip_public` MEDIUM on a four-part version number diff --git a/setup.ps1 b/setup.ps1 new file mode 100644 index 000000000..ab3b82386 --- /dev/null +++ b/setup.ps1 @@ -0,0 +1,54 @@ +#Requires -Version 5.1 +<# +.SYNOPSIS + Windows entry point for gstack's ./setup. + +.DESCRIPTION + Checks that Git for Windows, Bun and Node.js are on PATH, then runs ./setup + from this directory in Git Bash. Every argument goes to ./setup unchanged, + as separate arguments; none is pasted into shell source. + + .\setup.ps1 + .\setup.ps1 --host codex + + Set GSTACK_GIT_BASH to a bash.exe to use instead of Git for Windows' own. +#> +Set-StrictMode -Version Latest +$ErrorActionPreference = 'Stop' + +$missing = @( + @{ Cmd = 'git'; Name = 'Git for Windows'; Id = 'Git.Git' }, + @{ Cmd = 'bun'; Name = 'Bun'; Id = 'Oven-sh.Bun' }, + @{ Cmd = 'node'; Name = 'Node.js'; Id = 'OpenJS.NodeJS.LTS' } +) | Where-Object { -not (Get-Command $_.Cmd -CommandType Application -ErrorAction SilentlyContinue) } +if ($missing) { + Write-Host 'gstack setup needs these on PATH first:' + foreach ($tool in $missing) { Write-Host (' {0}: winget install --id {1} -e' -f $tool.Name, $tool.Id) } + Write-Host 'Install them, open a new PowerShell window and run .\setup.ps1 again. Nothing was installed or changed.' + exit 1 +} + +# Git Bash, found from git.exe (Git\cmd, Git\bin or Git\mingw64\bin). Never a +# bare `bash` from PATH: on Windows that is often WSL's, with a different home. +$bash = $env:GSTACK_GIT_BASH +if (-not $bash) { + $dir = Split-Path (Get-Command git -CommandType Application | Select-Object -First 1).Source + for ($i = 0; $i -lt 3 -and $dir; $i++) { + $candidate = Join-Path $dir 'bin\bash.exe' + if (Test-Path -LiteralPath $candidate -PathType Leaf) { $bash = $candidate; break } + $dir = Split-Path $dir + } +} +if (-not $bash -or -not (Test-Path -LiteralPath $bash -PathType Leaf)) { + Write-Host 'Git Bash (bash.exe) was not found next to git. Reinstall Git for Windows, or set GSTACK_GIT_BASH to its bash.exe.' + exit 1 +} + +Push-Location -LiteralPath $PSScriptRoot +try { + & $bash -c 'exec ./setup "$@"' setup @args + $code = $LASTEXITCODE +} finally { + Pop-Location +} +exit $code diff --git a/test/brain-sync-removals.test.ts b/test/brain-sync-removals.test.ts new file mode 100644 index 000000000..66b457375 --- /dev/null +++ b/test/brain-sync-removals.test.ts @@ -0,0 +1,153 @@ +/** + * gstack-brain-sync removals (artifacts_sync_removals): off by default, a + * synced file gone from disk stays published; on, the drain publishes its + * removal, and a drain that would remove more than 20 files at once publishes + * none of them until `--publish-removals --yes`. Temp GSTACK_HOME, local bare + * remote; no network. + */ +import { describe, test, expect, beforeEach, afterEach } from 'bun:test'; +import * as fs from 'fs'; +import * as os from 'os'; +import * as path from 'path'; +import { spawnSync } from 'child_process'; + +const ROOT = path.resolve(import.meta.dir, '..'); + +let home: string; +let remote: string; + +function run(argv: string[]) { + const r = spawnSync(path.join(ROOT, 'bin', argv[0]), argv.slice(1), { + env: { ...process.env, HOME: home, GSTACK_HOME: home }, + encoding: 'utf-8', + timeout: 60_000, + }); + return { stdout: r.stdout || '', stderr: r.stderr || '', status: r.status ?? -1 }; +} + +const remoteFiles = () => + spawnSync('git', ['--git-dir=' + remote, 'ls-tree', '-r', '--name-only', 'main'], { encoding: 'utf-8', timeout: 30_000 }) + .stdout.split('\n').filter((p) => p.startsWith('projects/') || p.startsWith('transcripts/')); +const status = () => JSON.parse(fs.readFileSync(path.join(home, '.brain-sync-status.json'), 'utf-8')); + +function write(p: string, body = '# doc\n') { + fs.mkdirSync(path.dirname(path.join(home, p)), { recursive: true }); + fs.writeFileSync(path.join(home, p), body); +} + +function syncAll(files: string[]) { + for (const f of files) write(f); + for (const f of files) run(['gstack-brain-enqueue', f]); + const r = run(['gstack-brain-sync', '--once']); + expect(r.status).toBe(0); + expect(remoteFiles()).toEqual(expect.arrayContaining(files)); +} + +const designs = (dir: string, n: number) => Array.from({ length: n }, (_, i) => `projects/${dir}/designs/d${i}.md`); + +beforeEach(() => { + home = fs.mkdtempSync(path.join(os.tmpdir(), 'brain-removals-home-')); + remote = fs.mkdtempSync(path.join(os.tmpdir(), 'brain-removals-remote-')); + spawnSync('git', ['init', '--bare', '-q', '-b', 'main', remote], { timeout: 30_000 }); + expect(run(['gstack-artifacts-init', '--remote', remote]).status).toBe(0); + run(['gstack-config', 'set', 'artifacts_sync_mode', 'full']); +}); + +afterEach(() => { + fs.rmSync(home, { recursive: true, force: true }); + fs.rmSync(remote, { recursive: true, force: true }); +}); + +describe('artifacts_sync_removals', () => { + test('off by default: a deleted synced file stays published', () => { + expect(run(['gstack-config', 'get', 'artifacts_sync_removals']).stdout.trim()).toBe('off'); + syncAll(['projects/p/designs/a.md', 'projects/p/designs/b.md']); + fs.rmSync(path.join(home, 'projects/p/designs/a.md')); + run(['gstack-brain-sync', '--once']); + expect(remoteFiles()).toContain('projects/p/designs/a.md'); + }); + + test('on: a slug migration removes the old copies with the queue empty or not', () => { + run(['gstack-config', 'set', 'artifacts_sync_removals', 'on']); + syncAll(['projects/acme/designs/x.md', 'projects/acme/designs/y.md']); + fs.renameSync(path.join(home, 'projects/acme'), path.join(home, 'projects/owner-acme')); + run(['gstack-brain-sync', '--discover-new']); + expect(run(['gstack-brain-sync', '--once']).status).toBe(0); + expect(remoteFiles().sort()).toEqual(['projects/owner-acme/designs/x.md', 'projects/owner-acme/designs/y.md']); + expect(status().message).toContain('published 2 removal(s)'); + + fs.rmSync(path.join(home, 'projects/owner-acme/designs/y.md')); + expect(run(['gstack-brain-sync', '--once']).status).toBe(0); + expect(remoteFiles()).toEqual(['projects/owner-acme/designs/x.md']); + expect(status().status).toBe('ok'); + }); + + test('on: tracked files outside the allowlist and transcript pages are never removed', () => { + run(['gstack-config', 'set', 'artifacts_sync_removals', 'on']); + syncAll(['projects/p/designs/a.md']); + write('projects/p/notes.txt'); + write('transcripts/run-1/s1.md'); + spawnSync('git', ['-C', home, 'add', '-f', 'projects/p/notes.txt', 'transcripts/run-1/s1.md'], { timeout: 30_000 }); + spawnSync('git', ['-C', home, '-c', 'user.email=u@example.com', '-c', 'user.name=u', 'commit', '-qm', 'manual'], { timeout: 30_000 }); + spawnSync('git', ['-C', home, 'push', '-q', 'origin', 'HEAD'], { timeout: 30_000 }); + for (const p of ['projects/p/notes.txt', 'transcripts/run-1/s1.md', 'projects/p/designs/a.md']) fs.rmSync(path.join(home, p)); + run(['gstack-brain-sync', '--once']); + expect(remoteFiles().sort()).toEqual(['projects/p/notes.txt', 'transcripts/run-1/s1.md']); + }); + + test('on: a decision log goes only together with its active snapshot', () => { + run(['gstack-config', 'set', 'artifacts_sync_removals', 'on']); + const LOG = 'projects/p/decisions.jsonl'; + const SNAP = 'projects/p/decisions.active.json'; + write(SNAP, JSON.stringify([{ id: 'a', kind: 'decide', note: '' }])); + write(LOG, JSON.stringify({ id: 'a', kind: 'decide', decision: 'd a', scope: 'repo', date: '2026-10-01', source: 'user' }) + '\n'); + run(['gstack-brain-enqueue', LOG]); + run(['gstack-brain-sync', '--once']); + expect(remoteFiles()).toEqual(expect.arrayContaining([LOG, SNAP])); + fs.rmSync(path.join(home, LOG)); + run(['gstack-brain-sync', '--once']); + expect(remoteFiles()).toEqual(expect.arrayContaining([LOG, SNAP])); + fs.rmSync(path.join(home, SNAP)); + run(['gstack-brain-sync', '--once']); + expect(remoteFiles()).toEqual([]); + }); + + test('on: more than 20 removals at once publishes none until --publish-removals --yes', () => { + run(['gstack-config', 'set', 'artifacts_sync_removals', 'on']); + const files = designs('big', 21); + syncAll(files); + fs.rmSync(path.join(home, 'projects/big'), { recursive: true }); + const drain = run(['gstack-brain-sync', '--once']); + expect(drain.stderr).toContain('holding 21 removal(s), more than 20 at once'); + expect(remoteFiles().sort()).toEqual([...files].sort()); + expect(status().message).toContain('none of the removals were published'); + + const listed = run(['gstack-brain-sync', '--publish-removals']); + expect(listed.stdout).toContain('21 synced file(s) are gone from disk:'); + expect(listed.stdout).toContain(' projects/big/designs/d0.md'); + expect(remoteFiles()).toHaveLength(21); + + const published = run(['gstack-brain-sync', '--publish-removals', '--yes']); + expect(published.status).toBe(0); + expect(published.stdout).toContain('published 21 removal(s)'); + expect(remoteFiles()).toEqual([]); + }); + + test('--publish-removals --yes works with the setting off and only touches removals', () => { + syncAll(['projects/p/designs/a.md', 'projects/p/designs/b.md']); + fs.rmSync(path.join(home, 'projects/p/designs/a.md')); + write('projects/p/designs/b.md', '# changed, not yet synced\n'); + const r = run(['gstack-brain-sync', '--publish-removals', '--yes']); + expect(r.status).toBe(0); + expect(remoteFiles()).toEqual(['projects/p/designs/b.md']); + const remoteB = spawnSync('git', ['--git-dir=' + remote, 'show', 'main:projects/p/designs/b.md'], { encoding: 'utf-8', timeout: 30_000 }); + expect(remoteB.stdout).toBe('# doc\n'); + expect(run(['gstack-brain-sync', '--publish-removals', '--yes']).stdout).toContain('no removals to publish'); + }); + + test('config rejects values other than on and off', () => { + const r = run(['gstack-config', 'set', 'artifacts_sync_removals', 'yes']); + expect(r.stderr).toContain("artifacts_sync_removals 'yes' not recognized"); + expect(run(['gstack-config', 'get', 'artifacts_sync_removals']).stdout.trim()).toBe('off'); + }); +}); diff --git a/test/redact-prepush-hook.test.ts b/test/redact-prepush-hook.test.ts index 36ced9365..66727b828 100644 --- a/test/redact-prepush-hook.test.ts +++ b/test/redact-prepush-hook.test.ts @@ -191,6 +191,100 @@ describe("release version exemption (#2856)", () => { }); }); +describe("committed allowlist (.gstack-redact-allowlist)", () => { + const HOOK_ALLOWLIST = ".gstack-redact-allowlist"; + const pushOf = (base: string, head: string) => `refs/heads/main ${head} refs/heads/main ${base}\n`; + const commitAllowlist = (content: string): string => { + return commit(HOOK_ALLOWLIST, content, "allowlist"); + }; + const DEV_URL = ["postgres://", "app:devpass", "@db:5432"].join(""); + + test("an exact committed span passes and the suppression is listed", () => { + const base = git(["rev-parse", "HEAD"]); + commit("config.txt", "key " + FAKE_AWS_KEY + "\n", "add key"); + const head = commitAllowlist("# docs sample key\n" + FAKE_AWS_KEY + "\n"); + const { code, stderr } = runHook(pushOf(base, head)); + expect(code).toBe(0); + expect(stderr).not.toContain("BLOCKED"); + expect(stderr).toContain(".gstack-redact-allowlist (1 entry) suppressed 2 finding(s) in this push:"); + expect(stderr).toContain("HIGH aws.access_key config.txt:1"); + expect(stderr).toContain("HIGH aws.access_key .gstack-redact-allowlist:2"); + }); + + test("a different credential still blocks next to an allowlisted one", () => { + const base = git(["rev-parse", "HEAD"]); + const otherKey = ["AKIA", "FEDCBA0987654321"].join(""); + commit("config.txt", "ok " + FAKE_AWS_KEY + "\nleak " + otherKey + "\n", "one benign one real"); + const head = commitAllowlist(FAKE_AWS_KEY + "\n"); + const { code, stderr } = runHook(pushOf(base, head)); + expect(code).toBe(1); + expect(stderr).toContain("BLOCKED"); + expect(stderr).toContain("suppressed 2 finding(s)"); + const blocked = stderr.slice(stderr.indexOf("BLOCKED")); + expect(blocked.match(/HIGH {2}aws\.access_key/g)).toHaveLength(1); + }); + + test("a substring of the matched span does not suppress it", () => { + const base = git(["rev-parse", "HEAD"]); + commit("compose.txt", "DATABASE_URL=" + DEV_URL + "\n", "dev url"); + const head = commitAllowlist("app:devpass\n"); + const { code, stderr } = runHook(pushOf(base, head)); + expect(code).toBe(1); + expect(stderr).toContain("db.url_with_password"); + expect(stderr).toContain("suppressed 0 finding(s) in this push."); + }); + + test("the whole matched URL suppresses that URL only", () => { + const base = git(["rev-parse", "HEAD"]); + const prodUrl = ["postgres://", "app:devpass", "@prod.internal:5432"].join(""); + commit("compose.txt", "DATABASE_URL=" + DEV_URL + "\nPROD_URL=" + prodUrl + "\n", "two urls"); + const head = commitAllowlist(DEV_URL + "\n"); + const { code, stderr } = runHook(pushOf(base, head)); + expect(code).toBe(1); + expect(stderr).toContain("HIGH db.url_with_password compose.txt:1"); + expect(stderr).toMatch(/BLOCKED[\s\S]*HIGH {2}db\.url_with_password/); + }); + + test("an uncommitted allowlist in the working tree suppresses nothing", () => { + const base = git(["rev-parse", "HEAD"]); + const head = commit("config.txt", "key " + FAKE_AWS_KEY + "\n", "add key"); + fs.writeFileSync(path.join(repo, HOOK_ALLOWLIST), FAKE_AWS_KEY + "\n"); + const { code, stderr } = runHook(pushOf(base, head)); + expect(code).toBe(1); + expect(stderr).not.toContain("suppressed"); + }); + + test("an oversized allowlist is ignored and the scan stays strict", () => { + const base = git(["rev-parse", "HEAD"]); + commit("config.txt", "key " + FAKE_AWS_KEY + "\n", "add key"); + const head = commitAllowlist(FAKE_AWS_KEY + "\n" + "#".repeat(65 * 1024) + "\n"); + const { code, stderr } = runHook(pushOf(base, head)); + expect(code).toBe(1); + expect(stderr).toContain("is over 65536 bytes; ignoring it"); + }); + + test("entries cover MEDIUM emails alongside gstack.redact.allowEmail", () => { + const base = git(["rev-parse", "HEAD"]); + git(["config", "--add", "gstack.redact.allowEmail", "carol@corp.io"]); + commit("notes.md", "contact bob@corp.io or carol@corp.io or dave@corp.io\n", "contacts"); + const head = commitAllowlist(" bob@corp.io \n"); + const { code, stderr } = runHook(pushOf(base, head)); + expect(code).toBe(0); + expect(stderr).toContain("MEDIUM pii.email notes.md:1"); + expect(stderr).toContain("suppressed 2 finding(s)"); + expect(stderr).toContain("1 MEDIUM finding(s) in pushed diff"); + }); + + test("the block message names the allowlist without one present", () => { + const base = git(["rev-parse", "HEAD"]); + const head = commit("config.txt", "key " + FAKE_AWS_KEY + "\n", "add key"); + const { code, stderr } = runHook(pushOf(base, head)); + expect(code).toBe(1); + expect(stderr).toContain("commit its exact matched text as a line of .gstack-redact-allowlist"); + expect(stderr).not.toContain("suppressed"); + }); +}); + describe("diff direction + special refs", () => { test("only NEW content is scanned (remote..local), not pre-existing", () => { // Put a secret in the FIRST commit (already on remote), then push a clean commit. diff --git a/test/setup-ps1.test.ts b/test/setup-ps1.test.ts new file mode 100644 index 000000000..abd1beff6 --- /dev/null +++ b/test/setup-ps1.test.ts @@ -0,0 +1,75 @@ +/** + * setup.ps1: the Windows entry point checks Git for Windows, Bun and Node.js, + * then runs ./setup in Git Bash with every argument passed as its own argv + * entry, never pasted into shell source. The behavioral cases copy the script + * next to a stub ./setup that prints its argv and run it under pwsh. + */ +import { describe, test, expect, afterEach } from 'bun:test'; +import * as fs from 'fs'; +import * as os from 'os'; +import * as path from 'path'; +import { spawnSync } from 'child_process'; + +const SCRIPT = path.resolve(import.meta.dir, '..', 'setup.ps1'); +const SOURCE = fs.readFileSync(SCRIPT, 'utf-8'); +const PWSH = Bun.which('pwsh'); +const BASH = Bun.which('bash'); + +const dirs: string[] = []; +afterEach(() => { + for (const d of dirs.splice(0)) fs.rmSync(d, { recursive: true, force: true }); +}); + +function stubCheckout(): string { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'setup-ps1-')); + dirs.push(dir); + fs.copyFileSync(SCRIPT, path.join(dir, 'setup.ps1')); + fs.writeFileSync(path.join(dir, 'setup'), '#!/usr/bin/env bash\nprintf \'cwd=%s\\n\' "$PWD"\nfor a in "$@"; do printf \'arg=[%s]\\n\' "$a"; done\nexit 7\n', { mode: 0o755 }); + return dir; +} + +function runPs1(dir: string, args: string[], env: Record = {}) { + const r = spawnSync(PWSH!, ['-NoProfile', '-NonInteractive', '-File', path.join(dir, 'setup.ps1'), ...args], { + cwd: os.tmpdir(), + encoding: 'utf-8', + env: { ...process.env, ...(process.platform === 'win32' ? {} : { GSTACK_GIT_BASH: BASH! }), ...env }, + timeout: 60_000, + }); + return { code: r.status, out: (r.stdout || '') + (r.stderr || '') }; +} + +describe('setup.ps1', () => { + test('passes arguments as an array to a constant bash script', () => { + expect(SOURCE).toContain(`& $bash -c 'exec ./setup "$@"' setup @args`); + expect(SOURCE).not.toMatch(/-join|Invoke-Expression|\biex\b|git-bash\.exe/); + expect([...SOURCE].every((c) => c.charCodeAt(0) < 128)).toBe(true); + }); + + test.skipIf(!PWSH || !BASH)('forwards every argument literally from the script directory and keeps the exit code', () => { + const dir = stubCheckout(); + const args = ['--host', 'codex', 'two words', '$(touch PWNED)', "it's", '"quoted"', ';echo injected', '`whoami`']; + const { code, out } = runPs1(dir, args); + expect(code).toBe(7); + const lines = out.split(/\r?\n/); + expect(lines.filter((l) => l.startsWith('arg=['))).toEqual(args.map((a) => `arg=[${a}]`)); + expect(lines.find((l) => l.startsWith('cwd='))).toContain(path.basename(dir)); + expect(out).not.toContain('injected\n'); + expect(fs.existsSync(path.join(dir, 'PWNED'))).toBe(false); + expect(fs.existsSync(path.join(os.tmpdir(), 'PWNED'))).toBe(false); + }); + + test.skipIf(!PWSH || !BASH || process.platform === 'win32')('a missing prerequisite names its winget command and runs nothing', () => { + const dir = stubCheckout(); + const bin = fs.mkdtempSync(path.join(os.tmpdir(), 'setup-ps1-bin-')); + dirs.push(bin); + for (const tool of ['git', 'node']) { + const real = Bun.which(tool); + if (real) fs.symlinkSync(real, path.join(bin, tool)); + } + const { code, out } = runPs1(dir, ['--host', 'codex'], { PATH: bin }); + expect(code).toBe(1); + expect(out).toContain('Bun: winget install --id Oven-sh.Bun -e'); + expect(out).not.toContain('Git for Windows:'); + expect(out).not.toContain('arg=['); + }); +});