diff --git a/bin/gstack-host-renders.sh b/bin/gstack-host-renders.sh index dd6c20b67..9c79a0ee7 100644 --- a/bin/gstack-host-renders.sh +++ b/bin/gstack-host-renders.sh @@ -148,3 +148,92 @@ EOF # shellcheck disable=SC2086 # word-split the host list on purpose gstack_render_hosts_add "$root" $found } + +# _gstack_hr_generated FILE TREE REAL — WEAK proof that FILE is a gstack +# render or copy: gen-skill-docs' two-line banner (setup's +# _gstack_generated_header rule), the generator's exact openai.yaml shape for +# its own skill directory, byte identity with this checkout (a Windows sidecar +# copy), or setup's runtime pointers naming this checkout. +_gstack_hr_generated() { + local f="$1" tree="$2" real="$3" skill rel first + case "$(head -c 8192 "$f" 2>/dev/null)" in + *''*) return 0 ;; + esac + case "$f" in + */agents/openai.yaml) + skill="${f%/agents/openai.yaml}"; skill="${skill##*/}" + [ "$(head -1 "$f" 2>/dev/null)" = "interface:" ] \ + && grep -qxF " display_name: \"$skill\"" "$f" \ + && grep -qxF " default_prompt: \"Use $skill for this task.\"" "$f" \ + && [ "$(wc -l < "$f" | tr -d ' ')" -le 6 ] && return 0 ;; + */.gstack-owned|*/.source-path) + first="$(head -1 "$f" 2>/dev/null)" + [ -n "$first" ] && [ "$(cd "$first" 2>/dev/null && pwd -P)" = "$real" ] && return 0 ;; + esac + case "$f" in + "$tree"/skills/gstack/*) + rel="${f#"$tree"/skills/gstack/}" + [ -f "$real/$rel" ] && cmp -s "$f" "$real/$rel" && return 0 ;; + esac + return 1 +} + +# gstack_prune_host_renders ROOT — prune the render dir of every host the +# record does not name. For each tree it changes, prints the kept paths and a +# summary; every path is listed in the backup's prune.log. A tree with nothing +# left to remove (only kept files) is silent. Returns 1 when a proven file +# could not be backed up (it is then left in place). Needs GSTACK_STATE_ROOT. +gstack_prune_host_renders() { + local root="$1" real recorded pair host dir tree backup e rel target rc=0 + local n_backed n_links n_kept n_dirs kept_lines lines nl=' +' + real="$(cd "$root" 2>/dev/null && pwd -P)" || return 0 + [ -f "$(gstack_render_hosts_file "$root")" ] || return 0 + recorded=" $(gstack_render_hosts_read "$root" | tr '\n' ' ') " + backup="${GSTACK_STATE_ROOT:?gstack-host-renders: GSTACK_STATE_ROOT is not set}/backups/host-renders/$(date -u +%Y%m%dT%H%M%SZ)-$(printf '%s' "$real" | cksum | awk '{print $1}')" + for pair in $GSTACK_HOST_RENDER_DIRS; do + host="${pair%%:*}"; dir="${pair#*:}"; tree="$real/$dir" + case "$recorded" in *" $host "*) continue ;; esac + if [ -L "$tree" ] || [ ! -d "$tree" ]; then continue; fi + n_backed=0; n_links=0; n_kept=0; n_dirs=0; kept_lines=""; lines="" + while IFS= read -r e; do + [ -n "$e" ] || continue + rel="${e#"$real"/}" + if [ -L "$e" ]; then + target="$(_gstack_hr_link_abs "$e")" || target="" + case "$target" in + "$real"|"$root") n_kept=$((n_kept + 1)); kept_lines="$kept_lines kept $rel: links to the checkout itself (bin/dev-setup)$nl"; lines="${lines}kept $rel links to the checkout itself$nl" ;; + "$real"/*|"$root"/*) rm -f "$e" && { n_links=$((n_links + 1)); lines="${lines}removed link $rel -> $target$nl"; } ;; + *) n_kept=$((n_kept + 1)); kept_lines="$kept_lines kept $rel: links outside this checkout$nl"; lines="${lines}kept $rel links outside this checkout$nl" ;; + esac + elif _gstack_hr_generated "$e" "$tree" "$real"; then + if mkdir -p "$backup/${rel%/*}" && mv -f "$e" "$backup/$rel"; then + n_backed=$((n_backed + 1)); lines="${lines}backed up $rel generated$nl" + else + rc=1; n_kept=$((n_kept + 1)); echo " kept $rel: could not back it up to $backup" >&2 + fi + else + n_kept=$((n_kept + 1)); kept_lines="$kept_lines kept $rel: not proven generated (no gstack banner or byte match)$nl" + lines="${lines}kept $rel not proven generated$nl" + fi + done </dev/null) +EOF + # -depth with -exec removes children first, so a directory emptied by its + # children's removal goes in the same pass. + while IFS= read -r e; do + [ -n "$e" ] || continue + n_dirs=$((n_dirs + 1)); lines="${lines}removed dir ${e#"$real"/} empty$nl" + done </dev/null) +EOF + [ $((n_backed + n_links + n_dirs)) -gt 0 ] || continue + mkdir -p "$backup" && printf '%s' "$lines" >> "$backup/prune.log" + printf '%s' "$kept_lines" + echo " pruned $dir ($host is not installed from this checkout): $n_backed generated files backed up, $n_links links removed, $n_dirs empty dirs removed, $n_kept kept" + done + if [ -f "$backup/prune.log" ]; then + echo " host-render backup: $backup (every path: $backup/prune.log; restore a file with mv)" + fi + return "$rc" +} diff --git a/gstack-upgrade/migrations/v1.91.50.0.sh b/gstack-upgrade/migrations/v1.91.50.0.sh new file mode 100755 index 000000000..7c4bf2c36 --- /dev/null +++ b/gstack-upgrade/migrations/v1.91.50.0.sh @@ -0,0 +1,31 @@ +#!/usr/bin/env bash +# Migration: v1.91.50.0 — install footprint (#1694). The release queue may +# rename this file at /ship; it must match VERSION. +# +# Installs used to render every host's skills inside the checkout, so a global +# Claude install carried ~576 extra SKILL.md files (32 MB) that Claude Code and +# Cursor-agent scan. Record the hosts this install already serves (registry +# rows, install roots that resolve here, host skills that link into its +# renders), then prune the renders of every other host with +# bin/gstack-host-renders.sh: generated files are backed up under +# $GSTACK_STATE_ROOT/backups/host-renders/, anything not proven generated stays. +# ./setup runs the same prune on every run; this covers an install upgraded +# without one. Idempotent and non-fatal. +set -u +_gstack_migration_dir="${BASH_SOURCE[0]//\\//}"; _gstack_migration_dir="${_gstack_migration_dir%/*}" +_gstack_bin="${_gstack_migration_dir}/../../bin" +. "$_gstack_bin/gstack-state-root.sh" 2>/dev/null || { echo "$0: cannot resolve the gstack state root: $_gstack_bin/gstack-state-root.sh is missing. fix: reinstall with ./setup or /gstack-upgrade (docs/state-root.md)" >&2; exit 1; } +gstack_state_root_select; GSTACK_STATE_ROOT="$_gstack_sr_root" +. "$_gstack_bin/gstack-install-registry.sh" 2>/dev/null || exit 0 +. "$_gstack_bin/gstack-host-renders.sh" 2>/dev/null || exit 0 + +INSTALL="${GSTACK_INSTALL_DIR:-${HOME:-}/.claude/skills/gstack}" +[ -f "$INSTALL/VERSION" ] && [ -f "$INSTALL/setup" ] || exit 0 + +gstack_render_hosts_seed "$INSTALL" +if [ ! -f "$(gstack_render_hosts_file "$INSTALL")" ]; then + echo "host renders: no install from $INSTALL was found, so its renders were left alone; ./setup --host records and prunes" + exit 0 +fi +gstack_prune_host_renders "$INSTALL" || echo "host renders: some generated files could not be backed up and were left in place; the next ./setup retries" >&2 +exit 0 diff --git a/setup b/setup index e3b80040b..0f57b37e4 100755 --- a/setup +++ b/setup @@ -255,6 +255,7 @@ case "$HOST" in echo "To integrate gstack with OpenClaw:" echo " 1. Tell your OpenClaw agent: 'install gstack for openclaw'" echo " 2. Or generate artifacts: bun run gen:skill-docs --host openclaw" + echo " (in an install, add 'openclaw' to .gstack-installed-hosts first, or ./setup prunes them)" echo " 3. See docs/OPENCLAW.md for the full architecture" print_instruction_tier exit 0 ;; @@ -268,6 +269,8 @@ case "$HOST" in echo " 2. Render Hermes-flavoured skills yourself (not certified; #2826):" echo " bun run gen:skill-docs --host hermes" echo " then point a Hermes profile's skills.external_dirs at .hermes/skills." + echo " Add 'hermes' to .gstack-installed-hosts in this checkout first: ./setup" + echo " prunes renders of hosts that file does not name (#1694)." print_instruction_tier exit 0 ;; gbrain) @@ -278,6 +281,7 @@ case "$HOST" in echo "" echo "To generate brain-aware skills:" echo " bun run gen:skill-docs --host gbrain" + echo " (in an install, add 'gbrain' to .gstack-installed-hosts first, or ./setup prunes them)" echo "" echo "GBrain setup and brain skills ship from the GBrain repo." echo "" @@ -3613,6 +3617,14 @@ if [ "$INSTALL_CODEX" -eq 1 ]; then _setup_arm_publish codex "$_CX_SCOPE" "$_CX_PROJECT" "$CODEX_SKILLS" "$CODEX_GSTACK" - "${_CODEX_RENDER_ROOT:-committed}" fi +# 7b. Prune the skill renders of hosts this checkout does not serve (#1694), +# on every run, so each install converges on its own recorded hosts even when +# several installs share one state root. Generated files go to the state +# root's backups; anything not proven generated stays in place. +if ! gstack_prune_host_renders "$SOURCE_GSTACK_DIR"; then + echo " warning: some generated host renders could not be backed up, so they were left in place; the next ./setup retries" >&2 +fi + # 8. Run pending version migrations # Migrations handle state fixes that ./setup alone can't cover (stale config, # orphaned files, directory structure changes). Each migration is idempotent. diff --git a/test/host-renders-prune.test.ts b/test/host-renders-prune.test.ts new file mode 100644 index 000000000..c7ec46940 --- /dev/null +++ b/test/host-renders-prune.test.ts @@ -0,0 +1,163 @@ +/** + * #1694: renders of hosts an install does not serve are pruned safely. + * + * Every ./setup (and the gstack-upgrade migration) removes the render dirs of + * hosts missing from the installed-hosts record with bin/gstack-relink's proof + * rules: a symlink into the checkout (STRONG) goes; a file proven generated + * (WEAK: gen-skill-docs' banner, the generator's openai.yaml shape, byte + * identity) is moved to $GSTACK_STATE_ROOT/backups/host-renders/; anything + * else stays; only emptied directories are removed. The legacy layout here is + * a real `--host all` render. + */ +import { afterAll, afterEach, describe, expect, test } from 'bun:test'; +import { spawnSync } from 'node:child_process'; +import * as fs from 'node:fs'; +import * as path from 'node:path'; +import { runGeneration } from '../scripts/gen-skill-docs'; +import { cleanupFixtures, cleanupSeed, makeFixture, makeSource, put, runSetup, type Fixture } from './helpers/install-fixture'; + +const ROOT = path.resolve(import.meta.dir, '..'); +const OTHER_DIRS = ['.agents', '.factory', '.kiro', '.opencode', '.cursor', '.copilot', '.slate', '.openclaw', '.hermes', '.gbrain']; + +afterEach(cleanupFixtures); +afterAll(cleanupSeed); + +const skillFiles = (dir: string): string[] => !fs.existsSync(dir) ? [] + : (fs.readdirSync(dir, { recursive: true }) as string[]).filter(rel => path.basename(rel) === 'SKILL.md' && fs.statSync(path.join(dir, rel)).isFile()); + +async function legacyRender(src: string): Promise { + const result = await runGeneration({ host: 'all', outputRoot: src }); + if (result.exitCode !== 0) throw new Error('legacy render failed'); +} + +function backups(f: Fixture): string[] { + const dir = path.join(f.home, '.gstack/backups/host-renders'); + return fs.existsSync(dir) ? fs.readdirSync(dir).map(d => path.join(dir, d)) : []; +} + +function setupOk(f: Fixture, src: string, args: string[] = []) { + const r = runSetup(f, path.join(src, 'setup'), args); + expect(r.status, r.stdout + r.stderr).toBe(0); + return r; +} + +describe.skipIf(process.platform === 'win32')('setup prunes unrecorded host renders without losing a file (#1694)', () => { + test('a legacy all-host render inside a Claude install: generated files backed up, user files kept, links handled', async () => { + const f = makeFixture(); + const src = makeSource(f, path.join(f.home, '.claude/skills/gstack')); + await legacyRender(src); + const before = skillFiles(src).length; + expect(before).toBeGreaterThan(600); + const customized = path.join(src, '.factory/skills/gstack-qa/SKILL.md'); + fs.appendFileSync(customized, '\nMy own QA note.\n'); + put(path.join(src, '.factory/notes.md'), 'my notes, no banner\n'); + put(path.join(src, '.kiro/skills/my-skill/SKILL.md'), '---\nname: my-skill\n---\nHand-written.\n'); + fs.mkdirSync(path.join(src, '.cursor/skills/gstack'), { recursive: true }); + fs.symlinkSync(path.join(src, 'bin'), path.join(src, '.cursor/skills/gstack/bin')); + fs.mkdirSync(path.join(f.dir, 'outside')); + fs.symlinkSync(path.join(f.dir, 'outside'), path.join(src, '.cursor/skills/gstack/elsewhere')); + + const r = setupOk(f, src); + expect(fs.readFileSync(path.join(src, '.gstack-installed-hosts'), 'utf8')).toEndWith('\nclaude\n'); + // Only the Claude skills and the hand-written look-alike remain. + const left = skillFiles(src).filter(rel => rel.startsWith('.')); + expect(left).toEqual(['.kiro/skills/my-skill/SKILL.md']); + expect(before - skillFiles(src).length).toBeGreaterThan(550); + expect(fs.readdirSync(path.join(src, '.factory'), { recursive: true })).toEqual(['notes.md']); + expect(fs.lstatSync(path.join(src, '.cursor/skills/gstack/elsewhere')).isSymbolicLink()).toBe(true); + expect(fs.existsSync(path.join(f.dir, 'outside'))).toBe(true); + expect(fs.existsSync(path.join(src, '.cursor/skills/gstack/bin'))).toBe(false); + expect(fs.existsSync(path.join(src, 'bin/gstack-config'))).toBe(true); + for (const dir of ['.agents', '.opencode', '.copilot', '.slate', '.openclaw', '.hermes', '.gbrain']) expect(fs.existsSync(path.join(src, dir)), dir).toBe(false); + + expect(r.stdout).toContain('kept .factory/notes.md: not proven generated (no gstack banner or byte match)'); + expect(r.stdout).toContain('kept .kiro/skills/my-skill/SKILL.md: not proven generated'); + expect(r.stdout).toContain('kept .cursor/skills/gstack/elsewhere: links outside this checkout'); + expect(r.stdout).toMatch(/pruned \.factory \(factory is not installed from this checkout\): \d+ generated files backed up, 0 links removed, \d+ empty dirs removed, 1 kept/); + const [backup] = backups(f); + expect(backups(f)).toHaveLength(1); + expect(r.stdout).toContain(`host-render backup: ${backup} (every path: ${backup}/prune.log; restore a file with mv)`); + expect(fs.readFileSync(path.join(backup!, '.factory/skills/gstack-qa/SKILL.md'), 'utf8')).toEndWith('\nMy own QA note.\n'); + const log = fs.readFileSync(path.join(backup!, 'prune.log'), 'utf8'); + expect(log).toContain('backed up\t.agents/skills/gstack-qa/agents/openai.yaml\tgenerated'); + expect(log).toContain('backed up\t.hermes/skills/gstack-qa/SKILL.md\tgenerated'); + expect(log).toContain(`removed link\t.cursor/skills/gstack/bin\t-> ${src}/bin`); + expect(log).toContain('kept\t.factory/notes.md\tnot proven generated'); + expect(log).toContain('removed dir\t.agents\tempty'); + + const again = setupOk(f, src); + expect(again.stdout).not.toContain('pruned '); + expect(again.stdout).not.toContain('host-render backup'); + expect(backups(f)).toHaveLength(1); + }, 240_000); + + test('two installs sharing one state root each end with only their own hosts', async () => { + const f = makeFixture(); + const a = makeSource(f, path.join(f.home, '.claude/skills/gstack')); + const b = makeSource(f, path.join(f.home, 'src-b/gstack')); + await legacyRender(a); + await legacyRender(b); + setupOk(f, a); + setupOk(f, b, ['--host', 'codex']); + for (const dir of OTHER_DIRS) expect(fs.existsSync(path.join(a, dir)), `a ${dir}`).toBe(false); + expect(skillFiles(path.join(b, '.agents')).length).toBeGreaterThan(40); + for (const dir of OTHER_DIRS.filter(d => d !== '.agents')) expect(fs.existsSync(path.join(b, dir)), `b ${dir}`).toBe(false); + expect(backups(f)).toHaveLength(2); + }, 240_000); + + test('a host whose skills link into the render is kept even with no registry row', async () => { + const f = makeFixture(); + const src = makeSource(f, path.join(f.home, '.claude/skills/gstack')); + await legacyRender(src); + fs.mkdirSync(path.join(f.home, '.factory/skills'), { recursive: true }); + fs.symlinkSync(path.join(src, '.factory/skills/gstack-qa'), path.join(f.home, '.factory/skills/gstack-qa')); + setupOk(f, src); + expect(fs.readFileSync(path.join(src, '.gstack-installed-hosts'), 'utf8')).toEndWith('\nclaude\nfactory\n'); + expect(skillFiles(path.join(src, '.factory')).length).toBeGreaterThan(40); + expect(fs.existsSync(path.join(f.home, '.factory/skills/gstack-qa/SKILL.md'))).toBe(true); + }, 240_000); +}); + +describe.skipIf(process.platform === 'win32')('gstack-upgrade migration prunes existing installs (#1694)', () => { + const migrations = fs.readdirSync(path.join(ROOT, 'gstack-upgrade/migrations')) + .filter(n => fs.readFileSync(path.join(ROOT, 'gstack-upgrade/migrations', n), 'utf8').includes('gstack-host-renders.sh')); + + test('exactly one migration owns the prune', () => { + expect(migrations).toEqual([expect.stringMatching(/^v\d+\.\d+\.\d+\.\d+\.sh$/)]); + }); + + function migrate(f: Fixture, install: string) { + return spawnSync('bash', [path.join(install, 'gstack-upgrade/migrations', migrations[0]!)], { + cwd: f.home, encoding: 'utf8', timeout: 60_000, env: { ...f.env, GSTACK_INSTALL_DIR: install }, + }); + } + + test('a pre-record Claude install: renders of other hosts are backed up and removed, a look-alike survives, rerun is a no-op', async () => { + const f = makeFixture(); + const src = makeSource(f, path.join(f.home, '.claude/skills/gstack')); + await legacyRender(src); + put(path.join(src, '.opencode/skills/mine/SKILL.md'), 'no banner\n'); + const r = migrate(f, src); + expect(r.status, r.stdout + r.stderr).toBe(0); + expect(fs.readFileSync(path.join(src, '.gstack-installed-hosts'), 'utf8')).toEndWith('\nclaude\n'); + expect(skillFiles(src).filter(rel => rel.startsWith('.'))).toEqual(['.opencode/skills/mine/SKILL.md']); + expect(r.stdout).toContain('kept .opencode/skills/mine/SKILL.md: not proven generated'); + expect(backups(f)).toHaveLength(1); + const again = migrate(f, src); + expect(again.status).toBe(0); + expect(again.stdout).not.toContain('pruned '); + expect(backups(f)).toHaveLength(1); + }, 240_000); + + test('a checkout no install points at is left alone', async () => { + const f = makeFixture(); + const src = makeSource(f, path.join(f.home, 'elsewhere/gstack')); + await legacyRender(src); + const before = skillFiles(src).length; + const r = migrate(f, src); + expect(r.status).toBe(0); + expect(r.stdout).toContain(`host renders: no install from ${src} was found, so its renders were left alone`); + expect(skillFiles(src).length).toBe(before); + expect(fs.existsSync(path.join(src, '.gstack-installed-hosts'))).toBe(false); + }, 240_000); +});