gstack/test/gen-skill-docs-prune-stale.test.ts

/**
 * gen-skill-docs prunes stale external-host renders.
 *
 * The generator only ever wrote outputs, so a skill deleted from the source
 * tree stayed rendered under every host's skills/ dir (and setup kept linking
 * it). Now a render removes `gstack-*` dirs it did not write — but only those
 * carrying the generated banner (proof it was ours), never after a host's
 * generation failed (a partial rendered set must not delete the rest), never
 * under --dry-run (a freshness check writes nothing and deletes nothing), and
 * never in ANOTHER host's skills/ dir (a `--host codex` run has no rendered
 * set for factory, so it has no basis to judge factory's entries).
 */
import { describe, test, expect } from 'bun:test';
import { spawnSync } from 'child_process';
import * as fs from 'fs';
import * as os from 'os';
import * as path from 'path';

const ROOT = path.resolve(import.meta.dir, '..');
const BANNER = '<!-- AUTO-GENERATED from SKILL.md.tmpl — do not edit directly -->';

/** A stale RENDER: the generator's banner is the proof of ownership the prune requires. */
function staleRender(name: string, body: string): string {
  return `---\nname: ${name}\n---\n${BANNER}\n${body}\n`;
}

function gen(out: string, ...extra: string[]) {
  return spawnSync('bun', ['run', 'scripts/gen-skill-docs.ts', '--host', 'codex', '--out-dir', out, ...extra], {
    cwd: ROOT, encoding: 'utf-8', timeout: 180_000,
    env: { ...process.env, GSTACK_DEFER_CLAUDE_RENAME_PRUNE: '0' },
  });
}

describe('gen-skill-docs stale-render prune', () => {
  test('a gstack-* render for a skill that no longer exists is removed; the sidecar symlink, real skills, un-bannered gstack-* dirs, and other hosts\' trees stay', () => {
    const out = fs.mkdtempSync(path.join(os.tmpdir(), 'gstack-prune-'));
    // Keep the sidecar's source disposable: a broken symlink-loop guard must
    // never overwrite this checkout's SKILL.md and poison parallel readers.
    const source = path.join(out, 'source');
    fs.mkdirSync(path.join(source, 'scripts'), { recursive: true });
    fs.copyFileSync(path.join(ROOT, 'scripts', 'gen-skill-docs.ts'), path.join(source, 'scripts', 'gen-skill-docs.ts'));
    for (const file of ['discover-skills.ts', 'external-skill-names.ts', 'gen-llms-txt.ts', 'gen-agents-digest.ts', 'gen-doctor-components.ts', 'models.ts']) {
      fs.symlinkSync(path.join(ROOT, 'scripts', file), path.join(source, 'scripts', file), 'file');
    }
    fs.symlinkSync(path.join(ROOT, 'scripts', 'resolvers'), path.join(source, 'scripts', 'resolvers'), 'dir');
    for (const dir of ['hosts', 'lib']) fs.symlinkSync(path.join(ROOT, dir), path.join(source, dir), 'dir');
    for (const skill of ['', 'ship']) {
      fs.mkdirSync(path.join(source, skill), { recursive: true });
      fs.writeFileSync(path.join(source, skill, 'SKILL.md.tmpl'), `---\nname: ${skill || 'gstack'}\ndescription: Fixture skill.\n---\nFixture body.\n`);
    }
    const canonical = path.join(source, 'SKILL.md');
    const metadata = path.join(source, 'agents', 'openai.yaml');
    fs.mkdirSync(path.dirname(metadata), { recursive: true });
    fs.writeFileSync(canonical, 'canonical skill sentinel\n');
    fs.writeFileSync(metadata, 'canonical metadata sentinel\n');
    const skills = path.join(out, '.agents', 'skills');
    fs.mkdirSync(path.join(skills, 'gstack-retired-zzz'), { recursive: true });
    fs.writeFileSync(path.join(skills, 'gstack-retired-zzz', 'SKILL.md'), staleRender('gstack-retired-zzz', 'stale'));
    // Someone's own skill that happens to use the gstack- prefix: no banner, never touched (#2119).
    fs.mkdirSync(path.join(skills, 'gstack-mine'), { recursive: true });
    fs.writeFileSync(path.join(skills, 'gstack-mine', 'SKILL.md'), '---\nname: gstack-mine\n---\nuser skill\n');
    // Two more shapes the banner gate must keep: a dir with no SKILL.md at all,
    // and a look-alike comment that is NOT the generator's exact banner.
    fs.mkdirSync(path.join(skills, 'gstack-nomd'), { recursive: true });
    fs.writeFileSync(path.join(skills, 'gstack-nomd', 'README.md'), 'no SKILL.md here\n');
    fs.mkdirSync(path.join(skills, 'gstack-lookalike'), { recursive: true });
    fs.writeFileSync(path.join(skills, 'gstack-lookalike', 'SKILL.md'), '---\nname: gstack-lookalike\n---\n<!-- auto-generated by someone else — do not edit -->\nmine\n');
    fs.mkdirSync(path.join(skills, 'not-ours'), { recursive: true });
    fs.symlinkSync(source, path.join(skills, 'gstack'), 'dir');
    // Host isolation: a bannered stale-looking render under ANOTHER host's tree
    // in the same out-dir is not a codex run's to prune (or to touch at all).
    const factorySkills = path.join(out, '.factory', 'skills');
    fs.mkdirSync(path.join(factorySkills, 'gstack-zzz'), { recursive: true });
    fs.writeFileSync(path.join(factorySkills, 'gstack-zzz', 'SKILL.md'), staleRender('gstack-zzz', 'stale factory'));
    try {
      const r = spawnSync('bun', ['run', path.join(source, 'scripts', 'gen-skill-docs.ts'), '--host', 'codex', '--out-dir', out], {
        cwd: source, encoding: 'utf-8', timeout: 180_000,
        env: { ...process.env, GSTACK_DEFER_CLAUDE_RENAME_PRUNE: '0' },
      });
      expect(r.status, r.stderr).toBe(0);
      // On Windows the old realpath + '/' comparison missed this alias,
      // rewriting BOTH source artifacts through the sidecar.
      expect(r.stdout.replace(/\\/g, '/')).toContain('SKIPPED (symlink loop): .agents/skills/gstack/SKILL.md');
      expect(fs.readFileSync(canonical, 'utf-8')).toBe('canonical skill sentinel\n');
      expect(fs.readFileSync(metadata, 'utf-8')).toBe('canonical metadata sentinel\n');
      expect(r.stdout).toContain('pruned stale codex render: gstack-retired-zzz');
      expect(fs.existsSync(path.join(skills, 'gstack-retired-zzz'))).toBe(false);
      // Exactly one prune in this run: the bannered stale render and nothing else.
      expect(r.stdout.match(/pruned stale /g)).toHaveLength(1);
      // Banner gate: three un-bannered gstack-* dirs survive, each named in the log.
      for (const kept of ['gstack-mine', 'gstack-nomd', 'gstack-lookalike']) {
        expect(r.stdout).toContain(`kept codex skills/${kept}: not a gstack render (no generated banner)`);
      }
      expect(fs.readFileSync(path.join(skills, 'gstack-mine', 'SKILL.md'), 'utf-8')).toContain('user skill');
      expect(fs.readFileSync(path.join(skills, 'gstack-nomd', 'README.md'), 'utf-8')).toContain('no SKILL.md here');
      expect(fs.readFileSync(path.join(skills, 'gstack-lookalike', 'SKILL.md'), 'utf-8')).toContain('mine');
      expect(fs.existsSync(path.join(skills, 'not-ours'))).toBe(true);
      expect(fs.lstatSync(path.join(skills, 'gstack')).isSymbolicLink()).toBe(true);
      expect(fs.existsSync(path.join(skills, 'gstack-ship', 'SKILL.md'))).toBe(true);
      // Host isolation: the factory tree is byte-identical to how we left it, and the log never mentions it.
      expect(fs.readFileSync(path.join(factorySkills, 'gstack-zzz', 'SKILL.md'), 'utf-8')).toBe(staleRender('gstack-zzz', 'stale factory'));
      expect(fs.readdirSync(factorySkills)).toEqual(['gstack-zzz']);
      expect(r.stdout).not.toContain('pruned stale factory');
      expect(r.stdout).not.toContain('gstack-zzz');
    } finally {
      fs.rmSync(out, { recursive: true, force: true });
    }
  }, 200_000);

  test('a failed host render keeps generated directories outside its partial inventory', () => {
    const out = fs.mkdtempSync(path.join(os.tmpdir(), 'gstack-prune-failed-'));
    const skills = path.join(out, '.agents', 'skills');
    const stale = path.join(skills, 'gstack-retired-zzz', 'SKILL.md');
    fs.mkdirSync(path.dirname(stale), { recursive: true });
    fs.writeFileSync(stale, staleRender('gstack-retired-zzz', 'retain after failure'));
    // An expected artifact that cannot be written interrupts the host render.
    fs.mkdirSync(path.join(skills, 'gstack-autoplan', 'SKILL.md'), { recursive: true });
    try {
      const r = gen(out);
      expect(r.status).toBe(1);
      expect(r.stderr).toContain('ERROR (codex)');
      expect(r.stderr).toContain('EISDIR');
      expect(r.stdout).not.toContain('pruned stale');
      expect(fs.readFileSync(stale, 'utf-8')).toBe(staleRender('gstack-retired-zzz', 'retain after failure'));
    } finally {
      fs.rmSync(out, { recursive: true, force: true });
    }
  }, 200_000);

  test('--dry-run never prunes: a bannered stale render stays byte-identical, no SKILL.md is written, and the run reports STALE', () => {
    const out = fs.mkdtempSync(path.join(os.tmpdir(), 'gstack-prune-dry-'));
    const skills = path.join(out, '.agents', 'skills');
    const stale = path.join(skills, 'gstack-retired-zzz', 'SKILL.md');
    fs.mkdirSync(path.dirname(stale), { recursive: true });
    fs.writeFileSync(stale, staleRender('gstack-retired-zzz', 'stale'));
    try {
      const r = gen(out, '--dry-run');
      // An empty out-dir is stale by definition: dry-run says so and exits 1 instead of writing.
      expect(r.status).toBe(1);
      expect(r.stdout).toContain('STALE: ');
      expect(r.stdout).not.toContain('GENERATED: ');
      expect(fs.existsSync(path.join(skills, 'gstack-ship', 'SKILL.md'))).toBe(false);
      // The prune step is skipped wholesale — no deletions, no "kept" verdicts either.
      expect(r.stdout).not.toContain('pruned stale');
      expect(r.stdout).not.toContain('kept codex skills/');
      expect(fs.readFileSync(stale, 'utf-8')).toBe(staleRender('gstack-retired-zzz', 'stale'));
    } finally {
      fs.rmSync(out, { recursive: true, force: true });
    }
  }, 200_000);

  test('setup can defer the renamed render only; standalone generation retires it without changing installed links', () => {
    const out = fs.mkdtempSync(path.join(os.tmpdir(), 'gstack-rename-prune-'));
    const skills = path.join(out, '.agents', 'skills');
    const old = path.join(skills, 'gstack-claude');
    const stale = path.join(skills, 'gstack-retired-zzz');
    const home = path.join(out, 'home');
    const installed = path.join(home, '.codex', 'skills', 'gstack-claude');
    for (const dir of [old, stale]) {
      fs.mkdirSync(dir, { recursive: true });
      fs.writeFileSync(path.join(dir, 'SKILL.md'), staleRender(path.basename(dir), 'old output'));
    }
    fs.mkdirSync(path.dirname(installed), { recursive: true });
    fs.symlinkSync(old, installed);
    try {
      const deferred = spawnSync('bun', ['run', 'scripts/gen-skill-docs.ts', '--host', 'codex', '--out-dir', out], {
        cwd: ROOT, encoding: 'utf8', timeout: 180_000,
        env: { ...process.env, HOME: home, GSTACK_DEFER_CLAUDE_RENAME_PRUNE: '1' },
      });
      expect(deferred.status).toBe(0);
      expect(fs.existsSync(path.join(old, 'SKILL.md'))).toBe(true);
      expect(fs.existsSync(stale)).toBe(false);
      expect(fs.existsSync(path.join(skills, 'gstack-claude-code', 'SKILL.md'))).toBe(true);
      expect(fs.readlinkSync(installed)).toBe(old);
      const standalone = gen(out);
      expect(standalone.status).toBe(0);
      expect(fs.existsSync(old)).toBe(false);
      expect(standalone.stdout).toContain('Run ./setup to migrate installed skill links');
      // Rendering has no authority to rewrite any user's installed skill tree.
      expect(fs.lstatSync(installed).isSymbolicLink()).toBe(true);
      expect(fs.readlinkSync(installed)).toBe(old);
    } finally { fs.rmSync(out, { recursive: true, force: true }); }
  }, 200_000);
});