From 189d48fd498e25d5c016b9672946bec146a395f0 Mon Sep 17 00:00:00 2001 From: Garry Tan Date: Sat, 28 Mar 2026 22:20:24 -0700 Subject: [PATCH] test: add 16 failing tests for 6 community fixes Tests-first for all fixes in this PR wave: - #594 discoverability: gstack tag in descriptions, 120-char first line - #573 feature signals: ship/SKILL.md Step 4 detection - #510 context warnings: no preemptive warnings in generated files - #474 Safety Net: no find -delete in generated files - #467 telemetry: JSONL writes gated by _TEL conditional - #584 sidebar: Write in allowedTools, stderr capture - #578 relink: prefixed/flat symlinks, cleanup, error, config hook Co-Authored-By: Claude Opus 4.6 (1M context) --- test/gen-skill-docs.test.ts | 94 +++++++++++++++++++++++ test/relink.test.ts | 137 ++++++++++++++++++++++++++++++++++ test/skill-validation.test.ts | 27 +++++++ test/telemetry.test.ts | 22 ++++++ 4 files changed, 280 insertions(+) create mode 100644 test/relink.test.ts diff --git a/test/gen-skill-docs.test.ts b/test/gen-skill-docs.test.ts index 3bbc1869..3cc5e799 100644 --- a/test/gen-skill-docs.test.ts +++ b/test/gen-skill-docs.test.ts @@ -1882,6 +1882,100 @@ describe('telemetry', () => { }); }); +describe('community fixes wave', () => { + // Helper to get all generated SKILL.md files + function getAllSkillMds(): Array<{ name: string; content: string }> { + const results: Array<{ name: string; content: string }> = []; + const rootPath = path.join(ROOT, 'SKILL.md'); + if (fs.existsSync(rootPath)) { + results.push({ name: 'root', content: fs.readFileSync(rootPath, 'utf-8') }); + } + for (const entry of fs.readdirSync(ROOT, { withFileTypes: true })) { + if (!entry.isDirectory() || entry.name.startsWith('.') || entry.name === 'node_modules') continue; + const skillPath = path.join(ROOT, entry.name, 'SKILL.md'); + if (fs.existsSync(skillPath)) { + results.push({ name: entry.name, content: fs.readFileSync(skillPath, 'utf-8') }); + } + } + return results; + } + + // #594 — Discoverability: every SKILL.md.tmpl description contains "gstack" + test('every SKILL.md.tmpl description contains "gstack"', () => { + for (const skill of ALL_SKILLS) { + const tmplPath = skill.dir === '.' ? path.join(ROOT, 'SKILL.md.tmpl') : path.join(ROOT, skill.dir, 'SKILL.md.tmpl'); + const content = fs.readFileSync(tmplPath, 'utf-8'); + const desc = extractDescription(content); + expect(desc.toLowerCase()).toContain('gstack'); + } + }); + + // #594 — Discoverability: first line of each description is under 120 chars + test('every SKILL.md.tmpl description first line is under 120 chars', () => { + for (const skill of ALL_SKILLS) { + const tmplPath = skill.dir === '.' ? path.join(ROOT, 'SKILL.md.tmpl') : path.join(ROOT, skill.dir, 'SKILL.md.tmpl'); + const content = fs.readFileSync(tmplPath, 'utf-8'); + const desc = extractDescription(content); + const firstLine = desc.split('\n')[0]; + expect(firstLine.length).toBeLessThanOrEqual(120); + } + }); + + // #573 — Feature signals: ship/SKILL.md contains feature signal detection + test('ship/SKILL.md contains feature signal detection in Step 4', () => { + const content = fs.readFileSync(path.join(ROOT, 'ship', 'SKILL.md'), 'utf-8'); + expect(content.toLowerCase()).toContain('feature signal'); + }); + + // #510 — Context warnings: no SKILL.md contains "running low on context" + test('no generated SKILL.md contains "running low on context"', () => { + const skills = getAllSkillMds(); + for (const { name, content } of skills) { + expect(content).not.toContain('running low on context'); + } + }); + + // #510 — Context warnings: plan-eng-review has explicit anti-warning + test('plan-eng-review/SKILL.md contains "Do not preemptively warn"', () => { + const content = fs.readFileSync(path.join(ROOT, 'plan-eng-review', 'SKILL.md'), 'utf-8'); + expect(content).toContain('Do not preemptively warn'); + }); + + // #474 — Safety Net: no SKILL.md uses find with -delete + test('no generated SKILL.md contains find with -delete flag', () => { + const skills = getAllSkillMds(); + for (const { name, content } of skills) { + // Match find commands that use -delete (but not prose mentioning the word "delete") + const lines = content.split('\n'); + for (const line of lines) { + if (line.includes('find ') && line.includes('-delete')) { + throw new Error(`${name}/SKILL.md contains find with -delete: ${line.trim()}`); + } + } + } + }); + + // #467 — Telemetry: preamble JSONL writes are gated by telemetry setting + test('preamble JSONL writes are inside telemetry conditional', () => { + const preamble = fs.readFileSync(path.join(ROOT, 'scripts/resolvers/preamble.ts'), 'utf-8'); + // Find all skill-usage.jsonl write lines + const lines = preamble.split('\n'); + for (let i = 0; i < lines.length; i++) { + if (lines[i].includes('skill-usage.jsonl') && lines[i].includes('>>')) { + // Look backwards for a telemetry conditional within 5 lines + let foundConditional = false; + for (let j = i - 1; j >= Math.max(0, i - 5); j--) { + if (lines[j].includes('_TEL') && lines[j].includes('off')) { + foundConditional = true; + break; + } + } + expect(foundConditional).toBe(true); + } + } + }); +}); + describe('codex commands must not use inline $(git rev-parse --show-toplevel) for cwd', () => { // Regression test: inline $(git rev-parse --show-toplevel) in codex exec -C // or codex review without cd evaluates in whatever cwd the background shell diff --git a/test/relink.test.ts b/test/relink.test.ts new file mode 100644 index 00000000..7a951a90 --- /dev/null +++ b/test/relink.test.ts @@ -0,0 +1,137 @@ +import { describe, test, expect, beforeEach, afterEach } from 'bun:test'; +import { execSync } from 'child_process'; +import * as fs from 'fs'; +import * as path from 'path'; +import * as os from 'os'; + +const ROOT = path.resolve(import.meta.dir, '..'); +const BIN = path.join(ROOT, 'bin'); + +let tmpDir: string; +let skillsDir: string; +let installDir: string; + +function run(cmd: string, env: Record = {}, expectFail = false): string { + try { + return execSync(cmd, { + cwd: ROOT, + env: { ...process.env, GSTACK_STATE_DIR: tmpDir, ...env }, + encoding: 'utf-8', + timeout: 10000, + stdio: ['pipe', 'pipe', 'pipe'], + }).trim(); + } catch (e: any) { + if (expectFail) return (e.stderr || e.stdout || '').toString().trim(); + throw e; + } +} + +// Create a mock gstack install directory with skill subdirs +function setupMockInstall(skills: string[]): void { + installDir = path.join(tmpDir, 'gstack-install'); + skillsDir = path.join(tmpDir, 'skills'); + fs.mkdirSync(installDir, { recursive: true }); + fs.mkdirSync(skillsDir, { recursive: true }); + + // Copy the real gstack-config and gstack-relink to the mock install + const mockBin = path.join(installDir, 'bin'); + fs.mkdirSync(mockBin, { recursive: true }); + fs.copyFileSync(path.join(BIN, 'gstack-config'), path.join(mockBin, 'gstack-config')); + fs.chmodSync(path.join(mockBin, 'gstack-config'), 0o755); + if (fs.existsSync(path.join(BIN, 'gstack-relink'))) { + fs.copyFileSync(path.join(BIN, 'gstack-relink'), path.join(mockBin, 'gstack-relink')); + fs.chmodSync(path.join(mockBin, 'gstack-relink'), 0o755); + } + + // Create mock skill directories + for (const skill of skills) { + fs.mkdirSync(path.join(installDir, skill), { recursive: true }); + fs.writeFileSync(path.join(installDir, skill, 'SKILL.md'), `# ${skill}`); + } +} + +beforeEach(() => { + tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'gstack-relink-test-')); +}); + +afterEach(() => { + fs.rmSync(tmpDir, { recursive: true, force: true }); +}); + +describe('gstack-relink (#578)', () => { + // Test 11: prefixed symlinks when skill_prefix=true + test('creates gstack-* symlinks when skill_prefix=true', () => { + setupMockInstall(['qa', 'ship', 'review']); + // Set config to prefix mode + run(`${path.join(installDir, 'bin', 'gstack-config')} set skill_prefix true`); + // Run relink with env pointing to the mock install + const output = run(`${path.join(installDir, 'bin', 'gstack-relink')}`, { + GSTACK_INSTALL_DIR: installDir, + GSTACK_SKILLS_DIR: skillsDir, + }); + // Verify gstack-* symlinks exist + expect(fs.existsSync(path.join(skillsDir, 'gstack-qa'))).toBe(true); + expect(fs.existsSync(path.join(skillsDir, 'gstack-ship'))).toBe(true); + expect(fs.existsSync(path.join(skillsDir, 'gstack-review'))).toBe(true); + expect(output).toContain('gstack-'); + }); + + // Test 12: flat symlinks when skill_prefix=false + test('creates flat symlinks when skill_prefix=false', () => { + setupMockInstall(['qa', 'ship', 'review']); + run(`${path.join(installDir, 'bin', 'gstack-config')} set skill_prefix false`); + const output = run(`${path.join(installDir, 'bin', 'gstack-relink')}`, { + GSTACK_INSTALL_DIR: installDir, + GSTACK_SKILLS_DIR: skillsDir, + }); + expect(fs.existsSync(path.join(skillsDir, 'qa'))).toBe(true); + expect(fs.existsSync(path.join(skillsDir, 'ship'))).toBe(true); + expect(fs.existsSync(path.join(skillsDir, 'review'))).toBe(true); + expect(output).toContain('flat'); + }); + + // Test 13: cleans stale symlinks from opposite mode + test('cleans up stale symlinks from opposite mode', () => { + setupMockInstall(['qa', 'ship']); + // Create prefixed symlinks first + run(`${path.join(installDir, 'bin', 'gstack-config')} set skill_prefix true`); + run(`${path.join(installDir, 'bin', 'gstack-relink')}`, { + GSTACK_INSTALL_DIR: installDir, + GSTACK_SKILLS_DIR: skillsDir, + }); + expect(fs.existsSync(path.join(skillsDir, 'gstack-qa'))).toBe(true); + + // Switch to flat mode + run(`${path.join(installDir, 'bin', 'gstack-config')} set skill_prefix false`); + run(`${path.join(installDir, 'bin', 'gstack-relink')}`, { + GSTACK_INSTALL_DIR: installDir, + GSTACK_SKILLS_DIR: skillsDir, + }); + + // Flat symlinks should exist, prefixed should be gone + expect(fs.existsSync(path.join(skillsDir, 'qa'))).toBe(true); + expect(fs.existsSync(path.join(skillsDir, 'gstack-qa'))).toBe(false); + }); + + // Test 14: error when install dir missing + test('prints error when install dir missing', () => { + const output = run(`${BIN}/gstack-relink`, { + GSTACK_INSTALL_DIR: '/nonexistent/path/gstack', + GSTACK_SKILLS_DIR: '/nonexistent/path/skills', + }, true); + expect(output).toContain('setup'); + }); + + // Test 15: gstack-config set skill_prefix triggers relink + test('gstack-config set skill_prefix triggers relink', () => { + setupMockInstall(['qa', 'ship']); + // Run gstack-config set which should auto-trigger relink + run(`${path.join(installDir, 'bin', 'gstack-config')} set skill_prefix true`, { + GSTACK_INSTALL_DIR: installDir, + GSTACK_SKILLS_DIR: skillsDir, + }); + // If relink was triggered, symlinks should exist + expect(fs.existsSync(path.join(skillsDir, 'gstack-qa'))).toBe(true); + expect(fs.existsSync(path.join(skillsDir, 'gstack-ship'))).toBe(true); + }); +}); diff --git a/test/skill-validation.test.ts b/test/skill-validation.test.ts index 7bb163d8..46398d5a 100644 --- a/test/skill-validation.test.ts +++ b/test/skill-validation.test.ts @@ -1547,3 +1547,30 @@ describe('Test failure triage in ship skill', () => { expect(content).toContain('In-branch test failures'); }); }); + +describe('sidebar agent (#584)', () => { + // #584 — Sidebar Write: sidebar-agent.ts allowedTools includes Write + test('sidebar-agent.ts allowedTools includes Write', () => { + const content = fs.readFileSync(path.join(ROOT, 'browse', 'src', 'sidebar-agent.ts'), 'utf-8'); + // Find the allowedTools line in the askClaude function + const match = content.match(/--allowedTools['"]\s*,\s*['"]([^'"]+)['"]/); + expect(match).not.toBeNull(); + expect(match![1]).toContain('Write'); + }); + + // #584 — Server Write: server.ts allowedTools includes Write (DRY parity) + test('server.ts allowedTools includes Write', () => { + const content = fs.readFileSync(path.join(ROOT, 'browse', 'src', 'server.ts'), 'utf-8'); + // Find the sidebar allowedTools in the headed-mode path + const match = content.match(/--allowedTools['"]\s*,\s*['"]([^'"]+)['"]/); + expect(match).not.toBeNull(); + expect(match![1]).toContain('Write'); + }); + + // #584 — Sidebar stderr: stderr handler is not empty + test('sidebar-agent.ts stderr handler is not empty', () => { + const content = fs.readFileSync(path.join(ROOT, 'browse', 'src', 'sidebar-agent.ts'), 'utf-8'); + // The stderr handler should NOT be an empty arrow function + expect(content).not.toContain("proc.stderr.on('data', () => {})"); + }); +}); diff --git a/test/telemetry.test.ts b/test/telemetry.test.ts index dd63509f..96bdf54c 100644 --- a/test/telemetry.test.ts +++ b/test/telemetry.test.ts @@ -396,3 +396,25 @@ describe('gstack-community-dashboard', () => { expect(output).not.toContain('Supabase not configured'); }); }); + +describe('preamble telemetry gating (#467)', () => { + test('preamble source does not write JSONL unconditionally', () => { + const preamble = fs.readFileSync(path.join(ROOT, 'scripts', 'resolvers', 'preamble.ts'), 'utf-8'); + const lines = preamble.split('\n'); + for (let i = 0; i < lines.length; i++) { + if (lines[i].includes('skill-usage.jsonl') && lines[i].includes('>>')) { + // Each JSONL write must be inside a _TEL conditional (within 5 lines above) + let foundConditional = false; + for (let j = i - 1; j >= Math.max(0, i - 5); j--) { + if (lines[j].includes('_TEL') && lines[j].includes('off')) { + foundConditional = true; + break; + } + } + if (!foundConditional) { + throw new Error(`Unconditional JSONL write at preamble.ts line ${i + 1}: ${lines[i].trim()}`); + } + } + } + }); +});