mirror of
https://github.com/garrytan/gstack.git
synced 2026-10-31 22:14:25 +01:00
Outside-review consent (codex_reviews) is enforced in gstack-codex-probe run-with-timeout, gstack-claude-code, gstack-outside-voice run and the autoplan runner (not_run / consent_missing, exit 3; #3102). Codex reasoning effort resolves GSTACK_CODEX_EFFORT, then codex_effort, then each gate's default inside run-with-timeout --effort (D1-A; #3098, closes #2934). README and telemetry privacy disclosures corrected (#1081, #1080, #1150). Co-authored-by: Lockyer <135391289+Lockyer228@users.noreply.github.com> Co-authored-by: Lubos Buracinsky <159481718+lubosxyz@users.noreply.github.com>
874 lines
36 KiB
TypeScript
874 lines
36 KiB
TypeScript
/**
|
|
* Host config system tests — 100% coverage of host-config.ts, hosts/index.ts,
|
|
* host-config-export.ts, and golden-file regression checks.
|
|
*/
|
|
|
|
import { describe, test, expect, beforeAll, afterAll } from 'bun:test';
|
|
import * as fs from 'fs';
|
|
import * as os from 'os';
|
|
import * as path from 'path';
|
|
import { validateHostConfig, validateAllConfigs, type HostConfig } from '../scripts/host-config';
|
|
import {
|
|
ALL_HOST_CONFIGS,
|
|
ALL_HOST_NAMES,
|
|
HOST_CONFIG_MAP,
|
|
getHostConfig,
|
|
resolveHostArg,
|
|
getExternalHosts,
|
|
claude,
|
|
codex,
|
|
factory,
|
|
kiro,
|
|
opencode,
|
|
slate,
|
|
cursor,
|
|
openclaw,
|
|
agy,
|
|
} from '../hosts/index';
|
|
import { HOST_PATHS } from '../scripts/resolvers/types';
|
|
import { RESOLVERS } from '../scripts/resolvers';
|
|
import { expectTokens } from './helpers/prompt-structure';
|
|
import { runGeneration } from '../scripts/gen-skill-docs';
|
|
|
|
const ROOT = path.resolve(import.meta.dir, '..');
|
|
const RESOLVER_NAMES = new Set(Object.keys(RESOLVERS));
|
|
|
|
// ─── hosts/index.ts ─────────────────────────────────────────
|
|
|
|
describe('hosts/index.ts', () => {
|
|
test('ALL_HOST_CONFIGS has 12 hosts', () => {
|
|
expect(ALL_HOST_CONFIGS.length).toBe(12);
|
|
});
|
|
|
|
test('ALL_HOST_NAMES matches config names', () => {
|
|
expect(ALL_HOST_NAMES).toEqual(ALL_HOST_CONFIGS.map(c => c.name));
|
|
});
|
|
|
|
test('HOST_CONFIG_MAP keys match names', () => {
|
|
for (const config of ALL_HOST_CONFIGS) {
|
|
expect(HOST_CONFIG_MAP[config.name]).toBe(config);
|
|
}
|
|
});
|
|
|
|
test('individual config re-exports match registry', () => {
|
|
expect(claude.name).toBe('claude');
|
|
expect(codex.name).toBe('codex');
|
|
expect(factory.name).toBe('factory');
|
|
expect(kiro.name).toBe('kiro');
|
|
expect(opencode.name).toBe('opencode');
|
|
expect(slate.name).toBe('slate');
|
|
expect(cursor.name).toBe('cursor');
|
|
expect(openclaw.name).toBe('openclaw');
|
|
expect(agy.name).toBe('agy');
|
|
});
|
|
|
|
test('getHostConfig returns correct config', () => {
|
|
const c = getHostConfig('codex');
|
|
expect(c.name).toBe('codex');
|
|
expect(c.displayName).toBe('OpenAI Codex CLI');
|
|
});
|
|
|
|
test('getHostConfig throws on unknown host', () => {
|
|
expect(() => getHostConfig('nonexistent')).toThrow('Unknown host');
|
|
});
|
|
|
|
test('resolveHostArg resolves direct names', () => {
|
|
for (const name of ALL_HOST_NAMES) {
|
|
expect(resolveHostArg(name)).toBe(name);
|
|
}
|
|
});
|
|
|
|
test('resolveHostArg resolves aliases', () => {
|
|
expect(resolveHostArg('agents')).toBe('codex');
|
|
expect(resolveHostArg('droid')).toBe('factory');
|
|
expect(resolveHostArg('antigravity')).toBe('agy');
|
|
});
|
|
|
|
test('resolveHostArg throws on unknown alias', () => {
|
|
expect(() => resolveHostArg('nonexistent')).toThrow('Unknown host');
|
|
});
|
|
|
|
test('getExternalHosts excludes claude', () => {
|
|
const external = getExternalHosts();
|
|
expect(external.find(c => c.name === 'claude')).toBeUndefined();
|
|
expect(external.length).toBe(ALL_HOST_CONFIGS.length - 1);
|
|
});
|
|
|
|
test('every host has a unique name', () => {
|
|
const names = new Set(ALL_HOST_NAMES);
|
|
expect(names.size).toBe(ALL_HOST_NAMES.length);
|
|
});
|
|
|
|
test('every host has a unique hostSubdir', () => {
|
|
const subdirs = new Set(ALL_HOST_CONFIGS.map(c => c.hostSubdir));
|
|
expect(subdirs.size).toBe(ALL_HOST_CONFIGS.length);
|
|
});
|
|
|
|
test('every host has a unique globalRoot', () => {
|
|
const roots = new Set(ALL_HOST_CONFIGS.map(c => c.globalRoot));
|
|
expect(roots.size).toBe(ALL_HOST_CONFIGS.length);
|
|
});
|
|
});
|
|
|
|
// ─── validateHostConfig ─────────────────────────────────────
|
|
|
|
describe('validateHostConfig', () => {
|
|
function makeValid(): HostConfig {
|
|
return {
|
|
name: 'test-host',
|
|
displayName: 'Test Host',
|
|
cliCommand: 'testcli',
|
|
defaultModel: 'claude',
|
|
tier: 'experimental',
|
|
capabilities: { toolExecution: true, questions: 'prose', planMode: false, delegation: false, browser: true, safetyHooks: 'advisory' },
|
|
globalRoot: '.test/skills/gstack',
|
|
localSkillRoot: '.test/skills/gstack',
|
|
hostSubdir: '.test',
|
|
usesEnvVars: true,
|
|
frontmatter: { mode: 'allowlist', keepFields: ['name', 'description'] },
|
|
generation: { generateMetadata: false },
|
|
pathRewrites: [],
|
|
runtimeRoot: { globalSymlinks: ['bin'] },
|
|
install: { linkingStrategy: 'symlink-generated' },
|
|
};
|
|
}
|
|
|
|
test('valid config passes', () => {
|
|
expect(validateHostConfig(makeValid())).toEqual([]);
|
|
});
|
|
|
|
test('invalid name is caught', () => {
|
|
const c = makeValid();
|
|
c.name = 'UPPER_CASE';
|
|
const errors = validateHostConfig(c);
|
|
expect(errors.some(e => e.includes('name'))).toBe(true);
|
|
});
|
|
|
|
test('name with special chars is caught', () => {
|
|
const c = makeValid();
|
|
c.name = 'has spaces';
|
|
expect(validateHostConfig(c).length).toBeGreaterThan(0);
|
|
});
|
|
|
|
test('empty displayName is caught', () => {
|
|
const c = makeValid();
|
|
c.displayName = '';
|
|
expect(validateHostConfig(c).some(e => e.includes('displayName'))).toBe(true);
|
|
});
|
|
|
|
test('invalid cliCommand is caught', () => {
|
|
const c = makeValid();
|
|
c.cliCommand = 'has spaces';
|
|
expect(validateHostConfig(c).some(e => e.includes('cliCommand'))).toBe(true);
|
|
});
|
|
|
|
test('invalid cliAlias is caught', () => {
|
|
const c = makeValid();
|
|
c.cliAliases = ['good', 'BAD!'];
|
|
expect(validateHostConfig(c).some(e => e.includes('cliAlias'))).toBe(true);
|
|
});
|
|
|
|
test('valid cliAliases pass', () => {
|
|
const c = makeValid();
|
|
c.cliAliases = ['alias-one', 'alias-two'];
|
|
expect(validateHostConfig(c)).toEqual([]);
|
|
});
|
|
|
|
test('invalid defaultModel is caught', () => {
|
|
const c = makeValid();
|
|
(c as any).defaultModel = 'llama-local';
|
|
expect(validateHostConfig(c).some(e => e.includes('defaultModel'))).toBe(true);
|
|
});
|
|
|
|
test('invalid globalRoot is caught', () => {
|
|
const c = makeValid();
|
|
c.globalRoot = 'path with spaces';
|
|
expect(validateHostConfig(c).some(e => e.includes('globalRoot'))).toBe(true);
|
|
});
|
|
|
|
test('invalid localSkillRoot is caught', () => {
|
|
const c = makeValid();
|
|
c.localSkillRoot = 'invalid<path>';
|
|
expect(validateHostConfig(c).some(e => e.includes('localSkillRoot'))).toBe(true);
|
|
});
|
|
|
|
test('invalid hostSubdir is caught', () => {
|
|
const c = makeValid();
|
|
c.hostSubdir = 'no spaces allowed';
|
|
expect(validateHostConfig(c).some(e => e.includes('hostSubdir'))).toBe(true);
|
|
});
|
|
|
|
test('invalid frontmatter.mode is caught', () => {
|
|
const c = makeValid();
|
|
(c.frontmatter as any).mode = 'invalid';
|
|
expect(validateHostConfig(c).some(e => e.includes('frontmatter.mode'))).toBe(true);
|
|
});
|
|
|
|
test('invalid linkingStrategy is caught', () => {
|
|
const c = makeValid();
|
|
(c.install as any).linkingStrategy = 'invalid';
|
|
expect(validateHostConfig(c).some(e => e.includes('linkingStrategy'))).toBe(true);
|
|
});
|
|
|
|
test('paths with $ and ~ are valid', () => {
|
|
const c = makeValid();
|
|
c.globalRoot = '$HOME/.test/skills/gstack';
|
|
c.localSkillRoot = '~/.test/skills/gstack';
|
|
expect(validateHostConfig(c)).toEqual([]);
|
|
});
|
|
|
|
test('shell injection attempt in cliCommand is caught', () => {
|
|
const c = makeValid();
|
|
c.cliCommand = 'opencode;rm -rf /';
|
|
expect(validateHostConfig(c).some(e => e.includes('cliCommand'))).toBe(true);
|
|
});
|
|
|
|
test('valid suppressedResolvers pass when resolver names provided', () => {
|
|
const c = makeValid();
|
|
c.suppressedResolvers = ['DESIGN_OUTSIDE_VOICES', 'REVIEW_ARMY'];
|
|
expect(validateHostConfig(c, RESOLVER_NAMES)).toEqual([]);
|
|
});
|
|
|
|
test('unknown suppressedResolvers entry is caught', () => {
|
|
const c = makeValid();
|
|
c.suppressedResolvers = ['DESIGN_OUTSIDE_VOICES', 'NONEXISTENT_RESOLVER'];
|
|
const errors = validateHostConfig(c, RESOLVER_NAMES);
|
|
expect(errors.some(e => e.includes('NONEXISTENT_RESOLVER'))).toBe(true);
|
|
});
|
|
|
|
test('suppressedResolvers unchecked when resolver names omitted', () => {
|
|
const c = makeValid();
|
|
c.suppressedResolvers = ['TYPO_RESOLVER'];
|
|
expect(validateHostConfig(c)).toEqual([]);
|
|
});
|
|
});
|
|
|
|
// ─── validateAllConfigs ─────────────────────────────────────
|
|
|
|
describe('validateAllConfigs', () => {
|
|
test('real configs all pass validation', () => {
|
|
const errors = validateAllConfigs(ALL_HOST_CONFIGS, RESOLVER_NAMES);
|
|
expect(errors).toEqual([]);
|
|
});
|
|
|
|
test('duplicate name detected', () => {
|
|
const dup = { ...codex, name: 'claude' } as HostConfig;
|
|
const errors = validateAllConfigs([claude, dup]);
|
|
expect(errors.some(e => e.includes('Duplicate name'))).toBe(true);
|
|
});
|
|
|
|
test('duplicate hostSubdir detected', () => {
|
|
const dup = { ...codex, name: 'dup-host', hostSubdir: '.claude', globalRoot: '.dup/skills/gstack' } as HostConfig;
|
|
const errors = validateAllConfigs([claude, dup]);
|
|
expect(errors.some(e => e.includes('Duplicate hostSubdir'))).toBe(true);
|
|
});
|
|
|
|
test('duplicate globalRoot detected', () => {
|
|
const dup = { ...codex, name: 'dup-host', hostSubdir: '.dup', globalRoot: '.claude/skills/gstack' } as HostConfig;
|
|
const errors = validateAllConfigs([claude, dup]);
|
|
expect(errors.some(e => e.includes('Duplicate globalRoot'))).toBe(true);
|
|
});
|
|
|
|
test('unknown suppressedResolvers entry surfaces with host-name prefix', () => {
|
|
const bad = { ...codex, name: 'bad-host', hostSubdir: '.bad', globalRoot: '.bad/skills/gstack', suppressedResolvers: ['BOGUS_RESOLVER'] } as HostConfig;
|
|
const errors = validateAllConfigs([bad], RESOLVER_NAMES);
|
|
expect(errors.some(e => e.startsWith('[bad-host]') && e.includes('BOGUS_RESOLVER'))).toBe(true);
|
|
});
|
|
|
|
test('per-config validation errors are prefixed with host name', () => {
|
|
const bad = { ...codex, name: 'BAD', cliCommand: 'also bad' } as HostConfig;
|
|
const errors = validateAllConfigs([bad]);
|
|
expect(errors.every(e => e.startsWith('[BAD]'))).toBe(true);
|
|
});
|
|
});
|
|
|
|
// ─── HOST_PATHS derivation ──────────────────────────────────
|
|
|
|
describe('HOST_PATHS derivation from configs', () => {
|
|
test('Claude uses literal home paths (no env vars)', () => {
|
|
expect(HOST_PATHS.claude.skillRoot).toBe('~/.claude/skills/gstack');
|
|
expect(HOST_PATHS.claude.binDir).toBe('~/.claude/skills/gstack/bin');
|
|
expect(HOST_PATHS.claude.browseDir).toBe('~/.claude/skills/gstack/browse/dist');
|
|
expect(HOST_PATHS.claude.designDir).toBe('~/.claude/skills/gstack/design/dist');
|
|
});
|
|
|
|
test('Codex uses $GSTACK_ROOT env vars', () => {
|
|
expect(HOST_PATHS.codex.skillRoot).toBe('$GSTACK_ROOT');
|
|
expect(HOST_PATHS.codex.binDir).toBe('$GSTACK_BIN');
|
|
expect(HOST_PATHS.codex.browseDir).toBe('$GSTACK_BROWSE');
|
|
expect(HOST_PATHS.codex.designDir).toBe('$GSTACK_DESIGN');
|
|
});
|
|
|
|
test('every host with usesEnvVars=true gets env var paths', () => {
|
|
for (const config of ALL_HOST_CONFIGS) {
|
|
if (config.usesEnvVars) {
|
|
expect(HOST_PATHS[config.name].skillRoot).toBe('$GSTACK_ROOT');
|
|
expect(HOST_PATHS[config.name].binDir).toBe('$GSTACK_BIN');
|
|
}
|
|
}
|
|
});
|
|
|
|
test('every host with usesEnvVars=false gets literal paths', () => {
|
|
for (const config of ALL_HOST_CONFIGS) {
|
|
if (!config.usesEnvVars) {
|
|
expect(HOST_PATHS[config.name].skillRoot).toContain('~/');
|
|
expect(HOST_PATHS[config.name].binDir).toContain('/bin');
|
|
}
|
|
}
|
|
});
|
|
|
|
test('localSkillRoot matches config for every host', () => {
|
|
for (const config of ALL_HOST_CONFIGS) {
|
|
expect(HOST_PATHS[config.name].localSkillRoot).toBe(config.localSkillRoot);
|
|
}
|
|
});
|
|
|
|
test('HOST_PATHS has entry for every registered host', () => {
|
|
for (const name of ALL_HOST_NAMES) {
|
|
expect(HOST_PATHS[name]).toBeDefined();
|
|
}
|
|
});
|
|
});
|
|
|
|
// ─── host-config-export.ts CLI ──────────────────────────────
|
|
|
|
describe('host-config-export.ts CLI', () => {
|
|
const EXPORT_SCRIPT = path.join(ROOT, 'scripts', 'host-config-export.ts');
|
|
|
|
function run(...args: string[]): { stdout: string; stderr: string; exitCode: number } {
|
|
const result = Bun.spawnSync(['bun', 'run', EXPORT_SCRIPT, ...args], {
|
|
cwd: ROOT, stdout: 'pipe', stderr: 'pipe', timeout: 30_000,
|
|
});
|
|
return {
|
|
stdout: result.stdout.toString().trim(),
|
|
stderr: result.stderr.toString().trim(),
|
|
exitCode: result.exitCode,
|
|
};
|
|
}
|
|
|
|
test('list prints all host names', () => {
|
|
const { stdout, exitCode } = run('list');
|
|
expect(exitCode).toBe(0);
|
|
const names = stdout.split('\n');
|
|
expect(names).toEqual(ALL_HOST_NAMES);
|
|
});
|
|
|
|
test('get returns string field', () => {
|
|
const { stdout, exitCode } = run('get', 'codex', 'globalRoot');
|
|
expect(exitCode).toBe(0);
|
|
expect(stdout).toBe('.codex/skills/gstack');
|
|
});
|
|
|
|
test('get returns boolean as 1/0', () => {
|
|
const { stdout: t } = run('get', 'claude', 'usesEnvVars');
|
|
expect(t).toBe('0');
|
|
const { stdout: f } = run('get', 'codex', 'usesEnvVars');
|
|
expect(f).toBe('1');
|
|
});
|
|
|
|
test('get with missing args exits 1', () => {
|
|
const { exitCode } = run('get', 'codex');
|
|
expect(exitCode).toBe(1);
|
|
});
|
|
|
|
test('get with unknown field exits 1', () => {
|
|
const { exitCode } = run('get', 'codex', 'nonexistent');
|
|
expect(exitCode).toBe(1);
|
|
});
|
|
|
|
test('get with unknown host exits 1', () => {
|
|
const { exitCode } = run('get', 'nonexistent', 'name');
|
|
expect(exitCode).not.toBe(0);
|
|
});
|
|
|
|
test('validate passes for real configs', () => {
|
|
const { stdout, exitCode } = run('validate');
|
|
expect(exitCode).toBe(0);
|
|
expect(stdout).toContain('configs valid');
|
|
});
|
|
|
|
test('symlinks returns asset list', () => {
|
|
const { stdout, exitCode } = run('symlinks', 'codex');
|
|
expect(exitCode).toBe(0);
|
|
const lines = stdout.split('\n');
|
|
expect(lines).toContain('bin');
|
|
expect(lines).toContain('ETHOS.md');
|
|
expect(lines).toContain('review/checklist.md');
|
|
});
|
|
|
|
test('opencode symlinks returns nested runtime assets', () => {
|
|
const { stdout, exitCode } = run('symlinks', 'opencode');
|
|
expect(exitCode).toBe(0);
|
|
const lines = stdout.split('\n');
|
|
expect(lines).toContain('bin');
|
|
expect(lines).toContain('browse/dist');
|
|
expect(lines).toContain('browse/bin');
|
|
expect(lines).toContain('review/design-checklist.md');
|
|
expect(lines).toContain('review/greptile-triage.md');
|
|
expect(lines).toContain('review/specialists');
|
|
expect(lines).toContain('qa/templates');
|
|
expect(lines).toContain('qa/references');
|
|
expect(lines).toContain('plan-devex-review/dx-hall-of-fame.md');
|
|
});
|
|
|
|
test('symlinks with missing host exits 1', () => {
|
|
const { exitCode } = run('symlinks');
|
|
expect(exitCode).toBe(1);
|
|
});
|
|
|
|
// Gated: the secretless free-tests CI lane deliberately installs no claude
|
|
// CLI, so "we are running in claude" is false there by design.
|
|
test.skipIf(!Bun.which('claude'))('detect finds claude (since we are running in claude)', () => {
|
|
const { stdout, exitCode } = run('detect');
|
|
expect(exitCode).toBe(0);
|
|
// claude binary should be on PATH in this environment
|
|
expect(stdout).toContain('claude');
|
|
});
|
|
|
|
test('unknown command exits 1', () => {
|
|
const { exitCode } = run('badcommand');
|
|
expect(exitCode).toBe(1);
|
|
});
|
|
});
|
|
|
|
// ─── Golden-file regression ─────────────────────────────────
|
|
|
|
describe('golden-file regression', () => {
|
|
const GOLDEN_DIR = path.join(ROOT, 'test', 'fixtures', 'golden');
|
|
|
|
// #2532 successor: the codex/factory goldens used to read gitignored
|
|
// .agents/ and .factory/ artifacts "produced by gen-skill-docs.test.ts" —
|
|
// an inter-test ordering dependency that failed with ENOENT on a clean
|
|
// clone or when this file ran in isolation. Severed: this describe
|
|
// UNCONDITIONALLY renders both hosts into its own --out-dir in beforeAll
|
|
// and reads its goldens only from that render — no when-missing check, no
|
|
// live-tree reads for the gitignored artifacts, no dependence on what any
|
|
// other test left on disk. Comparing a FRESH render to the golden is also
|
|
// strictly deterministic: a stale on-disk artifact can no longer mask (or
|
|
// fake) a generator regression.
|
|
const GOLDEN_OUT = fs.mkdtempSync(path.join(os.tmpdir(), 'gstack-golden-out-'));
|
|
|
|
beforeAll(() => {
|
|
for (const host of ['codex', 'factory', 'agy']) {
|
|
const result = Bun.spawnSync(
|
|
['bun', 'run', 'scripts/gen-skill-docs.ts', '--host', host, '--out-dir', GOLDEN_OUT],
|
|
{ cwd: ROOT, timeout: 120_000 },
|
|
);
|
|
if (result.exitCode !== 0) {
|
|
throw new Error(
|
|
`golden-file beforeAll: gen-skill-docs --host ${host} --out-dir failed (exit ${result.exitCode}):\n`
|
|
+ result.stderr.toString(),
|
|
);
|
|
}
|
|
}
|
|
});
|
|
|
|
afterAll(() => {
|
|
fs.rmSync(GOLDEN_OUT, { recursive: true, force: true });
|
|
});
|
|
|
|
test('every Claude outside-voice mode uses the restricted runner and exposes an explicit model override', () => {
|
|
const rendered = fs.readFileSync(path.join(GOLDEN_OUT, '.agents/skills/gstack-claude-code/SKILL.md'), 'utf8');
|
|
const calls = rendered.split('\n').filter(line => line.includes('"$CLAUDE_RUNNER" --user-invoked --cwd'));
|
|
expect(calls).toHaveLength(3);
|
|
expect(rendered.match(/CLAUDE_RUNNER="\$RUNTIME_ROOT\/bin\/gstack-claude-code"/g)).toHaveLength(3);
|
|
for (const call of calls) {
|
|
expect(call).toContain('--timeout-ms 600000');
|
|
}
|
|
expect(rendered).toContain('GSTACK_CLAUDE_MODEL=<model>');
|
|
expect(rendered).toContain('Without an override, retain Claude');
|
|
expect(fs.existsSync(path.join(GOLDEN_OUT, '.agents/skills/gstack-claude/SKILL.md'))).toBe(false);
|
|
});
|
|
|
|
test('Claude ship skill matches golden baseline', () => {
|
|
// Deliberately reads the TRACKED ship/SKILL.md (a read, not a write):
|
|
// the claude golden pins the committed render. Freshness of the tracked
|
|
// tree vs the templates is enforced by gen-skill-docs.test.ts. (An
|
|
// out-dir claude render would NOT byte-match this golden — --out-dir
|
|
// repoints section-base paths into the render by design.)
|
|
const golden = fs.readFileSync(path.join(GOLDEN_DIR, 'claude-ship-SKILL.md'), 'utf-8');
|
|
const current = fs.readFileSync(path.join(ROOT, 'ship', 'SKILL.md'), 'utf-8');
|
|
expect(current).toBe(golden);
|
|
});
|
|
|
|
test('Codex ship skill matches golden baseline', () => {
|
|
const golden = fs.readFileSync(path.join(GOLDEN_DIR, 'codex-ship-SKILL.md'), 'utf-8');
|
|
const current = fs.readFileSync(path.join(GOLDEN_OUT, '.agents', 'skills', 'gstack-ship', 'SKILL.md'), 'utf-8');
|
|
expect(current).toBe(golden);
|
|
});
|
|
|
|
test('Factory ship skill matches golden baseline', () => {
|
|
const golden = fs.readFileSync(path.join(GOLDEN_DIR, 'factory-ship-SKILL.md'), 'utf-8');
|
|
const current = fs.readFileSync(path.join(GOLDEN_OUT, '.factory', 'skills', 'gstack-ship', 'SKILL.md'), 'utf-8');
|
|
expect(current).toBe(golden);
|
|
});
|
|
|
|
test('Antigravity ship skill matches golden baseline', () => {
|
|
const golden = fs.readFileSync(path.join(GOLDEN_DIR, 'agy-ship-SKILL.md'), 'utf-8');
|
|
const current = fs.readFileSync(path.join(GOLDEN_OUT, '.agy', 'skills', 'gstack-ship', 'SKILL.md'), 'utf-8');
|
|
expect(current).toBe(golden);
|
|
});
|
|
});
|
|
|
|
// ─── Individual host config correctness ─────────────────────
|
|
|
|
describe('host config correctness', () => {
|
|
test('Codex renders the GPT overlay, Antigravity the Gemini overlay, every other host the Claude overlay', () => {
|
|
expect(codex.defaultModel).toBe('gpt');
|
|
expect(agy.defaultModel).toBe('gemini');
|
|
for (const host of ALL_HOST_CONFIGS.filter(h => h.name !== 'codex' && h.name !== 'agy')) {
|
|
expect(host.defaultModel).toBe('claude');
|
|
}
|
|
});
|
|
|
|
test('claude is the only host with real-dir-symlink strategy', () => {
|
|
for (const config of ALL_HOST_CONFIGS) {
|
|
if (config.name === 'claude') {
|
|
expect(config.install.linkingStrategy).toBe('real-dir-symlink');
|
|
} else {
|
|
expect(config.install.linkingStrategy).toBe('symlink-generated');
|
|
}
|
|
}
|
|
});
|
|
|
|
test('claude does not use env vars', () => {
|
|
expect(claude.usesEnvVars).toBe(false);
|
|
});
|
|
|
|
test('all external hosts use env vars', () => {
|
|
for (const config of getExternalHosts()) {
|
|
expect(config.usesEnvVars).toBe(true);
|
|
}
|
|
});
|
|
|
|
test('codex has 1024-char description limit with error behavior', () => {
|
|
expect(codex.frontmatter.descriptionLimit).toBe(1024);
|
|
expect(codex.frontmatter.descriptionLimitBehavior).toBe('error');
|
|
});
|
|
|
|
test('codex generates metadata (openai.yaml, format hardcoded in gen-skill-docs)', () => {
|
|
expect(codex.generation.generateMetadata).toBe(true);
|
|
});
|
|
|
|
test('codex rewrites CLAUDE.md to AGENTS.md', () => {
|
|
expect(codex.pathRewrites).toContainEqual({ from: 'CLAUDE.md', to: 'AGENTS.md' });
|
|
});
|
|
|
|
test('factory has tool rewrites', () => {
|
|
expect(factory.toolRewrites).toBeDefined();
|
|
expect(Object.keys(factory.toolRewrites!).length).toBeGreaterThan(0);
|
|
expect(factory.toolRewrites!['use the Bash tool']).toBe('run this command');
|
|
});
|
|
|
|
test('factory has conditional disable-model-invocation field', () => {
|
|
expect(factory.frontmatter.conditionalFields).toBeDefined();
|
|
expect(factory.frontmatter.conditionalFields!.length).toBe(1);
|
|
expect(factory.frontmatter.conditionalFields![0].if).toEqual({ sensitive: true });
|
|
expect(factory.frontmatter.conditionalFields![0].add).toEqual({ 'disable-model-invocation': true });
|
|
});
|
|
|
|
test('codex restores outside-review resolvers while retaining the Review Army restriction', () => {
|
|
expect(codex.suppressedResolvers).toContain('REVIEW_ARMY');
|
|
for (const resolver of ['CODEX_SECOND_OPINION', 'ADVERSARIAL_STEP', 'CODEX_PLAN_REVIEW', 'CODEX_DOC_REVIEW', 'DESIGN_OUTSIDE_VOICES']) {
|
|
expect(codex.suppressedResolvers).not.toContain(resolver);
|
|
}
|
|
});
|
|
|
|
test('codex has boundary instruction', () => {
|
|
expect(codex.boundaryInstruction).toBeDefined();
|
|
expect(codex.boundaryInstruction).toMatch(/do not read or execute any files under/i);
|
|
for (const glob of ['~/.claude/', '~/.agents/', '.claude/skills/', 'agents/']) {
|
|
expect(codex.boundaryInstruction).toContain(glob);
|
|
}
|
|
});
|
|
|
|
test('openclaw has tool rewrites for exec/read/write', () => {
|
|
expect(openclaw.toolRewrites).toBeDefined();
|
|
expect(openclaw.toolRewrites!['use the Bash tool']).toBe('use the exec tool');
|
|
expect(openclaw.toolRewrites!['use the Read tool']).toBe('use the read tool');
|
|
});
|
|
|
|
test('openclaw has CLAUDE.md→AGENTS.md path rewrite', () => {
|
|
expect(openclaw.pathRewrites.some(r => r.from === 'CLAUDE.md' && r.to === 'AGENTS.md')).toBe(true);
|
|
});
|
|
|
|
test('no host carries a no-op empty includeSkills allowlist', () => {
|
|
// includeSkills: [] was a no-op (the generator's `?.length` guard treats an
|
|
// empty allowlist as absent), so configs omit the field instead of
|
|
// shipping a lie about "no skills generated".
|
|
for (const config of ALL_HOST_CONFIGS) {
|
|
expect(config.generation.includeSkills).toBeUndefined();
|
|
}
|
|
});
|
|
|
|
test('every host has coAuthorTrailer or undefined', () => {
|
|
// Claude, Codex, Factory, OpenClaw have explicit trailers
|
|
expect(claude.coAuthorTrailer).toContain('Claude');
|
|
expect(codex.coAuthorTrailer).toContain('Codex');
|
|
expect(factory.coAuthorTrailer).toContain('Factory');
|
|
expect(openclaw.coAuthorTrailer).toContain('OpenClaw');
|
|
});
|
|
|
|
test('outside reviewer skills are omitted only from their own harness', () => {
|
|
for (const config of ALL_HOST_CONFIGS) {
|
|
const skipped = config.generation.skipSkills ?? [];
|
|
expect(skipped.includes('codex')).toBe(config.name === 'codex');
|
|
expect(skipped.includes('claude-code')).toBe(config.name === 'claude');
|
|
expect(skipped).not.toContain('claude');
|
|
}
|
|
});
|
|
|
|
test('every host has at least one pathRewrite (except claude)', () => {
|
|
for (const config of getExternalHosts()) {
|
|
expect(config.pathRewrites.length).toBeGreaterThan(0);
|
|
}
|
|
expect(claude.pathRewrites.length).toBe(0);
|
|
});
|
|
|
|
test('every host has runtimeRoot.globalSymlinks', () => {
|
|
for (const config of ALL_HOST_CONFIGS) {
|
|
expect(config.runtimeRoot.globalSymlinks.length).toBeGreaterThan(0);
|
|
expect(config.runtimeRoot.globalSymlinks).toContain('bin');
|
|
expect(config.runtimeRoot.globalSymlinks).toContain('ETHOS.md');
|
|
}
|
|
});
|
|
});
|
|
|
|
// ─── Host contract: tier + capabilities (docs/ADDING_A_HOST.md) ─────────────
|
|
|
|
describe('host contract: one tier vocabulary everywhere', () => {
|
|
const setupSrc = fs.readFileSync(path.join(ROOT, 'setup'), 'utf8');
|
|
const registrySrc = fs.readFileSync(path.join(ROOT, 'bin', 'gstack-install-registry.sh'), 'utf8');
|
|
const readme = fs.readFileSync(path.join(ROOT, 'README.md'), 'utf8');
|
|
const hostDoc = fs.readFileSync(path.join(ROOT, 'docs', 'ADDING_A_HOST.md'), 'utf8');
|
|
const byTier = (tier: string) => ALL_HOST_CONFIGS.filter(c => c.tier === tier).map(c => c.name);
|
|
|
|
test('every host declares a tier and coherent capabilities', () => {
|
|
for (const c of ALL_HOST_CONFIGS) {
|
|
expect(['full', 'experimental', 'instruction-only']).toContain(c.tier);
|
|
expect(validateHostConfig(c, RESOLVER_NAMES)).toEqual([]);
|
|
}
|
|
expect(validateHostConfig({ ...ALL_HOST_CONFIGS[0], capabilities: { ...ALL_HOST_CONFIGS[0].capabilities, toolExecution: false } }))
|
|
.toContain('capabilities.browser requires toolExecution');
|
|
expect(validateHostConfig({ ...ALL_HOST_CONFIGS[0], tier: 'beta' as never })).toContain("tier 'beta' must be one of full, experimental, instruction-only");
|
|
});
|
|
|
|
test('only hosts that run gstack hooks claim enforced safety', () => {
|
|
for (const c of ALL_HOST_CONFIGS) {
|
|
expect(c.capabilities.safetyHooks).toBe(c.name === 'claude' ? 'enforced' : 'advisory');
|
|
}
|
|
});
|
|
|
|
test('instruction-only hosts have no install arm; installable hosts do', () => {
|
|
const start = setupSrc.indexOf('case "$HOST" in');
|
|
const block = setupSrc.slice(start, setupSrc.indexOf('\nesac', start));
|
|
const installList = block.match(/^\s*([a-z|]+)\) ;;/m)![1].split('|').filter(h => h !== 'auto');
|
|
expect(installList.sort()).toEqual([...byTier('full'), ...byTier('experimental')].sort());
|
|
for (const name of byTier('instruction-only')) expect(installList).not.toContain(name);
|
|
});
|
|
|
|
test('gstack_host_tier (setup summaries, --status) matches hosts/index.ts', () => {
|
|
for (const c of ALL_HOST_CONFIGS) {
|
|
const r = Bun.spawnSync(['bash', '-c', `. "${path.join(ROOT, 'bin', 'gstack-install-registry.sh')}"; gstack_host_tier ${c.name}`], { timeout: 30_000 });
|
|
expect(r.stdout.toString().trim()).toBe(c.tier);
|
|
}
|
|
expect(registrySrc).toContain('gstack_host_tier()');
|
|
});
|
|
|
|
test('./setup --help lists every host under its tier', () => {
|
|
const r = Bun.spawnSync(['bash', path.join(ROOT, 'setup'), '--help'], { timeout: 30_000 });
|
|
const help = r.stdout.toString();
|
|
for (const tier of ['full', 'experimental', 'instruction-only']) {
|
|
const line = help.split('\n').find(l => l.trim().startsWith(`${tier}:`));
|
|
expect(line, `--help has a ${tier}: line`).toBeTruthy();
|
|
const listed = line!.split(':')[1].split(',').map(s => s.trim()).filter(Boolean);
|
|
expect(listed.sort()).toEqual(byTier(tier).sort());
|
|
}
|
|
expect(help).toContain('Default: claude');
|
|
expect(help).toContain('--host auto');
|
|
});
|
|
|
|
test('README host matrix has one row per host with its tier and install command', () => {
|
|
const start = readme.indexOf('| Agent | Tier |');
|
|
expect(start, 'README has the host matrix').toBeGreaterThan(-1);
|
|
const rows = readme.slice(start).split('\n').slice(2).filter(l => l.startsWith('|'));
|
|
for (const c of ALL_HOST_CONFIGS) {
|
|
const row = rows.find(r => r.includes(`\`--host ${c.name}\``));
|
|
expect(row, `README matrix row for ${c.name}`).toBeTruthy();
|
|
expect(row!.split('|')[2].trim().toLowerCase()).toBe(c.tier);
|
|
const safety = c.capabilities.safetyHooks === 'enforced' ? 'enforced' : 'advisory';
|
|
expect(row!.toLowerCase()).toContain(safety);
|
|
}
|
|
});
|
|
|
|
test('every full-tier host has a dated certification record', () => {
|
|
const section = hostDoc.slice(hostDoc.indexOf('## Certify your host'));
|
|
expect(section.length).toBeGreaterThan(20);
|
|
for (const name of byTier('full')) {
|
|
expect(section, `certification row for ${name}`).toMatch(new RegExp(`\\| ${name} \\|[^\\n]*\\d{4}-\\d{2}-\\d{2}`));
|
|
}
|
|
});
|
|
|
|
test('ADDING_A_HOST.md no longer claims zero setup code changes', () => {
|
|
expect(hostDoc).not.toMatch(/zero code changes to setup/i);
|
|
expect(readme).not.toMatch(/one TypeScript config file, zero code changes/i);
|
|
});
|
|
});
|
|
|
|
describe('hookless hosts render an honest "not enforced" safety line', () => {
|
|
const OUT = fs.mkdtempSync(path.join(os.tmpdir(), 'gstack-advisory-'));
|
|
beforeAll(() => {
|
|
const r = Bun.spawnSync(['bun', 'run', 'scripts/gen-skill-docs.ts', '--host', 'opencode', '--out-dir', OUT], { cwd: ROOT, timeout: 120_000 });
|
|
if (r.exitCode !== 0) throw new Error(r.stderr.toString());
|
|
});
|
|
afterAll(() => fs.rmSync(OUT, { recursive: true, force: true }));
|
|
|
|
for (const skill of ['careful', 'freeze', 'guard']) {
|
|
test(`${skill}: advisory, not blocked`, () => {
|
|
const md = fs.readFileSync(path.join(OUT, '.opencode', 'skills', `gstack-${skill}`, 'SKILL.md'), 'utf8');
|
|
expect(md).toContain('not enforced on OpenCode');
|
|
expect(md).toContain('advisory, not blocked');
|
|
expect(md.split('\n').slice(0, 12).join('\n')).not.toContain('hooks:');
|
|
const claude = fs.readFileSync(path.join(ROOT, skill, 'SKILL.md'), 'utf8');
|
|
expect(claude).not.toContain('not enforced on');
|
|
});
|
|
}
|
|
});
|
|
|
|
describe('Copilot host render (#393)', () => {
|
|
const OUT = fs.mkdtempSync(path.join(os.tmpdir(), 'gstack-copilot-'));
|
|
beforeAll(() => {
|
|
const r = Bun.spawnSync(['bun', 'run', 'scripts/gen-skill-docs.ts', '--host', 'copilot', '--out-dir', OUT], { cwd: ROOT, timeout: 120_000 });
|
|
if (r.exitCode !== 0) throw new Error(r.stderr.toString());
|
|
});
|
|
afterAll(() => fs.rmSync(OUT, { recursive: true, force: true }));
|
|
const read = (skill: string) => fs.readFileSync(path.join(OUT, '.copilot', 'skills', skill, 'SKILL.md'), 'utf8');
|
|
|
|
test('paths name the Copilot install, never Claude or Codex', () => {
|
|
for (const skill of ['gstack-review', 'gstack-ship', 'gstack-careful', 'gstack-upgrade']) {
|
|
const md = read(skill);
|
|
expect(md).not.toContain('~/.claude/skills/gstack');
|
|
expect(md).not.toContain('$HOME/.claude/skills/gstack');
|
|
expect(md).not.toContain('.codex/skills');
|
|
}
|
|
expect(read('gstack-upgrade')).toContain('$HOME/.copilot/skills/gstack/.source-path');
|
|
expect(read('gstack-upgrade')).toContain('./setup --host copilot --refresh-registered');
|
|
});
|
|
|
|
test('the tool-name glossary lands ahead of the preamble STATUS rules', () => {
|
|
const md = read('gstack-review');
|
|
const glossary = md.indexOf('**GitHub Copilot tool names:**');
|
|
expect(glossary).toBeGreaterThan(-1);
|
|
expect(glossary).toBeLessThan(md.indexOf('Read the echoed `KEY: value` STATUS lines'));
|
|
});
|
|
|
|
test('sensitive skills are not model-invocable', () => {
|
|
expect(read('gstack-ship')).toMatch(/^disable-model-invocation: true$/m);
|
|
});
|
|
});
|
|
|
|
describe('Antigravity CLI host render', () => {
|
|
const OUT = fs.mkdtempSync(path.join(os.tmpdir(), 'gstack-agy-'));
|
|
beforeAll(() => {
|
|
const r = Bun.spawnSync(['bun', 'run', 'scripts/gen-skill-docs.ts', '--host', 'agy', '--out-dir', OUT], { cwd: ROOT, timeout: 120_000 });
|
|
if (r.exitCode !== 0) throw new Error(r.stderr.toString());
|
|
});
|
|
afterAll(() => fs.rmSync(OUT, { recursive: true, force: true }));
|
|
const dir = path.join(OUT, '.agy', 'skills');
|
|
const read = (skill: string) => fs.readFileSync(path.join(dir, skill, 'SKILL.md'), 'utf8');
|
|
|
|
test('paths name the Antigravity CLI install, never Claude, Codex or Gemini CLI skill dirs', () => {
|
|
for (const skill of ['gstack-review', 'gstack-ship', 'gstack-careful', 'gstack-upgrade']) {
|
|
const md = read(skill);
|
|
expect(md).not.toContain('~/.claude/skills/gstack');
|
|
expect(md).not.toContain('$HOME/.claude/skills/gstack');
|
|
expect(md).not.toContain('.codex/skills');
|
|
expect(md).not.toMatch(/\.gemini\/(?:config\/)?skills/);
|
|
}
|
|
expect(read('gstack-review')).toContain('_r=~/.gemini/antigravity-cli/skills/gstack');
|
|
expect(read('gstack-upgrade')).toContain('$HOME/.gemini/antigravity-cli/skills/gstack/.source-path');
|
|
expect(read('gstack-upgrade')).toContain('./setup --host agy --refresh-registered');
|
|
});
|
|
|
|
test('the tool-name glossary names the documented tools ahead of the preamble STATUS rules', () => {
|
|
const md = read('gstack-review');
|
|
const glossary = md.indexOf('**Antigravity tool names:**');
|
|
expect(glossary).toBeGreaterThan(-1);
|
|
expect(glossary).toBeLessThan(md.indexOf('Read the echoed `KEY: value` STATUS lines'));
|
|
const line = md.slice(glossary, md.indexOf('\n', glossary));
|
|
expectTokens(line, ['`ask_question`', '`run_command`', '`view_file`', '`write_to_file`', '`replace_file_content`', '`grep_search`', '`find_by_name`', '`invoke_subagent`']);
|
|
});
|
|
|
|
test('frontmatter keeps only name and description, and every name equals its directory', () => {
|
|
const names = fs.readdirSync(dir).filter(d => fs.existsSync(path.join(dir, d, 'SKILL.md')));
|
|
expect(names.length).toBeGreaterThan(20);
|
|
for (const name of names) {
|
|
const fm = read(name).split('\n---\n')[0];
|
|
expect(fm.match(/^name:\s*(\S+)/m)![1]).toBe(name);
|
|
expect(fm.split('\n').filter(l => /^[a-z-]+:/.test(l)).map(l => l.split(':')[0])).toEqual(['name', 'description']);
|
|
}
|
|
});
|
|
});
|
|
|
|
describe('host renders name the host\'s own tools and identities (#2626, #2015, #2825, #2338)', () => {
|
|
const OUT = fs.mkdtempSync(path.join(os.tmpdir(), 'gstack-host-tools-'));
|
|
beforeAll(() => {
|
|
for (const host of ['opencode', 'hermes', 'codex']) {
|
|
const r = Bun.spawnSync(['bun', 'run', 'scripts/gen-skill-docs.ts', '--host', host, '--out-dir', OUT], { cwd: ROOT, timeout: 120_000 });
|
|
if (r.exitCode !== 0) throw new Error(r.stderr.toString());
|
|
}
|
|
});
|
|
afterAll(() => fs.rmSync(OUT, { recursive: true, force: true }));
|
|
const read = (subdir: string, skill: string) => fs.readFileSync(path.join(OUT, subdir, 'skills', skill, 'SKILL.md'), 'utf8');
|
|
|
|
test.each([['.opencode', '`question` tool'], ['.hermes', '`clarify` tool']])('%s preamble maps AskUserQuestion to %s', (subdir, tool) => {
|
|
const md = read(subdir, 'gstack-review');
|
|
const glossary = md.indexOf(tool);
|
|
expect(glossary).toBeGreaterThan(-1);
|
|
expect(glossary).toBeLessThan(md.indexOf('Read the echoed `KEY: value` STATUS lines'));
|
|
});
|
|
|
|
test('every Hermes skill\'s frontmatter name equals its directory (#2825)', () => {
|
|
const dir = path.join(OUT, '.hermes', 'skills');
|
|
const names = fs.readdirSync(dir).filter(d => fs.existsSync(path.join(dir, d, 'SKILL.md')));
|
|
expect(names.length).toBeGreaterThan(20);
|
|
for (const name of names) {
|
|
expect(fs.readFileSync(path.join(dir, name, 'SKILL.md'), 'utf8').match(/^name:\s*(\S+)/m)![1]).toBe(name);
|
|
}
|
|
});
|
|
|
|
test('Codex renders never name a ~/.Codex path (#2338)', () => {
|
|
const dir = path.join(OUT, '.agents', 'skills');
|
|
for (const name of fs.readdirSync(dir)) {
|
|
const md = path.join(dir, name, 'SKILL.md');
|
|
if (fs.existsSync(md)) expect(fs.readFileSync(md, 'utf8')).not.toMatch(/\.Codex\//);
|
|
}
|
|
});
|
|
});
|
|
|
|
describe('agent-runtime hosts never leak the Claude "Agent tool" name', () => {
|
|
const OUT = fs.mkdtempSync(path.join(os.tmpdir(), 'gstack-agent-tool-'));
|
|
afterAll(() => fs.rmSync(OUT, { recursive: true, force: true }));
|
|
|
|
test.each([
|
|
['hermes', 'delegate_task'],
|
|
['gbrain', 'sessions_spawn'],
|
|
['openclaw', 'sessions_spawn'],
|
|
['factory', 'delegation'],
|
|
])('%s renders map every Agent tool phrasing to %s', async (host, token) => {
|
|
const result = await runGeneration({ host: host as 'hermes', outputRoot: path.join(OUT, host), contentLinkRoot: null });
|
|
expect(result.exitCode).toBe(0);
|
|
const rendered = result.artifacts.filter(a => a.kind === 'skill' || a.kind === 'section');
|
|
expect(rendered.length).toBeGreaterThan(20);
|
|
const leaks: string[] = [];
|
|
let mentions = 0;
|
|
for (const artifact of rendered) {
|
|
const content = fs.readFileSync(path.join(OUT, host, artifact.relativePath), 'utf8');
|
|
if (/\bAgent tool\b/.test(content)) leaks.push(artifact.relativePath);
|
|
if (content.includes(`via ${token}`)) mentions++;
|
|
}
|
|
expect(leaks).toEqual([]);
|
|
expect(mentions).toBeGreaterThan(0);
|
|
}, 120_000);
|
|
});
|