test(eval-plan): eval-plan/SKILL.md names only defined codes and accepted flags

Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>
This commit is contained in:
garrytanandcapy-ai[bot] committed 2026-10-16 23:17:07 +00:00
1 parent 9335e45de2
commit a7a974d268
1 file changed
+15
+15
View File
@@ -11,6 +11,7 @@ import * as fs from 'node:fs';
import * as path from 'node:path';
import { formatEvalCalls } from '../lib/eval-plan';
import { lintArmModels, parseEvalModels, type ArmUse, type EvalModelRules } from '../lib/eval-models';
import { RESULT_CODES } from '../lib/result-codes';
import { capyHost, type CapyHost } from './helpers/capy-host';
const BIN = path.resolve(import.meta.dir, '..', 'bin', 'gstack-eval-plan');
@@ -82,6 +83,20 @@ describe('parseEvalModels', () => {
});
});
describe('eval-plan/SKILL.md', () => {
test('names only result codes the table defines and flags the bin accepts for the diagnostics run and the model rules', () => {
const skill = fs.readFileSync(path.resolve(import.meta.dir, '..', 'eval-plan', 'SKILL.md'), 'utf8');
const codes = [...new Set([...skill.matchAll(/\b(PREREG_[A-Z_]+|EVAL_MODELS?_[A-Z_]+)\b/g)].map(m => m[1]!))];
expect(codes).toEqual(expect.arrayContaining(['PREREG_DIAGNOSTIC_UNTESTED', 'EVAL_MODEL_DENIED', 'EVAL_MODEL_OLD_GENERATION', 'EVAL_MODEL_GENERATION_UNRESOLVED']));
expect(codes.filter(c => !(c in RESULT_CODES) && !['PREREG_DIAGNOSTIC', 'EVAL_MODEL'].includes(c))).toEqual([]);
expect(skill).toContain('prereg --check <path> --allow-repo-commands');
expect(skill).toContain('--scope smoke');
const help = spawnSync(BIN, ['--help'], { encoding: 'utf8', timeout: 30_000 }).stdout;
expect(help).toContain('prereg --check <file> [--allow-repo-commands]');
expect(help).toContain('--scope smoke|full');
});
});
describe('gstack-eval-plan lints every arm (bin, fixture repo with an origin)', () => {
let host: CapyHost;
let repo: string;