mirror of
https://github.com/garrytan/gstack.git
synced 2026-09-01 02:40:47 +02:00
feat(review): simplification specialist — advisory over-engineering lens with ponytail's tag vocabulary
New 8th Review Army specialist (DIFF_LINES > 100, --simplification force flag) hunting unrequested STRUCTURE only: delete/stdlib/native/speculative/shrink closed tags, one-line findings, lines_removable field. speculative: replaces ponytail's yagni: tag — we import the lens, not the posture; coverage stays sacred (Completeness Gaps owns it, suppressions inlined, shrink needs >=5 lines). Advisory carve-out in the merge step: advisory findings are excluded from quality_score and the findings-count header, render with an [ADVISORY] label, and are ASK-only in Fix-First. Zero-findings case prints the lens-scoped 'Simplification: lean already — nothing to cut.' from the PARENT (the specialist keeps the exact NO FINDINGS contract); with findings, the parent prints 'net: -N lines possible' summed from lines_removable. Tests: static pins for the carve-out + early-out contract (gen-skill-docs), two periodic e2e cases with planted fixtures — activation (over-build traps: hand-rolled Intl, one-impl abstract, dead config) and false-flag precision (a lean ETHOS 'choose A' diff must yield NO FINDINGS). Inspired by dietrichgebert/ponytail's /ponytail-review. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Fable 5
parent
848973007c
commit
cbb4d35792
+28
@@ -0,0 +1,28 @@
|
||||
// Lean-and-complete fixture: the false-flag precision case for the
|
||||
// simplification specialist. This is an ETHOS "choose A" diff — small,
|
||||
// covers the error path and edge cases, carries its own check. There is
|
||||
// nothing here to cut; a correct simplification pass returns NO FINDINGS.
|
||||
|
||||
function parsePort(value) {
|
||||
if (value === null || value === undefined || value === '') {
|
||||
throw new Error(`parsePort: missing value`);
|
||||
}
|
||||
const port = Number(value);
|
||||
if (!Number.isInteger(port) || port < 1 || port > 65535) {
|
||||
throw new Error(`parsePort: expected integer in 1-65535, got ${JSON.stringify(value)}`);
|
||||
}
|
||||
return port;
|
||||
}
|
||||
|
||||
// Self-check: the smallest thing that fails if the logic breaks.
|
||||
function testParsePort() {
|
||||
const assert = require('node:assert');
|
||||
assert.strictEqual(parsePort('8080'), 8080);
|
||||
assert.strictEqual(parsePort(443), 443);
|
||||
assert.throws(() => parsePort(''), /missing value/);
|
||||
assert.throws(() => parsePort('0'), /1-65535/);
|
||||
assert.throws(() => parsePort('65536'), /1-65535/);
|
||||
assert.throws(() => parsePort('abc'), /1-65535/);
|
||||
}
|
||||
|
||||
module.exports = { parsePort, testParsePort };
|
||||
+68
@@ -0,0 +1,68 @@
|
||||
// Planted over-engineering fixture for the simplification specialist.
|
||||
// Three deliberate invitations: a hand-rolled date formatter the platform
|
||||
// ships (native:), an abstract layer with exactly one implementation
|
||||
// (speculative:), and a config block nothing reads (delete:).
|
||||
|
||||
// INVITATION 1 (native:): Intl.DateTimeFormat does all of this in one line.
|
||||
class DateFormatter {
|
||||
constructor(locale) {
|
||||
this.locale = locale || 'en-US';
|
||||
this.monthNames = [
|
||||
'January', 'February', 'March', 'April', 'May', 'June',
|
||||
'July', 'August', 'September', 'October', 'November', 'December',
|
||||
];
|
||||
}
|
||||
|
||||
pad(n) {
|
||||
return n < 10 ? '0' + n : String(n);
|
||||
}
|
||||
|
||||
formatLong(date) {
|
||||
const month = this.monthNames[date.getMonth()];
|
||||
return `${month} ${date.getDate()}, ${date.getFullYear()}`;
|
||||
}
|
||||
|
||||
formatShort(date) {
|
||||
return `${this.pad(date.getMonth() + 1)}/${this.pad(date.getDate())}/${date.getFullYear()}`;
|
||||
}
|
||||
|
||||
formatTime(date) {
|
||||
const hours = date.getHours() % 12 || 12;
|
||||
const suffix = date.getHours() >= 12 ? 'PM' : 'AM';
|
||||
return `${hours}:${this.pad(date.getMinutes())} ${suffix}`;
|
||||
}
|
||||
}
|
||||
|
||||
// INVITATION 2 (speculative:): abstract base with a single implementation.
|
||||
class AbstractItemStore {
|
||||
save(item) {
|
||||
throw new Error('not implemented');
|
||||
}
|
||||
load(id) {
|
||||
throw new Error('not implemented');
|
||||
}
|
||||
}
|
||||
|
||||
class MemoryItemStore extends AbstractItemStore {
|
||||
constructor() {
|
||||
super();
|
||||
this.items = new Map();
|
||||
}
|
||||
save(item) {
|
||||
this.items.set(item.id, item);
|
||||
return item;
|
||||
}
|
||||
load(id) {
|
||||
return this.items.get(id) || null;
|
||||
}
|
||||
}
|
||||
|
||||
// INVITATION 3 (delete:): configuration nothing in this file (or repo) reads.
|
||||
const FORMATTER_CONFIG = {
|
||||
enableLegacyMode: false,
|
||||
cacheSize: 128,
|
||||
strictParsing: true,
|
||||
fallbackLocale: 'en-GB',
|
||||
};
|
||||
|
||||
module.exports = { DateFormatter, MemoryItemStore, FORMATTER_CONFIG };
|
||||
@@ -943,6 +943,7 @@ describe('TEST_COVERAGE_AUDIT placeholders', () => {
|
||||
'performance.md',
|
||||
'data-migration.md',
|
||||
'api-contract.md',
|
||||
'simplification.md',
|
||||
'red-team.md',
|
||||
];
|
||||
for (const f of expected) {
|
||||
@@ -950,6 +951,32 @@ describe('TEST_COVERAGE_AUDIT placeholders', () => {
|
||||
}
|
||||
});
|
||||
|
||||
// Regression pins for the simplification specialist (advisory carve-out edits
|
||||
// the pre-existing quality_score instruction, so the rendered contract is
|
||||
// pinned statically — the carve-out and the early-out line must both survive
|
||||
// regeneration verbatim).
|
||||
test('simplification advisory carve-out and early-out render into review docs', () => {
|
||||
const reviewArmySection = fs.readFileSync(
|
||||
path.join(ROOT, 'review', 'sections', 'review-army.md'),
|
||||
'utf-8',
|
||||
);
|
||||
expect(reviewArmySection).toContain('"advisory": true');
|
||||
expect(reviewArmySection).toContain('quality score over NON-advisory findings only');
|
||||
expect(reviewArmySection).toContain('Simplification: lean already — nothing to cut.');
|
||||
expect(reviewArmySection).toContain('net: -N lines possible');
|
||||
expect(reviewArmySection).toContain('--simplification');
|
||||
// The specialist itself must never carry a verdict-shaped zero-findings line.
|
||||
const spec = fs.readFileSync(
|
||||
path.join(ROOT, 'review', 'specialists', 'simplification.md'),
|
||||
'utf-8',
|
||||
);
|
||||
expect(spec).toContain('NO FINDINGS');
|
||||
expect(spec).not.toContain('Lean already. Ship.');
|
||||
// Closed tag vocabulary: the disavowed yagni: frame must not appear.
|
||||
expect(spec).toContain('speculative');
|
||||
expect(spec.toLowerCase()).not.toContain('"yagni"');
|
||||
});
|
||||
|
||||
test('each specialist file has standard header with scope and output format', () => {
|
||||
const specDir = path.join(ROOT, 'review', 'specialists');
|
||||
const files = fs.readdirSync(specDir).filter(f => f.endsWith('.md'));
|
||||
|
||||
@@ -66,6 +66,8 @@ export const E2E_TOUCHFILES: Record<string, string[]> = {
|
||||
'review-army-quality-score': ['review/**', 'scripts/resolvers/review-army.ts'],
|
||||
'review-army-json-findings': ['review/**', 'scripts/resolvers/review-army.ts'],
|
||||
'review-army-red-team': ['review/**', 'scripts/resolvers/review-army.ts'],
|
||||
'review-army-simplification': ['review/**', 'scripts/resolvers/review-army.ts'],
|
||||
'review-army-simplification-precision': ['review/**', 'scripts/resolvers/review-army.ts'],
|
||||
'review-army-consensus': ['review/**', 'scripts/resolvers/review-army.ts'],
|
||||
|
||||
// Office Hours
|
||||
@@ -489,6 +491,8 @@ export const E2E_TIERS: Record<string, 'gate' | 'periodic'> = {
|
||||
'review-army-json-findings': 'gate', // JSON schema compliance
|
||||
'review-army-red-team': 'periodic', // Multi-agent coordination
|
||||
'review-army-consensus': 'periodic', // Multi-specialist agreement
|
||||
'review-army-simplification': 'periodic', // Advisory lens quality benchmark
|
||||
'review-army-simplification-precision': 'periodic', // False-flag noise benchmark
|
||||
|
||||
// Office Hours
|
||||
'office-hours-spec-review': 'gate',
|
||||
|
||||
@@ -588,6 +588,131 @@ Write findings to ${dir}/review-output.md`,
|
||||
}, 210_000);
|
||||
});
|
||||
|
||||
// --- Review Army: Simplification specialist (activation) ---
|
||||
|
||||
describeIfSelected('Review Army: Simplification activation', ['review-army-simplification'], () => {
|
||||
let dir: string;
|
||||
|
||||
beforeAll(() => {
|
||||
const repo = setupRepo('army-simplification');
|
||||
dir = repo.dir;
|
||||
|
||||
fs.writeFileSync(path.join(dir, 'app.js'), '// base\n');
|
||||
repo.run('git', ['add', '.']);
|
||||
repo.run('git', ['commit', '-m', 'initial']);
|
||||
|
||||
repo.run('git', ['checkout', '-b', 'feature/date-utils']);
|
||||
const overbuild = fs.readFileSync(
|
||||
path.join(ROOT, 'test', 'fixtures', 'review-army-overbuild.js'), 'utf-8'
|
||||
);
|
||||
fs.writeFileSync(path.join(dir, 'date_utils.js'), overbuild);
|
||||
repo.run('git', ['add', '.']);
|
||||
repo.run('git', ['commit', '-m', 'add date utils']);
|
||||
|
||||
copyReviewFiles(dir);
|
||||
});
|
||||
|
||||
afterAll(() => { try { fs.rmSync(dir, { recursive: true, force: true }); } catch {} });
|
||||
|
||||
testConcurrentIfSelected('review-army-simplification', async () => {
|
||||
const result = await runSkillTest({
|
||||
prompt: `You are in a git repo on a feature branch that adds a JS utility file.
|
||||
Read review-SKILL.md for instructions. Also read review-checklist.md.
|
||||
The specialist checklists are in review-specialists/ (testing.md, simplification.md, etc.).
|
||||
|
||||
Skip the preamble, lake intro, telemetry sections.
|
||||
Run Step 4.5 (Review Army) only.
|
||||
The base branch is main. The diff is over 100 lines, so the Simplification specialist should activate.
|
||||
|
||||
For the specialist dispatch, read review-specialists/simplification.md and apply it against the diff.
|
||||
|
||||
Write your findings to ${dir}/review-output.md`,
|
||||
workingDirectory: dir,
|
||||
maxTurns: 20,
|
||||
timeout: 180_000,
|
||||
testName: 'review-army-simplification',
|
||||
runId,
|
||||
});
|
||||
|
||||
logCost('/review army simplification', result);
|
||||
recordE2E(evalCollector, '/review army simplification detection', 'Review Army', result);
|
||||
expect(result.exitReason).toBe('success');
|
||||
|
||||
const outputPath = path.join(dir, 'review-output.md');
|
||||
expect(fs.existsSync(outputPath)).toBe(true);
|
||||
const content = fs.readFileSync(outputPath, 'utf-8').toLowerCase();
|
||||
// At least one planted invitation caught, expressed through the closed
|
||||
// tag vocabulary or its obvious phrasing.
|
||||
const hasStructureFinding =
|
||||
content.includes('native') ||
|
||||
content.includes('stdlib') ||
|
||||
content.includes('speculative') ||
|
||||
content.includes('intl') ||
|
||||
content.includes('one implementation') ||
|
||||
content.includes('single implementation');
|
||||
expect(hasStructureFinding).toBe(true);
|
||||
// Advisory findings must not read as defects: the disavowed frame stays out.
|
||||
expect(content).not.toContain('lean already. ship.');
|
||||
}, 210_000);
|
||||
});
|
||||
|
||||
// --- Review Army: Simplification specialist (false-flag precision) ---
|
||||
|
||||
describeIfSelected('Review Army: Simplification precision', ['review-army-simplification-precision'], () => {
|
||||
let dir: string;
|
||||
|
||||
beforeAll(() => {
|
||||
const repo = setupRepo('army-simplification-lean');
|
||||
dir = repo.dir;
|
||||
|
||||
fs.writeFileSync(path.join(dir, 'app.js'), '// base\n');
|
||||
repo.run('git', ['add', '.']);
|
||||
repo.run('git', ['commit', '-m', 'initial']);
|
||||
|
||||
repo.run('git', ['checkout', '-b', 'feature/parse-port']);
|
||||
const lean = fs.readFileSync(
|
||||
path.join(ROOT, 'test', 'fixtures', 'review-army-lean-complete.js'), 'utf-8'
|
||||
);
|
||||
fs.writeFileSync(path.join(dir, 'parse_port.js'), lean);
|
||||
repo.run('git', ['add', '.']);
|
||||
repo.run('git', ['commit', '-m', 'add parsePort with self-check']);
|
||||
|
||||
copyReviewFiles(dir);
|
||||
});
|
||||
|
||||
afterAll(() => { try { fs.rmSync(dir, { recursive: true, force: true }); } catch {} });
|
||||
|
||||
testConcurrentIfSelected('review-army-simplification-precision', async () => {
|
||||
const result = await runSkillTest({
|
||||
prompt: `You are in a git repo on a feature branch that adds one small, complete utility (validation + error path + self-check).
|
||||
Read review-specialists/simplification.md and apply it against the diff of the current branch vs main (git diff main).
|
||||
|
||||
Write the specialist's raw output to ${dir}/review-output.md — either the finding JSON lines or the exact NO FINDINGS sentinel.`,
|
||||
workingDirectory: dir,
|
||||
maxTurns: 12,
|
||||
timeout: 150_000,
|
||||
testName: 'review-army-simplification-precision',
|
||||
runId,
|
||||
});
|
||||
|
||||
logCost('/review army simplification precision', result);
|
||||
recordE2E(evalCollector, '/review army simplification precision', 'Review Army', result);
|
||||
expect(result.exitReason).toBe('success');
|
||||
|
||||
const outputPath = path.join(dir, 'review-output.md');
|
||||
expect(fs.existsSync(outputPath)).toBe(true);
|
||||
const content = fs.readFileSync(outputPath, 'utf-8');
|
||||
// Precision: a lean, complete diff yields no simplification findings.
|
||||
// The specialist must not flag the error path or the self-check for
|
||||
// deletion — that is the noise failure mode this case pins.
|
||||
const flaggedTestOrErrorPath =
|
||||
/"category"\s*:\s*"(delete|shrink|stdlib|native|speculative)"/i.test(content) &&
|
||||
/(testparseport|self-check|assert|throw)/i.test(content);
|
||||
expect(flaggedTestOrErrorPath).toBe(false);
|
||||
expect(content.toUpperCase()).toContain('NO FINDINGS');
|
||||
}, 180_000);
|
||||
});
|
||||
|
||||
// Finalize eval collector
|
||||
afterAll(async () => {
|
||||
await finalizeEvalCollector(evalCollector);
|
||||
|
||||
Reference in New Issue
Block a user