feat: 3-tier eval suite with planted-bug outcome testing (EVALS=1)

Adds comprehensive eval infrastructure:
- Tier 1 (free): 13 new static tests — cross-skill path consistency, QA
  structure validation, greptile format, planted-bug fixture validation
- Tier 2 (Agent SDK E2E): /qa quick, /review with pre-built git repo,
  3 planted-bug outcome evals (static, SPA, checkout — each with 5 bugs)
- Tier 3 (LLM judge): QA workflow quality, health rubric clarity,
  cross-skill consistency, baseline score pinning

New fixtures: 3 HTML pages with 15 total planted bugs, ground truth JSON,
review-eval-vuln.rb, eval-baselines.json. Shared llm-judge.ts helper (DRY).

Unified EVALS=1 flag replaces SKILL_E2E + ANTHROPIC_API_KEY checks.
`bun run test:evals` runs everything that costs money (~$4/run).

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
Garry Tan
2026-03-14 01:17:36 -05:00
parent 5155fe3a28
commit 76803d789a
17 changed files with 1352 additions and 94 deletions
+73
View File
@@ -13,6 +13,7 @@
import { ALL_COMMANDS } from '../../browse/src/commands';
import { parseSnapshotArgs } from '../../browse/src/snapshot';
import * as fs from 'fs';
import * as path from 'path';
export interface BrowseCommand {
command: string;
@@ -131,3 +132,75 @@ export function validateSkill(skillPath: string): ValidationResult {
return result;
}
/**
* Extract all REMOTE_SLUG=$(...) assignment patterns from .md files in given subdirectories.
* Returns a Map from filename → array of full assignment lines found.
*/
export function extractRemoteSlugPatterns(rootDir: string, subdirs: string[]): Map<string, string[]> {
const results = new Map<string, string[]>();
const pattern = /^REMOTE_SLUG=\$\(.*\)$/;
for (const subdir of subdirs) {
const dir = path.join(rootDir, subdir);
if (!fs.existsSync(dir)) continue;
const files = fs.readdirSync(dir).filter(f => f.endsWith('.md'));
for (const file of files) {
const filePath = path.join(dir, file);
const content = fs.readFileSync(filePath, 'utf-8');
const matches: string[] = [];
for (const line of content.split('\n')) {
const trimmed = line.trim();
if (pattern.test(trimmed)) {
matches.push(trimmed);
}
}
if (matches.length > 0) {
results.set(`${subdir}/${file}`, matches);
}
}
}
return results;
}
/**
* Parse a markdown weight table anchored to a "### Weights" heading.
* Expects rows like: | Category | 15% |
* Returns Map<category, number> where number is the percentage (e.g., 15).
*/
export function extractWeightsFromTable(content: string): Map<string, number> {
const weights = new Map<string, number>();
// Find the ### Weights section
const weightsIdx = content.indexOf('### Weights');
if (weightsIdx === -1) return weights;
// Find the table within that section (stop at next heading or end)
const section = content.slice(weightsIdx);
const lines = section.split('\n');
for (let i = 1; i < lines.length; i++) {
const line = lines[i].trim();
// Stop at next heading
if (line.startsWith('#') && !line.startsWith('###')) break;
if (line.startsWith('### ') && i > 0) break;
// Parse table rows: | Category | N% |
const match = line.match(/^\|\s*(\w[\w\s]*\w|\w+)\s*\|\s*(\d+)%\s*\|$/);
if (match) {
const category = match[1].trim();
const pct = parseInt(match[2], 10);
// Skip header row
if (category !== 'Category' && !isNaN(pct)) {
weights.set(category, pct);
}
}
}
return weights;
}