mirror of
https://github.com/garrytan/gstack.git
synced 2026-07-08 09:07:55 +02:00
feat: 3-tier eval suite with planted-bug outcome testing (EVALS=1)
Adds comprehensive eval infrastructure: - Tier 1 (free): 13 new static tests — cross-skill path consistency, QA structure validation, greptile format, planted-bug fixture validation - Tier 2 (Agent SDK E2E): /qa quick, /review with pre-built git repo, 3 planted-bug outcome evals (static, SPA, checkout — each with 5 bugs) - Tier 3 (LLM judge): QA workflow quality, health rubric clarity, cross-skill consistency, baseline score pinning New fixtures: 3 HTML pages with 15 total planted bugs, ground truth JSON, review-eval-vuln.rb, eval-baselines.json. Shared llm-judge.ts helper (DRY). Unified EVALS=1 flag replaces SKILL_E2E + ANTHROPIC_API_KEY checks. `bun run test:evals` runs everything that costs money (~$4/run). Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
@@ -13,6 +13,7 @@
|
||||
import { ALL_COMMANDS } from '../../browse/src/commands';
|
||||
import { parseSnapshotArgs } from '../../browse/src/snapshot';
|
||||
import * as fs from 'fs';
|
||||
import * as path from 'path';
|
||||
|
||||
export interface BrowseCommand {
|
||||
command: string;
|
||||
@@ -131,3 +132,75 @@ export function validateSkill(skillPath: string): ValidationResult {
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
/**
|
||||
* Extract all REMOTE_SLUG=$(...) assignment patterns from .md files in given subdirectories.
|
||||
* Returns a Map from filename → array of full assignment lines found.
|
||||
*/
|
||||
export function extractRemoteSlugPatterns(rootDir: string, subdirs: string[]): Map<string, string[]> {
|
||||
const results = new Map<string, string[]>();
|
||||
const pattern = /^REMOTE_SLUG=\$\(.*\)$/;
|
||||
|
||||
for (const subdir of subdirs) {
|
||||
const dir = path.join(rootDir, subdir);
|
||||
if (!fs.existsSync(dir)) continue;
|
||||
|
||||
const files = fs.readdirSync(dir).filter(f => f.endsWith('.md'));
|
||||
for (const file of files) {
|
||||
const filePath = path.join(dir, file);
|
||||
const content = fs.readFileSync(filePath, 'utf-8');
|
||||
const matches: string[] = [];
|
||||
|
||||
for (const line of content.split('\n')) {
|
||||
const trimmed = line.trim();
|
||||
if (pattern.test(trimmed)) {
|
||||
matches.push(trimmed);
|
||||
}
|
||||
}
|
||||
|
||||
if (matches.length > 0) {
|
||||
results.set(`${subdir}/${file}`, matches);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return results;
|
||||
}
|
||||
|
||||
/**
|
||||
* Parse a markdown weight table anchored to a "### Weights" heading.
|
||||
* Expects rows like: | Category | 15% |
|
||||
* Returns Map<category, number> where number is the percentage (e.g., 15).
|
||||
*/
|
||||
export function extractWeightsFromTable(content: string): Map<string, number> {
|
||||
const weights = new Map<string, number>();
|
||||
|
||||
// Find the ### Weights section
|
||||
const weightsIdx = content.indexOf('### Weights');
|
||||
if (weightsIdx === -1) return weights;
|
||||
|
||||
// Find the table within that section (stop at next heading or end)
|
||||
const section = content.slice(weightsIdx);
|
||||
const lines = section.split('\n');
|
||||
|
||||
for (let i = 1; i < lines.length; i++) {
|
||||
const line = lines[i].trim();
|
||||
|
||||
// Stop at next heading
|
||||
if (line.startsWith('#') && !line.startsWith('###')) break;
|
||||
if (line.startsWith('### ') && i > 0) break;
|
||||
|
||||
// Parse table rows: | Category | N% |
|
||||
const match = line.match(/^\|\s*(\w[\w\s]*\w|\w+)\s*\|\s*(\d+)%\s*\|$/);
|
||||
if (match) {
|
||||
const category = match[1].trim();
|
||||
const pct = parseInt(match[2], 10);
|
||||
// Skip header row
|
||||
if (category !== 'Category' && !isNaN(pct)) {
|
||||
weights.set(category, pct);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return weights;
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user