mirror of
https://github.com/garrytan/gstack.git
synced 2026-08-06 22:38:38 +02:00
feat: 3-tier eval suite with planted-bug outcome testing (EVALS=1)
Adds comprehensive eval infrastructure: - Tier 1 (free): 13 new static tests — cross-skill path consistency, QA structure validation, greptile format, planted-bug fixture validation - Tier 2 (Agent SDK E2E): /qa quick, /review with pre-built git repo, 3 planted-bug outcome evals (static, SPA, checkout — each with 5 bugs) - Tier 3 (LLM judge): QA workflow quality, health rubric clarity, cross-skill consistency, baseline score pinning New fixtures: 3 HTML pages with 15 total planted bugs, ground truth JSON, review-eval-vuln.rb, eval-baselines.json. Shared llm-judge.ts helper (DRY). Unified EVALS=1 flag replaces SKILL_E2E + ANTHROPIC_API_KEY checks. `bun run test:evals` runs everything that costs money (~$4/run). Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 4.6
parent
5155fe3a28
commit
76803d789a
@@ -9,12 +9,21 @@ import { query } from '@anthropic-ai/claude-agent-sdk';
|
||||
import * as fs from 'fs';
|
||||
import * as path from 'path';
|
||||
|
||||
export interface CostEstimate {
|
||||
inputChars: number;
|
||||
outputChars: number;
|
||||
estimatedTokens: number;
|
||||
estimatedCost: number; // USD (approximate)
|
||||
turnsUsed: number;
|
||||
}
|
||||
|
||||
export interface SkillTestResult {
|
||||
messages: any[];
|
||||
toolCalls: Array<{ tool: string; input: any; output: string }>;
|
||||
browseErrors: string[];
|
||||
exitReason: string;
|
||||
duration: number;
|
||||
costEstimate: CostEstimate;
|
||||
}
|
||||
|
||||
const BROWSE_ERROR_PATTERNS = [
|
||||
@@ -36,7 +45,7 @@ export async function runSkillTest(options: {
|
||||
if (process.env.CLAUDECODE || process.env.CLAUDE_CODE_ENTRYPOINT) {
|
||||
throw new Error(
|
||||
'Cannot run E2E skill tests inside a Claude Code session. ' +
|
||||
'Run from a plain terminal: SKILL_E2E=1 bun test test/skill-e2e.test.ts'
|
||||
'Run from a plain terminal: EVALS=1 bun test test/skill-e2e.test.ts'
|
||||
);
|
||||
}
|
||||
|
||||
@@ -156,5 +165,39 @@ export async function runSkillTest(options: {
|
||||
}
|
||||
}
|
||||
|
||||
return { messages, toolCalls, browseErrors, exitReason, duration };
|
||||
// Estimate cost from message sizes (chars / 4 ≈ tokens, approximate)
|
||||
let inputChars = 0;
|
||||
let outputChars = 0;
|
||||
let turnsUsed = 0;
|
||||
|
||||
for (const msg of messages) {
|
||||
const content = msg.message?.content;
|
||||
if (!content) continue;
|
||||
const text = typeof content === 'string'
|
||||
? content
|
||||
: JSON.stringify(content);
|
||||
|
||||
if (msg.type === 'user') {
|
||||
inputChars += text.length;
|
||||
} else if (msg.type === 'assistant') {
|
||||
outputChars += text.length;
|
||||
turnsUsed++;
|
||||
}
|
||||
}
|
||||
|
||||
const estimatedTokens = Math.round((inputChars + outputChars) / 4);
|
||||
// Approximate pricing: sonnet input ~$3/M, output ~$15/M tokens
|
||||
const inputTokens = Math.round(inputChars / 4);
|
||||
const outputTokens = Math.round(outputChars / 4);
|
||||
const estimatedCost = (inputTokens * 3 + outputTokens * 15) / 1_000_000;
|
||||
|
||||
const costEstimate: CostEstimate = {
|
||||
inputChars,
|
||||
outputChars,
|
||||
estimatedTokens,
|
||||
estimatedCost: Math.round(estimatedCost * 100) / 100,
|
||||
turnsUsed,
|
||||
};
|
||||
|
||||
return { messages, toolCalls, browseErrors, exitReason, duration, costEstimate };
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user