feat: 3-tier eval suite with planted-bug outcome testing (EVALS=1)

Adds comprehensive eval infrastructure:
- Tier 1 (free): 13 new static tests — cross-skill path consistency, QA
  structure validation, greptile format, planted-bug fixture validation
- Tier 2 (Agent SDK E2E): /qa quick, /review with pre-built git repo,
  3 planted-bug outcome evals (static, SPA, checkout — each with 5 bugs)
- Tier 3 (LLM judge): QA workflow quality, health rubric clarity,
  cross-skill consistency, baseline score pinning

New fixtures: 3 HTML pages with 15 total planted bugs, ground truth JSON,
review-eval-vuln.rb, eval-baselines.json. Shared llm-judge.ts helper (DRY).

Unified EVALS=1 flag replaces SKILL_E2E + ANTHROPIC_API_KEY checks.
`bun run test:evals` runs everything that costs money (~$4/run).

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
Garry Tan
2026-03-14 01:17:36 -05:00
parent 5155fe3a28
commit 76803d789a
17 changed files with 1352 additions and 94 deletions
+7
View File
@@ -0,0 +1,7 @@
{
"command_reference": { "clarity": 4, "completeness": 4, "actionability": 4 },
"snapshot_flags": { "clarity": 4, "completeness": 4, "actionability": 4 },
"browse_skill": { "clarity": 4, "completeness": 4, "actionability": 4 },
"qa_workflow": { "clarity": 4, "completeness": 4, "actionability": 4 },
"qa_health_rubric": { "clarity": 4, "completeness": 4, "actionability": 4 }
}
+43
View File
@@ -0,0 +1,43 @@
{
"fixture": "qa-eval-checkout.html",
"bugs": [
{
"id": "broken-email-regex",
"category": "functional",
"severity": "high",
"description": "Email validation accepts 'user@' as valid — regex pattern [^@]+@[^@] is missing domain requirement",
"detection_hint": "email|regex|validation|accepts|invalid|user@|pattern"
},
{
"id": "nan-total",
"category": "functional",
"severity": "high",
"description": "Clearing the quantity field shows 'Total: $NaN' — parseInt on empty string returns NaN with no fallback",
"detection_hint": "NaN|total|quantity|empty|price|calculation|clear"
},
{
"id": "cc-field-overflow",
"category": "visual",
"severity": "medium",
"description": "Credit card input has no maxlength attribute — entering >20 characters causes text to overflow the container",
"detection_hint": "credit card|maxlength|overflow|cc|input|long|container"
},
{
"id": "missing-required-zip",
"category": "functional",
"severity": "medium",
"description": "Zip code field has no 'required' attribute — form can be submitted without a zip code",
"detection_hint": "zip|required|missing|form|submit|shipping|postal"
},
{
"id": "stripe-not-defined",
"category": "console",
"severity": "high",
"description": "Form submit triggers 'Uncaught ReferenceError: stripe is not defined' — payment SDK not loaded",
"detection_hint": "stripe|ReferenceError|not defined|console|error|submit|payment"
}
],
"total_bugs": 5,
"minimum_detection": 3,
"max_false_positives": 2
}
+43
View File
@@ -0,0 +1,43 @@
{
"fixture": "qa-eval.html",
"bugs": [
{
"id": "broken-link",
"category": "functional",
"severity": "medium",
"description": "Navigation link 'Resources' points to /nonexistent-404-page which returns 404",
"detection_hint": "link|404|broken|dead|nonexistent|Resources"
},
{
"id": "disabled-submit",
"category": "functional",
"severity": "high",
"description": "Contact form submit button has 'disabled' attribute permanently — form can never be submitted",
"detection_hint": "disabled|submit|button|form|cannot submit|contact"
},
{
"id": "content-overflow",
"category": "visual",
"severity": "medium",
"description": "Statistics text is clipped by overflow:hidden container — content wider than 400px container",
"detection_hint": "overflow|clipped|truncated|hidden|text cut|statistics"
},
{
"id": "missing-alt",
"category": "accessibility",
"severity": "medium",
"description": "Logo image (<img src='/logo.png'>) has no alt attribute",
"detection_hint": "alt|accessibility|image|a11y|missing alt|logo"
},
{
"id": "console-error",
"category": "console",
"severity": "high",
"description": "TypeError on page load: Cannot read properties of undefined (reading 'map')",
"detection_hint": "console|error|TypeError|undefined|map"
}
],
"total_bugs": 5,
"minimum_detection": 3,
"max_false_positives": 2
}
+43
View File
@@ -0,0 +1,43 @@
{
"fixture": "qa-eval-spa.html",
"bugs": [
{
"id": "broken-route",
"category": "functional",
"severity": "high",
"description": "Products nav link points to #/prodcts (typo) instead of #/products — shows 'Page not found'",
"detection_hint": "route|prodcts|typo|products|not found|broken link|navigation"
},
{
"id": "stale-cart-state",
"category": "functional",
"severity": "medium",
"description": "Cart count persists across route changes — never resets when navigating away from products",
"detection_hint": "cart|count|state|persist|reset|stale|navigation"
},
{
"id": "async-fetch-error",
"category": "functional",
"severity": "high",
"description": "Product list briefly loads then shows 'Error: Failed to fetch products from API' after 1 second",
"detection_hint": "error|fetch|products|API|loading|failed|async"
},
{
"id": "missing-aria-current",
"category": "accessibility",
"severity": "medium",
"description": "Navigation links have no aria-current attribute to indicate the active route",
"detection_hint": "aria|current|active|navigation|accessibility|a11y"
},
{
"id": "console-warn-leak",
"category": "console",
"severity": "medium",
"description": "console.warn fires on every route change: 'Possible memory leak detected: 11 event listeners'",
"detection_hint": "console|warn|memory leak|listener|event|warning"
}
],
"total_bugs": 5,
"minimum_detection": 3,
"max_false_positives": 2
}
+14
View File
@@ -0,0 +1,14 @@
class UserController < ApplicationController
def show
# SQL injection — interpolating user input directly into query
@user = User.where("id = #{params[:id]}").first
render json: @user
end
def promote
# Bypasses ActiveRecord validations — update_column skips callbacks + validation
@user = User.find(params[:id])
@user.update_column(:role, 'admin')
head :ok
end
end