mirror of
https://github.com/garrytan/gstack.git
synced 2026-07-09 09:38:09 +02:00
feat: 3-tier eval suite with planted-bug outcome testing (EVALS=1)
Adds comprehensive eval infrastructure: - Tier 1 (free): 13 new static tests — cross-skill path consistency, QA structure validation, greptile format, planted-bug fixture validation - Tier 2 (Agent SDK E2E): /qa quick, /review with pre-built git repo, 3 planted-bug outcome evals (static, SPA, checkout — each with 5 bugs) - Tier 3 (LLM judge): QA workflow quality, health rubric clarity, cross-skill consistency, baseline score pinning New fixtures: 3 HTML pages with 15 total planted bugs, ground truth JSON, review-eval-vuln.rb, eval-baselines.json. Shared llm-judge.ts helper (DRY). Unified EVALS=1 flag replaces SKILL_E2E + ANTHROPIC_API_KEY checks. `bun run test:evals` runs everything that costs money (~$4/run). Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
Vendored
+7
@@ -0,0 +1,7 @@
|
||||
{
|
||||
"command_reference": { "clarity": 4, "completeness": 4, "actionability": 4 },
|
||||
"snapshot_flags": { "clarity": 4, "completeness": 4, "actionability": 4 },
|
||||
"browse_skill": { "clarity": 4, "completeness": 4, "actionability": 4 },
|
||||
"qa_workflow": { "clarity": 4, "completeness": 4, "actionability": 4 },
|
||||
"qa_health_rubric": { "clarity": 4, "completeness": 4, "actionability": 4 }
|
||||
}
|
||||
@@ -0,0 +1,43 @@
|
||||
{
|
||||
"fixture": "qa-eval-checkout.html",
|
||||
"bugs": [
|
||||
{
|
||||
"id": "broken-email-regex",
|
||||
"category": "functional",
|
||||
"severity": "high",
|
||||
"description": "Email validation accepts 'user@' as valid — regex pattern [^@]+@[^@] is missing domain requirement",
|
||||
"detection_hint": "email|regex|validation|accepts|invalid|user@|pattern"
|
||||
},
|
||||
{
|
||||
"id": "nan-total",
|
||||
"category": "functional",
|
||||
"severity": "high",
|
||||
"description": "Clearing the quantity field shows 'Total: $NaN' — parseInt on empty string returns NaN with no fallback",
|
||||
"detection_hint": "NaN|total|quantity|empty|price|calculation|clear"
|
||||
},
|
||||
{
|
||||
"id": "cc-field-overflow",
|
||||
"category": "visual",
|
||||
"severity": "medium",
|
||||
"description": "Credit card input has no maxlength attribute — entering >20 characters causes text to overflow the container",
|
||||
"detection_hint": "credit card|maxlength|overflow|cc|input|long|container"
|
||||
},
|
||||
{
|
||||
"id": "missing-required-zip",
|
||||
"category": "functional",
|
||||
"severity": "medium",
|
||||
"description": "Zip code field has no 'required' attribute — form can be submitted without a zip code",
|
||||
"detection_hint": "zip|required|missing|form|submit|shipping|postal"
|
||||
},
|
||||
{
|
||||
"id": "stripe-not-defined",
|
||||
"category": "console",
|
||||
"severity": "high",
|
||||
"description": "Form submit triggers 'Uncaught ReferenceError: stripe is not defined' — payment SDK not loaded",
|
||||
"detection_hint": "stripe|ReferenceError|not defined|console|error|submit|payment"
|
||||
}
|
||||
],
|
||||
"total_bugs": 5,
|
||||
"minimum_detection": 3,
|
||||
"max_false_positives": 2
|
||||
}
|
||||
+43
@@ -0,0 +1,43 @@
|
||||
{
|
||||
"fixture": "qa-eval.html",
|
||||
"bugs": [
|
||||
{
|
||||
"id": "broken-link",
|
||||
"category": "functional",
|
||||
"severity": "medium",
|
||||
"description": "Navigation link 'Resources' points to /nonexistent-404-page which returns 404",
|
||||
"detection_hint": "link|404|broken|dead|nonexistent|Resources"
|
||||
},
|
||||
{
|
||||
"id": "disabled-submit",
|
||||
"category": "functional",
|
||||
"severity": "high",
|
||||
"description": "Contact form submit button has 'disabled' attribute permanently — form can never be submitted",
|
||||
"detection_hint": "disabled|submit|button|form|cannot submit|contact"
|
||||
},
|
||||
{
|
||||
"id": "content-overflow",
|
||||
"category": "visual",
|
||||
"severity": "medium",
|
||||
"description": "Statistics text is clipped by overflow:hidden container — content wider than 400px container",
|
||||
"detection_hint": "overflow|clipped|truncated|hidden|text cut|statistics"
|
||||
},
|
||||
{
|
||||
"id": "missing-alt",
|
||||
"category": "accessibility",
|
||||
"severity": "medium",
|
||||
"description": "Logo image (<img src='/logo.png'>) has no alt attribute",
|
||||
"detection_hint": "alt|accessibility|image|a11y|missing alt|logo"
|
||||
},
|
||||
{
|
||||
"id": "console-error",
|
||||
"category": "console",
|
||||
"severity": "high",
|
||||
"description": "TypeError on page load: Cannot read properties of undefined (reading 'map')",
|
||||
"detection_hint": "console|error|TypeError|undefined|map"
|
||||
}
|
||||
],
|
||||
"total_bugs": 5,
|
||||
"minimum_detection": 3,
|
||||
"max_false_positives": 2
|
||||
}
|
||||
+43
@@ -0,0 +1,43 @@
|
||||
{
|
||||
"fixture": "qa-eval-spa.html",
|
||||
"bugs": [
|
||||
{
|
||||
"id": "broken-route",
|
||||
"category": "functional",
|
||||
"severity": "high",
|
||||
"description": "Products nav link points to #/prodcts (typo) instead of #/products — shows 'Page not found'",
|
||||
"detection_hint": "route|prodcts|typo|products|not found|broken link|navigation"
|
||||
},
|
||||
{
|
||||
"id": "stale-cart-state",
|
||||
"category": "functional",
|
||||
"severity": "medium",
|
||||
"description": "Cart count persists across route changes — never resets when navigating away from products",
|
||||
"detection_hint": "cart|count|state|persist|reset|stale|navigation"
|
||||
},
|
||||
{
|
||||
"id": "async-fetch-error",
|
||||
"category": "functional",
|
||||
"severity": "high",
|
||||
"description": "Product list briefly loads then shows 'Error: Failed to fetch products from API' after 1 second",
|
||||
"detection_hint": "error|fetch|products|API|loading|failed|async"
|
||||
},
|
||||
{
|
||||
"id": "missing-aria-current",
|
||||
"category": "accessibility",
|
||||
"severity": "medium",
|
||||
"description": "Navigation links have no aria-current attribute to indicate the active route",
|
||||
"detection_hint": "aria|current|active|navigation|accessibility|a11y"
|
||||
},
|
||||
{
|
||||
"id": "console-warn-leak",
|
||||
"category": "console",
|
||||
"severity": "medium",
|
||||
"description": "console.warn fires on every route change: 'Possible memory leak detected: 11 event listeners'",
|
||||
"detection_hint": "console|warn|memory leak|listener|event|warning"
|
||||
}
|
||||
],
|
||||
"total_bugs": 5,
|
||||
"minimum_detection": 3,
|
||||
"max_false_positives": 2
|
||||
}
|
||||
Vendored
+14
@@ -0,0 +1,14 @@
|
||||
class UserController < ApplicationController
|
||||
def show
|
||||
# SQL injection — interpolating user input directly into query
|
||||
@user = User.where("id = #{params[:id]}").first
|
||||
render json: @user
|
||||
end
|
||||
|
||||
def promote
|
||||
# Bypasses ActiveRecord validations — update_column skips callbacks + validation
|
||||
@user = User.find(params[:id])
|
||||
@user.update_column(:role, 'admin')
|
||||
head :ok
|
||||
end
|
||||
end
|
||||
Reference in New Issue
Block a user