feat: 3-tier eval suite with planted-bug outcome testing (EVALS=1)

Adds comprehensive eval infrastructure: - Tier 1 (free): 13 new static tests — cross-skill path consistency, QA structure validation, greptile format, planted-bug fixture validation - Tier 2 (Agent SDK E2E): /qa quick, /review with pre-built git repo, 3 planted-bug outcome evals (static, SPA, checkout — each with 5 bugs) - Tier 3 (LLM judge): QA workflow quality, health rubric clarity, cross-skill consistency, baseline score pinning New fixtures: 3 HTML pages with 15 total planted bugs, ground truth JSON, review-eval-vuln.rb, eval-baselines.json. Shared llm-judge.ts helper (DRY). Unified EVALS=1 flag replaces SKILL_E2E + ANTHROPIC_API_KEY checks. `bun run test:evals` runs everything that costs money (~$4/run). Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
2026-08-05 05:48:36 +02:00 · 2026-03-14 01:17:36 -05:00
parent 5155fe3a28
commit 76803d789a
17 changed files with 1350 additions and 92 deletions
@@ -0,0 +1,108 @@
+<!DOCTYPE html>
+<html lang="en">
+<head>
+  <meta charset="utf-8">
+  <title>QA Eval — Checkout</title>
+  <style>
+    body { font-family: sans-serif; padding: 20px; }
+    .checkout-form { max-width: 500px; }
+    .form-group { margin-bottom: 15px; }
+    .form-group label { display: block; margin-bottom: 4px; font-weight: bold; }
+    .form-group input { width: 100%; padding: 8px; box-sizing: border-box; border: 1px solid #ccc; border-radius: 4px; }
+    .form-group input.invalid { border-color: red; }
+    .form-group .error-msg { color: red; font-size: 12px; display: none; }
+    .total { font-size: 24px; font-weight: bold; margin: 20px 0; }
+    button[type="submit"] { padding: 12px 24px; background: #0066cc; color: white; border: none; border-radius: 4px; cursor: pointer; font-size: 16px; }
+    .order-summary { background: #f5f5f5; padding: 15px; border-radius: 4px; margin-bottom: 20px; }
+  </style>
+</head>
+<body>
+  <h1>Checkout</h1>
+
+  <div class="order-summary">
+    <h2>Order Summary</h2>
+    <p>Widget Pro — $99.99 x <input type="number" id="quantity" value="1" min="1" style="width: 50px;"></p>
+    <p class="total" id="total">Total: $99.99</p>  <!-- BUG 2: shows $NaN when quantity is cleared -->
+  </div>
+
+  <form class="checkout-form" id="checkout-form">
+    <h2>Shipping Information</h2>
+
+    <div class="form-group">
+      <label for="email">Email</label>
+      <input type="text" id="email" name="email" placeholder="you@example.com" required
+             pattern="[^@]+@[^@]">  <!-- BUG 1: broken regex — accepts "user@" as valid -->
+      <span class="error-msg" id="email-error">Please enter a valid email</span>
+    </div>
+
+    <div class="form-group">
+      <label for="address">Address</label>
+      <input type="text" id="address" name="address" placeholder="123 Main St" required>
+    </div>
+
+    <div class="form-group">
+      <label for="city">City</label>
+      <input type="text" id="city" name="city" placeholder="San Francisco" required>
+    </div>
+
+    <div class="form-group">
+      <label for="zip">Zip Code</label>
+      <input type="text" id="zip" name="zip" placeholder="94105">  <!-- BUG 4: missing required attribute -->
+    </div>
+
+    <h2>Payment</h2>
+
+    <div class="form-group">
+      <label for="cc">Credit Card Number</label>
+      <input type="text" id="cc" name="cc" placeholder="4111 1111 1111 1111" required>
+      <!-- BUG 3: no maxlength — overflows container at >20 chars -->
+    </div>
+
+    <div class="form-group">
+      <label for="exp">Expiration</label>
+      <input type="text" id="exp" name="exp" placeholder="MM/YY" required maxlength="5">
+    </div>
+
+    <div class="form-group">
+      <label for="cvv">CVV</label>
+      <input type="text" id="cvv" name="cvv" placeholder="123" required maxlength="4">
+    </div>
+
+    <button type="submit">Place Order — $<span id="submit-total">99.99</span></button>
+  </form>
+
+  <script>
+    // Update total when quantity changes
+    const quantityInput = document.getElementById('quantity');
+    const totalEl = document.getElementById('total');
+    const submitTotalEl = document.getElementById('submit-total');
+
+    quantityInput.addEventListener('input', () => {
+      // BUG 2: parseInt on empty string returns NaN, no fallback
+      const qty = parseInt(quantityInput.value);
+      const total = (qty * 99.99).toFixed(2);
+      totalEl.textContent = 'Total: $' + total;
+      submitTotalEl.textContent = total;
+    });
+
+    // Email validation (broken)
+    const emailInput = document.getElementById('email');
+    emailInput.addEventListener('blur', () => {
+      // BUG 1: this regex accepts "user@" — missing domain part check
+      const valid = /[^@]+@/.test(emailInput.value);
+      emailInput.classList.toggle('invalid', !valid && emailInput.value.length > 0);
+      document.getElementById('email-error').style.display = (!valid && emailInput.value.length > 0) ? 'block' : 'none';
+    });
+
+    // Form submit
+    document.getElementById('checkout-form').addEventListener('submit', (e) => {
+      e.preventDefault();
+      // BUG 5: stripe is not defined — console error on submit
+      stripe.createPaymentMethod({
+        type: 'card',
+        card: { number: document.getElementById('cc').value }
+      });
+    });
+  </script>
+</body>
+</html>
@@ -0,0 +1,98 @@
+<!DOCTYPE html>
+<html lang="en">
+<head>
+  <meta charset="utf-8">
+  <title>QA Eval — SPA Store</title>
+  <style>
+    body { font-family: sans-serif; padding: 20px; margin: 0; }
+    nav { background: #333; padding: 10px 20px; }
+    nav a { color: white; margin-right: 15px; text-decoration: none; cursor: pointer; }
+    nav a:hover { text-decoration: underline; }
+    #app { padding: 20px; }
+    .product { border: 1px solid #ddd; padding: 10px; margin: 10px 0; border-radius: 4px; }
+    .product button { padding: 6px 12px; background: #0066cc; color: white; border: none; cursor: pointer; }
+    .cart-count { background: #cc0000; color: white; padding: 2px 8px; border-radius: 10px; font-size: 12px; }
+    .error { color: red; padding: 10px; }
+    .loading { color: #666; padding: 10px; }
+  </style>
+</head>
+<body>
+  <nav>
+    <a href="#/home">Home</a>
+    <a href="#/prodcts">Products</a>  <!-- BUG 1: broken route — typo "prodcts" instead of "products" -->
+    <a href="#/contact">Contact</a>
+    <span class="cart-count" id="cart-count">0</span>
+  </nav>
+
+  <div id="app">
+    <p>Welcome to SPA Store. Use the navigation above.</p>
+  </div>
+
+  <script>
+    let cartCount = 0;
+
+    // BUG 2: cart count never resets on route change — stale state
+    function addToCart() {
+      cartCount++;
+      document.getElementById('cart-count').textContent = cartCount;
+    }
+
+    function renderHome() {
+      document.getElementById('app').innerHTML = `
+        <h1>Welcome to SPA Store</h1>
+        <p>Browse our products using the navigation above.</p>
+      `;
+    }
+
+    function renderProducts() {
+      document.getElementById('app').innerHTML = '<p class="loading">Loading products...</p>';
+
+      // BUG 3: async race — shows data briefly, then shows error
+      setTimeout(() => {
+        document.getElementById('app').innerHTML = `
+          <h1>Products</h1>
+          <div class="product">
+            <h3>Widget A</h3>
+            <p>$29.99</p>
+            <button onclick="addToCart()">Add to Cart</button>
+          </div>
+          <div class="product">
+            <h3>Widget B</h3>
+            <p>$49.99</p>
+            <button onclick="addToCart()">Add to Cart</button>
+          </div>
+        `;
+      }, 300);
+
+      setTimeout(() => {
+        document.getElementById('app').innerHTML = '<p class="error">Error: Failed to fetch products from API</p>';
+      }, 1000);
+    }
+
+    function renderContact() {
+      document.getElementById('app').innerHTML = `
+        <h1>Contact Us</h1>
+        <p>Email: support@spastore.example.com</p>
+      `;
+    }
+
+    // BUG 4: nav links have no aria-current attribute on active route
+    function router() {
+      const hash = window.location.hash || '#/home';
+      switch (hash) {
+        case '#/home': renderHome(); break;
+        case '#/products': renderProducts(); break;
+        case '#/contact': renderContact(); break;
+        default:
+          document.getElementById('app').innerHTML = '<p>Page not found</p>';
+      }
+
+      // BUG 5: console.warn on every route change — simulates listener leak
+      console.warn('Possible memory leak detected: 11 event listeners added to window');
+    }
+
+    window.addEventListener('hashchange', router);
+    router();
+  </script>
+</body>
+</html>
@@ -0,0 +1,51 @@
+<!DOCTYPE html>
+<html lang="en">
+<head>
+  <meta charset="utf-8">
+  <title>QA Eval — Widget Dashboard</title>
+  <style>
+    body { font-family: sans-serif; padding: 20px; }
+    nav { margin-bottom: 20px; }
+    nav a { margin-right: 15px; color: #0066cc; }
+    form { margin: 20px 0; padding: 15px; border: 1px solid #ccc; border-radius: 4px; }
+    input { display: block; margin: 8px 0; padding: 6px; }
+    button { padding: 8px 16px; margin-top: 8px; }
+    .stats { margin: 20px 0; }
+    img { display: block; margin: 20px 0; }
+  </style>
+</head>
+<body>
+  <nav>
+    <a href="/">Home</a>
+    <a href="/about">About</a>
+    <a href="/nonexistent-404-page">Resources</a>  <!-- BUG 1: broken link (404) -->
+  </nav>
+
+  <h1>Widget Dashboard</h1>
+
+  <form id="contact">
+    <h2>Contact Us</h2>
+    <input type="text" name="name" placeholder="Name" required>
+    <input type="email" name="email" placeholder="Email" required>
+    <button type="submit" disabled>Submit</button>  <!-- BUG 2: submit button permanently disabled -->
+  </form>
+
+  <div class="stats" style="width: 400px; overflow: hidden;">
+    <h2>Statistics</h2>
+    <p style="white-space: nowrap; width: 600px;">
+      Revenue: $1,234,567.89 | Users: 45,678 | Conversion: 3.2% | Growth: +12.5% MoM | Retention: 87.3%
+    </p>  <!-- BUG 3: content overflow/clipping — text wider than container with overflow:hidden -->
+  </div>
+
+  <img src="/logo.png">  <!-- BUG 4: missing alt text on image -->
+
+  <footer>
+    <p>&copy; 2026 Widget Co. All rights reserved.</p>
+  </footer>
+
+  <script>
+    console.error("TypeError: Cannot read properties of undefined (reading 'map')");
+    // BUG 5: console error on page load
+  </script>
+</body>
+</html>