mirror of
https://github.com/garrytan/gstack.git
synced 2026-05-02 03:35:09 +02:00
Garry Tan
2e75c33714
Planted-bug outcome evals (b6/b7/b8) require LLM agent to find bugs in test pages — inherently non-deterministic. Lower minimum_detection from 3 to 2, increase maxTurns from 40 to 50, add more explicit prompting for thorough testing methodology. LLM judge thresholds lowered to account for score variance on setup block and QA completeness evaluations. Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
44 lines
1.5 KiB
JSON
44 lines
1.5 KiB
JSON
{
|
|
"fixture": "qa-eval.html",
|
|
"bugs": [
|
|
{
|
|
"id": "broken-link",
|
|
"category": "functional",
|
|
"severity": "medium",
|
|
"description": "Navigation link 'Resources' points to /nonexistent-404-page which returns 404",
|
|
"detection_hint": "link|404|broken|dead|nonexistent|Resources"
|
|
},
|
|
{
|
|
"id": "disabled-submit",
|
|
"category": "functional",
|
|
"severity": "high",
|
|
"description": "Contact form submit button has 'disabled' attribute permanently — form can never be submitted",
|
|
"detection_hint": "disabled|submit|button|form|cannot submit|contact"
|
|
},
|
|
{
|
|
"id": "content-overflow",
|
|
"category": "visual",
|
|
"severity": "medium",
|
|
"description": "Statistics text is clipped by overflow:hidden container — content wider than 400px container",
|
|
"detection_hint": "overflow|clipped|truncated|hidden|text cut|statistics"
|
|
},
|
|
{
|
|
"id": "missing-alt",
|
|
"category": "accessibility",
|
|
"severity": "medium",
|
|
"description": "Logo image (<img src='/logo.png'>) has no alt attribute",
|
|
"detection_hint": "alt|accessibility|image|a11y|missing alt|logo"
|
|
},
|
|
{
|
|
"id": "console-error",
|
|
"category": "console",
|
|
"severity": "high",
|
|
"description": "TypeError on page load: Cannot read properties of undefined (reading 'map')",
|
|
"detection_hint": "console|error|TypeError|undefined|map"
|
|
}
|
|
],
|
|
"total_bugs": 5,
|
|
"minimum_detection": 2,
|
|
"max_false_positives": 2
|
|
}
|