diff --git a/test/fixtures/judge-calibration/qa-health-rubric/runs/comparison-1.result.json b/test/fixtures/judge-calibration/qa-health-rubric/runs/comparison-1.result.json
new file mode 100644
index 000000000..f1d20a0bd
--- /dev/null
+++ b/test/fixtures/judge-calibration/qa-health-rubric/runs/comparison-1.result.json
@@ -0,0 +1,93 @@
+{
+  "config": "qa-health-rubric",
+  "model": "claude-fable-5-1",
+  "request_hash": {
+    "old": "c00326001bb0e5511ead24b7568cfeafc3a9ae4fe4a4f94a1ed5e556fae12497",
+    "new": "07ca93d3097140d5f7b4b0dcffff84f2eb20e5ddc4de92fe7693e6e1a011da81"
+  },
+  "complete": true,
+  "stopped": null,
+  "calls": 216,
+  "cost_usd": 10.6175,
+  "errors": {
+    "old": {
+      "refusal": 0,
+      "truncation": 0,
+      "parse": 0,
+      "schema": 0,
+      "transport": 0,
+      "samples": 144,
+      "rate": 0
+    },
+    "new": {
+      "refusal": 0,
+      "truncation": 0,
+      "parse": 0,
+      "schema": 0,
+      "transport": 0,
+      "samples": 72,
+      "rate": 0
+    }
+  },
+  "latency_p95_ms": {
+    "old": 17064,
+    "new": 18789
+  },
+  "all": {
+    "items": 24,
+    "old_flips": {
+      "flips": 0,
+      "compared": 24
+    },
+    "new_flips": {
+      "flips": 1,
+      "compared": 24
+    },
+    "agreement": {
+      "old-a": 22,
+      "old-b": 22,
+      "new": 23
+    },
+    "false_passes": {
+      "old-a": [
+        "a04"
+      ],
+      "old-b": [
+        "a04"
+      ],
+      "new": []
+    },
+    "new_false_passes": []
+  },
+  "heldout": {
+    "items": 8,
+    "old_flips": {
+      "flips": 0,
+      "compared": 8
+    },
+    "new_flips": {
+      "flips": 0,
+      "compared": 8
+    },
+    "agreement": {
+      "old-a": 8,
+      "old-b": 8,
+      "new": 8
+    },
+    "false_passes": {
+      "old-a": [],
+      "old-b": [],
+      "new": []
+    },
+    "new_false_passes": []
+  },
+  "landing": {
+    "lands": true,
+    "reasons": [],
+    "strict_no_flip_increase": false
+  },
+  "prose_step_triggered": {
+    "triggered": false,
+    "reasons": []
+  }
+}
diff --git a/test/fixtures/judge-calibration/qa-health-rubric/runs/comparison-1.samples.jsonl b/test/fixtures/judge-calibration/qa-health-rubric/runs/comparison-1.samples.jsonl
new file mode 100644
index 000000000..47580e2e4
--- /dev/null
+++ b/test/fixtures/judge-calibration/qa-health-rubric/runs/comparison-1.samples.jsonl
@@ -0,0 +1,216 @@
+{"config":"qa-health-rubric","item":"f05","split":"dev","phase":"old-b","sample":0,"status":"ok","value":{"clarity":1,"completeness":1,"actionability":1,"reasoning":"The rubric defines no scoring categories, no weights, and no specific deduction amounts. It only says to 'take off points' for vaguely described issues and 'more for worse problems,' leaving severity tiers, point values, and caps entirely to the agent's judgment. It even contradicts itself by stating a flawless site should score 'in the 90s' rather than 100, with no explanation of what the missing points represent. No edge cases (duplicate issues, intermittent errors, third-party scripts, minimum floor) are addressed. Two agents following this rubric would produce arbitrary, non-reproducible scores, and the agent cannot 'apply the weights correctly' because no weights exist."},"latency_ms":3949,"stop_reason":"end_turn","usage":{"input_tokens":397,"output_tokens":255},"cost_usd":0.01672}
+{"config":"qa-health-rubric","item":"f01","split":"dev","phase":"old-a","sample":1,"status":"ok","value":{"clarity":2,"completeness":2,"actionability":1,"reasoning":"Per-category scoring is reasonably clear (bucketed Console tiers, -15 per broken link, severity deductions with floors), and the primary-category precedence order for deduplication is useful. However, the rubric fails at the critical final step: it provides no numeric weights at all. It says functionality and accessibility 'should count for more' and tells the agent to 'use your judgment' per site, which makes the overall score non-reproducible and inconsistent across runs or agents. Severity levels (Critical/High/Medium/Low) are never defined, so classification is subjective. Edge cases are missing: what counts as a 'broken' link, whether a category with zero relevant surface area (e.g., no console) is scored 100 or excluded, how to handle a site where a category doesn't apply, and rounding rules. An agent cannot compute a correct, consistent final score from this rubric alone."},"latency_ms":6506,"stop_reason":"end_turn","usage":{"input_tokens":722,"output_tokens":317},"cost_usd":0.02307}
+{"config":"qa-health-rubric","item":"f03","split":"dev","phase":"old-a","sample":2,"status":"ok","value":{"clarity":2,"completeness":1,"actionability":1,"reasoning":"The weights are the only unambiguous element (they total 100% and are clearly tabulated). Everything else is vague: 'a few errors' vs 'lots of errors' with 'partial credit' and 'low score' gives no numeric values; link deductions have no point amounts and 'important link' is undefined; per-category scoring says 'deduct in proportion to how bad it is' and 'use a sensible floor' without specifying any point values or floor. The severity guide is tautological ('serious: anything that really hurts', 'minor: small stuff') and gives no concrete deduction per severity level. Edge cases (zero issues, many minor issues, mixed severity, what to do if a category isn't tested) are absent. 'Round sensibly' is undefined. Two agents following this would almost certainly produce different scores for the same site, so reproducibility is not achievable from the rubric alone."},"latency_ms":6667,"stop_reason":"end_turn","usage":{"input_tokens":698,"output_tokens":317},"cost_usd":0.02283}
+{"config":"qa-health-rubric","item":"f03","split":"dev","phase":"old-b","sample":1,"status":"ok","value":{"clarity":2,"completeness":1,"actionability":2,"reasoning":"The weight table is clear and sums to 100%, which is the rubric's main strength. But nearly every deduction criterion is vague: 'a few errors', 'lots of errors', 'partial credit', 'low score', 'more if it is an important link', 'in proportion to how bad it is', 'cost a lot', 'cost a little', 'use a sensible floor', 'round sensibly'. The severity guide ('really hurts the experience', 'noticeable problems', 'small stuff') gives no concrete point values or examples, so two agents would produce very different scores for the same site. No numeric deduction amounts, no floor value, no definition of 'important link', no rounding rule, no handling of edge cases (e.g., category not applicable, zero issues vs. untestable). An agent could mechanically apply the weights but cannot compute a consistent, reproducible score from this alone."},"latency_ms":6829,"stop_reason":"end_turn","usage":{"input_tokens":698,"output_tokens":315},"cost_usd":0.02273}
+{"config":"qa-health-rubric","item":"f01","split":"dev","phase":"new","sample":2,"status":"ok","value":{"clarity":2,"completeness":2,"actionability":2,"reasoning":"The per-category deduction tables (Console tiers, Links -15 each, severity deductions with floor 0) are concrete and unambiguous, and the deduplication/primary-category ordering rule is a nice touch. However, the rubric fails at its central purpose: the Final Score section provides no actual weights, instead telling the agent to 'use your judgment' about how much each category contributes. This makes the final score non-reproducible and inconsistent across runs or agents. Additional gaps: severity levels (Critical/High/Medium/Low) are never defined, so the same finding could be scored -25 or -3 depending on interpretation; 'broken link' is undefined (do redirects, soft 404s, timeouts, external links count?); Console does not specify whether warnings or non-reproducible errors count; and there is no guidance on how to handle a category that could not be tested or is not applicable. An agent can compute category sub-scores but cannot compute a correct, consistent overall score from this rubric alone."},"latency_ms":9037,"stop_reason":"end_turn","usage":{"input_tokens":1083,"output_tokens":568},"cost_usd":0.03923}
+{"config":"qa-health-rubric","item":"a03","split":"dev","phase":"old-a","sample":1,"status":"ok","value":{"clarity":3,"completeness":2,"actionability":3,"reasoning":"Deduction values, severity tiers, count thresholds, and the final formula are stated concretely, and the single-primary-category rule plus 'round only at the end' support reproducibility. However, the rubric has a structural inconsistency: Content is listed as a scored per-category area (and in the primary-category precedence order) but has no weight in the table, and the listed weights sum to 95% rather than 100%. An agent cannot know whether Content is weighted 5%, unweighted, or omitted, so two agents could produce different scores. Other gaps: 'tested' vs 'untested' is never defined (does one page count? one check?), 'compare only identical coverage' is unexplained, Performance issues lack guidance for mapping speed thresholds onto the Critical/High/Medium/Low tiers, and 'same root cause' deduplication relies on judgment. The score is computable, but not reliably consistent across agents."},"latency_ms":13265,"stop_reason":"end_turn","usage":{"input_tokens":1072,"output_tokens":838},"cost_usd":0.05262}
+{"config":"qa-health-rubric","item":"a01","split":"dev","phase":"old-a","sample":0,"status":"ok","value":{"clarity":4,"completeness":3,"actionability":4,"reasoning":"Categories, weights (sum to 100%), deduction values, severity tiers, formula, and rounding rule are all explicit and unambiguous. Weaknesses: 'Σ tested weights' implies categories can be untested but never defines what qualifies as tested/untested; no guidance on mapping Performance or Accessibility findings onto the Critical/High/Medium/Low tiers; no rule on deduplicating the same finding across pages or preventing a single defect from being deducted in multiple categories; 'reproducible/repeatable' and 'timeout' lack thresholds; exclusion of 'resource/API requests' from Links is fuzzy for broken images/assets. The trailing framework guidance and operational rules are unrelated to scoring and add noise. Despite gaps, an agent can compute a defensible score from the rubric alone, though reproducibility across agents will vary at severity-assignment boundaries."},"latency_ms":13759,"stop_reason":"end_turn","usage":{"input_tokens":1359,"output_tokens":917},"cost_usd":0.05944}
+{"config":"qa-health-rubric","item":"p05","split":"dev","phase":"old-a","sample":2,"status":"ok","value":{"clarity":4,"completeness":4,"actionability":4,"reasoning":"Strong rubric: ordered first-match category rule resolves overlap, severity tiers are defined, formulas are explicit, weights sum to 1.00, rounding (floor(x+0.5)) and the zero floor are specified, and untested/empty cases are handled with a provisional label. Remaining gaps: 'repeatable' 4xx/5xx and 'slow' performance have no thresholds; 'same root cause' merging and the Critical/High boundary require judgment that can vary between runs; it's not stated whether broken_links counts post-merge findings or raw URLs; 'tested' is undefined for partial passes; and 'Phase 5' is an external reference. These are minor and an agent can compute a correct, mostly reproducible score from the document, but not with perfect determinism."},"latency_ms":15862,"stop_reason":"end_turn","usage":{"input_tokens":1193,"output_tokens":1013},"cost_usd":0.06258}
+{"config":"qa-health-rubric","item":"p02","split":"dev","phase":"old-b","sample":2,"status":"ok","value":{"clarity":4,"completeness":4,"actionability":4,"reasoning":"Categories, weights (sum to 100%), deduction amounts, floors, and the final normalized formula with rounding rule are all explicit and unambiguous. Deduplication rules and a first-applicable category priority order prevent double-counting, and untested/partial coverage handling is defined. Weaknesses: severity definitions are task/workaround-oriented and map awkwardly to Performance, Visual, and Accessibility findings (e.g., how to grade a WCAG contrast violation or a 4s page load), 'reproducible'/'repeatable' and 'timeout' lack numeric thresholds, 'tested' vs 'untested' and 'coverage' are not operationally defined, and 'Compare only identical coverage' is cryptic. An agent can compute a correct score from this alone, but severity assignment in non-functional categories will introduce some inconsistency between runs."},"latency_ms":12486,"stop_reason":"end_turn","usage":{"input_tokens":1081,"output_tokens":786},"cost_usd":0.05011}
+{"config":"qa-health-rubric","item":"f04","split":"dev","phase":"new","sample":0,"status":"ok","value":{"clarity":1,"completeness":1,"actionability":1,"reasoning":"The rubric is internally contradictory and ambiguous throughout. Counting rules directly conflict: 'count every finding in every category it touches' vs 'no double deductions', and 'count each page separately' vs 'deduplicate findings that share a root cause'. Console thresholds overlap (1-5, 3-10, 8+), so 3, 4, 5, 8, 9, and 10 errors map to multiple scores. Functional/UX is a single 35% bucket but is described as two categories with 'deduct from whichever feels closer, or both', giving no way to compute one category score; Medium issue deduction is '-8 or -10'. Severity levels (Critical/High/Medium/Low) are never defined. Accessibility issues are said to also count as Visual, reintroducing double counting. Only Links specifies a floor of 0; other categories have no stated minimum. Visual/Content/Performance/Accessibility have no criteria for what counts as an issue. Weights do sum to 100%, but with these contradictions two agents cannot produce the same reproducible score."},"latency_ms":10493,"stop_reason":"end_turn","usage":{"input_tokens":1150,"output_tokens":752},"cost_usd":0.0491}
+{"config":"qa-health-rubric","item":"p04","split":"dev","phase":"old-a","sample":0,"status":"ok","value":{"clarity":4,"completeness":4,"actionability":5,"reasoning":"Categories, weights, and per-finding deductions are explicit and numeric, with a precedence rule for overlapping findings, a dedupe rule for shared elements, a 0 floor, untested-category handling, rounding timing, bands, and a worked example whose arithmetic checks out. Minor ambiguities remain: 'impaired' flow and 'meaningful images' require judgment; it is unclear whether expected 4xx (e.g., failed login attempt returning 401) count as Errors; a flow that doesn't exist on the site (no sign-up) vs. untested isn't addressed; performance measurement is undefined if the slowest page fails to load; and whether contrast failures count per element or per style is not stated. Despite these, an agent can reliably compute a reproducible score from the rubric alone."},"latency_ms":15400,"stop_reason":"end_turn","usage":{"input_tokens":1312,"output_tokens":1122},"cost_usd":0.06922}
+{"config":"qa-health-rubric","item":"p04","split":"dev","phase":"new","sample":1,"status":"ok","value":{"clarity":4,"completeness":4,"actionability":5,"reasoning":"Categories, weights, and per-finding deductions are explicit and numeric, with a precedence rule for overlapping findings, a dedupe rule for shared elements, a floor at 0, handling for untested categories, a rounding rule, and a worked example that checks out (86.75 → 87). Minor clarity gaps: 'meaningful images', 'time to interactive measured from the navigation timing entry' (nav timing doesn't directly expose TTI), and whether 'contrast failure' counts per element or per style require judgment. Completeness gaps: how to handle a page that fails to load and also blocks a core flow (precedence sends it to Availability only, which may understate flow impact), intermittent failures, performance measurement when the slowest page didn't load, and benign 4xx (e.g., favicon) counting as errors. Overall an agent can compute a consistent score from this document alone."},"latency_ms":19625,"stop_reason":"end_turn","usage":{"input_tokens":1673,"output_tokens":1265},"cost_usd":0.07998}
+{"config":"qa-health-rubric","item":"a06","split":"dev","phase":"new","sample":0,"status":"ok","value":{"clarity":3,"completeness":2,"actionability":2,"reasoning":"Weights are explicit and sum to 100%, and the A-F descriptors are reasonably distinct, but core mechanics are ambiguous: 'half a letter grade' has no defined intermediate value (is A minus 0.5 an A-? a B+?), 'High/Medium/Polish impact' findings are never defined, and there is no letter-to-number mapping, so a 'weighted average' of letter grades is undefined and the required numeric score out of 100 cannot be derived. Missing edge cases include rounding rules, how to average letters, what happens when a category has no findings vs. is not applicable, how two Medium findings combine with a half-step below D, and what the 'pithy verdict' for AI Slop should be. An agent could assign per-category letters somewhat consistently but could not produce a reproducible 0-100 score from this rubric alone."},"latency_ms":11014,"stop_reason":"end_turn","usage":{"input_tokens":1132,"output_tokens":654},"cost_usd":0.04402}
+{"config":"qa-health-rubric","item":"p01","split":"dev","phase":"old-b","sample":1,"status":"ok","value":{"clarity":4,"completeness":4,"actionability":4,"reasoning":"Categories, weights (summing to 100%), per-finding deductions, severity tiers, floors, and the final weighted-average formula with explicit rounding are all clearly stated, and the dedup/primary-category priority order prevents double counting. Gaps: no criteria for deciding when a category counts as 'tested' vs 'untested' (which changes the denominator), severity definitions are task-centric and map awkwardly to Performance and Visual (no latency/CLS thresholds), 'expected auth redirects' and 'Compare only identical coverage' are undefined, and the appended framework guidance and operational rules are unrelated to scoring and add noise. An agent can still compute a reproducible score in most cases, but coverage-determination and performance-severity judgments could cause variance between runs."},"latency_ms":18443,"stop_reason":"end_turn","usage":{"input_tokens":1507,"output_tokens":1144},"cost_usd":0.07227}
+{"config":"qa-health-rubric","item":"f05","split":"dev","phase":"new","sample":2,"status":"ok","value":{"clarity":1,"completeness":1,"actionability":1,"reasoning":"The rubric defines no scoring categories, no weights, and no specific deduction amounts. 'Take off points' and 'take off more for worse problems' are entirely subjective, so two agents testing the same site would produce wildly different scores. There are no severity tiers, no per-issue point values, no caps, no definition of what counts as a bug vs. cosmetic issue, and the only anchor ('no problems should score in the 90s') is itself a range rather than a fixed value and contradicts the idea of a perfect 100 baseline. Edge cases (duplicate issues, partially broken features, warnings vs. errors, uncounted categories like accessibility or performance) are unaddressed. An agent cannot compute a reproducible score from this text alone."},"latency_ms":5820,"stop_reason":"end_turn","usage":{"input_tokens":758,"output_tokens":274},"cost_usd":0.02128}
+{"config":"qa-health-rubric","item":"p05","split":"dev","phase":"old-b","sample":2,"status":"ok","value":{"clarity":4,"completeness":3,"actionability":4,"reasoning":"Categories use a clear first-match precedence list, severity tiers are distinct, and formulas/weights are given as explicit code with defined rounding (floor(x+0.5)) and a defined 'not scored' path, so an agent can compute a reproducible number in most cases. Gaps: 'slow' (Performance) and 'repeatable' (Links) have no thresholds; what constitutes a category's 'checks ran' for the tested/untested decision is not operationalized (partial passes, Quick mode specifics); how to count broken_links after root-cause merging (one finding vs. per-URL) is implied but not stated; and console 'error' vs warning scope is unaddressed. These leave some judgment calls that can produce inconsistent scores across runs."},"latency_ms":15290,"stop_reason":"end_turn","usage":{"input_tokens":1193,"output_tokens":934},"cost_usd":0.05863}
+{"config":"qa-health-rubric","item":"a01","split":"dev","phase":"old-b","sample":0,"status":"ok","value":{"clarity":4,"completeness":3,"actionability":4,"reasoning":"Weights sum to 100%, deduction values, floors, dedup rules, and the final formula (including rounding and normalization by tested weights) are explicit, so the arithmetic is reproducible once findings are classified. Severity tiers are reasonably defined. Gaps: no rule for assigning a finding to one category (Visual vs UX vs Functional) so double-counting is possible; 'defects scored elsewhere' and 'repeatable' 4xx/5xx leave judgment undefined; 'tested weights' never defines when a category counts as untested; no domain-specific severity anchors for Performance or Accessibility (e.g., thresholds, WCAG levels); severity tiers like Critical don't map naturally to Visual/Content. The framework guidance and operational rules reference external documents (BROWSER SETUP, LOOK/ACT) and add noise without affecting scoring. An agent can compute a correct score mechanically, but classification variance will limit consistency across runs."},"latency_ms":14414,"stop_reason":"end_turn","usage":{"input_tokens":1359,"output_tokens":923},"cost_usd":0.05974}
+{"config":"qa-health-rubric","item":"f04","split":"dev","phase":"old-a","sample":2,"status":"ok","value":{"clarity":1,"completeness":1,"actionability":1,"reasoning":"The rubric is riddled with direct contradictions: 'count every finding in every category it touches' vs 'no double deductions'; 'count each page separately' vs 'deduplicate findings that share a root cause'. Console tiers overlap (1-5, 3-10, 8+), so 4 or 9 errors map to two different scores. Medium issues deduct '-8 or -10' with no selection criterion, and Functional/UX says to deduct from 'whichever feels closer, or both'. Functional and UX are given a single 35% weight but no rule for combining or splitting them. Visual/Content/Performance/Accessibility have no explicit base score or floor, no severity definitions exist for Critical/High/Medium/Low, and the accessibility-color rule reintroduces double counting. An agent cannot compute a reproducible score from this."},"latency_ms":6303,"stop_reason":"end_turn","usage":{"input_tokens":789,"output_tokens":305},"cost_usd":0.02314}
+{"config":"qa-health-rubric","item":"f03","split":"dev","phase":"old-b","sample":0,"status":"ok","value":{"clarity":2,"completeness":1,"actionability":2,"reasoning":"Weights are explicit and sum to 100, but nearly every scoring instruction is vague: 'a few errors → partial credit', 'deduct for each broken link, more if important', 'deduct in proportion to how bad it is', 'use a sensible floor', 'round sensibly'. Severity definitions are circular ('serious: anything that really hurts'), and no point values are attached to Serious/Moderate/Minor. No numeric thresholds exist for console errors, no per-link deduction amounts, no definition of 'important link', no floor value, no rounding rule. An agent cannot reproduce a consistent score from this—two runs on the same site could diverge widely. The weighting step is actionable; the per-category scoring is not."},"latency_ms":5854,"stop_reason":"end_turn","usage":{"input_tokens":698,"output_tokens":269},"cost_usd":0.02043}
+{"config":"qa-health-rubric","item":"f04","split":"dev","phase":"new","sample":2,"status":"ok","value":{"clarity":1,"completeness":1,"actionability":1,"reasoning":"The rubric is internally contradictory and ambiguous at nearly every step. The Counting section directly conflicts with itself: 'count every finding in every category it touches' vs 'no double deductions', and 'count each page separately' vs 'deduplicate findings that share a root cause'. Console error bands overlap (1-5, 3-10, 8+), so counts of 3, 4, 5, 8, 9, 10 map to two different scores. Medium issues deduct '-8 or -10' with no rule for choosing. Functional and UX are given a '35% combined' weight but no instruction on whether they are one category or two and how to split or merge them; the guidance 'deduct from whichever feels closer, or both' is subjective and changes the final score. Severity levels (Critical/High/Medium/Low) are never defined, and Visual/Content/Performance/Accessibility simply inherit undefined deductions. No rounding, minimum-score, or edge-case guidance exists beyond the Links floor. Weights do sum to 100%, but an agent cannot produce a consistent, reproducible score from this document."},"latency_ms":10746,"stop_reason":"end_turn","usage":{"input_tokens":1150,"output_tokens":736},"cost_usd":0.0483}
+{"config":"qa-health-rubric","item":"a06","split":"dev","phase":"new","sample":1,"status":"ok","value":{"clarity":3,"completeness":2,"actionability":2,"reasoning":"Strengths: ten categories are named, weights are explicit and sum to 100%, the A–F descriptors are readable, and the deduction mechanic (High = -1 letter, Medium = -0.5 letter, Polish = no effect, floor at F) is stated plainly. Weaknesses that undermine reproducibility: (1) the rubric is letter-based while the agent must emit a numeric score out of 100—no letter-to-number mapping (e.g., A=100/90/95?) is given, so a 'weighted average' of letters is undefined; (2) 'half a letter grade' has no representation in the five-step A–F scale (no B+/B- defined), so one Medium finding yields an indeterminate grade; (3) no rounding rule for converting the weighted average back to a letter or number; (4) High/Medium/Polish severity is never defined, so the core deduction trigger is left to judgment; (5) category scopes (what distinguishes Visual Hierarchy from Spacing & Layout) are not described, inviting double-counting; (6) 'pithy verdict' and the standalone AI Slop grading method are unspecified. An agent cannot compute a consistent 0–100 score from this document alone without inventing several conventions."},"latency_ms":13651,"stop_reason":"end_turn","usage":{"input_tokens":1132,"output_tokens":816},"cost_usd":0.05212}
+{"config":"qa-health-rubric","item":"f01","split":"dev","phase":"new","sample":1,"status":"ok","value":{"clarity":2,"completeness":2,"actionability":2,"reasoning":"The per-category deduction values and console/link tiers are clearly specified, and the deduplication/primary-category ordering is a useful disambiguation rule. However, the Final Score section fatally undermines the rubric: no numeric weights are given, and the agent is told to 'use your judgment' about contribution per site, making the final score non-reproducible and inconsistent across runs — directly violating the stated goal. Severity levels (Critical/High/Medium/Low) are never defined, so classification of findings is subjective. Edge cases are also missing: what counts as 'reproducible' for console errors, whether warnings count, whether links include external/redirects/soft-404s, whether a page with no applicable category (e.g., no Performance testing) scores 100 or is excluded, and how rounding works. An agent cannot compute a correct, consistent final score from this rubric alone."},"latency_ms":6547,"stop_reason":"end_turn","usage":{"input_tokens":1083,"output_tokens":309},"cost_usd":0.02628}
+{"config":"qa-health-rubric","item":"p03","split":"dev","phase":"new","sample":0,"status":"ok","value":{"clarity":5,"completeness":4,"actionability":5,"reasoning":"Categories, weights, deduction amounts, severity tiers, priority order for single-category assignment, deduplication rules, floor, rounding (0.5 up), and the provisional/untested handling are all explicit and internally consistent; the worked example arithmetic (89.6 → 90; 79.6/0.90 → 88) checks out and demonstrates both the full and provisional paths. Minor completeness gaps: 'repeatable'/'reproducible' are not quantified (how many retries), Performance has no objective thresholds for what constitutes a finding or its severity, Accessibility severity mapping (e.g., WCAG level) is left to judgment, and the Medium ('impaired but workaround') vs Low ('friction, task completes') boundary can blur for UX findings. These could introduce some run-to-run variance in finding classification, but given a list of findings with severities, an agent can compute a correct, reproducible score from this rubric alone."},"latency_ms":18154,"stop_reason":"end_turn","usage":{"input_tokens":1964,"output_tokens":1196},"cost_usd":0.07944}
+{"config":"qa-health-rubric","item":"f02","split":"dev","phase":"old-b","sample":2,"status":"ok","value":{"clarity":2,"completeness":2,"actionability":1,"reasoning":"The rubric has direct internal contradictions: Console is labeled 20% in its header but 15% in the weights table, and Links is 15% in the header but 10% in the table. Worse, the weights table sums to 125%, not 100%, so a perfect site would score 125 despite the claim that 'a perfect site scores 100' — an agent cannot produce a correct or reproducible final score without guessing which weights to use or how to renormalize. Beyond the weight errors, severity levels (Critical/High/Medium/Low) are never defined, console 'errors' is ambiguous (warnings? per-page vs. deduplicated?), and there is no rounding guidance. The per-category deduction mechanics and dedup precedence are reasonably clear, which keeps clarity/completeness above 1, but the broken weighting makes the rubric non-actionable as written."},"latency_ms":8997,"stop_reason":"end_turn","usage":{"input_tokens":733,"output_tokens":703},"cost_usd":0.04248}
+{"config":"qa-health-rubric","item":"p03","split":"dev","phase":"old-a","sample":0,"status":"ok","value":{"clarity":5,"completeness":4,"actionability":5,"reasoning":"Categories, weights, deduction amounts, severity tiers, and tie-breaking (highest severity wins, first-matching category wins) are all explicit and unambiguous. Edge cases are well covered: deduplication of root causes, no double-counting via a precedence order, untested-category exclusion with renormalized denominator and 'provisional' labeling, 'not scored' when nothing tested, floors at 0, no credit back, rounding rule, link/console exclusions (auth redirects, warnings, API requests). The worked example is arithmetically correct (89.6 → 90; 79.6/0.90 = 88.44 → 88) and demonstrates the provisional path, so an agent can reproduce scores from the rubric alone. Minor completeness gaps: no concrete thresholds for what constitutes a Performance or Accessibility finding (agent must infer severity from task impact), no guidance for a site that fails to load entirely, and 'core app unusable for all users' vs 'core task blocked' can require judgment at the Critical/High boundary."},"latency_ms":14903,"stop_reason":"end_turn","usage":{"input_tokens":1603,"output_tokens":1051},"cost_usd":0.06858}
+{"config":"qa-health-rubric","item":"a01","split":"dev","phase":"new","sample":0,"status":"ok","value":{"clarity":4,"completeness":3,"actionability":4,"reasoning":"Categories, weights (summing to 100%), count thresholds for Console/Links, per-severity deductions, and the final weighted-average formula with explicit rounding are all clearly stated, so an agent can mechanically compute a score from a list of findings. Weaknesses: 'Σ tested weights' implies categories can be skipped but never defines when a category counts as untested; 'reproducible'/'repeatable' lack a repro count; deduplication is specified only for Console (not for Visual/Functional/UX/etc. findings recurring across pages); cross-category double-counting is only addressed for Console ('scored elsewhere') without a tiebreak rule for which category owns a defect; Performance and Accessibility have no concrete finding definitions or severity mapping (e.g., is a missing alt text Low or Medium?). The framework guidance and browser safety rules are irrelevant to score computation and add noise. Overall solid and usable, but boundary cases will produce inconsistent scores between runs."},"latency_ms":15223,"stop_reason":"end_turn","usage":{"input_tokens":1720,"output_tokens":909},"cost_usd":0.06265}
+{"config":"qa-health-rubric","item":"a04","split":"dev","phase":"old-b","sample":2,"status":"ok","value":{"clarity":4,"completeness":3,"actionability":4,"reasoning":"Weights sum to 100%, the deduplication/precedence order is explicit, severity tiers have concrete definitions, deduction amounts and floors are stated, and the final formula with tested-weight normalization and rounding rule is unambiguous. The main weakness is the Console category: 'a few', 'several', and 'many' have no numeric thresholds, so two agents could assign 70 vs 40 to the same error count, undermining reproducibility. Other gaps: no definition of what makes a category 'tested' vs 'untested' (or how much coverage qualifies), no guidance on mapping Performance findings (e.g., slow load times) to the severity tiers, 'core/major task' remains a judgment call, and 'compare only identical coverage' is terse. An agent can compute a correct score for most categories from this rubric alone, but Console scoring requires guesswork."},"latency_ms":11743,"stop_reason":"end_turn","usage":{"input_tokens":1071,"output_tokens":862},"cost_usd":0.05381}
+{"config":"qa-health-rubric","item":"p02","split":"dev","phase":"old-a","sample":2,"status":"ok","value":{"clarity":4,"completeness":4,"actionability":4,"reasoning":"Weights sum to 100 and are tabulated; deduction values, floors, category precedence for deduplication, untested-category handling, and the final weighted formula with rounding rule are all explicit, so an agent can compute a reproducible score. Severity tiers are defined by impact/workaround, though the Medium/Low boundary (task 'impaired' vs 'friction') and mapping of Performance or Accessibility findings onto task-based severity leave some judgment to the agent. Minor terse phrases ('Compare only identical coverage', what counts as 'tested' or 'reproducible') and no guidance on provisional-score computation beyond labeling keep it short of fully airtight, but overall it is a solid, mostly unambiguous rubric."},"latency_ms":16434,"stop_reason":"end_turn","usage":{"input_tokens":1081,"output_tokens":1083},"cost_usd":0.06496}
+{"config":"qa-health-rubric","item":"a05","split":"dev","phase":"old-b","sample":1,"status":"ok","value":{"clarity":3,"completeness":2,"actionability":3,"reasoning":"Weights sum to 100% and the arithmetic (per-category 0-100 then weighted sum) is straightforward, so the computation itself is actionable. However, the Console bands overlap at exactly 10 errors ('4-10 → 40' and '10+ → 10'), severity levels (Critical/High/Medium/Low) are never defined so classification is left to the agent's judgment, and 'error' and 'broken link' are undefined (warnings? duplicates across pages? site-wide vs per-page counts?). No guidance on rounding, on categories that weren't tested or are not applicable, on deduplicating repeated findings, or on what constitutes a finding in each of the six per-category areas. These gaps mean two agents with the same observations could produce materially different scores."},"latency_ms":11636,"stop_reason":"end_turn","usage":{"input_tokens":651,"output_tokens":716},"cost_usd":0.04231}
+{"config":"qa-health-rubric","item":"f03","split":"dev","phase":"new","sample":0,"status":"ok","value":{"clarity":2,"completeness":1,"actionability":1,"reasoning":"The rubric structure (categories and weights summing to 100%) is clear, but nearly every scoring instruction is vague: 'a few errors → partial credit', 'deduct for each broken link, more if important', 'deduct in proportion to how bad it is', 'use a sensible floor', 'round sensibly'. No numeric deduction values are given per severity, no thresholds define 'a few' vs 'lots', no floor is specified, and severity definitions are circular ('serious: anything that really hurts'). Edge cases (zero issues, duplicate issues, overlapping categories, how to classify a borderline issue, how to handle untestable categories) are undefined. Two agents following this rubric would produce materially different scores, so it fails the reproducibility requirement and cannot yield a correct score on its own."},"latency_ms":4399,"stop_reason":"end_turn","usage":{"input_tokens":1059,"output_tokens":293},"cost_usd":0.02524}
+{"config":"qa-health-rubric","item":"p05","split":"dev","phase":"old-b","sample":0,"status":"ok","value":{"clarity":4,"completeness":4,"actionability":5,"reasoning":"Categories use an explicit first-match precedence list, severity tiers are concrete, weights sum to 1.00 and are shown, and every formula (per-category scores, renormalization over tested categories, floor(score+0.5) rounding, max(0,...) floors) is spelled out so two agents would compute identical scores from the same finding list. Deduplication, untested-category handling, the empty-tested case, and provisional labeling are all covered. Minor gaps keep it from perfect: 'repeatable' 4xx/5xx and 'slow' lack thresholds, 'untested' is defined only by whether checks ran (no guidance for partially-run categories), and 'core task'/'core app' in severity tiers leave some judgment room; a worked example would also help. These affect finding classification more than score computation, so actionability remains very high."},"latency_ms":14703,"stop_reason":"end_turn","usage":{"input_tokens":1193,"output_tokens":864},"cost_usd":0.05513}
+{"config":"qa-health-rubric","item":"f05","split":"dev","phase":"old-b","sample":1,"status":"ok","value":{"clarity":1,"completeness":1,"actionability":1,"reasoning":"The rubric defines no scoring categories, no weights, and no deduction amounts—only a vague instruction to 'take off points' and 'take off more for worse problems.' Severity tiers, point values per issue type, caps, and boundaries are all absent, so two agents (or the same agent twice) would produce wildly different scores. The only concrete anchor is that a clean site scores 'in the 90s,' which is itself a range rather than a fixed value. An agent cannot compute a reproducible score from this text alone."},"latency_ms":3263,"stop_reason":"end_turn","usage":{"input_tokens":397,"output_tokens":198},"cost_usd":0.01387}
+{"config":"qa-health-rubric","item":"p05","split":"dev","phase":"new","sample":2,"status":"ok","value":{"clarity":4,"completeness":4,"actionability":5,"reasoning":"Categories use a first-match precedence list, severity tiers are defined, formulas are explicit with clamping and rounding rules, and weights are shown to sum to 1.00, so the computation is deterministic and reproducible. Untested categories, the 'not scored' case, renormalization of weights, and provisional labeling are all handled. Remaining gaps: 'slow' for Performance has no threshold, 'repeatable' 4xx/5xx has no repeat count, console 'error' vs warning and external vs internal links aren't addressed, and severity judgments (e.g., 'core task') still require interpretation, which slightly limits cross-run consistency. Otherwise an agent can compute the score from the rubric alone."},"latency_ms":17167,"stop_reason":"end_turn","usage":{"input_tokens":1554,"output_tokens":1134},"cost_usd":0.07224}
+{"config":"qa-health-rubric","item":"p04","split":"dev","phase":"old-b","sample":1,"status":"ok","value":{"clarity":4,"completeness":4,"actionability":5,"reasoning":"Categories, weights, deduction amounts, and the combination formula are explicit, weights sum to 100, precedence rule resolves most overlaps, and the worked example is arithmetically correct (86.75 → 87) with a stated rounding rule. Minor ambiguities remain: a page that fails to load and thereby blocks a core flow could be read as one finding (Availability only) or two; 'time to interactive from the navigation timing entry' doesn't name the exact metric; 'meaningful images' needs judgment; and Rule 2 (shared element counts once) slightly conflicts with Errors' 'unique by message + URL' counting. Edge cases like the 2.00–2.01s gap and whether bands apply to partial scores are unaddressed, but these are small; an agent can compute a reproducible score from this rubric alone."},"latency_ms":14809,"stop_reason":"end_turn","usage":{"input_tokens":1312,"output_tokens":1114},"cost_usd":0.06882}
+{"config":"qa-health-rubric","item":"a03","split":"dev","phase":"new","sample":1,"status":"ok","value":{"clarity":4,"completeness":3,"actionability":3,"reasoning":"Categories, severity tiers, count thresholds, dedup/primary-category ordering, normalization formula, and rounding rule are mostly explicit and unambiguous. However, there is a material internal inconsistency: Content is listed as a scored per-category bucket (and in the primary-category precedence list) but is absent from the weights table, and the listed weights sum to 95%, not 100%. An agent cannot know whether to drop Content, assign it a weight, or fold it into UX, which undermines reproducibility. Other gaps: 'untested' vs partially tested is not defined (no coverage threshold), Performance findings are forced into task-blocking severity definitions that fit poorly (no latency thresholds), and there is no guidance for findings that fit no category. The Σ tested weights normalization partially masks the weight-sum error, but an agent working from this rubric alone would still have to make an unguided judgment call about Content."},"latency_ms":14764,"stop_reason":"end_turn","usage":{"input_tokens":1433,"output_tokens":979},"cost_usd":0.06328}
+{"config":"qa-health-rubric","item":"f04","split":"dev","phase":"old-b","sample":2,"status":"ok","value":{"clarity":1,"completeness":1,"actionability":1,"reasoning":"The rubric is riddled with direct contradictions and gaps that make a reproducible score impossible. Counting rules conflict: 'count every finding in every category it touches' vs 'no double deductions'; 'count each page separately' vs 'deduplicate findings sharing a root cause'. Console bands overlap (1-5, 3-10, 8+), so 4 errors maps to both 70 and 40. Functional and UX are given a single combined 35% weight but 'deduct from whichever feels closer, or both' — unspecified how two sub-scores combine into one, and 'feels closer' is subjective. Medium deductions are '-8 or -10' with no rule for choosing. Severity levels (critical/high/medium/low) are never defined. Weights sum to 100% only if Functional/UX is one category, yet the text treats them as two. Several categories lack any starting value or floor statement, and no example calculation is given. An agent cannot compute a correct, consistent score from this alone."},"latency_ms":6909,"stop_reason":"end_turn","usage":{"input_tokens":789,"output_tokens":352},"cost_usd":0.02549}
+{"config":"qa-health-rubric","item":"f05","split":"dev","phase":"new","sample":1,"status":"ok","value":{"clarity":1,"completeness":1,"actionability":1,"reasoning":"The rubric defines no scoring categories, no weights, and no deduction amounts. 'Take off points' and 'take off more for worse problems' give no numeric guidance, so two agents testing the same site would produce wildly different scores. The only concrete anchor is 'no problems should score in the 90s,' which is itself a range rather than a value and oddly implies a flawless site cannot reach 100. There is no severity taxonomy (critical vs. minor), no floor/cap rules, no handling of edge cases like untestable pages, duplicate issues, or third-party errors, and no formula for combining deductions into a final out-of-100 score. An agent cannot compute a consistent, reproducible score from this text alone."},"latency_ms":5055,"stop_reason":"end_turn","usage":{"input_tokens":758,"output_tokens":263},"cost_usd":0.02073}
+{"config":"qa-health-rubric","item":"a04","split":"dev","phase":"old-a","sample":0,"status":"ok","value":{"clarity":4,"completeness":3,"actionability":4,"reasoning":"Weights sum to 100%, severity tiers have concrete definitions, deduction values and the final formula (including untested-weight normalization and rounding) are explicit, and a category precedence order prevents double counting. The main weakness is the Console category, which uses undefined buckets ('a few', 'several', 'many') with no numeric thresholds, making that 15% of the score non-reproducible. Other gaps: 'untested' is never defined (how much coverage counts as tested vs partial), 'same root cause' deduplication relies on judgment, and there is no guidance on how many findings constitute a single deduction when the same defect manifests in multiple ways within a category. An agent can compute a score from this rubric alone, but two agents could diverge on Console scoring and on tested/untested classification."},"latency_ms":10827,"stop_reason":"end_turn","usage":{"input_tokens":1071,"output_tokens":707},"cost_usd":0.04606}
+{"config":"qa-health-rubric","item":"p03","split":"dev","phase":"old-a","sample":2,"status":"ok","value":{"clarity":5,"completeness":4,"actionability":5,"reasoning":"Categories, weights (summing to 100%), deduction values, tiering, precedence order for single-category assignment, and the weighted-average formula are all explicit and internally consistent; the worked example is arithmetically correct and demonstrates both the full and provisional (untested-category) cases, plus rounding rules. Edge cases like deduplication, auth redirects, timeouts, floors, no-tested-categories, and provisional labeling are covered. Remaining gaps: Performance and Accessibility have no concrete thresholds or examples, so severity assignment there (and the 'core task' vs 'impaired with workaround' boundary generally) still relies on judgment; intermittent/non-reproducible findings are only addressed for Console and Links, not the severity-based categories. An agent can nonetheless compute a reproducible score from this document alone."},"latency_ms":15942,"stop_reason":"end_turn","usage":{"input_tokens":1603,"output_tokens":1065},"cost_usd":0.06928}
+{"config":"qa-health-rubric","item":"f03","split":"dev","phase":"old-b","sample":2,"status":"ok","value":{"clarity":2,"completeness":1,"actionability":1,"reasoning":"The weights table is clear and sums to 100%, which is the rubric's only strong point. Everything else is vague: 'a few errors → partial credit' and 'lots of errors → low score' give no numeric thresholds; 'deduct for each broken link, more if important' specifies no point values or definition of 'important'; the per-category scoring says 'deduct in proportion to how bad it is' with no point ranges per severity tier, and 'use a sensible floor' and 'round sensibly' leave both the floor and rounding rule undefined. The severity guide ('anything that really hurts', 'noticeable problems', 'small stuff') is circular and cannot distinguish Serious from Moderate reliably. No edge cases are addressed (e.g., what if a category has no issues to assess, how to handle duplicate issues, whether per-page or per-site, whether a 0 is possible). Two agents following this rubric would almost certainly produce different scores for the same site, so it fails the reproducibility requirement."},"latency_ms":7316,"stop_reason":"end_turn","usage":{"input_tokens":698,"output_tokens":356},"cost_usd":0.02478}
+{"config":"qa-health-rubric","item":"a04","split":"dev","phase":"new","sample":2,"status":"ok","value":{"clarity":4,"completeness":3,"actionability":3,"reasoning":"Strengths: weights sum to 100% and are tabulated, deduction values and floors are explicit, severity tiers have concrete definitions, a primary-category precedence order prevents double counting, and the final formula (including untested-weight normalization and rounding rule) is precisely specified. Weaknesses: the Console category uses vague bands ('a few', 'several', 'many') with no numeric thresholds, which directly undermines reproducibility for a 15%-weighted category; 'tested' vs 'untested' is never operationally defined, so the denominator of the formula is left to judgment; 'repeatable' for links and 'reproducible' for console errors lack retry criteria; the task-based severity scale maps awkwardly onto Visual, Content, and Performance findings (e.g., no guidance on how slow load times translate to severity); and the 'provisional/coverage' labeling has no format. An agent can compute a score but two agents could plausibly diverge on console bands and tested-category inclusion."},"latency_ms":12670,"stop_reason":"end_turn","usage":{"input_tokens":1432,"output_tokens":843},"cost_usd":0.05647}
+{"config":"qa-health-rubric","item":"a03","split":"dev","phase":"old-b","sample":2,"status":"ok","value":{"clarity":4,"completeness":3,"actionability":3,"reasoning":"Categories, severity tiers, deduction values, deduplication/primary-category ordering, floor, and rounding are mostly explicit, and the normalization formula for untested categories is clear. However, Content appears in the per-category scoring list and in the primary-category ordering but is missing from the weights table (weights sum to 95%), so an agent cannot compute a correct final score when Content findings exist. Other gaps: 'tested' vs 'untested' category is undefined, 'reproducible'/'repeatable' thresholds are unspecified, severity mapping for Performance/Visual findings (e.g., how slow is 'impaired') is left to judgment, and 'Compare only identical coverage' is vague. These undermine reproducibility despite otherwise solid structure."},"latency_ms":13037,"stop_reason":"end_turn","usage":{"input_tokens":1072,"output_tokens":841},"cost_usd":0.05277}
+{"config":"qa-health-rubric","item":"p04","split":"dev","phase":"old-a","sample":2,"status":"ok","value":{"clarity":5,"completeness":4,"actionability":5,"reasoning":"Categories, weights, and per-finding deductions are explicit and numeric; weights sum to 100, the tie-break rule (first matching row), de-duplication rule, 0 floor, untested-category handling, rounding method (half up, end only), and gap-free bands are all specified. The worked example is arithmetically correct and demonstrates the formula end to end, so an agent can reproduce a score reliably. Minor gaps keep completeness from a 5: 'time to interactive from the navigation timing entry' is not a precise metric, 'meaningful images' and 'distinct page' require some judgment, performance thresholds assume hundredth-second precision (no handling of e.g. 2.005s), there is no guidance when performance is measured on only some pages, and routing a 5xx on a checkout page solely to Availability (per rule 1) may understate Core-flow impact even though the rule itself is unambiguous."},"latency_ms":16440,"stop_reason":"end_turn","usage":{"input_tokens":1312,"output_tokens":1175},"cost_usd":0.07187}
+{"config":"qa-health-rubric","item":"p02","split":"dev","phase":"old-a","sample":0,"status":"ok","value":{"clarity":4,"completeness":4,"actionability":5,"reasoning":"Weights sum to 100% and are tabulated; deduction values, severity tiers, floors, dedup/primary-category precedence order, and the final formula with explicit rounding rule are all unambiguous. Untested-category handling (exclude, renormalize by tested weights, label provisional) and the none-tested case are covered. Minor gaps: no definition of what makes a category 'tested' vs partially tested, no threshold for 'repeatable' (how many retries) or 'reproducible', no guidance on third-party console errors or what constitutes a Performance finding (no latency thresholds), and 'compare only identical coverage' is terse. Some overlap between Functional/UX/Accessibility is resolved by the first-applicable ordering, though severity assignment still requires judgment. Overall an agent can compute a reproducible score from this alone."},"latency_ms":11985,"stop_reason":"end_turn","usage":{"input_tokens":1081,"output_tokens":770},"cost_usd":0.04931}
+{"config":"qa-health-rubric","item":"f02","split":"dev","phase":"old-b","sample":0,"status":"ok","value":{"clarity":2,"completeness":2,"actionability":1,"reasoning":"The rubric has direct internal contradictions: Console is labeled 20% in its section header but 15% in the weights table, and Links is 15% vs 10%. Worse, the weights table sums to 125%, so a perfect site would score 125, contradicting the stated 'perfect site scores 100' — the formula cannot yield a valid 0-100 score as written. Severity levels (Critical/High/Medium/Low) are never defined, 'console error' is not defined (warnings? per page or total?), and the deduplication rule gives a precedence order but no criteria for when a category 'applies'. The structure is reasonable and the per-category deduction values are concrete, but an agent following this alone cannot produce a correct or reproducible score."},"latency_ms":8106,"stop_reason":"end_turn","usage":{"input_tokens":733,"output_tokens":613},"cost_usd":0.03798}
+{"config":"qa-health-rubric","item":"a04","split":"dev","phase":"new","sample":0,"status":"ok","value":{"clarity":4,"completeness":3,"actionability":4,"reasoning":"Weights are explicit, sum to 100%, and the final formula (weighted average normalized by tested weights, round-half-up only at the end) is unambiguous. Severity tiers have concrete definitions with per-severity deductions and a floor of 0; Links has a precise per-item deduction with sensible exclusions; deduplication and primary-category precedence prevent double counting. Main weaknesses: the Console bands ('a few', 'several', 'many') lack numeric thresholds, so a 15%-weight category depends on subjective judgment and undermines reproducibility; 'tested' vs 'untested' is not defined (how much coverage counts as tested?); Performance and Visual have no domain-specific guidance for mapping observations (e.g., load times) to severity beyond the generic task-impact definitions; and 'Compare only identical coverage' is cryptic. An agent can compute a correct score from this alone for most categories, but Console scores will vary between runs."},"latency_ms":15637,"stop_reason":"end_turn","usage":{"input_tokens":1432,"output_tokens":998},"cost_usd":0.06422}
+{"config":"qa-health-rubric","item":"a05","split":"dev","phase":"old-a","sample":2,"status":"ok","value":{"clarity":3,"completeness":2,"actionability":3,"reasoning":"Weights sum correctly to 100% and the formula is explicit, so the arithmetic is mechanically clear. However, the Console bands overlap at exactly 10 errors (both '4-10' and '10+'), creating a direct ambiguity. Severity levels (Critical/High/Medium/Low) are never defined, so classification is left to agent judgment and undermines reproducibility. Missing edge cases: whether console warnings count, how duplicate/repeated errors or findings are counted, whether broken links are deduplicated, how to handle a category that was not tested (N/A), rounding of the final score, and any objective thresholds for Performance or Accessibility (e.g., Core Web Vitals, WCAG level mapping). An agent can compute a score once findings are classified, but two agents given the same site could diverge substantially due to undefined severity criteria and the boundary overlap."},"latency_ms":11764,"stop_reason":"end_turn","usage":{"input_tokens":651,"output_tokens":780},"cost_usd":0.04551}
+{"config":"qa-health-rubric","item":"p02","split":"dev","phase":"new","sample":0,"status":"ok","value":{"clarity":4,"completeness":4,"actionability":4,"reasoning":"Weights sum to 100% and are tabulated with an explicit decimal formula, untested-category normalization, floor, and rounding rule, so an agent can mechanically compute a score. Deduplication rules, a primary-category precedence order, and count-based tiers for Console/Links are unambiguous. Severity tiers are defined by user impact with 'use highest applicable,' which supports reproducibility. Gaps: no definition of what makes a category 'tested' (coverage threshold), 'repeatable'/'reproducible' lack a repro count, 'Compare only identical coverage' is cryptic, Performance has no numeric thresholds so severity mapping is judgment-heavy, and the Critical definition ('unusable for all users') arguably excludes accessibility-only blockers, creating ambiguity for that category. Visual issues will almost always map to Low under the task-based severity definitions, which may be intended but isn't stated."},"latency_ms":18029,"stop_reason":"end_turn","usage":{"input_tokens":1442,"output_tokens":1090},"cost_usd":0.06892}
+{"config":"qa-health-rubric","item":"a05","split":"dev","phase":"old-b","sample":0,"status":"ok","value":{"clarity":3,"completeness":2,"actionability":3,"reasoning":"Structure and weights are clear and sum to 100%, and the final formula is explicit. However, the Console tiers overlap ('4-10' and '10+' both include 10), which breaks reproducibility at that boundary. Severity levels (Critical/High/Medium/Low) are never defined, nor is what counts as a 'console error' (warnings? per page or total?) or a 'broken link' (404 only, timeouts, external links?). No guidance on rounding, duplicate findings, issues spanning multiple categories, or how to handle a category that wasn't tested. An agent can compute a score if findings are already classified, but consistency across runs is undermined by the undefined inputs and the boundary overlap."},"latency_ms":9733,"stop_reason":"end_turn","usage":{"input_tokens":651,"output_tokens":618},"cost_usd":0.03741}
+{"config":"qa-health-rubric","item":"p03","split":"dev","phase":"old-b","sample":1,"status":"ok","value":{"clarity":5,"completeness":4,"actionability":5,"reasoning":"Categories, weights, deduction amounts, and a strict precedence order for single-categorization are all explicit, and the severity tiers include a tie-break rule. Edge cases are largely handled: deduplication by root cause, untested categories (renormalized denominator, provisional label, 'not scored' when nothing tested), floors, no bonus points, rounding rule, and run-to-run comparability. The worked example is arithmetically correct (89.6 → 90; 79.6/0.90 → 88) and demonstrates both the full and provisional paths, so an agent can reproduce a score from the text alone. Minor gaps keep completeness from a 5: 'reproducible' console errors and 'ran to completion' checks are not quantified, no concrete performance thresholds or accessibility-to-severity mapping are given (left to judgment), and there is no instruction for a finding that fits none of the eight categories."},"latency_ms":16536,"stop_reason":"end_turn","usage":{"input_tokens":1603,"output_tokens":1164},"cost_usd":0.07423}
+{"config":"qa-health-rubric","item":"a05","split":"dev","phase":"new","sample":2,"status":"ok","value":{"clarity":3,"completeness":2,"actionability":3,"reasoning":"Structure and weighted-average formula are clear and weights sum to 100, but there are notable ambiguities: the Console bands overlap at exactly 10 errors (both '4-10 → 40' and '10+ → 10' apply); severity levels (Critical/High/Medium/Low) are never defined, so classification—and therefore the score—will vary between runs; and 'console error' is undefined (warnings? duplicates? third-party scripts? per page vs. site-wide?). Completeness gaps include no definition of 'broken link' (404 vs. timeout vs. redirect chains), no guidance on double-counting a finding across categories, no handling for untested/not-applicable categories, no rounding rule, and no page-vs-site aggregation rule. An agent can perform the arithmetic once findings are bucketed, but without severity definitions the result is not reliably reproducible from the rubric alone."},"latency_ms":9774,"stop_reason":"end_turn","usage":{"input_tokens":1012,"output_tokens":713},"cost_usd":0.04577}
+{"config":"qa-health-rubric","item":"a02","split":"dev","phase":"old-b","sample":1,"status":"ok","value":{"clarity":4,"completeness":3,"actionability":4,"reasoning":"Categories, weights (summing to 100%), thresholds, normalization formula, and rounding rule are explicit and unambiguous; the primary-category precedence list prevents double counting. However, the per-category scoring hinges on Critical/High/Medium/Low severity with no definitions, so two agents could classify the same finding differently and get different scores. Other soft spots: 'reproducible'/'repeatable' lack a repetition count, 'tested' vs 'untested' (and what qualifies as partial coverage) is undefined, and 'same root cause' deduplication requires judgment. Given pre-classified findings, an agent can compute a correct, reproducible score from the formula alone, but the severity gap limits consistency."},"latency_ms":12254,"stop_reason":"end_turn","usage":{"input_tokens":952,"output_tokens":923},"cost_usd":0.05567}
+{"config":"qa-health-rubric","item":"a03","split":"dev","phase":"old-b","sample":0,"status":"ok","value":{"clarity":3,"completeness":3,"actionability":3,"reasoning":"Severity tiers, deduction amounts, dedup rules, and the normalized weighted-average formula are precise and reproducible. However, 'Content' is defined as a scorable per-category (and in the primary-category ordering) yet is absent from the weights table, and the listed weights sum to 95% rather than 100%. The normalization by Σ tested weights masks the 95% issue but gives an agent no way to weight Content findings, so an agent cannot compute a correct score for any site with content defects without guessing. Secondary gaps: no definition of what makes a category 'tested' vs 'untested', 'compare only identical coverage' is vague, and Performance/Accessibility must be force-mapped onto generic task-blocking severity definitions."},"latency_ms":14498,"stop_reason":"end_turn","usage":{"input_tokens":1072,"output_tokens":886},"cost_usd":0.05502}
+{"config":"qa-health-rubric","item":"a04","split":"dev","phase":"old-b","sample":1,"status":"ok","value":{"clarity":3,"completeness":3,"actionability":4,"reasoning":"Weights (sum to 100%), per-finding deductions, floors, rounding, and the weighted-average formula with renormalization for untested categories are all explicit and computable. Severity tiers are reasonably defined and the dedup/primary-category precedence order prevents double counting. However, the Console category uses vague bands ('a few', 'several', 'many') with no numeric thresholds, which directly undermines reproducibility for a 15%-weight category. Other gaps: no definition of what makes a category 'tested' vs 'untested' or how to express 'coverage'; 'compare only identical coverage' is unexplained; no guidance mapping performance metrics (e.g., load times) to severity tiers; and deciding whether a console error is 'scored elsewhere' relies on judgment. An agent can produce a score from this rubric alone, but two agents could plausibly diverge on the Console band and on tested/untested determinations."},"latency_ms":11986,"stop_reason":"end_turn","usage":{"input_tokens":1071,"output_tokens":877},"cost_usd":0.05456}
+{"config":"qa-health-rubric","item":"a05","split":"dev","phase":"new","sample":1,"status":"ok","value":{"clarity":4,"completeness":2,"actionability":3,"reasoning":"Weights sum to 100 and the formula is explicit, deduction values are concrete, and the per-category mechanics are easy to follow. However, the Console tiers overlap at exactly 10 errors (both '4-10' and '10+' apply), creating a genuine ambiguity. Completeness is weak: severity levels (Critical/High/Medium/Low) are never defined, so the largest driver of the score is left to subjective judgment and will not be reproducible across runs; 'console error' is undefined (warnings? duplicates of the same error? third-party scripts?); 'broken link' is undefined (4xx only? 5xx? timeouts? redirects? external vs internal?); there is no guidance for categories that could not be tested (e.g., Performance not measured), for findings that span multiple categories, or for rounding/precision of the final score. An agent can mechanically compute a number from this rubric once findings are classified, but the missing severity definitions and the boundary overlap mean two agents testing the same site could plausibly produce materially different scores."},"latency_ms":11289,"stop_reason":"end_turn","usage":{"input_tokens":1012,"output_tokens":836},"cost_usd":0.05192}
+{"config":"qa-health-rubric","item":"f02","split":"dev","phase":"old-a","sample":1,"status":"ok","value":{"clarity":2,"completeness":2,"actionability":1,"reasoning":"The rubric has direct internal contradictions: Console is listed as 20% in its section header but 15% in the weights table, and Links is 15% vs 10%. Worse, the weights table sums to 125%, not 100%, so a perfect site would score 125 under the stated formula, contradicting 'A perfect site scores 100.' An agent cannot resolve which weights to use or how to normalize without guessing, making consistent reproduction impossible. Completeness is also weak: Critical/High/Medium/Low severity levels are never defined, 'console errors' is ambiguous (warnings? per-page vs. site-wide? unique vs. repeated), and the dedup rule's 'first applicable' category assignment lacks criteria for determining applicability. The per-category deduction mechanics and floor are clear, but the core arithmetic cannot be executed correctly from this document alone."},"latency_ms":9076,"stop_reason":"end_turn","usage":{"input_tokens":733,"output_tokens":653},"cost_usd":0.03998}
+{"config":"qa-health-rubric","item":"a01","split":"dev","phase":"old-a","sample":1,"status":"ok","value":{"clarity":4,"completeness":3,"actionability":4,"reasoning":"Categories, weights (summing to 100), deduction amounts, severity tiers, floors, and the final weighted-average formula with rounding rule are explicit and mostly unambiguous. Gaps: no criteria for what makes a category 'tested' vs untested (affects the denominator), no rule for assigning a finding to exactly one category or preventing double-counting across Visual/Functional/UX/Accessibility, no thresholds for what constitutes a Performance or Accessibility finding, and 'defects scored elsewhere' for Console is vague. The framework guidance and 'Important Rules' sections reference external docs (BROWSER SETUP, sentinel, LOOK vs ACT) and aren't scoring-relevant, slightly diluting focus. Given a list of classified findings, an agent can compute a reproducible score, but classification consistency depends on judgment the rubric doesn't fully constrain."},"latency_ms":13180,"stop_reason":"end_turn","usage":{"input_tokens":1359,"output_tokens":818},"cost_usd":0.05449}
+{"config":"qa-health-rubric","item":"p05","split":"dev","phase":"old-a","sample":0,"status":"ok","value":{"clarity":4,"completeness":4,"actionability":5,"reasoning":"Categories use explicit first-match precedence, severity tiers are defined, formulas and weights are spelled out (and sum to 1.00), rounding is specified, and untested-category handling plus the 'not scored' case are covered, so an agent can mechanically compute a reproducible score. Remaining ambiguities are judgment-based rather than structural: 'slow' has no performance threshold, the Critical/High boundary ('core app unusable' vs 'core task blocked') can blur, 'repeatable' for 4xx/5xx isn't quantified, and whether a category counts as 'tested' depends on an external Phase 5 definition. Edge cases like partial category coverage or console warnings vs errors are not addressed, but defaults are reasonably inferable."},"latency_ms":15160,"stop_reason":"end_turn","usage":{"input_tokens":1193,"output_tokens":940},"cost_usd":0.05893}
+{"config":"qa-health-rubric","item":"p01","split":"dev","phase":"old-b","sample":2,"status":"ok","value":{"clarity":4,"completeness":4,"actionability":4,"reasoning":"Categories, weights (summing to 100%), per-severity deductions, floors, dedup/primary-category ordering, exclusion of untested categories, and the final weighted formula with rounding rule are all explicitly defined, so an agent can compute a reproducible score. Minor gaps: 'tested' vs 'untested' for a category is not operationalized, 'reproducible'/'repeatable' and 'timeout' lack thresholds, 'Compare only identical coverage' is cryptic, and the unit of a 'finding' (one root cause vs. per-occurrence) relies on the dedup sentence rather than an explicit definition. The framework guidance and browser rules are irrelevant to scoring and add some noise, but do not undermine the core computation."},"latency_ms":14612,"stop_reason":"end_turn","usage":{"input_tokens":1507,"output_tokens":879},"cost_usd":0.05902}
+{"config":"qa-health-rubric","item":"a06","split":"dev","phase":"old-a","sample":1,"status":"ok","value":{"clarity":3,"completeness":2,"actionability":2,"reasoning":"Weights are explicit and sum to 100%, and the per-category grade descriptors are reasonably clear. However, the rubric is letter-based while the task requires a numeric score out of 100, and no letter-to-number mapping is given, so a 'weighted average' of letter grades cannot be computed deterministically. 'Half a letter grade' is undefined (does B- exist, or do two Mediums equal one High?). 'High-impact', 'Medium-impact', and 'Polish' findings are never defined, leaving severity classification to the agent's judgment and undermining reproducibility. Edge cases are missing: how to handle a category that is not applicable (e.g., Motion on a static site), how the standalone AI Slop grade is derived vs. its 5% contribution, how to round the final weighted result back to A-F or to 0-100, and whether the 'pithy verdict' has any scoring effect. An agent cannot produce a correct, consistent numeric score from this rubric alone."},"latency_ms":10839,"stop_reason":"end_turn","usage":{"input_tokens":771,"output_tokens":678},"cost_usd":0.04161}
+{"config":"qa-health-rubric","item":"p02","split":"dev","phase":"new","sample":2,"status":"ok","value":{"clarity":4,"completeness":3,"actionability":4,"reasoning":"Weights sum to 100% and are stated twice consistently; per-category deduction values, floors, count thresholds for Console/Links, severity definitions, dedup priority order, untested-category handling, and the exact weighted-average formula with rounding rule are all explicit, so an agent can reproduce a score from a given set of findings. Gaps: 'tested' vs 'untested' is never defined (how much coverage qualifies), Performance has no thresholds for what constitutes a finding and the severity ladder (task blocked/workaround) maps poorly to slowness, 'reproducible'/'repeatable'/'timeout' lack retry counts or time limits, and 'Compare only identical coverage' is opaque. Minor ambiguity in the one-word parenthetical category hints for dedup routing. These leave room for inconsistent inputs even though the arithmetic itself is unambiguous."},"latency_ms":14891,"stop_reason":"end_turn","usage":{"input_tokens":1442,"output_tokens":982},"cost_usd":0.06352}
+{"config":"qa-health-rubric","item":"f03","split":"dev","phase":"old-a","sample":1,"status":"ok","value":{"clarity":2,"completeness":1,"actionability":1,"reasoning":"The weights are the only unambiguous element (and they correctly sum to 100%). Everything else is vague: Console scoring uses 'a few' vs 'lots' with no numeric thresholds or point values; Links says 'deduct for each broken link' and 'more if important' without specifying amounts or defining 'important'; per-category scoring says 'in proportion to how bad it is' with no point values per severity tier, and the severity definitions ('really hurts', 'noticeable', 'small stuff') are circular and subjective. 'Use a sensible floor' and 'round sensibly' leave key boundaries undefined. No guidance on what counts as an issue in each category, how to handle zero issues, duplicate/overlapping issues, or categories that cannot be assessed. Two agents given identical findings would almost certainly produce different scores, so the rubric fails the reproducibility requirement."},"latency_ms":6680,"stop_reason":"end_turn","usage":{"input_tokens":698,"output_tokens":320},"cost_usd":0.02298}
+{"config":"qa-health-rubric","item":"a05","split":"dev","phase":"old-a","sample":0,"status":"ok","value":{"clarity":3,"completeness":2,"actionability":3,"reasoning":"Structure and weights are clear and sum to 100%, and the arithmetic is simple. However, the Console tiers overlap at exactly 10 errors ('4-10' vs '10+'), the formula doesn't state whether weights are decimals or percentages, and severity levels (Critical/High/Medium/Low) are never defined, leaving classification entirely to agent judgment and undermining reproducibility. Also undefined: what counts as a console 'error' (vs warning), what 'broken link' means (404 only? timeouts? redirects?), what constitutes a Performance finding, whether a finding can be counted in multiple categories, and rounding rules. An agent can compute a score if findings are pre-classified, but the rubric alone does not guarantee consistent results across agents."},"latency_ms":10732,"stop_reason":"end_turn","usage":{"input_tokens":651,"output_tokens":686},"cost_usd":0.04081}
+{"config":"qa-health-rubric","item":"f02","split":"dev","phase":"new","sample":2,"status":"ok","value":{"clarity":2,"completeness":2,"actionability":1,"reasoning":"The rubric contains direct contradictions: Console is labeled 20% in its section header but 15% in the weights table, and Links is 15% in the header but 10% in the table. Worse, the weights table sums to 125% (15+10+15+25+20+15+10+15), so a 'perfect' site would score 125, contradicting the stated 'perfect site scores 100.' An agent cannot compute a correct or reproducible score from this. Beyond the weight errors, severity levels (Critical/High/Medium/Low) are undefined, so deductions are subjective; 'console errors' is not scoped (warnings? third-party? repeated instances of the same error?); 'broken link' is undefined (4xx only? 5xx? timeouts? redirect chains?); and no rounding rule is given. The deduplication rule and per-category structure are reasonably clear, which is why clarity and completeness are not at the floor, but the core computation is broken."},"latency_ms":12357,"stop_reason":"end_turn","usage":{"input_tokens":1094,"output_tokens":877},"cost_usd":0.05479}
+{"config":"qa-health-rubric","item":"f05","split":"dev","phase":"new","sample":0,"status":"ok","value":{"clarity":1,"completeness":1,"actionability":1,"reasoning":"The rubric defines no scoring categories, no weights, and no per-issue deduction amounts. 'Take off more for worse problems' gives no severity tiers or point values, and 'anything that looks wrong' is entirely subjective. The only concrete anchor is that a flawless site 'should score in the 90s' (itself a 10-point range), with no guidance on how low scores can go for critical failures, how to handle partial test coverage, or how to cap deductions. An agent could not compute a reproducible score from this; two runs on the same findings would plausibly differ by tens of points. It fails all three task requirements: no categories to understand, no weights to apply, and no basis for consistency."},"latency_ms":5594,"stop_reason":"end_turn","usage":{"input_tokens":758,"output_tokens":252},"cost_usd":0.02018}
+{"config":"qa-health-rubric","item":"f04","split":"dev","phase":"old-a","sample":1,"status":"ok","value":{"clarity":1,"completeness":2,"actionability":1,"reasoning":"The rubric is internally contradictory: 'count every finding in every category it touches' directly conflicts with 'no double deductions', and 'count each page separately' conflicts with 'deduplicate findings that share a root cause'. Console bands overlap (1-5, 3-10, 8+), so 4 or 9 errors map to two different scores. Medium deductions are '-8 or -10' with no rule for choosing. Functional and UX share a 35% weight but the rubric says to deduct from 'whichever feels closer, or both' and never specifies whether they are one category or two, or how the 35% splits. Severity levels (Critical/High/Medium/Low) are never defined. Accessibility color issues are explicitly double-counted into Visual. Weights do sum to 100% and the aggregation formula is stated, which is the only solid part, but two agents following this rubric would almost certainly produce different scores for the same findings, so reproducibility is not achievable."},"latency_ms":9741,"stop_reason":"end_turn","usage":{"input_tokens":789,"output_tokens":570},"cost_usd":0.03639}
+{"config":"qa-health-rubric","item":"p04","split":"dev","phase":"new","sample":2,"status":"ok","value":{"clarity":5,"completeness":4,"actionability":5,"reasoning":"Categories, weights, and per-category deduction formulas are explicit and tabular; weights sum to 100 and the sum is stated. Tie-breaking for multi-category findings (first matching row), deduplication of shared elements, zero floor, untested-category reweighting, and rounding order are all specified. Bands use integer, non-overlapping ranges that align with the round-first rule. The worked example is arithmetically correct (8675/100 = 86.75 → 87) and demonstrates the full procedure. Minor gaps keep completeness from a 5: performance bands have a gap between 2.0s and 2.01s (sub-centisecond values undefined) and 'time to interactive from the navigation timing entry' is slightly imprecise since TTI isn't a native Navigation Timing field; it is not stated whether a default flow (e.g., checkout) that simply doesn't exist on the site is 'blocked', untested, or not applicable; whether a page crash that also blocks a flow counts only in Availability or also as a blocked flow is arguable under Rule 1; 'distinct page' is not defined (URL vs. template), and intermittent/flaky failures are not addressed. These are edge cases rather than core ambiguities, so an agent can reliably compute a reproducible score from the rubric alone."},"latency_ms":21493,"stop_reason":"end_turn","usage":{"input_tokens":1673,"output_tokens":1464},"cost_usd":0.08993}
+{"config":"qa-health-rubric","item":"a01","split":"dev","phase":"old-a","sample":2,"status":"ok","value":{"clarity":4,"completeness":3,"actionability":4,"reasoning":"Categories, per-finding deductions, count-based tiers, and weights (which sum to 100%) are stated explicitly, and the final formula with rounding rule is unambiguous. Severity definitions are reasonable but have soft boundaries (e.g., 'core app unusable for all users' vs. 'core task blocked' can overlap; a blocked non-core task has no clear tier). Completeness gaps: it never defines when a category counts as 'tested' (needed for the Σ tested weights denominator), how to avoid double-counting a single defect across Visual/Functional/UX/etc. beyond the console-only exclusion, what 'reproducible'/'repeatable' means (how many retries), timeout thresholds, or any objective criteria for Performance/Accessibility severity. Despite these gaps, an agent with a finding list can compute a deterministic score, so actionability is high; the framework guidance and browser rules are tangential noise for scoring but don't impair it."},"latency_ms":12947,"stop_reason":"end_turn","usage":{"input_tokens":1359,"output_tokens":804},"cost_usd":0.05379}
+{"config":"qa-health-rubric","item":"p02","split":"dev","phase":"old-b","sample":0,"status":"ok","value":{"clarity":4,"completeness":4,"actionability":4,"reasoning":"Categories, weights (sum to 100%), deduction values, severity tiers, dedup rules, and a priority order for assigning a finding to one category are all explicitly defined, and the final formula with tested-weight normalization and rounding rule is unambiguous. Edge cases like floors, minimum 0, untested categories, and double-deduction are covered. Gaps: 'tested' vs 'untested'/'partial' coverage is never operationally defined, 'Compare only identical coverage' is cryptic, finding granularity (what counts as one distinct finding vs. multiple instances) is left to judgment, and mapping Performance issues onto the task-blocking severity scale is not spelled out, which can affect reproducibility between agents."},"latency_ms":19407,"stop_reason":"end_turn","usage":{"input_tokens":1081,"output_tokens":1299},"cost_usd":0.07576}
+{"config":"qa-health-rubric","item":"a03","split":"dev","phase":"new","sample":0,"status":"ok","value":{"clarity":3,"completeness":3,"actionability":3,"reasoning":"The rubric is mostly well-structured: fixed deduction values, explicit severity tiers, a dedup/priority order, a normalization formula, and a rounding rule. However, there is a direct internal inconsistency: 'Content' is listed as a scorable category in both the priority order and the Per-Category Scoring section, but it is absent from the Weights table, and the listed weights sum to 95% rather than 100%. An agent cannot tell whether Content findings should be dropped, given an implied 5% weight, or folded elsewhere, which undermines reproducibility. Other gaps: 'tested' vs 'untested' is never operationalized, 'provisional with coverage' and 'compare only identical coverage' are vague, and the severity definitions are task-centric, making it unclear how a Performance or Visual finding maps to Critical/High (e.g., a 10s load time). The division by Σ tested weights lets an agent produce a number regardless, but two agents could reasonably diverge on the Content issue and on severity assignment for non-functional categories."},"latency_ms":14452,"stop_reason":"end_turn","usage":{"input_tokens":1433,"output_tokens":994},"cost_usd":0.06403}
+{"config":"qa-health-rubric","item":"p05","split":"dev","phase":"old-a","sample":1,"status":"ok","value":{"clarity":4,"completeness":4,"actionability":5,"reasoning":"The rubric is well-structured with ordered steps, first-match category precedence, explicit severity definitions, exact formulas, weights that sum to 1.00, and a defined rounding rule. Edge cases like untested categories, empty tested sets, duplicate merging, and console errors already covered by other findings are handled, and provisional labeling prevents misleading comparisons. Minor gaps keep it from perfect: 'repeatable' 4xx/5xx and 'slow' (Performance) are not quantified, 'same root cause' deduplication relies on judgment, and partial coverage within a category (some checks ran) is implicitly treated as tested without further guidance. Despite these, an agent can reproducibly compute a score from the rubric alone."},"latency_ms":12488,"stop_reason":"end_turn","usage":{"input_tokens":1193,"output_tokens":831},"cost_usd":0.05348}
+{"config":"qa-health-rubric","item":"f04","split":"dev","phase":"old-b","sample":1,"status":"ok","value":{"clarity":1,"completeness":1,"actionability":1,"reasoning":"The rubric is internally contradictory and non-reproducible. Counting rules directly conflict: 'count every finding in every category it touches' vs 'no double deductions'; 'count each page separately' vs 'deduplicate findings that share a root cause'. Console thresholds overlap (1-5 vs 3-10 vs 8+), so 4, 5, 8, 9, or 10 errors map to two different scores. Medium deductions are '-8 or -10' with no selection criterion. Functional/UX says 'deduct from whichever feels closer, or both,' making the 35% category inherently subjective and allowing double-counting that the counting section forbids. Severity levels (Critical/High/Medium/Low) are never defined. Visual/Content/Performance/Accessibility are given no starting value or floor, and the Accessibility-is-also-Visual rule again contradicts 'no double deductions'. Weights do sum to 100%, but Functional and UX are split in the rubric body yet merged in the table without guidance on how to combine two category scores into one 35% slot. An agent cannot compute a consistent score from this."},"latency_ms":6952,"stop_reason":"end_turn","usage":{"input_tokens":789,"output_tokens":397},"cost_usd":0.02774}
+{"config":"qa-health-rubric","item":"f01","split":"dev","phase":"new","sample":0,"status":"ok","value":{"clarity":3,"completeness":2,"actionability":2,"reasoning":"The per-category mechanics (Console tiers, Links -15 each, severity deductions with floors, dedup precedence order) are explicit and unambiguous. However, the Final Score section undermines the whole rubric: it provides no numeric weights, instead telling the agent to 'use your judgment about how much each category should contribute,' which makes the overall score non-reproducible across runs or agents. Severity levels (Critical/High/Medium/Low) are never defined, so classification of findings is subjective. Edge cases are missing: what to do when a category is not applicable or untested, how to round, whether Console/Links use the same weighting scheme as the other six categories, and what 'reproducible' means for console errors. An agent can compute each category score but cannot compute a correct, consistent final score from this rubric alone."},"latency_ms":8834,"stop_reason":"end_turn","usage":{"input_tokens":1083,"output_tokens":537},"cost_usd":0.03768}
+{"config":"qa-health-rubric","item":"a03","split":"dev","phase":"old-b","sample":1,"status":"ok","value":{"clarity":4,"completeness":3,"actionability":3,"reasoning":"Category definitions, severity tiers, count-based tables for Console/Links, deduplication order, floor, and rounding rules are mostly unambiguous and well specified. However, there is a significant internal inconsistency: Content is listed as a scored category (in the primary-category ordering and the Per-Category Scoring section) but is absent from the weights table, and the listed weights sum to only 95%. An agent cannot determine whether Content findings should be weighted, folded into another category, or discarded, which undermines reproducibility. Other gaps: 'tested' vs 'untested' is not operationally defined, 'reproducible'/'repeatable' lacks a retry threshold, 'Compare only identical coverage' is vague, and Performance has no objective thresholds (relies on severity judgment). The normalization formula over tested weights partially mitigates the missing weight, but a correct, consistent score still cannot be guaranteed from the rubric alone."},"latency_ms":14119,"stop_reason":"end_turn","usage":{"input_tokens":1072,"output_tokens":908},"cost_usd":0.05612}
+{"config":"qa-health-rubric","item":"a05","split":"dev","phase":"old-a","sample":1,"status":"ok","value":{"clarity":3,"completeness":3,"actionability":4,"reasoning":"Weights sum to 100% and the formula and per-finding deductions are explicit, so an agent can mechanically compute a score once findings are classified. However, the Console bands overlap at exactly 10 errors (both '4-10' and '10+' apply), severity levels (Critical/High/Medium/Low) are never defined, and there is no definition of what counts as a console 'error' (warnings? duplicate repeats?) or a 'broken link' (which status codes/timeouts). Also missing: rounding rules, whether a single finding can be counted in multiple categories, and how to handle categories with no applicable content. These gaps undermine reproducibility between agents even though the arithmetic itself is straightforward."},"latency_ms":10614,"stop_reason":"end_turn","usage":{"input_tokens":651,"output_tokens":630},"cost_usd":0.03801}
+{"config":"qa-health-rubric","item":"p03","split":"dev","phase":"old-a","sample":1,"status":"ok","value":{"clarity":5,"completeness":4,"actionability":5,"reasoning":"Categories, deduction values, weights, severity tiers, and tie-break rules (highest severity, first-applicable category order) are explicit and unambiguous; the worked example's arithmetic is correct and demonstrates both the full and provisional cases, including the rounding rule. Edge cases are well handled: deduplication by root cause, no double-deductions, floors at 0, no bonus points, untested-category exclusion with provisional labeling, 'not scored' when nothing tested, console error dedup key, auth-redirect and resource exclusions for links, and a concrete 10s timeout. Minor gaps keep completeness from a 5: no definition of how many repetitions make a console error 'reproducible', no guidance on mapping Performance measurements (e.g., load times) onto the task-impact severity scale, and 'check ran to completion' is not elaborated for partial crawls. Overall an agent can compute a correct, reproducible score from this document alone."},"latency_ms":16330,"stop_reason":"end_turn","usage":{"input_tokens":1603,"output_tokens":1137},"cost_usd":0.07288}
+{"config":"qa-health-rubric","item":"p01","split":"dev","phase":"old-a","sample":2,"status":"ok","value":{"clarity":4,"completeness":4,"actionability":4,"reasoning":"Categories, weights (sum to 100%), deduction amounts, severity tiers, dedup/primary-category ordering, floor, normalization over tested weights, and rounding are all explicit, so an agent can mechanically compute a score from a finding list. Gaps remain: 'tested' vs 'untested' and 'partial coverage' are undefined (no threshold for when a category counts as scored), 'reproducible'/'repeatable' lack retry counts, Performance and Accessibility reuse the generic severity tiers without concrete thresholds (e.g., what load time is Medium vs High), and 'Compare only identical coverage' is terse. The framework guidance and browser rules are operational noise unrelated to scoring but don't introduce scoring ambiguity."},"latency_ms":13110,"stop_reason":"end_turn","usage":{"input_tokens":1507,"output_tokens":874},"cost_usd":0.05877}
+{"config":"qa-health-rubric","item":"f03","split":"dev","phase":"new","sample":1,"status":"ok","value":{"clarity":2,"completeness":1,"actionability":1,"reasoning":"Weights are explicit and sum to 100%, which is the rubric's only concrete element. Everything else is vague: 'a few errors', 'lots of errors', 'partial credit', 'low score', 'deduct for each broken link, more if important', 'a sensible floor', and 'round sensibly' give no numeric values. The severity guide ('really hurts', 'noticeable', 'small stuff') provides no point deductions per severity level, no caps per category, no definition of what counts as an 'important' link, and no guidance on duplicate/overlapping issues or categories with no data. Two different agents (or the same agent twice) would produce materially different scores, so the rubric fails the reproducibility requirement and cannot be used to compute a correct score on its own."},"latency_ms":4513,"stop_reason":"end_turn","usage":{"input_tokens":1059,"output_tokens":277},"cost_usd":0.02444}
+{"config":"qa-health-rubric","item":"f03","split":"dev","phase":"new","sample":2,"status":"ok","value":{"clarity":2,"completeness":1,"actionability":2,"reasoning":"The weights table is explicit and sums to 100%, and the overall structure (per-category score, then weighted average) is understandable. However, nearly every deduction criterion is vague: 'a few errors' vs 'lots of errors' has no thresholds; 'partial credit' and 'low score' have no numeric values; link deductions have no point amounts and 'important link' is undefined; per-category deductions are 'in proportion to how bad it is' with no point ranges per severity tier; the severity guide ('really hurts', 'noticeable', 'small stuff') is circular and non-operational; 'sensible floor' and 'round sensibly' are undefined. There is no guidance on edge cases such as duplicate issues, issues spanning categories, warnings vs errors, or a site with zero testable links. Two agents following this rubric would almost certainly produce materially different scores, so reproducibility is poor despite the weights being correctly specified."},"latency_ms":5117,"stop_reason":"end_turn","usage":{"input_tokens":1059,"output_tokens":334},"cost_usd":0.02729}
+{"config":"qa-health-rubric","item":"p04","split":"dev","phase":"old-a","sample":1,"status":"ok","value":{"clarity":5,"completeness":4,"actionability":5,"reasoning":"Categories, weights, and per-category scoring formulas are explicit and tabular; blocked vs impaired flows, error uniqueness (message + URL), performance thresholds, and deduction sizes are all concrete. Rules cover the key edge cases: overlap resolution (first matching row), repeated shared elements counting once, floor at 0, untested categories with reweighting, and rounding only at the end. The worked example confirms the arithmetic. Minor gaps keep completeness from 5: 'meaningful images' and 'distinct content issue' still require some judgment, performance boundaries leave tiny gaps (e.g., 2.005s), it is unclear how to treat a default core flow (e.g., sign-up) that simply does not exist on the site, and whether bands apply to 'partial' scores is implied rather than stated. An agent can nonetheless compute a correct, reproducible score from this rubric alone."},"latency_ms":13910,"stop_reason":"end_turn","usage":{"input_tokens":1312,"output_tokens":1002},"cost_usd":0.06322}
+{"config":"qa-health-rubric","item":"a02","split":"dev","phase":"old-a","sample":1,"status":"ok","value":{"clarity":4,"completeness":3,"actionability":4,"reasoning":"Weights sum to 100% and are tabulated, deduction values are numeric, the final formula with renormalization over tested weights and the rounding rule are explicit, and dedup/primary-category precedence is clear. Main gaps: severity levels (Critical/High/Medium/Low) are never defined, so the largest driver of six categories is left to judgment and undermines reproducibility; 'tested' vs 'untested' per category and partial coverage thresholds are not specified; 'reproducible' console errors and what constitutes a Performance finding lack criteria; 'compare only identical coverage' is terse. Given pre-classified findings, an agent can compute a correct score mechanically, but severity ambiguity limits consistency across runs."},"latency_ms":9876,"stop_reason":"end_turn","usage":{"input_tokens":952,"output_tokens":667},"cost_usd":0.04287}
+{"config":"qa-health-rubric","item":"a02","split":"dev","phase":"old-a","sample":2,"status":"ok","value":{"clarity":4,"completeness":3,"actionability":4,"reasoning":"Weights sum to 100%, the formula with tested-weight normalization and rounding rule is explicit, Console/Links have concrete thresholds, and the category-precedence order plus dedup rule prevent double counting. Main gaps: Critical/High/Medium/Low are never defined, so the per-category deductions (covering 75% of the weight) hinge on subjective severity judgments that undermine reproducibility; 'tested' vs 'untested' is not operationalized; no performance thresholds or examples; 'compare only identical coverage' is terse. An agent can compute a score mechanically once severities are assigned, but two agents could legitimately diverge on severity."},"latency_ms":9836,"stop_reason":"end_turn","usage":{"input_tokens":952,"output_tokens":671},"cost_usd":0.04307}
+{"config":"qa-health-rubric","item":"a04","split":"dev","phase":"old-a","sample":2,"status":"ok","value":{"clarity":3,"completeness":3,"actionability":4,"reasoning":"Weights are explicit and sum to 100, deduction values and severity tiers are concrete, the final formula with normalization by tested weights and rounding rule is unambiguous, and deduplication/category-precedence rules are given. The main weakness is the Console category, which uses undefined buckets ('a few', 'several', 'many') with no numeric thresholds, undermining reproducibility for 15% of the score. Other gaps: 'core/major task' and 'reproducible' are undefined, Performance issues lack guidance on mapping speed to severity tiers, and partially-tested categories within a single category (e.g., some pages only) aren't addressed beyond a 'provisional' label. An agent can compute a correct score for most categories but will produce inconsistent Console scores."},"latency_ms":9961,"stop_reason":"end_turn","usage":{"input_tokens":1071,"output_tokens":716},"cost_usd":0.04651}
+{"config":"qa-health-rubric","item":"p04","split":"dev","phase":"new","sample":0,"status":"ok","value":{"clarity":5,"completeness":4,"actionability":5,"reasoning":"Categories, weights, and deductions are laid out in a single table with explicit per-item penalties or banded thresholds, weights are stated to sum to 100, a precedence rule resolves overlapping findings, and a fully worked example confirms the arithmetic and rounding. Edge cases are mostly covered: floor at 0, repeated shared elements counted once, untested categories dropped with reweighting and a 'partial' flag, unique-error definition, and explicit rounding timing. Remaining gaps are small: the unit for a 'contrast failure' (per element vs per page) and 'meaningful images' require judgment; 'distinct page' isn't defined for URL variants; a 5xx on a checkout step counts only under Availability by rule 1, which is deterministic but may surprise; and whether partial scores still receive a band is implied rather than stated. An agent can nonetheless compute a correct, reproducible score from the rubric alone."},"latency_ms":17590,"stop_reason":"end_turn","usage":{"input_tokens":1673,"output_tokens":1062},"cost_usd":0.06983}
+{"config":"qa-health-rubric","item":"f02","split":"dev","phase":"old-a","sample":0,"status":"ok","value":{"clarity":2,"completeness":2,"actionability":1,"reasoning":"The rubric has internal contradictions that make it impossible to apply correctly: Console is stated as 20% in its section header but 15% in the weights table; Links is 15% in the header but 10% in the table. The table weights sum to 125%, not 100%, so the formula cannot produce a score 'out of 100' and a perfect site would score 125, contradicting the stated claim. Beyond the weight errors, severity levels (Critical/High/Medium/Low) are undefined, 'console error' is not scoped (warnings? per page or total? does deduplication apply?), 'broken link' is undefined (4xx only? 5xx? timeouts?), and rounding is unspecified. The deduplication rule and per-category deduction values are reasonably clear, but an agent following this rubric alone cannot compute a consistent, correct score."},"latency_ms":9457,"stop_reason":"end_turn","usage":{"input_tokens":733,"output_tokens":702},"cost_usd":0.04243}
+{"config":"qa-health-rubric","item":"f01","split":"dev","phase":"old-a","sample":2,"status":"ok","value":{"clarity":3,"completeness":2,"actionability":2,"reasoning":"Category scoring rules are mostly clear and numeric (Console tiers, -15 per broken link, severity deductions with floor 0), and the dedup/primary-category ordering is a reasonable tiebreaker. However, the rubric fails at its most important step: the Final Score section gives no weights at all, instead telling the agent to 'use your judgment' about how much each category contributes. This makes the overall score non-reproducible and subjective, directly undermining the goal of a consistent numeric result. Other gaps: no definitions for Critical/High/Medium/Low severity, no guidance on what counts as a 'broken link' (soft 404s, redirects, external vs internal), no rule for when a category has no findings vs. wasn't tested, and the Console dedup key (message+source) differs from the root-cause dedup rule without reconciliation. An agent can compute per-category scores but cannot compute a correct, repeatable final score from this rubric alone."},"latency_ms":7217,"stop_reason":"end_turn","usage":{"input_tokens":722,"output_tokens":341},"cost_usd":0.02427}
+{"config":"qa-health-rubric","item":"f01","split":"dev","phase":"old-b","sample":0,"status":"ok","value":{"clarity":3,"completeness":2,"actionability":2,"reasoning":"Per-category mechanics are explicit (fixed deductions, floors, console tiers, link penalty, dedup/precedence order), but the Final Score section abandons specificity: no numeric weights are given, and the agent is told to 'use your judgment,' which makes the overall score non-reproducible. Severity levels (Critical/High/Medium/Low) are never defined, there is no guidance for untested or N/A categories, and 'broken link' and 'error' scope (e.g., warnings, redirects, soft 404s) are unspecified. An agent can compute category subscores consistently but cannot derive a single correct final score from this rubric alone."},"latency_ms":8577,"stop_reason":"end_turn","usage":{"input_tokens":722,"output_tokens":447},"cost_usd":0.02957}
+{"config":"qa-health-rubric","item":"a05","split":"dev","phase":"old-b","sample":2,"status":"ok","value":{"clarity":3,"completeness":3,"actionability":4,"reasoning":"Structure is simple and weights sum to 100, and the formula is explicit, so an agent can generally compute a score. However, the Console tiers overlap at exactly 10 errors ('4-10' and '10+' both include 10), which breaks reproducibility for that boundary. Severity levels (Critical/High/Medium/Low) are never defined, nor is what counts as a console 'error' (warnings? duplicates of the same error?) or a 'broken link' (4xx only? 5xx? timeouts?). Rounding of the final score is unspecified. The 'Each category starts at 100' rule is stated but no guidance exists for whether the same issue appearing in multiple categories is counted once or per category."},"latency_ms":10019,"stop_reason":"end_turn","usage":{"input_tokens":651,"output_tokens":643},"cost_usd":0.03866}
+{"config":"qa-health-rubric","item":"p01","split":"dev","phase":"old-a","sample":0,"status":"ok","value":{"clarity":4,"completeness":4,"actionability":4,"reasoning":"Categories, weights (summing to 100%), deduction amounts, severity tiers, dedup/primary-category ordering, floors, rounding, and the normalized formula for untested categories are all explicitly specified, so an agent can reproduce a score. Minor ambiguities remain: what threshold makes a category 'tested' vs 'untested' (and how 'coverage' is quantified for provisional labels), 'Compare only identical coverage' is cryptic, 'expected auth redirects' and 'same root cause' require judgment, and 3xx/redirect-loop handling for Links is unstated. The appended framework guidance and operational rules are unrelated to score computation and add noise without clarifying scoring."},"latency_ms":13790,"stop_reason":"end_turn","usage":{"input_tokens":1507,"output_tokens":865},"cost_usd":0.05832}
+{"config":"qa-health-rubric","item":"a03","split":"dev","phase":"old-a","sample":2,"status":"ok","value":{"clarity":3,"completeness":3,"actionability":3,"reasoning":"Deduction values, severity tiers, console/link count bands, rounding rule, and the normalization formula for untested categories are all specific and mostly unambiguous. However, the rubric has a structural defect: Content is listed as a dedup priority category and in the per-category scoring section, but has no weight in the Weights table (which sums to 95%, not 100%). An agent cannot know whether Content findings should be scored, dropped, or folded into another category, undermining reproducibility. Other gaps: 'partial' vs 'untested' coverage is not defined (what threshold makes a category tested?), 'Compare only identical coverage' is vague, severity definitions are task-oriented and map awkwardly to Performance findings, and there is no guidance on counting repeated findings of the same severity within a category after dedup. Overall computable for most cases, but the missing Content weight will produce inconsistent scores across agents."},"latency_ms":13088,"stop_reason":"end_turn","usage":{"input_tokens":1072,"output_tokens":838},"cost_usd":0.05262}
+{"config":"qa-health-rubric","item":"p04","split":"dev","phase":"old-b","sample":0,"status":"ok","value":{"clarity":4,"completeness":4,"actionability":5,"reasoning":"Categories, weights, and deductions are fully numeric, a tie-break order for overlapping findings is given, deduplication for shared elements is specified, and a worked example confirms the arithmetic and rounding. Minor ambiguities remain: 'impaired flow' and 'meaningful images' require judgment; performance band boundaries have small gaps (e.g., 2.005s falls between ≤2.0 and 2.01); and rule 1 is unclear on whether a 5xx page that also blocks checkout counts only in Availability or also as a blocked flow, since the flow blockage is arguably a separate finding. Edge cases like untested categories, floor at 0, and rounding are covered, but 'all categories untested' and missing navigation timing data are not. Overall an agent can compute a reproducible score from this alone."},"latency_ms":13963,"stop_reason":"end_turn","usage":{"input_tokens":1312,"output_tokens":988},"cost_usd":0.06252}
+{"config":"qa-health-rubric","item":"a01","split":"dev","phase":"new","sample":1,"status":"ok","value":{"clarity":4,"completeness":3,"actionability":4,"reasoning":"Categories, weights (consistent between inline and table, summing to 100%), deduction amounts, severity definitions, floor, and rounding are all explicit and mostly unambiguous. The formula with normalization by tested weights is clear. Gaps: 'tested weights' is never defined (when is a category considered untested vs. scoring 100?); only Console says to exclude defects scored elsewhere, so a single bug could be double-counted across Functional/UX/Visual/Accessibility; Performance and Accessibility have no objective thresholds (e.g., Core Web Vitals, WCAG level) so severity assignment there is subjective; Links doesn't address 3xx loops or how many retries make a failure 'repeatable'; no guidance on counting repeated instances of the same per-category issue (one finding vs. many). The framework guidance and safety rules are useful but don't affect the arithmetic. Overall an agent can compute a correct score mechanically from this rubric, with reproducibility limited mainly by classification ambiguity rather than math."},"latency_ms":14838,"stop_reason":"end_turn","usage":{"input_tokens":1720,"output_tokens":902},"cost_usd":0.0623}
+{"config":"qa-health-rubric","item":"f02","split":"dev","phase":"old-a","sample":2,"status":"ok","value":{"clarity":2,"completeness":2,"actionability":1,"reasoning":"The rubric has direct internal contradictions: Console is labeled 20% in its section header but 15% in the weights table, and Links is 15% in the header but 10% in the table. Worse, the weights table sums to 125% (15+10+15+25+20+15+10+15), so a perfect site would score 125, contradicting the stated 'perfect site scores 100.' An agent cannot compute a correct, reproducible score without guessing which weights to use and how to normalize. Beyond the weight errors, severity levels (Critical/High/Medium/Low) are never defined, 'console errors' is ambiguous (warnings? per page or deduplicated? the dedup rule doesn't say how it interacts with raw error counts), the dedup priority order is stated but criteria for 'applicable' are not, and rounding is unspecified. The per-category deduction mechanics themselves are clear, which is why clarity and completeness aren't at the floor."},"latency_ms":10810,"stop_reason":"end_turn","usage":{"input_tokens":733,"output_tokens":799},"cost_usd":0.04728}
+{"config":"qa-health-rubric","item":"p03","split":"dev","phase":"new","sample":2,"status":"ok","value":{"clarity":4,"completeness":4,"actionability":4,"reasoning":"Categories, weights, deduction values, severity tiers, dedup rules, primary-category precedence, floors, rounding, and provisional handling are all explicit, and the worked example is arithmetically correct (89.6 → 90; 79.6/0.90 = 88.44 → 88), which strongly supports reproducibility. Remaining gaps: category parentheticals (e.g., 'Accessibility (access barriers)' vs 'Functional (behavior)') are thin for boundary cases; 'reproducible'/'repeatable' is not quantified (how many retries?); severity definitions are task-centric, leaving Performance (no timing thresholds) and Accessibility (no WCAG-level mapping) findings hard to grade consistently; and partial coverage within a tested category isn't addressed. An agent can compute a correct, consistent score from this rubric alone once findings are classified, but classification of perf/a11y severity will vary between runs."},"latency_ms":17804,"stop_reason":"end_turn","usage":{"input_tokens":1964,"output_tokens":1204},"cost_usd":0.07984}
+{"config":"qa-health-rubric","item":"f04","split":"dev","phase":"new","sample":1,"status":"ok","value":{"clarity":1,"completeness":2,"actionability":1,"reasoning":"The rubric is riddled with direct contradictions that make consistent scoring impossible. The Counting section says to count every finding in every category it touches AND says 'no double deductions'; it says to count each page separately AND to deduplicate findings sharing a root cause. Console bands overlap (1-5, 3-10, 8+), so 4 or 9 errors map to two different scores. Functional and UX are given a combined 35% weight but with the instruction to deduct 'from whichever feels closer, or both,' with no definition of whether they are one category or two in the final sum. Medium issues are '-8 or -10' with no decision rule. Accessibility color issues are explicitly double-counted into Visual, again conflicting with 'no double deductions.' Weights sum to 100% only if Functional/UX is treated as a single category, which is undefined. No severity definitions for Critical/High/Medium/Low, no starting score stated for deduction categories (presumably 100), no floor stated for most categories. An agent cannot produce a reproducible score from this rubric."},"latency_ms":7522,"stop_reason":"end_turn","usage":{"input_tokens":1150,"output_tokens":373},"cost_usd":0.03015}
+{"config":"qa-health-rubric","item":"a06","split":"dev","phase":"old-b","sample":2,"status":"ok","value":{"clarity":3,"completeness":2,"actionability":2,"reasoning":"Categories and weights are clearly listed and sum to 100%, and the deduction mechanism (High = -1 letter, Medium = -0.5 letter, Polish = none) is simple. However, the rubric never defines what qualifies a finding as High vs. Medium vs. Polish impact, so the core deduction step is subjective. 'Half a letter grade' is undefined (A- vs B+ vs numeric 0.5?), and the output is a letter grade (A-F) with no numeric mapping, yet the agent must produce a score out of 100. Computing a 'weighted average' of letter grades requires an unspecified letter-to-number conversion and a rounding/rebinning rule back to A-F. The standalone AI Slop grade has no computation rule, and 'pithy verdict' is unspecified. Without severity definitions and a numeric scale, two agents would not reproduce the same score, and no agent can derive a 0-100 number from this rubric alone."},"latency_ms":10547,"stop_reason":"end_turn","usage":{"input_tokens":771,"output_tokens":684},"cost_usd":0.04191}
+{"config":"qa-health-rubric","item":"a04","split":"dev","phase":"new","sample":1,"status":"ok","value":{"clarity":4,"completeness":4,"actionability":4,"reasoning":"The rubric is largely precise: weights sum to 100% and are tabulated, per-category deduction values and severity definitions are explicit, dedup rules give a deterministic primary-category ordering, the final formula normalizes over tested weights, and rounding (0.5 up) is specified. The main flaw is the Console category (15% weight), which uses vague bins ('a few', 'several', 'many') with no numeric thresholds, so two agents could score the same error count differently, undermining reproducibility. Minor gaps: what constitutes a category being 'tested' (how much coverage qualifies) is not defined, severity tiers are task-framed and may be awkward to map onto Performance/Visual findings, and 'compare only identical coverage' is slightly unclear in intent. Otherwise an agent can compute a correct, consistent score from this document alone."},"latency_ms":13745,"stop_reason":"end_turn","usage":{"input_tokens":1432,"output_tokens":943},"cost_usd":0.06147}
+{"config":"qa-health-rubric","item":"p01","split":"dev","phase":"new","sample":0,"status":"ok","value":{"clarity":4,"completeness":3,"actionability":4,"reasoning":"Categories, weights (sum to 100%), per-finding deductions, count-based thresholds for Console/Links, a priority order for primary-category assignment, a floor, a final weighted formula normalized by tested weights, and a rounding rule are all stated explicitly, so an agent can reproducibly compute a score. Weaknesses: 'tested' vs 'untested' is never operationally defined (how much coverage counts as tested?), 'Compare only identical coverage' and the 'provisional with coverage' labeling are vague, and severity mapping is awkward for some categories—Performance has no latency/CLS thresholds, Accessibility barriers for a subset of users don't fit the 'all users' Critical definition cleanly, and Visual/Content findings will almost always collapse to Low. Handling of intermittent/non-reproducible issues and findings with multiple root causes on one page is only implicit. The trailing framework guidance and operational rules are useful context but unrelated to score computation. Overall solid and usable, with moderate gaps in boundary definitions."},"latency_ms":17882,"stop_reason":"end_turn","usage":{"input_tokens":1868,"output_tokens":1050},"cost_usd":0.07118}
+{"config":"qa-health-rubric","item":"f01","split":"dev","phase":"old-b","sample":2,"status":"ok","value":{"clarity":3,"completeness":2,"actionability":2,"reasoning":"The per-category deduction amounts, console tiers, and link penalties are clearly specified, and the deduplication/priority ordering is explicit. However, the Final Score section fails on the core requirement: it gives no numeric weights, only vague guidance ('weight them accordingly', 'use your judgment'), making the final score non-reproducible across runs or agents. Severity definitions (critical/high/medium/low) are not provided, so the same finding could be scored differently. Edge cases like rounding, how Console/Links interact with the 'first applicable' category ordering when a finding spans categories, and whether categories with zero findings should be included in the weighted average are undefined. An agent can compute category scores but cannot reliably compute a correct or consistent overall score from this rubric alone."},"latency_ms":8862,"stop_reason":"end_turn","usage":{"input_tokens":722,"output_tokens":283},"cost_usd":0.02137}
+{"config":"qa-health-rubric","item":"a06","split":"dev","phase":"old-a","sample":2,"status":"ok","value":{"clarity":3,"completeness":2,"actionability":2,"reasoning":"Categories, weights (summing to 100%), and per-letter descriptions are stated clearly, and the finding-to-grade-drop rule is simple. However, the rubric is letter-based and never defines a numeric mapping, so a 'score out of 100' cannot be computed; 'half a letter grade' is undefined (A-? B+? is a B-minus a thing?), there is no rule for converting the weighted average back to A-F or handling fractional results, and 'High/Medium/Polish' impact levels are referenced but never defined. The categories themselves (e.g., what counts as a 'Visual Hierarchy' vs 'Spacing & Layout' finding) have no criteria, and the 'pithy verdict' for AI Slop is unspecified. Two agents applying this would likely produce different results, so reproducibility is weak."},"latency_ms":10901,"stop_reason":"end_turn","usage":{"input_tokens":771,"output_tokens":645},"cost_usd":0.03996}
+{"config":"qa-health-rubric","item":"a02","split":"dev","phase":"old-a","sample":0,"status":"ok","value":{"clarity":4,"closeness":null,"completeness":3,"actionability":4,"reasoning":"Weights sum to 100%, the final formula with tested-weight normalization and rounding rule is explicit, Console/Links thresholds are concrete, and the dedup priority order resolves category overlap. Main gaps: Critical/High/Medium/Low severities are never defined, so per-category deductions (the bulk of the score) depend on unguided judgment and will not be reproducible across agents; 'untested' vs 'tested' is not defined (e.g., partially tested categories); 'provisional with coverage' and 'compare only identical coverage' are terse without format or thresholds; category boundary definitions are one-word hints. Arithmetic is fully actionable, but consistency suffers from the missing severity criteria."},"latency_ms":11734,"stop_reason":"end_turn","usage":{"input_tokens":952,"output_tokens":830},"cost_usd":0.05102}
+{"config":"qa-health-rubric","item":"p02","split":"dev","phase":"old-a","sample":1,"status":"ok","value":{"clarity":4,"completeness":4,"actionability":4,"reasoning":"Weights sum to 100% and are given with explicit decimal conversion; deduction values, floors, count bands, rounding rule, and the normalization formula for untested categories are all unambiguous. The 'first applicable' primary-category ordering cleanly resolves double-counting. Remaining ambiguities: 'root cause' deduplication and severity assignment (Critical vs High for 'core' issues) still require judgment; 'Compare only identical coverage' is cryptic; the boundary between 'untested' and 'partial/provisional' coverage is undefined; Performance/Accessibility have no thresholds for what constitutes a finding; link 'timeouts' have no duration threshold. No worked example is provided, which slightly hurts reproducibility, but an agent can compute a correct score from the formula once findings are classified."},"latency_ms":15026,"stop_reason":"end_turn","usage":{"input_tokens":1081,"output_tokens":916},"cost_usd":0.05661}
+{"config":"qa-health-rubric","item":"p03","split":"dev","phase":"new","sample":1,"status":"ok","value":{"clarity":5,"completeness":4,"actionability":5,"reasoning":"Categories, deduction values, weights, and the formula are explicit and internally consistent (the Links floor matches 7×15>100; the worked example arithmetic checks out including the provisional recomputation). A clear precedence order prevents double-counting, severity tiers have mutually exclusive definitions with a tie-break rule, and rounding/precision rules are stated. Edge cases are well covered: deduplication by root cause, untested categories with renormalized weights and 'provisional' labeling, the no-categories-tested case, floors, and cross-run comparability. Remaining gaps are minor: 'a check ran to completion' is not defined, 'reproducible' console error is not operationalized (how many repro attempts), and Performance/Accessibility findings must be mapped onto task-impact severity without category-specific thresholds (e.g., load-time cutoffs), leaving some judgment variance. An agent can nonetheless compute a correct, reproducible score from this document alone."},"latency_ms":17126,"stop_reason":"end_turn","usage":{"input_tokens":1964,"output_tokens":1064},"cost_usd":0.07284}
+{"config":"qa-health-rubric","item":"p01","split":"dev","phase":"old-b","sample":0,"status":"ok","value":{"clarity":4,"completeness":3,"actionability":4,"reasoning":"Categories, weights, per-finding deductions, floors, rounding, and the weighted-average formula (normalized by tested weights) are explicit, and the primary-category priority order resolves cross-category deduplication. Severity tiers are defined with reasonably distinct boundaries. Gaps remain: 'tested' vs 'untested' vs 'partial' is never operationalized (how much coverage makes a category tested, how to express 'provisional with coverage'); Performance has no concrete thresholds (e.g., what LCP/TTI or timeout counts as Medium vs High); link 'timeout' duration and 'repeatable' are undefined; Accessibility lacks any mapping from WCAG criteria to severity, so two agents may classify the same finding differently. The framework guidance and browser rules are operational noise that don't affect scoring. An agent can compute a reproducible score once findings are classified, but classification consistency is the weak point."},"latency_ms":15158,"stop_reason":"end_turn","usage":{"input_tokens":1507,"output_tokens":950},"cost_usd":0.06257}
+{"config":"qa-health-rubric","item":"p01","split":"dev","phase":"new","sample":2,"status":"ok","value":{"clarity":4,"completeness":3,"actionability":4,"reasoning":"Categories, weights (sum to 100%), deduction amounts, severity tiers, dedup precedence order, floors, and the normalized weighted-average formula with explicit rounding are all clearly stated, making the arithmetic fully reproducible. Minor ambiguities remain: 'first applicable' category assignment relies on judgment, 'Compare only identical coverage' is terse, and 'reproducible'/'repeatable' lack a repetition count. Completeness gaps: no definition of what makes a category 'tested' vs untested or how to express 'coverage' for provisional scores; no metric thresholds for Performance or Accessibility severity (e.g., LCP/CLS, WCAG level mapping); Visual issues almost always map to Low by the severity definitions, which may be unintended; no guidance on mixed-severity root causes spanning categories beyond precedence. The framework guidance and Important Rules sections are useful operationally but tangential to scoring. Overall an agent can compute a correct score from the rubric, with severity/categorization judgment being the main source of variance."},"latency_ms":16219,"stop_reason":"end_turn","usage":{"input_tokens":1868,"output_tokens":1055},"cost_usd":0.07143}
+{"config":"qa-health-rubric","item":"a04","split":"dev","phase":"old-b","sample":0,"status":"ok","value":{"clarity":4,"completeness":3,"actionability":4,"reasoning":"Weights sum to 100%, the formula with renormalization over tested weights is explicit, severity tiers have concrete definitions, deduction amounts and floors are stated, and rounding is specified. The main weakness is the Console category, whose bands ('a few', 'several', 'many') lack numeric thresholds, undermining reproducibility for 15% of the score. Other gaps: no definition of what constitutes a Performance finding or a link 'timeout', 'reproducible' is not quantified, and 'provisional with coverage' for partially tested categories lacks concrete guidance. Despite these, an agent can compute a score end-to-end from the rubric alone, with console being the only materially inconsistent part."},"latency_ms":11096,"stop_reason":"end_turn","usage":{"input_tokens":1071,"output_tokens":757},"cost_usd":0.04856}
+{"config":"qa-health-rubric","item":"f05","split":"dev","phase":"old-a","sample":1,"status":"ok","value":{"clarity":1,"completeness":1,"actionability":1,"reasoning":"The rubric defines no scoring categories, no weights, no per-issue deduction amounts, and no severity tiers—only a vague instruction to 'take off more for worse problems.' It provides a single anchor (clean site scores in the 90s) but leaves the starting baseline, the ceiling (why not 100?), and the floor undefined. Edge cases like duplicate issues, issues of mixed severity, or how 'anything that looks wrong' is bounded are unaddressed. An agent cannot compute a reproducible number from this; two runs on the same site could plausibly produce 60 and 85. It is a vague directive, not a scoring rubric."},"latency_ms":3930,"stop_reason":"end_turn","usage":{"input_tokens":397,"output_tokens":244},"cost_usd":0.01617}
+{"config":"qa-health-rubric","item":"p02","split":"dev","phase":"old-b","sample":1,"status":"ok","value":{"clarity":4,"completeness":4,"actionability":5,"reasoning":"Categories, severity tiers, deduction amounts, count thresholds, and weights (summing to 100%) are explicit, with a clear priority order for root-cause categorization and an unambiguous final-score formula including renormalization for untested categories and a rounding rule. Minor gaps: 'tested' vs 'untested' is not operationally defined (e.g., partial coverage within a category), 'reproducible' and 'same root cause' require judgment, and Performance lacks concrete thresholds for what constitutes a finding. Severity assignment inherently needs judgment, but 'highest applicable severity' and the impact-based definitions make it reproducible enough that an agent can compute a correct score from the rubric alone."},"latency_ms":13681,"stop_reason":"end_turn","usage":{"input_tokens":1081,"output_tokens":860},"cost_usd":0.05381}
+{"config":"qa-health-rubric","item":"f05","split":"dev","phase":"old-a","sample":2,"status":"ok","value":{"clarity":1,"completeness":1,"actionability":1,"reasoning":"The rubric defines no scoring categories, no weights, no per-issue deduction amounts, and no severity tiers—only a vague instruction to 'take off more for worse problems.' It is internally inconsistent (a flawless site scores 'in the 90s' rather than 100, with no explanation of what the missing points represent). No edge cases are addressed (duplicate issues, caps per category, floor at 0, how to treat warnings vs. errors, cosmetic vs. functional). An agent cannot derive a reproducible number from this; two runs on the same site would almost certainly yield different scores. The only concrete guidance is where to place the score in the report."},"latency_ms":4133,"stop_reason":"end_turn","usage":{"input_tokens":397,"output_tokens":243},"cost_usd":0.01612}
+{"config":"qa-health-rubric","item":"a02","split":"dev","phase":"new","sample":1,"status":"ok","value":{"clarity":4,"completeness":3,"actionability":4,"reasoning":"Weights sum to 100%, the formula with renormalization over tested weights and the rounding rule are explicit, and the first-applicable category ordering plus deduplication rules are stated clearly. Console and Links bands are unambiguous. The main gaps: Critical/High/Medium/Low severity is never defined, so the per-category deductions (which drive 75% of the score) depend on agent judgment and undermine reproducibility; 'root cause' deduplication, 'reproducible', and what constitutes a 'tested' vs 'untested' or 'partial' category are not operationalized; Performance has no thresholds for what counts as a finding; and 'compare only identical coverage' and 'label provisional with coverage' lack concrete output formats. An agent can mechanically compute a score from this, but two agents could diverge on severity assignment."},"latency_ms":11067,"stop_reason":"end_turn","usage":{"input_tokens":1313,"output_tokens":857},"cost_usd":0.05598}
+{"config":"qa-health-rubric","item":"a02","split":"dev","phase":"new","sample":0,"status":"ok","value":{"clarity":4,"completeness":3,"actionability":4,"reasoning":"Weights, deduction values, tier thresholds, floors, the untested-category normalization formula, and rounding are all explicit and mostly unambiguous, with a clear precedence order for deduplicating root causes. The main gap is that Critical/High/Medium/Low severity is never defined, yet six of eight categories depend on it, which undermines reproducibility. Other undefined boundaries: what counts as a 'tested' vs partially tested category, how to express 'coverage' on provisional scores, timeout thresholds for links, and how many repro attempts make an error 'reproducible'. Given severity-classified findings, an agent can mechanically compute a correct score, but consistency across agents is not guaranteed by the rubric alone."},"latency_ms":10275,"stop_reason":"end_turn","usage":{"input_tokens":1313,"output_tokens":752},"cost_usd":0.05073}
+{"config":"qa-health-rubric","item":"a02","split":"dev","phase":"old-b","sample":2,"status":"ok","value":{"clarity":4,"completeness":3,"actionability":4,"reasoning":"Weights sum to 100, thresholds, floors, dedup rules, category precedence order, and the normalization formula with explicit rounding are all unambiguous, so an agent can mechanically compute a score once findings are categorized. The main gap is that Critical/High/Medium/Low severity is never defined for the six per-category deductions, which drives most of the score and will cause inconsistent results across runs. Also undefined: what qualifies a category as 'tested' vs 'untested' (partial coverage threshold), what 'reproducible'/'repeatable' means in practice (how many attempts), and the terse 'compare only identical coverage' guidance. Otherwise edge cases (0 floors, dedup, exclusions, no-tested-categories case) are reasonably covered."},"latency_ms":11310,"stop_reason":"end_turn","usage":{"input_tokens":952,"output_tokens":761},"cost_usd":0.04757}
+{"config":"qa-health-rubric","item":"a05","split":"dev","phase":"new","sample":0,"status":"ok","value":{"clarity":3,"completeness":2,"actionability":3,"reasoning":"Weights sum to 100 and the arithmetic formula is simple, so an agent can mechanically compute a score once findings are classified. However, several ambiguities undermine reproducibility: Console bands '4-10' and '10+' overlap at exactly 10 errors; 'errors' is undefined (do warnings, repeated identical errors, or third-party script errors count?); 'broken link' is undefined (4xx only? 5xx? timeouts? redirects? external vs internal?). The severity levels Critical/High/Medium/Low have no definitions, leaving the largest driver of the score (six categories, 75% of weight) to subjective judgment. No guidance on deduplication of findings, how to handle a category that wasn't tested or doesn't apply, rounding of the final score, or Performance thresholds (e.g., load time cutoffs). The rubric is usable but will produce inconsistent scores across runs."},"latency_ms":9556,"stop_reason":"end_turn","usage":{"input_tokens":1012,"output_tokens":693},"cost_usd":0.04477}
+{"config":"qa-health-rubric","item":"a06","split":"dev","phase":"old-b","sample":0,"status":"ok","value":{"clarity":3,"completeness":2,"actionability":2,"reasoning":"The rubric clearly lists 10 categories with weights summing to 100% and gives readable A–F descriptors, but key terms are undefined: what constitutes a High vs. Medium vs. Polish finding, what a 'half letter grade' is (A- or B+? how many halves before the next full drop?), and what each category actually covers. Critically, there is no mapping from letter grades to numbers, so a 'weighted average' of letters cannot be computed and no numeric score out of 100 can be produced without the agent inventing a conversion scale. Also unspecified: rounding, how the standalone AI Slop grade is derived vs. its 5% component, what the 'pithy verdict' is, and how to handle categories that are not applicable (e.g., no motion on the site). Two agents following this rubric would likely produce different numeric results."},"latency_ms":10576,"stop_reason":"end_turn","usage":{"input_tokens":771,"output_tokens":642},"cost_usd":0.03981}
+{"config":"qa-health-rubric","item":"p05","split":"dev","phase":"old-b","sample":1,"status":"ok","value":{"clarity":4,"completeness":4,"actionability":5,"reasoning":"Categories use an explicit first-match priority order, severity levels are defined with distinct boundaries, weights are listed and sum to 1.00, and every formula (including clamping to 0, renormalization over tested categories, and floor(x+0.5) rounding) is written as code, so an agent can compute a deterministic score from a finding list. Edge cases for untested categories, empty tested sets, and provisional labeling are handled. Minor gaps keep it from perfect: 'repeatable' 4xx/5xx and 'slow' (Performance) have no quantitative thresholds, so classification of borderline findings may vary between runs; it is slightly unclear whether a 404'd image is Links or Visual; and 'tested' is defined only by example. These affect finding classification consistency more than the arithmetic itself."},"latency_ms":17831,"stop_reason":"end_turn","usage":{"input_tokens":1193,"output_tokens":1078},"cost_usd":0.06583}
+{"config":"qa-health-rubric","item":"f05","split":"dev","phase":"old-b","sample":2,"status":"ok","value":{"clarity":1,"completeness":1,"actionability":1,"reasoning":"The rubric defines no scoring categories, no weights, no deduction amounts, and no severity tiers—only vague guidance to 'take off points' and 'take off more for worse problems.' The only concrete anchor is that a problem-free site scores 'in the 90s,' which itself is ambiguous (why not 100?) and leaves the entire 0–89 range undefined. Edge cases (e.g., duplicate errors, third-party console noise, warnings vs. errors, what counts as 'looks wrong') are unaddressed. An agent cannot derive a reproducible numeric score from this; two agents testing the same site would produce arbitrarily different results."},"latency_ms":3347,"stop_reason":"end_turn","usage":{"input_tokens":397,"output_tokens":246},"cost_usd":0.01627}
+{"config":"qa-health-rubric","item":"a03","split":"dev","phase":"new","sample":2,"status":"ok","value":{"clarity":4,"completeness":3,"actionability":3,"reasoning":"Categories, severity tiers, console buckets, link deductions, dedup priority order, floor, and rounding rule are all stated concretely, and the normalization formula handles untested categories. However, there is a material defect: Content appears in the per-category scoring list and the dedup priority order but has no weight in the Weights table, and the listed weights sum to 95%, not 100%. An agent cannot determine whether Content is scored at all, what weight to use, or whether the 95% total is intentional (relying on normalization) or an omission—this directly undermines reproducibility. Smaller gaps: 'untested' is not operationally defined, severity mapping for Performance findings (e.g., slow load vs. 'task blocked') is left to judgment, 'Compare only identical coverage' and provisional labeling format are vague, and 'reproducible'/'repeatable' thresholds for console errors and link failures are unspecified. Mostly computable, but the Content weight gap would produce inconsistent scores across agents."},"latency_ms":13138,"stop_reason":"end_turn","usage":{"input_tokens":1433,"output_tokens":929},"cost_usd":0.06078}
+{"config":"qa-health-rubric","item":"a03","split":"dev","phase":"old-a","sample":0,"status":"ok","value":{"clarity":3,"completeness":3,"actionability":3,"reasoning":"Strengths: explicit deduction tables, severity definitions with impact/workaround criteria, a deduplication priority order, a normalization formula for untested categories, and a rounding rule. Key flaw: Content is listed as a scorable per-category (and in the dedup priority order) but is absent from the weights table, which sums to only 95%; an agent cannot know whether to drop Content, assign it a weight, or treat the 5% gap as intentional, undermining reproducibility. Other gaps: 'tested' vs 'untested' is never defined, Performance and Accessibility lack any criteria for what constitutes a finding (no thresholds or WCAG reference), 'timeouts' and 'expected auth redirects' lack thresholds/definitions, and 'compare only identical coverage' is vague. Severity classification (e.g., Medium vs Low for friction) will also vary between agents. The score is computable for most categories but the Content/weight inconsistency prevents a fully correct, consistent result from the rubric alone."},"latency_ms":15125,"stop_reason":"end_turn","usage":{"input_tokens":1072,"output_tokens":1029},"cost_usd":0.06217}
+{"config":"qa-health-rubric","item":"a06","split":"dev","phase":"new","sample":2,"status":"ok","value":{"clarity":3,"completeness":2,"actionability":2,"reasoning":"Strengths: the 10 categories are named, weights are explicit and sum to 100%, and the deduction mechanic (A start, High = -1 letter, Medium = -0.5 letter, Polish = no effect, floor at F) is simple. Weaknesses: the task requires a numeric score out of 100, but the rubric only yields A–F letters with no letter-to-number mapping, so a 'weighted average' of letter grades cannot actually be computed without inventing a scale (A=4? A=100? A=90-100?). 'Half a letter grade' is undefined (A-? B+? 3.5?) and there is no rounding rule for converting the weighted average back to a letter. High/Medium/Polish impact levels are never defined, so the core deduction trigger is left to judgment. The qualitative per-category descriptions (A = 'delightful', C = 'generic') provide a second, potentially conflicting grading path alongside the count-based computation, with no rule for which wins. The AI Slop standalone grade's 'pithy verdict' has no criteria. Edge cases like many Medium findings exceeding the F floor, an E grade gap, or zero findings in a category with no evaluable content are unaddressed. Net: an agent could not produce a consistent, reproducible 0–100 score from this rubric alone."},"latency_ms":18334,"stop_reason":"end_turn","usage":{"input_tokens":1132,"output_tokens":963},"cost_usd":0.05947}
+{"config":"qa-health-rubric","item":"p03","split":"dev","phase":"old-b","sample":2,"status":"ok","value":{"clarity":5,"completeness":4,"actionability":5,"reasoning":"Categories, weights (summing to 100%), deduction values, and the final formula are explicit and the worked example is arithmetically correct, including the provisional/untested case and rounding rule. Deduplication, primary-category ordering, severity tie-breaking, floors, and the 'no bonus points' rule remove most ambiguity. Minor gaps remain: severity boundaries for Performance and Visual are judgment-based (no thresholds like LCP limits), 'repeatable'/'reproducible' lacks a retry count, third-party script errors and what constitutes a 'check' aren't defined, and Low vs Medium ('friction' vs 'impaired') has a gray zone. Given a list of findings, an agent can compute a reproducible score from this rubric alone."},"latency_ms":19905,"stop_reason":"end_turn","usage":{"input_tokens":1603,"output_tokens":1486},"cost_usd":0.09033}
+{"config":"qa-health-rubric","item":"a06","split":"dev","phase":"old-a","sample":0,"status":"ok","value":{"clarity":3,"completeness":2,"actionability":2,"reasoning":"Weights are explicit and sum to 100%, and the basic deduction rule (High = one letter, Medium = half letter, Polish = none) is easy to state. However, key terms are undefined: 'half a letter grade' has no representation on an A-F-only scale (no +/- grades specified), High/Medium/Polish impact tiers are never defined, and the per-category grade descriptors (A-F) conflict with the mechanical deduction rule without saying which governs. Critically, the rubric never gives a numeric mapping for letters (e.g., A=100, B=80), so 'weighted average of all 10 categories' cannot actually be computed, nor can the result be converted back to a headline letter or expressed as a score out of 100 as the task requires. Rounding rules, how D drops (to F directly?), whether half-grades round up or down, and how the standalone AI Slop grade/verdict is derived are all missing. An agent could not produce a consistent, reproducible numeric score from this alone."},"latency_ms":13289,"stop_reason":"end_turn","usage":{"input_tokens":771,"output_tokens":818},"cost_usd":0.04861}
+{"config":"qa-health-rubric","item":"p02","split":"dev","phase":"new","sample":1,"status":"ok","value":{"clarity":4,"completeness":4,"actionability":4,"reasoning":"Categories, weights (summing to 100%), per-finding deductions, severity tiers, floors, and the final weighted formula with an explicit rounding rule are all stated unambiguously, and a precedence order resolves cross-category assignment. Remaining ambiguities: 'reproducible'/'repeatable' thresholds are undefined, 'same root cause' dedup requires judgment, the severity ladder is task-oriented and maps awkwardly to Performance/Visual findings (e.g., how slow is Medium vs Low), and 'tested' vs 'untested' coverage criteria and the meaning of 'provisional with coverage' / 'compare only identical coverage' are not operationalized. Edge cases like floor 0, no categories tested, and partial coverage renormalization are handled, but what constitutes sufficient testing for a category to count (and thus score 100 with no findings) is missing. An agent can compute a reproducible score from this rubric alone, with variance mainly arising from severity classification rather than arithmetic."},"latency_ms":16640,"stop_reason":"end_turn","usage":{"input_tokens":1442,"output_tokens":1029},"cost_usd":0.06587}
+{"config":"qa-health-rubric","item":"a06","split":"dev","phase":"old-b","sample":1,"status":"ok","value":{"clarity":3,"completeness":2,"actionability":2,"reasoning":"Categories, letter descriptors, and weights (which correctly sum to 100%) are clearly stated, but key mechanics are ambiguous: 'half a letter grade' has no defined representation (A-? B+? a numeric step?), and 'weighted average' of letter grades is undefined without a letter-to-number mapping. The rubric never defines what qualifies as a High vs Medium vs Polish impact finding, so deductions are not reproducible across agents. Most critically, the task requires a numeric score out of 100, yet the rubric only yields A–F letters with no conversion scale, so an agent cannot compute the required numeric score from this document alone. Edge cases like grades below F after many deductions, rounding of half-steps, or how the standalone AI Slop grade interacts with its 5% weight are also unaddressed."},"latency_ms":11229,"stop_reason":"end_turn","usage":{"input_tokens":771,"output_tokens":654},"cost_usd":0.04041}
+{"config":"qa-health-rubric","item":"p01","split":"dev","phase":"old-a","sample":1,"status":"ok","value":{"clarity":4,"completeness":3,"actionability":4,"reasoning":"Categories, weights (sum to 100%), deduction amounts, floors, and the final formula with explicit rounding and tested-weight normalization are clear and mostly unambiguous. The dedup/primary-category precedence order and severity ladder are well specified. Gaps: 'tested' vs 'untested' category is never defined (what minimum coverage counts?), 'reproducible' console error and link 'timeout' thresholds are undefined, severity mapping for Performance/Visual findings (e.g., what makes a slow page 'High' vs 'Medium') is left to judgment, and 'Compare only identical coverage' is terse. Trailing framework guidance and operational rules (BROWSER SETUP, 'LOOK is not ACT') reference external context and add noise but don't affect score computation. An agent can compute a correct score from the formula, though severity judgment calls may reduce reproducibility."},"latency_ms":14845,"stop_reason":"end_turn","usage":{"input_tokens":1507,"output_tokens":932},"cost_usd":0.06167}
+{"config":"qa-health-rubric","item":"p04","split":"dev","phase":"old-b","sample":2,"status":"ok","value":{"clarity":4,"completeness":4,"actionability":5,"reasoning":"Categories, weights, deduction amounts, and the combination formula are explicit, with a tie-break rule for overlapping findings, a floor at 0, a defined rounding step, an untested-category procedure, and a worked example that an agent can reproduce. Remaining ambiguities are mostly judgment calls rather than structural gaps: 'impaired flow' vs 'blocked' hinges on what counts as a workaround, 'meaningful images' and 'distinct page' are undefined, failed-request uniqueness by 'message' is unclear for HTTP errors, and Performance is undefined if the slowest page never loads. Edge cases not covered include default flows that don't exist on the site (e.g., no checkout), flows that are blocked solely because a page fails Availability, and whether contrast failures count per element or per shared style. None of these prevent computing a score, but they could produce small inconsistencies between runs."},"latency_ms":17822,"stop_reason":"end_turn","usage":{"input_tokens":1312,"output_tokens":1233},"cost_usd":0.07477}
+{"config":"qa-health-rubric","item":"a01","split":"dev","phase":"old-b","sample":2,"status":"ok","value":{"clarity":4,"completeness":3,"actionability":4,"reasoning":"Categories, tiers, deduction amounts, and weights (summing to 100%) are explicit, and the final formula with rounding rule is unambiguous, so an agent can mechanically compute a score from a list of findings. Gaps: the severity definitions (data loss, task blocked, workaround) are task-centric and map poorly to Performance and Accessibility findings, leaving severity assignment there subjective; 'Σ tested weights' is used but what qualifies a category as 'tested' vs. untested is never defined; 'reproducible'/'repeatable' thresholds for console errors and link failures are unspecified; and cross-category double-counting is only addressed for Console (not e.g. Functional vs. UX). The appended framework guidance and operational rules are irrelevant to scoring and add noise without harming the arithmetic."},"latency_ms":19236,"stop_reason":"end_turn","usage":{"input_tokens":1359,"output_tokens":939},"cost_usd":0.06054}
+{"config":"qa-health-rubric","item":"f04","split":"dev","phase":"old-b","sample":0,"status":"ok","value":{"clarity":1,"completeness":1,"actionability":1,"reasoning":"The rubric is riddled with direct contradictions and ambiguities. Counting rules conflict: 'count every finding in every category it touches' vs 'no double deductions'; 'count each page separately' vs 'deduplicate findings that share a root cause'. Console error bands overlap (1-5, 3-10, 8+), so 3-5 or 8-10 errors map to two different scores. Medium issues deduct '-8 or -10' with no rule for choosing. Functional and UX are a single 35% weight but scored by 'whichever feels closer, or both', with no guidance on whether they are one combined category score or two averaged scores. Severity levels (Critical/High/Medium/Low) are never defined. Accessibility/Visual overlap is introduced without resolving double-counting. No floor is stated for most categories, no rounding rule, and no handling of unapplicable categories. An agent cannot derive a reproducible score; two agents following this rubric would routinely produce different numbers."},"latency_ms":8631,"stop_reason":"end_turn","usage":{"input_tokens":789,"output_tokens":557},"cost_usd":0.03574}
+{"config":"qa-health-rubric","item":"a04","split":"dev","phase":"old-a","sample":1,"status":"ok","value":{"clarity":4,"completeness":3,"actionability":4,"reasoning":"Weights are explicit, sum to 100%, and the final formula with tested-weight normalization and rounding rule is unambiguous. Severity tiers have concrete definitions and fixed deductions, the primary-category precedence order prevents double counting, and Links has a precise per-item deduction. The main weakness is the Console category: 'a few / several / many' are undefined, so two agents could bucket the same error count differently, undermining reproducibility for a 15%-weight category. Other gaps: no numeric thresholds for what counts as a Performance finding or 'repeatable' timeout, no definition of when a category is 'tested' vs 'untested' (e.g., partial coverage), and no guidance on mapping accessibility violations to severity tiers. An agent can compute a correct score for most categories from this alone, but Console scoring will be inconsistent."},"latency_ms":10784,"stop_reason":"end_turn","usage":{"input_tokens":1071,"output_tokens":732},"cost_usd":0.04731}
+{"config":"qa-health-rubric","item":"p03","split":"dev","phase":"old-b","sample":0,"status":"ok","value":{"clarity":5,"completeness":4,"actionability":5,"reasoning":"Categories, deduction amounts, weights, and precedence order for categorizing findings are explicit and unambiguous; rounding, floors, deduplication, and no-double-deduction rules are spelled out. Edge cases like untested categories, 'not scored', expected auth redirects, timeouts, and provisional labeling are handled, and the worked example (including the provisional variant) lets an agent verify its arithmetic. Minor completeness gaps: severity tiers still require judgment (what counts as a 'core task'; Medium 'impaired' vs Low 'friction' can overlap), 'reproducible'/'repeatable' lacks a retry count, and Performance/Accessibility have no concrete thresholds for what constitutes a finding, so inter-run consistency depends partly on tester judgment rather than the rubric alone."},"latency_ms":14450,"stop_reason":"end_turn","usage":{"input_tokens":1603,"output_tokens":959},"cost_usd":0.06398}
+{"config":"qa-health-rubric","item":"a01","split":"dev","phase":"new","sample":2,"status":"ok","value":{"clarity":4,"completeness":3,"actionability":4,"reasoning":"Categories, weights (sum to 100%), count-based tiers for Console/Links, and severity deductions are explicit, and the final formula with rounding rule is well specified. Ambiguities remain: 'defects scored elsewhere' for Console exclusion is undefined, 'Σ tested weights' implies categories can be untested but no criteria define when a category counts as untested or how to record it, and nothing prevents one finding from being deducted in multiple per-category buckets (e.g., Visual vs UX vs Accessibility). 'Reproducible/repeatable' lacks a threshold (how many retries), Performance/Accessibility findings must be force-fit into task-impact severity tiers with no guidance (e.g., thresholds for LCP/CLS or WCAG level mapping), and framework guidance lists symptoms without mapping them to severity. The trailing operational rules (browser usage, redaction, AskUserQuestion) are unrelated to scoring and add noise. Overall an agent can compute a plausible, mostly reproducible score, but category assignment and untested-category handling will cause variance."},"latency_ms":14700,"stop_reason":"end_turn","usage":{"input_tokens":1720,"output_tokens":888},"cost_usd":0.0616}
+{"config":"qa-health-rubric","item":"p05","split":"dev","phase":"new","sample":0,"status":"ok","value":{"clarity":4,"completeness":4,"actionability":5,"reasoning":"Strong rubric: categories use a first-match precedence rule that resolves overlaps, severity has four defined tiers, every formula is explicit (including tier boundaries, floor-based rounding, and weights that are stated to sum to 1.00), and untested categories are handled via renormalization rather than defaulting to 100, with a 'not scored' fallback and provisional labeling. Minor clarity gaps: 'repeatable' 4xx/5xx and 'slow' performance have no concrete thresholds, and 'same root cause' deduplication requires judgment. Completeness gaps: no definition of partially tested categories, reliance on external context ('Phase 5', 'Quick mode'), and no guidance on whether a category with zero findings counts as tested only if its checks ran (implied but not stated for console). Despite these, the computational pipeline is fully specified and an agent can produce a reproducible score from the rubric alone."},"latency_ms":18301,"stop_reason":"end_turn","usage":{"input_tokens":1554,"output_tokens":1186},"cost_usd":0.07484}
+{"config":"qa-health-rubric","item":"f02","split":"dev","phase":"old-b","sample":1,"status":"ok","value":{"clarity":2,"completeness":2,"actionability":1,"reasoning":"The rubric is internally contradictory: Console is listed as 20% in its section header but 15% in the weights table, and Links is 15% vs 10%. Worse, the weights table sums to 125%, so a perfect site would score 125, directly contradicting the stated 'out of 100 / perfect site scores 100' claim. An agent cannot compute a correct, reproducible score without guessing which weights to use and how to normalize. Beyond the arithmetic failures, severity levels (Critical/High/Medium/Low) are never defined, 'console error' is undefined (warnings? unique vs. per-page?), broken-link counting (per unique URL vs. per occurrence) is unclear, and 'same root cause' deduplication has no operational test. The structure (category floors, per-finding deductions, priority ordering for dedup) is a reasonable skeleton, but the weight inconsistencies make the rubric unusable as written."},"latency_ms":10392,"stop_reason":"end_turn","usage":{"input_tokens":733,"output_tokens":783},"cost_usd":0.04648}
+{"config":"qa-health-rubric","item":"f05","split":"dev","phase":"old-a","sample":0,"status":"ok","value":{"clarity":1,"completeness":1,"actionability":1,"reasoning":"The rubric defines no scoring categories, no weights, and no specific deduction amounts. 'Take off more for worse problems' gives no severity tiers or point values, and 'anything that looks wrong' is entirely subjective. It never explains why a flawless site should score in the 90s rather than 100, nor defines any lower boundaries (e.g., what score a critically broken site gets). Two agents testing the same site would produce wildly different scores, so it fails the consistency/reproducibility requirement completely. An agent cannot compute a correct, defensible score from this text alone."},"latency_ms":3531,"stop_reason":"end_turn","usage":{"input_tokens":397,"output_tokens":218},"cost_usd":0.01487}
+{"config":"qa-health-rubric","item":"f04","split":"dev","phase":"old-a","sample":0,"status":"ok","value":{"clarity":1,"completeness":1,"actionability":1,"reasoning":"The rubric is riddled with direct contradictions and ambiguities. Counting rules contradict themselves: 'count every finding in every category it touches' vs 'no double deductions'; 'count each page separately' vs 'deduplicate findings that share a root cause'. Console error bands overlap (1-5, 3-10, 8+) so counts of 3-5 and 8-10 map to two different scores. Medium issue deduction is '-8 or -10' with no selection criterion. Functional and UX are given a combined 35% weight but the agent is told to 'deduct from whichever feels closer, or both', with no guidance on how to compute a single combined score (one bucket? two averaged? which weights?). Severity levels (critical/high/medium/low) are never defined. The accessibility/visual overlap rule compounds the double-counting contradiction. No floor stated for non-Links categories, no handling of zero-finding categories vs untested categories, and no rounding rule. Two agents following this rubric would almost certainly produce different scores from identical findings, so it fails the reproducibility goal entirely."},"latency_ms":7405,"stop_reason":"end_turn","usage":{"input_tokens":789,"output_tokens":388},"cost_usd":0.02729}
+{"config":"qa-health-rubric","item":"a02","split":"dev","phase":"new","sample":2,"status":"ok","value":{"clarity":4,"completeness":3,"actionability":4,"reasoning":"Categories, weights (sum to 100%), fixed-tier deductions for Console/Links, floors, rounding rule, and the untested-category normalization formula are all explicit and unambiguous. The dedup priority order prevents double counting. Main gaps: severity levels (Critical/High/Medium/Low) are never defined, which is the biggest threat to reproducibility across six categories; 'tested' vs 'untested' is not operationalized; 'reproducible'/'repeatable' thresholds and timeout durations are unspecified; and 'compare only identical coverage' is terse. An agent can compute a mechanically correct score from this rubric alone, but consistency between agents will vary with severity judgment."},"latency_ms":9254,"stop_reason":"end_turn","usage":{"input_tokens":1313,"output_tokens":660},"cost_usd":0.04613}
+{"config":"qa-health-rubric","item":"f02","split":"dev","phase":"new","sample":1,"status":"ok","value":{"clarity":2,"completeness":2,"actionability":1,"reasoning":"The rubric has direct internal contradictions: Console is labeled 20% in its section header but 15% in the weights table; Links is labeled 15% in its header but 10% in the table. Worse, the weights table sums to 125% (15+10+15+25+20+15+10+15), so the stated 'final score is out of 100' and 'perfect site scores 100' are mathematically false — a perfect site would score 125. An agent cannot compute a correct, reproducible score from this rubric alone. Beyond the weight errors, severity levels (Critical/High/Medium/Low) are never defined, 'console errors' is ambiguous regarding warnings and repeated identical errors, the root-cause deduplication rule gives a priority order but no criteria for determining what counts as the same root cause, and no rounding rule is specified. The per-category deduction structure and floor are clearly stated, but the core arithmetic is broken."},"latency_ms":10401,"stop_reason":"end_turn","usage":{"input_tokens":1094,"output_tokens":691},"cost_usd":0.04549}
+{"config":"qa-health-rubric","item":"p05","split":"dev","phase":"new","sample":1,"status":"ok","value":{"clarity":4,"completeness":4,"actionability":5,"reasoning":"Categories use an explicit first-match ordering, severity tiers are defined with concrete criteria, formulas are given as code with weights that sum to 1.00, and rounding (floor(score+0.5)) and the 0 floor are explicit. Step 3 correctly excludes Links and Console from severity, matching the Step 4 formulas, so there is no dangling input. Edge cases are handled: deduplication rules, untested categories (with a definition and an explicit 'never score as 100'), empty tested set → 'not scored', provisional labeling, and comparability. Remaining gaps are mostly in finding classification rather than arithmetic: 'slow' for Performance has no threshold, 'repeatable' 4xx/5xx has no retry count, and severity boundaries (e.g., 'impaired' vs 'blocked') still require judgment, so two agents could disagree on inputs. Also unstated whether a category with zero findings scores 100 (implied by the formula but not said) and how partially-run checks are treated beyond 'none ran'. Given the inputs, an agent can compute the score deterministically from this document alone."},"latency_ms":15399,"stop_reason":"end_turn","usage":{"input_tokens":1554,"output_tokens":995},"cost_usd":0.06529}
+{"config":"qa-health-rubric","item":"p01","split":"dev","phase":"new","sample":1,"status":"ok","value":{"clarity":4,"completeness":4,"actionability":4,"reasoning":"Categories, weights (summing to 100%), deduction amounts, severity tiers, floor, rounding, deduplication precedence order, and the untested-category normalization formula are all explicitly defined, so an agent can mechanically compute a score from a list of findings. Weaknesses: severity definitions are task-blockage oriented and map poorly to Performance and Accessibility (e.g., no thresholds for slow loads or guidance on how a11y barriers for a subset of users are rated), so most perf findings default ambiguously; 'tested' vs 'untested' coverage and 'provisional' labeling lack a concrete threshold; Console 'reproducible' and Links 'timeout' have no defined repro count or duration; 'same root cause' dedup and 'Compare only identical coverage' require judgment. The appended framework guidance and operational rules are unrelated to scoring and slightly dilute focus, with cryptic phrasing like 'LOOK is not ACT'. Overall solid and computable, but severity classification remains the main source of inter-run variance."},"latency_ms":18138,"stop_reason":"end_turn","usage":{"input_tokens":1868,"output_tokens":1132},"cost_usd":0.07528}
+{"config":"qa-health-rubric","item":"f01","split":"dev","phase":"old-a","sample":0,"status":"ok","value":{"clarity":2,"completeness":2,"actionability":1,"reasoning":"Per-category scoring is reasonably clear (fixed deductions by severity, console tiers, link penalties, floor at 0). However, the Final Score section provides no actual weights—it tells the agent to 'use your judgment,' which makes the overall score non-reproducible and inconsistent across runs or agents. Severity definitions (Critical/High/Medium/Low) are undefined, so classification is subjective. The deduplication rule is helpful but the 'first applicable' category list ordering could still produce ambiguity (e.g., a broken link that is also an access barrier). Edge cases like missing categories, sites with no measurable performance data, or whether Links/Console use the same weighting as other categories are unaddressed. Because the critical final-combination step is left entirely to agent discretion, an agent cannot compute a correct, consistent score from this rubric alone."},"latency_ms":7057,"stop_reason":"end_turn","usage":{"input_tokens":722,"output_tokens":312},"cost_usd":0.02282}
+{"config":"qa-health-rubric","item":"a02","split":"dev","phase":"old-b","sample":0,"status":"ok","value":{"clarity":4,"completeness":3,"actionability":4,"reasoning":"Structure is clear: weights sum to 100%, each category has explicit deduction values, the dedup/primary-category ordering is specified, the normalization formula for untested categories is given, and the rounding rule is precise. The main gap is that Critical/High/Medium/Low severities are never defined, so the largest driver of six categories (60% of weight) depends on agent judgment, hurting reproducibility. Also undefined: what qualifies a category as 'tested', what constitutes a Performance or Accessibility finding (no thresholds or WCAG level), and how to classify a finding that straddles severity levels. 'Compare only identical coverage' is terse. Given classified findings, the arithmetic is fully mechanical and unambiguous, so actionability is high despite the severity gap."},"latency_ms":12055,"stop_reason":"end_turn","usage":{"input_tokens":952,"output_tokens":856},"cost_usd":0.05232}
+{"config":"qa-health-rubric","item":"f01","split":"dev","phase":"old-b","sample":1,"status":"ok","value":{"clarity":3,"completeness":2,"actionability":2,"reasoning":"The category-level scoring is mostly clear: Console uses defined bands, Links uses a fixed per-item deduction, and the six per-category scores use explicit severity deductions with a floor. The deduplication rule and priority order for assigning root causes are also well specified. However, the rubric fails at the critical final step: it provides no actual weights, instead telling the agent to 'use your judgment' about category contributions. This makes the final score non-reproducible across runs or agents, which defeats the task's requirement for a consistent score. Additional gaps: no definition of what constitutes Critical/High/Medium/Low severity for the six categories, no guidance on how to count 'reproducible' console errors or what counts as a broken link (e.g., 404 vs timeout vs redirect), and no handling of categories with no applicable findings. An agent cannot compute a correct, deterministic score from this rubric alone."},"latency_ms":7368,"stop_reason":"end_turn","usage":{"input_tokens":722,"output_tokens":323},"cost_usd":0.02337}
+{"config":"qa-health-rubric","item":"f03","split":"dev","phase":"old-a","sample":0,"status":"ok","value":{"clarity":2,"completeness":1,"actionability":1,"reasoning":"The weights are explicit and sum to 100%, which is the rubric's only concrete element. Everything else is vague: 'a few errors → partial credit' and 'lots of errors → low score' give no numeric thresholds; link deductions have no point values and 'important link' is undefined; per-category scoring says 'deduct in proportion to how bad it is' with no point ranges per severity tier; the severity definitions are circular ('serious: anything that really hurts'); 'use a sensible floor' and 'round sensibly' leave the floor value and rounding rule unspecified. No edge cases are covered (e.g., console warnings vs errors, redirect vs 404, duplicate issues, zero-content categories). Two agents following this rubric would almost certainly produce different scores for the same site, so it fails the reproducibility goal."},"latency_ms":8502,"stop_reason":"end_turn","usage":{"input_tokens":698,"output_tokens":300},"cost_usd":0.02198}
+{"config":"qa-health-rubric","item":"f02","split":"dev","phase":"new","sample":0,"status":"ok","value":{"clarity":2,"completeness":2,"actionability":2,"reasoning":"The rubric has internal contradictions that make it ambiguous: Console is listed as 20% in its section header but 15% in the weights table, and Links is 15% in its header but 10% in the table. The weights table sums to 125%, not 100%, so a perfect site would score 125 despite the claim that the final score is out of 100 — the agent cannot know whether to renormalize, drop a category, or trust the headers. Severity levels (Critical/High/Medium/Low) are never defined, so per-category deductions are not reproducible across agents. 'Console error' is undefined (warnings? per page or site-wide total?), 'broken link' is undefined (4xx only? 5xx? timeouts? redirect chains?), and 'same root cause' deduplication is left to judgment. Rounding and weight format (percent vs decimal) are unspecified. The basic structure (per-category 0-100, weighted sum, floor at 0) is clear, which keeps this from being a 1, but an agent cannot compute a single correct, consistent score from this rubric as written."},"latency_ms":13455,"stop_reason":"end_turn","usage":{"input_tokens":1094,"output_tokens":915},"cost_usd":0.05669}
+{"config":"qa-health-rubric","item":"a01","split":"dev","phase":"old-b","sample":1,"status":"ok","value":{"clarity":4,"completeness":3,"actionability":4,"reasoning":"Categories, weights (summing to 100%), tiered deductions, floor, and rounding are explicit and the final formula is given, so an agent can compute a reproducible number from a list of classified findings. Gaps: 'tested weights' is never defined (no criteria for when a category counts as untested); no guidance on assigning a finding to one category to avoid double-counting across Visual/Functional/UX/etc.; 'reproducible'/'repeatable' thresholds for console/link errors are undefined; severity ladder has a hole (non-core task blocked with no workaround fits neither High nor Medium); Performance/Accessibility lack any concrete thresholds for severity. The framework guidance and operational rules are unrelated to score computation and slightly dilute focus."},"latency_ms":17757,"stop_reason":"end_turn","usage":{"input_tokens":1359,"output_tokens":865},"cost_usd":0.05684}
+{"config":"qa-health-rubric","item":"f06","split":"heldout","phase":"new","sample":0,"status":"ok","value":{"clarity":2,"completeness":1,"actionability":1,"reasoning":"The weights and final formula are clear and sum to 100%, which is the rubric's only concrete strength. Everything else is impressionistic: categories are defined by vague feelings ('how noisy and worrying', 'how good the copy is', 'how accessible the site seems') with no concrete deduction criteria, no mapping from observed defects (e.g., number of broken links, console errors, failed flows) to point deductions, and only three loose anchor bands (90s/70s/<50) with undefined gaps between them. 'Trust your instincts' is the opposite of a reproducible scoring procedure. No edge cases are addressed (empty categories, untestable features, severity vs. count of issues, catastrophic failures in one area). An agent cannot derive a consistent, reproducible score from this rubric alone; two runs would produce materially different numbers."},"latency_ms":4568,"stop_reason":"end_turn","usage":{"input_tokens":1082,"output_tokens":305},"cost_usd":0.02607}
+{"config":"qa-health-rubric","item":"f08","split":"heldout","phase":"new","sample":0,"status":"ok","value":{"clarity":3,"completeness":2,"actionability":1,"reasoning":"The structure is clear: a weighted-average formula, a deduplication rule with a defined category precedence, floor at 0, provisional labeling for untested categories, and a round-only-at-the-end instruction. However, every numeric input needed to actually compute a score—severity penalties, per-link penalty, console error bands, and category weights—is delegated to an external 'team QA scoring profile' that is not included. Additional ambiguities remain: no criteria for classifying findings as Critical/High/Medium/Low, no definition of 'reproducible' console errors, no definition of when a profile is 'out of date', and no fallback if neither profile nor prior report exists. Because the agent cannot compute any score from this document alone, actionability is minimal despite the sound skeleton."},"latency_ms":9119,"stop_reason":"end_turn","usage":{"input_tokens":1064,"output_tokens":528},"cost_usd":0.03704}
+{"config":"qa-health-rubric","item":"p08","split":"heldout","phase":"old-a","sample":1,"status":"ok","value":{"clarity":4,"completeness":3,"actionability":4,"reasoning":"Categories, weights, per-defect deduction amounts, floors, rounding, redirect handling, double-width counting, decorative-alt exemption, overlap precedence, and partial-score handling are all explicit, so an agent can compute a reproducible number from counts. Residual ambiguities limit it: the unit for contrast deductions (per unique pair per page vs. sitewide), how duplicate titles are counted (one page or both), whether broken/unreachable pages are still checked for rendering/content/a11y, what happens with zero crawled pages or a broken homepage, how 'repeat request' timing works, and inherently subjective items like 'typo' and 'layout break' without a detection standard. These gaps could produce different scores between runs or agents, but the core math and thresholds are unambiguous."},"latency_ms":12708,"stop_reason":"end_turn","usage":{"input_tokens":1084,"output_tokens":918},"cost_usd":0.05674}
+{"config":"qa-health-rubric","item":"p06","split":"heldout","phase":"new","sample":1,"status":"ok","value":{"clarity":4,"completeness":4,"actionability":4,"reasoning":"Strong rubric: single table defines every category with explicit weight, inclusion/exclusion criteria, and a deterministic score rule; severity points, level definitions, a tiebreaker ('use the higher'), a precedence order for multi-category findings, dedupe rules, untested-category handling, and the exact final formula with rounding are all specified. Remaining gaps: 'repeatable'/'reproducible' are not quantified (how many retries?), 'same-origin' vs subdomains is unstated, 'core task' is subjective without examples, the UX row ('flows that still complete') conflicts with Critical/High severity definitions (which require blockage), 'a check ran' for a category is undefined for partial testing, and third-party script errors in Console are not addressed. No worked example is provided, which would help reproducibility. An agent can nonetheless compute a correct, mostly consistent score from this alone."},"latency_ms":14413,"stop_reason":"end_turn","usage":{"input_tokens":1519,"output_tokens":923},"cost_usd":0.06134}
+{"config":"qa-health-rubric","item":"a08","split":"heldout","phase":"new","sample":0,"status":"ok","value":{"clarity":2,"completeness":3,"actionability":2,"reasoning":"The structure is mostly solid: weights sum to 100, the final formula handles untested categories via normalization, rounding is specified, and Console/Links thresholds are explicit. However, there is a critical internal inconsistency: the per-category deduction table uses three tiers (Blocker -25, Major -15, Minor -5) while the severity definitions list four tiers (Critical, High, Medium, Low) with different names. There is no mapping between them, so an agent cannot determine what a 'Medium' finding deducts (is it Major or Minor?) or whether Critical and High both map to Blocker. This single defect makes reproducible scoring impossible without guessing. Secondary gaps: 'tested' vs 'untested' vs 'partial coverage' is undefined (how much coverage counts as tested, how to label 'provisional'); Performance and Accessibility have no finding criteria or thresholds; within-page duplicate handling and how to treat a finding whose root cause spans a category boundary beyond the first-applicable rule are not addressed. The rubric is close to usable but the severity-to-deduction mismatch must be fixed before an agent can compute a correct score from it alone."},"latency_ms":14641,"stop_reason":"end_turn","usage":{"input_tokens":1429,"output_tokens":930},"cost_usd":0.06079}
+{"config":"qa-health-rubric","item":"a08","split":"heldout","phase":"old-a","sample":2,"status":"ok","value":{"clarity":3,"completeness":3,"actionability":3,"reasoning":"Weights sum to 100%, the final formula with untested-weight normalization and rounding rule is explicit, Console/Links tiers are unambiguous, and the primary-category precedence prevents double counting. However, there is a core inconsistency: deductions are defined for three tiers (Blocker -25, Major -15, Minor -5) while severity definitions use four different labels (Critical, High, Medium, Low) with no mapping between them. An agent cannot determine whether Low deducts -5 or 0, or whether Critical and High both map to Blocker, making per-category scores non-reproducible. Other gaps: no guidance on performance thresholds, no definition of 'reproducible' or 'repeatable', and no rule for findings spanning multiple severities on the same root cause beyond 'highest applicable'."},"latency_ms":10243,"stop_reason":"end_turn","usage":{"input_tokens":1068,"output_tokens":725},"cost_usd":0.04693}
+{"config":"qa-health-rubric","item":"p07","split":"heldout","phase":"new","sample":1,"status":"ok","value":{"clarity":4,"completeness":4,"actionability":5,"reasoning":"Categories, point values, and per-finding deductions are laid out in a single table with explicit speed/console thresholds, boundary values, a 0-floor rule, and a table-order precedence rule for overlapping defects. The untested-category rescaling formula and rounding rule are explicit, and the worked example (including a tie for largest deduction) makes the computation reproducible. Minor ambiguities remain: whether a single element that is both too small and non-functional takes −5 only or −5 and −2; how to count 'overlapping' (per element vs per pair); what qualifies as a 'label' (aria-label, placeholder); whether 'smaller than 44×44' means either dimension; whether a crashed page is excluded from the speed measurement; and partial-blank rendering. These are edge cases rather than core gaps, so an agent can compute a correct score from the rubric alone in nearly all realistic situations."},"latency_ms":15084,"stop_reason":"end_turn","usage":{"input_tokens":1453,"output_tokens":937},"cost_usd":0.06138}
+{"config":"qa-health-rubric","item":"a07","split":"heldout","phase":"new","sample":1,"status":"ok","value":{"clarity":4,"completeness":3,"actionability":4,"reasoning":"Categories, per-item deduction amounts, floors, and weights are explicit and the 'first area that fits' rule resolves most cross-category overlaps (e.g., 5xx in Stability vs Navigation). However, several boundaries are undefined: the combination method is only implied ('combine using the weights') rather than stated as a weighted average; no rounding rule for the whole-number result; no instruction on whether to renormalize remaining weights when an unchecked area is 'left out'; unclear whether a crashing page that also throws an exception incurs both −30 and −10; 'repeatable' is required for Navigation but intermittent 5xx/crashes in Stability are unaddressed; and the single-count rule creates ambiguity when a crash blocks a flow (Stability only, or also a Flows failure?). An agent can produce a reasonable score from this alone, but reproducibility across agents will suffer on the exclusion/renormalization and double-counting edge cases."},"latency_ms":16429,"stop_reason":"end_turn","usage":{"input_tokens":1243,"output_tokens":1082},"cost_usd":0.06653}
+{"config":"qa-health-rubric","item":"p08","split":"heldout","phase":"new","sample":1,"status":"ok","value":{"clarity":4,"completeness":4,"actionability":4,"reasoning":"Categories, weights, and deduction amounts are concrete and numeric, weights sum to 100, and rounding, floors, double-counting (lower-numbered category wins), and partial-score handling are explicitly defined. Remaining ambiguities: crawl order/dedup rules for the 50-page cap (query strings, fragments, trailing slashes) affect which pages are scored; 'typo', 'meaningful image', and 'overlap/clipped text' require judgment; 'contrast pair' is unclear as per-occurrence vs per-unique-pair; duplicate titles unclear whether both pages are deducted; 'unique error messages' doesn't address near-duplicates or third-party script errors. Edge cases like zero pages crawled or an unreachable homepage are not covered. Despite these gaps an agent can compute a defensible, mostly reproducible score from the formula, tie-breaker, and thresholds provided."},"latency_ms":17764,"stop_reason":"end_turn","usage":{"input_tokens":1445,"output_tokens":1116},"cost_usd":0.07025}
+{"config":"qa-health-rubric","item":"f08","split":"heldout","phase":"old-a","sample":1,"status":"ok","value":{"clarity":3,"completeness":2,"actionability":1,"reasoning":"The procedural structure is clearly described (dedup rules, category precedence, floor at 0, weighted-average formula, rounding only at the end), but every numeric input needed to actually score—per-link penalty, severity penalties, console score bands, and category weights—is delegated to an external 'team QA scoring profile' that is not included. Severity definitions (what makes a finding Critical vs. Low), what counts as a 'reproducible' console error, and how to determine a profile is 'out of date' are also undefined. Because the agent cannot compute any category score or weighting from this document alone, reproducibility depends entirely on an absent artifact, making the rubric essentially non-actionable in isolation."},"latency_ms":7759,"stop_reason":"end_turn","usage":{"input_tokens":703,"output_tokens":444},"cost_usd":0.02923}
+{"config":"qa-health-rubric","item":"p07","split":"heldout","phase":"old-b","sample":0,"status":"ok","value":{"clarity":5,"completeness":4,"actionability":5,"reasoning":"Categories, point weights, and per-finding deductions are stated in a single table with explicit magnitudes; boundary values for speed buckets are spelled out, category floors are defined, and a worked example shows the full computation including rescaling and rounding. Counting rules resolve the main cross-category ambiguities (precedence by table order, dedup across pages, console dedup by message text). Minor gaps: whether a single control that is both too small and non-functional takes both the −5 and −2 within the same category, whether 'smaller than 44×44' means either dimension or both, and what 'largest deduction' means when a single finding (e.g., −8 form) is smaller than a category's cumulative loss—the example implies per-category totals but doesn't say so. These are small enough that an agent can still compute a reproducible score from the rubric alone."},"latency_ms":16039,"stop_reason":"end_turn","usage":{"input_tokens":1092,"output_tokens":953},"cost_usd":0.05857}
+{"config":"qa-health-rubric","item":"a08","split":"heldout","phase":"new","sample":1,"status":"ok","value":{"clarity":3,"completeness":3,"actionability":2,"reasoning":"Strengths: weights sum to 100%, the final-score formula handles untested categories via normalization, rounding rule is explicit, Console/Links thresholds and dedup/primary-category ordering are clear. Critical flaw: the per-category deductions use a three-tier scale (Blocker -25 / Major -15 / Minor -5) while the severity definitions use a four-tier scale (Critical / High / Medium / Low) with no mapping between them. An agent cannot determine whether Critical and High both map to Blocker, or whether Medium is Major or Minor, so six of eight categories cannot be scored deterministically from the rubric alone. Secondary gaps: 'untested' vs 'partially tested' is not defined, Performance/Accessibility have no measurable criteria for severity, 'Compare only identical coverage' is underspecified, and 'remaining errors' for Console relies on the reader inferring the exclusion logic. Reproducibility is undermined mainly by the severity-tier mismatch."},"latency_ms":14499,"stop_reason":"end_turn","usage":{"input_tokens":1429,"output_tokens":993},"cost_usd":0.06394}
+{"config":"qa-health-rubric","item":"p07","split":"heldout","phase":"old-a","sample":1,"status":"ok","value":{"clarity":4,"completeness":4,"actionability":5,"reasoning":"Categories, point values, and deductions are laid out in a single table with explicit floors (never below 0), explicit speed boundary rules (exactly 3.0s/6.0s/10.0s), a cross-category precedence rule (first fitting category in table order), console deduplication by message text, and a round-half-up rescaling formula for untested categories, all reinforced by a fully worked example whose arithmetic checks out (67/85 → 79). Remaining gaps are minor within-category stacking questions: whether a single control that is both too small and non-functional takes −7 or −5, whether a field that is both unlabeled and has the wrong keyboard takes −4 or −8, how to measure 'slowest page load' when a page crashes and never fires load, and whether two mutually overlapping elements count as one or two findings. These could cause small score variance between agents but do not block computing a defensible, reproducible score from the rubric alone."},"latency_ms":14072,"stop_reason":"end_turn","usage":{"input_tokens":1092,"output_tokens":898},"cost_usd":0.05582}
+{"config":"qa-health-rubric","item":"p06","split":"heldout","phase":"new","sample":2,"status":"ok","value":{"clarity":5,"completeness":4,"actionability":5,"reasoning":"The single table makes categories, weights (summing to 1.00), and per-category score rules explicit; Console uses clear count tiers, Links a clear linear formula, and the six severity categories share one point scale with concrete level definitions and a tie-break rule ('use the higher'). Overlap handling is deterministic via a fixed precedence order, dedupe rules (message+source; same root cause = one finding) reduce double-counting, and the final formula with renormalization over tested categories plus explicit rounding (x.5 up) is fully specified. Minor gaps: no objective thresholds for Performance ('slow'), partial coverage within a category (some pages tested, others not) isn't addressed, and 'same root cause' judgment leaves some room for inconsistency between runs. Otherwise an agent can compute a reproducible score from this document alone."},"latency_ms":15801,"stop_reason":"end_turn","usage":{"input_tokens":1519,"output_tokens":969},"cost_usd":0.06364}
+{"config":"qa-health-rubric","item":"a07","split":"heldout","phase":"old-b","sample":0,"status":"ok","value":{"clarity":4,"completeness":3,"actionability":4,"reasoning":"Categories, per-item deductions, floors, and weights are stated explicitly, and the 'first area that fits' precedence rule plus 'count once across pages/viewports' resolve most double-counting questions. Gaps: the weighted-combination formula is implied rather than written (weighted average by weight/100), the rounding rule for the whole-number result is unspecified, and the instruction to 'leave out' an unchecked area doesn't say whether remaining weights are renormalized to 100 or the overall is reported on a reduced scale. Some judgment terms remain ('distinct issue', 'workaround', whether a crash that blocks a flow also counts as an incomplete flow). An agent can compute a reproducible score in the common case, but these edge cases could produce inconsistent results."},"latency_ms":14802,"stop_reason":"end_turn","usage":{"input_tokens":882,"output_tokens":995},"cost_usd":0.05857}
+{"config":"qa-health-rubric","item":"p07","split":"heldout","phase":"new","sample":2,"status":"ok","value":{"clarity":4,"completeness":4,"actionability":5,"reasoning":"Categories, point allocations, deduction amounts, floors at 0, bracket boundaries, rounding rule, first-fit precedence, deduplication, and rescaling for untested categories are all explicitly defined, and the worked example is arithmetically correct and demonstrates tie handling for the largest deduction. Minor gaps: the 'one defect counts once across pages' rule can conflict with the 'per page with horizontal scroll' deduction when a shared element causes scroll on multiple pages; it is unclear whether a control that is both undersized and non-functional incurs −5, −2, or −7 within the Tap targets category; 'overlapping or clipped element' is somewhat subjective; and behavior when the load event never fires (e.g., crashed page) for the Speed category is not stated. These are small and an agent can still compute a reproducible score from this rubric alone."},"latency_ms":15564,"stop_reason":"end_turn","usage":{"input_tokens":1453,"output_tokens":997},"cost_usd":0.06438}
+{"config":"qa-health-rubric","item":"p07","split":"heldout","phase":"new","sample":0,"status":"ok","value":{"clarity":4,"completeness":4,"actionability":5,"reasoning":"Categories, point weights, and per-finding deductions are laid out in a single table with explicit floors (no category below 0), tie-break ordering across categories, dedupe rules, boundary definitions for Speed, and a rescaling formula with rounding rule; the worked example is arithmetically correct and demonstrates the untested-category path. Remaining ambiguities are mostly within-category stacking: whether a control that is both too small and non-functional takes −7 or −5, whether a field with both wrong keyboard and no label takes −4 or −8, whether unsubmittable-form deductions stack with its field deductions, and whether console warnings count as errors. Edge cases like a page that never finishes loading (crash vs. speed) and degenerate cases where most categories are removed are not addressed, but the cross-category 'first fit' rule covers the main overlaps. An agent can compute a reproducible score from this rubric alone in nearly all realistic cases."},"latency_ms":18902,"stop_reason":"end_turn","usage":{"input_tokens":1453,"output_tokens":1342},"cost_usd":0.08163}
+{"config":"qa-health-rubric","item":"a08","split":"heldout","phase":"old-b","sample":1,"status":"ok","value":{"clarity":2,"completeness":3,"actionability":2,"reasoning":"Structure, weights (sum to 100%), dedup rules, and the final formula with rounding are well specified. However, the core scoring mechanic is broken by a terminology mismatch: deductions are defined for Blocker/Major/Minor (three tiers) while the severity definitions use Critical/High/Medium/Low (four tiers). There is no mapping between them, so an agent cannot determine whether Low gets -5 or 0, or whether Critical and High both map to Blocker, making scores non-reproducible. Additional gaps: 'untested' category and 'coverage' are undefined, the 'provisional' label has no criteria, and 'Compare only identical coverage' is vague. Console and Links are fully actionable, but the six per-category scores cannot be computed without guessing."},"latency_ms":13045,"stop_reason":"end_turn","usage":{"input_tokens":1068,"output_tokens":886},"cost_usd":0.05498}
+{"config":"qa-health-rubric","item":"f07","split":"heldout","phase":"new","sample":0,"status":"ok","value":{"clarity":2,"completeness":2,"actionability":1,"reasoning":"Individual category deduction tables are clear and weights sum to 100, but the rubric contains two direct self-contradictions that make a reproducible score impossible: (1) the intro says the final score is the lowest category score, while the Final Score section says to take the weighted average; (2) the Counting section says untested categories count as 0, while the Final Score section says to exclude them. These produce wildly different results for the same findings. Additional gaps: severity levels (critical/high/medium/low) are undefined, 'same root cause' deduplication is vague, console warnings vs errors and what counts as a 'broken' link (redirects, 4xx vs 5xx, external links) are unspecified, and Quick mode does not say whether the Console/Links average is simple or weighted. An agent cannot compute a single correct score from this rubric alone."},"latency_ms":8333,"stop_reason":"end_turn","usage":{"input_tokens":1108,"output_tokens":653},"cost_usd":0.04373}
+{"config":"qa-health-rubric","item":"f06","split":"heldout","phase":"old-b","sample":2,"status":"ok","value":{"clarity":2,"completeness":1,"actionability":1,"reasoning":"Weights are explicit and sum to 100%, and the final formula is clear, but the category scoring is entirely impressionistic ('how noisy and worrying', 'trust your instincts'). There are no concrete deduction criteria (e.g., points per console error, per broken link, per WCAG violation), no thresholds (what performance timing earns 90 vs 70), no definition of what evidence to count, and no handling of edge cases (no console output, untestable categories, N/A weights). Anchor bands (90s/70s/<50) are vague and leave huge gaps. Two agents testing the same site could produce scores differing by 20+ points, so the rubric fails the reproducibility requirement and cannot yield a correct score on its own."},"latency_ms":5825,"stop_reason":"end_turn","usage":{"input_tokens":721,"output_tokens":269},"cost_usd":0.02066}
+{"config":"qa-health-rubric","item":"a07","split":"heldout","phase":"new","sample":0,"status":"ok","value":{"clarity":4,"completeness":3,"actionability":4,"reasoning":"Categories, deduction amounts, floors, and weights are explicit and the first-fit rule resolves most overlap (e.g., 5xx in Stability vs Navigation). Gaps: 'combine using the weights' never states the formula (weighted average Σ(score×weight)/100 is implied but not written); rounding method for the whole number is unspecified; when an area is omitted it says to leave it out but not whether to renormalize remaining weights to 100 or report out of a smaller total; the once-only rule is ambiguous when a crash also blocks a flow (is the blocked flow a separate defect or the same one?). An agent could still compute a reproducible score in the normal case, but omitted areas and rounding could yield inconsistent results."},"latency_ms":13508,"stop_reason":"end_turn","usage":{"input_tokens":1243,"output_tokens":838},"cost_usd":0.05433}
+{"config":"qa-health-rubric","item":"a07","split":"heldout","phase":"old-a","sample":0,"status":"ok","value":{"clarity":4,"completeness":3,"actionability":4,"reasoning":"Categories, per-area deductions, floors, and weights are specific and the overlap rule ('area listed first that fits') resolves double-counting. Weights sum to 100, so weighted-average combination is strongly implied though not stated as an explicit formula. Gaps: when an area is skipped, the rubric says to omit it but does not say how to renormalize remaining weights (rescale to 100 vs. raw sum), which would produce inconsistent overall scores; rounding method for the whole-number result is unspecified; 'distinct issue' in Presentation/Accessibility/Copy is not defined (per element vs. per defect type), and 'per page that crashes' leaves per-URL vs. per-template ambiguous. An agent can still compute a reproducible score in the common case where all six areas are checked."},"latency_ms":13061,"stop_reason":"end_turn","usage":{"input_tokens":882,"output_tokens":833},"cost_usd":0.05047}
+{"config":"qa-health-rubric","item":"p06","split":"heldout","phase":"old-b","sample":2,"status":"ok","value":{"clarity":4,"completeness":4,"actionability":4,"reasoning":"Categories, weights, and per-category score rules are laid out in a single table with explicit numeric formulas, severity point values, level definitions, a tie-break rule ('use the higher'), a precedence order for overlapping findings, root-cause dedupe, untested-category handling, and exact rounding. This makes the final computation mechanical and reproducible. Remaining gaps: Performance has no numeric thresholds for mapping slowness to severity; 'reproducible'/'repeatable' is undefined (how many attempts?); UX is defined as flows that 'still complete' yet shares severity levels where only Low explicitly matches 'task completes', creating mild tension; and what constitutes a category having been 'tested' (vs. partially tested) is not specified. No worked example is provided, though the formula is simple enough that one is not strictly needed."},"latency_ms":17064,"stop_reason":"end_turn","usage":{"input_tokens":1158,"output_tokens":1120},"cost_usd":0.06758}
+{"config":"qa-health-rubric","item":"a08","split":"heldout","phase":"old-a","sample":0,"status":"ok","value":{"clarity":3,"completeness":2,"actionability":2,"reasoning":"Weights sum to 100%, the final formula with tested-weight normalization is explicit, and Console/Links count-based bands are unambiguous. However, the core per-category scoring is internally inconsistent: deductions are defined for Blocker/Major/Minor (three tiers), while severity is defined as Critical/High/Medium/Low (four tiers) with no mapping between them. An agent cannot determine whether a 'Critical' or 'High' finding is a Blocker or Major, or what a 'Low' deducts, so category scores for six of eight categories are not reproducible. Additional gaps: 'Compare only identical coverage' is unexplained, 'provisional with coverage' lacks a format, and the dedup rule doesn't say how to handle a finding that fits no listed category."},"latency_ms":11588,"stop_reason":"end_turn","usage":{"input_tokens":1068,"output_tokens":707},"cost_usd":0.04603}
+{"config":"qa-health-rubric","item":"f08","split":"heldout","phase":"old-a","sample":2,"status":"ok","value":{"clarity":2,"completeness":2,"actionability":1,"reasoning":"The rubric is structurally clean (counting rules, dedup order, formula, rounding) but defers every numeric parameter—deduction amounts, severity penalties, per-link penalty, console bands, and category weights—to an external 'team profile' that isn't included. An agent cannot compute any score from this document alone. Clarity suffers from unresolved references (which profile, what if it's missing, what counts as 'out of date'), and completeness is weak because key edge cases (no prior QA report exists, severity assignment criteria, what 'provisional' means numerically, performance measurement method) are undefined."},"latency_ms":5554,"stop_reason":"end_turn","usage":{"input_tokens":703,"output_tokens":226},"cost_usd":0.01833}
+{"config":"qa-health-rubric","item":"f06","split":"heldout","phase":"old-a","sample":0,"status":"ok","value":{"clarity":2,"completeness":1,"actionability":2,"reasoning":"The weights are clearly specified, sum to 100%, and the final formula is unambiguous — that is the rubric's only strength. Every category is defined purely by subjective impression ('how noisy and worrying', 'how pleasant', 'how good the copy is', 'seems accessible'), with no concrete deduction criteria (e.g., points per console error, per broken link, per failed WCAG check). The anchor guidance (90s/70s/below 50) is vague and leaves the 50-70 and 80-90 ranges undefined, and 'trust your instincts' explicitly invites non-reproducible scoring. No edge cases are addressed: what to do when a category can't be assessed, how to handle zero-issue vs. catastrophic states, or how Visual's comparison to 'well-designed sites in the same space' should be grounded. An agent can mechanically compute a weighted average, but the inputs are arbitrary, so the same site would yield widely different scores across runs."},"latency_ms":8170,"stop_reason":"end_turn","usage":{"input_tokens":721,"output_tokens":346},"cost_usd":0.02451}
+{"config":"qa-health-rubric","item":"f07","split":"heldout","phase":"old-a","sample":1,"status":"ok","value":{"clarity":2,"completeness":2,"actionability":1,"reasoning":"The rubric contradicts itself on the two most important aggregation rules: the intro says the final score is the lowest category score, while the Final Score section says to take a weighted average; and the Counting section says untested categories count as 0, while the Final Score section says to exclude them. These are not edge-case ambiguities but direct conflicts on the core computation, so two agents could legitimately produce very different scores. Additional gaps: severity levels (critical/high/medium/low) are undefined; 'deduplicate the same root cause' is not operationalized (does it apply to console errors and broken links?); Quick mode doesn't state whether 'average' is simple or weighted (15/10); console warnings are unaddressed. The individual category tables and weights (which sum to 100) are clear, but the rubric cannot yield a reproducible final score as written."},"latency_ms":8412,"stop_reason":"end_turn","usage":{"input_tokens":747,"output_tokens":666},"cost_usd":0.04077}
+{"config":"qa-health-rubric","item":"p07","split":"heldout","phase":"old-a","sample":2,"status":"ok","value":{"clarity":4,"completeness":4,"actionability":5,"reasoning":"Categories, point values, per-finding deductions, the 0-point floor, speed/console boundaries, and the half-up rounding are all stated explicitly, and the worked example verifies the arithmetic (67/85 → 79). Cross-category double counting is resolved by the 'first category in table order' rule, and untested categories have a defined rescale formula. Remaining gaps: within-category stacking is undefined (a control that is both <44px and non-functional — −7 or −5?; a field with both wrong keyboard and no label — −4 or −8?), 'control' vs 'tap target' are not distinguished, and the 'one defect counts once' rule sits in mild tension with '−5 per page with horizontal scroll' when a single shared element causes overflow on every page. 'Overlapping or clipped' also leaves some judgment to the tester. These are minor; an agent can produce a correct, reproducible score from the document alone in nearly all cases."},"latency_ms":16530,"stop_reason":"end_turn","usage":{"input_tokens":1092,"output_tokens":1030},"cost_usd":0.06242}
+{"config":"qa-health-rubric","item":"p08","split":"heldout","phase":"old-b","sample":0,"status":"ok","value":{"clarity":4,"completeness":3,"actionability":4,"reasoning":"Categories, weights (summing to 100), deduction sizes, floors, rounding, and the overlap/partial rules are explicit and mostly unambiguous, so an agent can compute a score and apply the ship thresholds. Gaps remain around the counting unit for repeated defects: whether the same broken image, missing-alt image, or low-contrast CSS pair appearing on many pages counts once or per page (and whether broken images at both widths count once), how 'unique' console errors are deduplicated, and judgment calls like 'typo' and 'meaningful image'. Edge cases such as a failing homepage, zero pages crawled, or non-HTML link targets are undefined. These can materially change scores between runs, limiting reproducibility, but the rubric is otherwise well-specified and usable."},"latency_ms":15613,"stop_reason":"end_turn","usage":{"input_tokens":1084,"output_tokens":1071},"cost_usd":0.06439}
+{"config":"qa-health-rubric","item":"f08","split":"heldout","phase":"new","sample":1,"status":"ok","value":{"clarity":3,"completeness":2,"actionability":1,"reasoning":"The structure, deduplication rules, per-category start-at-100-with-floor approach, provisional labeling, and final weighted-average formula are clearly stated and unambiguous as a process. However, every actual number needed to score is delegated to an external 'team QA scoring profile' that is not included: per-link penalty, severity penalties (Critical/High/Medium/Low), console error bands, and category weights. Severity levels themselves are undefined (what makes a finding Critical vs High), 'reproducible' console errors is not defined, 'out of date' profile has no criterion, and there is no fallback if neither a profile nor a prior QA report exists. Edge cases like untested categories, floors, and rounding are handled, which earns some completeness credit. Because the agent cannot produce any numeric score from this document alone, actionability is minimal — the rubric is a procedural skeleton requiring an external dependency to function."},"latency_ms":11022,"stop_reason":"end_turn","usage":{"input_tokens":1064,"output_tokens":653},"cost_usd":0.04329}
+{"config":"qa-health-rubric","item":"f06","split":"heldout","phase":"new","sample":2,"status":"ok","value":{"clarity":2,"completeness":1,"actionability":1,"reasoning":"The weights and final formula are clear and sum to 100%, but the category criteria are entirely impressionistic ('how noisy and worrying', 'how well things work overall', 'trust your instincts'). No deduction criteria are defined: nothing specifies how many console errors, broken links, failed flows, or accessibility violations map to a given score, nor how severity is treated. The anchors (90s/70s/below 50) are vague and leave gaps (50-69, 80-89). Edge cases such as empty categories, untestable features, or critical blockers are absent. An agent cannot produce a consistent, reproducible score from this rubric alone; two runs would likely yield materially different results."},"latency_ms":3622,"stop_reason":"end_turn","usage":{"input_tokens":1082,"output_tokens":252},"cost_usd":0.02342}
+{"config":"qa-health-rubric","item":"a07","split":"heldout","phase":"old-b","sample":2,"status":"ok","value":{"clarity":4,"completeness":3,"actionability":4,"reasoning":"Categories, per-item deductions, floors, and weights are stated explicitly, and the Notes resolve key ambiguities (weights sum to 100, defects counted once in the first-listed fitting area, viewport duplicates counted once, round to whole number). Weaknesses: the combination formula is only implied ('combine using the weights') rather than written out; when an unchecked area is dropped, it is unclear whether remaining weights are renormalized; 'distinct issue' in Presentation/Accessibility/Copy lacks a tie-breaker for borderline grouping; the overlap rule means a flow blocked by a 5xx counts only in Stability, which is defined but may surprise; no rounding convention (half-up vs banker's) and no handling for flows that complete but with visible bugs short of a workaround. An agent can produce a reproducible score in most cases, with some risk of divergence on the unchecked-area and grouping edge cases."},"latency_ms":15582,"stop_reason":"end_turn","usage":{"input_tokens":882,"output_tokens":984},"cost_usd":0.05802}
+{"config":"qa-health-rubric","item":"f07","split":"heldout","phase":"old-a","sample":2,"status":"ok","value":{"clarity":2,"completeness":2,"actionability":1,"reasoning":"The rubric directly contradicts itself on the two things that matter most: the final score is defined both as 'the lowest category score' and as 'the weighted average of the category scores', and untested categories are both 'count as 0' and 'excluded so partial runs are not penalized'. An agent cannot resolve these without guessing, so scores will not be reproducible. Additional gaps: Quick mode says 'average of Console and Links' without specifying simple vs weighted (15/10); severity levels (critical/high/medium/low) are never defined; 'deduplicate the same root cause' has no criteria; it's unclear whether console warnings count or whether deduplication applies to console/link counts. The per-category deduction tables and weight table are individually clear, which keeps clarity/completeness above the floor, but the computation path is fundamentally ambiguous."},"latency_ms":7227,"stop_reason":"end_turn","usage":{"input_tokens":747,"output_tokens":525},"cost_usd":0.03372}
+{"config":"qa-health-rubric","item":"p06","split":"heldout","phase":"old-a","sample":2,"status":"ok","value":{"clarity":4,"completeness":4,"actionability":4,"reasoning":"The rubric is compact and largely unambiguous: weights sum to 1.00, each category has an explicit score formula, severity points and level definitions are concrete, a tie-break order resolves multi-category findings, the 'same root cause = one finding' rule prevents double counting, and untested-category handling and end-only rounding (x.5 up) are specified. Weak spots that cost consistency: Performance has no numeric thresholds for what counts as 'slow'; 'reproducible'/'repeatable' are undefined (how many retries?); UX vs Functional boundary relies on 'still completes' which is workable but judgment-based; and what constitutes a category being 'tested' (partial coverage) is not defined. These gaps mean two agents could plausibly diverge on Performance and severity assignments, but the formula itself is fully computable from the document."},"latency_ms":14937,"stop_reason":"end_turn","usage":{"input_tokens":1158,"output_tokens":913},"cost_usd":0.05723}
+{"config":"qa-health-rubric","item":"f07","split":"heldout","phase":"old-a","sample":0,"status":"ok","value":{"clarity":2,"completeness":2,"actionability":1,"reasoning":"The rubric directly contradicts itself on the two most important rules: the intro says the final score is the lowest category score, while the Final Score section says to take a weighted average. It also says untested categories count as 0 (Counting section) and then says to exclude untested categories (Final Score section). Quick mode's 'average of Console and Links' is ambiguous (simple vs. weighted). Severity levels (Critical/High/Medium/Low) are never defined, and 'deduplicate the same root cause' lacks criteria. Per-category deduction tables and weights are individually clear and the weights sum to 100%, but an agent cannot produce a reproducible score because it must guess which of the conflicting aggregation and untested-category rules to apply."},"latency_ms":5585,"stop_reason":"end_turn","usage":{"input_tokens":747,"output_tokens":420},"cost_usd":0.02847}
+{"config":"qa-health-rubric","item":"p06","split":"heldout","phase":"old-a","sample":1,"status":"ok","value":{"clarity":5,"completeness":4,"actionability":5,"reasoning":"Categories, weights, and per-category score rules are defined in a single table with explicit thresholds, severity point values, dedupe rules, a deterministic tie-break order for overlapping findings, floor-at-zero, and precise rounding (x.5 up). Untested-category handling and the 'not scored' case are specified, and the renormalized weighted-average formula is unambiguous, so an agent can reproduce a score from the rubric alone. Minor completeness gaps: severity levels are defined in task-blocking terms that map awkwardly onto Performance/Visual/Content findings (a slow load or placeholder copy rarely 'blocks a core task'), partial coverage within a category is not addressed, and it is slightly unclear whether a JS error that causes a functional bug is still counted in Console after the tie-break assigns the finding to Functional."},"latency_ms":16952,"stop_reason":"end_turn","usage":{"input_tokens":1158,"output_tokens":1076},"cost_usd":0.06538}
+{"config":"qa-health-rubric","item":"a07","split":"heldout","phase":"new","sample":2,"status":"ok","value":{"clarity":4,"completeness":3,"actionability":4,"reasoning":"Categories, per-item deductions, floors, and weights are explicit and the weights verifiably sum to 100; the 'count once, in the first area that fits' rule gives a clear precedence for overlapping defects (e.g., a 5xx page goes to Stability, not Navigation). Weaknesses: the combination method is only implied ('combine using the weights' rather than an explicit weighted-average formula), the rounding rule for the whole number is unspecified, 'distinct issue' and 'workaround' are undefined, and when an area is omitted it is unclear whether remaining weights are renormalized to 100 or simply summed (which changes the result materially). Despite these gaps, an agent could compute a largely consistent score from this rubric with minimal assumptions."},"latency_ms":13020,"stop_reason":"end_turn","usage":{"input_tokens":1243,"output_tokens":881},"cost_usd":0.05648}
+{"config":"qa-health-rubric","item":"p06","split":"heldout","phase":"old-b","sample":1,"status":"ok","value":{"clarity":4,"completeness":4,"actionability":5,"reasoning":"Categories, weights, and score rules are laid out in a single table with explicit numeric thresholds; severity points have defined values and criteria, with a tie-break rule ('use the higher') and an explicit precedence order for multi-category findings. The final formula, renormalization over tested categories, floor, and rounding rule are all unambiguous, so an agent can mechanically compute a score. Minor gaps: 'reproducible'/'repeatable' lack a count, 'root cause' consolidation is judgment-based, Performance has no concrete slowness thresholds, and 'core task' is undefined, so severity assignment remains partly subjective and could reduce cross-run reproducibility. Edge cases (untested, nothing tested, floors) are otherwise well handled."},"latency_ms":14959,"stop_reason":"end_turn","usage":{"input_tokens":1158,"output_tokens":917},"cost_usd":0.05743}
+{"config":"qa-health-rubric","item":"f08","split":"heldout","phase":"old-b","sample":1,"status":"ok","value":{"clarity":2,"completeness":2,"actionability":1,"reasoning":"The rubric's structure is clear (start at 100, deduct per finding, weighted average over tested categories, dedupe by root cause with a defined category precedence, round only at the end), but all the actual numbers—severity penalties, per-link penalty, console bands, and category weights—are delegated to an external 'team profile' that is not provided. Severity definitions (what makes a finding Critical vs. Low), performance scoring criteria, and what 'out of date' means for a profile are also undefined, and the fallback to 'the last QA report' is another external dependency. An agent cannot compute a reproducible numeric score from this document alone."},"latency_ms":5674,"stop_reason":"end_turn","usage":{"input_tokens":703,"output_tokens":235},"cost_usd":0.01878}
+{"config":"qa-health-rubric","item":"f06","split":"heldout","phase":"old-a","sample":2,"status":"ok","value":{"clarity":2,"completeness":1,"actionability":1,"reasoning":"The weights table and final formula are clear and sum to 100%, but the deduction criteria are entirely subjective ('how noisy', 'how pleasant', 'trust your instincts'). No concrete deductions, thresholds, or examples are defined for any category (e.g., what counts as a console error vs. warning, how many broken links drop Links to 50, what accessibility checks apply). The anchor guidance (90s/70s/<50) is vague and non-exhaustive, with undefined bands between 50-70 and 80-90. No edge cases are handled (e.g., categories not testable, sites with no console output, partial page loads). An agent cannot produce a consistent or reproducible score from this rubric alone—two runs would almost certainly yield different results."},"latency_ms":6103,"stop_reason":"end_turn","usage":{"input_tokens":721,"output_tokens":276},"cost_usd":0.02101}
+{"config":"qa-health-rubric","item":"p07","split":"heldout","phase":"old-b","sample":2,"status":"ok","value":{"clarity":4,"completeness":4,"actionability":5,"reasoning":"Categories, point weights, per-finding deductions, floor-at-zero, boundary values (3.0s/6.0s/10.0s), dedup rule for console errors, table-order precedence, rescaling formula with explicit rounding, and a fully worked example make this highly computable and reproducible. Minor ambiguities remain: the 'one defect counts once' rule (shared footer example) conflicts with '−5 per page with horizontal scroll' when a shared element causes overflow on every page; it is unclear whether a control that is both too small and non-functional stacks −7 or counts once; 'per overlapping element' doesn't say whether a pair counts as one or two. Unaddressed edge cases include speed when navigation timing is unavailable and whether a crashed page's other defects are still surveyed. These are small gaps relative to an otherwise tight, agent-ready rubric."},"latency_ms":15708,"stop_reason":"end_turn","usage":{"input_tokens":1092,"output_tokens":1036},"cost_usd":0.06272}
+{"config":"qa-health-rubric","item":"f07","split":"heldout","phase":"old-b","sample":2,"status":"ok","value":{"clarity":2,"completeness":2,"actionability":1,"reasoning":"The per-category deduction tables and weight table are individually clear, but the rubric contradicts itself on the two most important rules. Final score is described both as 'the lowest category score' and as 'the weighted average of the category scores.' Untested categories are said to 'count as 0 so skipped coverage is visible' and also to be 'excluded so partial runs are not penalized.' An agent cannot resolve these without guessing, so scores will not be reproducible. Further gaps: severity levels (Critical/High/Medium/Low) are never defined; 'same root cause' deduplication is undefined; Quick mode 'average of Console and Links' doesn't say whether it is a simple or weighted (15/10) average; no guidance on whether console warnings count, whether duplicate console errors across pages are deduped, or on rounding. Because the aggregation step itself is contradictory, an agent cannot compute a correct score from this rubric alone."},"latency_ms":8392,"stop_reason":"end_turn","usage":{"input_tokens":747,"output_tokens":657},"cost_usd":0.04032}
+{"config":"qa-health-rubric","item":"f07","split":"heldout","phase":"new","sample":2,"status":"ok","value":{"clarity":2,"completeness":2,"actionability":1,"reasoning":"The rubric contradicts itself on the two most important rules. The intro says the final score is the lowest category score (min), while the Final Score section says to take a weighted average; the Counting section says untested categories count as 0, while the Final Score section says to exclude them so partial runs aren't penalized. An agent cannot resolve these without guessing, so two agents would produce different numbers. Quick mode says 'average of Console and Links' without specifying simple vs. weighted (15/10). Severity levels (Critical/High/Medium/Low) are never defined, 'deduplicate the same root cause' has no operational criteria, console scoring doesn't address warnings vs. errors or whether dedup applies to console errors, and rounding/renormalization of weights after exclusion is unspecified. The per-category tables and weights themselves are clear and sum to 100%, which keeps clarity and completeness above the floor, but the core aggregation logic is unusable as written."},"latency_ms":7734,"stop_reason":"end_turn","usage":{"input_tokens":1108,"output_tokens":562},"cost_usd":0.03918}
+{"config":"qa-health-rubric","item":"a08","split":"heldout","phase":"new","sample":2,"status":"ok","value":{"clarity":3,"completeness":3,"actionability":3,"reasoning":"The rubric is well-structured in many respects: weights sum to 100%, the final formula correctly normalizes by tested weights, rounding is specified, Console and Links use clear count-based tables, and the deduplication order across categories is explicit. However, there is a critical inconsistency: the per-category deduction table uses three severity labels (Blocker/Major/Minor) while the severity definitions use four different labels (Critical/High/Medium/Low). No mapping is given, so an agent cannot determine what a 'Medium' finding deducts or whether 'Critical' equals 'Blocker'. This directly undermines reproducibility for six of the eight categories. Secondary gaps: 'tested' vs 'untested' is not operationally defined (e.g., partially tested categories), Performance has no objective thresholds for what constitutes a finding, and the handling of a finding that could be both Low in one category and higher in another is only partly addressed by the primary-category ordering. Console/Links and the aggregation are fully actionable; the severity-to-deduction ambiguity holds the overall scores to the middle."},"latency_ms":15919,"stop_reason":"end_turn","usage":{"input_tokens":1429,"output_tokens":1034},"cost_usd":0.06599}
+{"config":"qa-health-rubric","item":"p08","split":"heldout","phase":"old-b","sample":2,"status":"ok","value":{"clarity":4,"completeness":4,"actionability":4,"reasoning":"Categories, weights, deduction amounts, floors, rounding, precedence between categories, and the partial-score fallback are all stated explicitly, so an agent can compute a score with a defined formula. Remaining ambiguities reduce reproducibility: 'typo' and 'meaningful image' require judgment; it's unclear whether a contrast 'pair' or a repeated template defect (e.g., a placeholder in a shared footer) counts once or per page/instance; 'unique error messages' doesn't define uniqueness when messages differ only by URL or line number; and the zero-pages-crawled case (homepage unreachable) causes a division by zero in Reachability. Otherwise the rubric is tight and well-bounded."},"latency_ms":11309,"stop_reason":"end_turn","usage":{"input_tokens":1084,"output_tokens":817},"cost_usd":0.05169}
+{"config":"qa-health-rubric","item":"a08","split":"heldout","phase":"old-b","sample":2,"status":"ok","value":{"clarity":2,"completeness":3,"actionability":2,"reasoning":"Structure, weights (sum to 100%), console/links tiers, and the final formula with rounding are clear and reproducible. However, the per-category scoring is internally inconsistent: deductions are defined for Blocker/Major/Minor (three tiers) while severities are defined as Critical/High/Medium/Low (four tiers) with no mapping between them, so an agent cannot determine how much to deduct for any finding in six of eight categories—this breaks reproducibility for the bulk of the score. Minor gaps also remain: 'coverage' for provisional scores and 'untested' are undefined, root-cause deduplication and category boundaries (e.g., Visual vs UX) rely on judgment, and 'reproducible' console errors lacks a threshold."},"latency_ms":14067,"stop_reason":"end_turn","usage":{"input_tokens":1068,"output_tokens":974},"cost_usd":0.05938}
+{"config":"qa-health-rubric","item":"a07","split":"heldout","phase":"old-a","sample":1,"status":"ok","value":{"clarity":4,"completeness":3,"actionability":4,"reasoning":"Each area has explicit starting value, per-defect deductions, and a floor; weights are stated and sum to 100; the 'first area that fits' rule resolves overlap (e.g., 5xx pages go to Stability, not Navigation), and the viewport note resolves Presentation double-counting. Gaps: the combination step says 'combine using the weights' without writing the weighted-average formula; 'whole number' doesn't specify rounding method; omitting an unchecked area doesn't say whether to renormalize remaining weights; 'crashes' isn't qualified as repeatable the way Navigation's 4xx/5xx is, so intermittent failures are ambiguous; 'distinct issue' in Copy/Accessibility relies on judgment. An agent can produce a mostly reproducible score, but these unaddressed boundaries can cause small inconsistencies."},"latency_ms":15271,"stop_reason":"end_turn","usage":{"input_tokens":882,"output_tokens":997},"cost_usd":0.05867}
+{"config":"qa-health-rubric","item":"f07","split":"heldout","phase":"old-b","sample":1,"status":"ok","value":{"clarity":2,"completeness":2,"actionability":1,"reasoning":"The rubric directly contradicts itself twice: it first says the final score is the lowest category score, then says to take a weighted average; it says untested categories count as 0 'so skipped coverage is visible,' then says to exclude untested categories 'so partial runs are not penalized.' An agent cannot resolve these without guessing, so scores will not be reproducible. Quick mode is also ambiguous (simple vs. weighted average of Console/Links, and whether the 'lowest category' rule applies). Severity levels (Critical/High/Medium/Low) are undefined, 'same root cause' deduplication has no criteria, console warnings vs. errors are unaddressed, and no rounding rule is given. The deduction tables themselves are clear, but the aggregation layer is broken."},"latency_ms":6938,"stop_reason":"end_turn","usage":{"input_tokens":747,"output_tokens":511},"cost_usd":0.03302}
+{"config":"qa-health-rubric","item":"p06","split":"heldout","phase":"new","sample":0,"status":"ok","value":{"clarity":5,"completeness":4,"actionability":4,"reasoning":"Categories, weights (summing to 1.00), per-category score rules, severity point values, and the final weighted-average formula with explicit rounding are all unambiguous. The tie-break order for multi-category findings, dedupe rules for Console and Links, floor-at-0, untested-category handling (renormalize weights, mark provisional, 'not scored' when nothing tested), and 'when unsure, go higher' are well specified. Gaps: Performance has no numeric thresholds for what counts as 'slow', so finding identification there is subjective and hurts reproducibility; 'reproducible' for console errors has no repetition count; what constitutes a category being 'tested' (e.g., partial coverage) is undefined; 'same root cause' consolidation requires judgment. Severity assignment is inherently judgment-based but the four tiers are reasonably distinct. An agent can compute a correct, consistent score from this rubric alone once findings are classified, with only moderate interpretive risk."},"latency_ms":17264,"stop_reason":"end_turn","usage":{"input_tokens":1519,"output_tokens":1006},"cost_usd":0.06549}
+{"config":"qa-health-rubric","item":"a07","split":"heldout","phase":"old-b","sample":1,"status":"ok","value":{"clarity":4,"completeness":3,"actionability":4,"reasoning":"Categories, per-item deductions, floors, and weights are explicit and the weights are stated to sum to 100, so the structure is largely unambiguous. The first-fit dedup rule resolves most cross-area overlaps (e.g., a 5xx page counts under Stability, not Navigation). Gaps remain: 'combine using the weights' never states the formula (weighted average via Σ score×weight/100 is implied, not written); 'leave it out of the overall score' doesn't say whether to renormalize the remaining weights, which materially changes the result; a crashing page that also throws an uncaught exception could plausibly be double-deducted (−30 and −10) with no rule against it; 'workaround' for Flows and 'distinct issue' are judgment-dependent; and rounding method for the whole-number result is unspecified. An agent can produce a mostly correct score from this alone, but reproducibility suffers when an area is skipped or when crash/exception overlap occurs."},"latency_ms":13934,"stop_reason":"end_turn","usage":{"input_tokens":882,"output_tokens":913},"cost_usd":0.05447}
+{"config":"qa-health-rubric","item":"a07","split":"heldout","phase":"old-a","sample":2,"status":"ok","value":{"clarity":4,"completeness":3,"actionability":4,"reasoning":"Each area has a concrete starting value, explicit per-defect deductions, and a floor, and weights are stated and sum to 100. The 'count once, in the first area that fits' rule resolves most double-counting conflicts (e.g., a 5xx link goes to Stability, not Navigation). Weak spots: the combination step says 'combine using the weights' without stating the formula (weighted average = Σ score×weight/100 is implied, not written); 'whole number' gives no rounding rule; the 'leave it out' instruction for unchecked areas does not say how to renormalize the remaining weights, which could produce different overall scores across agents; 'workaround' and 'distinct issue' are undefined; and there is no handling for a flow that completes but with non-blocking bugs. These gaps are small enough that an agent would usually produce the right number, but reproducibility suffers at the edges."},"latency_ms":13482,"stop_reason":"end_turn","usage":{"input_tokens":882,"output_tokens":838},"cost_usd":0.05072}
+{"config":"qa-health-rubric","item":"p07","split":"heldout","phase":"old-b","sample":1,"status":"ok","value":{"clarity":4,"completeness":4,"actionability":5,"reasoning":"Weights, per-category caps, deduction amounts, and speed/console boundaries are explicit and tabulated, with a cross-category precedence rule and a correct worked example (67/85 → 79). Minor ambiguities remain within categories: whether a single element can stack deductions (e.g., a tiny button that also does nothing: −5 and −2? a field with both wrong keyboard and no label: −4 or −8?), what 'smaller than 44×44' means when only one dimension is short, and whether a crashed page's other layout/speed findings still count. Edge cases like SPAs without a load event, console warnings vs. errors, and the (unlikely) all-categories-removed divide-by-zero are undefined. Despite these gaps, the formula, rescaling procedure, rounding rule, and output format let an agent compute a reproducible score from the rubric alone."},"latency_ms":15695,"stop_reason":"end_turn","usage":{"input_tokens":1092,"output_tokens":1095},"cost_usd":0.06567}
+{"config":"qa-health-rubric","item":"f07","split":"heldout","phase":"new","sample":1,"status":"ok","value":{"clarity":2,"completeness":2,"actionability":1,"reasoning":"The rubric contains direct internal contradictions that make reproducible scoring impossible: the intro says the final score is the lowest category score, while the Final Score section says to take a weighted average; the Counting section says untested categories count as 0, while the Final Score section says to exclude them. Quick mode's 'average of Console and Links' is ambiguous (simple vs. weighted). Weights do sum to 100 and the per-category deduction tables are individually clear, but severity levels (Critical/High/Medium/Low) are undefined, 'console error' vs. warning is unspecified, deduplication of 'same root cause' is subjective, and rounding is not addressed. An agent following this alone could legitimately produce several different scores for the same findings."},"latency_ms":7043,"stop_reason":"end_turn","usage":{"input_tokens":1108,"output_tokens":562},"cost_usd":0.03918}
+{"config":"qa-health-rubric","item":"a08","split":"heldout","phase":"old-b","sample":0,"status":"ok","value":{"clarity":2,"completeness":3,"actionability":2,"reasoning":"The rubric is well-structured in most respects: weights sum to 100%, dedup/primary-category ordering is explicit, Console and Links use clear count-based tiers, floors and rounding rules are defined, and the weighted-average formula handles untested categories. However, there is a critical label mismatch at the heart of the per-category scoring: deductions are defined for three tiers (Blocker -25, Major -15, Minor -5) while severity is defined with four different labels (Critical, High, Medium, Low). There is no mapping between them, so an agent cannot determine whether a 'High' or 'Medium' finding costs -25, -15, or -5, nor how four severities collapse into three deduction tiers. Since six of eight categories (65% of weight) depend on this, scores are not reproducible from the rubric alone. Secondary gaps: no definition of what makes a category 'tested' vs 'untested' or how partial coverage affects the category score beyond labeling it provisional."},"latency_ms":14868,"stop_reason":"end_turn","usage":{"input_tokens":1068,"output_tokens":877},"cost_usd":0.05453}
+{"config":"qa-health-rubric","item":"p06","split":"heldout","phase":"old-a","sample":0,"status":"ok","value":{"clarity":4,"completeness":4,"actionability":4,"reasoning":"Strong rubric: single table with explicit weights (sum 1.00), per-category score rules, numeric severity points with level definitions, a tie-break rule ('use the higher'), an explicit one-finding-one-category precedence order, dedup/root-cause rules, floor-at-0, untested-category handling with renormalized weights, and precise rounding (x.5 up at the end). An agent can compute a reproducible number. Deductions: 'slow loads and interactions' has no thresholds or mapping to severity levels; 'confusing/high-friction' UX and 'repeatable'/'reproducible' are subjective without a retry count; 'tested' vs partially tested category is undefined; Visual/Content/Performance severity tiers (e.g., what Critical means for Content) are left to inference. Minor ambiguity remains in classification, not in the arithmetic."},"latency_ms":15356,"stop_reason":"end_turn","usage":{"input_tokens":1158,"output_tokens":985},"cost_usd":0.06083}
+{"config":"qa-health-rubric","item":"p08","split":"heldout","phase":"old-a","sample":0,"status":"ok","value":{"clarity":4,"completeness":3,"actionability":4,"reasoning":"Categories, weights, per-defect deduction amounts, floors, and the combining formula are explicit, and useful edge rules are present (both-widths counts once, redirects to 200 are working, decorative alt='' is fine, one-category-only attribution with lower-number-wins, partial-score handling, round-half-up at end). Gaps remain: 'repeat request' doesn't say how many retries; zero pages crawled (homepage down) yields divide-by-zero; a page with both no <h1> and no <main> (or both missing title and duplicate heading) is ambiguous between one and two deductions; 'text/background pair' isn't scoped per page vs. site-wide unique; 'unique error messages' lacks a matching rule; and 'typo' and 'meaningful image' depend on judgment, which weakens reproducibility. An agent can still compute a defensible score from this alone in most cases."},"latency_ms":12905,"stop_reason":"end_turn","usage":{"input_tokens":1084,"output_tokens":870},"cost_usd":0.05434}
+{"config":"qa-health-rubric","item":"f08","split":"heldout","phase":"new","sample":2,"status":"ok","value":{"clarity":3,"completeness":2,"actionability":1,"reasoning":"The structural logic is clearly laid out (start at 100, deduct, floor at 0, weighted average over tested weights, round only at the end, dedup by root cause with a fixed category precedence), and the formula is explicit. However, every concrete number—per-link penalty, severity penalties, console error bands, and category weights—is delegated to an external 'team QA scoring profile' that is not included, so the rubric is not self-contained. Several terms are undefined: what makes a profile 'out of date', how to classify Critical/High/Medium/Low severity, what to do if neither a profile nor a prior QA report exists, and how 'provisional' labeling affects the output. Edge cases like dedup, untested categories, flooring, and rounding are handled, but the core scoring boundaries are missing. Because an agent literally cannot compute any category score or apply weights from this document alone, actionability is minimal and reproducibility depends entirely on external artifacts."},"latency_ms":9319,"stop_reason":"end_turn","usage":{"input_tokens":1064,"output_tokens":505},"cost_usd":0.03589}
+{"config":"qa-health-rubric","item":"f06","split":"heldout","phase":"new","sample":1,"status":"ok","value":{"clarity":2,"completeness":1,"actionability":1,"reasoning":"The weights and final formula are clear and sum to 100%, but the category criteria are entirely impressionistic ('how noisy and worrying', 'how good the copy is', 'trust your instincts') with no defined deductions, thresholds, or measurable signals. No edge cases are addressed (e.g., what counts as a console error vs. warning, how many broken links drop a score, how to handle untestable categories, what 'acceptable' means concretely). An agent cannot produce a consistent or reproducible score from this; two runs on the same site would yield arbitrarily different numbers."},"latency_ms":3590,"stop_reason":"end_turn","usage":{"input_tokens":1082,"output_tokens":219},"cost_usd":0.02177}
+{"config":"qa-health-rubric","item":"p07","split":"heldout","phase":"old-a","sample":0,"status":"ok","value":{"clarity":4,"completeness":4,"actionability":5,"reasoning":"Categories, point weights, deduction amounts, and speed/console boundaries are explicitly tabulated, with a floor-at-0 rule, table-order precedence for multi-category defects, deduplication rules, a rescaling formula for untested categories, a rounding rule, and a fully worked example — an agent can reproduce a score from this alone. Remaining gaps: 'smaller than 44×44' doesn't say whether failing one dimension counts; 'overlapping element' is ambiguous between per-element and per-pair counting; 'per page with horizontal scroll' sits in slight tension with 'one defect counts once even across pages' when a shared element causes the overflow; and a dead submit button lands in Tap targets (−5) rather than Forms (−8) by the precedence rule, which is deterministic but may surprise. Edge cases like removing the Crashes category or ties for largest deduction are only implicitly handled."},"latency_ms":15588,"stop_reason":"end_turn","usage":{"input_tokens":1092,"output_tokens":973},"cost_usd":0.05957}
+{"config":"qa-health-rubric","item":"a08","split":"heldout","phase":"old-a","sample":1,"status":"ok","value":{"clarity":2,"completeness":3,"actionability":2,"reasoning":"Weights sum to 100, Console/Links count tiers are unambiguous, dedup order and the weighted-average/rounding formula are explicit, and untested-category handling is defined. However, the core per-category scoring is broken: deductions are defined for three labels (Blocker/Major/Minor) while severity definitions use four different labels (Critical/High/Medium/Low) with no mapping between them. An agent cannot determine whether Critical and High both map to Blocker, whether Low maps to Minor or to no deduction, etc., so scores for six of eight categories are not reproducible. Lesser gaps: 'tested' vs 'untested' coverage is undefined, and 'provisional' labeling has no threshold. The label mismatch alone prevents computing a correct score from the rubric as written."},"latency_ms":13925,"stop_reason":"end_turn","usage":{"input_tokens":1068,"output_tokens":915},"cost_usd":0.05643}
+{"config":"qa-health-rubric","item":"f06","split":"heldout","phase":"old-a","sample":1,"status":"ok","value":{"clarity":2,"completeness":1,"actionability":2,"reasoning":"Weights are explicit, sum to 100%, and the aggregation formula with rounding is clear — that part is unambiguous. However, the category scoring is entirely impressionistic: 'how noisy and worrying the console looks', 'how well things work overall', 'trust your instincts'. There are no concrete deduction criteria (e.g., points per broken link, per console error, per failed form), no definition of what a specific finding costs, and only vague anchors (90s/70s/<50) with no boundaries for the gaps between them. Edge cases such as a category with no observable data, severity tiers, or how to handle a single catastrophic failure are undefined. An agent can mechanically compute the weighted average, but two agents testing the same site would almost certainly produce different category scores, so the result is neither consistent nor reproducible."},"latency_ms":7531,"stop_reason":"end_turn","usage":{"input_tokens":721,"output_tokens":296},"cost_usd":0.02201}
+{"config":"qa-health-rubric","item":"f08","split":"heldout","phase":"old-b","sample":2,"status":"ok","value":{"clarity":3,"completeness":2,"actionability":1,"reasoning":"The structure is clear (counting rules, dedup, floor at 0, weighted-average formula, round only at the end), but every concrete number—severity penalties, per-link penalty, console score bands, and category weights—is delegated to an external 'team QA scoring profile' that is not part of the rubric. Severity classification criteria (what makes a finding Critical vs. Low), what 'out of date' means for a profile, and how to handle a missing profile or prior report are undefined. Because the agent has no deduction values or weights, it cannot compute a score from this document alone; results would not be reproducible."},"latency_ms":8799,"stop_reason":"end_turn","usage":{"input_tokens":703,"output_tokens":411},"cost_usd":0.02758}
+{"config":"qa-health-rubric","item":"p06","split":"heldout","phase":"old-b","sample":0,"status":"ok","value":{"clarity":4,"completeness":4,"actionability":5,"reasoning":"Categories, weights, score rules, severity point values, and the final weighted-average formula are all explicit, and a deterministic precedence order resolves multi-category findings. Edge cases like untested categories, nothing tested, floor 0, dedupe, and rounding are covered. Remaining gaps: no numeric threshold for what counts as 'slow' in Performance, 'core task' and 'confusing' are left to judgment (partially mitigated by the 'use the higher' tiebreak), and there's no guidance when one root cause presents with different severities across pages. Still, an agent can compute a reproducible score from this alone given a list of findings."},"latency_ms":13736,"stop_reason":"end_turn","usage":{"input_tokens":1158,"output_tokens":822},"cost_usd":0.05268}
+{"config":"qa-health-rubric","item":"p08","split":"heldout","phase":"new","sample":2,"status":"ok","value":{"clarity":4,"completeness":3,"actionability":4,"reasoning":"Categories, weights, deduction amounts, floors, rounding, and the tie-break rule for cross-category defects are stated explicitly, and the partial-score formula is well defined. Remaining ambiguities: 'duplicates another page's' title doesn't say whether both pages are penalized; Accessibility 'per text/background pair' doesn't say per unique pair or per occurrence; whether a page missing both <h1> and <main> costs 10 or 20 isn't stated (Rendering has an explicit once-only clause, Accessibility does not); 'typo' and 'meaningful image' require judgment. Completeness gaps: no rule for which 50 pages to crawl when the site has more (hurts reproducibility), no handling of zero crawled pages (division by zero if homepage fails), no statement of whether broken pages are excluded from Rendering/Content/A11y checks, and 'same-origin' vs subdomains is undefined. Despite these, an agent can compute a defensible, mostly reproducible score from the rubric alone."},"latency_ms":14767,"stop_reason":"end_turn","usage":{"input_tokens":1445,"output_tokens":1011},"cost_usd":0.065}
+{"config":"qa-health-rubric","item":"p08","split":"heldout","phase":"old-a","sample":2,"status":"ok","value":{"clarity":4,"completeness":4,"actionability":4,"reasoning":"Categories, weights, per-defect deductions, floors, and the combining formula are explicit and concrete (e.g., exact point values, timeout, retry rule, rounding direction, precedence rule for cross-category defects, partial-score handling). Remaining ambiguities: crawl scope/order when >50 pages exist (depth-1 vs recursive, which 50), how many 'repeat requests' before a page is broken, whether a page missing both <h1> and <main> is one or two deductions, whether duplicate titles are charged per page or per pair, whether contrast failures are per unique pair or per occurrence, what 'unique' console message means (exact string match?), and inherently subjective items like 'typo', 'meaningful image', and 'layout break'. Homepage-unreachable case is undefined. Despite these gaps, an agent can produce a reproducible numeric score from this rubric with only minor interpretive choices."},"latency_ms":15044,"stop_reason":"end_turn","usage":{"input_tokens":1084,"output_tokens":935},"cost_usd":0.05759}
+{"config":"qa-health-rubric","item":"f08","split":"heldout","phase":"old-b","sample":0,"status":"ok","value":{"clarity":3,"completeness":2,"actionability":1,"reasoning":"The structure is clear (categories, dedup rule, floor at 0, weighted-average formula with normalization over tested weights, round-only-at-end), but every actual number is delegated to an external 'team QA scoring profile' that is not included: per-link penalty, severity penalties, console bands, and category weights. Severity levels (Critical/High/Medium/Low) are named but never defined, so classification is subjective. Edge cases like a missing or stale profile fall back to 'the last QA report,' which may not exist, and there is no rule for when neither is available or when the profile is only partially out of date. Because the agent cannot compute any category score or weight from the text alone, it is not actionable as a standalone rubric and scores will not be reproducible without the referenced profile."},"latency_ms":8306,"stop_reason":"end_turn","usage":{"input_tokens":703,"output_tokens":408},"cost_usd":0.02743}
+{"config":"qa-health-rubric","item":"f07","split":"heldout","phase":"old-b","sample":0,"status":"ok","value":{"clarity":2,"completeness":2,"actionability":1,"reasoning":"The rubric directly contradicts itself on the two most important rules. Final score: the intro says 'the final score is the lowest category score' while the Final Score section says 'take the weighted average'. Untested categories: Counting says they 'count as 0 so skipped coverage is visible' while Final Score says 'exclude untested categories so partial runs are not penalized'. These are mutually exclusive, so two agents following the same rubric will produce different numbers. Secondary gaps: no definitions of Critical/High/Medium/Low severity for the six per-category areas; 'deduplicate the same root cause across pages' is undefined (does one issue on 5 pages count once?); Quick mode 'average of Console and Links' doesn't say whether it is a simple average or weighted by 15/10; weights are listed twice (inline and in the table) inviting drift; no rounding rule. The individual deduction tables (Console tiers, Links -15, severity deductions) are clear and computable, which keeps clarity/completeness from being 1, but an agent cannot determine the correct final score from this rubric alone."},"latency_ms":8319,"stop_reason":"end_turn","usage":{"input_tokens":747,"output_tokens":584},"cost_usd":0.03667}
+{"config":"qa-health-rubric","item":"f08","split":"heldout","phase":"old-a","sample":0,"status":"ok","value":{"clarity":3,"completeness":2,"actionability":1,"reasoning":"The structure is clear: per-category 0-100 scores, start-at-100-and-deduct, one primary category per root cause, weighted average over tested weights, round only at the end. But every concrete number is deferred to an external 'team QA scoring profile' that is not included: severity penalties, per-link penalty, console score bands, and category weights. Severity classification criteria (what makes a finding Critical vs. High) and how to detect an 'out of date' profile are also undefined. Because an agent cannot derive any numeric value from this document alone, it cannot compute a correct or reproducible score without the missing profile."},"latency_ms":8267,"stop_reason":"end_turn","usage":{"input_tokens":703,"output_tokens":453},"cost_usd":0.02968}
+{"config":"qa-health-rubric","item":"f06","split":"heldout","phase":"old-b","sample":1,"status":"ok","value":{"clarity":2,"completeness":1,"actionability":1,"reasoning":"The weights and final formula are clear and sum to 100%, and the category list is explicit. However, the deduction criteria are entirely impressionistic ('how noisy and worrying', 'how pleasant', 'trust your instincts') with no concrete definitions of what constitutes a deduction, no point values per issue, and no severity tiers. Completeness is very poor: no edge cases are addressed (e.g., what if a category cannot be tested, how to handle zero issues, how many broken links map to what score, what counts as a console 'error' vs 'warning'), and the only anchors are vague bands (90s/70s/below 50). Actionability is minimal because two agents following this rubric would produce widely divergent scores for the same site; it cannot yield a consistent, reproducible result from the rubric alone."},"latency_ms":7132,"stop_reason":"end_turn","usage":{"input_tokens":721,"output_tokens":291},"cost_usd":0.02176}
+{"config":"qa-health-rubric","item":"f06","split":"heldout","phase":"old-b","sample":0,"status":"ok","value":{"clarity":2,"completeness":1,"actionability":2,"reasoning":"Weights and the final formula are explicit and sum to 100%, which is the rubric's only strong point. However, the category definitions are purely impressionistic ('how noisy and worrying', 'how pleasant', 'trust your instincts') with no concrete deduction criteria, thresholds, or examples (e.g., what counts as a broken link, how many console errors drop a score to 70). The anchor guidance (90s/70s/<50) is vague and leaves large undefined gaps. No edge cases are addressed (empty categories, N/A categories, sites with no console output, partial loads). Two agents following this would produce widely divergent scores, so it fails the reproducibility and consistency requirement."},"latency_ms":6090,"stop_reason":"end_turn","usage":{"input_tokens":721,"output_tokens":257},"cost_usd":0.02006}
+{"config":"qa-health-rubric","item":"p08","split":"heldout","phase":"old-b","sample":1,"status":"ok","value":{"clarity":4,"completeness":4,"actionability":4,"reasoning":"Strong rubric: explicit weights, fixed deduction amounts, worked-out formula with rounding rule, floor 0, redirect handling, double-width rule, precedence rule for overlapping defects, and a defined partial-score procedure. Remaining ambiguities: crawl order/selection when more than 50 pages exist (affects reproducibility), whether broken pages are still 'crawled' for Rendering/Content/Accessibility counts, zero-pages-crawled edge case, whether contrast deductions are per unique pair or per instance, whether console errors are deduped across pages, and inherently judgment-laden terms ('typo', 'meaningful image', 'layout break'). An agent can compute a correct score in nearly all cases, with small drift risk at the margins."},"latency_ms":13777,"stop_reason":"end_turn","usage":{"input_tokens":1084,"output_tokens":1021},"cost_usd":0.06189}
+{"config":"qa-health-rubric","item":"p08","split":"heldout","phase":"new","sample":0,"status":"ok","value":{"clarity":4,"completeness":3,"actionability":4,"reasoning":"Categories, weights, and deduction amounts are explicit and numeric, with a clear combining formula, rounding rule, overlap-resolution rule, and partial-score handling. Remaining ambiguities: 'typo' and 'meaningful image' are subjective; 'per text/background pair' doesn't specify unique pair vs. per instance; 'unique error messages' lacks a normalization rule; duplicate-title deductions don't say whether both pages or only the second are penalized; and site-wide defects (e.g., a broken header image or missing <main> in a shared template) could be deducted once per page, letting a single bug zero a category. Edge cases like zero crawlable pages, a broken homepage, or pages discovered beyond the 50-page cap are not addressed. Despite these gaps, an agent can compute a reproducible score from the rubric alone in most cases."},"latency_ms":18789,"stop_reason":"end_turn","usage":{"input_tokens":1445,"output_tokens":939},"cost_usd":0.0614}
diff --git a/test/fixtures/judge-calibration/qa-health-rubric/runs/comparison-1.stop.json b/test/fixtures/judge-calibration/qa-health-rubric/runs/comparison-1.stop.json
new file mode 100644
index 000000000..6339c6830
--- /dev/null
+++ b/test/fixtures/judge-calibration/qa-health-rubric/runs/comparison-1.stop.json
@@ -0,0 +1 @@
+{"stopped":null}
diff --git a/test/skill-llm-eval.test.ts b/test/skill-llm-eval.test.ts
index 7d842d0a7..856ca5cb9 100644
--- a/test/skill-llm-eval.test.ts
+++ b/test/skill-llm-eval.test.ts
@@ -14,7 +14,7 @@ import { afterAll, expect } from 'bun:test';
 import { JUDGE_MS } from './helpers/eval-budgets';
 import * as fs from 'fs';
 import * as path from 'path';
-import { callJudge, judge, JudgeRefusalError, buildQaWorkflowJudgePrompt, buildQaHealthRubricJudgePrompt, buildQaAntiRefusalJudgePrompt, buildCrossSkillConsistencyJudgePrompt, buildVoiceDirectiveJudgePrompt, DEFAULT_JUDGE_MAX_TOKENS, judgePanel, judgePanelMean, judgePanelMajority, judgePanelReasoning, JUDGE_SCORE_DIMENSIONS } from './helpers/llm-judge';
+import { callJudge, judge, JudgeRefusalError, JUDGE_SCORE_SCHEMA, buildQaWorkflowJudgePrompt, buildQaHealthRubricJudgePrompt, buildQaAntiRefusalJudgePrompt, buildCrossSkillConsistencyJudgePrompt, buildVoiceDirectiveJudgePrompt, DEFAULT_JUDGE_MAX_TOKENS, judgePanel, judgePanelMean, judgePanelMajority, judgePanelReasoning, JUDGE_SCORE_DIMENSIONS } from './helpers/llm-judge';
 import { ASK_QUESTIONS_HEADING, ENG_REVIEW_EXCERPT } from './helpers/workflow-excerpt';
 import type { JudgeScore } from './helpers/llm-judge';
 import { readWorkflowJudgeInput, buildWorkflowJudgePrompt, QA_DISCOVERY_REFERENCES, WORKFLOW_JUDGE_RESPONSE_SCHEMA, type WorkflowJudgeInput } from './helpers/workflow-judge-input';
@@ -231,7 +231,7 @@ describeIfSelected('QA skill quality evals', ['qa/SKILL.md workflow', 'qa/SKILL.
     const t0 = Date.now();
     const section = sliceQaPatterns('## Health Score Rubric');
 
-    const samples = await judgePanel(() => callJudge<JudgeScore>(buildQaHealthRubricJudgePrompt(section)));
+    const samples = await judgePanel(() => callJudge<JudgeScore>(buildQaHealthRubricJudgePrompt(section), undefined, { jsonSchema: JUDGE_SCORE_SCHEMA }));
     const scores = judgePanelMean(samples, JUDGE_SCORE_DIMENSIONS);
     console.log('QA health rubric panel:', JSON.stringify({ mean: scores, samples }, null, 2));