diff --git a/test/fixtures/judge-calibration/qa-anti-refusal/runs/comparison-1.result.json b/test/fixtures/judge-calibration/qa-anti-refusal/runs/comparison-1.result.json new file mode 100644 index 000000000..4e128068a --- /dev/null +++ b/test/fixtures/judge-calibration/qa-anti-refusal/runs/comparison-1.result.json @@ -0,0 +1,95 @@ +{ + "config": "qa-anti-refusal", + "model": "claude-fable-5-1", + "request_hash": { + "old": "c00326001bb0e5511ead24b7568cfeafc3a9ae4fe4a4f94a1ed5e556fae12497", + "new": "082cf5eb55dddaf8358a52935ee3477c3e67ad5019137241d9e25f7363d69280" + }, + "complete": true, + "stopped": null, + "calls": 216, + "cost_usd": 7.0201, + "errors": { + "old": { + "refusal": 0, + "truncation": 0, + "parse": 0, + "schema": 0, + "transport": 0, + "samples": 144, + "rate": 0 + }, + "new": { + "refusal": 0, + "truncation": 0, + "parse": 0, + "schema": 0, + "transport": 0, + "samples": 72, + "rate": 0 + } + }, + "latency_p95_ms": { + "old": 12551, + "new": 13420 + }, + "all": { + "items": 24, + "old_flips": { + "flips": 0, + "compared": 24 + }, + "new_flips": { + "flips": 0, + "compared": 24 + }, + "agreement": { + "old-a": 23, + "old-b": 23, + "new": 23 + }, + "false_passes": { + "old-a": [ + "a03" + ], + "old-b": [ + "a03" + ], + "new": [ + "a03" + ] + }, + "new_false_passes": [] + }, + "heldout": { + "items": 8, + "old_flips": { + "flips": 0, + "compared": 8 + }, + "new_flips": { + "flips": 0, + "compared": 8 + }, + "agreement": { + "old-a": 8, + "old-b": 8, + "new": 8 + }, + "false_passes": { + "old-a": [], + "old-b": [], + "new": [] + }, + "new_false_passes": [] + }, + "landing": { + "lands": true, + "reasons": [], + "strict_no_flip_increase": true + }, + "prose_step_triggered": { + "triggered": false, + "reasons": [] + } +} diff --git a/test/fixtures/judge-calibration/qa-anti-refusal/runs/comparison-1.samples.jsonl b/test/fixtures/judge-calibration/qa-anti-refusal/runs/comparison-1.samples.jsonl new file mode 100644 index 000000000..8cb97b412 --- /dev/null +++ b/test/fixtures/judge-calibration/qa-anti-refusal/runs/comparison-1.samples.jsonl @@ -0,0 +1,216 @@ +{"config":"qa-anti-refusal","item":"f05","split":"dev","phase":"old-b","sample":0,"status":"ok","value":{"would_browse":false,"fallback_behavior":"The agent would not launch a browser. It would run the project's test command (bun test, npm test, pytest, or whatever CLAUDE.md specifies) plus any eval suite covering the changed prompt template and config files, then report those results as the QA outcome.","confidence":5,"reasoning":"The document explicitly gates browser launch on a file-path/extension grep filter for UI files. Prompt templates and config files would not match the filter, and the 'No matches' branch directly instructs: 'do not launch the browser' and instead run tests/evals. The Important Rules reinforce this with 'no browser session unless the diff filter above matched.' There is no contradictory instruction to always browse, so the agent would decline to browse and fall back to running tests and evals."},"latency_ms":4109,"stop_reason":"end_turn","usage":{"input_tokens":810,"output_tokens":275},"cost_usd":0.02185} +{"config":"qa-anti-refusal","item":"p05","split":"dev","phase":"old-a","sample":2,"status":"ok","value":{"would_browse":true,"fallback_behavior":"When the diff maps to no pages, the agent falls back to Quick mode: it opens the browser and tests the homepage plus the top 5 navigation links, screenshotting any issues and checking the console after each interaction.","confidence":5,"reasoning":"Both excerpts are explicit and consistent. The diff-aware mode states 'Either way, the browser is always opened' and prescribes a concrete fallback (homepage + top 5 nav links). The Important Rules reinforce this with 'Always use the browser. No diff, however backend-only, is a reason to skip it.' There is no language permitting the agent to decline browsing or substitute evals/unit tests."},"latency_ms":5014,"stop_reason":"end_turn","usage":{"input_tokens":640,"output_tokens":247},"cost_usd":0.01875} +{"config":"qa-anti-refusal","item":"f01","split":"dev","phase":"old-a","sample":1,"status":"ok","value":{"would_browse":false,"fallback_behavior":"The agent would determine that the diff (prompt templates + config files) maps to no identifiable pages, skip browser testing entirely, and recommend running the unit tests or evals that cover the changed files instead.","confidence":5,"reasoning":"Excerpt 1 explicitly states: 'No identifiable pages: the change has no browser surface. Skip browser testing and recommend the unit tests or evals that cover the changed files instead.' The scenario's diff contains only prompt templates and config files — none of the listed mappable categories (controllers/routes/views/components/models/services/styles). Nothing in Excerpt 2 overrides this; it only governs behavior during browser discovery once browsing is underway. The document unambiguously permits (and instructs) declining to browse in this case."},"latency_ms":5298,"stop_reason":"end_turn","usage":{"input_tokens":1113,"output_tokens":285},"cost_usd":0.02538} +{"config":"qa-anti-refusal","item":"f01","split":"dev","phase":"new","sample":2,"status":"ok","value":{"would_browse":false,"fallback_behavior":"The agent would run the diff commands, attempt to map changed files (prompt templates, config) to pages, find no identifiable browser surface, and then skip browser testing entirely, recommending that the user run the unit tests or evals that cover the changed files instead.","confidence":5,"reasoning":"Excerpt 1 explicitly states: 'No identifiable pages: the change has no browser surface. Skip browser testing and recommend the unit tests or evals that cover the changed files instead.' Prompt templates and config files do not map to controllers/routes/views/components/models/services/styles, so the agent would follow this branch and decline to browse. Nothing in Excerpt 2 overrides this; it only governs behavior during browser discovery once browsing has been chosen."},"latency_ms":6730,"stop_reason":"end_turn","usage":{"input_tokens":1424,"output_tokens":268},"cost_usd":0.02764} +{"config":"qa-anti-refusal","item":"f03","split":"dev","phase":"old-b","sample":1,"status":"ok","value":{"would_browse":false,"fallback_behavior":"Excerpt 1 says to fall back to Quick mode (homepage + top 5 nav targets) when no pages are identified, but Important Rule #2 specifically carves out this exact scenario (prompt templates, config, backend-only) and instructs the agent to skip browser testing and run evals or unit tests instead. The agent would most likely cite Rule #2, decline to open the browser, and suggest/run evals or unit tests.","confidence":2,"reasoning":"The document is internally contradictory. Excerpt 1 says 'always use the browser anyway' when no pages are identified. However, Important Rule #2 explicitly names prompt templates and configuration changes and says to skip browser testing in favor of evals/unit tests. Rules sections typically take precedence and Rule #2 matches the scenario precisely, so the agent would most likely not browse. Rule #4 ('never refuse when the user asks') and Excerpt 1 create genuine ambiguity, especially since invoking /qa could be read as asking for the browser, hence low confidence. Because the document permits skipping browser testing for non-UI changes, would_browse is false."},"latency_ms":9324,"stop_reason":"end_turn","usage":{"input_tokens":739,"output_tokens":667},"cost_usd":0.04074} +{"config":"qa-anti-refusal","item":"f03","split":"dev","phase":"old-a","sample":2,"status":"ok","value":{"would_browse":false,"fallback_behavior":"The document is self-contradictory. Excerpt 1 says that if no pages are identified from the diff the agent should 'always use the browser anyway' and fall back to Quick mode (homepage + top 5 nav targets). But Important Rule 2 specifically addresses this exact scenario (backend-only, config, or prompt template changes) and instructs the agent to skip browser testing and run the project's evals or unit tests instead. Because Rule 2 is a specific, directly matching 'Important Rule' while Excerpt 1 is a generic fallback, the agent would most likely decline to browse and suggest/run evals or unit tests, possibly citing Rule 2. Rule 4 ('never refuse when the user asks') and Excerpt 1 give the agent grounds to browse, so some fraction of runs would open the browser in Quick mode.","confidence":2,"reasoning":"The two excerpts directly conflict: Excerpt 1 mandates browsing even when no pages are identified, while Rule 2 explicitly permits (indeed instructs) skipping the browser for precisely the kind of diff in this scenario (prompt templates + config). Since the document allows skipping browser testing for non-UI changes, would_browse is false per the evaluation rules, but the contradiction with Excerpt 1 and Rule 4 makes the actual agent behavior unpredictable, hence low confidence."},"latency_ms":9781,"stop_reason":"end_turn","usage":{"input_tokens":739,"output_tokens":766},"cost_usd":0.04569} +{"config":"qa-anti-refusal","item":"p04","split":"dev","phase":"old-a","sample":0,"status":"ok","value":{"would_browse":true,"fallback_behavior":"The agent would first search the app's pages for where the changed prompt templates and config affect visible output and test those pages. If it cannot locate such a page, it falls back to Quick mode: test the homepage plus the top 5 navigation targets, checking the console after each interaction, then report findings with screenshots scoped to the branch.","confidence":5,"reasoning":"Excerpt 1 step 3 explicitly addresses backend/config/prompt-only diffs and says to 'treat them as browser work, not as an exception,' providing a concrete fallback (Quick mode) when no affected page can be found. Excerpt 2 reinforces this with an explicit rule: 'Never refuse to use the browser' and forbids the exact response 'no UI changes, run the unit tests instead.' The document is unambiguous."},"latency_ms":5445,"stop_reason":"end_turn","usage":{"input_tokens":1130,"output_tokens":294},"cost_usd":0.026} +{"config":"qa-anti-refusal","item":"p04","split":"dev","phase":"new","sample":1,"status":"ok","value":{"would_browse":true,"fallback_behavior":"The agent would first search the app's pages to locate where the changed prompt templates and config values surface in the UI and test those pages. If no such page can be identified, it falls back to Quick mode: load the homepage plus the top 5 navigation targets, checking the console after each interaction, and report findings with screenshots scoped to the branch.","confidence":5,"reasoning":"Excerpt 1 explicitly addresses 'Backend, config and prompt-only diffs' and says to 'treat them as browser work, not as an exception,' providing a concrete fallback (Quick mode) when no UI page can be mapped. Excerpt 2 reinforces this with an unambiguous rule: 'Never refuse to use the browser' and explicitly forbids answering 'no UI changes, run the unit tests instead.' The two excerpts are consistent and directly cover this exact scenario, leaving no room to decline browser testing."},"latency_ms":6683,"stop_reason":"end_turn","usage":{"input_tokens":1441,"output_tokens":317},"cost_usd":0.03026} +{"config":"qa-anti-refusal","item":"a03","split":"dev","phase":"old-a","sample":1,"status":"ok","value":{"would_browse":true,"fallback_behavior":"When the diff yields no mapped UI pages, the agent would still treat the running web app as the selected browser surface: it would check commits/PR intent and TODOs to infer which flows the prompt/config changes affect, discover a local app via the port scan (3000/4000/8080) or use the supplied/staging URL (asking if none is found), open the app in the browser, and exercise realistic end-to-end flows and adjacent pages (plus browser-surface API probes via fetch) that would be influenced by the changed prompts/config — rather than suggesting evals or unit tests as a substitute.","confidence":4,"reasoning":"Excerpt 2 explicitly states 'Never refuse to use the browser for a selected browser surface, even backend-only app changes. Tests/curl cannot replace it.' Since the user invoked /qa (a browser-based skill) against a web application, the app itself is the selected browser surface, and prompt-template/config changes fall squarely under 'backend-only app changes.' Excerpt 1 also provides a fallback path (commit intent, TODO bugs, local app discovery, adjacent flows) when the diff doesn't map directly to pages. The only ambiguity is the clause that API/CLI/job/worker/webhook targets 'do not select' the browser surface — if the agent misclassified the changed files as a pure job/worker target it might hesitate — but the scenario describes an application with UI, so the dominant instruction is to browse."},"latency_ms":12097,"stop_reason":"end_turn","usage":{"input_tokens":1129,"output_tokens":782},"cost_usd":0.05039} +{"config":"qa-anti-refusal","item":"p01","split":"dev","phase":"old-b","sample":1,"status":"ok","value":{"would_browse":true,"fallback_behavior":"When the diff maps to no identifiable pages, the agent falls back to running the Quick test plan plus any interactions it discovers in the live app, explicitly including backend/config/infrastructure-only changes. It locates the local app (probing ports 3000/4000/8080 or using a supplied/staging URL, asking if none is found), opens the browser, exercises realistic end-to-end flows while checking the console, and flags any new bugs in the Phase 6 report rather than suggesting evals or unit tests.","confidence":5,"reasoning":"Both excerpts directly address this case. Excerpt 1 states: 'No identifiable pages: use Quick plus discovered interactions, even for backend/config/infrastructure changes.' Excerpt 2 states: 'Never refuse to use the browser for a selected browser surface, even backend-only app changes. Tests/curl cannot replace it.' Prompt template and config changes are backend/config changes to an app with a browser surface, so the agent would browse. The only carve-out is for API/CLI/job/worker/webhook targets that don't select a browser surface, which does not apply here."},"latency_ms":7208,"stop_reason":"end_turn","usage":{"input_tokens":1168,"output_tokens":399},"cost_usd":0.03163} +{"config":"qa-anti-refusal","item":"p02","split":"dev","phase":"old-b","sample":2,"status":"ok","value":{"would_browse":true,"fallback_behavior":"When no obvious pages/routes are identified from the diff, the agent falls back to Quick mode: detect the running app on common local ports (3000/4000/8080) or ask for a URL, navigate to the homepage, follow the top 5 navigation targets, check the console for errors after each interaction, test any interactive elements found, take screenshots and show them via the Read tool, and report findings. It would not suggest evals or unit tests as a substitute.","confidence":5,"reasoning":"Both excerpts explicitly and unambiguously address this exact scenario. Excerpt 1 states 'If no obvious pages/routes are identified from the diff: Do not skip browser testing' and prescribes the Quick mode fallback, noting that backend/config changes still affect app behavior. Excerpt 2 Rule 12 says 'Never refuse to use the browser' and 'Never suggest evals, unit tests, or other alternatives as a substitute. Even if the diff appears to have no UI changes... always open the browser and test.' There is no contradictory language permitting the agent to skip browsing for non-UI diffs."},"latency_ms":7123,"stop_reason":"end_turn","usage":{"input_tokens":1958,"output_tokens":393},"cost_usd":0.03923} +{"config":"qa-anti-refusal","item":"a01","split":"dev","phase":"old-a","sample":0,"status":"ok","value":{"would_browse":false,"fallback_behavior":"With no routes/views/components/CSS in the diff, the diff-aware mapping step yields 'no identifiable pages.' The document then tells the agent to 'consider a Quick pass of the homepage if time permits; otherwise note that no UI surface was found.' Since the 'never refuse' rule is explicitly gated on a browser surface having been selected (and 'API/CLI/job/worker/webhook targets do not select it'), a purely prompt-template/config diff gives the agent a sanctioned path to skip browsing, write up that no UI surface was found, and suggest evals/unit tests instead—possibly with an optional, discretionary homepage Quick pass.","confidence":3,"reasoning":"Excerpt 2's 'never refuse the browser, even for backend-only app changes' sounds absolute, but it is conditioned on 'a selected browser surface,' and selection happens in Excerpt 1's mapping step. Excerpt 1 provides an explicit escape hatch for 'no identifiable pages' ('if time permits; otherwise note that no UI surface was found'), which an agent would readily invoke for a diff touching only prompt templates and config. The two excerpts are in tension, so confidence is moderate, but the 'no UI surface found' off-ramp is the governing instruction for this scenario."},"latency_ms":14677,"stop_reason":"end_turn","usage":{"input_tokens":1172,"output_tokens":987},"cost_usd":0.06107} +{"config":"qa-anti-refusal","item":"f04","split":"dev","phase":"new","sample":0,"status":"ok","value":{"would_browse":false,"fallback_behavior":"The agent would classify the diff as functional-surface only (prompt templates + config), skip loading browser setup entirely, and instead run repository-native tools: the project's unit tests, eval suite, CLI commands, and API calls to verify the changes.","confidence":5,"reasoning":"Excerpt 1 explicitly lists prompt templates and configuration under 'Functional surface' with 'Do not load browser setup', and states that a diff containing only functional-surface files selects no browser surface so the browser is never opened. Excerpt 2's 'never refuse the browser' rule applies only to a selected browser surface, and it explicitly says prompt/config targets do not select it and the browser is not used. Both excerpts consistently direct the agent away from browsing for this scenario."},"latency_ms":5662,"stop_reason":"end_turn","usage":{"input_tokens":1135,"output_tokens":274},"cost_usd":0.02505} +{"config":"qa-anti-refusal","item":"p05","split":"dev","phase":"old-b","sample":2,"status":"ok","value":{"would_browse":true,"fallback_behavior":"When the diff maps to no pages (as here, with only prompt templates and config files changed), the agent falls back to Quick mode: it opens the browser and tests the homepage plus the top 5 navigation links, screenshotting any issues and checking the console after every interaction.","confidence":5,"reasoning":"Both excerpts are explicit and consistent. The diff-aware mode says that if the diff maps to no pages, the agent should still test the homepage and top 5 nav links, and states 'the browser is always opened.' The Important Rules reinforce this with 'Always use the browser. No diff, however backend-only, is a reason to skip it.' There is no language permitting a fallback to evals or unit tests instead of browsing."},"latency_ms":5623,"stop_reason":"end_turn","usage":{"input_tokens":640,"output_tokens":266},"cost_usd":0.0197} +{"config":"qa-anti-refusal","item":"f05","split":"dev","phase":"new","sample":2,"status":"ok","value":{"would_browse":false,"fallback_behavior":"The agent would not launch the browser. It would run the project's test command (bun test, npm test, pytest, or the command specified in CLAUDE.md) plus any eval suite covering the changed prompt template and config files, then report those results as the QA outcome.","confidence":5,"reasoning":"The diff-aware mode explicitly runs a grep filter for UI-related paths and extensions; prompt templates and config files would not match. The document unambiguously says 'No matches: do not launch the browser' and instead run tests/evals. The Important Rules reinforce this with 'no browser session unless the diff filter above matched.' There is no contradictory instruction to always browse."},"latency_ms":5115,"stop_reason":"end_turn","usage":{"input_tokens":1121,"output_tokens":237},"cost_usd":0.02306} +{"config":"qa-anti-refusal","item":"p03","split":"dev","phase":"new","sample":0,"status":"ok","value":{"would_browse":true,"fallback_behavior":"When the diff maps to no identifiable pages, the agent runs Quick mode — homepage plus the top 5 navigation targets plus discovered interactions — and reports what it saw. It may mention unit tests or evals as additional coverage only after completing the browser pass, and must not ask the user whether browser testing is needed.","confidence":5,"reasoning":"Both excerpts explicitly address this exact scenario. Excerpt 1 says 'No identifiable pages: use Quick plus discovered interactions, even for backend/config/infrastructure changes.' Excerpt 2 Rule 3 says 'Always open the browser... whatever the diff contains: prompt templates, config files...' and Rule 4 forbids substituting tests/evals for the browser pass or asking the user whether browsing is needed. The document is unambiguous."},"latency_ms":5665,"stop_reason":"end_turn","usage":{"input_tokens":1570,"output_tokens":294},"cost_usd":0.0304} +{"config":"qa-anti-refusal","item":"a06","split":"dev","phase":"new","sample":0,"status":"ok","value":{"would_browse":true,"fallback_behavior":"When the diff maps to no identifiable pages (prompt templates and config only), the agent would still open the browser and run Quick mode — homepage plus top 5 navigation targets — but keep that browser pass capped at about 30 seconds per the Important Rules, then spend the remainder of the run executing the project's unit tests to verify the logic changes more precisely.","confidence":4,"reasoning":"Excerpt 1 explicitly mandates browser use via Quick mode 'even for backend, config, or prompt-only changes,' so the agent would not refuse to browse. Excerpt 2 does not override this; it only time-boxes the browser pass to 30 seconds for backend-only diffs and redirects remaining effort to unit tests. The two excerpts are consistent that browsing happens, though the 30-second cap and unit-test emphasis introduce mild tension about how much browser testing occurs, hence not a 5."},"latency_ms":8232,"stop_reason":"end_turn","usage":{"input_tokens":1056,"output_tokens":493},"cost_usd":0.03521} +{"config":"qa-anti-refusal","item":"f04","split":"dev","phase":"new","sample":2,"status":"ok","value":{"would_browse":false,"fallback_behavior":"The agent would classify the diff as functional-surface only (prompt templates and config files), skip loading browser setup entirely, and instead test the changes using repository-native tools: the project's unit tests, eval suite, CLI commands, and API calls.","confidence":5,"reasoning":"Excerpt 1 explicitly lists prompt templates and configuration under 'Functional surface' with the instruction 'Do not load browser setup,' and states that a diff containing only functional-surface files 'selects no browser surface, so the browser is never opened for it.' Excerpt 2 reinforces this: the 'never refuse' rule applies only to a selected browser surface, and it explicitly says prompt/config targets do not select the browser surface and that repository-native tests and evals are the correct method. Both excerpts are consistent and unambiguous for this scenario."},"latency_ms":5804,"stop_reason":"end_turn","usage":{"input_tokens":1135,"output_tokens":298},"cost_usd":0.02625} +{"config":"qa-anti-refusal","item":"f04","split":"dev","phase":"old-a","sample":2,"status":"ok","value":{"would_browse":false,"fallback_behavior":"The agent would classify the diff as functional-surface only (prompt templates + config), skip loading browser setup entirely, and instead run repository-native tests: the project's unit tests, eval suite, CLI commands, and/or API calls against the changed behavior.","confidence":5,"reasoning":"Excerpt 1 explicitly lists prompt templates and configuration as 'Functional surface' with the instruction 'Do not load browser setup' and states 'A diff that contains only functional-surface files selects no browser surface, so the browser is never opened for it.' Excerpt 2 reinforces this: the 'never refuse the browser' rule applies only to a *selected* browser surface, and it explicitly says prompt/config targets 'do not select it' and 'the browser is not used.' Both excerpts are consistent and unambiguous for this scenario."},"latency_ms":5947,"stop_reason":"end_turn","usage":{"input_tokens":824,"output_tokens":308},"cost_usd":0.02364} +{"config":"qa-anti-refusal","item":"a01","split":"dev","phase":"old-b","sample":0,"status":"ok","value":{"would_browse":false,"fallback_behavior":"When the diff maps to no identifiable pages, the agent may optionally do a Quick pass of the homepage 'if time permits', but is explicitly permitted to instead just note that no UI surface was found. Since the 'never refuse' rule is conditioned on 'a selected browser surface' and the diff-aware flow only browses 'after selecting and isolating a browser surface', a prompt-template/config-only diff gives the agent a sanctioned path to skip browsing and report no UI surface (likely suggesting evals/unit tests instead).","confidence":3,"reasoning":"Excerpt 2's 'Never refuse to use the browser... even backend-only app changes' sounds absolute, but it is scoped to 'a selected browser surface', and the same paragraph says API/CLI/job/worker/webhook targets do not select it. Excerpt 1 explicitly provides an opt-out: 'No identifiable pages: consider a Quick pass of the homepage if time permits; otherwise note that no UI surface was found.' That 'otherwise' clause permits declining to browse for a purely backend diff. The document is somewhat contradictory (strong rule vs. optional fallback), hence moderate confidence."},"latency_ms":10346,"stop_reason":"end_turn","usage":{"input_tokens":1172,"output_tokens":733},"cost_usd":0.04837} +{"config":"qa-anti-refusal","item":"f01","split":"dev","phase":"new","sample":1,"status":"ok","value":{"would_browse":false,"fallback_behavior":"The agent would run the git diff, find only prompt template and config files with no mappable pages/routes/views/components, conclude the change has no browser surface, skip browser testing, and recommend running the unit tests or evals that cover the changed files instead.","confidence":5,"reasoning":"Excerpt 1 explicitly states: 'No identifiable pages: the change has no browser surface. Skip browser testing and recommend the unit tests or evals that cover the changed files instead.' Prompt templates and config files do not map to controllers/routes/views/components/models/services/styles, so the diff-aware mode would classify this as no browser surface and decline to browse. Excerpt 2 contains no rule overriding this (it only governs behavior during browser discovery once browsing is underway)."},"latency_ms":5762,"stop_reason":"end_turn","usage":{"input_tokens":1424,"output_tokens":272},"cost_usd":0.02784} +{"config":"qa-anti-refusal","item":"p03","split":"dev","phase":"old-a","sample":0,"status":"ok","value":{"would_browse":true,"fallback_behavior":"When the diff maps to no identifiable pages, the agent runs Quick mode: it opens the browser on the homepage plus the top 5 navigation targets (plus discovered interactions), checks the console after each interaction, and reports what it saw. It may only mention evals/unit tests as additional coverage after completing the browser pass, and must not ask the user whether browser testing is needed.","confidence":5,"reasoning":"Rule 3 explicitly says 'Always open the browser' on every run regardless of diff contents, naming prompt templates and config files specifically. Excerpt 1 reinforces this: 'No identifiable pages: use Quick plus discovered interactions, even for backend/config/infrastructure changes.' Rule 4 forbids substituting tests/evals for the browser pass and forbids asking the user whether browsing is needed. The document is unambiguous and consistent."},"latency_ms":5906,"stop_reason":"end_turn","usage":{"input_tokens":1259,"output_tokens":320},"cost_usd":0.02859} +{"config":"qa-anti-refusal","item":"a06","split":"dev","phase":"new","sample":1,"status":"ok","value":{"would_browse":true,"fallback_behavior":"With no identifiable pages from the diff (prompt templates and config only), the agent would fall back to Quick mode in the browser — loading the homepage plus the top 5 navigation targets — but would keep that browser pass short (capped at ~30 seconds per the Important Rules), screenshotting any issues and checking the console, then spend the remainder of the run executing the project's unit tests to verify the logic/config changes more precisely.","confidence":4,"reasoning":"Excerpt 1 explicitly mandates browser use via Quick mode 'even for backend, config, or prompt-only changes,' so the agent would not refuse to browse. Excerpt 2 does not create an exception to skip the browser; it only caps the browser pass at 30 seconds for backend-only diffs and redirects remaining effort to unit tests. The two excerpts are consistent in requiring at least a brief browser pass, though the 30-second cap and emphasis on unit tests being 'more precise' introduces some risk an agent might treat the browser step as token or skippable, which is why confidence is 4 rather than 5."},"latency_ms":9982,"stop_reason":"end_turn","usage":{"input_tokens":1056,"output_tokens":565},"cost_usd":0.03881} +{"config":"qa-anti-refusal","item":"f02","split":"dev","phase":"old-b","sample":2,"status":"ok","value":{"would_browse":false,"fallback_behavior":"The agent would classify the diff as Non-UI (prompt templates and config fall explicitly in that category), not open the browser, report 'No UI changes detected on this branch; browser QA skipped', stop, and suggest the user run the project's unit tests or evals for coverage of the changed files.","confidence":5,"reasoning":"Excerpt 1 explicitly lists 'config' and 'prompt templates' as Non-UI diff examples and gives a direct instruction for that case: 'browser QA is not applicable. Do not open the browser.' Excerpt 2 reinforces this with 'Scope to the diff' and 'Don't browse for nothing.' There is no contradicting instruction to always browse, so the document unambiguously causes the agent to refuse browser testing in this scenario."},"latency_ms":6100,"stop_reason":"end_turn","usage":{"input_tokens":875,"output_tokens":290},"cost_usd":0.02325} +{"config":"qa-anti-refusal","item":"f03","split":"dev","phase":"old-b","sample":0,"status":"ok","value":{"would_browse":false,"fallback_behavior":"Per Important Rule 2, the agent would conclude there is nothing for a browser to see, skip browser testing, and run the project's evals or unit tests instead — though Excerpt 1 directly contradicts this by instructing it to fall back to Quick mode (homepage + top 5 nav targets) in the browser.","confidence":2,"reasoning":"The document is internally contradictory. Excerpt 1 says 'always use the browser anyway' when no pages are identified from the diff. But Important Rule 2 explicitly names this exact scenario (backend code, configuration, prompt templates) and instructs the agent to skip browser testing and run evals/unit tests. Because Rule 2 is a specific, scenario-matching permission to skip browsing, the document 'allows the agent to skip browser testing for non-UI changes,' so would_browse is false per the evaluation criteria. Rule 4 ('never refuse when the user asks') could be read to override since the user invoked /qa, but Rule 2's specificity makes refusal the more likely outcome. Confidence is low due to the direct conflict."},"latency_ms":8652,"stop_reason":"end_turn","usage":{"input_tokens":739,"output_tokens":715},"cost_usd":0.04314} +{"config":"qa-anti-refusal","item":"a04","split":"dev","phase":"old-b","sample":2,"status":"ok","value":{"would_browse":false,"fallback_behavior":"The agent would likely run a Quick pass of the homepage, but the document explicitly grants it discretion to skip browser testing and defer to unit tests or evals for backend-only branches like prompt templates and config changes.","confidence":4,"reasoning":"Excerpt 1 says browser testing is 'generally preferred' and a homepage Quick pass is 'usually still worthwhile,' but it immediately hedges with 'Use your judgment' and states that 'for some backend-only branches the unit tests or evals already give better signal.' Per the evaluation rules, since the document permits skipping the browser for non-UI changes, would_browse is false. The soft language makes it plausible the agent would decline and suggest evals for a prompt-template-only diff."},"latency_ms":3596,"stop_reason":"end_turn","usage":{"input_tokens":733,"output_tokens":276},"cost_usd":0.02113} +{"config":"qa-anti-refusal","item":"p02","split":"dev","phase":"old-a","sample":2,"status":"ok","value":{"would_browse":true,"fallback_behavior":"When no obvious pages/routes are identified from the diff, the agent falls back to Quick mode: detect the running app on common local ports (3000/4000/8080) or a staging URL, navigate to the homepage, follow the top 5 navigation targets, check the console for errors after each interaction, test any interactive elements found, take screenshots (and Read them so the user sees them), and report findings — never suggesting evals or unit tests as a substitute.","confidence":5,"reasoning":"Both excerpts explicitly and redundantly address this exact scenario. Excerpt 1 step 2 says 'If no obvious pages/routes are identified from the diff: Do not skip browser testing... Fall back to Quick mode' and notes that backend/config changes affect app behavior. Excerpt 2 rule 12 says 'Never refuse to use the browser... Even if the diff appears to have no UI changes... always open the browser and test' and forbids suggesting evals or unit tests as alternatives. There is no ambiguity or contradiction."},"latency_ms":6823,"stop_reason":"end_turn","usage":{"input_tokens":1958,"output_tokens":368},"cost_usd":0.03798} +{"config":"qa-anti-refusal","item":"p05","split":"dev","phase":"new","sample":2,"status":"ok","value":{"would_browse":true,"fallback_behavior":"When the diff maps to no UI pages, the agent falls back to Quick mode: it opens the browser and tests the homepage plus the top 5 navigation links, screenshotting any issues and checking the console after each interaction.","confidence":5,"reasoning":"Both excerpts are explicit and consistent: the diff-aware section states 'Either way, the browser is always opened' and prescribes a concrete fallback (homepage + top 5 nav links) when no pages are affected, and the Important Rules state 'Always use the browser. No diff, however backend-only, is a reason to skip it.' There is no provision allowing the agent to decline browsing or substitute evals/unit tests."},"latency_ms":5537,"stop_reason":"end_turn","usage":{"input_tokens":951,"output_tokens":242},"cost_usd":0.02161} +{"config":"qa-anti-refusal","item":"p05","split":"dev","phase":"old-b","sample":0,"status":"ok","value":{"would_browse":true,"fallback_behavior":"When the diff maps to no UI pages (prompt templates and config files only), the agent falls back to Quick mode: it opens the browser and tests the homepage plus the top 5 navigation links, screenshotting any issues and checking the console after every interaction.","confidence":5,"reasoning":"Both excerpts are explicit and consistent. Excerpt 1 states that if the diff maps to no pages, the agent should test the homepage and top 5 nav links in the browser, and that 'the browser is always opened.' Excerpt 2 reinforces this with 'Always use the browser. No diff, however backend-only, is a reason to skip it.' There is no language permitting a fallback to evals or unit tests, so the agent would browse."},"latency_ms":5603,"stop_reason":"end_turn","usage":{"input_tokens":640,"output_tokens":269},"cost_usd":0.01985} +{"config":"qa-anti-refusal","item":"a05","split":"dev","phase":"old-b","sample":1,"status":"ok","value":{"would_browse":true,"fallback_behavior":"When no identifiable pages map from the diff, the agent runs the Quick test plan plus discovered interactions against the local/staging app (locating it via localhost port scan or asking for a URL if none responds), exploring realistic end-to-end flows and checking console after each interaction, then flags any new bugs in the Phase 6 report.","confidence":5,"reasoning":"Excerpt 1 explicitly states that with no identifiable pages the agent should 'use Quick plus discovered interactions, even for backend/config/infrastructure changes.' Excerpt 2 reinforces this with 'Never refuse to use the browser for a selected browser surface, even backend-only app changes. Tests/curl cannot replace it.' The only exception is the user explicitly asking not to open the browser, which does not apply in this scenario. Prompt template and config changes fall squarely under the backend/config case the document says still requires browsing."},"latency_ms":6107,"stop_reason":"end_turn","usage":{"input_tokens":1226,"output_tokens":329},"cost_usd":0.02871} +{"config":"qa-anti-refusal","item":"a01","split":"dev","phase":"new","sample":0,"status":"ok","value":{"would_browse":false,"fallback_behavior":"With no identifiable pages mapped from the diff, the agent would most likely either do a discretionary Quick pass of the homepage 'if time permits' or simply note that no UI surface was found and suggest evals/unit tests, since the 'never refuse' rule is scoped to an already 'selected browser surface' and the diff-aware section explicitly offers a non-browser exit path.","confidence":2,"reasoning":"The two excerpts conflict. Excerpt 2 says 'Never refuse to use the browser for a selected browser surface, even backend-only app changes,' which pushes toward browsing. But it is conditioned on a surface being 'selected,' and Excerpt 1's diff-aware flow makes selection depend on mapping changed files to pages; prompt templates and config files map to nothing. The explicit fallback — 'consider a Quick pass of the homepage if time permits; otherwise note that no UI surface was found' — gives the agent permission to skip browsing entirely. Because the mandatory rule is gated on a selection step that this scenario fails, and the fallback is discretionary, the document permits (and arguably steers toward) declining to browse, though an agent could reasonably read it either way."},"latency_ms":9715,"stop_reason":"end_turn","usage":{"input_tokens":1483,"output_tokens":734},"cost_usd":0.05153} +{"config":"qa-anti-refusal","item":"p04","split":"dev","phase":"old-b","sample":1,"status":"ok","value":{"would_browse":true,"fallback_behavior":"The agent would first search the app for pages where the changed prompt template output or config-controlled features appear and test those. If no such page can be identified, it falls back to Quick mode: test the homepage plus the top 5 navigation targets, checking the browser console after each interaction, and report findings with screenshots scoped to the branch.","confidence":5,"reasoning":"Excerpt 1 step 3 explicitly addresses backend/config/prompt-only diffs and says to 'treat them as browser work, not as an exception,' with a concrete fallback (Quick mode) when no affected page is found. Excerpt 2 reinforces this with a hard rule: 'Never refuse to use the browser' and explicitly forbids the response 'no UI changes, run the unit tests instead.' The document is unambiguous and directly covers this exact scenario."},"latency_ms":6608,"stop_reason":"end_turn","usage":{"input_tokens":1130,"output_tokens":303},"cost_usd":0.02645} +{"config":"qa-anti-refusal","item":"f05","split":"dev","phase":"old-b","sample":1,"status":"ok","value":{"would_browse":false,"fallback_behavior":"The agent would not launch the browser. It would run the project's test command (bun test, npm test, pytest, or whatever CLAUDE.md specifies) plus any eval suite covering the changed prompt template and config files, then report those results as the QA outcome.","confidence":5,"reasoning":"The document explicitly gates browser use behind a diff filter matching view/component/page/CSS/template files. Prompt template and config file changes would not match that grep pattern, and the 'No matches' branch directly instructs: 'do not launch the browser' and run tests/evals instead. The Important Rules reinforce this with 'no browser session unless the diff filter above matched.' Both excerpts are consistent and unambiguous."},"latency_ms":3791,"stop_reason":"end_turn","usage":{"input_tokens":810,"output_tokens":263},"cost_usd":0.02125} +{"config":"qa-anti-refusal","item":"p03","split":"dev","phase":"old-a","sample":2,"status":"ok","value":{"would_browse":true,"fallback_behavior":"When the diff maps to no identifiable pages, the agent runs Quick mode — opens the homepage plus the top 5 navigation targets, exercises discovered interactions, checks the console after each interaction, and reports what it saw. It may mention afterward that evals or unit tests would add coverage, but only after completing the browser pass, and it must not ask the user whether browser testing is needed.","confidence":5,"reasoning":"Both excerpts are explicit and consistent. Excerpt 1 says 'No identifiable pages: use Quick plus discovered interactions, even for backend/config/infrastructure changes.' Excerpt 2, Rule 3 ('Always open the browser... whatever the diff contains: prompt templates, config files...') directly names this scenario, and Rule 4 forbids substituting tests/evals for the browser pass or asking the user whether browsing is needed. There is no ambiguity or contradiction permitting the agent to skip the browser."},"latency_ms":6092,"stop_reason":"end_turn","usage":{"input_tokens":1259,"output_tokens":347},"cost_usd":0.02994} +{"config":"qa-anti-refusal","item":"a03","split":"dev","phase":"new","sample":1,"status":"ok","value":{"would_browse":true,"fallback_behavior":"When the diff maps to no specific pages, the agent would still treat the running web app as the selected browser surface, locate the local app (probe ports 3000/4000/8080 or ask for a URL), check commit/PR intent and TODOS.md to infer which user-facing flows the prompt templates and config affect (e.g., AI/chat features that consume the templates), and exercise realistic end-to-end flows on those and adjacent pages while watching the console, rather than deferring to evals or unit tests.","confidence":4,"reasoning":"Excerpt 2 explicitly states 'Never refuse to use the browser for a selected browser surface, even backend-only app changes. Tests/curl cannot replace it.' Prompt template and config changes are backend-only app changes, not API/CLI/job/worker/webhook targets, so the exclusion clause does not apply. The only ambiguity is the phrase 'selected browser surface' and the 'After selecting and isolating a browser surface' step in Excerpt 1, which could be read as a gate; however, since /qa is a browser-based skill and the app itself is the surface, the clear intent is to browse regardless of diff content."},"latency_ms":10930,"stop_reason":"end_turn","usage":{"input_tokens":1440,"output_tokens":692},"cost_usd":0.049} +{"config":"qa-anti-refusal","item":"f03","split":"dev","phase":"new","sample":0,"status":"ok","value":{"would_browse":false,"fallback_behavior":"The agent would most likely cite Rule 2 ('Respect the diff'), note that the diff touches only prompt templates and config files with no UI surface, skip browser testing, and run the project's evals or unit tests instead — though Excerpt 1's 'always use the browser anyway' fallback (Quick mode: homepage + top 5 nav targets) and Rule 4 give a competing instruction that some agents might follow.","confidence":2,"reasoning":"The two excerpts directly contradict each other for this exact scenario. Excerpt 1 says that when no pages are identified from the diff the agent should 'always use the browser anyway' and fall back to Quick mode. Excerpt 2 Rule 2 explicitly names backend code, configuration, and prompt-template-only changes and instructs the agent to skip browser testing in favor of evals/unit tests. Because Rule 2 is more specific to the scenario (it literally enumerates 'prompt templates' and 'configuration') and is framed as an 'Important Rule', it provides clear permission to skip browsing, which per the evaluation criteria makes would_browse false. Rule 4 ('never refuse when the user asks') could be read as overriding since the user invoked /qa, but the user did not explicitly request a browser session, so Rule 2 likely governs. The document is contradictory, hence low confidence."},"latency_ms":11021,"stop_reason":"end_turn","usage":{"input_tokens":1050,"output_tokens":676},"cost_usd":0.0443} +{"config":"qa-anti-refusal","item":"f04","split":"dev","phase":"old-b","sample":2,"status":"ok","value":{"would_browse":false,"fallback_behavior":"The agent would classify the diff as functional-surface only (prompt templates + config), skip loading browser setup entirely, and instead test the changes using repository-native tools: the project's unit tests, eval suite, CLI commands, and API calls.","confidence":5,"reasoning":"Excerpt 1 explicitly lists prompt templates and configuration under 'Functional surface' with the instruction 'Do not load browser setup' and states that a diff containing only functional-surface files 'selects no browser surface, so the browser is never opened for it.' Excerpt 2 reinforces this: the 'never refuse' rule applies only to a selected browser surface, and prompt/config targets explicitly 'do not select it.' Both excerpts are consistent and unambiguous that the agent should not browse in this scenario."},"latency_ms":5621,"stop_reason":"end_turn","usage":{"input_tokens":824,"output_tokens":294},"cost_usd":0.02294} +{"config":"qa-anti-refusal","item":"f05","split":"dev","phase":"new","sample":1,"status":"ok","value":{"would_browse":false,"fallback_behavior":"The agent would not launch the browser. It would run the project's test command (bun test, npm test, pytest, or the command in CLAUDE.md) plus any eval suite covering the changed prompt template and config files, then report those results as the QA outcome.","confidence":5,"reasoning":"The document explicitly instructs that when the diff filter (views/components/pages/public paths and UI file extensions) finds no matches, the agent must not launch the browser and should instead run tests/evals. Prompt template and config file changes would not match the grep pattern. The Important Rules section reinforces this: 'no browser session unless the diff filter above matched.' The instructions are unambiguous and consistent."},"latency_ms":5220,"stop_reason":"end_turn","usage":{"input_tokens":1121,"output_tokens":243},"cost_usd":0.02336} +{"config":"qa-anti-refusal","item":"a04","split":"dev","phase":"old-a","sample":0,"status":"ok","value":{"would_browse":false,"fallback_behavior":"The agent would run the diff and log commands, find that no changed files map to routes, views, components, or styles, and then exercise the judgment the document grants it. Given that the changes are prompt templates and config files with no UI surface, it would likely conclude this is one of the 'backend-only branches' where evals or unit tests give better signal, and decline or deprioritize the browser pass, suggesting evals/unit tests instead. It might offer a Quick homepage pass as optional, but the document does not require it.","confidence":4,"reasoning":"Excerpt 1 explicitly permits skipping browser testing: 'Use your judgment about whether it adds value for this particular diff; for some backend-only branches the unit tests or evals already give better signal.' The scenario (prompt templates + config, no UI files) is precisely the case that language carves out. While the document hedges ('generally preferred', 'usually still worthwhile'), it does not mandate browser use regardless of diff content, so the agent is allowed—and likely—to decline browsing. The Important Rules in Excerpt 2 only govern how to test once browsing, not whether to browse. Confidence is 4 rather than 5 because the soft 'generally preferred' wording leaves some room for an agent to opt into a Quick homepage pass."},"latency_ms":8479,"stop_reason":"end_turn","usage":{"input_tokens":733,"output_tokens":620},"cost_usd":0.03833} +{"config":"qa-anti-refusal","item":"p04","split":"dev","phase":"old-a","sample":2,"status":"ok","value":{"would_browse":true,"fallback_behavior":"The agent would first search the app's pages for where the changed prompt templates and config affect UI output and test those pages. If it cannot locate such a page, it falls back to Quick mode: testing the homepage plus the top 5 navigation targets, checking the console after each interaction, and reporting findings with screenshots scoped to the branch.","confidence":5,"reasoning":"Excerpt 1 explicitly addresses 'Backend, config and prompt-only diffs' and instructs the agent to 'treat them as browser work, not as an exception,' providing a concrete fallback (Quick mode) when no affected page can be identified. Excerpt 2 reinforces this with the rule 'Never refuse to use the browser' and directly forbids the exact response 'no UI changes, run the unit tests instead.' The document is unambiguous that the agent must open the browser in this scenario."},"latency_ms":6087,"stop_reason":"end_turn","usage":{"input_tokens":1130,"output_tokens":315},"cost_usd":0.02705} +{"config":"qa-anti-refusal","item":"p02","split":"dev","phase":"old-a","sample":0,"status":"ok","value":{"would_browse":true,"fallback_behavior":"When no obvious pages/routes are identified from the diff, the agent falls back to Quick mode: detect the running app on common local ports, navigate to the homepage, follow the top 5 navigation targets, check the console for errors, and test any interactive elements found, then report findings with screenshots.","confidence":5,"reasoning":"Both excerpts explicitly and unambiguously instruct the agent to never skip browser testing. Excerpt 1 states 'If no obvious pages/routes are identified from the diff: Do not skip browser testing... Fall back to Quick mode' and Excerpt 2's Rule 12 says 'Never refuse to use the browser... Never suggest evals, unit tests, or other alternatives as a substitute. Even if the diff appears to have no UI changes... always open the browser and test.' The scenario (only prompt templates and config changes) maps directly to this fallback case."},"latency_ms":5769,"stop_reason":"end_turn","usage":{"input_tokens":1958,"output_tokens":325},"cost_usd":0.03583} +{"config":"qa-anti-refusal","item":"p02","split":"dev","phase":"new","sample":0,"status":"ok","value":{"would_browse":true,"fallback_behavior":"When no obvious pages/routes are identified from the diff, the agent falls back to Quick mode: it detects the running app on common local ports (3000/4000/8080) or a staging URL, navigates to the homepage, follows the top 5 navigation targets, checks the console for JS errors after each interaction, tests any interactive elements found, takes screenshots and shows them to the user, cross-references commit messages/PR description for intent, checks TODOS.md, and reports findings scoped to the branch.","confidence":5,"reasoning":"Both excerpts explicitly address this exact scenario. Excerpt 1 says 'If no obvious pages/routes are identified from the diff: Do not skip browser testing... Fall back to Quick mode' and that backend/config changes must still be verified. Excerpt 2 Rule 12 says 'Never refuse to use the browser... Never suggest evals, unit tests, or other alternatives as a substitute. Even if the diff appears to have no UI changes... always open the browser and test.' The instructions are unambiguous and consistent, so the agent would open the browser rather than decline."},"latency_ms":6393,"stop_reason":"end_turn","usage":{"input_tokens":2269,"output_tokens":384},"cost_usd":0.04189} +{"config":"qa-anti-refusal","item":"a04","split":"dev","phase":"new","sample":0,"status":"ok","value":{"would_browse":false,"fallback_behavior":"The agent would note that no changed files map to a page, weigh whether a Quick pass of the homepage adds value for this prompt-template/config-only diff, and likely decline or skip browser testing in favor of suggesting the unit tests or evals already give better signal — possibly offering a minimal homepage smoke check as optional.","confidence":4,"reasoning":"Excerpt 1 explicitly permits skipping the browser: 'Use your judgment about whether it adds value for this particular diff; for some backend-only branches the unit tests or evals already give better signal.' While it says browser testing is 'generally preferred' and a homepage Quick pass is 'usually still worthwhile,' the hedged language gives the agent latitude to decline for a diff consisting only of prompt templates and config. Nothing in the Important Rules overrides this. Since the question is whether the document mandates browsing regardless of diff content, the answer is no — it allows skipping for non-UI changes."},"latency_ms":6774,"stop_reason":"end_turn","usage":{"input_tokens":1044,"output_tokens":335},"cost_usd":0.02719} +{"config":"qa-anti-refusal","item":"f02","split":"dev","phase":"old-b","sample":0,"status":"ok","value":{"would_browse":false,"fallback_behavior":"The agent would classify the diff as Non-UI (prompt templates and config files are explicitly listed as non-UI), not open the browser, report 'No UI changes detected on this branch; browser QA skipped', stop, and suggest the user run the project's unit tests or evals for coverage of the changed files.","confidence":5,"reasoning":"Excerpt 1 explicitly classifies prompt templates and config as Non-UI and gives a direct instruction: 'browser QA is not applicable. Do not open the browser.' Excerpt 2 reinforces this with 'Don't browse for nothing' and 'Scope to the diff.' There is no contradictory language; the document unambiguously directs the agent to skip browser testing in this scenario."},"latency_ms":5485,"stop_reason":"end_turn","usage":{"input_tokens":875,"output_tokens":274},"cost_usd":0.02245} +{"config":"qa-anti-refusal","item":"f03","split":"dev","phase":"old-b","sample":2,"status":"ok","value":{"would_browse":false,"fallback_behavior":"When the diff maps to no UI pages, the agent would most likely invoke Important Rule #2 ('Respect the diff'), decline to open the browser, and instead run the project's evals or unit tests, explaining that prompt-template/config changes have no browser-visible surface. It would likely offer to browse if the user explicitly requests it (per Rule #4).","confidence":2,"reasoning":"The document is internally contradictory. Excerpt 1 says 'always use the browser anyway' and fall back to Quick mode when no pages are identified. But Excerpt 2's Rule #2 directly and specifically addresses this exact scenario (backend code, configuration, prompt templates) and instructs the agent to skip browser testing in favor of evals/unit tests. Because Rule #2 is the more specific instruction, sits under an 'Important Rules' heading, and matches the scenario precisely, an agent would most likely follow it over the general diff-aware fallback. Rule #4 ('never refuse when the user asks for it') could push toward browsing since the user invoked /qa, but it is phrased as a response to an explicit user request rather than an override of Rule #2. The contradiction makes the outcome uncertain, hence low confidence."},"latency_ms":8594,"stop_reason":"end_turn","usage":{"input_tokens":739,"output_tokens":654},"cost_usd":0.04009} +{"config":"qa-anti-refusal","item":"p03","split":"dev","phase":"old-b","sample":1,"status":"ok","value":{"would_browse":true,"fallback_behavior":"When the diff maps to no identifiable pages, the agent runs Quick mode (homepage plus the top 5 navigation targets) along with discovered interactions, reports what it saw, and may only afterward mention that unit tests or evals would add coverage — never instead of the browser pass, and without asking the user whether browser testing is needed.","confidence":5,"reasoning":"Rule 3 explicitly states 'Always open the browser' on every run regardless of diff contents, specifically naming prompt templates and config files as examples. Excerpt 1's 'No identifiable pages' clause reinforces this by instructing Quick mode even for backend/config/infrastructure changes. Rule 4 forbids substituting tests/evals for the browser pass and forbids asking the user whether browsing is needed. The document is unambiguous and consistent."},"latency_ms":5807,"stop_reason":"end_turn","usage":{"input_tokens":1259,"output_tokens":304},"cost_usd":0.02779} +{"config":"qa-anti-refusal","item":"a03","split":"dev","phase":"old-b","sample":2,"status":"ok","value":{"would_browse":true,"fallback_behavior":"When the diff yields no obvious UI pages, the agent would still treat the app as the selected browser surface (since /qa is a browser QA skill and the changes are app changes, not API/CLI/job/worker/webhook targets). It would check commit/PR intent to infer which flows the prompt templates and config affect, discover a local app via the port scan (3000/4000/8080) or ask for a URL, then open the browser and run realistic end-to-end flows on the pages/features most likely influenced by the backend changes (e.g., AI-driven features, forms, flows using the changed config), checking console after each interaction and reporting bugs in Phase 6.","confidence":4,"reasoning":"Excerpt 2 explicitly states 'Never refuse to use the browser for a selected browser surface, even backend-only app changes. Tests/curl cannot replace it.' Prompt templates and config files are backend-only app changes, which this rule directly covers. The only exemption is for API/CLI/job/worker/webhook targets, which don't apply here. Excerpt 1 reinforces this by instructing the agent to use commit/PR intent, find a local app, and test changed/adjacent flows rather than bail out. Minor ambiguity exists around what counts as a 'selected' browser surface, but invoking a browser-based /qa skill on a web app strongly implies selection, so the agent would browse rather than suggest evals/unit tests."},"latency_ms":12789,"stop_reason":"end_turn","usage":{"input_tokens":1129,"output_tokens":861},"cost_usd":0.05434} +{"config":"qa-anti-refusal","item":"a05","split":"dev","phase":"new","sample":2,"status":"ok","value":{"would_browse":true,"fallback_behavior":"When the diff maps to no identifiable pages, the agent falls back to the Quick test plan plus discovered interactions: it locates the local app (probing ports 3000/4000/8080 or asking for a URL), opens the browser, exercises realistic end-to-end flows on the main/adjacent pages, checks the console after each interaction, and reports findings. It does not substitute evals, unit tests, or curl for browser coverage.","confidence":5,"reasoning":"Excerpt 1 explicitly states that with no identifiable pages the agent should 'use Quick plus discovered interactions, even for backend/config/infrastructure changes.' Excerpt 2 reinforces this with 'Never refuse to use the browser for a selected browser surface, even backend-only app changes. Tests/curl cannot replace it.' The only carve-out is when the user explicitly says the change is backend-only and asks not to open the browser, which is not the case in this scenario. Prompt template and config changes to a web app fall squarely under 'backend/config' changes, so the agent would browse."},"latency_ms":5122,"stop_reason":"end_turn","usage":{"input_tokens":1537,"output_tokens":368},"cost_usd":0.03377} +{"config":"qa-anti-refusal","item":"a05","split":"dev","phase":"old-a","sample":2,"status":"ok","value":{"would_browse":true,"fallback_behavior":"When no identifiable pages map from the diff (prompt templates and config only), the agent runs the Quick test tier plus any discovered interactions on the running app — finding a local app on ports 3000/4000/8080 or using a supplied/staging URL, asking only if none responds — and flags new bugs in the Phase 6 report. It would not substitute evals or unit tests for browser coverage unless the user explicitly asked it not to open the browser in this request.","confidence":5,"reasoning":"Both excerpts are explicit: 'No identifiable pages: use Quick plus discovered interactions, even for backend/config/infrastructure changes' and 'Never refuse to use the browser for a selected browser surface, even backend-only app changes. Tests/curl cannot replace it.' The only carve-out is a user explicitly stating the change is backend-only and asking not to open the browser, which did not occur in this scenario. Prompt template and config changes fall squarely under the backend/config case the document says still requires browsing."},"latency_ms":6677,"stop_reason":"end_turn","usage":{"input_tokens":1226,"output_tokens":359},"cost_usd":0.03021} +{"config":"qa-anti-refusal","item":"a04","split":"dev","phase":"old-b","sample":1,"status":"ok","value":{"would_browse":false,"fallback_behavior":"The agent would note that nothing in the diff maps to a page, consider a Quick pass of the homepage as 'usually still worthwhile', but then exercise judgment per the explicit allowance that 'for some backend-only branches the unit tests or evals already give better signal.' Given the diff is purely prompt templates and config with no UI surface, the agent would likely decline to browse and suggest running evals or unit tests instead, possibly offering an optional homepage smoke test.","confidence":4,"reasoning":"Excerpt 1 explicitly grants the agent discretion to skip browser testing: 'Use your judgment about whether it adds value for this particular diff; for some backend-only branches the unit tests or evals already give better signal.' The language is hedged ('generally preferred', 'usually still worthwhile') rather than mandatory. A prompt-template-and-config diff is the archetypal case this escape hatch describes, so an agent is more likely to defer to evals/unit tests than to open the browser. The Important Rules in Excerpt 2 govern how to test, not whether to test, so they do not override the discretion. Confidence is 4 rather than 5 because the document does lean toward browsing as the default, so some agents might still do a quick homepage pass."},"latency_ms":5875,"stop_reason":"end_turn","usage":{"input_tokens":733,"output_tokens":439},"cost_usd":0.02928} +{"config":"qa-anti-refusal","item":"a02","split":"dev","phase":"old-b","sample":1,"status":"ok","value":{"would_browse":false,"fallback_behavior":"The agent would note that the diff contains only prompt-template and config changes, conclude that no browser surface is selected, and recommend running the eval suite and unit tests instead of opening the browser. It might fall back to 'Quick mode' browsing only if it ignored Excerpt 2's guidance.","confidence":3,"reasoning":"Excerpt 2 explicitly states that prompt-template and config changes are 'usually exercised through the eval suite and unit tests' and that non-UI targets 'do not select the browser surface,' which gives the agent clear permission to skip browsing. Excerpt 1's 'After selecting and isolating a browser surface' gating also implies browsing only happens once a browser surface is selected. However, Excerpt 1 also says 'No identifiable pages: use Quick mode,' which pushes toward browsing anyway, creating some tension; the more specific rule in Excerpt 2 about prompt/config changes would likely dominate, so the agent would decline to browse."},"latency_ms":8459,"stop_reason":"end_turn","usage":{"input_tokens":1136,"output_tokens":671},"cost_usd":0.04491} +{"config":"qa-anti-refusal","item":"p05","split":"dev","phase":"old-a","sample":0,"status":"ok","value":{"would_browse":true,"fallback_behavior":"When the diff maps to no UI pages, the agent falls back to Quick mode: it opens the browser and tests the homepage plus the top 5 navigation links, screenshotting any issues and checking the console after each interaction.","confidence":5,"reasoning":"Both excerpts are explicit and consistent. The diff-aware instructions state that if the diff maps to no pages, the agent must still test the homepage and top 5 nav links, and that 'the browser is always opened.' The Important Rules reinforce this: 'Always use the browser. No diff, however backend-only, is a reason to skip it.' There is no provision allowing the agent to decline browsing or substitute evals/unit tests for prompt-template-only or config-only changes."},"latency_ms":5211,"stop_reason":"end_turn","usage":{"input_tokens":640,"output_tokens":270},"cost_usd":0.0199} +{"config":"qa-anti-refusal","item":"a05","split":"dev","phase":"new","sample":1,"status":"ok","value":{"would_browse":true,"fallback_behavior":"When no identifiable pages map from the diff, the agent falls back to Quick mode plus discovered interactions: it locates the local app (probing ports 3000/4000/8080 or using a supplied/staging URL, asking if none found), opens the browser, exercises realistic end-to-end flows and adjacent pages, checks console after each interaction, and reports findings. It only skips browser coverage if the user explicitly states the change is backend-only and asks not to open the browser.","confidence":5,"reasoning":"Excerpt 1 explicitly says 'No identifiable pages: use Quick plus discovered interactions, even for backend/config/infrastructure changes.' Excerpt 2 reinforces with 'Never refuse to use the browser for a selected browser surface, even backend-only app changes. Tests/curl cannot replace it.' The only exception requires an explicit user instruction not to browse, which is absent in this scenario. Prompt template and config changes are app changes (not API/CLI/job/worker/webhook targets), so the browser surface remains selected."},"latency_ms":4931,"stop_reason":"end_turn","usage":{"input_tokens":1537,"output_tokens":367},"cost_usd":0.03372} +{"config":"qa-anti-refusal","item":"a04","split":"dev","phase":"new","sample":2,"status":"ok","value":{"would_browse":false,"fallback_behavior":"The agent would likely run a Quick pass of the homepage as a default, but the document explicitly permits it to skip browser testing and defer to unit tests or evals for backend-only branches where browsing adds no value, so it may decline and suggest running evals/unit tests instead.","confidence":4,"reasoning":"Excerpt 1 says browser testing is 'generally preferred' and a homepage Quick pass is 'usually still worthwhile,' but it explicitly adds 'Use your judgment' and states that 'for some backend-only branches the unit tests or evals already give better signal.' This hedged language grants the agent discretion to skip the browser for prompt-template/config-only diffs, so the document does not mandate browsing regardless of diff content. The scenario (prompt templates + config) is a textbook case the escape hatch is written for, making a refusal/deferral plausible even though the default lean is toward a quick homepage check."},"latency_ms":26362,"stop_reason":"end_turn","usage":{"input_tokens":1044,"output_tokens":324},"cost_usd":0.02664} +{"config":"qa-anti-refusal","item":"f02","split":"dev","phase":"old-a","sample":1,"status":"ok","value":{"would_browse":false,"fallback_behavior":"The agent would classify the diff as Non-UI (prompt templates and config files are explicitly listed in that category), not open the browser, report \"No UI changes detected on this branch; browser QA skipped\", stop, and suggest the user run the project's unit tests or evals for coverage of the changed files.","confidence":5,"reasoning":"Excerpt 1 explicitly classifies prompt templates and config as Non-UI diffs and instructs the agent to 'not open the browser' and stop with a skip message plus a suggestion to run unit tests/evals. Excerpt 2 reinforces this with 'Don't browse for nothing' and 'Scope to the diff.' There is no contradictory instruction to always browse, so the document unambiguously leads to refusing browser QA in this scenario."},"latency_ms":5767,"stop_reason":"end_turn","usage":{"input_tokens":875,"output_tokens":291},"cost_usd":0.0233} +{"config":"qa-anti-refusal","item":"p01","split":"dev","phase":"old-b","sample":2,"status":"ok","value":{"would_browse":true,"fallback_behavior":"When no identifiable pages map from the diff, the agent falls back to running the Quick test plan plus discovered interactions against the running app (locating it on localhost:3000/4000/8080 or using a supplied/staging URL), rather than declining. It would open the browser, exercise realistic end-to-end flows, check the console after each interaction, and flag any new bugs in the Phase 6 report.","confidence":5,"reasoning":"Both excerpts explicitly cover this case. Excerpt 1 states: 'No identifiable pages: use Quick plus discovered interactions, even for backend/config/infrastructure changes.' Excerpt 2 states: 'Never refuse to use the browser for a selected browser surface, even backend-only app changes. Tests/curl cannot replace it.' Prompt template and config changes are backend/config changes to a web app with a browser surface, so the agent is directed to browse rather than suggest evals or unit tests. The only exception is when the target itself is an API/CLI/job/worker/webhook, which does not apply to a web app QA run."},"latency_ms":7083,"stop_reason":"end_turn","usage":{"input_tokens":1168,"output_tokens":381},"cost_usd":0.03073} +{"config":"qa-anti-refusal","item":"p02","split":"dev","phase":"new","sample":2,"status":"ok","value":{"would_browse":true,"fallback_behavior":"When no obvious pages/routes are identified from the diff, the agent falls back to Quick mode: it detects the running app on common local ports (3000/4000/8080) or asks for a URL, navigates to the homepage, follows the top 5 navigation targets, checks the console for errors after each interaction, tests any interactive elements found, takes screenshots and shows them to the user, and reports findings — treating the prompt template and config changes as backend changes that could affect app behavior.","confidence":5,"reasoning":"Both excerpts explicitly and unambiguously instruct the agent never to skip browser testing. Excerpt 1 step 2 says 'If no obvious pages/routes are identified from the diff: Do not skip browser testing... Fall back to Quick mode' and 'Backend, config, and infrastructure changes affect app behavior — always verify the app still works.' Excerpt 2 rule 12 says 'Never refuse to use the browser... Never suggest evals, unit tests, or other alternatives as a substitute. Even if the diff appears to have no UI changes... always open the browser and test.' The scenario of prompt-template and config-only changes is directly covered by these instructions, so the agent would open the browser rather than suggest evals or unit tests."},"latency_ms":6869,"stop_reason":"end_turn","usage":{"input_tokens":2269,"output_tokens":431},"cost_usd":0.04424} +{"config":"qa-anti-refusal","item":"p04","split":"dev","phase":"new","sample":2,"status":"ok","value":{"would_browse":true,"fallback_behavior":"The agent would search the app for pages where the changed prompt templates/config output appears and test those; if none can be found, it falls back to Quick mode: testing the homepage plus the top 5 navigation targets, checking the console on each, and reporting findings with screenshots.","confidence":5,"reasoning":"Excerpt 1 step 3 explicitly addresses 'Backend, config and prompt-only diffs' and says to treat them as browser work, not an exception, with a concrete fallback (Quick mode) when no affected page can be identified. Excerpt 2 reinforces this with an unambiguous rule: 'Never refuse to use the browser' and explicitly forbids answering 'no UI changes, run the unit tests instead.' Both excerpts are consistent and directly cover this scenario."},"latency_ms":5611,"stop_reason":"end_turn","usage":{"input_tokens":1441,"output_tokens":280},"cost_usd":0.02841} +{"config":"qa-anti-refusal","item":"a01","split":"dev","phase":"old-a","sample":1,"status":"ok","value":{"would_browse":false,"fallback_behavior":"When the diff maps to no pages, the agent is told to 'consider a Quick pass of the homepage if time permits; otherwise note that no UI surface was found.' Since this is discretionary ('consider', 'if time permits') and the fallback explicitly sanctions reporting that no UI surface exists, the agent would likely conclude that prompt-template/config changes do not 'select' a browser surface, skip browsing, and report no UI surface found (plausibly suggesting evals/unit tests instead).","confidence":3,"reasoning":"Excerpt 2's 'Never refuse to use the browser... even backend-only app changes' sounds absolute, but it is gated on 'a selected browser surface,' and the same paragraph says non-UI targets 'do not select it.' Excerpt 1 further conditions app discovery on 'After selecting and isolating a browser surface' and provides an explicit escape hatch for 'No identifiable pages' that merely suggests (not mandates) a homepage pass and permits noting no UI surface was found. Together these give the agent permission to decline browsing for a diff of only prompt templates and config files. The tension between 'even backend-only app changes' and the discretionary fallback makes the document somewhat contradictory, hence moderate confidence."},"latency_ms":12378,"stop_reason":"end_turn","usage":{"input_tokens":1172,"output_tokens":865},"cost_usd":0.05497} +{"config":"qa-anti-refusal","item":"a03","split":"dev","phase":"old-b","sample":0,"status":"ok","value":{"would_browse":true,"fallback_behavior":"When the diff maps to no specific pages, the agent would still treat the web app as the selected browser surface, check commits/PR intent and TODOS.md for related items, locate the local app by probing ports 3000/4000/8080 (or ask for a URL if none responds), then open the browser and run realistic end-to-end flows on adjacent/core pages (e.g., any flow that exercises the prompt templates or configured behavior), checking the console after each interaction and reporting any new bugs in the Phase 6 report. It would not substitute evals, curl, or unit tests for the browser session.","confidence":4,"reasoning":"Excerpt 2 explicitly states 'Never refuse to use the browser for a selected browser surface, even backend-only app changes. Tests/curl cannot replace it.' Prompt-template and config changes to a web app are exactly 'backend-only app changes', so the agent is directed to browse anyway. The only carve-out is for targets that are themselves API/CLI/job/worker/webhook, which does not describe a web app whose backend prompts/config changed. Excerpt 1 further instructs the agent to find the local app and 'test changed/adjacent pages and flows' rather than bail out. The minor ambiguity is around what counts as a 'selected browser surface', since excerpt 1 references a selection step not fully shown, but the 'even backend-only app changes' clause strongly covers this scenario."},"latency_ms":17760,"stop_reason":"end_turn","usage":{"input_tokens":1129,"output_tokens":881},"cost_usd":0.05534} +{"config":"qa-anti-refusal","item":"p02","split":"dev","phase":"old-b","sample":0,"status":"ok","value":{"would_browse":true,"fallback_behavior":"When no obvious pages/routes are identified from the diff, the agent falls back to Quick mode: detect the running app on common local ports, navigate to the homepage, follow the top 5 navigation targets, check console for errors after each interaction, test any interactive elements found, take screenshots (and Read them so the user sees them), and report findings. It would not suggest evals or unit tests as a substitute.","confidence":5,"reasoning":"Both excerpts explicitly address this exact scenario. Excerpt 1 step 2 says 'If no obvious pages/routes are identified from the diff: Do not skip browser testing... Fall back to Quick mode' and notes that backend/config changes affect app behavior. Excerpt 2 Rule 12 states 'Never refuse to use the browser... Never suggest evals, unit tests, or other alternatives as a substitute. Even if the diff appears to have no UI changes... always open the browser and test.' The instructions are unambiguous and non-contradictory."},"latency_ms":5712,"stop_reason":"end_turn","usage":{"input_tokens":1958,"output_tokens":357},"cost_usd":0.03743} +{"config":"qa-anti-refusal","item":"f02","split":"dev","phase":"new","sample":2,"status":"ok","value":{"would_browse":false,"fallback_behavior":"The agent would classify the diff as Non-UI (prompt templates and config are explicitly listed as non-UI), not open the browser, report 'No UI changes detected on this branch; browser QA skipped', stop, and suggest the user run the project's unit tests or evals for coverage of the changed files.","confidence":5,"reasoning":"Excerpt 1 explicitly lists config and prompt templates under Non-UI diff and instructs: 'browser QA is not applicable. Do not open the browser.' Excerpt 2 reinforces this with 'Don't browse for nothing' and 'Scope to the diff.' The document is unambiguous that the agent should skip browsing in this scenario."},"latency_ms":5144,"stop_reason":"end_turn","usage":{"input_tokens":1186,"output_tokens":250},"cost_usd":0.02436} +{"config":"qa-anti-refusal","item":"a05","split":"dev","phase":"old-a","sample":0,"status":"ok","value":{"would_browse":true,"fallback_behavior":"With no identifiable pages from the diff, the agent would fall back to the Quick test mode plus discovered interactions: locate the local app (port scan 3000/4000/8080 or use supplied/staging URL, asking if none found), open the browser, exercise realistic end-to-end flows on the running app, check console after each interaction, and flag any new bugs in the Phase 6 report. It would not substitute evals, unit tests, or curl for browser coverage.","confidence":5,"reasoning":"Excerpt 1 explicitly states that when there are no identifiable pages, the agent should 'use Quick plus discovered interactions, even for backend/config/infrastructure changes.' Excerpt 2 reinforces this with 'Never refuse to use the browser for a selected browser surface, even backend-only app changes. Tests/curl cannot replace it.' The only carve-out is when the user explicitly says the change is backend-only AND asks not to open the browser, which is not the case in this scenario. Prompt templates and config files are backend/config changes to a web app (a browser surface), not an API/CLI/job/worker/webhook target, so the browser is required."},"latency_ms":7124,"stop_reason":"end_turn","usage":{"input_tokens":1226,"output_tokens":407},"cost_usd":0.03261} +{"config":"qa-anti-refusal","item":"a06","split":"dev","phase":"old-a","sample":1,"status":"ok","value":{"would_browse":true,"fallback_behavior":"With no identifiable pages from the diff, the agent would open the browser in Quick mode (homepage plus top 5 navigation targets), keep that browser pass short (capped around 30 seconds per the backend-only rule), screenshot any issues and check the console, and then spend the remainder of the run executing the project's unit tests to verify the prompt/config logic changes.","confidence":4,"reasoning":"Excerpt 1 explicitly mandates a browser Quick-mode pass 'even for backend, config, or prompt-only changes' when no pages can be identified. Excerpt 2 does not permit skipping the browser; it only caps the browser pass at 30 seconds and then redirects remaining effort to unit tests. So the agent would still browse, just briefly, rather than refusing. The 30-second cap and unit-test emphasis introduce a mild tension that could tempt an agent to minimize browsing, but neither excerpt authorizes declining to browse, hence confidence 4 rather than 5."},"latency_ms":7617,"stop_reason":"end_turn","usage":{"input_tokens":745,"output_tokens":574},"cost_usd":0.03615} +{"config":"qa-anti-refusal","item":"f03","split":"dev","phase":"old-a","sample":1,"status":"ok","value":{"would_browse":false,"fallback_behavior":"Per Important Rule 2, the agent would conclude there is nothing for a browser to see and skip browser testing, running the project's evals or unit tests instead. This conflicts with the Diff-aware section, which says to always use the browser anyway and fall back to Quick mode (homepage plus top 5 navigation targets), so some agents might instead open the browser and do a Quick-mode pass.","confidence":2,"reasoning":"The document is internally contradictory. Excerpt 1 says 'If no pages are identified from the diff: always use the browser anyway.' But Excerpt 2 Rule 2 explicitly names this exact scenario (backend code, configuration, prompt templates) and instructs the agent to skip browser testing in favor of evals/unit tests. Because Rule 2 is the more specific instruction and directly matches the scenario, and because the evaluation criterion says would_browse is false whenever the document allows skipping browser testing for non-UI changes, the likely outcome is that the agent declines to browse. Rule 4 ('never refuse when the user asks') could be read as overriding since the user invoked /qa, but it is weaker than the explicit Rule 2 carve-out. Low confidence due to the contradiction."},"latency_ms":9134,"stop_reason":"end_turn","usage":{"input_tokens":739,"output_tokens":709},"cost_usd":0.04284} +{"config":"qa-anti-refusal","item":"p05","split":"dev","phase":"old-a","sample":1,"status":"ok","value":{"would_browse":true,"fallback_behavior":"When the diff maps to no pages, the agent falls back to Quick mode: it opens the browser and tests the homepage plus the top 5 navigation links, screenshotting any issues and checking the console after every interaction.","confidence":5,"reasoning":"Both excerpts are explicit and consistent. The diff-aware section states that if the diff maps to no pages, the agent should test the homepage and top 5 nav links in the browser, and that 'the browser is always opened.' The Important Rules reinforce this: 'Always use the browser. No diff, however backend-only, is a reason to skip it.' There is no language permitting a fallback to evals or unit tests, so the agent would browse rather than decline."},"latency_ms":5262,"stop_reason":"end_turn","usage":{"input_tokens":640,"output_tokens":259},"cost_usd":0.01935} +{"config":"qa-anti-refusal","item":"f05","split":"dev","phase":"new","sample":0,"status":"ok","value":{"would_browse":false,"fallback_behavior":"The agent would not launch the browser. It would run the project's test command (bun test, npm test, pytest, or the command specified in CLAUDE.md) plus any eval suite covering the changed prompt template and config files, then report those results as the QA outcome.","confidence":5,"reasoning":"The document explicitly instructs that the browser should only be launched when the diff filter matches UI-related paths or extensions. Prompt templates and config files would not match the grep pattern (views, components, pages, public, or css/html/tsx/jsx/vue/erb extensions). The 'No matches' branch directly says 'do not launch the browser' and to run tests/evals instead, and the Important Rules reinforce this with 'no browser session unless the diff filter above matched.' The document is unambiguous."},"latency_ms":5764,"stop_reason":"end_turn","usage":{"input_tokens":1121,"output_tokens":277},"cost_usd":0.02506} +{"config":"qa-anti-refusal","item":"f04","split":"dev","phase":"old-a","sample":1,"status":"ok","value":{"would_browse":false,"fallback_behavior":"The agent would classify the diff as functional-surface only (prompt templates and config), skip loading browser setup entirely, and instead test the changes with repository-native tools: the project's unit tests, eval suite, CLI commands, and API calls. It would not open a browser or attempt to find UI pages to test.","confidence":5,"reasoning":"Excerpt 1 explicitly lists prompt templates and configuration as 'Functional surface' and instructs 'Do not load browser setup' for them, adding that a diff with only functional-surface files 'selects no browser surface, so the browser is never opened for it.' Excerpt 2 reinforces this: the 'never refuse the browser' rule applies only to a selected browser surface, and it explicitly states prompt/config targets do not select the browser, with tests and evals being the correct method. Both excerpts are consistent and unambiguous for this scenario."},"latency_ms":6433,"stop_reason":"end_turn","usage":{"input_tokens":824,"output_tokens":323},"cost_usd":0.02439} +{"config":"qa-anti-refusal","item":"a05","split":"dev","phase":"old-b","sample":0,"status":"ok","value":{"would_browse":true,"fallback_behavior":"With no identifiable pages from the diff, the agent would fall back to the Quick test mode plus discovered interactions: locate the local app (ports 3000/4000/8080 or supplied/staging URL, asking if none found), open it in the browser, exercise realistic end-to-end flows and discovered interactions, check the console after each interaction, and report any new bugs in the Phase 6 report — recording browser coverage as run rather than suggesting evals or unit tests instead.","confidence":5,"reasoning":"Both excerpts explicitly address this exact case. Excerpt 1 says 'No identifiable pages: use Quick plus discovered interactions, even for backend/config/infrastructure changes.' Excerpt 2 states 'Never refuse to use the browser for a selected browser surface, even backend-only app changes. Tests/curl cannot replace it.' The only exception is when the user explicitly asks not to open the browser in the current request, which does not apply here. Prompt template and config changes are backend/config changes to a web app, not API/CLI/job/worker/webhook targets, so the browser surface remains selected."},"latency_ms":29711,"stop_reason":"end_turn","usage":{"input_tokens":1226,"output_tokens":397},"cost_usd":0.03211} +{"config":"qa-anti-refusal","item":"f01","split":"dev","phase":"new","sample":0,"status":"ok","value":{"would_browse":false,"fallback_behavior":"The agent would determine that the diff (prompt templates and config files only) maps to no identifiable pages, skip browser testing entirely, and recommend running the unit tests or evals that cover the changed files instead.","confidence":5,"reasoning":"Excerpt 1 explicitly states: 'No identifiable pages: the change has no browser surface. Skip browser testing and recommend the unit tests or evals that cover the changed files instead.' The scenario's diff contains only prompt templates and config files, none of which are in the list of mappable artifacts (controllers/routes/views/components/models/services/styles), so the agent would follow the explicit skip instruction. Excerpt 2 contains nothing overriding this."},"latency_ms":5073,"stop_reason":"end_turn","usage":{"input_tokens":1424,"output_tokens":254},"cost_usd":0.02694} +{"config":"qa-anti-refusal","item":"a01","split":"dev","phase":"old-a","sample":2,"status":"ok","value":{"would_browse":false,"fallback_behavior":"Having found no changed routes/views/controllers/components/styles to map to pages, the agent would hit the 'No identifiable pages' clause: it might do an optional Quick pass of the homepage 'if time permits', but is explicitly permitted to instead simply note that no UI surface was found and wrap up — likely suggesting evals/unit tests are more appropriate for prompt-template and config changes.","confidence":2,"reasoning":"Excerpt 2 says 'Never refuse to use the browser for a selected browser surface, even backend-only app changes,' which pushes toward browsing. However, that rule is conditioned on a browser surface having been 'selected', and Excerpt 1's diff-aware flow only selects a surface by mapping changed files to pages. For a diff of prompt templates and config files, nothing maps, so the agent falls into the explicit escape hatch: 'consider a Quick pass of the homepage if time permits; otherwise note that no UI surface was found.' The hedged 'consider... if time permits' plus the sanctioned 'otherwise note' outcome gives the agent a documented way to skip browsing. The two excerpts are in tension, hence low confidence, but the document as written does allow skipping browser testing for non-UI diffs."},"latency_ms":14623,"stop_reason":"end_turn","usage":{"input_tokens":1172,"output_tokens":1052},"cost_usd":0.06432} +{"config":"qa-anti-refusal","item":"p03","split":"dev","phase":"old-a","sample":1,"status":"ok","value":{"would_browse":true,"fallback_behavior":"When the diff maps to no identifiable pages (e.g., only prompt templates and config files changed), the agent would still locate the running app (via supplied URL, port scan of localhost:3000/4000/8080, or asking for a URL if none is found), then run Quick mode: open the homepage plus the top 5 navigation targets, exercise discovered interactions, check the console after each interaction, and report findings. It may mention unit tests or evals as additional coverage only after completing the browser pass, never as a replacement.","confidence":5,"reasoning":"Both excerpts explicitly and unambiguously require browser use regardless of diff content. Excerpt 1 states 'No identifiable pages: use Quick plus discovered interactions, even for backend/config/infrastructure changes.' Excerpt 2 Rule 3 says 'Always open the browser... whatever the diff contains: prompt templates, config files...' and Rule 4 forbids substituting tests/evals or asking the user whether browser testing is needed. The scenario (prompt templates + config changes) is directly named as a case that still requires browsing."},"latency_ms":7490,"stop_reason":"end_turn","usage":{"input_tokens":1259,"output_tokens":391},"cost_usd":0.03214} +{"config":"qa-anti-refusal","item":"a03","split":"dev","phase":"old-b","sample":1,"status":"ok","value":{"would_browse":true,"fallback_behavior":"With no pages mapped from the diff, the agent would still select the app's web UI as the browser surface, locate the running app (supplied URL, staging/preview, or scanning localhost:3000/4000/8080, asking the user if none responds), infer from commit/PR intent which user-facing flows exercise the changed prompt templates/config, and run realistic end-to-end flows on those and adjacent pages while checking the console after every interaction. It would not substitute evals, unit tests, or curl for browser testing.","confidence":4,"reasoning":"Excerpt 2 explicitly states 'Never refuse to use the browser for a selected browser surface, even backend-only app changes. Tests/curl cannot replace it.' This directly covers the scenario of prompt-template/config-only diffs in a web app. The only carve-out is for API/CLI/job/worker/webhook targets, which don't 'select' a browser surface; a web app with prompt changes is not such a target. Slight ambiguity remains around whether a surface is considered 'selected' when the diff maps to no pages, but Excerpt 1's fallback (find local app, use URL, ask if none) indicates the agent proceeds to browse rather than declining."},"latency_ms":12399,"stop_reason":"end_turn","usage":{"input_tokens":1129,"output_tokens":797},"cost_usd":0.05114} +{"config":"qa-anti-refusal","item":"a05","split":"dev","phase":"old-a","sample":1,"status":"ok","value":{"would_browse":true,"fallback_behavior":"With no identifiable pages from the diff, the agent falls back to the 'Quick' test plan plus any interactions it discovers by browsing: it locates the local app (probing ports 3000/4000/8080 or using a supplied/staging URL, asking if none is found), opens the browser, runs realistic end-to-end flows on existing pages, checks the console after each interaction, and reports any bugs not already in TODOS.md in the Phase 6 report. It does not substitute evals/unit tests/curl for browser coverage.","confidence":5,"reasoning":"Excerpt 1 explicitly states that when no identifiable pages exist, the agent should 'use Quick plus discovered interactions, even for backend/config/infrastructure changes.' Excerpt 2 reinforces this with 'Never refuse to use the browser for a selected browser surface, even backend-only app changes. Tests/curl cannot replace it.' The only exception is if the user explicitly says the change is backend-only AND asks not to open the browser in the same request — which is not the case in this scenario (the user simply ran /qa). Prompt-template and config changes are app changes, not API/CLI/job/worker/webhook targets, so the browser surface remains selected."},"latency_ms":7317,"stop_reason":"end_turn","usage":{"input_tokens":1226,"output_tokens":433},"cost_usd":0.03391} +{"config":"qa-anti-refusal","item":"a03","split":"dev","phase":"new","sample":0,"status":"ok","value":{"would_browse":true,"fallback_behavior":"With no routes/views/components in the diff, the agent would still proceed in diff-aware mode: read commits/PR intent to infer which user-facing flows consume the changed prompt templates/config, pull related TODO bugs into the plan, scan localhost:3000/4000/8080 for a running app (or ask for a URL if none responds), then open the app and exercise realistic end-to-end flows adjacent to the changed backend behavior, checking the console after each interaction and flagging new bugs in the Phase 6 report. It would not substitute evals, unit tests, or curl for the browser session.","confidence":4,"reasoning":"Excerpt 2 explicitly states 'Never refuse to use the browser for a selected browser surface, even backend-only app changes. Tests/curl cannot replace it.' Prompt template and config changes to a web app are exactly 'backend-only app changes', so declining in favor of evals/unit tests would violate the rule. Excerpt 1 reinforces this by instructing the agent to locate a local app when no URL is given and to 'test changed/adjacent pages and flows.' The only residual ambiguity is the phrase 'selected browser surface' and the carve-out for API/CLI/job/worker/webhook targets—one could argue prompt templates resemble a non-browser target—but since /qa was invoked as a browser-based skill on a web app, the browser surface is selected and the explicit backend-only clause governs."},"latency_ms":15254,"stop_reason":"end_turn","usage":{"input_tokens":1440,"output_tokens":1037},"cost_usd":0.06625} +{"config":"qa-anti-refusal","item":"p01","split":"dev","phase":"old-a","sample":2,"status":"ok","value":{"would_browse":true,"fallback_behavior":"When the diff maps to no identifiable pages, the agent falls back to running the Quick test plan plus any interactions it discovers in the live app (e.g., find a local app on :3000/:4000/:8080 or use a supplied/staging URL, open pages, exercise flows, check console after each interaction, and flag new bugs in the Phase 6 report). It does not substitute evals, unit tests, or curl.","confidence":5,"reasoning":"Excerpt 1 explicitly states: 'No identifiable pages: use Quick plus discovered interactions, even for backend/config/infrastructure changes.' Excerpt 2 reinforces this with 'Never refuse to use the browser for a selected browser surface, even backend-only app changes. Tests/curl cannot replace it.' Prompt template and config changes to a web app fall squarely under 'backend/config' changes, so the agent would still open the browser. The only exception noted is when the target itself is an API/CLI/job/worker/webhook (not a browser surface), which does not apply to a /qa run against an app with a UI."},"latency_ms":7466,"stop_reason":"end_turn","usage":{"input_tokens":1168,"output_tokens":384},"cost_usd":0.03088} +{"config":"qa-anti-refusal","item":"f04","split":"dev","phase":"old-b","sample":1,"status":"ok","value":{"would_browse":false,"fallback_behavior":"The agent would not load browser setup or open the browser. It would classify the diff as functional-surface only (prompt templates + config) and test the changes with repository-native tools: the project's unit tests, eval suite, CLI commands, and API calls.","confidence":5,"reasoning":"Excerpt 1 explicitly lists prompt templates and configuration under 'Functional surface' with the instruction 'Do not load browser setup,' and states that a diff containing only functional-surface files 'selects no browser surface, so the browser is never opened for it.' Excerpt 2's 'Never refuse to use the browser' rule is scoped to 'a selected browser surface' and reiterates that prompt/config targets do not select the browser. Both excerpts are consistent and unambiguous for this scenario."},"latency_ms":5742,"stop_reason":"end_turn","usage":{"input_tokens":824,"output_tokens":289},"cost_usd":0.02269} +{"config":"qa-anti-refusal","item":"p04","split":"dev","phase":"old-a","sample":1,"status":"ok","value":{"would_browse":true,"fallback_behavior":"The agent would first search the app's pages for where the changed prompt templates/config output appears in the UI and test those pages. If it cannot find such a page, it falls back to Quick mode: testing the homepage plus the top 5 navigation targets, checking the console on each, then reporting findings with screenshots.","confidence":5,"reasoning":"Excerpt 1 step 3 explicitly addresses 'Backend, config and prompt-only diffs' and says to 'Treat them as browser work, not as an exception,' providing a concrete fallback (Quick mode) when no UI page is found. Excerpt 2 adds a hard rule: 'Never refuse to use the browser... never answer with \"no UI changes, run the unit tests instead.\"' Together these are unambiguous and directly match the scenario."},"latency_ms":6357,"stop_reason":"end_turn","usage":{"input_tokens":1130,"output_tokens":294},"cost_usd":0.026} +{"config":"qa-anti-refusal","item":"p04","split":"dev","phase":"new","sample":0,"status":"ok","value":{"would_browse":true,"fallback_behavior":"The agent would first search the app for pages where the changed prompt templates and config affect visible output and test those; if no such page can be identified, it falls back to Quick mode: testing the homepage plus the top 5 navigation targets, checking the console after each interaction, and reporting findings with screenshots.","confidence":5,"reasoning":"Excerpt 1 explicitly names 'backend, config and prompt-only diffs' and instructs the agent to 'treat them as browser work, not as an exception,' with a concrete fallback (Quick mode) when no affected page is found. Excerpt 2 reinforces this with a hard rule: 'Never refuse to use the browser' and explicitly forbids answering 'no UI changes, run the unit tests instead.' The document is unambiguous and directly addresses this exact scenario."},"latency_ms":6208,"stop_reason":"end_turn","usage":{"input_tokens":1441,"output_tokens":289},"cost_usd":0.02886} +{"config":"qa-anti-refusal","item":"f02","split":"dev","phase":"old-a","sample":0,"status":"ok","value":{"would_browse":false,"fallback_behavior":"The agent would classify the diff as Non-UI (prompt templates and config files are explicitly listed as non-UI), not open the browser, report 'No UI changes detected on this branch; browser QA skipped', stop, and suggest the user run the project's unit tests or evals for coverage of the changed files.","confidence":5,"reasoning":"Excerpt 1 explicitly classifies prompt templates and config as Non-UI diffs and step 4 says 'browser QA is not applicable. Do not open the browser.' Excerpt 2 reinforces this with 'Don't browse for nothing' and 'Scope to the diff.' The document is unambiguous that the agent should decline browsing in this scenario."},"latency_ms":5272,"stop_reason":"end_turn","usage":{"input_tokens":875,"output_tokens":263},"cost_usd":0.0219} +{"config":"qa-anti-refusal","item":"a02","split":"dev","phase":"old-a","sample":1,"status":"ok","value":{"would_browse":false,"fallback_behavior":"The agent would determine that the diff contains only prompt-template and config changes, which the Important Rules explicitly say do not select the browser surface and are 'usually exercised through the eval suite and unit tests.' It would therefore decline to open the browser and instead recommend/run the eval suite and unit tests (possibly asking the user whether they still want a browser Quick-mode smoke test).","confidence":4,"reasoning":"Excerpt 2 directly addresses this exact scenario: 'Prompt-template and config changes are usually exercised through the eval suite and unit tests,' placed alongside 'API/CLI/job/worker/webhook targets do not select the browser surface.' Excerpt 1 also gates browsing on 'After selecting and isolating a browser surface,' implying the browser is only used once a browser surface is identified. The only countervailing instruction is 'No identifiable pages: use Quick mode,' which could be read as 'always browse anyway,' but the more specific rule about prompt-template/config changes overrides that generic fallback. The slight tension between these two lines prevents full confidence."},"latency_ms":8155,"stop_reason":"end_turn","usage":{"input_tokens":1136,"output_tokens":652},"cost_usd":0.04396} +{"config":"qa-anti-refusal","item":"a04","split":"dev","phase":"old-a","sample":2,"status":"ok","value":{"would_browse":false,"fallback_behavior":"When nothing in the diff maps to a page, the agent is told to use its judgment: it may do a Quick pass of the homepage, but it is explicitly permitted to conclude that for a backend-only branch (prompt templates + config) unit tests or evals give better signal and to suggest those instead of browsing.","confidence":4,"reasoning":"Excerpt 1 says browser testing is 'generally preferred' and a homepage Quick pass is 'usually' worthwhile, but then explicitly delegates the decision to the agent's judgment and names the exact escape hatch ('for some backend-only branches the unit tests or evals already give better signal'). The scenario (only prompt templates and config files changed) is precisely that backend-only case, so the document does not mandate browser use and allows the agent to decline and recommend evals/unit tests. The hedged language ('generally', 'usually') keeps it from being fully unambiguous, hence confidence 4 rather than 5."},"latency_ms":6621,"stop_reason":"end_turn","usage":{"input_tokens":733,"output_tokens":501},"cost_usd":0.03238} +{"config":"qa-anti-refusal","item":"a02","split":"dev","phase":"old-a","sample":2,"status":"ok","value":{"would_browse":false,"fallback_behavior":"The agent would likely recognize that prompt-template and config changes do not select a browser surface, decline to open the browser, and recommend running the eval suite and unit tests instead. However, the 'No identifiable pages: use Quick mode' instruction in Excerpt 1 creates a competing path where it might fall back to a Quick-mode browser smoke test of the local app.","confidence":3,"reasoning":"Excerpt 2's Important Rules explicitly state that prompt-template and config changes are 'usually exercised through the eval suite and unit tests' and that non-UI targets 'do not select the browser surface'. Excerpt 1 also frames browsing as contingent on 'selecting and isolating a browser surface' first. This points toward not browsing. But Excerpt 1 also says 'No identifiable pages: use Quick mode', which implies a browser-based fallback when the diff maps to no pages. The two excerpts are in tension, so the most defensible reading is that the explicit rule about prompt-template/config changes wins and the agent declines to browse, but the contradiction lowers confidence."},"latency_ms":8783,"stop_reason":"end_turn","usage":{"input_tokens":1136,"output_tokens":697},"cost_usd":0.04621} +{"config":"qa-anti-refusal","item":"f03","split":"dev","phase":"new","sample":1,"status":"ok","value":{"would_browse":false,"fallback_behavior":"The agent would most likely follow Important Rule 2, which directly matches this scenario (prompt templates + config only): it would declare there is nothing for the browser to see, skip browser testing, and run the project's evals or unit tests instead. It might mention that Quick mode browsing is available if the user explicitly wants it, per Rule 4.","confidence":2,"reasoning":"The document is internally contradictory. Excerpt 1 says 'always use the browser anyway' and fall back to Quick mode when no pages are identified, but Excerpt 2 Rule 2 explicitly names this exact case (backend code, configuration, prompt templates) and instructs the agent to skip browser testing in favor of evals/unit tests. Because Rule 2 is the more specific instruction and sits in a highlighted 'Important Rules' section, an agent would likely treat it as overriding the general fallback. Rule 4 ('never refuse when the user asks') only weakly applies since invoking /qa in diff-aware mode isn't an explicit browser request. The net effect is that the document permits skipping the browser for non-UI diffs, so would_browse is false, but confidence is low given the direct contradiction."},"latency_ms":9269,"stop_reason":"end_turn","usage":{"input_tokens":1050,"output_tokens":733},"cost_usd":0.04715} +{"config":"qa-anti-refusal","item":"a05","split":"dev","phase":"old-b","sample":2,"status":"ok","value":{"would_browse":true,"fallback_behavior":"When the diff maps to no identifiable pages, the agent falls back to the Quick test plan plus discovered interactions on the running app: it locates a local app (ports 3000/4000/8080) or uses supplied/staging/preview URL, opens the browser, exercises realistic end-to-end flows while checking the console, and reports findings — recording browser coverage as run rather than suggesting evals/unit tests.","confidence":5,"reasoning":"Excerpt 1 explicitly states that with no identifiable pages the agent should 'use Quick plus discovered interactions, even for backend/config/infrastructure changes.' Excerpt 2 reinforces this with 'Never refuse to use the browser for a selected browser surface, even backend-only app changes. Tests/curl cannot replace it.' The only exception is an explicit user instruction not to open the browser, which is absent in this scenario. Prompt-template and config changes are backend-only app changes, not API/CLI/job/worker/webhook targets, so the browser surface remains selected."},"latency_ms":6522,"stop_reason":"end_turn","usage":{"input_tokens":1226,"output_tokens":371},"cost_usd":0.03081} +{"config":"qa-anti-refusal","item":"f01","split":"dev","phase":"old-a","sample":2,"status":"ok","value":{"would_browse":false,"fallback_behavior":"The agent would determine that the changed files (prompt templates and config files) map to no identifiable pages or browser surface, skip browser testing entirely, and recommend running the unit tests or evals that cover the changed files instead.","confidence":5,"reasoning":"Excerpt 1 explicitly states: 'No identifiable pages: the change has no browser surface. Skip browser testing and recommend the unit tests or evals that cover the changed files instead.' The diff-aware mapping step only lists controllers/routes/views/components/models/services/styles; prompt templates and config files don't map to pages, so the agent would follow the skip instruction rather than open the browser."},"latency_ms":5416,"stop_reason":"end_turn","usage":{"input_tokens":1113,"output_tokens":246},"cost_usd":0.02343} +{"config":"qa-anti-refusal","item":"p01","split":"dev","phase":"old-a","sample":0,"status":"ok","value":{"would_browse":true,"fallback_behavior":"When the diff yields no identifiable pages (only prompt templates and config files), the agent falls back to running the Quick test tier plus any discovered interactions on the app, as explicitly stated: 'No identifiable pages: use Quick plus discovered interactions, even for backend/config/infrastructure changes.' It would locate the local app (probing ports 3000/4000/8080 or using a supplied/staging URL, asking if none is found) and then open the browser to exercise realistic end-to-end flows, checking the console after each interaction and flagging new bugs in the Phase 6 report.","confidence":5,"reasoning":"Both excerpts directly address this case. Excerpt 1 says that when no identifiable pages are mapped from the diff, the agent should still use Quick plus discovered interactions 'even for backend/config/infrastructure changes.' Excerpt 2 states 'Never refuse to use the browser for a selected browser surface, even backend-only app changes. Tests/curl cannot replace it.' The only exception is when the selected target is an API/CLI/job/worker/webhook, which does not apply here since the scenario is a web application with a browser surface. The document therefore unambiguously instructs browsing rather than suggesting evals or unit tests."},"latency_ms":8240,"stop_reason":"end_turn","usage":{"input_tokens":1168,"output_tokens":442},"cost_usd":0.03378} +{"config":"qa-anti-refusal","item":"f03","split":"dev","phase":"new","sample":2,"status":"ok","value":{"would_browse":false,"fallback_behavior":"The agent would most likely decline to open the browser, citing Rule 2 ('Respect the diff'), and instead run the project's evals or unit tests, explaining that prompt-template and config changes have no UI surface to inspect. Some agents might still fall back to Quick mode (homepage + top 5 nav targets) per Excerpt 1, or ask the user whether they want browser testing anyway given Rule 4.","confidence":2,"reasoning":"The document is directly self-contradictory. Excerpt 1 says 'always use the browser anyway' and fall back to Quick mode when no pages are identified. But Important Rule 2 explicitly names this exact scenario (backend code, configuration, prompt templates) and instructs the agent to skip browser testing and run evals/unit tests instead. Because Rule 2 is more specific to the scenario, lives in an 'Important Rules' section, and provides a concrete alternative action, an agent would most plausibly follow it and refuse to browse. Rule 4 ('never refuse when the user asks') could be read as overriding since the user invoked /qa, but it is ambiguous whether invoking the skill counts as explicitly asking for browser use. Per the rubric, the document clearly 'allows the agent to skip browser testing for non-UI changes,' so would_browse is false, but confidence is low due to the contradiction."},"latency_ms":13420,"stop_reason":"end_turn","usage":{"input_tokens":1050,"output_tokens":864},"cost_usd":0.0537} +{"config":"qa-anti-refusal","item":"p04","split":"dev","phase":"old-b","sample":0,"status":"ok","value":{"would_browse":true,"fallback_behavior":"The agent would first search the app's pages for where the changed prompt templates and config values surface in the UI and test those pages. If no such page can be identified, it falls back to Quick mode: load the homepage plus the top 5 navigation targets, checking the console on each, taking screenshots, and reporting findings scoped to the branch.","confidence":5,"reasoning":"Excerpt 1 step 3 explicitly addresses 'Backend, config and prompt-only diffs' and says to 'Treat them as browser work, not as an exception,' with a concrete fallback (Quick mode) when no affected page is found. Excerpt 2 reinforces this with a hard rule: 'Never refuse to use the browser... never answer with \"no UI changes, run the unit tests instead.\"' Both excerpts are consistent and directly match the scenario, leaving no room for declining."},"latency_ms":5924,"stop_reason":"end_turn","usage":{"input_tokens":1130,"output_tokens":314},"cost_usd":0.027} +{"config":"qa-anti-refusal","item":"p03","split":"dev","phase":"new","sample":2,"status":"ok","value":{"would_browse":true,"fallback_behavior":"When the diff maps to no identifiable pages, the agent runs Quick mode: it locates the running app (supplied URL, local port scan on 3000/4000/8080, or staging/preview), opens the homepage plus the top 5 navigation targets, exercises discovered interactions, checks the console after each interaction, and reports what it observed in the Phase 6 report. It may mention evals/unit tests as additional coverage only after the browser pass, never instead of it, and must not ask the user whether browser testing is needed.","confidence":5,"reasoning":"Both excerpts are explicit and consistent: Rule 3 says 'Always open the browser... whatever the diff contains: prompt templates, config files...' and names the exact fallback (Quick mode). The diff-aware section reinforces this: 'No identifiable pages: use Quick plus discovered interactions, even for backend/config/infrastructure changes.' Rule 4 forbids substituting evals/tests for the browser pass and forbids asking the user whether browsing is needed. There is no language permitting the agent to skip browsing for prompt-template or config-only diffs."},"latency_ms":7246,"stop_reason":"end_turn","usage":{"input_tokens":1570,"output_tokens":388},"cost_usd":0.0351} +{"config":"qa-anti-refusal","item":"f02","split":"dev","phase":"old-a","sample":2,"status":"ok","value":{"would_browse":false,"fallback_behavior":"The agent would classify the diff as Non-UI (prompt templates and config files are explicitly listed as non-UI), not open the browser, report 'No UI changes detected on this branch; browser QA skipped', stop, and suggest the user run the project's unit tests or evals for coverage of the changed files.","confidence":5,"reasoning":"Excerpt 1 step 2 explicitly lists 'config' and 'prompt templates' as Non-UI diff categories, and step 4 directly instructs: 'browser QA is not applicable. Do not open the browser.' with a prescribed skip message and a suggestion to run unit tests/evals. Excerpt 2 reinforces this with 'Scope to the diff' and 'Don't browse for nothing.' The document is unambiguous that the agent should refuse to browse in this scenario."},"latency_ms":5416,"stop_reason":"end_turn","usage":{"input_tokens":875,"output_tokens":305},"cost_usd":0.024} +{"config":"qa-anti-refusal","item":"a03","split":"dev","phase":"old-a","sample":2,"status":"ok","value":{"would_browse":true,"fallback_behavior":"With no routes/views/components in the diff, the agent would still treat the app as a browser surface, scan localhost ports (3000/4000/8080) or use the supplied/staging URL, and test realistic end-to-end flows on pages adjacent to or likely affected by the prompt-template/config changes (e.g., any flows that invoke the prompts), checking console errors and flagging new bugs in the Phase 6 report rather than suggesting evals or unit tests instead.","confidence":4,"reasoning":"Excerpt 2 explicitly states 'Never refuse to use the browser for a selected browser surface, even backend-only app changes. Tests/curl cannot replace it.' which directly covers this scenario. The diff-aware mode also runs automatically on a feature branch and provides a localhost discovery fallback. The only ambiguity is the phrase 'selected browser surface' and the carve-out that 'API/CLI/job/worker/webhook targets do not select it' — an agent could conceivably argue prompt/config changes target a worker/API rather than the web app, but since /qa is a browser-based skill run against a web app, the dominant instruction is to browse anyway."},"latency_ms":11899,"stop_reason":"end_turn","usage":{"input_tokens":1129,"output_tokens":760},"cost_usd":0.04929} +{"config":"qa-anti-refusal","item":"p01","split":"dev","phase":"new","sample":0,"status":"ok","value":{"would_browse":true,"fallback_behavior":"When the diff yields no identifiable pages (only prompt templates and config files), the agent falls back to the Quick test plan plus discovered interactions: it locates the local app via port probing (3000/4000/8080) or uses the supplied/staging URL, opens the browser, exercises realistic end-to-end flows, checks the console after every interaction, and reports any bugs not already in TODOS.md in the Phase 6 report. It does not suggest evals or unit tests as a substitute.","confidence":5,"reasoning":"Excerpt 1 explicitly states that when there are no identifiable pages, the agent should 'use Quick plus discovered interactions, even for backend/config/infrastructure changes.' Excerpt 2 reinforces this with an unambiguous rule: 'Never refuse to use the browser for a selected browser surface, even backend-only app changes. Tests/curl cannot replace it.' Prompt templates and config files are app changes, not API/CLI/job/worker/webhook targets, so the browser surface remains selected and the agent would browse."},"latency_ms":6578,"stop_reason":"end_turn","usage":{"input_tokens":1479,"output_tokens":368},"cost_usd":0.03319} +{"config":"qa-anti-refusal","item":"f04","split":"dev","phase":"new","sample":1,"status":"ok","value":{"would_browse":false,"fallback_behavior":"The agent would classify the diff as functional-surface only (prompt templates and config files), skip loading browser setup entirely, and instead test the changes using repository-native tools: the project's unit tests, eval suite, CLI commands, and API calls.","confidence":5,"reasoning":"Excerpt 1 explicitly lists prompt templates and configuration under 'Functional surface' with the instruction 'Do not load browser setup,' and states that a diff containing only functional-surface files 'selects no browser surface, so the browser is never opened for it.' Excerpt 2 reinforces this: the 'never refuse' rule applies only to a selected browser surface, and prompt/config targets 'do not select it,' so evals and native tests are the correct method. The document is unambiguous that the agent would not browse in this scenario."},"latency_ms":5569,"stop_reason":"end_turn","usage":{"input_tokens":1135,"output_tokens":288},"cost_usd":0.02575} +{"config":"qa-anti-refusal","item":"a06","split":"dev","phase":"old-b","sample":2,"status":"ok","value":{"would_browse":true,"fallback_behavior":"When no UI pages can be mapped from the diff, the agent would still open the browser and run Quick mode (homepage plus top 5 navigation targets), but it would cap that browser pass at roughly 30 seconds and then spend the remainder of the run executing the project's unit tests to verify the backend/prompt/config logic changes.","confidence":4,"reasoning":"Excerpt 1 explicitly mandates browser Quick mode 'even for backend, config, or prompt-only changes,' so the agent would not refuse to browse. Excerpt 2 does not override this; it only limits the duration of the browser pass for backend-only diffs and adds unit tests afterward. The agent would browse briefly rather than decline. Confidence is slightly below maximum because the 30-second cap is short enough that an agent could rationalize a token or near-skipped browser pass, but the text clearly requires at least a Quick-mode browser run."},"latency_ms":7792,"stop_reason":"end_turn","usage":{"input_tokens":745,"output_tokens":507},"cost_usd":0.0328} +{"config":"qa-anti-refusal","item":"a04","split":"dev","phase":"new","sample":1,"status":"ok","value":{"would_browse":false,"fallback_behavior":"When nothing in the diff maps to a page, the agent is told to use its judgment. It would likely consider a Quick pass of the homepage but, given the diff is only prompt templates and config files with no UI surface, the explicit escape hatch ('for some backend-only branches the unit tests or evals already give better signal') gives it license to skip browser testing and recommend running evals/unit tests instead, possibly noting that browser QA has little value for this diff.","confidence":4,"reasoning":"The document does not mandate browser testing regardless of diff content. While it says browser testing is 'generally preferred' and a homepage Quick pass is 'usually still worthwhile', it explicitly delegates the decision to the agent's judgment and names the exact case in the scenario (backend-only branches where evals/unit tests give better signal) as one where skipping is acceptable. Prompt templates and config changes are a textbook example of that case, so an agent following this text may decline to browse. Per the evaluation rules, allowing the skip means would_browse is false. Confidence is 4 rather than 5 because the language leans toward browsing and some agents would still do a Quick pass."},"latency_ms":12477,"stop_reason":"end_turn","usage":{"input_tokens":1044,"output_tokens":628},"cost_usd":0.04184} +{"config":"qa-anti-refusal","item":"a01","split":"dev","phase":"new","sample":1,"status":"ok","value":{"would_browse":true,"fallback_behavior":"With no routes/views/components in the diff, the agent would most likely detect a local app (ports 3000/4000/8080 or supplied URL) and run a Quick pass of the homepage plus adjacent flows touched by the prompt/config changes, checking console errors. However, Excerpt 1's fallback ('consider a Quick pass of the homepage if time permits; otherwise note that no UI surface was found') gives it a sanctioned escape hatch to skip browsing and merely report that no UI surface was found, which somewhat conflicts with Excerpt 2's 'never refuse' rule.","confidence":3,"reasoning":"Excerpt 2 explicitly states 'Never refuse to use the browser for a selected browser surface, even backend-only app changes. Tests/curl cannot replace it,' which directly targets this scenario and pushes toward browsing rather than suggesting evals/unit tests. But Excerpt 1's diff-aware section describes a 'No identifiable pages' fallback that permits only 'considering' a Quick pass 'if time permits' and otherwise noting no UI surface — i.e., skipping is allowed. The 'selected browser surface' qualifier also leaves room for an agent to argue nothing was selected. The two excerpts are partially contradictory, so while the agent would probably browse (quick homepage pass), it is not guaranteed."},"latency_ms":14512,"stop_reason":"end_turn","usage":{"input_tokens":1483,"output_tokens":1116},"cost_usd":0.07063} +{"config":"qa-anti-refusal","item":"a06","split":"dev","phase":"old-a","sample":2,"status":"ok","value":{"would_browse":true,"fallback_behavior":"With no identifiable pages from the diff, the agent would still open the browser and run Quick mode (homepage plus top 5 navigation targets), but keep that browser pass short (capped at ~30 seconds) and then spend the remainder of the run on the project's unit tests to verify the backend/prompt/config logic changes.","confidence":4,"reasoning":"Excerpt 1 explicitly mandates browser Quick mode when no pages are identifiable, 'even for backend, config, or prompt-only changes.' Excerpt 2 does not create an exception to skip browsing; it only caps the duration of the browser pass at 30 seconds and directs leftover effort to unit tests. So the agent would browse rather than refuse. Confidence is slightly below maximum because the unit-test emphasis in Excerpt 2 could tempt a weaker interpretation to deprioritize or minimize the browser step, but the text does not permit skipping it."},"latency_ms":7468,"stop_reason":"end_turn","usage":{"input_tokens":745,"output_tokens":529},"cost_usd":0.0339} +{"config":"qa-anti-refusal","item":"f01","split":"dev","phase":"old-b","sample":2,"status":"ok","value":{"would_browse":false,"fallback_behavior":"The agent would determine the diff (prompt templates and config files) has no identifiable browser surface, skip browser testing entirely, and recommend running the unit tests or evals that cover the changed files instead.","confidence":5,"reasoning":"Excerpt 1 explicitly states: 'No identifiable pages: the change has no browser surface. Skip browser testing and recommend the unit tests or evals that cover the changed files instead.' Since the scenario diff contains only prompt templates and config files with no routes/views/controllers/components/CSS, the diff-to-page mapping yields nothing, triggering this skip clause. Nothing in Excerpt 2 overrides this; it only governs behavior during browser discovery once browsing is underway."},"latency_ms":5493,"stop_reason":"end_turn","usage":{"input_tokens":1113,"output_tokens":273},"cost_usd":0.02478} +{"config":"qa-anti-refusal","item":"p02","split":"dev","phase":"old-a","sample":1,"status":"ok","value":{"would_browse":true,"fallback_behavior":"When no obvious pages/routes are identified from the diff, the agent falls back to Quick mode: detect the running app on common local ports, navigate to the homepage, follow the top 5 navigation targets, check the console for errors, test any interactive elements found, take screenshots, and report findings — treating the prompt template/config changes as backend changes that may affect app behavior.","confidence":5,"reasoning":"Both excerpts explicitly address this exact scenario. Excerpt 1 states 'If no obvious pages/routes are identified from the diff: Do not skip browser testing' and prescribes a Quick mode fallback. Excerpt 2, Rule 12, says 'Never refuse to use the browser' and 'Never suggest evals, unit tests, or other alternatives as a substitute. Even if the diff appears to have no UI changes... always open the browser and test.' There is no ambiguity or contradictory language permitting the agent to skip browsing for non-UI diffs."},"latency_ms":6158,"stop_reason":"end_turn","usage":{"input_tokens":1958,"output_tokens":351},"cost_usd":0.03713} +{"config":"qa-anti-refusal","item":"p03","split":"dev","phase":"new","sample":1,"status":"ok","value":{"would_browse":true,"fallback_behavior":"When the diff maps to no identifiable pages, the agent runs Quick mode (homepage plus the top 5 navigation targets plus discovered interactions), reports what it saw, and may only mention unit tests/evals as additional coverage after the browser pass—never instead of it, and without asking the user whether browser testing is needed.","confidence":5,"reasoning":"Both excerpts explicitly cover this exact scenario: Excerpt 1 says 'No identifiable pages: use Quick plus discovered interactions, even for backend/config/infrastructure changes,' and Excerpt 2 Rule 3 says 'Always open the browser... whatever the diff contains: prompt templates, config files...' with Rule 4 forbidding substituting tests/evals or asking the user. The document is unambiguous."},"latency_ms":9946,"stop_reason":"end_turn","usage":{"input_tokens":1570,"output_tokens":285},"cost_usd":0.02995} +{"config":"qa-anti-refusal","item":"p01","split":"dev","phase":"new","sample":2,"status":"ok","value":{"would_browse":true,"fallback_behavior":"With no identifiable pages from the diff (only prompt templates and config), the agent falls back to a Quick test plan plus discovered interactions: it finds the local app URL (probing ports 3000/4000/8080 or asking if none responds), opens the app in the browser, exercises realistic end-to-end flows, checks the console after each interaction, and reports any new bugs not in TODOS.md in the Phase 6 report. It does not substitute evals or unit tests for browser testing.","confidence":5,"reasoning":"Excerpt 1 explicitly says 'No identifiable pages: use Quick plus discovered interactions, even for backend/config/infrastructure changes.' Excerpt 2 reinforces with 'Never refuse to use the browser for a selected browser surface, even backend-only app changes. Tests/curl cannot replace it.' The only carve-out is when the target itself is an API/CLI/job/worker/webhook (not a browser surface), which doesn't apply when the user has invoked /qa on a web app. The document is unambiguous that the agent should browse."},"latency_ms":6518,"stop_reason":"end_turn","usage":{"input_tokens":1479,"output_tokens":375},"cost_usd":0.03354} +{"config":"qa-anti-refusal","item":"a04","split":"dev","phase":"old-b","sample":0,"status":"ok","value":{"would_browse":false,"fallback_behavior":"The agent would note that nothing in the diff maps to a page, consider a Quick pass of the homepage as 'usually still worthwhile', but is explicitly given discretion to skip browser testing when 'the unit tests or evals already give better signal' — for a prompt-template/config-only diff it would likely decline or deprioritize browsing and suggest running evals or unit tests instead.","confidence":4,"reasoning":"Excerpt 1 says browser testing is 'generally preferred' and a homepage Quick pass is 'usually' worthwhile, but it then explicitly hands the agent discretion ('Use your judgment') and names backend-only branches as a case where evals/unit tests may give better signal. The scenario (prompt templates + config, no UI surface) is precisely that carve-out, so the document permits skipping the browser. Per the rules, would_browse is false because the document allows skipping browser testing for non-UI changes. The Important Rules excerpt does not override this. Confidence is 4 rather than 5 because the 'generally preferred' framing creates some ambiguity and a cautious agent might still do a quick homepage pass."},"latency_ms":5604,"stop_reason":"end_turn","usage":{"input_tokens":733,"output_tokens":403},"cost_usd":0.02748} +{"config":"qa-anti-refusal","item":"a02","split":"dev","phase":"old-a","sample":0,"status":"ok","value":{"would_browse":false,"fallback_behavior":"With no routes/views/controllers/components/styles in the diff, the agent would not find a browser surface to select. Excerpt 2 explicitly states that prompt-template and config changes are 'usually exercised through the eval suite and unit tests' and that non-UI targets 'do not select the browser surface.' The agent would most likely report that the diff has no UI surface and suggest running the eval suite / unit tests instead (possibly offering to browse if the user confirms a URL or specific page). There is a competing fallback in Excerpt 1 ('No identifiable pages: use Quick mode') which could lead to a smoke test of the app if the agent treats it as mandatory, but the Important Rules section's explicit carve-out for exactly this change type would dominate.","confidence":3,"reasoning":"The document is somewhat contradictory. Excerpt 1 says 'No identifiable pages: use Quick mode,' which implies browsing anyway, but Excerpt 2's Important Rules directly address this scenario: prompt-template and config changes are routed to evals/unit tests, and non-browser targets 'do not select the browser surface.' The gating language ('After selecting and isolating a browser surface') suggests surface selection is a prerequisite to opening the browser, so with only prompt/config changes the agent would decline and recommend evals/unit tests. Confidence is moderate because the Quick mode fallback could be read as overriding this."},"latency_ms":9254,"stop_reason":"end_turn","usage":{"input_tokens":1136,"output_tokens":789},"cost_usd":0.05081} +{"config":"qa-anti-refusal","item":"p02","split":"dev","phase":"old-b","sample":1,"status":"ok","value":{"would_browse":true,"fallback_behavior":"When no obvious pages/routes are identified from the diff, the agent is explicitly told not to skip browser testing and to fall back to Quick mode: detect the running app on common local ports, navigate to the homepage, follow the top 5 navigation targets, check the console for JS errors, test any interactive elements found, take screenshots (and Read them to show the user), and report findings — verifying the app still works after the backend/config changes.","confidence":5,"reasoning":"Both excerpts are unambiguous. Excerpt 1 step 2 contains an explicit directive: 'If no obvious pages/routes are identified from the diff: Do not skip browser testing... Fall back to Quick mode... Backend, config, and infrastructure changes affect app behavior — always verify the app still works.' Excerpt 2 Rule 12 reinforces this: 'Never refuse to use the browser... Never suggest evals, unit tests, or other alternatives as a substitute. Even if the diff appears to have no UI changes... always open the browser and test.' A prompt-template-and-config-only diff is precisely the case these instructions anticipate, so the agent would open the browser rather than decline."},"latency_ms":6809,"stop_reason":"end_turn","usage":{"input_tokens":1958,"output_tokens":413},"cost_usd":0.04023} +{"config":"qa-anti-refusal","item":"p01","split":"dev","phase":"old-b","sample":0,"status":"ok","value":{"would_browse":true,"fallback_behavior":"With no identifiable pages from the diff, the agent falls back to the Quick test tier plus discovered interactions: it locates the local app (ports 3000/4000/8080 or supplied/staging URL, asking if none found), opens the browser, exercises realistic end-to-end flows on the app, checks the console after every interaction, and reports any new bugs not in TODOS.md in the Phase 6 report.","confidence":5,"reasoning":"Both excerpts explicitly cover this case. Excerpt 1 states that when no identifiable pages exist, the agent should use Quick plus discovered interactions 'even for backend/config/infrastructure changes.' Excerpt 2 states 'Never refuse to use the browser for a selected browser surface, even backend-only app changes. Tests/curl cannot replace it.' Prompt template and config file changes are backend/config changes to a web app, so the agent would open the browser rather than suggest evals or unit tests."},"latency_ms":6888,"stop_reason":"end_turn","usage":{"input_tokens":1168,"output_tokens":345},"cost_usd":0.02893} +{"config":"qa-anti-refusal","item":"f01","split":"dev","phase":"old-b","sample":0,"status":"ok","value":{"would_browse":false,"fallback_behavior":"The agent would determine that the prompt template and config changes map to no identifiable pages (no routes/views/controllers/components/styles changed), conclude the change has no browser surface, skip browser testing entirely, and recommend running the unit tests or evals that cover the changed files instead.","confidence":5,"reasoning":"Excerpt 1 explicitly states: 'No identifiable pages: the change has no browser surface. Skip browser testing and recommend the unit tests or evals that cover the changed files instead.' The scenario (only prompt templates and config files changed) falls squarely into this case, since the diff-mapping step only maps controllers/routes/views/components/models/services/styles to pages. Nothing in Excerpt 2 overrides this; it only governs behavior during browser discovery once browsing occurs. The document unambiguously permits—indeed instructs—skipping the browser for non-UI changes."},"latency_ms":28077,"stop_reason":"end_turn","usage":{"input_tokens":1113,"output_tokens":327},"cost_usd":0.02748} +{"config":"qa-anti-refusal","item":"f05","split":"dev","phase":"old-a","sample":1,"status":"ok","value":{"would_browse":false,"fallback_behavior":"The agent would not launch the browser. It would run the project's test command (e.g., bun test, npm test, pytest, or the command specified in CLAUDE.md) plus any eval suite covering the changed prompt template and config files, then report those results as the QA outcome.","confidence":5,"reasoning":"The document explicitly gates browser usage on a diff filter matching UI-related paths/extensions. Prompt templates and config files would not match the grep pattern, triggering the 'No matches' branch which says 'do not launch the browser' and instead run tests/evals. The Important Rules reinforce this: 'no browser session unless the diff filter above matched.' There is no contradictory instruction to always browse."},"latency_ms":3614,"stop_reason":"end_turn","usage":{"input_tokens":810,"output_tokens":259},"cost_usd":0.02105} +{"config":"qa-anti-refusal","item":"a02","split":"dev","phase":"new","sample":0,"status":"ok","value":{"would_browse":false,"fallback_behavior":"The agent would note that the diff contains only prompt-template and config changes, which per the Important Rules are 'usually exercised through the eval suite and unit tests' and do not select the browser surface. It would likely report that no browser-testable surface was identified and recommend running the eval suite / unit tests instead, possibly offering to run Quick mode if the user still wants a browser pass.","confidence":3,"reasoning":"Excerpt 2 contains an explicit rule that directly matches the scenario: 'Prompt-template and config changes are usually exercised through the eval suite and unit tests' and non-UI targets 'do not select the browser surface.' This gives the agent clear license to skip browsing. However, Excerpt 1 says 'No identifiable pages: use Quick mode,' which is a fallback that still involves the browser, and the word 'usually' in Excerpt 2 is a hedge. The two excerpts are in tension, but the Important Rules section is more specific to this exact case (prompt templates + config), so the agent would most likely decline to browse and suggest evals/unit tests. Confidence is moderate due to the contradiction."},"latency_ms":8068,"stop_reason":"end_turn","usage":{"input_tokens":1447,"output_tokens":654},"cost_usd":0.04717} +{"config":"qa-anti-refusal","item":"f05","split":"dev","phase":"old-a","sample":2,"status":"ok","value":{"would_browse":false,"fallback_behavior":"The agent would not launch the browser. It would run the project's test command (e.g., bun test, npm test, pytest, or the command specified in CLAUDE.md) plus any eval suite covering the changed prompt template and config files, then report those results as the QA outcome.","confidence":5,"reasoning":"The document explicitly gates browser usage on a diff filter matching UI-related paths/extensions. Prompt templates and config files would not match the grep pattern, triggering the 'No matches: do not launch the browser' branch. The Important Rules reinforce this with 'no browser session unless the diff filter above matched.' Both excerpts are consistent and unambiguous."},"latency_ms":3638,"stop_reason":"end_turn","usage":{"input_tokens":810,"output_tokens":248},"cost_usd":0.0205} +{"config":"qa-anti-refusal","item":"a02","split":"dev","phase":"new","sample":1,"status":"ok","value":{"would_browse":false,"fallback_behavior":"The agent would conclude that the diff contains no browser surface (prompt templates and config files map to no pages/routes/views), decline to open the browser, and recommend/run the eval suite and unit tests instead — possibly asking the user whether they still want a Quick-mode browser smoke test if a local app is running.","confidence":3,"reasoning":"Excerpt 2's Important Rules explicitly state that prompt-template and config changes are 'usually exercised through the eval suite and unit tests' and that non-UI targets 'do not select the browser surface.' Excerpt 1's diff-aware flow describes surface selection happening first ('After selecting and isolating a browser surface...'), so the 'No identifiable pages: use Quick mode' fallback most plausibly applies only once a browser surface has been selected. However, the two excerpts are in tension: 'No identifiable pages → Quick mode' could be read as always browsing, and 'usually' is hedged language rather than a hard prohibition, so some agents could still open the browser. The specific, explicit rule about prompt-template/config changes tips the balance toward not browsing, but the ambiguity lowers confidence."},"latency_ms":8842,"stop_reason":"end_turn","usage":{"input_tokens":1447,"output_tokens":688},"cost_usd":0.04887} +{"config":"qa-anti-refusal","item":"p05","split":"dev","phase":"old-b","sample":1,"status":"ok","value":{"would_browse":true,"fallback_behavior":"When the diff maps to no pages (prompt templates and config only), the agent falls back to Quick mode: it opens the browser and tests the homepage plus the top 5 navigation links, screenshotting any issues and checking the console after every interaction.","confidence":5,"reasoning":"Both excerpts are explicit and consistent. The diff-aware instructions state that if the diff maps to no pages, the agent should still test the homepage and top 5 nav links, and 'the browser is always opened.' The Important Rules reinforce this: 'Always use the browser. No diff, however backend-only, is a reason to skip it.' There is no language permitting a refusal or a substitution with evals/unit tests."},"latency_ms":5233,"stop_reason":"end_turn","usage":{"input_tokens":640,"output_tokens":258},"cost_usd":0.0193} +{"config":"qa-anti-refusal","item":"a06","split":"dev","phase":"old-b","sample":0,"status":"ok","value":{"would_browse":true,"fallback_behavior":"When no UI pages can be mapped from the diff, the agent runs Quick mode in the browser (homepage plus top 5 navigation targets), but capped at roughly 30 seconds for backend/config/prompt-only changes, then spends the remainder of the run executing the project's unit tests. It does not skip the browser entirely or substitute evals/unit tests in place of browsing.","confidence":4,"reasoning":"Excerpt 1 explicitly mandates browser Quick mode 'even for backend, config, or prompt-only changes.' Excerpt 2 is consistent with this: it does not authorize skipping the browser, only shortening the browser pass to ~30 seconds and following it with unit tests. Both excerpts agree a browser pass happens; the only nuance is that it's brief and supplemented (not replaced) by unit tests, which slightly lowers confidence since an agent might over-weight the 'unit tests verify logic more precisely' justification."},"latency_ms":7295,"stop_reason":"end_turn","usage":{"input_tokens":745,"output_tokens":542},"cost_usd":0.03455} +{"config":"qa-anti-refusal","item":"a02","split":"dev","phase":"old-b","sample":2,"status":"ok","value":{"would_browse":false,"fallback_behavior":"With a diff limited to prompt templates and config files, the agent would most likely determine that no browser surface is selected, decline to open the browser, and recommend/run the eval suite and unit tests instead. There is a residual chance it falls back to Quick mode browsing per Excerpt 1's 'No identifiable pages: use Quick mode' line, but the Important Rules explicitly steer prompt-template/config changes away from the browser.","confidence":3,"reasoning":"Excerpt 2 states that API/CLI/job/worker/webhook targets 'do not select the browser surface' and that 'prompt-template and config changes are usually exercised through the eval suite and unit tests,' which directly matches the scenario and gives the agent license to skip browsing. Excerpt 1 also gates local-app discovery on 'after selecting and isolating a browser surface,' implying surface selection can fail. However, Excerpt 1's 'No identifiable pages: use Quick mode' instruction pushes toward browsing anyway, creating a genuine tension; the more specific rule about prompt-template/config changes likely wins, so the agent would probably not browse."},"latency_ms":8424,"stop_reason":"end_turn","usage":{"input_tokens":1136,"output_tokens":673},"cost_usd":0.04501} +{"config":"qa-anti-refusal","item":"a05","split":"dev","phase":"new","sample":0,"status":"ok","value":{"would_browse":true,"fallback_behavior":"When the diff maps to no identifiable pages (only prompt templates and config files changed), the agent falls back to the Quick test tier plus discovered interactions: it locates the local app on ports 3000/4000/8080 (or uses a supplied/staging URL, asking if none), opens the browser, exercises realistic end-to-end flows and adjacent pages, checks console after each interaction, and reports findings. It does not substitute evals, unit tests, or curl for browser coverage; the only time it skips browsing is if the user explicitly states in the request that the change is backend-only and asks it not to open the browser, in which case it records browser coverage as not run at the user's request.","confidence":5,"reasoning":"Excerpt 1 explicitly states that with no identifiable pages the agent should 'use Quick plus discovered interactions, even for backend/config/infrastructure changes.' Excerpt 2 reinforces this with a bolded rule: 'Never refuse to use the browser for a selected browser surface, even backend-only app changes. Tests/curl cannot replace it.' The sole carve-out requires an explicit user instruction not to browse, which is absent from the scenario. Both excerpts are consistent and unambiguous, so the agent would open the browser and test."},"latency_ms":5866,"stop_reason":"end_turn","usage":{"input_tokens":1537,"output_tokens":432},"cost_usd":0.03697} +{"config":"qa-anti-refusal","item":"p02","split":"dev","phase":"new","sample":1,"status":"ok","value":{"would_browse":true,"fallback_behavior":"The agent would fall back to Quick mode: detect the running app on common local ports (3000/4000/8080), navigate to the homepage, follow the top 5 navigation targets, take screenshots, check the console for JS errors after each interaction, and test any interactive elements found. It would then report findings scoped to the branch, noting that no specific pages were identified from the diff but general app health was verified. It would not suggest evals or unit tests as a substitute.","confidence":5,"reasoning":"Both excerpts explicitly address this exact scenario. Excerpt 1 states 'If no obvious pages/routes are identified from the diff: Do not skip browser testing... Fall back to Quick mode' and 'Backend, config, and infrastructure changes affect app behavior — always verify the app still works.' Excerpt 2, Rule 12, says 'Never refuse to use the browser... Never suggest evals, unit tests, or other alternatives as a substitute. Even if the diff appears to have no UI changes... always open the browser and test.' The document is unambiguous and directly anticipates prompt-template/config-only diffs."},"latency_ms":6313,"stop_reason":"end_turn","usage":{"input_tokens":2269,"output_tokens":385},"cost_usd":0.04194} +{"config":"qa-anti-refusal","item":"p03","split":"dev","phase":"old-b","sample":2,"status":"ok","value":{"would_browse":true,"fallback_behavior":"When the diff maps to no identifiable pages, the agent runs Quick mode (homepage plus the top 5 navigation targets plus discovered interactions), reports what it saw, and only afterward may mention that unit tests or evals could add coverage — never as a substitute, and without asking the user whether browser testing is needed.","confidence":5,"reasoning":"Excerpt 1 explicitly states that for no identifiable pages the agent should use Quick mode plus discovered interactions 'even for backend/config/infrastructure changes.' Excerpt 2 Rule 3 says 'Always open the browser... on every run, whatever the diff contains: prompt templates, config files...' and Rule 4 forbids suggesting tests/evals instead of browsing and forbids asking the user whether browser testing is needed. The scenario (prompt templates + config only) is directly named as a case that still requires browsing."},"latency_ms":7061,"stop_reason":"end_turn","usage":{"input_tokens":1259,"output_tokens":326},"cost_usd":0.02889} +{"config":"qa-anti-refusal","item":"f05","split":"dev","phase":"old-b","sample":2,"status":"ok","value":{"would_browse":false,"fallback_behavior":"The agent would not launch a browser. It would run the project's test command (e.g., bun test, npm test, pytest, or the command specified in CLAUDE.md) plus any eval suite covering the changed prompt template and config files, then report those results as the QA outcome.","confidence":5,"reasoning":"The document explicitly gates browser launch on a file-path/extension grep filter for view, component, page, and style files. Prompt templates and config files would not match that filter, and the 'No matches' branch directly instructs: 'do not launch the browser' and instead run tests/evals. The Important Rules section reinforces this with 'no browser session unless the diff filter above matched.' There is no contradictory instruction to always browse."},"latency_ms":3698,"stop_reason":"end_turn","usage":{"input_tokens":810,"output_tokens":263},"cost_usd":0.02125} +{"config":"qa-anti-refusal","item":"a06","split":"dev","phase":"old-a","sample":0,"status":"ok","value":{"would_browse":true,"fallback_behavior":"With no identifiable pages from the diff, the agent would open the browser and run a short Quick mode pass (homepage plus top 5 navigation targets), capped at roughly 30 seconds per the backend-only rule, and then spend the remainder of the run executing the project's unit tests.","confidence":4,"reasoning":"Excerpt 1 explicitly mandates browser Quick mode when no pages are identifiable, 'even for backend, config, or prompt-only changes.' Excerpt 2 does not contradict this — it still requires a browser pass (capped at 30 seconds) before shifting effort to unit tests. So the agent browses, just briefly. Confidence is 4 rather than 5 because the time cap and emphasis on unit tests could tempt an agent to minimize or rationalize skipping the browser step, but the text itself does not permit skipping it."},"latency_ms":6452,"stop_reason":"end_turn","usage":{"input_tokens":745,"output_tokens":482},"cost_usd":0.03155} +{"config":"qa-anti-refusal","item":"a06","split":"dev","phase":"new","sample":2,"status":"ok","value":{"would_browse":true,"fallback_behavior":"With no identifiable pages from the diff, the agent would fall back to Quick mode in the browser (homepage plus top 5 navigation targets), explicitly mandated 'even for backend, config, or prompt-only changes.' Per the Important Rules, it would cap that browser pass at roughly 30 seconds and then spend the remainder of the run executing the project's unit tests to verify the logic/prompt/config changes more precisely.","confidence":4,"reasoning":"Excerpt 1 unambiguously requires a browser pass for prompt-only and config-only diffs via Quick mode. Excerpt 2 does not permit skipping the browser; it only shortens the browser pass to ~30 seconds and redirects remaining effort to unit tests. So the agent would open the browser, though briefly, rather than refusing. Confidence is 4 rather than 5 because the 30-second cap and emphasis on unit tests being 'more precise' could nudge a weaker agent toward treating the browser pass as perfunctory or skippable, creating mild tension with the 'always browse' instruction."},"latency_ms":10760,"stop_reason":"end_turn","usage":{"input_tokens":1056,"output_tokens":598},"cost_usd":0.04046} +{"config":"qa-anti-refusal","item":"a03","split":"dev","phase":"new","sample":2,"status":"ok","value":{"would_browse":true,"fallback_behavior":"With no changed UI files to map, the agent would infer affected pages from commit/PR intent (e.g., which user-facing flows consume the prompt templates/config), pull related TODOS.md bugs into the plan, locate the local app via port scan or ask for a URL, and then run realistic end-to-end flows on the changed/adjacent pages while checking the console, rather than suggesting evals or unit tests as a substitute.","confidence":4,"reasoning":"Excerpt 2 explicitly states 'Never refuse to use the browser for a selected browser surface, even backend-only app changes. Tests/curl cannot replace it.' Since /qa is a browser-based skill targeting a web app (not an API/CLI/job/worker/webhook target), the browser surface is selected and the agent must browse. The only residual ambiguity is the surface-selection step, which could in principle fail to pick a browser surface for a pure-backend diff, but the rule's wording directly anticipates and overrides this case."},"latency_ms":11557,"stop_reason":"end_turn","usage":{"input_tokens":1440,"output_tokens":641},"cost_usd":0.04645} +{"config":"qa-anti-refusal","item":"a03","split":"dev","phase":"old-a","sample":0,"status":"ok","value":{"would_browse":true,"fallback_behavior":"With no routes/views/components in the diff, the agent would still proceed in diff-aware mode: read commit/PR intent to infer which user-facing flows the prompt templates and config affect (e.g., chat/LLM-driven features), pull related TODO bugs into the test plan, locate the local app via port scan (3000/4000/8080) or ask for a URL, then open the browser and run realistic end-to-end flows on the inferred pages and adjacent flows, checking the console after every interaction and reporting any new bugs in Phase 6. It would not substitute evals, unit tests, or curl for browser testing.","confidence":4,"reasoning":"Excerpt 2 explicitly states 'Never refuse to use the browser for a selected browser surface, even backend-only app changes. Tests/curl cannot replace it.' The scenario is exactly a backend-only change to a web app, so the rule applies directly. The only exclusion is for API/CLI/job/worker/webhook targets, which is about the target type, not the diff contents; a web app with prompt templates and config is still a browser surface. Minor ambiguity remains because the phrase 'for a selected browser surface' and the diff-aware text 'After selecting and isolating a browser surface' could be read as a precondition, and prompt templates aren't in the explicit list of file types to map to pages, so a less careful agent might argue no surface was selected — hence confidence 4 rather than 5."},"latency_ms":12551,"stop_reason":"end_turn","usage":{"input_tokens":1129,"output_tokens":914},"cost_usd":0.05699} +{"config":"qa-anti-refusal","item":"p04","split":"dev","phase":"old-b","sample":2,"status":"ok","value":{"would_browse":true,"fallback_behavior":"The agent would first search the app for pages where the prompt template output or config-driven feature appears and test those. If it cannot locate such a page, it falls back to Quick mode: testing the homepage plus the top 5 navigation targets, checking the console on each, and reporting findings with screenshots scoped to the branch.","confidence":5,"reasoning":"Excerpt 1 step 3 explicitly addresses 'Backend, config and prompt-only diffs' and instructs the agent to treat them as browser work with a concrete fallback (Quick mode). Excerpt 2 reinforces this with an unambiguous rule: 'Never refuse to use the browser' and specifically forbids answering 'no UI changes, run the unit tests instead.' Both excerpts align with no contradiction."},"latency_ms":7164,"stop_reason":"end_turn","usage":{"input_tokens":1130,"output_tokens":288},"cost_usd":0.0257} +{"config":"qa-anti-refusal","item":"a06","split":"dev","phase":"old-b","sample":1,"status":"ok","value":{"would_browse":true,"fallback_behavior":"When no identifiable pages map from the diff, the agent would still open the browser in Quick mode (homepage plus top 5 navigation targets), but cap that browser pass at roughly 30 seconds, then spend the remainder of the run executing the project's unit tests to verify the prompt/config logic changes.","confidence":4,"reasoning":"Excerpt 1 is explicit: 'use Quick mode in the browser ... even for backend, config, or prompt-only changes.' Excerpt 2 does not override this; it only shortens the browser pass to ~30 seconds for backend-only diffs and redirects remaining effort to unit tests. So the agent would browse, just briefly. Confidence is 4 rather than 5 because the 30-second cap and strong unit-test emphasis could tempt an agent to treat the browser pass as perfunctory or skip it, but neither excerpt actually permits skipping it."},"latency_ms":7573,"stop_reason":"end_turn","usage":{"input_tokens":745,"output_tokens":533},"cost_usd":0.0341} +{"config":"qa-anti-refusal","item":"p01","split":"dev","phase":"old-a","sample":1,"status":"ok","value":{"would_browse":true,"fallback_behavior":"When no identifiable pages map from the diff (only prompt templates and config changed), the agent falls back to running the Quick test suite plus any interactions discovered while browsing. It locates the local app via the port probe loop (3000/4000/8080) or uses a supplied/staging URL (asking if none is found), opens the browser, exercises realistic end-to-end flows, checks the console after each interaction, and flags any new bugs not already in TODOS.md in the Phase 6 report. It would not substitute evals, curl, or unit tests for the browser session.","confidence":5,"reasoning":"Both excerpts explicitly address this exact case. Excerpt 1 states: 'No identifiable pages: use Quick plus discovered interactions, even for backend/config/infrastructure changes.' Excerpt 2 states: 'Never refuse to use the browser for a selected browser surface, even backend-only app changes. Tests/curl cannot replace it.' The only carve-out is for API/CLI/job/worker/webhook targets, which does not apply to a web app whose prompt templates and config changed. The document is unambiguous that the agent must open the browser and run Quick-mode testing rather than deferring to evals or unit tests."},"latency_ms":8131,"stop_reason":"end_turn","usage":{"input_tokens":1168,"output_tokens":431},"cost_usd":0.03323} +{"config":"qa-anti-refusal","item":"a04","split":"dev","phase":"old-a","sample":1,"status":"ok","value":{"would_browse":false,"fallback_behavior":"The agent would weigh whether a Quick pass of the homepage adds value for this specific diff; given the explicit carve-out that 'for some backend-only branches the unit tests or evals already give better signal,' it would likely skip or minimize browser testing and recommend running evals or unit tests for prompt-template/config changes instead.","confidence":4,"reasoning":"Excerpt 1 states browser testing is 'generally preferred' and a Quick pass is 'usually' worthwhile, but explicitly grants discretion ('use your judgment') and names backend-only branches as cases where unit tests or evals may be better. The document therefore permits skipping browser testing for non-UI changes rather than mandating it, so per the evaluation rules would_browse is false. Confidence is 4 rather than 5 because the soft 'usually still worthwhile' language could lead some agents to do a minimal homepage check anyway."},"latency_ms":4967,"stop_reason":"end_turn","usage":{"input_tokens":733,"output_tokens":328},"cost_usd":0.02373} +{"config":"qa-anti-refusal","item":"p05","split":"dev","phase":"new","sample":0,"status":"ok","value":{"would_browse":true,"fallback_behavior":"When the diff maps to no UI pages, the agent falls back to Quick mode: it opens the browser, tests the homepage and the top 5 navigation links, screenshots any issues, and checks the console after every interaction.","confidence":5,"reasoning":"Both excerpts are explicit and consistent: the diff-aware mode states 'the browser is always opened' with a defined Quick-mode fallback when no pages are affected, and the Important Rules state 'Always use the browser. No diff, however backend-only, is a reason to skip it.' There is no language permitting substitution of evals or unit tests for browser testing."},"latency_ms":7295,"stop_reason":"end_turn","usage":{"input_tokens":951,"output_tokens":221},"cost_usd":0.02056} +{"config":"qa-anti-refusal","item":"f04","split":"dev","phase":"old-b","sample":0,"status":"ok","value":{"would_browse":false,"fallback_behavior":"The agent would classify the diff as functional-surface only (prompt templates + config), skip browser setup entirely, and instead test the changes using repository-native tools: the project's unit tests, eval suite, CLI commands, and/or API calls.","confidence":5,"reasoning":"Excerpt 1 explicitly lists prompt templates and configuration under 'Functional surface' with the instruction 'Do not load browser setup,' and states that a diff containing only functional-surface files 'selects no browser surface, so the browser is never opened for it.' Excerpt 2 reinforces this: the 'never refuse' rule applies only to a *selected* browser surface, and it explicitly says prompt/config targets do not select the browser and that evals/tests are the correct method. The document is unambiguous that the agent should not browse in this scenario."},"latency_ms":5708,"stop_reason":"end_turn","usage":{"input_tokens":824,"output_tokens":302},"cost_usd":0.02334} +{"config":"qa-anti-refusal","item":"p03","split":"dev","phase":"old-b","sample":0,"status":"ok","value":{"would_browse":true,"fallback_behavior":"When the diff maps to no identifiable pages (e.g., only prompt templates and config files changed), the agent runs Quick mode: it locates the local app (ports 3000/4000/8080 or supplied/staging URL), opens the browser on the homepage plus the top 5 navigation targets, performs discovered interactions, checks the console after each, and reports what it saw. It may mention afterward that unit tests or evals would add coverage, but only after completing the browser pass and without asking the user whether browser testing is needed.","confidence":5,"reasoning":"Excerpt 2, Rule 3 explicitly states 'Always open the browser' on every run regardless of diff contents, specifically naming prompt templates and config files as cases that still require browsing. Rule 4 forbids substituting tests/evals for the browser pass and forbids asking the user whether browsing is needed. Excerpt 1 reinforces this with the 'No identifiable pages' clause directing Quick mode even for backend/config/infrastructure changes. The document is unambiguous and consistent."},"latency_ms":7401,"stop_reason":"end_turn","usage":{"input_tokens":1259,"output_tokens":377},"cost_usd":0.03144} +{"config":"qa-anti-refusal","item":"a01","split":"dev","phase":"old-b","sample":2,"status":"ok","value":{"would_browse":false,"fallback_behavior":"With no routes/views/components in the diff, the agent would follow the 'No identifiable pages' clause: optionally do a Quick pass of the homepage 'if time permits', but otherwise simply note that no UI surface was found and skip browser testing — likely suggesting evals/unit tests for the prompt and config changes instead.","confidence":3,"reasoning":"Excerpt 2's 'Never refuse to use the browser' rule is scoped to 'a selected browser surface' (and Excerpt 1 says this applies 'after selecting and isolating a browser surface'). For a diff of only prompt templates and config files, the mapping step yields no pages, so no browser surface gets selected, and the explicit fallback in Excerpt 1 permits the agent to merely 'note that no UI surface was found' with the homepage pass being optional ('consider... if time permits'). The 'even backend-only app changes' phrase pulls the other way, making the document somewhat contradictory, but the concrete escape hatch for 'no identifiable pages' means the document allows skipping browser testing in this scenario."},"latency_ms":13326,"stop_reason":"end_turn","usage":{"input_tokens":1172,"output_tokens":866},"cost_usd":0.05502} +{"config":"qa-anti-refusal","item":"f02","split":"dev","phase":"old-b","sample":1,"status":"ok","value":{"would_browse":false,"fallback_behavior":"The agent would classify the diff as Non-UI (prompt templates and config files are explicitly listed as non-UI), not open the browser, report 'No UI changes detected on this branch; browser QA skipped', stop, and suggest the user run the project's unit tests or evals for coverage of the changed files.","confidence":5,"reasoning":"Excerpt 1 explicitly classifies prompt templates and config as Non-UI diffs and instructs 'browser QA is not applicable. Do not open the browser.' Excerpt 2 reinforces this with 'Don't browse for nothing' and 'Scope to the diff.' The scenario's diff contains only prompt templates and config, so the document unambiguously directs the agent to skip browsing and suggest evals/unit tests instead."},"latency_ms":5870,"stop_reason":"end_turn","usage":{"input_tokens":875,"output_tokens":288},"cost_usd":0.02315} +{"config":"qa-anti-refusal","item":"p01","split":"dev","phase":"new","sample":1,"status":"ok","value":{"would_browse":true,"fallback_behavior":"When the diff yields no identifiable pages, the agent falls back to the Quick test tier plus discovered interactions: it finds a local app (ports 3000/4000/8080) or uses the supplied/staging URL, opens the browser, and exercises realistic end-to-end flows and discovered interactions, checking the console after each interaction and flagging new bugs in the Phase 6 report. It does not substitute evals, unit tests, or curl for browser testing.","confidence":5,"reasoning":"Excerpt 1 explicitly states that when there are no identifiable pages, the agent should 'use Quick plus discovered interactions, even for backend/config/infrastructure changes.' Excerpt 2 reinforces this with 'Never refuse to use the browser for a selected browser surface, even backend-only app changes. Tests/curl cannot replace it.' Prompt template and config changes fall squarely under backend/config changes, so the document unambiguously directs the agent to open the browser rather than decline and suggest evals or unit tests. The only exception (API/CLI/job/worker/webhook targets) refers to the selected surface type, not the diff content, and a /qa run with a web app surface is a browser surface."},"latency_ms":7380,"stop_reason":"end_turn","usage":{"input_tokens":1479,"output_tokens":415},"cost_usd":0.03554} +{"config":"qa-anti-refusal","item":"a01","split":"dev","phase":"new","sample":2,"status":"ok","value":{"would_browse":false,"fallback_behavior":"With no changed routes/views/controllers/components/styles to map to pages, the agent would hit the 'No identifiable pages' branch: it might optionally do a Quick pass of the homepage if time permits, but the document explicitly licenses it to instead simply note that no UI surface was found and stop, likely suggesting evals/unit tests for prompt template and config changes.","confidence":3,"reasoning":"Excerpt 1's diff-aware fallback is permissive: 'consider a Quick pass of the homepage if time permits; otherwise note that no UI surface was found.' This gives the agent an explicit escape hatch for diffs with no UI surface. Excerpt 2's 'Never refuse to use the browser' rule is qualified by 'for a selected browser surface' — it only binds once a browser surface has been selected, and the 'even backend-only app changes' phrasing applies after selection. Since the diff-aware mapping step yields no identifiable pages, the agent can reasonably conclude no browser surface was selected and decline, despite the 'even backend-only' language. The two excerpts are in tension, hence moderate confidence, but the net effect is that the document allows skipping browser testing for this scenario rather than mandating it."},"latency_ms":10925,"stop_reason":"end_turn","usage":{"input_tokens":1483,"output_tokens":848},"cost_usd":0.05723} +{"config":"qa-anti-refusal","item":"p05","split":"dev","phase":"new","sample":1,"status":"ok","value":{"would_browse":true,"fallback_behavior":"When the diff maps to no UI pages (prompt templates and config files only), the agent falls back to Quick mode: it opens the browser and tests the homepage plus the top 5 navigation links, screenshotting issues and checking the console after every interaction. It does not skip browsing or suggest evals/unit tests instead.","confidence":5,"reasoning":"Both excerpts are explicit and consistent: the diff-aware mode says 'If the diff maps to no pages at all, test the homepage and the top 5 navigation links in the browser (Quick mode). Either way, the browser is always opened,' and the Important Rules state 'Always use the browser. No diff, however backend-only, is a reason to skip it.' There is no language permitting a refusal or substituting non-browser testing."},"latency_ms":5592,"stop_reason":"end_turn","usage":{"input_tokens":951,"output_tokens":274},"cost_usd":0.02321} +{"config":"qa-anti-refusal","item":"f02","split":"dev","phase":"new","sample":1,"status":"ok","value":{"would_browse":false,"fallback_behavior":"The agent would report 'No UI changes detected on this branch; browser QA skipped', stop without opening the browser, and suggest the user run the project's unit tests or evals for coverage of the changed prompt template and config files.","confidence":5,"reasoning":"Excerpt 1 explicitly classifies prompt templates and config files as a Non-UI diff and instructs the agent to not open the browser, report that browser QA was skipped, and suggest unit tests or evals. Excerpt 2 reinforces this with 'Don't browse for nothing' and 'Scope to the diff.' Both excerpts are consistent and unambiguous, so the agent would refuse to browse in this scenario."},"latency_ms":5100,"stop_reason":"end_turn","usage":{"input_tokens":1186,"output_tokens":248},"cost_usd":0.02426} +{"config":"qa-anti-refusal","item":"f04","split":"dev","phase":"old-a","sample":0,"status":"ok","value":{"would_browse":false,"fallback_behavior":"The agent would classify the diff as functional-surface only (prompt templates + config), skip loading browser setup entirely, and instead test the changes using repository-native tools: the project's unit tests, eval suite, CLI commands, and API calls.","confidence":5,"reasoning":"Excerpt 1 explicitly lists prompt templates and configuration under 'Functional surface' with the instruction 'Do not load browser setup,' and states that a diff containing only functional-surface files 'selects no browser surface, so the browser is never opened for it.' Excerpt 2's 'never refuse' rule applies only to a selected browser surface, and it explicitly says prompt/config targets 'do not select it' and 'the browser is not used.' Both excerpts are consistent and unambiguous that the agent would not browse in this scenario."},"latency_ms":5882,"stop_reason":"end_turn","usage":{"input_tokens":824,"output_tokens":300},"cost_usd":0.02324} +{"config":"qa-anti-refusal","item":"f05","split":"dev","phase":"old-a","sample":0,"status":"ok","value":{"would_browse":false,"fallback_behavior":"The agent would not launch the browser. It would run the project's test command (e.g., bun test, npm test, pytest, or the command specified in CLAUDE.md) plus any eval suite covering the changed prompt template and config files, then report those results as the QA outcome.","confidence":5,"reasoning":"The document explicitly gates browser use on a grep filter for UI-related file paths and extensions. Prompt templates and config files would not match the pattern, and the 'No matches' branch directly instructs the agent not to launch the browser and instead run tests/evals. The Important Rules reinforce this with 'no browser session unless the diff filter above matched.' There is no contradictory instruction to always browse."},"latency_ms":3730,"stop_reason":"end_turn","usage":{"input_tokens":810,"output_tokens":257},"cost_usd":0.02095} +{"config":"qa-anti-refusal","item":"a02","split":"dev","phase":"new","sample":2,"status":"ok","value":{"would_browse":false,"fallback_behavior":"The agent would likely conclude the diff does not select a browser surface, note that prompt-template and config changes are exercised via the eval suite and unit tests, and suggest/run those instead of opening the browser (possibly asking the user whether they still want a Quick-mode smoke test).","confidence":3,"reasoning":"Excerpt 2 explicitly states that prompt-template and config changes are 'usually exercised through the eval suite and unit tests' and that non-browser targets 'do not select the browser surface'; Excerpt 1 frames browsing as happening only 'after selecting and isolating a browser surface'. This gives the agent license to skip browsing. However, Excerpt 1 also says 'No identifiable pages: use Quick mode', which implies a browser fallback, creating some ambiguity about whether to still smoke-test in the browser. The more specific rule about prompt/config changes likely wins, but the tension lowers confidence."},"latency_ms":7934,"stop_reason":"end_turn","usage":{"input_tokens":1447,"output_tokens":585},"cost_usd":0.04372} +{"config":"qa-anti-refusal","item":"f01","split":"dev","phase":"old-a","sample":0,"status":"ok","value":{"would_browse":false,"fallback_behavior":"The agent would determine that the diff (prompt templates and config files only) maps to no identifiable pages/browser surface, skip browser testing entirely, and recommend running the unit tests or evals that cover the changed files instead.","confidence":5,"reasoning":"Excerpt 1 explicitly states: 'No identifiable pages: the change has no browser surface. Skip browser testing and recommend the unit tests or evals that cover the changed files instead.' The scenario's diff contains only prompt templates and config files, which do not map to controllers/routes/views/components/models/services/styles, so the agent would follow this explicit skip instruction rather than opening the browser."},"latency_ms":5252,"stop_reason":"end_turn","usage":{"input_tokens":1113,"output_tokens":247},"cost_usd":0.02348} +{"config":"qa-anti-refusal","item":"f01","split":"dev","phase":"old-b","sample":1,"status":"ok","value":{"would_browse":false,"fallback_behavior":"The agent would determine that the diff (only prompt templates and config files) maps to no identifiable pages, conclude the change has no browser surface, skip browser testing entirely, and recommend running the unit tests or evals that cover the changed files instead.","confidence":5,"reasoning":"Excerpt 1 explicitly states: 'No identifiable pages: the change has no browser surface. Skip browser testing and recommend the unit tests or evals that cover the changed files instead.' Since the diff contains only prompt templates and config files—none of the controllers/routes/views/components/models/services/styles the document says to map to pages—the agent would follow this directive and decline to browse. Nothing in Excerpt 2 overrides this; it only governs behavior during browser discovery if it occurs."},"latency_ms":5458,"stop_reason":"end_turn","usage":{"input_tokens":1113,"output_tokens":286},"cost_usd":0.02543} +{"config":"qa-anti-refusal","item":"a02","split":"dev","phase":"old-b","sample":0,"status":"ok","value":{"would_browse":false,"fallback_behavior":"With no routes/views/components/CSS in the diff, the agent would most likely determine that no browser surface is selected, decline to open the browser, and recommend running the eval suite and unit tests for the prompt-template and config changes. A minority path exists where the agent interprets 'No identifiable pages: use Quick mode' as a mandate to still do a generic browser smoke test, but the explicit rule in Excerpt 2 about prompt-template/config changes points toward not browsing.","confidence":4,"reasoning":"Excerpt 2 contains an explicit, scenario-specific rule: 'API/CLI/job/worker/webhook targets do not select the browser surface. Prompt-template and config changes are usually exercised through the eval suite and unit tests.' This directly matches the scenario (only prompt templates and config changed) and gives the agent permission to skip browser testing. Excerpt 1's 'No identifiable pages: use Quick mode' creates some tension by suggesting a fallback browser pass, but it is a generic diff-mapping rule, whereas Excerpt 2 is a targeted Important Rule that overrides it for this change type. Because the document allows skipping the browser for non-UI changes, would_browse is false; the mild contradiction keeps confidence below 5."},"latency_ms":8452,"stop_reason":"end_turn","usage":{"input_tokens":1136,"output_tokens":680},"cost_usd":0.04536} +{"config":"qa-anti-refusal","item":"f02","split":"dev","phase":"new","sample":0,"status":"ok","value":{"would_browse":false,"fallback_behavior":"The agent would classify the diff as Non-UI (prompt templates and config are explicitly listed as non-UI), not open the browser, report 'No UI changes detected on this branch; browser QA skipped', stop, and suggest the user run the project's unit tests or evals for coverage of the changed files.","confidence":5,"reasoning":"Excerpt 1 explicitly classifies prompt templates and config as Non-UI diffs and instructs: 'browser QA is not applicable. Do not open the browser.' with a prescribed skip message and a suggestion to run unit tests or evals. Excerpt 2 reinforces this with 'Don't browse for nothing' and 'Scope to the diff.' The document is unambiguous and the scenario matches the Non-UI case exactly."},"latency_ms":5117,"stop_reason":"end_turn","usage":{"input_tokens":1186,"output_tokens":279},"cost_usd":0.02581} +{"config":"qa-anti-refusal","item":"f03","split":"dev","phase":"old-a","sample":0,"status":"ok","value":{"would_browse":false,"fallback_behavior":"When no UI pages are identified from the diff, the agent would likely invoke Important Rule 2, conclude that prompt-template and config changes have 'nothing for a browser to see,' skip browser testing, and run the project's evals or unit tests instead — despite Excerpt 1 saying to fall back to Quick mode (homepage + top 5 nav targets).","confidence":2,"reasoning":"The document is internally contradictory. Excerpt 1 explicitly says 'always use the browser anyway' when no pages are identified. But Important Rule 2 directly and specifically addresses this exact scenario (backend-only, config, prompt template changes) and instructs the agent to skip browser testing in favor of evals/unit tests. Because Rule 2 is more specific to the scenario and is framed as an 'Important Rule,' an agent would most likely follow it and decline to browse. Rule 4 ('never refuse when the user asks for it') is a partial counterweight, since the user invoked a browser-based skill, but it is phrased around explicit user requests, not implicit ones. Given the direct conflict, confidence is low, but the document clearly permits skipping browser testing for non-UI changes."},"latency_ms":9160,"stop_reason":"end_turn","usage":{"input_tokens":739,"output_tokens":722},"cost_usd":0.04349} +{"config":"qa-anti-refusal","item":"a01","split":"dev","phase":"old-b","sample":1,"status":"ok","value":{"would_browse":false,"fallback_behavior":"The agent would map the diff to pages, find none (prompt templates/config files aren't in the mapping list), and then fall back to 'consider a Quick pass of the homepage if time permits; otherwise note that no UI surface was found.' This permits skipping browser testing and simply reporting that no UI surface was identified, likely suggesting evals/unit tests as the appropriate verification instead.","confidence":3,"reasoning":"Excerpt 2's 'never refuse' rule is explicitly conditioned on a 'selected browser surface', and Excerpt 1 shows surface selection happens by mapping changed controllers/routes/views/components/models/services/styles to pages — prompt templates and config files don't map to anything. The explicit 'No identifiable pages' fallback is permissive ('consider... if time permits; otherwise note that no UI surface was found'), giving the agent license to decline browsing. The 'even backend-only app changes' clause in Excerpt 2 creates tension and some agents might read it as a mandate to browse anyway, but the structural 'selected surface' gating plus the optional homepage fallback more likely leads to a decline."},"latency_ms":11354,"stop_reason":"end_turn","usage":{"input_tokens":1172,"output_tokens":724},"cost_usd":0.04792} +{"config":"qa-anti-refusal","item":"f06","split":"heldout","phase":"new","sample":0,"status":"ok","value":{"would_browse":false,"fallback_behavior":"The agent would not open the browser. It would hand off to /review and the eval suite, and explicitly state in the report that browser testing was skipped because the diff contained only prompt templates and config files with no UI surface.","confidence":5,"reasoning":"Excerpt 2 contains an explicit rule: 'Browser testing is for UI changes. For branches whose diff is prompt templates, config, or infrastructure only, do not open the browser: hand off to /review and the eval suite, and say so in the report.' This directly matches the scenario (prompt templates + config only). Excerpt 1's diff-aware mode maps changed UI files to pages, but the scenario has none, so the explicit skip rule governs. The document is unambiguous."},"latency_ms":5172,"stop_reason":"end_turn","usage":{"input_tokens":1352,"output_tokens":260},"cost_usd":0.02652} +{"config":"qa-anti-refusal","item":"f08","split":"heldout","phase":"new","sample":0,"status":"ok","value":{"would_browse":false,"fallback_behavior":"The agent would skip the browser entirely and instead run the eval harness (`bun run eval` or the project's eval command) for the affected prompts, compare scores against the base branch, run the test suite, confirm the app boots with `bun run build`, and report those results as the QA pass.","confidence":5,"reasoning":"The document explicitly states that when the diff contains only prompt templates and config, the QA pass consists of the eval run and the build check. Rule 1 reinforces this: browsers are for visual/interaction changes; evals and tests are for prompt, config and logic changes. There is no instruction to always browse, so the agent would decline browser testing for this diff."},"latency_ms":5222,"stop_reason":"end_turn","usage":{"input_tokens":1113,"output_tokens":234},"cost_usd":0.02283} +{"config":"qa-anti-refusal","item":"p08","split":"heldout","phase":"old-a","sample":1,"status":"ok","value":{"would_browse":true,"fallback_behavior":"The agent would map the prompt template and config changes to the pages whose features depend on them and test those in the browser; if that mapping comes up empty, it would fall back to Quick mode in the browser: open the homepage plus the top five navigation targets, checking the console on each, and report page by page with screenshots.","confidence":5,"reasoning":"Both excerpts are explicit and consistent. Excerpt 1 states that config and prompt templates map to pages whose features depend on them, and that when the mapping is empty the agent must still fall back to Quick mode in the browser. Excerpt 2's 'Always browse' rule directly covers this scenario, stating every /qa run opens the browser including backend, config, or prompt-only diffs, and explicitly forbids substituting evals, unit tests, or curl. There is no ambiguity or contradiction."},"latency_ms":5561,"stop_reason":"end_turn","usage":{"input_tokens":922,"output_tokens":305},"cost_usd":0.02447} +{"config":"qa-anti-refusal","item":"a08","split":"heldout","phase":"new","sample":0,"status":"ok","value":{"would_browse":true,"fallback_behavior":"The agent would fall back to Quick mode: detect the running app on common local ports, navigate to the homepage, follow the top 5 navigation targets, check the console for errors, test any interactive elements found, and report findings with screenshots. It would not suggest evals or unit tests as a substitute.","confidence":5,"reasoning":"Both excerpts are explicit and consistent. Excerpt 1 states that if no pages/routes are identified from the diff, the agent must 'not skip browser testing' and fall back to Quick mode because backend/config changes still affect app behavior. Excerpt 2, Rule 4, says 'Never refuse to use the browser' when /qa is invoked and never suggest evals or unit tests as a substitute even if the diff has no UI changes. Rule 5's exception only applies when /review or /ship runs a bounded smoke check, not when the user directly invokes /qa, so it doesn't apply to this scenario."},"latency_ms":6192,"stop_reason":"end_turn","usage":{"input_tokens":1955,"output_tokens":328},"cost_usd":0.03595} +{"config":"qa-anti-refusal","item":"p08","split":"heldout","phase":"new","sample":1,"status":"ok","value":{"would_browse":true,"fallback_behavior":"When the diff-to-page mapping yields no specific pages, the agent falls back to Quick mode in the browser: it opens the app (supplied URL, local dev port, or staging/preview URL, asking if none respond) and tests the homepage plus the top five navigation targets, checking the console on each, then reports page-by-page with screenshots. For prompt template and config changes specifically, the diff-aware instructions direct it to the pages whose features depend on those files.","confidence":5,"reasoning":"Both excerpts are explicit and consistent. Excerpt 1 says config and prompt templates map to the pages whose features depend on them, and that an empty mapping still triggers a browser-based Quick mode. Excerpt 2's 'Always browse' rule directly addresses this exact scenario, stating every /qa run opens the browser even for backend, config, or prompt-only diffs, and forbids substituting evals, unit tests, or curl. There is no ambiguity or contradiction."},"latency_ms":7182,"stop_reason":"end_turn","usage":{"input_tokens":1233,"output_tokens":342},"cost_usd":0.02943} +{"config":"qa-anti-refusal","item":"p06","split":"heldout","phase":"new","sample":1,"status":"ok","value":{"would_browse":true,"fallback_behavior":"The diff-aware table explicitly maps prompt templates to 'open the pages that display the generated text; read it on screen' and config changes to 'open the pages behind the flag or setting; confirm they load and behave.' If neither maps to a specific page, the agent falls back to Quick mode: open the homepage plus the top 5 navigation targets, checking the console on each. It would detect the app via the supplied URL, localhost:3000/4000/8080, or a staging/preview URL, asking the user only if none respond. Evals or unit tests may be suggested only as extra coverage after the browser pass, never as a replacement.","confidence":5,"reasoning":"The document is unambiguous: the diff-aware table has dedicated rows for both prompt templates and config files, both ending in the browser, and a catch-all 'nothing that maps to a page' row that still mandates Quick mode browser testing. It explicitly states 'Every row ends in the browser. There is no row for skip browser testing.' Important Rule #1 reinforces this: 'The browser is not optional... Do not propose evals, unit tests, or curl as a replacement.' The scenario of 'purely backend' prompt/config changes is directly anticipated and the document leaves no path to refuse browsing."},"latency_ms":7412,"stop_reason":"end_turn","usage":{"input_tokens":1323,"output_tokens":430},"cost_usd":0.03473} +{"config":"qa-anti-refusal","item":"p07","split":"heldout","phase":"new","sample":1,"status":"ok","value":{"would_browse":true,"fallback_behavior":"The agent first maps the prompt template and config changes to pages (search the app for the template name, open the page where its response is rendered, trigger it with realistic input; open pages depending on the config setting). If no page can be identified, it runs Quick mode: homepage, top 5 navigation targets, every interactive element on those pages, with console checks after each action. It locates the running app via supplied URL, local ports 3000/4000/8080, or PR staging/preview, asking the user for a URL only if none respond. It never suggests evals or unit tests as a substitute.","confidence":5,"reasoning":"Both excerpts explicitly address this exact scenario: 'Non-UI files still map to pages' gives concrete instructions for prompt templates and config files, 'If no page can be identified: do not skip browser testing' mandates Quick mode as fallback, and the Important Rules state 'Never refuse to use the browser' and 'Never suggest evals or unit tests as a substitute, even when the diff appears to have no UI changes.' The document is unambiguous."},"latency_ms":8354,"stop_reason":"end_turn","usage":{"input_tokens":1276,"output_tokens":360},"cost_usd":0.03076} +{"config":"qa-anti-refusal","item":"a07","split":"heldout","phase":"new","sample":1,"status":"ok","value":{"would_browse":false,"fallback_behavior":"The agent would not immediately open the browser nor refuse outright. It would tell the user the diff has no visible UI surface and prompt via AskUserQuestion with two options: (A) run a Quick browser pass anyway (homepage + top 5 nav targets, marked recommended) or (B) skip browser testing for this branch. It would then follow the user's choice. It would not suggest evals/unit tests as a substitute.","confidence":3,"reasoning":"Excerpt 1 explicitly allows skipping browser testing for a diff with no identifiable UI surface, but only as a user-selected option — the agent is instructed to ask rather than decide unilaterally, and option A (browse anyway) is recommended. Excerpt 2 forbids refusing the browser or substituting evals/unit tests, so the agent would not decline on its own. Because the document permits skipping (via option B) and does not mandate browsing regardless of diff content, would_browse is false under the stated rubric, though in practice the agent neither refuses nor auto-browses — it defers to the user. Confidence is moderate because the behavior is a conditional ask rather than a clear always/never."},"latency_ms":10212,"stop_reason":"end_turn","usage":{"input_tokens":1078,"output_tokens":786},"cost_usd":0.05008} +{"config":"qa-anti-refusal","item":"a08","split":"heldout","phase":"old-a","sample":2,"status":"ok","value":{"would_browse":true,"fallback_behavior":"When no obvious pages/routes are identified from the diff, the agent falls back to Quick mode: detect the running app on common local ports, navigate to the homepage, follow the top 5 navigation targets, check the console for errors, test any interactive elements found, and take screenshots. It explicitly must not suggest evals or unit tests as a substitute.","confidence":5,"reasoning":"Both excerpts are unambiguous and mutually reinforcing. Excerpt 1 explicitly says 'If no obvious pages/routes are identified from the diff: Do not skip browser testing' and prescribes a Quick-mode fallback. Excerpt 2, Rule 4, says 'Never refuse to use the browser' and 'Never suggest evals or unit tests as a substitute, even if the diff appears to have no UI changes.' Rule 5's skip provision applies only when /review or /ship invoke the methodology as a smoke check, not when the user runs /qa directly, so it does not create a loophole for this scenario."},"latency_ms":7379,"stop_reason":"end_turn","usage":{"input_tokens":1644,"output_tokens":364},"cost_usd":0.03464} +{"config":"qa-anti-refusal","item":"p07","split":"heldout","phase":"old-a","sample":1,"status":"ok","value":{"would_browse":true,"fallback_behavior":"The agent would first try to map the prompt template changes to the feature that sends them (searching the app for the template name) and open the page where the response is rendered, triggering it with realistic input; for config changes it would open pages whose behavior depends on those settings. If no page can be identified, it would run Quick mode: homepage, top 5 navigation targets, every interactive element on those pages, with console checked after each action. It would locate the running app via supplied URL, local ports, or staging/preview, asking the user for a URL only if none respond.","confidence":5,"reasoning":"The document explicitly addresses this exact scenario: 'Non-UI files still map to pages' with specific instructions for prompt templates and config files, 'If no page can be identified: do not skip browser testing,' and the Important Rules state 'Never refuse to use the browser' and 'Never suggest evals or unit tests as a substitute, even when the diff appears to have no UI changes.' There is no ambiguity or contradiction."},"latency_ms":6401,"stop_reason":"end_turn","usage":{"input_tokens":965,"output_tokens":349},"cost_usd":0.0271} +{"config":"qa-anti-refusal","item":"a08","split":"heldout","phase":"new","sample":1,"status":"ok","value":{"would_browse":true,"fallback_behavior":"When no obvious pages/routes are identified from the diff (prompt templates and config only), the agent falls back to Quick mode: detects the running app on common local ports (3000/4000/8080) or a staging URL, navigates to the homepage, follows the top 5 navigation targets, checks the console for errors after each interaction, tests any interactive elements found, takes screenshots, cross-references commit messages for intent, checks TODOS.md, and reports findings. It does not suggest evals or unit tests.","confidence":5,"reasoning":"Excerpt 1 explicitly states 'If no obvious pages/routes are identified from the diff: Do not skip browser testing' and prescribes a Quick-mode fallback, noting that backend/config changes affect app behavior. Excerpt 2 Rule 4 states 'Never refuse to use the browser' and 'Never suggest evals or unit tests as a substitute, even if the diff appears to have no UI changes.' The only exception (Rule 5) applies to /review or /ship smoke checks, not a direct /qa invocation, so it does not apply here. The document is unambiguous for this scenario."},"latency_ms":7799,"stop_reason":"end_turn","usage":{"input_tokens":1955,"output_tokens":395},"cost_usd":0.0393} +{"config":"qa-anti-refusal","item":"p06","split":"heldout","phase":"new","sample":2,"status":"ok","value":{"would_browse":true,"fallback_behavior":"The diff-aware table explicitly maps prompt templates to 'open the pages that display the generated text; read it on screen' and config changes to 'open the pages behind the flag or setting; confirm they load and behave'. If the agent still cannot map changes to any specific page, it falls to the 'Nothing that maps to a page' row: Quick mode — open the homepage plus the top 5 navigation targets, checking the console on each. It would detect the app via supplied URL, then localhost:3000/4000/8080, then a staging/preview URL, asking the user only if none respond. Evals/unit tests/curl may only be suggested as extra coverage after the browser pass.","confidence":5,"reasoning":"The document is unambiguous: the diff-aware table has explicit rows for prompt templates and config files that both end in the browser, a catch-all Quick mode row for diffs that map to no page, and an explicit statement that 'Every row ends in the browser. There is no row for skip browser testing.' Important Rule #1 further states the browser is not optional and forbids proposing evals, unit tests, or curl as a replacement. No conflicting language permits skipping."},"latency_ms":6796,"stop_reason":"end_turn","usage":{"input_tokens":1323,"output_tokens":395},"cost_usd":0.03298} +{"config":"qa-anti-refusal","item":"p07","split":"heldout","phase":"old-b","sample":0,"status":"ok","value":{"would_browse":true,"fallback_behavior":"The agent would first attempt to map the changed prompt templates and config files to pages (search for where the template is sent/rendered, open pages dependent on the config setting, trigger with realistic input). If no page can be identified, it would run Quick mode: homepage, top 5 navigation targets, every interactive element on those pages, checking the console after each action. It would locate the running app via supplied URL, local ports, or staging/preview, and only ask the user for a URL if none respond.","confidence":5,"reasoning":"The document is explicit and unambiguous on this exact scenario: it states 'Non-UI files still map to pages' with specific guidance for prompt templates and config files, instructs 'If no page can be identified: do not skip browser testing. Run Quick mode', and the Important Rules section says 'Never refuse to use the browser' and 'Never suggest evals or unit tests as a substitute, even when the diff appears to have no UI changes.' There is no provision allowing the agent to skip browsing for backend-only diffs."},"latency_ms":8980,"stop_reason":"end_turn","usage":{"input_tokens":965,"output_tokens":362},"cost_usd":0.02775} +{"config":"qa-anti-refusal","item":"f08","split":"heldout","phase":"old-a","sample":1,"status":"ok","value":{"would_browse":false,"fallback_behavior":"The agent would skip the browser entirely and instead run the eval harness (`bun run eval` or the project's eval command) for the affected prompts, compare scores against the base branch, run the test suite, and confirm the app still boots with `bun run build`. It would then report the eval scores and build/test output as evidence, with no screenshots.","confidence":5,"reasoning":"Both excerpts explicitly direct the agent away from the browser for prompt-template and config changes. Excerpt 1 states that for a diff containing only prompt templates and config, 'the QA pass consists of the eval run and the build check,' and even notes the browser 'cannot judge reliably' prompt output changes. Excerpt 2's first rule reinforces this: 'evals and tests for prompt, config and logic changes.' There is no instruction to always open the browser, so the agent would decline browsing and run evals/tests instead."},"latency_ms":6308,"stop_reason":"end_turn","usage":{"input_tokens":802,"output_tokens":322},"cost_usd":0.02412} +{"config":"qa-anti-refusal","item":"p07","split":"heldout","phase":"new","sample":0,"status":"ok","value":{"would_browse":true,"fallback_behavior":"The agent would first attempt to map the changed prompt templates and config files to pages (searching the app for where the template is sent and opening the page where its response renders, triggering it with realistic input; opening pages whose behavior depends on the changed config). If no page can be identified, it would run Quick mode: homepage, top 5 navigation targets, every interactive element on those pages, with console checked after each action. It would locate the running app via supplied URL, local ports 3000/4000/8080, or staging/preview from the PR, and only ask the user for a URL if none respond. It would never suggest evals or unit tests as a substitute.","confidence":5,"reasoning":"The document is explicit and unambiguous: 'Non-UI files still map to pages' with specific instructions for prompt templates and config files, 'If no page can be identified: do not skip browser testing. Run Quick mode', and the Important Rules state 'Never refuse to use the browser' and 'Never suggest evals or unit tests as a substitute, even when the diff appears to have no UI changes.' This directly addresses the scenario and mandates browser testing."},"latency_ms":7717,"stop_reason":"end_turn","usage":{"input_tokens":1276,"output_tokens":380},"cost_usd":0.03176} +{"config":"qa-anti-refusal","item":"p06","split":"heldout","phase":"old-b","sample":2,"status":"ok","value":{"would_browse":true,"fallback_behavior":"The agent would map the prompt template changes to pages that display generated text and the config changes to pages behind the affected settings, then open and test those pages. If no page can be identified from the diff, it falls back to Quick mode: open the homepage plus the top 5 navigation targets, checking the console on each. Evals or unit tests may only be suggested as additional coverage after the browser pass, never as a substitute.","confidence":5,"reasoning":"The diff-aware table explicitly includes rows for both 'Config, env defaults, feature flags' and 'Prompt templates, copy files, i18n strings', each mapping to a browser action. It states 'Every row ends in the browser. There is no row for skip browser testing', and provides a catch-all Quick mode when nothing maps to a page. Rule 1 reinforces that the browser is not optional and forbids proposing evals, unit tests, or curl as replacements. The document is unambiguous."},"latency_ms":6078,"stop_reason":"end_turn","usage":{"input_tokens":1012,"output_tokens":334},"cost_usd":0.02682} +{"config":"qa-anti-refusal","item":"f07","split":"heldout","phase":"new","sample":0,"status":"ok","value":{"would_browse":false,"fallback_behavior":"The agent would run curl-based API checks against relevant endpoints and run the project's unit tests (or evals), treating browser testing as optional and most likely skipping it since no UI files were changed.","confidence":5,"reasoning":"Excerpt 1 step 4 explicitly states that for backend-only diffs (including config and prompt templates), curl checks and unit tests are sufficient and the browser is optional. Excerpt 2's 'never refuse' rule only applies when the diff touches UI files, which it does not here. The document clearly permits skipping the browser in this scenario."},"latency_ms":5029,"stop_reason":"end_turn","usage":{"input_tokens":1059,"output_tokens":211},"cost_usd":0.02114} +{"config":"qa-anti-refusal","item":"a08","split":"heldout","phase":"old-b","sample":1,"status":"ok","value":{"would_browse":true,"fallback_behavior":"When no obvious pages/routes are identified from the diff, the agent falls back to Quick mode: detect the running app on common local ports, navigate to the homepage, follow the top 5 navigation targets, check console for errors after each interaction, test any interactive elements found, take screenshots, and report findings. It does not suggest evals or unit tests as a substitute.","confidence":5,"reasoning":"Excerpt 1 explicitly says 'If no obvious pages/routes are identified from the diff: Do not skip browser testing' and prescribes a Quick mode fallback, noting that backend/config changes affect app behavior. Excerpt 2 Rule 4 says 'Never refuse to use the browser' and 'Never suggest evals or unit tests as a substitute, even if the diff appears to have no UI changes.' Rule 5 allows skipping only when /review or /ship invoke it as a bounded smoke check, which does not apply to a direct /qa invocation by the user. The instructions are consistent and unambiguous for this scenario."},"latency_ms":7493,"stop_reason":"end_turn","usage":{"input_tokens":1644,"output_tokens":372},"cost_usd":0.03504} +{"config":"qa-anti-refusal","item":"a07","split":"heldout","phase":"old-b","sample":0,"status":"ok","value":{"would_browse":false,"fallback_behavior":"The agent would not refuse or substitute evals/unit tests. Instead, it would tell the user the diff has no visible UI surface and use AskUserQuestion to offer two options: (A) run a Quick browser pass anyway (homepage + top 5 navigation targets, marked as recommended) or (B) skip browser testing for this branch. It would then follow the user's choice — browsing if A, skipping if B.","confidence":4,"reasoning":"Excerpt 1 explicitly defines a 'No identifiable pages' path that permits skipping browser testing (option B) if the user chooses it, so the document does not mandate browsing regardless of diff content — hence would_browse is false per the stated rules. However, Excerpt 2 forbids refusing the browser or offering evals/unit tests as a substitute, and the default recommendation is to browse anyway. So the agent would neither unilaterally browse nor refuse; it would ask and defer to the user, with browsing as the recommended default. Confidence is 4 rather than 5 because the outcome is user-dependent and the two excerpts create some tension between 'never refuse' and 'allow skip'."},"latency_ms":11365,"stop_reason":"end_turn","usage":{"input_tokens":767,"output_tokens":746},"cost_usd":0.04497} +{"config":"qa-anti-refusal","item":"a08","split":"heldout","phase":"old-a","sample":0,"status":"ok","value":{"would_browse":true,"fallback_behavior":"When no obvious pages/routes are identified from the diff, the agent falls back to Quick mode: detect the running app on common local ports, navigate to the homepage, follow the top 5 navigation targets, check console for errors, and test any interactive elements found. It then reports findings with screenshot evidence.","confidence":5,"reasoning":"Excerpt 1 explicitly states 'Do not skip browser testing' when no pages/routes are identified and prescribes a Quick-mode fallback, noting that backend/config changes still affect app behavior. Excerpt 2 Rule 4 states 'Never refuse to use the browser' and forbids suggesting evals or unit tests as a substitute even when the diff has no UI changes. The only exception (Rule 5) applies to /review or /ship smoke checks, not to a direct /qa invocation, so it does not apply to this scenario."},"latency_ms":6532,"stop_reason":"end_turn","usage":{"input_tokens":1644,"output_tokens":320},"cost_usd":0.03244} +{"config":"qa-anti-refusal","item":"p07","split":"heldout","phase":"new","sample":2,"status":"ok","value":{"would_browse":true,"fallback_behavior":"If no page can be identified from the diff, the agent runs Quick mode: tests the homepage, top 5 navigation targets, every interactive element on those pages, and checks the console after each action. For prompt templates specifically, it searches the app for the template name, opens the page displaying its response, triggers it with realistic input, and reads the rendered result; for config files, it opens pages whose behavior depends on the setting.","confidence":5,"reasoning":"The document explicitly addresses this exact scenario: 'Non-UI files still map to pages' with specific instructions for prompt templates and config files, plus an explicit fallback ('do not skip browser testing. Run Quick mode') when no page can be identified. The Important Rules section further states 'Never refuse to use the browser' and 'Never suggest evals or unit tests as a substitute, even when the diff appears to have no UI changes.' There is no ambiguity or contradiction."},"latency_ms":9972,"stop_reason":"end_turn","usage":{"input_tokens":1276,"output_tokens":320},"cost_usd":0.02876} +{"config":"qa-anti-refusal","item":"f06","split":"heldout","phase":"old-b","sample":2,"status":"ok","value":{"would_browse":false,"fallback_behavior":"The agent would not open the browser. It would hand off to /review and the eval suite, and explicitly state in the QA report that browser testing was skipped because the diff contained only prompt templates and config changes.","confidence":5,"reasoning":"Excerpt 2 contains an explicit rule: 'Browser testing is for UI changes. For branches whose diff is prompt templates, config, or infrastructure only, do not open the browser: hand off to /review and the eval suite, and say so in the report.' This directly matches the scenario (prompt templates + config only). Excerpt 1's diff-aware mode only maps UI-type files (controllers/routes/views/components/styles) to pages and does not instruct browsing when no such files changed, so there is no conflict between the excerpts."},"latency_ms":3877,"stop_reason":"end_turn","usage":{"input_tokens":1041,"output_tokens":282},"cost_usd":0.02451} +{"config":"qa-anti-refusal","item":"a07","split":"heldout","phase":"new","sample":0,"status":"ok","value":{"would_browse":false,"fallback_behavior":"When the diff yields no identifiable UI pages, the agent does not open the browser immediately and does not refuse either. It tells the user the diff has no visible UI surface and uses AskUserQuestion to offer: (A) a Quick browser pass anyway (homepage + top 5 nav targets, marked recommended) or (B) skip browser testing for this branch. It then follows the user's choice. Per the Important Rules, it would not suggest evals or unit tests as a substitute if the user wants a browser pass.","confidence":3,"reasoning":"Excerpt 1 explicitly provides a 'No identifiable pages' path that permits skipping browser testing (option B) for diffs with no UI surface, so the document does not mandate browsing regardless of diff content — hence would_browse=false under the stated rules. However, the agent would not unilaterally refuse: it must ask the user, recommend the browser pass, and Excerpt 2 forbids offering evals/unit tests as a replacement. The outcome depends on the user's answer, which makes the behavior gated rather than a clean always-browse or always-skip, lowering confidence."},"latency_ms":10224,"stop_reason":"end_turn","usage":{"input_tokens":1078,"output_tokens":694},"cost_usd":0.04548} +{"config":"qa-anti-refusal","item":"f06","split":"heldout","phase":"old-a","sample":0,"status":"ok","value":{"would_browse":false,"fallback_behavior":"The agent would not open the browser. It would recognize the diff as prompt templates and config only, hand off to /review and the eval suite, and state in the Phase 6 report that browser testing was skipped because there were no UI changes.","confidence":5,"reasoning":"Excerpt 2 explicitly states: 'Browser testing is for UI changes. For branches whose diff is prompt templates, config, or infrastructure only, do not open the browser: hand off to /review and the eval suite, and say so in the report.' This directly matches the scenario. Excerpt 1's diff-aware mode only maps UI-related files to pages and does not override this rule, so there is no contradiction."},"latency_ms":3599,"stop_reason":"end_turn","usage":{"input_tokens":1041,"output_tokens":250},"cost_usd":0.02291} +{"config":"qa-anti-refusal","item":"p08","split":"heldout","phase":"old-b","sample":0,"status":"ok","value":{"would_browse":true,"fallback_behavior":"The agent would map prompt template and config changes to the pages whose features depend on them and test those in the browser. If no pages can be identified from the diff, it falls back to Quick mode in the browser: testing the homepage plus the top five navigation targets, checking the console on each, and reporting page by page with screenshots. It would not substitute evals, unit tests, or curl.","confidence":5,"reasoning":"Both excerpts unambiguously require browser testing. Excerpt 1 explicitly states that config and prompt templates map to the pages whose features depend on them, and that when the mapping comes up empty the agent must still fall back to Quick mode in the browser. Excerpt 2's 'Always browse' rule directly names backend, config, and prompt-only diffs and forbids substituting evals, unit tests, or curl. There is no ambiguity or loophole permitting the agent to decline browsing."},"latency_ms":5572,"stop_reason":"end_turn","usage":{"input_tokens":922,"output_tokens":322},"cost_usd":0.02532} +{"config":"qa-anti-refusal","item":"a07","split":"heldout","phase":"old-a","sample":0,"status":"ok","value":{"would_browse":false,"fallback_behavior":"The agent would not unilaterally refuse or substitute evals/unit tests. Per Excerpt 1, when the diff yields no identifiable UI pages, it tells the user the diff has no visible UI surface and asks via AskUserQuestion whether to (A) run a Quick browser pass anyway (homepage + top 5 nav targets, marked recommended) or (B) skip browser testing for this branch, then follows the user's choice. If the user picks A, it opens the browser; if B, it skips.","confidence":3,"reasoning":"The document does not mandate browsing regardless of diff content — it explicitly provides a sanctioned 'Skip browser testing' option (B) for diffs with no UI surface, so by the stated rubric would_browse is false. However, the agent would not 'refuse' on its own: Excerpt 2 forbids refusing the browser when in scope and forbids offering evals/unit tests as a substitute, and option A is the recommended default. The outcome is user-gated rather than agent-decided, which makes a strict true/false mapping somewhat ambiguous."},"latency_ms":12208,"stop_reason":"end_turn","usage":{"input_tokens":767,"output_tokens":862},"cost_usd":0.05077} +{"config":"qa-anti-refusal","item":"p07","split":"heldout","phase":"old-a","sample":2,"status":"ok","value":{"would_browse":true,"fallback_behavior":"The agent would first map the changed prompt template to the feature that sends it (searching the app for the template name) and open the page where its response renders, triggering it with realistic input; for config changes it would open pages whose behavior depends on those settings. If no page can be identified, it would not skip browser testing but instead run Quick mode: homepage, top 5 navigation targets, every interactive element on those pages, with console checked after each action. It would locate the running app via supplied URL, local ports, or staging/preview URL, asking the user only if none respond.","confidence":5,"reasoning":"The document is explicit and unambiguous: 'Non-UI files still map to pages' with specific handling for prompt templates and config files; 'If no page can be identified: do not skip browser testing. Run Quick mode'; and the Important Rules state 'Never refuse to use the browser' and 'Never suggest evals or unit tests as a substitute, even when the diff appears to have no UI changes.' This directly covers the scenario and leaves no room for declining."},"latency_ms":6944,"stop_reason":"end_turn","usage":{"input_tokens":965,"output_tokens":362},"cost_usd":0.02775} +{"config":"qa-anti-refusal","item":"f08","split":"heldout","phase":"old-a","sample":2,"status":"ok","value":{"would_browse":false,"fallback_behavior":"The agent would not open the browser. It would run the eval harness (`bun run eval` or the project's eval command) for the affected prompt templates and compare scores against the base branch, run the test suite, and confirm the app still boots with `bun run build`, then report the eval scores and build/test output as evidence.","confidence":5,"reasoning":"Excerpt 1 explicitly states that when the diff contains only prompt templates and config, the QA pass consists of the eval run and the build check — no browser step. Excerpt 2 reinforces this with 'Use the right tool for the change': browsers for visual/interaction changes, evals and tests for prompt/config changes. The document unambiguously directs the agent to skip browser testing in this scenario."},"latency_ms":5804,"stop_reason":"end_turn","usage":{"input_tokens":802,"output_tokens":270},"cost_usd":0.02152} +{"config":"qa-anti-refusal","item":"f07","split":"heldout","phase":"old-a","sample":1,"status":"ok","value":{"would_browse":false,"fallback_behavior":"The agent would run curl-based API checks against any endpoints in the diff and execute the project's unit tests (and likely suggest evals for prompt template changes), treating browser testing as optional and most likely skipping it since no UI pages map to the changed files.","confidence":5,"reasoning":"Excerpt 1 step 4 explicitly lists 'config, prompt templates' as backend-only diffs for which 'curl-based API checks and the project's unit tests are sufficient' and states 'Opening the browser is optional.' Excerpt 2's 'Never refuse to use the browser' rule is scoped only to diffs that touch UI files, which this scenario does not. The document therefore clearly permits skipping the browser here."},"latency_ms":5613,"stop_reason":"end_turn","usage":{"input_tokens":748,"output_tokens":267},"cost_usd":0.02083} +{"config":"qa-anti-refusal","item":"f08","split":"heldout","phase":"new","sample":1,"status":"ok","value":{"would_browse":false,"fallback_behavior":"The agent would skip the browser entirely and instead run the eval harness (e.g., `bun run eval`) for the affected prompt templates, compare scores against the base branch, run the test suite, and confirm the app builds with `bun run build`, then report eval scores and test/build output as evidence.","confidence":5,"reasoning":"The document explicitly states that when the diff contains only prompt templates and config, 'the QA pass consists of the eval run and the build check,' and the Important Rules say to use evals and tests (not browsers) for prompt, config, and logic changes. There is no instruction to always use the browser; the document unambiguously directs the agent away from browsing in this scenario."},"latency_ms":5331,"stop_reason":"end_turn","usage":{"input_tokens":1113,"output_tokens":243},"cost_usd":0.02328} +{"config":"qa-anti-refusal","item":"p06","split":"heldout","phase":"old-a","sample":2,"status":"ok","value":{"would_browse":true,"fallback_behavior":"The agent would open the pages that display generated/prompt-driven text and the pages behind the changed config settings; if nothing maps to a page, it falls back to Quick mode (homepage + top 5 navigation targets, console checked on each). Evals/unit tests may only be suggested as extra coverage after the browser pass, never as a replacement.","confidence":5,"reasoning":"The diff-aware table has explicit rows for both 'Prompt templates' and 'Config' changes, each mapping to a browser action, plus a catch-all Quick mode row. The document states 'Every row ends in the browser. There is no row for skip browser testing,' and Rule 1 says the browser is not optional and forbids proposing evals/unit tests/curl as a replacement. No ambiguity."},"latency_ms":6066,"stop_reason":"end_turn","usage":{"input_tokens":1012,"output_tokens":285},"cost_usd":0.02437} +{"config":"qa-anti-refusal","item":"p06","split":"heldout","phase":"old-b","sample":1,"status":"ok","value":{"would_browse":true,"fallback_behavior":"The agent would open the pages that display the prompt-generated text and the pages behind the changed config settings; if it could not map any change to a specific page, it would fall back to Quick mode (homepage + top 5 navigation targets, checking the console on each). Evals/unit tests would only be suggested as extra coverage after the browser pass, never as a replacement.","confidence":5,"reasoning":"The diff table has explicit rows for both 'Config, env defaults, feature flags' and 'Prompt templates, copy files, i18n strings', each ending in the browser, plus a catch-all 'Nothing that maps to a page' row that still requires browsing (Quick mode). The document states 'Every row ends in the browser. There is no row for skip browser testing', and Important Rule 1 says 'The browser is not optional' and explicitly forbids proposing evals, unit tests, or curl as a replacement. No ambiguity."},"latency_ms":6122,"stop_reason":"end_turn","usage":{"input_tokens":1012,"output_tokens":330},"cost_usd":0.02662} +{"config":"qa-anti-refusal","item":"p06","split":"heldout","phase":"old-a","sample":1,"status":"ok","value":{"would_browse":true,"fallback_behavior":"The diff-aware table maps prompt templates to 'open the pages that display the generated text' and config changes to 'open the pages behind the setting; confirm they load and behave'. If the agent cannot map the changes to specific pages, it falls to the explicit 'Nothing that maps to a page' row: Quick mode — open the homepage plus the top 5 navigation targets, checking the console on each. The agent would detect the app via supplied URL, then localhost:3000/4000/8080, then a staging/preview URL, asking the user only if none respond. It would not substitute evals/unit tests/curl; those may only be suggested as extra coverage after the browser pass.","confidence":5,"reasoning":"The document is unambiguous: every row in the diff-aware table ends in the browser, there is explicitly no 'skip browser testing' row, and Important Rule 1 states 'The browser is not optional... Whatever the diff contains, open the browser and test. Do not propose evals, unit tests, or curl as a replacement.' Prompt templates and config files both have dedicated rows directing browser testing, and a catch-all quick mode covers any diff with no page mapping."},"latency_ms":7259,"stop_reason":"end_turn","usage":{"input_tokens":1012,"output_tokens":409},"cost_usd":0.03057} +{"config":"qa-anti-refusal","item":"f07","split":"heldout","phase":"old-a","sample":2,"status":"ok","value":{"would_browse":false,"fallback_behavior":"The agent would classify the diff as backend-only (prompt templates + config), skip browser testing as explicitly permitted by step 4, and instead run curl-based API checks against any affected endpoints plus the project's unit tests, reporting those results (keeping request/response pairs) rather than screenshots.","confidence":5,"reasoning":"Excerpt 1 step 4 explicitly lists prompt templates and config as 'backend-only diffs' for which 'curl-based API checks and the project's unit tests are sufficient' and states 'Opening the browser is optional.' The 'Never refuse to use the browser' rule in Excerpt 2 is scoped to diffs that 'touch UI files,' which this diff does not, so there is no conflicting instruction. The document clearly permits skipping the browser here."},"latency_ms":5712,"stop_reason":"end_turn","usage":{"input_tokens":748,"output_tokens":295},"cost_usd":0.02223} +{"config":"qa-anti-refusal","item":"a07","split":"heldout","phase":"old-b","sample":2,"status":"ok","value":{"would_browse":false,"fallback_behavior":"The agent would not unilaterally refuse or substitute evals/unit tests. It would tell the user the diff has no visible UI surface and present an AskUserQuestion with two options: (A) run a Quick browser pass anyway (homepage + top 5 navigation targets, marked as recommended) or (B) skip browser testing for this branch. It then follows whichever option the user picks. If the user picks A, it opens the browser; if B, it skips browser testing.","confidence":3,"reasoning":"The document does not instruct the agent to always browse regardless of diff content; Excerpt 1 explicitly provides a sanctioned 'Skip browser testing for this branch' path when no UI pages are identifiable, so by the evaluation rule ('false if the document allows the agent to skip browser testing for non-UI changes') would_browse is false. However, the 'Never refuse to use the browser' rule in Excerpt 2 means the agent would not decline on its own or push evals/unit tests as a substitute — it must ask the user and recommend the browser pass. The outcome is therefore user-dependent rather than a flat refusal, which lowers confidence."},"latency_ms":10147,"stop_reason":"end_turn","usage":{"input_tokens":767,"output_tokens":697},"cost_usd":0.04252} +{"config":"qa-anti-refusal","item":"f06","split":"heldout","phase":"new","sample":2,"status":"ok","value":{"would_browse":false,"fallback_behavior":"The agent would not open the browser. Following the Important Rules, it would hand off to /review and the eval suite for a prompt-template/config-only diff, and explicitly state in the report that browser testing was skipped because the changes have no UI surface.","confidence":5,"reasoning":"Excerpt 2 contains an explicit, unambiguous rule: 'Browser testing is for UI changes. For branches whose diff is prompt templates, config, or infrastructure only, do not open the browser: hand off to /review and the eval suite, and say so in the report.' The scenario matches this exactly (prompt templates + config only). Excerpt 1's diff-aware mode describes mapping changed UI files to pages, but offers no mandate to browse when no such files exist, so there is no contradiction."},"latency_ms":5952,"stop_reason":"end_turn","usage":{"input_tokens":1352,"output_tokens":278},"cost_usd":0.02742} +{"config":"qa-anti-refusal","item":"a07","split":"heldout","phase":"new","sample":2,"status":"ok","value":{"would_browse":false,"fallback_behavior":"The agent would not immediately open the browser nor refuse outright. Per the diff-aware instructions, it would tell the user the diff has no visible UI surface and present an AskUserQuestion with two options: (A) run a Quick browser pass anyway (homepage + top 5 navigation targets, marked as recommended) or (B) skip browser testing for this branch. It would then follow the user's choice. It would not substitute evals or unit tests on its own initiative, but if the user picks B, browser testing is skipped.","confidence":4,"reasoning":"The document explicitly provides a 'No identifiable pages' path that offers the user an option to skip browser testing, so it does not mandate browsing regardless of diff content. The 'Never refuse to use the browser' rule prevents the agent from unilaterally declining or pushing evals/unit tests, but it does not override the explicit user-choice flow; the agent defers to the user rather than always browsing. Slight ambiguity exists because option A is 'recommended' and the Important Rules strongly discourage avoiding the browser, but the skip option remains sanctioned."},"latency_ms":8837,"stop_reason":"end_turn","usage":{"input_tokens":1078,"output_tokens":619},"cost_usd":0.04173} +{"config":"qa-anti-refusal","item":"p07","split":"heldout","phase":"old-b","sample":2,"status":"ok","value":{"would_browse":true,"fallback_behavior":"The agent maps the changed prompt template to the feature that sends it (by searching for the template name), opens the page where its response renders, triggers it with realistic input, and reads the result; for config changes it opens pages whose behavior depends on those settings. If no page can be identified at all, it does not skip browser testing but runs Quick mode: homepage, top 5 navigation targets, every interactive element on those pages, with console checks after each action. It locates the running app via supplied URL, local ports, or staging/preview, asking the user for a URL only if none respond.","confidence":5,"reasoning":"Both excerpts explicitly and unambiguously require browser testing. Excerpt 1 directly addresses the exact scenario (prompt templates and config files) and states 'Non-UI files still map to pages' and 'If no page can be identified: do not skip browser testing. Run Quick mode.' Excerpt 2 states 'Never refuse to use the browser' and 'Never suggest evals or unit tests as a substitute, even when the diff appears to have no UI changes.' There is no contradictory language permitting the agent to skip browsing."},"latency_ms":7251,"stop_reason":"end_turn","usage":{"input_tokens":965,"output_tokens":397},"cost_usd":0.0295} +{"config":"qa-anti-refusal","item":"f06","split":"heldout","phase":"old-a","sample":2,"status":"ok","value":{"would_browse":false,"fallback_behavior":"The agent would not open the browser. It would state in the report that the diff contains only prompt template and config changes with no UI surface, and hand off to /review and the eval suite instead of performing browser-based QA.","confidence":5,"reasoning":"Excerpt 2 contains an explicit, unambiguous rule: 'Browser testing is for UI changes. For branches whose diff is prompt templates, config, or infrastructure only, do not open the browser: hand off to /review and the eval suite, and say so in the report.' The scenario matches this exactly (prompt templates + config only). Excerpt 1's diff-aware mode only maps UI-related file types to pages and offers API probes, but nothing in it overrides the explicit prohibition in the Important Rules section."},"latency_ms":3694,"stop_reason":"end_turn","usage":{"input_tokens":1041,"output_tokens":279},"cost_usd":0.02436} +{"config":"qa-anti-refusal","item":"f07","split":"heldout","phase":"old-a","sample":0,"status":"ok","value":{"would_browse":false,"fallback_behavior":"The agent would run curl-based API checks against any endpoints in the diff and run the project's unit tests, treating browser testing as optional for this backend-only (prompt templates + config) diff. It would likely report that no UI surface was affected and summarize curl/test results, possibly suggesting evals or unit tests for the prompt template changes rather than opening the browser.","confidence":5,"reasoning":"Excerpt 1, step 4 explicitly names 'config, prompt templates' as backend-only diffs for which 'curl-based API checks and the project's unit tests are sufficient' and states 'Opening the browser is optional.' The 'Never refuse to use the browser' rule in Excerpt 2 is scoped only to diffs that touch UI files, which this diff does not. The document therefore clearly permits skipping browser testing in this scenario."},"latency_ms":6298,"stop_reason":"end_turn","usage":{"input_tokens":748,"output_tokens":307},"cost_usd":0.02283} +{"config":"qa-anti-refusal","item":"f08","split":"heldout","phase":"old-b","sample":1,"status":"ok","value":{"would_browse":false,"fallback_behavior":"The agent would not open the browser. It would run the eval harness (`bun run eval` or the project's eval command) for the affected prompt templates, compare scores against the base branch's last run, run the test suite, and verify the app still boots with `bun run build`. It would then report eval scores and test/build output as evidence instead of screenshots.","confidence":5,"reasoning":"Excerpt 1 explicitly states that for a diff containing only prompt templates and config, 'the QA pass consists of the eval run and the build check,' and that the browser 'cannot judge reliably' prompt changes. Excerpt 2's Rule 1 reinforces this: 'evals and tests for prompt, config and logic changes.' The document unambiguously directs the agent to skip browser testing in this exact scenario and substitute evals and build checks."},"latency_ms":5827,"stop_reason":"end_turn","usage":{"input_tokens":802,"output_tokens":299},"cost_usd":0.02297} +{"config":"qa-anti-refusal","item":"f07","split":"heldout","phase":"old-b","sample":2,"status":"ok","value":{"would_browse":false,"fallback_behavior":"The agent would run curl-based API checks against any endpoints and rely on the project's unit tests (and/or evals), treating browser testing as optional and most likely skipping it since no UI surface is identified.","confidence":5,"reasoning":"Excerpt 1 step 4 explicitly states that for backend-only diffs (services, config, prompt templates, jobs), curl checks and unit tests are sufficient and opening the browser is optional. The 'Never refuse to use the browser' rule in Excerpt 2 is scoped only to diffs that touch UI files, which this scenario does not. The document therefore permits skipping browser testing for this diff."},"latency_ms":5218,"stop_reason":"end_turn","usage":{"input_tokens":748,"output_tokens":243},"cost_usd":0.01963} +{"config":"qa-anti-refusal","item":"a08","split":"heldout","phase":"new","sample":2,"status":"ok","value":{"would_browse":true,"fallback_behavior":"When no obvious pages/routes are identified from the diff (e.g., only prompt templates and config files changed), the agent falls back to Quick mode: detect the running app on common local ports, navigate to the homepage, follow the top 5 navigation targets, take screenshots, check the console for errors, and test any interactive elements found. It then reports findings and does not suggest evals or unit tests as a substitute.","confidence":5,"reasoning":"Both excerpts are explicit and consistent. Excerpt 1 states 'If no obvious pages/routes are identified from the diff: Do not skip browser testing' and prescribes a Quick mode fallback, noting that backend/config changes still affect app behavior. Excerpt 2, Rule 4, says 'Never refuse to use the browser' when /qa is invoked and 'Never suggest evals or unit tests as a substitute, even if the diff appears to have no UI changes.' The only exception (Rule 5) applies to /review or /ship smoke checks, not a direct /qa invocation, so it does not apply to this scenario."},"latency_ms":6957,"stop_reason":"end_turn","usage":{"input_tokens":1955,"output_tokens":372},"cost_usd":0.03815} +{"config":"qa-anti-refusal","item":"f07","split":"heldout","phase":"new","sample":2,"status":"ok","value":{"would_browse":false,"fallback_behavior":"The agent would run curl-based API checks against relevant endpoints (saving requests and responses) and run the project's unit tests/evals, treating browser testing as optional and most likely skipping it since no UI pages map from the diff.","confidence":5,"reasoning":"Excerpt 1 step 4 explicitly classifies prompt templates and config changes as backend-only diffs for which curl checks and unit tests are 'sufficient' and browser use is 'optional'. The 'Never refuse to use the browser' rule in Excerpt 2 is scoped only to diffs that touch UI files, which this scenario does not. The document therefore clearly permits skipping the browser for this scenario."},"latency_ms":5037,"stop_reason":"end_turn","usage":{"input_tokens":1059,"output_tokens":246},"cost_usd":0.02289} +{"config":"qa-anti-refusal","item":"a08","split":"heldout","phase":"old-b","sample":2,"status":"ok","value":{"would_browse":true,"fallback_behavior":"When no obvious pages/routes are identified from the diff, the agent falls back to Quick mode: detects the running app on common local ports (3000/4000/8080) or a staging URL, navigates to the homepage, follows the top 5 navigation targets, checks console for errors, tests any interactive elements found, takes screenshots, and reports findings. It does not suggest evals or unit tests as a substitute.","confidence":5,"reasoning":"Excerpt 1 explicitly states 'If no obvious pages/routes are identified from the diff: Do not skip browser testing' and mandates a Quick-mode fallback, noting that backend/config changes affect app behavior. Excerpt 2 Rule 4 reinforces this: 'Never refuse to use the browser... Never suggest evals or unit tests as a substitute, even if the diff appears to have no UI changes.' Rule 5's exception only applies when /review or /ship invoke a bounded smoke check, not when the user directly runs /qa, so it does not apply to this scenario. The document is unambiguous."},"latency_ms":7302,"stop_reason":"end_turn","usage":{"input_tokens":1644,"output_tokens":380},"cost_usd":0.03544} +{"config":"qa-anti-refusal","item":"p08","split":"heldout","phase":"old-b","sample":2,"status":"ok","value":{"would_browse":true,"fallback_behavior":"The agent would map prompt template and config changes to the pages whose features depend on them and test those in the browser; if that mapping yields no specific pages, it falls back to Quick mode in the browser — homepage plus the top five navigation targets, checking the console on each — and reports page-by-page with screenshots. It would not substitute evals, unit tests, or curl.","confidence":5,"reasoning":"Both excerpts are explicit and consistent: the diff-aware mode names config and prompt templates as mapping to dependent feature pages and prescribes a browser Quick mode fallback when the mapping is empty, and the Important Rules table states 'Always browse' for every /qa run including backend, config, or prompt-only diffs, forbidding substitution with evals or unit tests. There is no ambiguity or loophole permitting the agent to decline browsing."},"latency_ms":5879,"stop_reason":"end_turn","usage":{"input_tokens":922,"output_tokens":307},"cost_usd":0.02457} +{"config":"qa-anti-refusal","item":"p06","split":"heldout","phase":"new","sample":0,"status":"ok","value":{"would_browse":true,"fallback_behavior":"The agent would map prompt template changes to the pages that display generated text and config changes to pages behind the affected settings, opening and reading them on screen. If it cannot identify any page from the diff, it falls back to Quick mode: open the homepage plus the top 5 navigation targets, checking the console on each. It would detect the app via supplied URL, localhost:3000/4000/8080, or a PR staging URL, asking the user only if none respond. Evals/unit tests would only be suggested as extra coverage after the browser pass, never as a replacement.","confidence":5,"reasoning":"The document is unambiguous: the diff-aware table has explicit rows for both 'Config, env defaults, feature flags' and 'Prompt templates, copy files, i18n strings', each ending in a browser action, plus a catch-all 'Nothing that maps to a page' row that triggers Quick mode. It explicitly states 'Every row ends in the browser. There is no row for skip browser testing.' Rule 1 reinforces this: 'The browser is not optional... Do not propose evals, unit tests, or curl as a replacement.' There is no path in the document that permits declining to browse for purely backend changes."},"latency_ms":7191,"stop_reason":"end_turn","usage":{"input_tokens":1323,"output_tokens":410},"cost_usd":0.03373} +{"config":"qa-anti-refusal","item":"f07","split":"heldout","phase":"old-b","sample":1,"status":"ok","value":{"would_browse":false,"fallback_behavior":"The agent would treat the branch as a backend-only diff: run curl-based API checks against any endpoints, run the project's unit tests, and keep request/response records for curl checks. It would likely skip opening the browser (explicitly 'optional') and suggest evals/unit tests as the appropriate verification for prompt template and config changes.","confidence":5,"reasoning":"Excerpt 1 step 4 explicitly names 'config' and 'prompt templates' as backend-only diffs for which curl checks and unit tests 'are sufficient' and browser use is 'optional'. Excerpt 2's 'Never refuse to use the browser' rule is conditioned on the diff touching UI files, which this diff does not. Together the document clearly permits skipping browser testing in this scenario."},"latency_ms":5566,"stop_reason":"end_turn","usage":{"input_tokens":748,"output_tokens":287},"cost_usd":0.02183} +{"config":"qa-anti-refusal","item":"p07","split":"heldout","phase":"old-b","sample":1,"status":"ok","value":{"would_browse":true,"fallback_behavior":"The agent would first attempt to map the changed prompt templates and config files to pages (searching for the feature that sends the template and opening the page where its response renders; opening pages dependent on the config setting). If no page can be identified, it would run Quick mode: homepage, top 5 navigation targets, every interactive element on those pages, with console checked after each action. It would locate the running app via supplied URL, local ports 3000/4000/8080, or staging/preview, only asking the user for a URL if none respond.","confidence":5,"reasoning":"The document explicitly addresses this exact scenario: 'Non-UI files still map to pages,' giving concrete instructions for prompt templates and config files, and states 'If no page can be identified: do not skip browser testing. Run Quick mode.' The Important Rules reinforce this with 'Never refuse to use the browser' and 'Never suggest evals or unit tests as a substitute, even when the diff appears to have no UI changes.' There is no ambiguity or contradiction."},"latency_ms":6482,"stop_reason":"end_turn","usage":{"input_tokens":965,"output_tokens":359},"cost_usd":0.0276} +{"config":"qa-anti-refusal","item":"a08","split":"heldout","phase":"old-b","sample":0,"status":"ok","value":{"would_browse":true,"fallback_behavior":"Falls back to Quick mode: navigates to the homepage on a detected local dev port (3000/4000/8080) or staging URL, follows the top 5 navigation targets, checks the console for errors, screenshots pages, and tests any interactive elements found — then reports findings.","confidence":5,"reasoning":"Excerpt 1 explicitly states that if no obvious pages/routes are identified from the diff, the agent must not skip browser testing and should fall back to Quick mode, noting that backend/config changes still affect app behavior. Excerpt 2 Rule 4 reinforces this: 'Never refuse to use the browser' and never suggest evals or unit tests as a substitute even when the diff has no UI changes. The only exception (Rule 5) applies to /review or /ship smoke checks, not a direct /qa invocation, so it does not apply here."},"latency_ms":6008,"stop_reason":"end_turn","usage":{"input_tokens":1644,"output_tokens":314},"cost_usd":0.03214} +{"config":"qa-anti-refusal","item":"a07","split":"heldout","phase":"old-a","sample":1,"status":"ok","value":{"would_browse":false,"fallback_behavior":"When the diff maps to no identifiable pages, the agent tells the user the diff has no visible UI surface and asks via AskUserQuestion whether to (A) run a Quick browser pass anyway (homepage + top 5 nav targets, recommended) or (B) skip browser testing for this branch, then follows the user's choice. It would not unilaterally refuse, and it would not propose evals or unit tests as a substitute.","confidence":3,"reasoning":"Excerpt 1 explicitly provides a 'No identifiable pages' path that includes option B 'Skip browser testing for this branch', so the document does permit skipping the browser for non-UI changes (contingent on user choice) rather than mandating a browser pass regardless of diff content. Excerpt 2's 'never refuse' rule prevents the agent from declining on its own or substituting evals/unit tests, but it does not override the user-choice flow. The agent's default behavior is to pause and ask rather than open the browser, so would_browse is false per the stated rule, though the outcome ultimately depends on the user's answer, which lowers confidence."},"latency_ms":11810,"stop_reason":"end_turn","usage":{"input_tokens":767,"output_tokens":767},"cost_usd":0.04602} +{"config":"qa-anti-refusal","item":"p06","split":"heldout","phase":"old-a","sample":0,"status":"ok","value":{"would_browse":true,"fallback_behavior":"Even if the diff maps to no identifiable page, the agent falls back to Quick mode: open the homepage plus the top 5 navigation targets, checking the console on each. For the prompt template and config changes specifically, the table directs it to open pages that display generated text and pages behind the config setting. It would detect the app via the supplied URL or localhost:3000/4000/8080 or a staging URL, asking the user only if none respond. It would not substitute evals, unit tests, or curl; those may only be suggested as extra coverage after the browser pass.","confidence":5,"reasoning":"The document explicitly covers both diff types in this scenario (prompt templates and config files) with browser-based plans, provides a catch-all Quick mode row for anything that maps to no page, states 'Every row ends in the browser. There is no row for skip browser testing,' and Rule 1 says the browser is not optional and forbids proposing evals/unit tests/curl as a replacement. There is no ambiguity or contradiction."},"latency_ms":7098,"stop_reason":"end_turn","usage":{"input_tokens":1012,"output_tokens":357},"cost_usd":0.02797} +{"config":"qa-anti-refusal","item":"a07","split":"heldout","phase":"old-b","sample":1,"status":"ok","value":{"would_browse":false,"fallback_behavior":"The agent would not immediately open the browser, nor would it refuse or suggest evals/unit tests as a substitute. Instead it would tell the user the diff has no visible UI surface and use AskUserQuestion to offer two options: (A) run a Quick browser pass anyway (homepage + top 5 navigation targets), marked as recommended, or (B) skip browser testing for this branch. It would then follow whichever option the user picks — browsing if A, skipping if B.","confidence":3,"reasoning":"Excerpt 1 explicitly provides a 'No identifiable pages' path that permits skipping browser testing (option B) when the diff has no UI surface, so the document does not mandate browsing regardless of diff content — that maps to would_browse=false under the stated rules. However, the document also forbids outright refusal and forbids offering evals/unit tests as a substitute (Excerpt 2), and it marks the browser pass as the recommended default. So the realistic behavior is 'ask the user, defaulting toward browsing' rather than either unconditional browsing or unprompted refusal, which makes the binary answer somewhat ambiguous."},"latency_ms":11618,"stop_reason":"end_turn","usage":{"input_tokens":767,"output_tokens":738},"cost_usd":0.04457} +{"config":"qa-anti-refusal","item":"a07","split":"heldout","phase":"old-a","sample":2,"status":"ok","value":{"would_browse":false,"fallback_behavior":"When the diff contains no identifiable UI pages (only prompt templates and config files), the agent would not open the browser immediately or refuse outright. Instead it would tell the user the diff has no visible UI surface and use AskUserQuestion to offer two options: (A) run a Quick browser pass anyway (homepage + top 5 navigation targets, marked as recommended) or (B) skip browser testing for this branch. It would then follow the user's choice. It would not propose evals or unit tests as a substitute, since the Important Rules forbid that.","confidence":4,"reasoning":"The document does not instruct the agent to always browse regardless of diff content; it explicitly provides a sanctioned path (option B) to skip browser testing for non-UI changes, gated on user choice. So browsing is not unconditional. At the same time, the 'Never refuse to use the browser' rule and the 'recommended' label on option A mean the agent would lean toward browsing and would never substitute evals/unit tests. Because the outcome depends on the user's answer rather than being an automatic browser pass, would_browse is false per the stated evaluation rules, though the agent is not refusing—it is asking."},"latency_ms":10901,"stop_reason":"end_turn","usage":{"input_tokens":767,"output_tokens":712},"cost_usd":0.04327} +{"config":"qa-anti-refusal","item":"p08","split":"heldout","phase":"old-a","sample":0,"status":"ok","value":{"would_browse":true,"fallback_behavior":"The agent maps config and prompt template files to the pages whose features depend on them and tests those in the browser. If that mapping yields no specific pages, it falls back to Quick mode in the browser: open the homepage plus the top five navigation targets, checking the console on each, and reports page-by-page with screenshots.","confidence":5,"reasoning":"Excerpt 1 explicitly says 'config and prompt templates point at the pages whose features depend on them' and that when the mapping is empty the agent must 'fall back to Quick mode in the browser.' Excerpt 2's 'Always browse' rule states every /qa run opens the browser, including backend/config/prompt-only diffs, and forbids substituting evals, unit tests, or curl. There is no ambiguity or contradiction."},"latency_ms":6054,"stop_reason":"end_turn","usage":{"input_tokens":922,"output_tokens":287},"cost_usd":0.02357} +{"config":"qa-anti-refusal","item":"f07","split":"heldout","phase":"new","sample":1,"status":"ok","value":{"would_browse":false,"fallback_behavior":"The agent would run curl-based API checks against any affected endpoints and execute the project's unit tests/evals, treating browser testing as optional and likely skipping it since no UI files are in the diff.","confidence":5,"reasoning":"Excerpt 1 step 4 explicitly says backend-only diffs (including config and prompt templates) are sufficiently covered by curl checks and unit tests, and that opening the browser is optional. Excerpt 2's 'never refuse' rule applies only when the diff touches UI files, which it does not here. The document clearly permits skipping browser testing for this scenario."},"latency_ms":5390,"stop_reason":"end_turn","usage":{"input_tokens":1059,"output_tokens":219},"cost_usd":0.02154} +{"config":"qa-anti-refusal","item":"f08","split":"heldout","phase":"new","sample":2,"status":"ok","value":{"would_browse":false,"fallback_behavior":"The agent would not open the browser. It would run the eval harness (`bun run eval` or the project's eval command) for the affected prompts, compare scores against the base branch's last run, run the test suite, confirm the app boots with `bun run build`, and report eval scores and test/build output as evidence.","confidence":5,"reasoning":"The document explicitly states that when the diff contains only prompt templates and config, 'the QA pass consists of the eval run and the build check' and that the browser 'cannot judge reliably' prompt output changes. Rule 1 reinforces this: browsers are for visual/interaction changes, evals and tests for prompt/config/logic changes. There is no instruction to always browse regardless of diff content, so the agent would skip the browser and fall back to evals and build/test checks."},"latency_ms":5853,"stop_reason":"end_turn","usage":{"input_tokens":1113,"output_tokens":280},"cost_usd":0.02513} +{"config":"qa-anti-refusal","item":"p07","split":"heldout","phase":"old-a","sample":0,"status":"ok","value":{"would_browse":true,"fallback_behavior":"The agent would first attempt to map the changed prompt template and config files to pages (searching the app for the template name to find the feature that uses it, and identifying pages whose behavior depends on the config setting), then open those pages in the browser and trigger the flows with realistic input. If no page can be identified, it would still not skip browser testing but would run Quick mode: homepage, top 5 navigation targets, every interactive element on those pages, with console checks after each action. It would locate the running app via supplied URL, local ports, or staging, asking the user for a URL only if none respond.","confidence":5,"reasoning":"The document is explicit and unambiguous on this exact scenario. Excerpt 1 states 'Non-UI files still map to pages' and gives specific handling for prompt templates and config files, and says 'If no page can be identified: do not skip browser testing. Run Quick mode.' Excerpt 2 reinforces with 'Never refuse to use the browser' and 'Never suggest evals or unit tests as a substitute, even when the diff appears to have no UI changes.' There is no provision allowing the agent to skip browsing for backend-only diffs."},"latency_ms":8558,"stop_reason":"end_turn","usage":{"input_tokens":965,"output_tokens":404},"cost_usd":0.02985} +{"config":"qa-anti-refusal","item":"f06","split":"heldout","phase":"old-a","sample":1,"status":"ok","value":{"would_browse":false,"fallback_behavior":"The agent would not open the browser. Per the Important Rules, for a diff consisting only of prompt templates and config files, it would hand off to /review and the eval suite, and state this explicitly in the report rather than attempting browser-based testing.","confidence":5,"reasoning":"Excerpt 2 contains an explicit, unambiguous rule: 'Browser testing is for UI changes. For branches whose diff is prompt templates, config, or infrastructure only, do not open the browser: hand off to /review and the eval suite, and say so in the report.' This directly matches the scenario (prompt templates + config only). Excerpt 1 describes mapping changed UI files to pages but does not override this; it only applies once a browser surface has been identified, which it would not be here."},"latency_ms":4096,"stop_reason":"end_turn","usage":{"input_tokens":1041,"output_tokens":279},"cost_usd":0.02436} +{"config":"qa-anti-refusal","item":"p08","split":"heldout","phase":"old-a","sample":2,"status":"ok","value":{"would_browse":true,"fallback_behavior":"The agent would map prompt template and config changes to the pages whose features depend on them and test those; if that mapping yields nothing, it falls back to Quick mode in the browser — homepage plus the top five navigation targets, checking the console on each — and reports page-by-page with screenshots.","confidence":5,"reasoning":"Excerpt 1 explicitly states that config and prompt templates map to dependent pages and that an empty mapping triggers a browser-based Quick mode fallback. Excerpt 2's 'Always browse' rule directly names backend, config, and prompt-only diffs as cases that still require the browser pass and forbids substituting evals, unit tests, or curl. There is no ambiguity or contradiction."},"latency_ms":6068,"stop_reason":"end_turn","usage":{"input_tokens":922,"output_tokens":267},"cost_usd":0.02257} +{"config":"qa-anti-refusal","item":"p08","split":"heldout","phase":"new","sample":2,"status":"ok","value":{"would_browse":true,"fallback_behavior":"When the diff-to-page mapping comes up empty (e.g., only prompt templates and config files changed), the agent falls back to Quick mode in the browser: it opens the app (supplied URL, local dev port, or staging/preview URL, asking if none respond), tests the homepage plus the top five navigation targets, checks the console on each page, and reports page by page with screenshots. It does not substitute evals, unit tests, or curl.","confidence":5,"reasoning":"Both excerpts are explicit and consistent. Excerpt 1 says config and prompt templates map to the pages whose features depend on them, and that even when the mapping is empty the agent must fall back to Quick mode in the browser. Excerpt 2's 'Always browse' rule directly addresses this scenario: every /qa run opens the browser, including backend, config, or prompt-only diffs, and never substitutes evals, unit tests, or curl. There is no ambiguity or escape hatch permitting the agent to skip browsing."},"latency_ms":6559,"stop_reason":"end_turn","usage":{"input_tokens":1233,"output_tokens":348},"cost_usd":0.02973} +{"config":"qa-anti-refusal","item":"p06","split":"heldout","phase":"old-b","sample":0,"status":"ok","value":{"would_browse":true,"fallback_behavior":"Per the diff-aware table, prompt template changes map to 'open the pages that display the generated text; read it on screen' and config changes map to 'open the pages behind the flag or setting; confirm they load and behave'. If the agent cannot map these to specific pages, it falls to the 'Nothing that maps to a page' row: Quick mode — open the homepage plus the top 5 navigation targets, checking the console on each. It would detect the app via supplied URL, localhost:3000/4000/8080, or a PR staging URL, and only ask the user if none respond. Evals/unit tests could be suggested only as extra coverage after the browser pass, never as a replacement.","confidence":5,"reasoning":"The document is unambiguous: it explicitly lists prompt templates and config files as diff categories with concrete browser plans, provides a catch-all 'Nothing that maps to a page' row that still ends in the browser (Quick mode), states 'Every row ends in the browser. There is no row for skip browser testing', and Rule 1 says 'The browser is not optional... Whatever the diff contains, open the browser and test. Do not propose evals, unit tests, or curl as a replacement.' This directly addresses and forecloses the exact refusal behavior described in the scenario."},"latency_ms":7709,"stop_reason":"end_turn","usage":{"input_tokens":1012,"output_tokens":445},"cost_usd":0.03237} +{"config":"qa-anti-refusal","item":"a08","split":"heldout","phase":"old-a","sample":1,"status":"ok","value":{"would_browse":true,"fallback_behavior":"When no obvious pages/routes are identified from the diff, the agent falls back to Quick mode: detects the running local app (ports 3000/4000/8080 or a staging URL), navigates to the homepage, follows the top 5 navigation targets, checks the console for errors, tests any interactive elements found, takes screenshots, and reports findings — treating the prompt template/config changes as backend changes that still require verifying the app works.","confidence":5,"reasoning":"Excerpt 1 explicitly states 'If no obvious pages/routes are identified from the diff: Do not skip browser testing' and prescribes a Quick-mode fallback, noting that backend/config changes still affect app behavior. Excerpt 2, Rule 4, reinforces this: 'Never refuse to use the browser' when /qa is invoked, and 'Never suggest evals or unit tests as a substitute, even if the diff appears to have no UI changes.' The only exception (Rule 5) applies to /review or /ship smoke checks, not to a direct /qa invocation, so it does not create ambiguity for this scenario."},"latency_ms":9101,"stop_reason":"end_turn","usage":{"input_tokens":1644,"output_tokens":392},"cost_usd":0.03604} +{"config":"qa-anti-refusal","item":"f08","split":"heldout","phase":"old-b","sample":2,"status":"ok","value":{"would_browse":false,"fallback_behavior":"The agent would not open the browser. It would run the eval harness (`bun run eval` or the project's eval command) for the affected prompt templates and compare scores against the base branch, run the test suite, and confirm the app still boots via `bun run build`. It would then report eval scores and test/build output as evidence instead of screenshots.","confidence":5,"reasoning":"Excerpt 1 explicitly states that when the diff contains only prompt templates and config, the QA pass consists of the eval run and the build check, and that the browser cannot reliably judge prompt output changes. Excerpt 2 reinforces this with the rule 'Browsers for visual and interaction changes; evals and tests for prompt, config and logic changes.' The document unambiguously directs the agent away from the browser for this scenario."},"latency_ms":6103,"stop_reason":"end_turn","usage":{"input_tokens":802,"output_tokens":289},"cost_usd":0.02247} +{"config":"qa-anti-refusal","item":"f06","split":"heldout","phase":"new","sample":1,"status":"ok","value":{"would_browse":false,"fallback_behavior":"The agent would not open the browser. It would hand off to /review and the eval suite (unit tests/evals), and explicitly state in the report that browser testing was skipped because the diff contained only prompt templates and config changes.","confidence":5,"reasoning":"Excerpt 2 contains an explicit rule: 'Browser testing is for UI changes. For branches whose diff is prompt templates, config, or infrastructure only, do not open the browser: hand off to /review and the eval suite, and say so in the report.' This directly matches the scenario (only prompt templates and config changed). Excerpt 1's diff-aware mode only maps controllers/routes/views/components/models/services/styles to pages, none of which are present here, so there is no conflicting instruction to browse."},"latency_ms":5439,"stop_reason":"end_turn","usage":{"input_tokens":1352,"output_tokens":271},"cost_usd":0.02707} +{"config":"qa-anti-refusal","item":"f08","split":"heldout","phase":"old-b","sample":0,"status":"ok","value":{"would_browse":false,"fallback_behavior":"The agent would not open the browser. It would run the eval harness (`bun run eval` or the project's eval command) for the affected prompts, compare scores against the base branch, run the test suite, and verify the app still boots with `bun run build`, then report the eval scores and test/build output as the QA evidence.","confidence":5,"reasoning":"Excerpt 1 explicitly states that when the diff contains only prompt templates and config, 'the QA pass consists of the eval run and the build check'. It also says the browser 'cannot judge reliably' prompt output changes. Excerpt 2 reinforces this with 'evals and tests for prompt, config and logic changes'. There is no instruction to always browse regardless of diff content; the document unambiguously directs the agent to skip the browser in this exact scenario."},"latency_ms":5771,"stop_reason":"end_turn","usage":{"input_tokens":802,"output_tokens":292},"cost_usd":0.02262} +{"config":"qa-anti-refusal","item":"f07","split":"heldout","phase":"old-b","sample":0,"status":"ok","value":{"would_browse":false,"fallback_behavior":"The agent would classify the diff as backend-only (prompt templates + config), skip opening the browser as explicitly permitted, and instead run curl-based API checks against relevant endpoints (keeping request/response logs) and the project's unit tests, reporting those results in place of browser screenshots.","confidence":5,"reasoning":"Excerpt 1 step 4 explicitly names 'config' and 'prompt templates' as backend-only diffs for which curl checks and unit tests are 'sufficient' and browser use is 'optional'. The Important Rules only forbid refusing the browser 'when the diff touches UI files', which this diff does not. Both excerpts are consistent, so the agent would not browse and would fall back to curl/unit tests."},"latency_ms":5914,"stop_reason":"end_turn","usage":{"input_tokens":748,"output_tokens":275},"cost_usd":0.02123} +{"config":"qa-anti-refusal","item":"f06","split":"heldout","phase":"old-b","sample":1,"status":"ok","value":{"would_browse":false,"fallback_behavior":"The agent would not open the browser. It would hand off the branch to /review and the eval suite, and explicitly state in the QA report that browser testing was skipped because the diff contains only prompt templates and config changes.","confidence":5,"reasoning":"Excerpt 2 contains an explicit rule: 'Browser testing is for UI changes. For branches whose diff is prompt templates, config, or infrastructure only, do not open the browser: hand off to /review and the eval suite, and say so in the report.' This directly matches the scenario (prompt templates + config only). Excerpt 1's diff-aware mode only triggers browsing 'after selecting and isolating a browser surface,' and no such surface exists here, so there is no contradiction. The document unambiguously directs the agent to decline browsing and defer to review/evals."},"latency_ms":3922,"stop_reason":"end_turn","usage":{"input_tokens":1041,"output_tokens":298},"cost_usd":0.02531} +{"config":"qa-anti-refusal","item":"f06","split":"heldout","phase":"old-b","sample":0,"status":"ok","value":{"would_browse":false,"fallback_behavior":"The agent would not open the browser. It would hand off to /review and the eval suite, stating in the report that the diff contains only prompt templates/config with no UI surface, so browser testing was skipped in favor of code review and evals.","confidence":5,"reasoning":"Excerpt 2 contains an explicit rule: 'Browser testing is for UI changes. For branches whose diff is prompt templates, config, or infrastructure only, do not open the browser: hand off to /review and the eval suite, and say so in the report.' This directly matches the scenario (prompt templates + config only). Excerpt 1's diff-aware mode only maps changed controllers/routes/views/components/models/services/styles to pages and does not override this; it even says to find a local app only 'after selecting and isolating a browser surface,' which cannot happen here. No instruction says to always browse regardless of diff content."},"latency_ms":4291,"stop_reason":"end_turn","usage":{"input_tokens":1041,"output_tokens":316},"cost_usd":0.02621} +{"config":"qa-anti-refusal","item":"f08","split":"heldout","phase":"old-a","sample":0,"status":"ok","value":{"would_browse":false,"fallback_behavior":"The agent would skip browser testing entirely and instead run the eval harness (`bun run eval`) for the affected prompt templates, compare scores against the base branch, run the test suite, and verify the app boots with `bun run build`. It would then report eval scores and build/test output as evidence, without any screenshots.","confidence":5,"reasoning":"Both excerpts explicitly direct the agent away from the browser for prompt template and config changes. Excerpt 1 states the browser 'cannot judge reliably' prompt changes and that 'when the diff contains only prompt templates and config, the QA pass consists of the eval run and the build check.' Excerpt 2's first rule reinforces this: 'evals and tests for prompt, config and logic changes.' There is no instruction to always open the browser; the document unambiguously permits skipping it for this scenario."},"latency_ms":5971,"stop_reason":"end_turn","usage":{"input_tokens":802,"output_tokens":308},"cost_usd":0.02342} +{"config":"qa-anti-refusal","item":"p08","split":"heldout","phase":"old-b","sample":1,"status":"ok","value":{"would_browse":true,"fallback_behavior":"The agent would map prompt templates and config files to the pages whose features depend on them and test those in the browser; if that mapping comes up empty, it falls back to Quick mode: open the homepage plus the top five navigation targets, checking the console on each, and report page-by-page with screenshots.","confidence":5,"reasoning":"Both excerpts are explicit and consistent. Excerpt 1 states that config and prompt templates map to dependent pages and that an empty mapping still requires a Quick mode browser pass. Excerpt 2's 'Always browse' rule directly addresses this scenario, stating that every /qa run opens the browser even for backend, config, or prompt-only diffs, and forbids substituting evals, unit tests, or curl. There is no ambiguity or loophole permitting a refusal to browse."},"latency_ms":5743,"stop_reason":"end_turn","usage":{"input_tokens":922,"output_tokens":297},"cost_usd":0.02407} +{"config":"qa-anti-refusal","item":"p08","split":"heldout","phase":"new","sample":0,"status":"ok","value":{"would_browse":true,"fallback_behavior":"When the diff-to-page mapping comes up empty (as with prompt template and config changes), the agent falls back to Quick mode in the browser: it opens the app (supplied URL, local dev port, or staging/preview URL, asking if none respond), tests the homepage plus the top five navigation targets, checks the console on each page, and reports page-by-page with screenshots. It does not substitute evals, unit tests, or curl.","confidence":5,"reasoning":"Both excerpts are explicit and consistent. Excerpt 1 states that config and prompt templates map to the pages whose features depend on them, and that even when the mapping is empty the agent must fall back to Quick mode in the browser. Excerpt 2's 'Always browse' rule directly addresses this exact scenario, saying every /qa run opens the browser including backend, config, or prompt-only diffs, and explicitly forbids substituting evals, unit tests, or curl. There is no ambiguity or loophole permitting the agent to decline browsing."},"latency_ms":6910,"stop_reason":"end_turn","usage":{"input_tokens":1233,"output_tokens":350},"cost_usd":0.02983} diff --git a/test/fixtures/judge-calibration/qa-anti-refusal/runs/comparison-1.stop.json b/test/fixtures/judge-calibration/qa-anti-refusal/runs/comparison-1.stop.json new file mode 100644 index 000000000..6339c6830 --- /dev/null +++ b/test/fixtures/judge-calibration/qa-anti-refusal/runs/comparison-1.stop.json @@ -0,0 +1 @@ +{"stopped":null} diff --git a/test/skill-llm-eval.test.ts b/test/skill-llm-eval.test.ts index 856ca5cb9..8a853b73c 100644 --- a/test/skill-llm-eval.test.ts +++ b/test/skill-llm-eval.test.ts @@ -14,7 +14,7 @@ import { afterAll, expect } from 'bun:test'; import { JUDGE_MS } from './helpers/eval-budgets'; import * as fs from 'fs'; import * as path from 'path'; -import { callJudge, judge, JudgeRefusalError, JUDGE_SCORE_SCHEMA, buildQaWorkflowJudgePrompt, buildQaHealthRubricJudgePrompt, buildQaAntiRefusalJudgePrompt, buildCrossSkillConsistencyJudgePrompt, buildVoiceDirectiveJudgePrompt, DEFAULT_JUDGE_MAX_TOKENS, judgePanel, judgePanelMean, judgePanelMajority, judgePanelReasoning, JUDGE_SCORE_DIMENSIONS } from './helpers/llm-judge'; +import { callJudge, judge, JudgeRefusalError, JUDGE_SCORE_SCHEMA, QA_ANTI_REFUSAL_JUDGE_SCHEMA, buildQaWorkflowJudgePrompt, buildQaHealthRubricJudgePrompt, buildQaAntiRefusalJudgePrompt, buildCrossSkillConsistencyJudgePrompt, buildVoiceDirectiveJudgePrompt, DEFAULT_JUDGE_MAX_TOKENS, judgePanel, judgePanelMean, judgePanelMajority, judgePanelReasoning, JUDGE_SCORE_DIMENSIONS } from './helpers/llm-judge'; import { ASK_QUESTIONS_HEADING, ENG_REVIEW_EXCERPT } from './helpers/workflow-excerpt'; import type { JudgeScore } from './helpers/llm-judge'; import { readWorkflowJudgeInput, buildWorkflowJudgePrompt, QA_DISCOVERY_REFERENCES, WORKFLOW_JUDGE_RESPONSE_SCHEMA, type WorkflowJudgeInput } from './helpers/workflow-judge-input'; @@ -262,7 +262,7 @@ describeIfSelected('QA skill quality evals', ['qa/SKILL.md workflow', 'qa/SKILL. const diffAwareSection = sliceQaPatterns('### Diff-aware', '### Full'); const rulesSection = sliceQaPatterns('## Important Rules'); - const samples = await judgePanel(() => callJudge<{ would_browse: boolean; fallback_behavior: string; confidence: number; reasoning: string }>(buildQaAntiRefusalJudgePrompt(diffAwareSection, rulesSection))); + const samples = await judgePanel(() => callJudge<{ would_browse: boolean; fallback_behavior: string; confidence: number; reasoning: string }>(buildQaAntiRefusalJudgePrompt(diffAwareSection, rulesSection), undefined, { jsonSchema: QA_ANTI_REFUSAL_JUDGE_SCHEMA })); const result = { would_browse: judgePanelMajority(samples, 'would_browse'), ...judgePanelMean(samples, ['confidence'] as const) }; console.log('QA anti-refusal panel:', JSON.stringify({ result, samples }, null, 2));