refactor(qa): /qa and /qa-only drive Aside, fall back to $B

QA_METHODOLOGY runs every phase as Aside scripts (orient, explore, document, re-test, mobile viewport via CDP emulation, links via HEAD fetch); the authenticate phase is 'you are already signed in'; a 13th rule requires consent before mutating actions on non-local targets; the fallback section translates each step onto $B. The qa E2E tests run on whichever engine is present.

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>

test/skill-e2e-qa-bugs.test.ts

@@ -9,6 +9,7 @@ import {
   copyDirSync, setupBrowseShims, logCost, recordE2E, dumpOutcomeDiagnostic,
   createEvalCollector, finalizeEvalCollector,
 } from './helpers/e2e-helpers';
+import { asideAvailable } from './helpers/aside-available';
 import { startTestServer } from '../browse/test/test-server';
 import { spawnSync } from 'child_process';
 import * as fs from 'fs';
@@ -20,7 +21,25 @@ const evalCollector = createEvalCollector('e2e-qa-bugs');
 // --- B6/B7/B8: Planted-bug outcome evals ---
 
 // Outcome evals also need ANTHROPIC_API_KEY for the LLM judge
-const describeOutcome = (evalsEnabled && hasApiKey) ? describe : describe.skip;
+// ...and a browser: a live Aside (primary) or a built browse/dist/browse
+// (fallback). Neither → skip, never fail.
+const describeOutcome = (evalsEnabled && hasApiKey && (asideAvailable() || fs.existsSync(browseBin))) ? describe : describe.skip;
+
+/**
+ * The BROWSER SETUP section qa/SKILL.md renders (Aside probe + browse fallback
+ * + driving rules). The agent gets just this, not the 1500-line skill, so the
+ * driver decision is the skill's own text, not the prompt's.
+ */
+function browserSetupSection(): string {
+  const skill = fs.readFileSync(path.join(ROOT, 'qa', 'SKILL.md'), 'utf-8');
+  const start = skill.indexOf('## BROWSER SETUP');
+  // The Aside contract is followed by its own H2, '## Browser fallback: ...' — the
+  // fixture must carry both so a run without Aside can take the $B path.
+  const fallback = skill.indexOf('\n## Browser fallback', start + 3);
+  const end = skill.indexOf('\n## ', (fallback > 0 ? fallback : start) + 3);
+  if (start < 0 || end < 0) throw new Error('qa/SKILL.md: BROWSER SETUP section not found — regenerate with: bun run gen:skill-docs');
+  return skill.slice(start, end);
+}
 
 // Wrap describeOutcome with selection — skip if no planted-bug tests are selected
 const outcomeTestNames = ['qa-b6-static', 'qa-b7-spa', 'qa-b8-checkout'];
@@ -59,20 +78,20 @@ let testServer: ReturnType<typeof startTestServer>;
     fs.mkdirSync(path.join(reportDir, 'screenshots'), { recursive: true });
     const reportPath = path.join(reportDir, 'qa-report.md');
 
-    // Direct bug-finding with browse. Keep prompt concise — no reading long SKILL.md docs.
+    fs.writeFileSync(path.join(testWorkDir, 'BROWSER-SETUP.md'), browserSetupSection());
+
+    // Direct bug-finding. Keep prompt concise — no reading long SKILL.md docs.
     // "Write early, update later" pattern ensures report exists even if agent hits max turns.
     const targetUrl = `${testServer.url}/${fixture}`;
     const result = await runSkillTest({
       prompt: `Find bugs on this page: ${targetUrl}
 
-Browser binary: B="${browseBin}"
+Browser: read BROWSER-SETUP.md in this directory and follow it exactly — it probes for Aside first and falls back to the gstack browse binary. If it falls back, the binary is at ${browseBin} (B="${browseBin}"). Do not look for any other browser. The target is LOCAL, so submitting its forms needs no consent question.
 
-PHASE 1 — Quick scan (5 commands max):
-$B goto ${targetUrl}
-$B console --errors
-$B snapshot -i
-$B snapshot -c
-$B accessibility
+PHASE 1 — Quick scan (5 browser steps max):
+- Load ${targetUrl} and capture the console errors
+- Take an interactive snapshot (clickable/fillable elements) and read the page text
+- Accessibility pass: img elements without alt text, form controls without a label or aria-label
 
 PHASE 2 — Write initial report to ${reportPath}:
 Write every bug you found so far. Format each as:
@@ -80,12 +99,12 @@ Write every bug you found so far. Format each as:
 - Severity: high / medium / low
 - Evidence: what you observed
 
-PHASE 3 — Interactive testing (targeted — max 15 commands):
-- Test email: type "user@" (no domain) and blur — does it validate?
+PHASE 3 — Interactive testing (targeted — max 15 browser steps):
+- Test email: fill "user@" (no domain) and blur — does it validate?
 - Test quantity: clear the field entirely — check the total display
-- Test credit card: type a 25-character string — check for overflow
+- Test credit card: fill a 25-character string — check for overflow
 - Submit the form with zip code empty — does it require zip?
-- Submit a valid form and run $B console --errors
+- Submit a valid form and capture the console errors again
 - After finding more bugs, UPDATE ${reportPath} with new findings
 
 PHASE 4 — Finalize report:
@@ -128,7 +147,7 @@ CRITICAL RULES:
       // Agent may have named it differently — find any .md in reportDir or testWorkDir
       for (const searchDir of [reportDir, testWorkDir]) {
         try {
-          const mdFiles = fs.readdirSync(searchDir).filter(f => f.endsWith('.md'));
+          const mdFiles = fs.readdirSync(searchDir).filter(f => f.endsWith('.md') && f !== 'BROWSER-SETUP.md');
           if (mdFiles.length > 0) {
             report = fs.readFileSync(path.join(searchDir, mdFiles[0]), 'utf-8');
             break;

test/skill-e2e-qa-workflow.test.ts

@@ -2,11 +2,12 @@ import { describe, test, expect, beforeAll, afterAll } from 'bun:test';
 import { JUDGE_MS, CAPTURE_MS, CAPTURE_LONG_MS } from './helpers/eval-budgets';
 import { runSkillTest } from './helpers/session-runner';
 import {
-  ROOT, browseBin, runId, evalsEnabled,
+  ROOT, browseBin, runId, evalsEnabled, selectedTests,
   describeIfSelected, testConcurrentIfSelected,
   copyDirSync, setupBrowseShims, logCost, recordE2E,
   createEvalCollector, finalizeEvalCollector,
 } from './helpers/e2e-helpers';
+import { asideAvailable } from './helpers/aside-available';
 import { startTestServer } from '../browse/test/test-server';
 import { spawnSync } from 'child_process';
 import * as fs from 'fs';
@@ -15,6 +16,18 @@ import * as os from 'os';
 
 const evalCollector = createEvalCollector('e2e-qa-workflow');
 
+// /qa and /qa-only drive the Aside browser first and fall back to the gstack
+// browse binary. The browser-driving describes need one of the two — a live
+// Aside or a built browse/dist/browse (CI builds it, so the Linux lane runs
+// the fallback path). Neither → skip, never fail. qa-bootstrap opens no
+// browser and is not gated.
+const browserSelected = evalsEnabled && (asideAvailable() || fs.existsSync(browseBin)) ? selectedTests : [];
+
+// The skill's BROWSER SETUP decides Aside vs fallback; the prompt only tells the
+// agent where the fallback binary is (the hermetic HOME has no global install).
+const browserPrompt = (skillMd: string) =>
+  `Follow the BROWSER SETUP section in ${skillMd} exactly: it probes for Aside first and falls back to the gstack browse binary. If it falls back, the browse binary is at ${browseBin} (B="${browseBin}"; find-browse is shimmed under browse/bin in this directory). Do not look for any other browser.`;
+
 // --- B4: QA skill E2E ---
 
 describeIfSelected('QA skill E2E', ['qa-quick'], () => {
@@ -40,7 +53,7 @@ describeIfSelected('QA skill E2E', ['qa-quick'], () => {
 
   testConcurrentIfSelected('qa-quick', async () => {
     const result = await runSkillTest({
-      prompt: `B="${browseBin}"
+      prompt: `${browserPrompt('qa/SKILL.md')}
 
 The test server is already running at: ${testServer.url}
 Target page: ${testServer.url}/basic.html
@@ -71,7 +84,7 @@ Write your report to ${qaDir}/qa-reports/qa-report.md`,
     // Accept error_max_turns — the agent doing thorough QA work is not a failure
     expect(['success', 'error_max_turns']).toContain(result.exitReason);
   }, CAPTURE_MS);
-});
+}, browserSelected);
 
 // --- QA-Only E2E (report-only, no fixes) ---
 
@@ -112,9 +125,7 @@ describeIfSelected('QA-Only skill E2E', ['qa-only-no-fix'], () => {
 
   testConcurrentIfSelected('qa-only-no-fix', async () => {
     const result = await runSkillTest({
-      prompt: `IMPORTANT: The browse binary is already assigned below as B. Do NOT search for it or run the SKILL.md setup block — just use $B directly.
-
-B="${browseBin}"
+      prompt: `${browserPrompt('qa-only/SKILL.md')}
 
 Read the file qa-only/SKILL.md for the QA-only workflow instructions.
 Skip the preamble bash block, lake intro, telemetry, and contributor mode sections — go straight to the QA workflow.
@@ -158,7 +169,7 @@ Write your report to ${qaOnlyDir}/qa-reports/qa-only-report.md`,
     );
     expect(statusLines.filter((l: string) => l.startsWith(' M') || l.startsWith('M '))).toHaveLength(0);
   }, CAPTURE_MS);
-});
+}, browserSelected);
 
 // --- QA Fix Loop E2E ---
 
@@ -233,7 +244,7 @@ describeIfSelected('QA Fix Loop E2E', ['qa-fix-loop'], () => {
     const qaFixUrl = `http://127.0.0.1:${qaFixServer!.port}`;
 
     const result = await runSkillTest({
-      prompt: `You have a browse binary at ${browseBin}. Assign it to B variable like: B="${browseBin}"
+      prompt: `${browserPrompt('qa/SKILL.md')}
 
 Read the file qa/SKILL.md for the QA workflow instructions.
 qa is a carved skill: when SKILL.md tells you to Read ~/.claude/skills/gstack/qa/sections/<file>, read qa/sections/<file> in this working directory instead (same content, local copy).
@@ -273,7 +284,7 @@ This is a test+fix loop: find bugs, fix them in the source code, commit each fix
     const editCalls = result.toolCalls.filter(tc => tc.tool === 'Edit');
     expect(editCalls.length).toBeGreaterThan(0);
   }, CAPTURE_LONG_MS);
-});
+}, browserSelected);
 
 // --- Test Bootstrap E2E ---