diff --git a/qa-only/SKILL.md.tmpl b/qa-only/SKILL.md.tmpl index 75c4123cc..38f512948 100644 --- a/qa-only/SKILL.md.tmpl +++ b/qa-only/SKILL.md.tmpl @@ -39,13 +39,15 @@ You are a QA engineer. Test web applications like a real user — click everythi | Mode | full | `--quick`, `--regression .gstack/qa-reports/baseline.json` | | Output dir | `.gstack/qa-reports/` | `Output to /tmp/qa` | | Scope | Full app (or diff-scoped) | `Focus on the billing page` | -| Auth | None | `Sign in to user@example.com`, `Import cookies from cookies.json` | +| Auth | Your Aside session (already signed in) | If a sign-in wall appears, you sign in yourself in Aside — no credentials in chat (see BROWSER SETUP). Fallback browser only: /setup-browser-cookies or `$B handoff` | **If no URL is given and you're on a feature branch:** Automatically enter **diff-aware mode** (see Modes below). This is the most common case — the user just shipped code on a branch and wants to verify it works. -**Find the browse binary:** +**Browser: Aside** -{{BROWSE_SETUP}} +{{ASIDE_SETUP}} + +{{BROWSE_FALLBACK}} **Create output directories:** @@ -95,9 +97,10 @@ Write to `~/.gstack/projects/{slug}/{user}-{branch}-test-outcome-{datetime}.md` .gstack/qa-reports/ ├── qa-report-{domain}-{YYYY-MM-DD}.md # Structured report ├── screenshots/ -│ ├── initial.png # Landing page annotated screenshot -│ ├── issue-001-step-1.png # Per-issue evidence -│ ├── issue-001-result.png +│ ├── initial.jpg # Landing page screenshot +│ ├── issue-001-step-1.jpg # Per-issue evidence +│ ├── issue-001-result.jpg +│ ├── issue-002.png # Annotated screenshot (static bugs) │ └── ... └── baseline.json # For regression mode ``` diff --git a/qa/SKILL.md.tmpl b/qa/SKILL.md.tmpl index d15a0c5d1..23934afde 100644 --- a/qa/SKILL.md.tmpl +++ b/qa/SKILL.md.tmpl @@ -57,7 +57,7 @@ You are a QA engineer AND a bug-fix engineer. Test web applications like a real | Mode | full | `--regression .gstack/qa-reports/baseline.json` | | Output dir | `.gstack/qa-reports/` | `Output to /tmp/qa` | | Scope | Full app (or diff-scoped) | `Focus on the billing page` | -| Auth | None | `Sign in to user@example.com`, `Import cookies from cookies.json` | +| Auth | Your Aside session (already signed in) | If a sign-in wall appears, you sign in yourself in Aside — no credentials in chat (see BROWSER SETUP). Fallback browser only: /setup-browser-cookies or `$B handoff` | **Tiers determine which issues get fixed:** - **Quick:** Fix critical + high severity only @@ -66,12 +66,6 @@ You are a QA engineer AND a bug-fix engineer. Test web applications like a real **If no URL is given and you're on a feature branch:** Automatically enter **diff-aware mode** (see Modes below). This is the most common case — the user just shipped code on a branch and wants to verify it works. -**CDP mode detection:** Before starting, check if the browse server is connected to the user's real browser: -```bash -$B status 2>/dev/null | grep -q "Mode: cdp" && echo "CDP_MODE=true" || echo "CDP_MODE=false" -``` -If `CDP_MODE=true`: skip cookie import prompts (the real browser already has cookies), skip user-agent overrides (real browser has real user-agent), and skip headless detection workarounds. The user's real auth sessions are already available. - **Check for clean working tree:** ```bash @@ -90,9 +84,11 @@ RECOMMENDATION: Choose A because uncommitted work should be preserved as a commi After the user chooses, execute their choice (commit or stash), then continue with setup. -**Find the browse binary:** +**Browser: Aside** -{{BROWSE_SETUP}} +{{ASIDE_SETUP}} + +{{BROWSE_FALLBACK}} **Check test framework (bootstrap if needed):** @@ -101,7 +97,8 @@ After the user chooses, execute their choice (commit or stash), then continue wi **Create output directories:** ```bash -mkdir -p .gstack/qa-reports/screenshots +REPORT_DIR=".gstack/qa-reports" +mkdir -p "$REPORT_DIR/screenshots" ``` --- @@ -137,11 +134,11 @@ Record baseline health score at end of Phase 6 (per the Health Score Rubric in t .gstack/qa-reports/ ├── qa-report-{domain}-{YYYY-MM-DD}.md # Structured report ├── screenshots/ -│ ├── initial.png # Landing page annotated screenshot -│ ├── issue-001-step-1.png # Per-issue evidence -│ ├── issue-001-result.png -│ ├── issue-001-before.png # Before fix (if fixed) -│ ├── issue-001-after.png # After fix (if fixed) +│ ├── initial.jpg # Landing page screenshot +│ ├── issue-001-step-1.jpg # Per-issue evidence +│ ├── issue-001-result.jpg +│ ├── issue-002.png # Annotated screenshot (static bugs) +│ ├── issue-001-after.jpg # After fix (if fixed); the Phase 5 evidence is the before │ └── ... └── baseline.json # For regression mode ``` @@ -209,17 +206,36 @@ git commit -m "fix(qa): ISSUE-NNN — short description" ### 8d. Re-test - Navigate back to the affected page -- Take **before/after screenshot pair** +- Take **before/after screenshot pair** — the Phase 5 evidence is the before; capture the after now - Check console for errors -- Use `snapshot -D` to verify the change had the expected effect +- Compare the snapshot tree and `CONSOLE_ERRORS=` against the Phase 5 evidence to verify the change had the expected effect + +One flow, one script (tabs close when the script ends, so re-navigate from the URL): ```bash -$B goto -$B screenshot "$REPORT_DIR/screenshots/issue-NNN-after.png" -$B console --errors -$B snapshot -D +aside repl ' +const HOOK = `(() => { window.__gstackErrs = window.__gstackErrs || []; const oe = console.error; console.error = (...a) => { window.__gstackErrs.push(a.map(String).join(" ")); oe.apply(console, a); }; window.addEventListener("error", e => window.__gstackErrs.push("uncaught: " + e.message)); })()`; +const pg = await openTab("about:blank"); +await pg._sendToTarget("Page.addScriptToEvaluateOnNewDocument", { source: HOOK }); +await pg.goto(""); +const s = await snapshot(pg, { interactive: true }); +console.log(s.tree); +console.log("CONSOLE_ERRORS=" + JSON.stringify(await pg.evaluate(() => window.__gstackErrs))); +await pg.screenshot({ path: "issue-NNN-after.jpg", type: "jpeg", quality: 60, fullPage: true }); +console.log("ASIDE_DIR=" + pwd); +await closeTab(pg); +console.log("GSTACK_STEP_OK"); +' ``` +Then copy the evidence out of the `ASIDE_DIR` the script printed: + +```bash +cp "/issue-NNN-after.jpg" "$REPORT_DIR/screenshots/issue-NNN-after.jpg" +``` + +Read `$REPORT_DIR/screenshots/issue-NNN-after.jpg` so the user sees the after state inline. If the bug needed an interaction to reproduce, re-run the Phase 5 Drive-a-flow script instead and compare its `DIFF` and `CONSOLE_ERRORS=` lines with the original evidence. + ### 8e. Classify - **verified**: re-test confirms the fix works, no new errors introduced diff --git a/qa/references/issue-taxonomy.md b/qa/references/issue-taxonomy.md index 05c574166..796815a20 100644 --- a/qa/references/issue-taxonomy.md +++ b/qa/references/issue-taxonomy.md @@ -75,11 +75,11 @@ For each page visited during a QA session: -1. **Visual scan** — Take annotated screenshot (`snapshot -i -a -o`). Look for layout issues, broken images, alignment. +1. **Visual scan** — Take a screenshot (the Read-a-page script; `annotatedScreenshot(pg)` when you need ref labels). Look for layout issues, broken images, alignment. 2. **Interactive elements** — Click every button, link, and control. Does each do what it says? -3. **Forms** — Fill and submit. Test empty submission, invalid data, edge cases (long text, special characters). +3. **Forms** — Fill and submit (non-local target: consent first — rule 13). Test empty submission, invalid data, edge cases (long text, special characters). 4. **Navigation** — Check all paths in/out. Breadcrumbs, back button, deep links, mobile menu. 5. **States** — Check empty state, loading state, error state, full/overflow state. -6. **Console** — Run `console --errors` after interactions. Any new JS errors or failed requests? +6. **Console** — Print `CONSOLE_ERRORS=` after interactions. Any new JS errors or failed requests? 7. **Responsiveness** — If relevant, check mobile and tablet viewports. -8. **Auth boundaries** — What happens when logged out? Different user roles? +8. **Auth boundaries** — Never sign the user out or switch accounts yourself. If the signed-out or other-role view matters, ask the user to sign out / switch in Aside and re-run the page scripts. diff --git a/qa/templates/qa-report-template.md b/qa/templates/qa-report-template.md index 6aa309433..439a80ba4 100644 --- a/qa/templates/qa-report-template.md +++ b/qa/templates/qa-report-template.md @@ -64,11 +64,10 @@ **Repro Steps:** 1. Navigate to {URL} - ![Step 1](screenshots/issue-001-step-1.png) + ![Step 1](screenshots/issue-001-step-1.jpg) 2. {Action} - ![Step 2](screenshots/issue-001-step-2.png) 3. **Observe:** {what goes wrong} - ![Result](screenshots/issue-001-result.png) + ![Result](screenshots/issue-001-result.jpg) --- @@ -81,8 +80,8 @@ ### Before/After Evidence #### ISSUE-NNN: {title} -**Before:** ![Before](screenshots/issue-NNN-before.png) -**After:** ![After](screenshots/issue-NNN-after.png) +**Before:** ![Before](screenshots/issue-NNN-result.jpg) — the Phase 5 evidence (`issue-NNN.png` for a static bug) +**After:** ![After](screenshots/issue-NNN-after.jpg) --- diff --git a/scripts/resolvers/utility.ts b/scripts/resolvers/utility.ts index 911b01d8f..1f5bebd0a 100644 --- a/scripts/resolvers/utility.ts +++ b/scripts/resolvers/utility.ts @@ -117,25 +117,32 @@ This is the **primary mode** for developers verifying their work. When the user - View/template/component files → which pages render them - Model/service files → which pages use those models (check controllers that reference them) - CSS/style files → which pages include those stylesheets - - API endpoints → test them directly with \`$B js "await fetch('/api/...')"\` + - API endpoints → call them with the session's own cookies from one \`aside repl\` script: + \`\`\`bash + aside repl ' + const pg = await openTab(""); + const r = await fetch("/api/...", { method: "GET" }); + console.log("API_STATUS=" + r.status); + console.log("API_BODY_START"); console.log((await r.text()).slice(0, 4000)); console.log("API_BODY_END"); + await closeTab(pg); console.log("GSTACK_STEP_OK"); + ' + \`\`\` - Static pages (markdown, HTML) → navigate to them directly **If no obvious pages/routes are identified from the diff:** Do not skip browser testing. The user invoked /qa because they want browser-based verification. Fall back to Quick mode — navigate to the homepage, follow the top 5 navigation targets, check console for errors, and test any interactive elements found. Backend, config, and infrastructure changes affect app behavior — always verify the app still works. -3. **Detect the running app** — check common local dev ports: +3. **Detect the running app** — probe common local dev ports (no browser needed to find a port): \`\`\`bash - $B goto http://localhost:3000 2>/dev/null && echo "Found app on :3000" || \\ - $B goto http://localhost:4000 2>/dev/null && echo "Found app on :4000" || \\ - $B goto http://localhost:8080 2>/dev/null && echo "Found app on :8080" + for p in 3000 4000 8080; do curl -sI --max-time 3 "http://localhost:$p" >/dev/null 2>&1 && echo "Found app on :$p"; done \`\`\` - If no local app is found, check for a staging/preview URL in the PR or environment. If nothing works, ask the user for the URL. + Open the first URL that answers in Aside. If no local app is found, check for a staging/preview URL in the PR or environment. If nothing works, ask the user for the URL. 4. **Test each affected page/route:** - - Navigate to the page + - Navigate to the page (the Read-a-page script in Phase 3) - Take a screenshot - - Check console for errors + - Check console for errors (the \`CONSOLE_ERRORS=\` line) - If the change was interactive (forms, buttons, flows), test the interaction end-to-end - - Use \`snapshot -D\` before and after actions to verify the change had the expected effect + - Snapshot before acting and print the diff after (the Drive-a-flow script in Phase 5) to verify the change had the expected effect 5. **Cross-reference with commit messages and PR description** to understand *intent* — what should the change do? Verify it actually does that. @@ -163,77 +170,87 @@ Run full mode, then load \`baseline.json\` from a previous run. Diff: which issu ### Phase 1: Initialize -1. Find browse binary (see Setup above) +1. Confirm Aside is READY (see BROWSER SETUP above). If it printed \`NEEDS_ASIDE\` or \`ASIDE_NOT_RUNNING\`, the Browser fallback section applies: find \`$B\` there and translate every \`aside repl\` script below through its table. 2. Create output directories 3. Copy report template from \`qa/templates/qa-report-template.md\` to output dir 4. Start timer for duration tracking ### Phase 2: Authenticate (if needed) -**If the user specified auth credentials:** +Aside is the user's real browser, so the session is already signed in wherever the user is signed in. You never authenticate — the user does. In the fallback browser there is no session to inherit: import one with /setup-browser-cookies, or \`$B handoff\` for a human sign-in and \`$B resume\` when they're done. -\`\`\`bash -$B goto -$B snapshot -i # find the login form -$B fill @e3 "user@example.com" -$B fill @e4 "[REDACTED]" # NEVER include real passwords in report -$B click @e5 # submit -$B snapshot -D # verify login succeeded -\`\`\` +**If a sign-in wall appears:** stop and tell the user: "Sign in to in Aside yourself (open it in a new Aside tab), then tell me you're done." Then re-run the step — the browser's cookies now apply. Never type passwords, one-time codes, or payment details, and never read or print cookies, tokens, or localStorage. -**If the user provided a cookie file:** +**If 2FA/OTP is required:** The user completes it in the Aside window, then tells you to continue. -\`\`\`bash -$B cookie-import cookies.json -$B goto -\`\`\` - -**If 2FA/OTP is required:** Ask the user for the code and wait. - -**If CAPTCHA blocks you:** Tell the user: "Please complete the CAPTCHA in the browser, then tell me to continue." +**If CAPTCHA blocks you:** Tell the user: "Please complete the CAPTCHA in Aside, then tell me to continue." ### Phase 3: Orient -Get a map of the application: +Get a map of the application. One script reads the landing page — console errors from load, the interactive snapshot tree, the visible text, and a screenshot: \`\`\`bash -$B goto -$B snapshot -i -a -o "$REPORT_DIR/screenshots/initial.png" -$B links # map navigation structure -$B console --errors # any errors on landing? +aside repl ' +const HOOK = \`(() => { window.__gstackErrs = window.__gstackErrs || []; const oe = console.error; console.error = (...a) => { window.__gstackErrs.push(a.map(String).join(" ")); oe.apply(console, a); }; window.addEventListener("error", e => window.__gstackErrs.push("uncaught: " + e.message)); window.addEventListener("unhandledrejection", e => window.__gstackErrs.push("unhandledrejection: " + (e.reason && e.reason.message || e.reason))); })()\`; +const pg = await openTab("about:blank"); +await pg._sendToTarget("Page.addScriptToEvaluateOnNewDocument", { source: HOOK }); +await pg.goto(""); +const s = await snapshot(pg, { interactive: true }); +console.log(s.tree); +console.log("CONSOLE_ERRORS=" + JSON.stringify(await pg.evaluate(() => window.__gstackErrs))); +console.log("TEXT_START"); console.log((await pg.evaluate(() => document.body.innerText)).slice(0, 20000)); console.log("TEXT_END"); +await pg.screenshot({ path: "initial.jpg", type: "jpeg", quality: 60, fullPage: true }); +console.log("ASIDE_DIR=" + pwd); +await closeTab(pg); +console.log("GSTACK_STEP_OK"); +' \`\`\` +Then copy the screenshot out of the printed directory and show it: \`cp "/initial.jpg" "$REPORT_DIR/screenshots/initial.jpg"\`, then Read it. + +Map the navigation structure with the links script (same-origin, read-only HEAD requests with the user's cookies): + +\`\`\`bash +aside repl ' +const pg = await openTab(""); +const links = await pg.evaluate(() => [...new Set([...document.querySelectorAll("a[href]")].map(a => a.href))].filter(h => h.startsWith(location.origin) && !/logout|signout|delete|remove|cancel|unsubscribe/i.test(h))); +for (const l of links) { const r = await fetch(l, { method: "HEAD" }).catch(e => ({ status: "ERR " + e.message })); console.log("LINK", r.status, l); } +await closeTab(pg); console.log("GSTACK_STEP_OK"); +' +\`\`\` + +Every \`LINK\` line with a 4xx/5xx or \`ERR\` status is a broken link for the Links score. + **Detect framework** (note in report metadata): - \`__next\` in HTML or \`_next/data\` requests → Next.js - \`csrf-token\` meta tag → Rails - \`wp-content\` in URLs → WordPress - Client-side routing with no page reloads → SPA -**For SPAs:** The \`links\` command may return few results because navigation is client-side. Use \`snapshot -i\` to find nav elements (buttons, menu items) instead. +**For SPAs:** The links script may return few results because navigation is client-side. Use \`snapshot(pg, { interactive: true })\` to find nav elements (buttons, menu items) instead. ### Phase 4: Explore -Visit pages systematically. At each page: - -\`\`\`bash -$B goto -$B snapshot -i -a -o "$REPORT_DIR/screenshots/page-name.png" -$B console --errors -\`\`\` +Visit pages systematically. At each page, run the Read-a-page script from Phase 3 against the page URL with \`page-.jpg\` as the screenshot path, copy it into \`$REPORT_DIR/screenshots/\`, and Read it. Then follow the **per-page exploration checklist** (see \`qa/references/issue-taxonomy.md\`): -1. **Visual scan** — Look at the annotated screenshot for layout issues +1. **Visual scan** — Look at the screenshot for layout issues (use the annotated-screenshot script when you need ref labels on the page) 2. **Interactive elements** — Click buttons, links, controls. Do they work? 3. **Forms** — Fill and submit. Test empty, invalid, edge cases 4. **Navigation** — Check all paths in and out 5. **States** — Empty state, loading, error, overflow -6. **Console** — Any new JS errors after interactions? -7. **Responsiveness** — Check mobile viewport if relevant: +6. **Console** — Any new JS errors after interactions? Print \`CONSOLE_ERRORS=\` after every action +7. **Responsiveness** — Check the mobile viewport if relevant: \`\`\`bash - $B viewport 375x812 - $B screenshot "$REPORT_DIR/screenshots/page-mobile.png" - $B viewport 1280x720 + aside repl ' + const pg = await openTab(""); + await pg._sendToTarget("Emulation.setDeviceMetricsOverride", { width: 375, height: 812, deviceScaleFactor: 2, mobile: true }); + await sleep(300); + await pg.screenshot({ path: "page-mobile.jpg", type: "jpeg", quality: 60, fullPage: true }); + await pg._sendToTarget("Emulation.clearDeviceMetricsOverride", {}); + console.log("ASIDE_DIR=" + pwd); await closeTab(pg); console.log("GSTACK_STEP_OK"); + ' \`\`\` **Depth judgment:** Spend more time on core features (homepage, dashboard, checkout, search) and less on secondary pages (about, terms, privacy). @@ -246,26 +263,47 @@ Document each issue **immediately when found** — don't batch them. **Two evidence tiers:** -**Interactive bugs** (broken flows, dead buttons, form failures): +**Interactive bugs** (broken flows, dead buttons, form failures) — one script per flow, because tabs close when the script ends: 1. Take a screenshot before the action 2. Perform the action 3. Take a screenshot showing the result -4. Use \`snapshot -D\` to show what changed +4. Print the snapshot diff to show what changed 5. Write repro steps referencing screenshots \`\`\`bash -$B screenshot "$REPORT_DIR/screenshots/issue-001-step-1.png" -$B click @e5 -$B screenshot "$REPORT_DIR/screenshots/issue-001-result.png" -$B snapshot -D +aside repl ' +const HOOK = \`(() => { window.__gstackErrs = window.__gstackErrs || []; const oe = console.error; console.error = (...a) => { window.__gstackErrs.push(a.map(String).join(" ")); oe.apply(console, a); }; window.addEventListener("error", e => window.__gstackErrs.push("uncaught: " + e.message)); })()\`; +const pg = await openTab("about:blank"); +await pg._sendToTarget("Page.addScriptToEvaluateOnNewDocument", { source: HOOK }); +await pg.goto(""); +await snapshot(pg, { interactive: true }); // baseline for .diff; refs like e12 name the elements +await pg.screenshot({ path: "issue-001-step-1.jpg", type: "jpeg", quality: 60 }); +await pg.locator("e12").click(); // or pg.fill("#email", "qa@example.com"), pg.getByRole("button", { name: "Save" }).click() +await sleep(500); // or await pg.waitForSelector("#done"); await pg.waitForURL(/dashboard/) +const s = await snapshot(pg); +console.log("DIFF_START"); console.log(s.diff); console.log("DIFF_END"); +console.log("URL=" + pg.url()); +console.log("CONSOLE_ERRORS=" + JSON.stringify(await pg.evaluate(() => window.__gstackErrs))); +await pg.screenshot({ path: "issue-001-result.jpg", type: "jpeg", quality: 60 }); +console.log("ASIDE_DIR=" + pwd); +await closeTab(pg); +console.log("GSTACK_STEP_OK"); +' \`\`\` +Copy both screenshots out of the printed \`ASIDE_DIR\` into \`$REPORT_DIR/screenshots/\` and Read them. + **Static bugs** (typos, layout issues, missing images): 1. Take a single annotated screenshot showing the problem 2. Describe what's wrong \`\`\`bash -$B snapshot -i -a -o "$REPORT_DIR/screenshots/issue-002.png" +aside repl ' +const pg = await openTab(""); +const a = await annotatedScreenshot(pg); +await fs.writeFile(path.join(pwd, "issue-002.png"), Buffer.from(a.base64Image, "base64")); +console.log("ASIDE_DIR=" + pwd); await closeTab(pg); console.log("GSTACK_STEP_OK"); +' \`\`\` **Write each issue to the report immediately** using the template format from \`qa/templates/qa-report-template.md\`. @@ -356,7 +394,7 @@ Minimum 0 per category. - Check for mixed content warnings (common with WP) ### General SPA (React, Vue, Angular) -- Use \`snapshot -i\` for navigation — \`links\` command misses client-side routes +- Use \`snapshot(pg, { interactive: true })\` for navigation — the links script misses client-side routes - Check for stale state (navigate away and back — does data refresh?) - Test browser back/forward — does the app handle history correctly? - Check for memory leaks (monitor console after extended use) @@ -367,16 +405,17 @@ Minimum 0 per category. 1. **Repro is everything.** Every issue needs at least one screenshot. No exceptions. 2. **Verify before documenting.** Retry the issue once to confirm it's reproducible, not a fluke. -3. **Never include credentials.** Write \`[REDACTED]\` for passwords in repro steps. +3. **Never include credentials.** You never type them — the user signs in inside Aside. Write \`[REDACTED]\` if a repro step has to mention one. 4. **Write incrementally.** Append each issue to the report as you find it. Don't batch. 5. **Never read source code.** Test as a user, not a developer. 6. **Check console after every interaction.** JS errors that don't surface visually are still bugs. 7. **Test like a user.** Use realistic data. Walk through complete workflows end-to-end. 8. **Depth over breadth.** 5-10 well-documented issues with evidence > 20 vague descriptions. 9. **Never delete output files.** Screenshots and reports accumulate — that's intentional. -10. **Use \`snapshot -C\` for tricky UIs.** Finds clickable divs that the accessibility tree misses. -11. **Show screenshots to the user.** After every \`$B screenshot\`, \`$B snapshot -a -o\`, or \`$B responsive\` command, use the Read tool on the output file(s) so the user can see them inline. For \`responsive\` (3 files), Read all three. This is critical — without it, screenshots are invisible to the user. -12. **Never refuse to use the browser.** When the user invokes /qa or /qa-only, they are requesting browser-based testing. Never suggest evals, unit tests, or other alternatives as a substitute. Even if the diff appears to have no UI changes, backend changes affect app behavior — always open the browser and test.`; +10. **Use \`annotatedScreenshot(pg)\` when the tree misses a clickable element.** Ref labels drawn on the page find clickable divs the accessibility tree skips; then click by ref or CSS selector. +11. **Show screenshots to the user.** After every script that saves a screenshot, \`cp\` it out of the printed \`ASIDE_DIR\` into \`$REPORT_DIR/screenshots/\` and use the Read tool on the copied file so the user can see it inline. This is critical — without it, screenshots are invisible to the user. +12. **Never refuse to use the browser.** When the user invokes /qa or /qa-only, they are requesting browser-based testing in Aside. Never suggest evals, unit tests, curl, or other alternatives as a substitute. Even if the diff appears to have no UI changes, backend changes affect app behavior — always open the app in the browser and test. +13. **Mutating actions on a non-local target need consent.** Submitting, creating, deleting, purchasing, or changing settings on anything that is not LOCAL follows the "Invocation is consent to LOOK, not to ACT" rule in BROWSER SETUP — one AskUserQuestion per run, before the first such action.`; } export function generateCoAuthorTrailer(ctx: TemplateContext): string { diff --git a/test/skill-e2e-qa-bugs.test.ts b/test/skill-e2e-qa-bugs.test.ts index c2fcff155..7858173d7 100644 --- a/test/skill-e2e-qa-bugs.test.ts +++ b/test/skill-e2e-qa-bugs.test.ts @@ -9,6 +9,7 @@ import { copyDirSync, setupBrowseShims, logCost, recordE2E, dumpOutcomeDiagnostic, createEvalCollector, finalizeEvalCollector, } from './helpers/e2e-helpers'; +import { asideAvailable } from './helpers/aside-available'; import { startTestServer } from '../browse/test/test-server'; import { spawnSync } from 'child_process'; import * as fs from 'fs'; @@ -20,7 +21,25 @@ const evalCollector = createEvalCollector('e2e-qa-bugs'); // --- B6/B7/B8: Planted-bug outcome evals --- // Outcome evals also need ANTHROPIC_API_KEY for the LLM judge -const describeOutcome = (evalsEnabled && hasApiKey) ? describe : describe.skip; +// ...and a browser: a live Aside (primary) or a built browse/dist/browse +// (fallback). Neither → skip, never fail. +const describeOutcome = (evalsEnabled && hasApiKey && (asideAvailable() || fs.existsSync(browseBin))) ? describe : describe.skip; + +/** + * The BROWSER SETUP section qa/SKILL.md renders (Aside probe + browse fallback + * + driving rules). The agent gets just this, not the 1500-line skill, so the + * driver decision is the skill's own text, not the prompt's. + */ +function browserSetupSection(): string { + const skill = fs.readFileSync(path.join(ROOT, 'qa', 'SKILL.md'), 'utf-8'); + const start = skill.indexOf('## BROWSER SETUP'); + // The Aside contract is followed by its own H2, '## Browser fallback: ...' — the + // fixture must carry both so a run without Aside can take the $B path. + const fallback = skill.indexOf('\n## Browser fallback', start + 3); + const end = skill.indexOf('\n## ', (fallback > 0 ? fallback : start) + 3); + if (start < 0 || end < 0) throw new Error('qa/SKILL.md: BROWSER SETUP section not found — regenerate with: bun run gen:skill-docs'); + return skill.slice(start, end); +} // Wrap describeOutcome with selection — skip if no planted-bug tests are selected const outcomeTestNames = ['qa-b6-static', 'qa-b7-spa', 'qa-b8-checkout']; @@ -59,20 +78,20 @@ let testServer: ReturnType; fs.mkdirSync(path.join(reportDir, 'screenshots'), { recursive: true }); const reportPath = path.join(reportDir, 'qa-report.md'); - // Direct bug-finding with browse. Keep prompt concise — no reading long SKILL.md docs. + fs.writeFileSync(path.join(testWorkDir, 'BROWSER-SETUP.md'), browserSetupSection()); + + // Direct bug-finding. Keep prompt concise — no reading long SKILL.md docs. // "Write early, update later" pattern ensures report exists even if agent hits max turns. const targetUrl = `${testServer.url}/${fixture}`; const result = await runSkillTest({ prompt: `Find bugs on this page: ${targetUrl} -Browser binary: B="${browseBin}" +Browser: read BROWSER-SETUP.md in this directory and follow it exactly — it probes for Aside first and falls back to the gstack browse binary. If it falls back, the binary is at ${browseBin} (B="${browseBin}"). Do not look for any other browser. The target is LOCAL, so submitting its forms needs no consent question. -PHASE 1 — Quick scan (5 commands max): -$B goto ${targetUrl} -$B console --errors -$B snapshot -i -$B snapshot -c -$B accessibility +PHASE 1 — Quick scan (5 browser steps max): +- Load ${targetUrl} and capture the console errors +- Take an interactive snapshot (clickable/fillable elements) and read the page text +- Accessibility pass: img elements without alt text, form controls without a label or aria-label PHASE 2 — Write initial report to ${reportPath}: Write every bug you found so far. Format each as: @@ -80,12 +99,12 @@ Write every bug you found so far. Format each as: - Severity: high / medium / low - Evidence: what you observed -PHASE 3 — Interactive testing (targeted — max 15 commands): -- Test email: type "user@" (no domain) and blur — does it validate? +PHASE 3 — Interactive testing (targeted — max 15 browser steps): +- Test email: fill "user@" (no domain) and blur — does it validate? - Test quantity: clear the field entirely — check the total display -- Test credit card: type a 25-character string — check for overflow +- Test credit card: fill a 25-character string — check for overflow - Submit the form with zip code empty — does it require zip? -- Submit a valid form and run $B console --errors +- Submit a valid form and capture the console errors again - After finding more bugs, UPDATE ${reportPath} with new findings PHASE 4 — Finalize report: @@ -128,7 +147,7 @@ CRITICAL RULES: // Agent may have named it differently — find any .md in reportDir or testWorkDir for (const searchDir of [reportDir, testWorkDir]) { try { - const mdFiles = fs.readdirSync(searchDir).filter(f => f.endsWith('.md')); + const mdFiles = fs.readdirSync(searchDir).filter(f => f.endsWith('.md') && f !== 'BROWSER-SETUP.md'); if (mdFiles.length > 0) { report = fs.readFileSync(path.join(searchDir, mdFiles[0]), 'utf-8'); break; diff --git a/test/skill-e2e-qa-workflow.test.ts b/test/skill-e2e-qa-workflow.test.ts index a07315d7f..ddba5c954 100644 --- a/test/skill-e2e-qa-workflow.test.ts +++ b/test/skill-e2e-qa-workflow.test.ts @@ -2,11 +2,12 @@ import { describe, test, expect, beforeAll, afterAll } from 'bun:test'; import { JUDGE_MS, CAPTURE_MS, CAPTURE_LONG_MS } from './helpers/eval-budgets'; import { runSkillTest } from './helpers/session-runner'; import { - ROOT, browseBin, runId, evalsEnabled, + ROOT, browseBin, runId, evalsEnabled, selectedTests, describeIfSelected, testConcurrentIfSelected, copyDirSync, setupBrowseShims, logCost, recordE2E, createEvalCollector, finalizeEvalCollector, } from './helpers/e2e-helpers'; +import { asideAvailable } from './helpers/aside-available'; import { startTestServer } from '../browse/test/test-server'; import { spawnSync } from 'child_process'; import * as fs from 'fs'; @@ -15,6 +16,18 @@ import * as os from 'os'; const evalCollector = createEvalCollector('e2e-qa-workflow'); +// /qa and /qa-only drive the Aside browser first and fall back to the gstack +// browse binary. The browser-driving describes need one of the two — a live +// Aside or a built browse/dist/browse (CI builds it, so the Linux lane runs +// the fallback path). Neither → skip, never fail. qa-bootstrap opens no +// browser and is not gated. +const browserSelected = evalsEnabled && (asideAvailable() || fs.existsSync(browseBin)) ? selectedTests : []; + +// The skill's BROWSER SETUP decides Aside vs fallback; the prompt only tells the +// agent where the fallback binary is (the hermetic HOME has no global install). +const browserPrompt = (skillMd: string) => + `Follow the BROWSER SETUP section in ${skillMd} exactly: it probes for Aside first and falls back to the gstack browse binary. If it falls back, the browse binary is at ${browseBin} (B="${browseBin}"; find-browse is shimmed under browse/bin in this directory). Do not look for any other browser.`; + // --- B4: QA skill E2E --- describeIfSelected('QA skill E2E', ['qa-quick'], () => { @@ -40,7 +53,7 @@ describeIfSelected('QA skill E2E', ['qa-quick'], () => { testConcurrentIfSelected('qa-quick', async () => { const result = await runSkillTest({ - prompt: `B="${browseBin}" + prompt: `${browserPrompt('qa/SKILL.md')} The test server is already running at: ${testServer.url} Target page: ${testServer.url}/basic.html @@ -71,7 +84,7 @@ Write your report to ${qaDir}/qa-reports/qa-report.md`, // Accept error_max_turns — the agent doing thorough QA work is not a failure expect(['success', 'error_max_turns']).toContain(result.exitReason); }, CAPTURE_MS); -}); +}, browserSelected); // --- QA-Only E2E (report-only, no fixes) --- @@ -112,9 +125,7 @@ describeIfSelected('QA-Only skill E2E', ['qa-only-no-fix'], () => { testConcurrentIfSelected('qa-only-no-fix', async () => { const result = await runSkillTest({ - prompt: `IMPORTANT: The browse binary is already assigned below as B. Do NOT search for it or run the SKILL.md setup block — just use $B directly. - -B="${browseBin}" + prompt: `${browserPrompt('qa-only/SKILL.md')} Read the file qa-only/SKILL.md for the QA-only workflow instructions. Skip the preamble bash block, lake intro, telemetry, and contributor mode sections — go straight to the QA workflow. @@ -158,7 +169,7 @@ Write your report to ${qaOnlyDir}/qa-reports/qa-only-report.md`, ); expect(statusLines.filter((l: string) => l.startsWith(' M') || l.startsWith('M '))).toHaveLength(0); }, CAPTURE_MS); -}); +}, browserSelected); // --- QA Fix Loop E2E --- @@ -233,7 +244,7 @@ describeIfSelected('QA Fix Loop E2E', ['qa-fix-loop'], () => { const qaFixUrl = `http://127.0.0.1:${qaFixServer!.port}`; const result = await runSkillTest({ - prompt: `You have a browse binary at ${browseBin}. Assign it to B variable like: B="${browseBin}" + prompt: `${browserPrompt('qa/SKILL.md')} Read the file qa/SKILL.md for the QA workflow instructions. qa is a carved skill: when SKILL.md tells you to Read ~/.claude/skills/gstack/qa/sections/, read qa/sections/ in this working directory instead (same content, local copy). @@ -273,7 +284,7 @@ This is a test+fix loop: find bugs, fix them in the source code, commit each fix const editCalls = result.toolCalls.filter(tc => tc.tool === 'Edit'); expect(editCalls.length).toBeGreaterThan(0); }, CAPTURE_LONG_MS); -}); +}, browserSelected); // --- Test Bootstrap E2E ---