mirror of
https://github.com/garrytan/gstack.git
synced 2026-09-09 14:38:59 +02:00
refactor(qa): /qa and /qa-only drive Aside, fall back to $B
QA_METHODOLOGY runs every phase as Aside scripts (orient, explore, document, re-test, mobile viewport via CDP emulation, links via HEAD fetch); the authenticate phase is 'you are already signed in'; a 13th rule requires consent before mutating actions on non-local targets; the fallback section translates each step onto $B. The qa E2E tests run on whichever engine is present. Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
This commit is contained in:
Sina
@@ -39,13 +39,15 @@ You are a QA engineer. Test web applications like a real user — click everythi
|
||||
| Mode | full | `--quick`, `--regression .gstack/qa-reports/baseline.json` |
|
||||
| Output dir | `.gstack/qa-reports/` | `Output to /tmp/qa` |
|
||||
| Scope | Full app (or diff-scoped) | `Focus on the billing page` |
|
||||
| Auth | None | `Sign in to user@example.com`, `Import cookies from cookies.json` |
|
||||
| Auth | Your Aside session (already signed in) | If a sign-in wall appears, you sign in yourself in Aside — no credentials in chat (see BROWSER SETUP). Fallback browser only: /setup-browser-cookies or `$B handoff` |
|
||||
|
||||
**If no URL is given and you're on a feature branch:** Automatically enter **diff-aware mode** (see Modes below). This is the most common case — the user just shipped code on a branch and wants to verify it works.
|
||||
|
||||
**Find the browse binary:**
|
||||
**Browser: Aside**
|
||||
|
||||
{{BROWSE_SETUP}}
|
||||
{{ASIDE_SETUP}}
|
||||
|
||||
{{BROWSE_FALLBACK}}
|
||||
|
||||
**Create output directories:**
|
||||
|
||||
@@ -95,9 +97,10 @@ Write to `~/.gstack/projects/{slug}/{user}-{branch}-test-outcome-{datetime}.md`
|
||||
.gstack/qa-reports/
|
||||
├── qa-report-{domain}-{YYYY-MM-DD}.md # Structured report
|
||||
├── screenshots/
|
||||
│ ├── initial.png # Landing page annotated screenshot
|
||||
│ ├── issue-001-step-1.png # Per-issue evidence
|
||||
│ ├── issue-001-result.png
|
||||
│ ├── initial.jpg # Landing page screenshot
|
||||
│ ├── issue-001-step-1.jpg # Per-issue evidence
|
||||
│ ├── issue-001-result.jpg
|
||||
│ ├── issue-002.png # Annotated screenshot (static bugs)
|
||||
│ └── ...
|
||||
└── baseline.json # For regression mode
|
||||
```
|
||||
|
||||
+37
-21
@@ -57,7 +57,7 @@ You are a QA engineer AND a bug-fix engineer. Test web applications like a real
|
||||
| Mode | full | `--regression .gstack/qa-reports/baseline.json` |
|
||||
| Output dir | `.gstack/qa-reports/` | `Output to /tmp/qa` |
|
||||
| Scope | Full app (or diff-scoped) | `Focus on the billing page` |
|
||||
| Auth | None | `Sign in to user@example.com`, `Import cookies from cookies.json` |
|
||||
| Auth | Your Aside session (already signed in) | If a sign-in wall appears, you sign in yourself in Aside — no credentials in chat (see BROWSER SETUP). Fallback browser only: /setup-browser-cookies or `$B handoff` |
|
||||
|
||||
**Tiers determine which issues get fixed:**
|
||||
- **Quick:** Fix critical + high severity only
|
||||
@@ -66,12 +66,6 @@ You are a QA engineer AND a bug-fix engineer. Test web applications like a real
|
||||
|
||||
**If no URL is given and you're on a feature branch:** Automatically enter **diff-aware mode** (see Modes below). This is the most common case — the user just shipped code on a branch and wants to verify it works.
|
||||
|
||||
**CDP mode detection:** Before starting, check if the browse server is connected to the user's real browser:
|
||||
```bash
|
||||
$B status 2>/dev/null | grep -q "Mode: cdp" && echo "CDP_MODE=true" || echo "CDP_MODE=false"
|
||||
```
|
||||
If `CDP_MODE=true`: skip cookie import prompts (the real browser already has cookies), skip user-agent overrides (real browser has real user-agent), and skip headless detection workarounds. The user's real auth sessions are already available.
|
||||
|
||||
**Check for clean working tree:**
|
||||
|
||||
```bash
|
||||
@@ -90,9 +84,11 @@ RECOMMENDATION: Choose A because uncommitted work should be preserved as a commi
|
||||
|
||||
After the user chooses, execute their choice (commit or stash), then continue with setup.
|
||||
|
||||
**Find the browse binary:**
|
||||
**Browser: Aside**
|
||||
|
||||
{{BROWSE_SETUP}}
|
||||
{{ASIDE_SETUP}}
|
||||
|
||||
{{BROWSE_FALLBACK}}
|
||||
|
||||
**Check test framework (bootstrap if needed):**
|
||||
|
||||
@@ -101,7 +97,8 @@ After the user chooses, execute their choice (commit or stash), then continue wi
|
||||
**Create output directories:**
|
||||
|
||||
```bash
|
||||
mkdir -p .gstack/qa-reports/screenshots
|
||||
REPORT_DIR=".gstack/qa-reports"
|
||||
mkdir -p "$REPORT_DIR/screenshots"
|
||||
```
|
||||
|
||||
---
|
||||
@@ -137,11 +134,11 @@ Record baseline health score at end of Phase 6 (per the Health Score Rubric in t
|
||||
.gstack/qa-reports/
|
||||
├── qa-report-{domain}-{YYYY-MM-DD}.md # Structured report
|
||||
├── screenshots/
|
||||
│ ├── initial.png # Landing page annotated screenshot
|
||||
│ ├── issue-001-step-1.png # Per-issue evidence
|
||||
│ ├── issue-001-result.png
|
||||
│ ├── issue-001-before.png # Before fix (if fixed)
|
||||
│ ├── issue-001-after.png # After fix (if fixed)
|
||||
│ ├── initial.jpg # Landing page screenshot
|
||||
│ ├── issue-001-step-1.jpg # Per-issue evidence
|
||||
│ ├── issue-001-result.jpg
|
||||
│ ├── issue-002.png # Annotated screenshot (static bugs)
|
||||
│ ├── issue-001-after.jpg # After fix (if fixed); the Phase 5 evidence is the before
|
||||
│ └── ...
|
||||
└── baseline.json # For regression mode
|
||||
```
|
||||
@@ -209,17 +206,36 @@ git commit -m "fix(qa): ISSUE-NNN — short description"
|
||||
### 8d. Re-test
|
||||
|
||||
- Navigate back to the affected page
|
||||
- Take **before/after screenshot pair**
|
||||
- Take **before/after screenshot pair** — the Phase 5 evidence is the before; capture the after now
|
||||
- Check console for errors
|
||||
- Use `snapshot -D` to verify the change had the expected effect
|
||||
- Compare the snapshot tree and `CONSOLE_ERRORS=` against the Phase 5 evidence to verify the change had the expected effect
|
||||
|
||||
One flow, one script (tabs close when the script ends, so re-navigate from the URL):
|
||||
|
||||
```bash
|
||||
$B goto <affected-url>
|
||||
$B screenshot "$REPORT_DIR/screenshots/issue-NNN-after.png"
|
||||
$B console --errors
|
||||
$B snapshot -D
|
||||
aside repl '
|
||||
const HOOK = `(() => { window.__gstackErrs = window.__gstackErrs || []; const oe = console.error; console.error = (...a) => { window.__gstackErrs.push(a.map(String).join(" ")); oe.apply(console, a); }; window.addEventListener("error", e => window.__gstackErrs.push("uncaught: " + e.message)); })()`;
|
||||
const pg = await openTab("about:blank");
|
||||
await pg._sendToTarget("Page.addScriptToEvaluateOnNewDocument", { source: HOOK });
|
||||
await pg.goto("<affected-url>");
|
||||
const s = await snapshot(pg, { interactive: true });
|
||||
console.log(s.tree);
|
||||
console.log("CONSOLE_ERRORS=" + JSON.stringify(await pg.evaluate(() => window.__gstackErrs)));
|
||||
await pg.screenshot({ path: "issue-NNN-after.jpg", type: "jpeg", quality: 60, fullPage: true });
|
||||
console.log("ASIDE_DIR=" + pwd);
|
||||
await closeTab(pg);
|
||||
console.log("GSTACK_STEP_OK");
|
||||
'
|
||||
```
|
||||
|
||||
Then copy the evidence out of the `ASIDE_DIR` the script printed:
|
||||
|
||||
```bash
|
||||
cp "<ASIDE_DIR>/issue-NNN-after.jpg" "$REPORT_DIR/screenshots/issue-NNN-after.jpg"
|
||||
```
|
||||
|
||||
Read `$REPORT_DIR/screenshots/issue-NNN-after.jpg` so the user sees the after state inline. If the bug needed an interaction to reproduce, re-run the Phase 5 Drive-a-flow script instead and compare its `DIFF` and `CONSOLE_ERRORS=` lines with the original evidence.
|
||||
|
||||
### 8e. Classify
|
||||
|
||||
- **verified**: re-test confirms the fix works, no new errors introduced
|
||||
|
||||
@@ -75,11 +75,11 @@
|
||||
|
||||
For each page visited during a QA session:
|
||||
|
||||
1. **Visual scan** — Take annotated screenshot (`snapshot -i -a -o`). Look for layout issues, broken images, alignment.
|
||||
1. **Visual scan** — Take a screenshot (the Read-a-page script; `annotatedScreenshot(pg)` when you need ref labels). Look for layout issues, broken images, alignment.
|
||||
2. **Interactive elements** — Click every button, link, and control. Does each do what it says?
|
||||
3. **Forms** — Fill and submit. Test empty submission, invalid data, edge cases (long text, special characters).
|
||||
3. **Forms** — Fill and submit (non-local target: consent first — rule 13). Test empty submission, invalid data, edge cases (long text, special characters).
|
||||
4. **Navigation** — Check all paths in/out. Breadcrumbs, back button, deep links, mobile menu.
|
||||
5. **States** — Check empty state, loading state, error state, full/overflow state.
|
||||
6. **Console** — Run `console --errors` after interactions. Any new JS errors or failed requests?
|
||||
6. **Console** — Print `CONSOLE_ERRORS=` after interactions. Any new JS errors or failed requests?
|
||||
7. **Responsiveness** — If relevant, check mobile and tablet viewports.
|
||||
8. **Auth boundaries** — What happens when logged out? Different user roles?
|
||||
8. **Auth boundaries** — Never sign the user out or switch accounts yourself. If the signed-out or other-role view matters, ask the user to sign out / switch in Aside and re-run the page scripts.
|
||||
|
||||
@@ -64,11 +64,10 @@
|
||||
**Repro Steps:**
|
||||
|
||||
1. Navigate to {URL}
|
||||
|
||||
|
||||
2. {Action}
|
||||
|
||||
3. **Observe:** {what goes wrong}
|
||||
|
||||
|
||||
|
||||
---
|
||||
|
||||
@@ -81,8 +80,8 @@
|
||||
### Before/After Evidence
|
||||
|
||||
#### ISSUE-NNN: {title}
|
||||
**Before:** 
|
||||
**After:** 
|
||||
**Before:**  — the Phase 5 evidence (`issue-NNN.png` for a static bug)
|
||||
**After:** 
|
||||
|
||||
---
|
||||
|
||||
|
||||
@@ -117,25 +117,32 @@ This is the **primary mode** for developers verifying their work. When the user
|
||||
- View/template/component files → which pages render them
|
||||
- Model/service files → which pages use those models (check controllers that reference them)
|
||||
- CSS/style files → which pages include those stylesheets
|
||||
- API endpoints → test them directly with \`$B js "await fetch('/api/...')"\`
|
||||
- API endpoints → call them with the session's own cookies from one \`aside repl\` script:
|
||||
\`\`\`bash
|
||||
aside repl '
|
||||
const pg = await openTab("<base-url>");
|
||||
const r = await fetch("<base-url>/api/...", { method: "GET" });
|
||||
console.log("API_STATUS=" + r.status);
|
||||
console.log("API_BODY_START"); console.log((await r.text()).slice(0, 4000)); console.log("API_BODY_END");
|
||||
await closeTab(pg); console.log("GSTACK_STEP_OK");
|
||||
'
|
||||
\`\`\`
|
||||
- Static pages (markdown, HTML) → navigate to them directly
|
||||
|
||||
**If no obvious pages/routes are identified from the diff:** Do not skip browser testing. The user invoked /qa because they want browser-based verification. Fall back to Quick mode — navigate to the homepage, follow the top 5 navigation targets, check console for errors, and test any interactive elements found. Backend, config, and infrastructure changes affect app behavior — always verify the app still works.
|
||||
|
||||
3. **Detect the running app** — check common local dev ports:
|
||||
3. **Detect the running app** — probe common local dev ports (no browser needed to find a port):
|
||||
\`\`\`bash
|
||||
$B goto http://localhost:3000 2>/dev/null && echo "Found app on :3000" || \\
|
||||
$B goto http://localhost:4000 2>/dev/null && echo "Found app on :4000" || \\
|
||||
$B goto http://localhost:8080 2>/dev/null && echo "Found app on :8080"
|
||||
for p in 3000 4000 8080; do curl -sI --max-time 3 "http://localhost:$p" >/dev/null 2>&1 && echo "Found app on :$p"; done
|
||||
\`\`\`
|
||||
If no local app is found, check for a staging/preview URL in the PR or environment. If nothing works, ask the user for the URL.
|
||||
Open the first URL that answers in Aside. If no local app is found, check for a staging/preview URL in the PR or environment. If nothing works, ask the user for the URL.
|
||||
|
||||
4. **Test each affected page/route:**
|
||||
- Navigate to the page
|
||||
- Navigate to the page (the Read-a-page script in Phase 3)
|
||||
- Take a screenshot
|
||||
- Check console for errors
|
||||
- Check console for errors (the \`CONSOLE_ERRORS=\` line)
|
||||
- If the change was interactive (forms, buttons, flows), test the interaction end-to-end
|
||||
- Use \`snapshot -D\` before and after actions to verify the change had the expected effect
|
||||
- Snapshot before acting and print the diff after (the Drive-a-flow script in Phase 5) to verify the change had the expected effect
|
||||
|
||||
5. **Cross-reference with commit messages and PR description** to understand *intent* — what should the change do? Verify it actually does that.
|
||||
|
||||
@@ -163,77 +170,87 @@ Run full mode, then load \`baseline.json\` from a previous run. Diff: which issu
|
||||
|
||||
### Phase 1: Initialize
|
||||
|
||||
1. Find browse binary (see Setup above)
|
||||
1. Confirm Aside is READY (see BROWSER SETUP above). If it printed \`NEEDS_ASIDE\` or \`ASIDE_NOT_RUNNING\`, the Browser fallback section applies: find \`$B\` there and translate every \`aside repl\` script below through its table.
|
||||
2. Create output directories
|
||||
3. Copy report template from \`qa/templates/qa-report-template.md\` to output dir
|
||||
4. Start timer for duration tracking
|
||||
|
||||
### Phase 2: Authenticate (if needed)
|
||||
|
||||
**If the user specified auth credentials:**
|
||||
Aside is the user's real browser, so the session is already signed in wherever the user is signed in. You never authenticate — the user does. In the fallback browser there is no session to inherit: import one with /setup-browser-cookies, or \`$B handoff\` for a human sign-in and \`$B resume\` when they're done.
|
||||
|
||||
\`\`\`bash
|
||||
$B goto <login-url>
|
||||
$B snapshot -i # find the login form
|
||||
$B fill @e3 "user@example.com"
|
||||
$B fill @e4 "[REDACTED]" # NEVER include real passwords in report
|
||||
$B click @e5 # submit
|
||||
$B snapshot -D # verify login succeeded
|
||||
\`\`\`
|
||||
**If a sign-in wall appears:** stop and tell the user: "Sign in to <origin> in Aside yourself (open it in a new Aside tab), then tell me you're done." Then re-run the step — the browser's cookies now apply. Never type passwords, one-time codes, or payment details, and never read or print cookies, tokens, or localStorage.
|
||||
|
||||
**If the user provided a cookie file:**
|
||||
**If 2FA/OTP is required:** The user completes it in the Aside window, then tells you to continue.
|
||||
|
||||
\`\`\`bash
|
||||
$B cookie-import cookies.json
|
||||
$B goto <target-url>
|
||||
\`\`\`
|
||||
|
||||
**If 2FA/OTP is required:** Ask the user for the code and wait.
|
||||
|
||||
**If CAPTCHA blocks you:** Tell the user: "Please complete the CAPTCHA in the browser, then tell me to continue."
|
||||
**If CAPTCHA blocks you:** Tell the user: "Please complete the CAPTCHA in Aside, then tell me to continue."
|
||||
|
||||
### Phase 3: Orient
|
||||
|
||||
Get a map of the application:
|
||||
Get a map of the application. One script reads the landing page — console errors from load, the interactive snapshot tree, the visible text, and a screenshot:
|
||||
|
||||
\`\`\`bash
|
||||
$B goto <target-url>
|
||||
$B snapshot -i -a -o "$REPORT_DIR/screenshots/initial.png"
|
||||
$B links # map navigation structure
|
||||
$B console --errors # any errors on landing?
|
||||
aside repl '
|
||||
const HOOK = \`(() => { window.__gstackErrs = window.__gstackErrs || []; const oe = console.error; console.error = (...a) => { window.__gstackErrs.push(a.map(String).join(" ")); oe.apply(console, a); }; window.addEventListener("error", e => window.__gstackErrs.push("uncaught: " + e.message)); window.addEventListener("unhandledrejection", e => window.__gstackErrs.push("unhandledrejection: " + (e.reason && e.reason.message || e.reason))); })()\`;
|
||||
const pg = await openTab("about:blank");
|
||||
await pg._sendToTarget("Page.addScriptToEvaluateOnNewDocument", { source: HOOK });
|
||||
await pg.goto("<target-url>");
|
||||
const s = await snapshot(pg, { interactive: true });
|
||||
console.log(s.tree);
|
||||
console.log("CONSOLE_ERRORS=" + JSON.stringify(await pg.evaluate(() => window.__gstackErrs)));
|
||||
console.log("TEXT_START"); console.log((await pg.evaluate(() => document.body.innerText)).slice(0, 20000)); console.log("TEXT_END");
|
||||
await pg.screenshot({ path: "initial.jpg", type: "jpeg", quality: 60, fullPage: true });
|
||||
console.log("ASIDE_DIR=" + pwd);
|
||||
await closeTab(pg);
|
||||
console.log("GSTACK_STEP_OK");
|
||||
'
|
||||
\`\`\`
|
||||
|
||||
Then copy the screenshot out of the printed directory and show it: \`cp "<ASIDE_DIR>/initial.jpg" "$REPORT_DIR/screenshots/initial.jpg"\`, then Read it.
|
||||
|
||||
Map the navigation structure with the links script (same-origin, read-only HEAD requests with the user's cookies):
|
||||
|
||||
\`\`\`bash
|
||||
aside repl '
|
||||
const pg = await openTab("<target-url>");
|
||||
const links = await pg.evaluate(() => [...new Set([...document.querySelectorAll("a[href]")].map(a => a.href))].filter(h => h.startsWith(location.origin) && !/logout|signout|delete|remove|cancel|unsubscribe/i.test(h)));
|
||||
for (const l of links) { const r = await fetch(l, { method: "HEAD" }).catch(e => ({ status: "ERR " + e.message })); console.log("LINK", r.status, l); }
|
||||
await closeTab(pg); console.log("GSTACK_STEP_OK");
|
||||
'
|
||||
\`\`\`
|
||||
|
||||
Every \`LINK\` line with a 4xx/5xx or \`ERR\` status is a broken link for the Links score.
|
||||
|
||||
**Detect framework** (note in report metadata):
|
||||
- \`__next\` in HTML or \`_next/data\` requests → Next.js
|
||||
- \`csrf-token\` meta tag → Rails
|
||||
- \`wp-content\` in URLs → WordPress
|
||||
- Client-side routing with no page reloads → SPA
|
||||
|
||||
**For SPAs:** The \`links\` command may return few results because navigation is client-side. Use \`snapshot -i\` to find nav elements (buttons, menu items) instead.
|
||||
**For SPAs:** The links script may return few results because navigation is client-side. Use \`snapshot(pg, { interactive: true })\` to find nav elements (buttons, menu items) instead.
|
||||
|
||||
### Phase 4: Explore
|
||||
|
||||
Visit pages systematically. At each page:
|
||||
|
||||
\`\`\`bash
|
||||
$B goto <page-url>
|
||||
$B snapshot -i -a -o "$REPORT_DIR/screenshots/page-name.png"
|
||||
$B console --errors
|
||||
\`\`\`
|
||||
Visit pages systematically. At each page, run the Read-a-page script from Phase 3 against the page URL with \`page-<name>.jpg\` as the screenshot path, copy it into \`$REPORT_DIR/screenshots/\`, and Read it.
|
||||
|
||||
Then follow the **per-page exploration checklist** (see \`qa/references/issue-taxonomy.md\`):
|
||||
|
||||
1. **Visual scan** — Look at the annotated screenshot for layout issues
|
||||
1. **Visual scan** — Look at the screenshot for layout issues (use the annotated-screenshot script when you need ref labels on the page)
|
||||
2. **Interactive elements** — Click buttons, links, controls. Do they work?
|
||||
3. **Forms** — Fill and submit. Test empty, invalid, edge cases
|
||||
4. **Navigation** — Check all paths in and out
|
||||
5. **States** — Empty state, loading, error, overflow
|
||||
6. **Console** — Any new JS errors after interactions?
|
||||
7. **Responsiveness** — Check mobile viewport if relevant:
|
||||
6. **Console** — Any new JS errors after interactions? Print \`CONSOLE_ERRORS=\` after every action
|
||||
7. **Responsiveness** — Check the mobile viewport if relevant:
|
||||
\`\`\`bash
|
||||
$B viewport 375x812
|
||||
$B screenshot "$REPORT_DIR/screenshots/page-mobile.png"
|
||||
$B viewport 1280x720
|
||||
aside repl '
|
||||
const pg = await openTab("<page-url>");
|
||||
await pg._sendToTarget("Emulation.setDeviceMetricsOverride", { width: 375, height: 812, deviceScaleFactor: 2, mobile: true });
|
||||
await sleep(300);
|
||||
await pg.screenshot({ path: "page-mobile.jpg", type: "jpeg", quality: 60, fullPage: true });
|
||||
await pg._sendToTarget("Emulation.clearDeviceMetricsOverride", {});
|
||||
console.log("ASIDE_DIR=" + pwd); await closeTab(pg); console.log("GSTACK_STEP_OK");
|
||||
'
|
||||
\`\`\`
|
||||
|
||||
**Depth judgment:** Spend more time on core features (homepage, dashboard, checkout, search) and less on secondary pages (about, terms, privacy).
|
||||
@@ -246,26 +263,47 @@ Document each issue **immediately when found** — don't batch them.
|
||||
|
||||
**Two evidence tiers:**
|
||||
|
||||
**Interactive bugs** (broken flows, dead buttons, form failures):
|
||||
**Interactive bugs** (broken flows, dead buttons, form failures) — one script per flow, because tabs close when the script ends:
|
||||
1. Take a screenshot before the action
|
||||
2. Perform the action
|
||||
3. Take a screenshot showing the result
|
||||
4. Use \`snapshot -D\` to show what changed
|
||||
4. Print the snapshot diff to show what changed
|
||||
5. Write repro steps referencing screenshots
|
||||
|
||||
\`\`\`bash
|
||||
$B screenshot "$REPORT_DIR/screenshots/issue-001-step-1.png"
|
||||
$B click @e5
|
||||
$B screenshot "$REPORT_DIR/screenshots/issue-001-result.png"
|
||||
$B snapshot -D
|
||||
aside repl '
|
||||
const HOOK = \`(() => { window.__gstackErrs = window.__gstackErrs || []; const oe = console.error; console.error = (...a) => { window.__gstackErrs.push(a.map(String).join(" ")); oe.apply(console, a); }; window.addEventListener("error", e => window.__gstackErrs.push("uncaught: " + e.message)); })()\`;
|
||||
const pg = await openTab("about:blank");
|
||||
await pg._sendToTarget("Page.addScriptToEvaluateOnNewDocument", { source: HOOK });
|
||||
await pg.goto("<page-url>");
|
||||
await snapshot(pg, { interactive: true }); // baseline for .diff; refs like e12 name the elements
|
||||
await pg.screenshot({ path: "issue-001-step-1.jpg", type: "jpeg", quality: 60 });
|
||||
await pg.locator("e12").click(); // or pg.fill("#email", "qa@example.com"), pg.getByRole("button", { name: "Save" }).click()
|
||||
await sleep(500); // or await pg.waitForSelector("#done"); await pg.waitForURL(/dashboard/)
|
||||
const s = await snapshot(pg);
|
||||
console.log("DIFF_START"); console.log(s.diff); console.log("DIFF_END");
|
||||
console.log("URL=" + pg.url());
|
||||
console.log("CONSOLE_ERRORS=" + JSON.stringify(await pg.evaluate(() => window.__gstackErrs)));
|
||||
await pg.screenshot({ path: "issue-001-result.jpg", type: "jpeg", quality: 60 });
|
||||
console.log("ASIDE_DIR=" + pwd);
|
||||
await closeTab(pg);
|
||||
console.log("GSTACK_STEP_OK");
|
||||
'
|
||||
\`\`\`
|
||||
|
||||
Copy both screenshots out of the printed \`ASIDE_DIR\` into \`$REPORT_DIR/screenshots/\` and Read them.
|
||||
|
||||
**Static bugs** (typos, layout issues, missing images):
|
||||
1. Take a single annotated screenshot showing the problem
|
||||
2. Describe what's wrong
|
||||
|
||||
\`\`\`bash
|
||||
$B snapshot -i -a -o "$REPORT_DIR/screenshots/issue-002.png"
|
||||
aside repl '
|
||||
const pg = await openTab("<page-url>");
|
||||
const a = await annotatedScreenshot(pg);
|
||||
await fs.writeFile(path.join(pwd, "issue-002.png"), Buffer.from(a.base64Image, "base64"));
|
||||
console.log("ASIDE_DIR=" + pwd); await closeTab(pg); console.log("GSTACK_STEP_OK");
|
||||
'
|
||||
\`\`\`
|
||||
|
||||
**Write each issue to the report immediately** using the template format from \`qa/templates/qa-report-template.md\`.
|
||||
@@ -356,7 +394,7 @@ Minimum 0 per category.
|
||||
- Check for mixed content warnings (common with WP)
|
||||
|
||||
### General SPA (React, Vue, Angular)
|
||||
- Use \`snapshot -i\` for navigation — \`links\` command misses client-side routes
|
||||
- Use \`snapshot(pg, { interactive: true })\` for navigation — the links script misses client-side routes
|
||||
- Check for stale state (navigate away and back — does data refresh?)
|
||||
- Test browser back/forward — does the app handle history correctly?
|
||||
- Check for memory leaks (monitor console after extended use)
|
||||
@@ -367,16 +405,17 @@ Minimum 0 per category.
|
||||
|
||||
1. **Repro is everything.** Every issue needs at least one screenshot. No exceptions.
|
||||
2. **Verify before documenting.** Retry the issue once to confirm it's reproducible, not a fluke.
|
||||
3. **Never include credentials.** Write \`[REDACTED]\` for passwords in repro steps.
|
||||
3. **Never include credentials.** You never type them — the user signs in inside Aside. Write \`[REDACTED]\` if a repro step has to mention one.
|
||||
4. **Write incrementally.** Append each issue to the report as you find it. Don't batch.
|
||||
5. **Never read source code.** Test as a user, not a developer.
|
||||
6. **Check console after every interaction.** JS errors that don't surface visually are still bugs.
|
||||
7. **Test like a user.** Use realistic data. Walk through complete workflows end-to-end.
|
||||
8. **Depth over breadth.** 5-10 well-documented issues with evidence > 20 vague descriptions.
|
||||
9. **Never delete output files.** Screenshots and reports accumulate — that's intentional.
|
||||
10. **Use \`snapshot -C\` for tricky UIs.** Finds clickable divs that the accessibility tree misses.
|
||||
11. **Show screenshots to the user.** After every \`$B screenshot\`, \`$B snapshot -a -o\`, or \`$B responsive\` command, use the Read tool on the output file(s) so the user can see them inline. For \`responsive\` (3 files), Read all three. This is critical — without it, screenshots are invisible to the user.
|
||||
12. **Never refuse to use the browser.** When the user invokes /qa or /qa-only, they are requesting browser-based testing. Never suggest evals, unit tests, or other alternatives as a substitute. Even if the diff appears to have no UI changes, backend changes affect app behavior — always open the browser and test.`;
|
||||
10. **Use \`annotatedScreenshot(pg)\` when the tree misses a clickable element.** Ref labels drawn on the page find clickable divs the accessibility tree skips; then click by ref or CSS selector.
|
||||
11. **Show screenshots to the user.** After every script that saves a screenshot, \`cp\` it out of the printed \`ASIDE_DIR\` into \`$REPORT_DIR/screenshots/\` and use the Read tool on the copied file so the user can see it inline. This is critical — without it, screenshots are invisible to the user.
|
||||
12. **Never refuse to use the browser.** When the user invokes /qa or /qa-only, they are requesting browser-based testing in Aside. Never suggest evals, unit tests, curl, or other alternatives as a substitute. Even if the diff appears to have no UI changes, backend changes affect app behavior — always open the app in the browser and test.
|
||||
13. **Mutating actions on a non-local target need consent.** Submitting, creating, deleting, purchasing, or changing settings on anything that is not LOCAL follows the "Invocation is consent to LOOK, not to ACT" rule in BROWSER SETUP — one AskUserQuestion per run, before the first such action.`;
|
||||
}
|
||||
|
||||
export function generateCoAuthorTrailer(ctx: TemplateContext): string {
|
||||
|
||||
@@ -9,6 +9,7 @@ import {
|
||||
copyDirSync, setupBrowseShims, logCost, recordE2E, dumpOutcomeDiagnostic,
|
||||
createEvalCollector, finalizeEvalCollector,
|
||||
} from './helpers/e2e-helpers';
|
||||
import { asideAvailable } from './helpers/aside-available';
|
||||
import { startTestServer } from '../browse/test/test-server';
|
||||
import { spawnSync } from 'child_process';
|
||||
import * as fs from 'fs';
|
||||
@@ -20,7 +21,25 @@ const evalCollector = createEvalCollector('e2e-qa-bugs');
|
||||
// --- B6/B7/B8: Planted-bug outcome evals ---
|
||||
|
||||
// Outcome evals also need ANTHROPIC_API_KEY for the LLM judge
|
||||
const describeOutcome = (evalsEnabled && hasApiKey) ? describe : describe.skip;
|
||||
// ...and a browser: a live Aside (primary) or a built browse/dist/browse
|
||||
// (fallback). Neither → skip, never fail.
|
||||
const describeOutcome = (evalsEnabled && hasApiKey && (asideAvailable() || fs.existsSync(browseBin))) ? describe : describe.skip;
|
||||
|
||||
/**
|
||||
* The BROWSER SETUP section qa/SKILL.md renders (Aside probe + browse fallback
|
||||
* + driving rules). The agent gets just this, not the 1500-line skill, so the
|
||||
* driver decision is the skill's own text, not the prompt's.
|
||||
*/
|
||||
function browserSetupSection(): string {
|
||||
const skill = fs.readFileSync(path.join(ROOT, 'qa', 'SKILL.md'), 'utf-8');
|
||||
const start = skill.indexOf('## BROWSER SETUP');
|
||||
// The Aside contract is followed by its own H2, '## Browser fallback: ...' — the
|
||||
// fixture must carry both so a run without Aside can take the $B path.
|
||||
const fallback = skill.indexOf('\n## Browser fallback', start + 3);
|
||||
const end = skill.indexOf('\n## ', (fallback > 0 ? fallback : start) + 3);
|
||||
if (start < 0 || end < 0) throw new Error('qa/SKILL.md: BROWSER SETUP section not found — regenerate with: bun run gen:skill-docs');
|
||||
return skill.slice(start, end);
|
||||
}
|
||||
|
||||
// Wrap describeOutcome with selection — skip if no planted-bug tests are selected
|
||||
const outcomeTestNames = ['qa-b6-static', 'qa-b7-spa', 'qa-b8-checkout'];
|
||||
@@ -59,20 +78,20 @@ let testServer: ReturnType<typeof startTestServer>;
|
||||
fs.mkdirSync(path.join(reportDir, 'screenshots'), { recursive: true });
|
||||
const reportPath = path.join(reportDir, 'qa-report.md');
|
||||
|
||||
// Direct bug-finding with browse. Keep prompt concise — no reading long SKILL.md docs.
|
||||
fs.writeFileSync(path.join(testWorkDir, 'BROWSER-SETUP.md'), browserSetupSection());
|
||||
|
||||
// Direct bug-finding. Keep prompt concise — no reading long SKILL.md docs.
|
||||
// "Write early, update later" pattern ensures report exists even if agent hits max turns.
|
||||
const targetUrl = `${testServer.url}/${fixture}`;
|
||||
const result = await runSkillTest({
|
||||
prompt: `Find bugs on this page: ${targetUrl}
|
||||
|
||||
Browser binary: B="${browseBin}"
|
||||
Browser: read BROWSER-SETUP.md in this directory and follow it exactly — it probes for Aside first and falls back to the gstack browse binary. If it falls back, the binary is at ${browseBin} (B="${browseBin}"). Do not look for any other browser. The target is LOCAL, so submitting its forms needs no consent question.
|
||||
|
||||
PHASE 1 — Quick scan (5 commands max):
|
||||
$B goto ${targetUrl}
|
||||
$B console --errors
|
||||
$B snapshot -i
|
||||
$B snapshot -c
|
||||
$B accessibility
|
||||
PHASE 1 — Quick scan (5 browser steps max):
|
||||
- Load ${targetUrl} and capture the console errors
|
||||
- Take an interactive snapshot (clickable/fillable elements) and read the page text
|
||||
- Accessibility pass: img elements without alt text, form controls without a label or aria-label
|
||||
|
||||
PHASE 2 — Write initial report to ${reportPath}:
|
||||
Write every bug you found so far. Format each as:
|
||||
@@ -80,12 +99,12 @@ Write every bug you found so far. Format each as:
|
||||
- Severity: high / medium / low
|
||||
- Evidence: what you observed
|
||||
|
||||
PHASE 3 — Interactive testing (targeted — max 15 commands):
|
||||
- Test email: type "user@" (no domain) and blur — does it validate?
|
||||
PHASE 3 — Interactive testing (targeted — max 15 browser steps):
|
||||
- Test email: fill "user@" (no domain) and blur — does it validate?
|
||||
- Test quantity: clear the field entirely — check the total display
|
||||
- Test credit card: type a 25-character string — check for overflow
|
||||
- Test credit card: fill a 25-character string — check for overflow
|
||||
- Submit the form with zip code empty — does it require zip?
|
||||
- Submit a valid form and run $B console --errors
|
||||
- Submit a valid form and capture the console errors again
|
||||
- After finding more bugs, UPDATE ${reportPath} with new findings
|
||||
|
||||
PHASE 4 — Finalize report:
|
||||
@@ -128,7 +147,7 @@ CRITICAL RULES:
|
||||
// Agent may have named it differently — find any .md in reportDir or testWorkDir
|
||||
for (const searchDir of [reportDir, testWorkDir]) {
|
||||
try {
|
||||
const mdFiles = fs.readdirSync(searchDir).filter(f => f.endsWith('.md'));
|
||||
const mdFiles = fs.readdirSync(searchDir).filter(f => f.endsWith('.md') && f !== 'BROWSER-SETUP.md');
|
||||
if (mdFiles.length > 0) {
|
||||
report = fs.readFileSync(path.join(searchDir, mdFiles[0]), 'utf-8');
|
||||
break;
|
||||
|
||||
@@ -2,11 +2,12 @@ import { describe, test, expect, beforeAll, afterAll } from 'bun:test';
|
||||
import { JUDGE_MS, CAPTURE_MS, CAPTURE_LONG_MS } from './helpers/eval-budgets';
|
||||
import { runSkillTest } from './helpers/session-runner';
|
||||
import {
|
||||
ROOT, browseBin, runId, evalsEnabled,
|
||||
ROOT, browseBin, runId, evalsEnabled, selectedTests,
|
||||
describeIfSelected, testConcurrentIfSelected,
|
||||
copyDirSync, setupBrowseShims, logCost, recordE2E,
|
||||
createEvalCollector, finalizeEvalCollector,
|
||||
} from './helpers/e2e-helpers';
|
||||
import { asideAvailable } from './helpers/aside-available';
|
||||
import { startTestServer } from '../browse/test/test-server';
|
||||
import { spawnSync } from 'child_process';
|
||||
import * as fs from 'fs';
|
||||
@@ -15,6 +16,18 @@ import * as os from 'os';
|
||||
|
||||
const evalCollector = createEvalCollector('e2e-qa-workflow');
|
||||
|
||||
// /qa and /qa-only drive the Aside browser first and fall back to the gstack
|
||||
// browse binary. The browser-driving describes need one of the two — a live
|
||||
// Aside or a built browse/dist/browse (CI builds it, so the Linux lane runs
|
||||
// the fallback path). Neither → skip, never fail. qa-bootstrap opens no
|
||||
// browser and is not gated.
|
||||
const browserSelected = evalsEnabled && (asideAvailable() || fs.existsSync(browseBin)) ? selectedTests : [];
|
||||
|
||||
// The skill's BROWSER SETUP decides Aside vs fallback; the prompt only tells the
|
||||
// agent where the fallback binary is (the hermetic HOME has no global install).
|
||||
const browserPrompt = (skillMd: string) =>
|
||||
`Follow the BROWSER SETUP section in ${skillMd} exactly: it probes for Aside first and falls back to the gstack browse binary. If it falls back, the browse binary is at ${browseBin} (B="${browseBin}"; find-browse is shimmed under browse/bin in this directory). Do not look for any other browser.`;
|
||||
|
||||
// --- B4: QA skill E2E ---
|
||||
|
||||
describeIfSelected('QA skill E2E', ['qa-quick'], () => {
|
||||
@@ -40,7 +53,7 @@ describeIfSelected('QA skill E2E', ['qa-quick'], () => {
|
||||
|
||||
testConcurrentIfSelected('qa-quick', async () => {
|
||||
const result = await runSkillTest({
|
||||
prompt: `B="${browseBin}"
|
||||
prompt: `${browserPrompt('qa/SKILL.md')}
|
||||
|
||||
The test server is already running at: ${testServer.url}
|
||||
Target page: ${testServer.url}/basic.html
|
||||
@@ -71,7 +84,7 @@ Write your report to ${qaDir}/qa-reports/qa-report.md`,
|
||||
// Accept error_max_turns — the agent doing thorough QA work is not a failure
|
||||
expect(['success', 'error_max_turns']).toContain(result.exitReason);
|
||||
}, CAPTURE_MS);
|
||||
});
|
||||
}, browserSelected);
|
||||
|
||||
// --- QA-Only E2E (report-only, no fixes) ---
|
||||
|
||||
@@ -112,9 +125,7 @@ describeIfSelected('QA-Only skill E2E', ['qa-only-no-fix'], () => {
|
||||
|
||||
testConcurrentIfSelected('qa-only-no-fix', async () => {
|
||||
const result = await runSkillTest({
|
||||
prompt: `IMPORTANT: The browse binary is already assigned below as B. Do NOT search for it or run the SKILL.md setup block — just use $B directly.
|
||||
|
||||
B="${browseBin}"
|
||||
prompt: `${browserPrompt('qa-only/SKILL.md')}
|
||||
|
||||
Read the file qa-only/SKILL.md for the QA-only workflow instructions.
|
||||
Skip the preamble bash block, lake intro, telemetry, and contributor mode sections — go straight to the QA workflow.
|
||||
@@ -158,7 +169,7 @@ Write your report to ${qaOnlyDir}/qa-reports/qa-only-report.md`,
|
||||
);
|
||||
expect(statusLines.filter((l: string) => l.startsWith(' M') || l.startsWith('M '))).toHaveLength(0);
|
||||
}, CAPTURE_MS);
|
||||
});
|
||||
}, browserSelected);
|
||||
|
||||
// --- QA Fix Loop E2E ---
|
||||
|
||||
@@ -233,7 +244,7 @@ describeIfSelected('QA Fix Loop E2E', ['qa-fix-loop'], () => {
|
||||
const qaFixUrl = `http://127.0.0.1:${qaFixServer!.port}`;
|
||||
|
||||
const result = await runSkillTest({
|
||||
prompt: `You have a browse binary at ${browseBin}. Assign it to B variable like: B="${browseBin}"
|
||||
prompt: `${browserPrompt('qa/SKILL.md')}
|
||||
|
||||
Read the file qa/SKILL.md for the QA workflow instructions.
|
||||
qa is a carved skill: when SKILL.md tells you to Read ~/.claude/skills/gstack/qa/sections/<file>, read qa/sections/<file> in this working directory instead (same content, local copy).
|
||||
@@ -273,7 +284,7 @@ This is a test+fix loop: find bugs, fix them in the source code, commit each fix
|
||||
const editCalls = result.toolCalls.filter(tc => tc.tool === 'Edit');
|
||||
expect(editCalls.length).toBeGreaterThan(0);
|
||||
}, CAPTURE_LONG_MS);
|
||||
});
|
||||
}, browserSelected);
|
||||
|
||||
// --- Test Bootstrap E2E ---
|
||||
|
||||
|
||||
Reference in New Issue
Block a user