mirror of
https://github.com/garrytan/gstack.git
synced 2026-09-16 09:55:29 +02:00
refactor(qa): /qa and /qa-only drive Aside, fall back to $B
QA_METHODOLOGY runs every phase as Aside scripts (orient, explore, document, re-test, mobile viewport via CDP emulation, links via HEAD fetch); the authenticate phase is 'you are already signed in'; a 13th rule requires consent before mutating actions on non-local targets; the fallback section translates each step onto $B. The qa E2E tests run on whichever engine is present. Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
This commit is contained in:
@@ -2,11 +2,12 @@ import { describe, test, expect, beforeAll, afterAll } from 'bun:test';
|
||||
import { JUDGE_MS, CAPTURE_MS, CAPTURE_LONG_MS } from './helpers/eval-budgets';
|
||||
import { runSkillTest } from './helpers/session-runner';
|
||||
import {
|
||||
ROOT, browseBin, runId, evalsEnabled,
|
||||
ROOT, browseBin, runId, evalsEnabled, selectedTests,
|
||||
describeIfSelected, testConcurrentIfSelected,
|
||||
copyDirSync, setupBrowseShims, logCost, recordE2E,
|
||||
createEvalCollector, finalizeEvalCollector,
|
||||
} from './helpers/e2e-helpers';
|
||||
import { asideAvailable } from './helpers/aside-available';
|
||||
import { startTestServer } from '../browse/test/test-server';
|
||||
import { spawnSync } from 'child_process';
|
||||
import * as fs from 'fs';
|
||||
@@ -15,6 +16,18 @@ import * as os from 'os';
|
||||
|
||||
const evalCollector = createEvalCollector('e2e-qa-workflow');
|
||||
|
||||
// /qa and /qa-only drive the Aside browser first and fall back to the gstack
|
||||
// browse binary. The browser-driving describes need one of the two — a live
|
||||
// Aside or a built browse/dist/browse (CI builds it, so the Linux lane runs
|
||||
// the fallback path). Neither → skip, never fail. qa-bootstrap opens no
|
||||
// browser and is not gated.
|
||||
const browserSelected = evalsEnabled && (asideAvailable() || fs.existsSync(browseBin)) ? selectedTests : [];
|
||||
|
||||
// The skill's BROWSER SETUP decides Aside vs fallback; the prompt only tells the
|
||||
// agent where the fallback binary is (the hermetic HOME has no global install).
|
||||
const browserPrompt = (skillMd: string) =>
|
||||
`Follow the BROWSER SETUP section in ${skillMd} exactly: it probes for Aside first and falls back to the gstack browse binary. If it falls back, the browse binary is at ${browseBin} (B="${browseBin}"; find-browse is shimmed under browse/bin in this directory). Do not look for any other browser.`;
|
||||
|
||||
// --- B4: QA skill E2E ---
|
||||
|
||||
describeIfSelected('QA skill E2E', ['qa-quick'], () => {
|
||||
@@ -40,7 +53,7 @@ describeIfSelected('QA skill E2E', ['qa-quick'], () => {
|
||||
|
||||
testConcurrentIfSelected('qa-quick', async () => {
|
||||
const result = await runSkillTest({
|
||||
prompt: `B="${browseBin}"
|
||||
prompt: `${browserPrompt('qa/SKILL.md')}
|
||||
|
||||
The test server is already running at: ${testServer.url}
|
||||
Target page: ${testServer.url}/basic.html
|
||||
@@ -71,7 +84,7 @@ Write your report to ${qaDir}/qa-reports/qa-report.md`,
|
||||
// Accept error_max_turns — the agent doing thorough QA work is not a failure
|
||||
expect(['success', 'error_max_turns']).toContain(result.exitReason);
|
||||
}, CAPTURE_MS);
|
||||
});
|
||||
}, browserSelected);
|
||||
|
||||
// --- QA-Only E2E (report-only, no fixes) ---
|
||||
|
||||
@@ -112,9 +125,7 @@ describeIfSelected('QA-Only skill E2E', ['qa-only-no-fix'], () => {
|
||||
|
||||
testConcurrentIfSelected('qa-only-no-fix', async () => {
|
||||
const result = await runSkillTest({
|
||||
prompt: `IMPORTANT: The browse binary is already assigned below as B. Do NOT search for it or run the SKILL.md setup block — just use $B directly.
|
||||
|
||||
B="${browseBin}"
|
||||
prompt: `${browserPrompt('qa-only/SKILL.md')}
|
||||
|
||||
Read the file qa-only/SKILL.md for the QA-only workflow instructions.
|
||||
Skip the preamble bash block, lake intro, telemetry, and contributor mode sections — go straight to the QA workflow.
|
||||
@@ -158,7 +169,7 @@ Write your report to ${qaOnlyDir}/qa-reports/qa-only-report.md`,
|
||||
);
|
||||
expect(statusLines.filter((l: string) => l.startsWith(' M') || l.startsWith('M '))).toHaveLength(0);
|
||||
}, CAPTURE_MS);
|
||||
});
|
||||
}, browserSelected);
|
||||
|
||||
// --- QA Fix Loop E2E ---
|
||||
|
||||
@@ -233,7 +244,7 @@ describeIfSelected('QA Fix Loop E2E', ['qa-fix-loop'], () => {
|
||||
const qaFixUrl = `http://127.0.0.1:${qaFixServer!.port}`;
|
||||
|
||||
const result = await runSkillTest({
|
||||
prompt: `You have a browse binary at ${browseBin}. Assign it to B variable like: B="${browseBin}"
|
||||
prompt: `${browserPrompt('qa/SKILL.md')}
|
||||
|
||||
Read the file qa/SKILL.md for the QA workflow instructions.
|
||||
qa is a carved skill: when SKILL.md tells you to Read ~/.claude/skills/gstack/qa/sections/<file>, read qa/sections/<file> in this working directory instead (same content, local copy).
|
||||
@@ -273,7 +284,7 @@ This is a test+fix loop: find bugs, fix them in the source code, commit each fix
|
||||
const editCalls = result.toolCalls.filter(tc => tc.tool === 'Edit');
|
||||
expect(editCalls.length).toBeGreaterThan(0);
|
||||
}, CAPTURE_LONG_MS);
|
||||
});
|
||||
}, browserSelected);
|
||||
|
||||
// --- Test Bootstrap E2E ---
|
||||
|
||||
|
||||
Reference in New Issue
Block a user