feat: add provider-aware browser QA setup

Detect host-native browser tools before offering the isolated local Chromium fallback, add a common readiness fixture, harden managed browser startup, and verify standards installs expose one canonical QA skill.
This commit is contained in:
Sinabina
2026-07-20 16:01:24 -07:00
parent f14445bb00
commit d6ef673e4d
163 changed files with 5272 additions and 990 deletions
@@ -0,0 +1,94 @@
import { afterEach, describe, expect, test } from "bun:test";
import fs from "node:fs";
import path from "node:path";
import {
BROWSER_PROVIDER_CONTRACTS,
renderBrowserProviderContract,
} from "../scripts/gstack2/browser-provider-contract";
import { createReadinessServer } from "../runtime/browser-provider-smoke.mjs";
const ROOT = path.resolve(import.meta.dir, "..");
const TREES = ["plan", "design", "qa", "debug", "review", "ship"];
const fixtures: Array<ReturnType<typeof createReadinessServer>> = [];
afterEach(async () => {
await Promise.all(fixtures.splice(0).map((fixture) => fixture.stop()));
});
describe("browser provider setup contract", () => {
test("defines every verified host plus Gemini without inventing Kimi or Pi core automation", () => {
expect(BROWSER_PROVIDER_CONTRACTS.map((provider) => provider.id)).toEqual([
"claude",
"codex",
"gemini",
"cursor",
"github-copilot",
"openclaw",
"kimi",
"pi",
]);
expect(BROWSER_PROVIDER_CONTRACTS.map((provider) => provider.kind)).toEqual([
"native-extension",
"native-in-app",
"native-agent",
"native-mcp",
"native-in-app",
"native-plugin",
"no-native-automation",
"extension-only",
]);
const rendered = renderBrowserProviderContract();
expect(rendered).toContain("Never install an extension");
expect(rendered).toContain("Tool names or metadata alone are insufficient evidence");
expect(rendered).toContain("Inherited `CODEX_*`");
expect(rendered).toContain("Never report `ready` merely because");
expect(rendered).toContain("Gemini CLI has an experimental bundled `browser_agent`");
expect(rendered).toContain("current VS Code can expose built-in browser agent tools");
expect(rendered).toContain("OpenClaw includes a browser plugin");
expect(rendered).toContain("Do not describe `kimi web` as browser automation");
expect(rendered).toContain("Pi intentionally has no core interactive browser tool");
expect(rendered).toContain("never silently add a browser MCP or alternate backend");
expect(rendered).toContain("GStack's existing local Chromium/Playwright implementation remains the only bundled browser backend");
expect(rendered).toContain("Do not run `./setup`");
});
test("generates the same provider contract and dependency-free probe for all six skills", () => {
const expectedContract = `<!-- GENERATED by scripts/gstack2/generate-skill-tree.ts; do not edit. -->\n${renderBrowserProviderContract()}`;
const expectedProbe = fs.readFileSync(path.join(ROOT, "runtime", "browser-provider-smoke.mjs"));
for (const tree of TREES) {
expect(fs.readFileSync(path.join(ROOT, "skills", tree, "references", "BROWSER-PROVIDERS.md"), "utf8"))
.toBe(expectedContract);
expect(fs.readFileSync(path.join(ROOT, "skills", tree, "references", "support", "browser-provider-smoke.mjs")))
.toEqual(expectedProbe);
expect(fs.readFileSync(path.join(ROOT, "skills", tree, "references", "RUNTIME.md"), "utf8"))
.toContain("Before interactive browser work, read `references/BROWSER-PROVIDERS.md` in full");
}
});
test("readiness fixture proves navigation, protected interaction, and durable completion state", async () => {
const token = "a".repeat(48);
const fixture = createReadinessServer({ token });
fixtures.push(fixture);
const started = await fixture.start();
expect(started.baseUrl).toMatch(/^http:\/\/127\.0\.0\.1:\d+$/);
const page = await fetch(started.url);
expect(page.status).toBe(200);
const html = await page.text();
expect(html).toContain("GStack browser readiness");
expect(html).toContain('id="gstack-readiness-action"');
expect(html).toContain('id="gstack-readiness-status"');
const denied = await fetch(`${started.baseUrl}/proof?token=${"b".repeat(48)}`, { method: "POST" });
expect(denied.status).toBe(403);
const before = await fetch(`${started.baseUrl}/status?token=${token}`).then((response) => response.json());
expect(before).toEqual({ ok: true, completed: false });
const proof = await fetch(`${started.baseUrl}/proof?token=${token}`, { method: "POST" });
expect(proof.status).toBe(200);
expect(await proof.json()).toEqual({ ok: true, status: "READY" });
const after = await fetch(`${started.baseUrl}/status?token=${token}`).then((response) => response.json());
expect(after).toEqual({ ok: true, completed: true });
});
});
+4 -1
View File
@@ -62,6 +62,7 @@ describe('GStack 2 standard installer surface', () => {
expect(AGENT_MATRIX.map((entry) => entry.agent)).toEqual([
'claude-code',
'codex',
'kimi-code-cli',
'cursor',
'pi',
'openclaw',
@@ -105,7 +106,9 @@ describe('GStack 2 standard installer surface', () => {
expect(result.summary.passed).toBe(true);
expect(result.discovery).toMatchObject({ count: 6, names: [...PUBLIC_SKILLS], passed: true });
expect(result.installs).toHaveLength(AGENT_MATRIX.length * 2 + 2);
// Two default scopes per host plus project/global collision selection,
// the OpenClaw single-skill case, and the opt-in compatibility alias.
expect(result.installs).toHaveLength(AGENT_MATRIX.length * 2 + 4);
expect(result.removals).toHaveLength(2);
expect(fs.existsSync(output)).toBe(true);
}, 600_000);
+39 -2
View File
@@ -168,7 +168,7 @@ describe("GStack 2 managed runtime installer", () => {
const phases: string[] = [];
const run = async (_command: string, args: string[], options: { env?: Record<string, string>; superviseTree?: boolean; timeoutMs?: number } = {}) => {
if (!args[0]?.endsWith(path.join("node_modules", "playwright", "cli.js"))) return { code: 0, stdout: "", stderr: "" };
expect(args.slice(1)).toEqual(["install", "--no-shell", "chromium"]);
expect(args.slice(1)).toEqual(["install", "--only-shell", "chromium"]);
expect(options.superviseTree).toBe(true);
expect(options.timeoutMs).toBe(15 * 60_000);
const target = options.env?.PLAYWRIGHT_BROWSERS_PATH;
@@ -234,7 +234,7 @@ describe("GStack 2 managed runtime installer", () => {
capabilities: { browse: ".gstack-runtime-browsers/fresh/chrome" },
runCommand: async (_command: string, args: string[], options: { env?: Record<string, string> } = {}) => {
downloads += 1;
expect(args.slice(1)).toEqual(["install", "--no-shell", "chromium"]);
expect(args.slice(1)).toEqual(["install", "--only-shell", "chromium"]);
const target = options.env?.PLAYWRIGHT_BROWSERS_PATH;
if (!target) throw new Error("missing fixture browser destination");
await fs.mkdir(path.join(target, "fresh"), { recursive: true });
@@ -309,6 +309,43 @@ describe("GStack 2 managed runtime installer", () => {
});
});
test("browser-visible materializes full Chromium without also downloading the headless shell", async () => {
await withFixture(async ({ home }) => {
const installModes: string[] = [];
const result = await installManagedRuntime({
sourceDir: REPO_ROOT,
home,
version: "browser-visible-only",
capabilityIds: ["browser-visible"],
bunCommand: process.execPath,
runCommand: async (command: string, args: string[], options: { env?: Record<string, string> } = {}) => {
if (args[0] === "--eval" && args[1]?.includes("process.execPath")) {
return { code: 0, stdout: process.execPath, stderr: "" };
}
if (args[0]?.endsWith(path.join("node_modules", "playwright", "cli.js"))) {
installModes.push(args[2]);
expect(args.slice(1)).toEqual(["install", "--no-shell", "chromium"]);
const browserRoot = options.env?.PLAYWRIGHT_BROWSERS_PATH;
if (!browserRoot) throw new Error("fixture browser root missing");
await fs.mkdir(path.join(browserRoot, "chromium-fixture"), { recursive: true });
await fs.writeFile(path.join(browserRoot, "chromium-fixture", "chrome"), "fixture\n", { mode: 0o755 });
return { code: 0, stdout: "", stderr: "" };
}
if (args[0] === "--version" && (command === process.execPath || command.includes(".gstack-runtime-tools"))) {
return { code: 0, stdout: "1.3.14\n", stderr: "" };
}
return { code: 0, stdout: "", stderr: "" };
},
smokeTest: async () => {},
});
expect(installModes).toEqual(["--no-shell"]);
const bundleManifest = await readJson(path.join(result.path, ".gstack-bundle.json"));
expect(bundleManifest.selectedCapabilities).toEqual(["browser-visible"]);
expect(bundleManifest.runtimeComponents).toContain("browser-visible");
expect(bundleManifest.runtimeComponents).not.toContain("browser-headless");
}, { createDefaultSource: false });
}, FULL_RUNTIME_TEST_TIMEOUT_MS);
test("default capability builds never regenerate the Agent Skills tree", async () => {
const calls: Array<{ command: string; args: string[] }> = [];
await defaultBunBuilder({
+38 -31
View File
@@ -9,7 +9,10 @@ import { resolveRuntimePaths } from "../runtime/paths.js";
import { setupRuntime } from "../runtime/setup.js";
import { bashCandidates, resolveBashCommand } from "../runtime/tooling.js";
import {
BOOTSTRAP_SCHEMA_VERSION,
BOOTSTRAP_RUNTIME_VERSION,
CAPABILITY_COMPONENTS,
COMPONENT_DEPENDENCIES,
OFFICIAL_MANIFEST_URL,
main as bootstrapMain,
} from "../runtime/runtime-bootstrap.mjs";
@@ -19,6 +22,29 @@ function capture() {
return { stream: { write: (chunk: string) => { value += chunk; } }, value: () => value };
}
function officialManifestFixture(target: string, customize?: (component: string, artifact: Record<string, unknown>) => void) {
const components = Object.fromEntries(Object.keys(COMPONENT_DEPENDENCIES)
.filter((component) => component !== "ios" || target.startsWith("darwin-"))
.map((component) => {
const artifact: Record<string, unknown> = {
url: `https://github.com/time-attack/gstack/releases/download/v${BOOTSTRAP_RUNTIME_VERSION}/${component}.tar.gz`,
sha256: "0".repeat(64),
bytes: 8,
format: "tar.gz",
};
customize?.(component, artifact);
return [component, artifact];
}));
return {
schemaVersion: BOOTSTRAP_SCHEMA_VERSION,
version: BOOTSTRAP_RUNTIME_VERSION,
skillApi: "2.0",
capabilityComponents: CAPABILITY_COMPONENTS,
componentDependencies: COMPONENT_DEPENDENCIES,
targets: { [target]: { components } },
};
}
describe("GStack runtime setup UX", () => {
test("capability selection keeps the core and excludes unselected heavyweight surfaces", () => {
const surface = runtimeSurfaceForCapabilities(["browser"]);
@@ -221,7 +247,7 @@ describe("GStack runtime setup UX", () => {
}
});
test("doctor resolves the exact Chromium executable from the managed slot", async () => {
test("doctor launches the managed headless Chromium slot instead of requiring full Chromium", async () => {
const root = await fs.mkdtemp(path.join(os.tmpdir(), "gstack-doctor-browser-"));
const home = path.join(root, "home");
try {
@@ -231,15 +257,13 @@ describe("GStack runtime setup UX", () => {
const browserRoot = path.join(active, ".gstack-runtime-browsers");
const managedBun = path.join(active, ".gstack-runtime-tools", process.platform === "win32" ? "bun.exe" : "bun");
const playwright = path.join(active, "node_modules", "playwright");
const executable = path.join(browserRoot, "chromium-fixture", process.platform === "win32" ? "chrome.exe" : "chrome");
await fs.mkdir(path.dirname(executable), { recursive: true });
await fs.mkdir(path.join(browserRoot, "chromium-headless-shell-fixture"), { recursive: true });
await fs.mkdir(path.dirname(managedBun), { recursive: true });
await fs.mkdir(playwright, { recursive: true });
await fs.writeFile(executable, "fixture\n", { mode: 0o755 });
await fs.copyFile(process.execPath, managedBun);
if (process.platform !== "win32") await fs.chmod(managedBun, 0o755);
await fs.writeFile(path.join(playwright, "index.mjs"),
`export const chromium = { executablePath: () => ${JSON.stringify(executable)} };\n`);
`export const chromium = { launch: async ({ headless }) => { if (headless !== true) throw new Error("expected headless"); return { version: () => "fixture-chromium", close: async () => {} }; } };\n`);
await fs.writeFile(path.join(active, ".gstack-bundle.json"), JSON.stringify({
compatibility: { skillApi: "2.0" },
selectedCapabilities: ["browser"],
@@ -252,7 +276,7 @@ describe("GStack runtime setup UX", () => {
const report = await runDoctor({ home, cwd: root, nodeCommand: process.execPath });
expect(report.checks.find((check) => check.id === "capability:browser")).toMatchObject({
status: "pass",
details: { executable },
details: { browserRoot, version: "fixture-chromium" },
});
expect(report.checks.find((check) => check.id === "runtime-tool:bun")).toMatchObject({ status: "pass" });
} finally {
@@ -289,7 +313,6 @@ describe("GStack runtime setup UX", () => {
test("bootstrap refuses an artifact whose SHA-256 does not match the official manifest", async () => {
const output = capture();
const target = `${process.platform === "win32" ? "windows" : process.platform}-${process.arch}`;
const artifactUrl = `https://github.com/time-attack/gstack/releases/download/v${BOOTSTRAP_RUNTIME_VERSION}/fixture.tar.gz`;
let calls = 0;
const fetch_ = async (url: string) => {
calls += 1;
@@ -297,21 +320,16 @@ describe("GStack runtime setup UX", () => {
return {
ok: true,
url,
json: async () => ({
schemaVersion: 1,
version: BOOTSTRAP_RUNTIME_VERSION,
skillApi: "2.0",
artifacts: { [target]: { url: artifactUrl, sha256: "0".repeat(64), bytes: 8, format: "tar.gz" } },
}),
json: async () => officialManifestFixture(target),
};
}
return {
ok: true,
url: artifactUrl,
url,
arrayBuffer: async () => new TextEncoder().encode("tampered").buffer,
};
};
expect(await bootstrapMain(["install", "--capability", "browser"], {
expect(await bootstrapMain(["install", "--capability", "browser", "--yes"], {
stdout: output.stream,
stderr: output.stream,
fetch: fetch_,
@@ -352,9 +370,8 @@ describe("GStack runtime setup UX", () => {
test("declared Cosign bundles must bind the official release workflow identity", async () => {
const output = capture();
const target = `${process.platform === "win32" ? "windows" : process.platform}-${process.arch}`;
const artifactUrl = `https://github.com/time-attack/gstack/releases/download/v${BOOTSTRAP_RUNTIME_VERSION}/fixture.tar.gz`;
let calls = 0;
expect(await bootstrapMain(["install", "--capability", "browser"], {
expect(await bootstrapMain(["install", "--capability", "browser", "--yes"], {
stdout: output.stream,
stderr: output.stream,
fetch: async (url: string) => {
@@ -362,19 +379,9 @@ describe("GStack runtime setup UX", () => {
return {
ok: true,
url,
json: async () => ({
schemaVersion: 1,
version: BOOTSTRAP_RUNTIME_VERSION,
skillApi: "2.0",
artifacts: {
[target]: {
url: artifactUrl,
sha256: "0".repeat(64),
bytes: 8,
format: "tar.gz",
cosignBundleUrl: `${artifactUrl}.sigstore.json`,
},
},
json: async () => officialManifestFixture(target, (component, artifact) => {
if (component !== "core") return;
artifact.cosignBundleUrl = `${artifact.url}.sigstore.json`;
}),
};
},
@@ -396,7 +403,7 @@ describe("GStack runtime setup UX", () => {
process.env.BOOTSTRAP_TEST_LOG = log;
try {
expect(await bootstrapMain([
"install", "--source", root, "--capability", "pdf", "--home", path.join(root, "home"),
"install", "--source", root, "--capability", "pdf", "--home", path.join(root, "home"), "--yes",
], { stdout: output.stream, stderr: output.stream })).toBe(0);
} finally {
if (previous == null) delete process.env.BOOTSTRAP_TEST_LOG;