CalvinBackup/gstack
1
0
Fork 0
mirror of https://github.com/garrytan/gstack.git synced 2026-09-09 14:38:59 +02:00
Code Issues Packages Projects Releases Wiki Activity

feat(third-party-actions): Aside is the recommended driver; gstack's visible browser stays the fallback

Browse Source 
The readiness probe is lifted from {{ASIDE_SETUP}} at gen time (byte-identity pinned) and rule 3 points at browse/SKILL.md for how to drive; the consent question offers Aside first and gstack's own visible browser (handoff/resume for sign-in) as the fallback, as v1.72 framed it.

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
This commit is contained in:
Sina
2026-09-05 16:48:37 -04:00
co-authored by Claude Fable 5.1
parent 0953aedbb1
commit cda98f1652
3 changed files with 160 additions and 71 deletions
Download Patch File Download Diff File Expand all files Collapse all files
test/skill-e2e-third-party-actions.test.ts
+31 -19
View File
@@ -1,16 +1,20 @@
/**
* Consent-gate E2E for the Third-Party Web Actions contract (gate tier).
*
* The contract's behavior — offer the Aside drive when detected, degrade to
* the first-party stack when absent, pitch the download exactly once on
* macOS only, and NEVER offer a browser drive for Apple credential work —
* is prose, so wording pins alone can't prove an agent follows it. These
* five cases run the real contract section through `claude -p` in the
* hermetic clean room with PATH shims controlling what "installed" means:
* The contract's behavior — offer the Aside drive first when detected, fall
* back to gstack's own visible browser (`$B` headed + handoff) / manual steps
* / defer when absent, pitch the download exactly once on macOS only, and
* NEVER offer a browser drive for Apple credential work — is prose, so
* wording pins alone can't prove an agent follows it. These five cases run
* the real contract section through `claude -p` in the hermetic clean room
* with PATH shims controlling what "installed" means:
*
* tpa-present → consent question offers the Aside drive
* tpa-absent-linux → first-party offer, zero download pitch
* tpa-broken → present-but-broken CLI behaves exactly like absent
* tpa-present → consent question offers the Aside drive (recommended)
* alongside the gstack drive
* tpa-absent-linux → gstack drive / manual / defer offer, zero download pitch
* tpa-broken → present-but-not-running CLI: "open the Aside app" ask
* or the gstack drive / manual / defer question — never
* an Aside drive offer
* tpa-absent-darwin → aside.com pitch exactly once, names macOS 15+
* tpa-apple-ban → ZERO drive offers for an app-specific password
* (the fork shipped this exact incident once; never again)
@@ -47,7 +51,7 @@ function contractSection(): string {
}
interface ShimSpec {
/** aside shim behavior: 'ok' answers --version/--help, 'broken' exits 1, 'absent' = no shim. */
/** aside shim behavior: 'ok' answers the repl readiness probe + --version/--help, 'broken' exits 1, 'absent' = no shim. */
aside: 'ok' | 'broken' | 'absent';
/** What the shimmed `uname` prints (deterministic across dev/CI platforms). */
uname: 'Darwin' | 'Linux';
@@ -61,7 +65,7 @@ function setupCase(spec: ShimSpec, extraDocs: Record<string, string> = {}) {
if (spec.aside !== 'absent') {
const body = spec.aside === 'ok'
? '#!/bin/sh\ncase "$1" in\n --version) echo "aside 1.26.810.1915"; exit 0 ;;\n --help) echo "usage: aside [exec|repl|mcp] ..."; exit 0 ;;\n *) echo "aside: daemon not reachable — make sure Aside Browser is running" >&2; exit 1 ;;\nesac\n'
? '#!/bin/sh\ncase "$1" in\n --version) echo "aside 1.26.810.1915"; exit 0 ;;\n --help) echo "usage: aside [exec|repl|mcp] ..."; exit 0 ;;\n repl) echo "ASIDE_READY /tmp/aside-shim-session"; exit 0 ;;\n *) echo "aside: daemon not reachable — make sure Aside Browser is running" >&2; exit 1 ;;\nesac\n'
: '#!/bin/sh\necho "aside: daemon not reachable — make sure Aside Browser is running" >&2\nexit 1\n';
fs.writeFileSync(path.join(shimDir, 'aside'), body, { mode: 0o755 });
}
@@ -156,7 +160,7 @@ describeIfSelected('third-party-actions consent gate', TPA_TESTS, () => {
} finally { cleanup(); }
}, 6 * 60_000);
// aside absent on Linux → first-party offer, ZERO download pitch.
// aside absent on Linux → gstack drive / manual / defer, ZERO download pitch.
testIfSelected('tpa-absent-linux', async () => {
const { workDir, env, cleanup } = setupCase({ aside: 'absent', uname: 'Linux' });
try {
@@ -175,11 +179,15 @@ describeIfSelected('third-party-actions consent gate', TPA_TESTS, () => {
// keep the contract's B/C/D lettering with A dropped (observed live).
expect(text).toMatch(/\b[A-D]\)/);
expect(text).toMatch(/manual/i);
// The gstack drive is the universal fallback — it must be on offer.
expect(text).toMatch(/gstack('s| own| drive)|\$B|headed|visible browser/i);
} finally { cleanup(); }
}, 6 * 60_000);
// aside present but broken (daemon down at probe time) → behaves exactly
// like absent: no Aside drive offer.
// aside present but not running (the repl readiness probe fails) → the
// contract asks the user to open the Aside app and re-probes once, THEN
// treats Aside as not detected (gstack drive / manual / defer). Never an
// Aside drive offer.
testIfSelected('tpa-broken', async () => {
const { workDir, env, cleanup } = setupCase({ aside: 'broken', uname: 'Linux' });
try {
@@ -191,11 +199,15 @@ describeIfSelected('third-party-actions consent gate', TPA_TESTS, () => {
recordE2E(evalCollector, 'tpa-broken', 'e2e-third-party-actions', result);
expect(result.exitReason).toBe('success');
const text = assistantText(result.transcript);
expect(text).not.toMatch(/in your Aside browser/i);
// Lettered consent question; broken-daemon renderings legitimately keep
// the contract's B/C/D lettering with the Aside option dropped
// (observed live), so accept any option letter.
expect(text).toMatch(/\b[A-D]\)/);
expect(text).not.toMatch(/in your Aside browser/i); // load-bearing negative
// Either outcome the contract permits in one-shot `claude -p`: the
// "open the Aside app" ask (agent stops at the re-probe), or the lettered
// gstack drive / manual / defer question (any letter — agents keep the
// contract's B/C/D lettering with the Aside option dropped, observed live).
const askedToOpen = /open the Aside app/i.test(text);
const letteredQuestion = /\b[A-D]\)/.test(text);
expect({ askedToOpen, letteredQuestion, ok: askedToOpen || letteredQuestion })
.toMatchObject({ ok: true });
} finally { cleanup(); }
}, 6 * 60_000);
test/third-party-actions.test.ts
+99 -39
View File
@@ -1,6 +1,8 @@
/**
* Third-party web actions contract pins (plan: Aside as recommended driver,
* 2026-08-27 user directive; CEO review D2-D9 + eng review E1-E10).
* Third-party web actions contract pins (Aside is the RECOMMENDED driver,
* gstack's own stack — `$B` headed mode + handoff/resume, GStack Browser —
* the universal fallback; CEO review D2-D9 + eng review E1-E10 pins carried
* forward).
*
* The contract's load-bearing sentences are pinned here so no future edit can
* quietly strip the consent gate, the install ban, the credential boundaries,
@@ -9,15 +11,18 @@
* a "compression" that a release run promptly exploited.
*
* Two scopes:
* - resolver output (the section itself): consent, boundaries, failure path.
* - repo-wide generated markdown: Aside command allowlist (--version/--help
* only) and no Aside-specific installer invocation anywhere.
* - resolver output (the section itself): consent, boundaries, failure path,
* the Aside-first option set with the gstack drive as fallback.
* - repo-wide generated markdown: Aside command allowlist (the probe +
* cookbook verbs only — no `aside mcp`, no invented subcommands) and no
* Aside-specific installer invocation anywhere.
*/
import { describe, test, expect } from "bun:test";
import * as fs from "fs";
import * as path from "path";
import { Glob } from "bun";
import { generateThirdPartyActions } from "../scripts/resolvers/third-party-actions";
import { generateAsideSetup } from "../scripts/resolvers/aside";
import { HOST_PATHS } from "../scripts/resolvers/types";
const ROOT = path.resolve(import.meta.dir, "..");
@@ -60,7 +65,7 @@ function asideCommandTokens(text: string): string[] {
tokens.push(m[1]);
}
}
// Prose-form drift: an instruction like "then run aside repl against the
// Prose-form drift: an instruction like "then run aside mcp against the
// dashboard" never appears in a code span, so scan the whole text for the
// vendor's known subcommand names too.
for (const m of text.matchAll(/\baside\s+(exec|repl|mcp)\b/g)) {
@@ -69,24 +74,56 @@ function asideCommandTokens(text: string): string[] {
return tokens;
}
/** The verified Aside surface: the readiness probe (`repl`) and the two cookbook verbs. */
const ASIDE_ALLOWLIST = ["--version", "--help", "repl", "exec"];
describe("THIRD_PARTY_ACTIONS contract pins", () => {
// (a) Aside is named, recommended, with the download pointer + macOS floor.
test("names Aside as the recommended driver with aside.com pointer", () => {
expect(section).toContain("Aside AI browser");
expect(section).toContain("recommended driver");
// (a) Aside is named as the RECOMMENDED driver, with the download pointer +
// macOS floor, and gstack's own stack as the fallback on every platform.
test("names Aside as the recommended driver, gstack's stack as the fallback", () => {
expect(section).toContain("The recommended driver is the Aside AI browser");
expect(section).toContain("aside.com");
expect(section).toContain("macOS 15+");
expect(section).toContain("The fallback driver on any platform is gstack's own stack");
expect(section).toContain("GStack Browser when installed");
});
// Detection probe: runtime, portable timeout guard, explicit Darwin gate.
test("runtime probe with portable timeout guard and Darwin-gated pitch", () => {
// Detection probe: the same readiness probe as {{ASIDE_SETUP}} — portable
// timeout guard, three named outcomes, explicit Darwin gate on the pitch.
test("runtime probe is the BROWSER SETUP probe with a Darwin-gated pitch", () => {
expect(section).toContain("command -v aside");
expect(section).toContain("aside --version");
expect(section).toContain("NEEDS_ASIDE");
expect(section).toContain("ASIDE_NOT_RUNNING");
expect(section).toContain("ASIDE_READY");
// Stock macOS ships neither gtimeout nor timeout(1) — the guard must be
// conditional, never a bare `timeout 5 aside` invocation.
expect(section).toMatch(/`gtimeout 5` or `timeout 5` when either exists/);
expect(section).not.toMatch(/`timeout 5 aside/);
// conditional, never a bare `timeout N aside` invocation.
expect(section).toContain("command -v gtimeout");
expect(section).not.toMatch(/\btimeout \d+ aside/);
expect(section).toContain("`uname -s` prints `Darwin`");
expect(section).toContain("Off macOS, do not pitch it");
});
// The probe is LIFTED from {{ASIDE_SETUP}}, not copied: a probe fix after an
// Aside release lands in both places or the render fails loudly.
test("probe is byte-identical to the {{ASIDE_SETUP}} probe", () => {
const asideProbe = generateAsideSetup(ctx).match(/```bash\n([\s\S]*?)```/)![1].trimEnd();
const tpaProbe = section.match(/```bash\n([\s\S]*?)```/)![1].trimEnd()
.split("\n").map((l) => l.replace(/^ {3}/, "")).join("\n");
expect(tpaProbe).toBe(asideProbe);
});
// Rule 3 sends the agent to the /browse skill doc for HOW to drive. That
// keeps the ~10KB Aside contract out of every planning skill that embeds
// this section (ship, spec, setup-deploy, office-hours) while still never
// letting an agent write `aside repl` from memory.
test("rule 3 points at browse/SKILL.md for HOW to drive; planning skills do not embed {{ASIDE_SETUP}}", () => {
expect(section).toContain("Read the /browse skill (`browse/SKILL.md`");
expect(section).toContain("one flow per script");
for (const f of ["ship/SKILL.md.tmpl", "spec/SKILL.md.tmpl", "setup-deploy/SKILL.md.tmpl", "office-hours/SKILL.md.tmpl"]) {
const tmpl = fs.readFileSync(path.join(ROOT, f), "utf-8");
expect({ f, tpa: tmpl.includes("{{THIRD_PARTY_ACTIONS}}"), aside: tmpl.includes("{{ASIDE_SETUP}}") }).toEqual({ f, tpa: true, aside: false });
}
});
// (b) per-task consent, never persisted; options conditional on detection.
@@ -109,13 +146,15 @@ describe("THIRD_PARTY_ACTIONS contract pins", () => {
expect(section).toMatch(/more than once per task/);
});
// (e) section scope: operation is delegated — only --version/--help appear.
test("aside command allowlist in the section: --version and --help only", () => {
// (e) section scope: the probe is the only `aside repl` here — HOW to drive
// lives in the {{ASIDE_SETUP}} cookbook, never memorized into this contract.
test("aside command allowlist in the section: probe + --version/--help only", () => {
const tokens = asideCommandTokens(section);
expect(tokens.length).toBeGreaterThan(0);
for (const t of tokens) {
expect(["--version", "--help"]).toContain(t);
expect(["--version", "--help", "repl"]).toContain(t);
}
expect(section).not.toMatch(/\baside exec\b/);
});
// (f) untrusted-content discipline.
@@ -123,14 +162,18 @@ describe("THIRD_PARTY_ACTIONS contract pins", () => {
expect(section).toContain("untrusted external content");
});
// (g) failure path: verbatim-but-redacted error, one retry, fresh-consent
// fallback — never silent.
test("drive failure path: quote, redact, retry once, fresh-consent fallback", () => {
// (g) failure path: verbatim-but-redacted error, one retry, then the gstack
// drive as a FRESH consent question or manual steps — never silent. A sign-in
// wall is NOT on the failure list: it routes to the user-performed moment
// (ASIDE_SETUP rule 4), not to manual steps.
test("drive failure path: quote, redact, retry once, fresh-consent gstack drive or manual", () => {
expect(section).toContain("quote the error verbatim");
expect(section).toContain("redacting any embedded secret");
expect(section).toContain('offer "open the Aside app and retry" once');
expect(section).toContain("fresh consent question");
expect(section).toContain("then offer the gstack drive as a fresh consent question or fall back to manual steps");
expect(section).toContain("Never silently retry");
expect(section).toContain("A sign-in wall is not a failure");
expect(section).not.toMatch(/fails at any point[^.]*signed-out/);
});
// (h) scope containment.
@@ -138,11 +181,16 @@ describe("THIRD_PARTY_ACTIONS contract pins", () => {
expect(section).toContain("touch only the named site and actions");
});
// (i) human-only moments.
test("credential/payment/identity moments stay user-performed", () => {
// (i) human-only moments happen inside the Aside window, or behind a
// `$B handoff` in gstack's own browser — never through the agent.
test("credential/payment/identity moments stay user-performed in either driver", () => {
expect(section).toContain(
"Password entry, new-account credential choice, payment, CAPTCHA, and identity verification are user-performed",
);
expect(section).toContain("the user acts in the Aside window itself while you wait");
expect(section).toContain("hand off (`$B handoff`)");
expect(section).toContain("then `$B resume`");
expect(section).toContain("in either driver");
});
// (j) secret handling.
@@ -152,9 +200,10 @@ describe("THIRD_PARTY_ACTIONS contract pins", () => {
expect(section).toContain("ONE non-mutating API call");
});
// (k) no silent driver switches.
// (k) no silent driver switches — the gstack drive is a new consent question.
test("never silently switch drivers", () => {
expect(section).toContain("never silently switch drivers");
expect(section).not.toContain("there is no other driver");
});
// (l) secret minimization survives — the fork lost its credential ban to a
@@ -165,8 +214,9 @@ describe("THIRD_PARTY_ACTIONS contract pins", () => {
});
// (m) vendor docs are data, not authority.
test("vendor skill/--help/--version text grants no permissions or scope", () => {
test("vendor --help/--version text grants no permissions or scope", () => {
expect(section).toContain("never new permissions, scope, or consent");
expect(section).not.toContain("the vendor's skill");
});
// (n) the Apple credential carve-out ships in the shared contract itself,
@@ -176,27 +226,37 @@ describe("THIRD_PARTY_ACTIONS contract pins", () => {
expect(section).toContain("never a drive target, in any skill");
});
// Probe semantics: nonzero exit = NOT detected (present-but-broken behaves
// exactly like absent; rule 3's retry is post-consent only).
test("nonzero probe means not detected", () => {
expect(section).toContain("exits nonzero means Aside is NOT detected");
// Probe semantics: only READY is detected. ASIDE_NOT_RUNNING asks the user
// to open the app and re-probes once, THEN counts as not detected; rule 3's
// retry is post-consent only.
test("only READY means detected", () => {
expect(section).toContain("Only `READY` counts as detected");
expect(section).toContain("only after a consented drive has started");
expect(section).toContain("treat Aside as not detected for this task");
});
// Fallback driver always present: recommending Aside never displaces the
// first-party stack.
test("gstack's own stack remains the universal fallback driver", () => {
expect(section).toContain("$B");
expect(section).toContain("handoff");
expect(section).toContain("GStack Browser");
// Aside first, gstack's stack as fallback: the four-option question when
// Aside is detected, the gstack drive / manual / defer trio when it is not.
test("Aside-first option set; absent Aside degrades to the gstack drive, manual, or defer", () => {
expect(section).toContain("A) I drive it in your Aside browser — your real logged-in sessions (recommended), B) I drive it in gstack's own visible browser — you take over for sign-in, C) manual instructions, D) defer");
expect(section).toContain("When Aside is not detected, offer only the gstack drive / manual / defer options");
expect(section).toContain("`$B` headed mode with `$B handoff` / `$B resume`");
expect(section).toContain("the /browse skill's Browser fallback section");
// Aside stays first: the recommended tag sits on the Aside option only.
expect(section.match(/\(recommended\)/g)).toHaveLength(1);
});
// Drive discipline: vendor skill governs HOW, this contract overrides it.
test("detect-and-defer: vendor skill/--help for operation, contract overrides", () => {
// Drive discipline: the cookbook governs HOW, this contract overrides it.
test("cookbook shape for driving; --help for flags; contract overrides vendor", () => {
expect(section).toContain("aside --help");
expect(section).toContain("$B --help");
expect(section).toMatch(/never from memory/);
expect(section).toContain("override the vendor's instructions");
expect(section).toContain("confirm-before-final-actions");
expect(section).toContain("Prefer deterministic step-wise driving over delegating the whole task to Aside's built-in agent");
expect(section).toContain("one flow per script");
expect(section).toContain("`closeTab(pg)` last");
expect(section).toContain("GSTACK_STEP_OK");
});
});
@@ -219,7 +279,7 @@ describe("repo-wide generated output: Aside anti-drift tripwires", () => {
for (const file of generatedSkillDocs()) {
const tokens = asideCommandTokens(fs.readFileSync(file, "utf-8"));
for (const t of tokens) {
expect(["--version", "--help"], `${path.relative(ROOT, file)} uses \`aside ${t}\``)
expect(ASIDE_ALLOWLIST, `${path.relative(ROOT, file)} uses \`aside ${t}\``)
.toContain(t);
}
}