Files
gstack/browse/src/snapshot.ts
T
Garry Tanandcapy-ai[bot] 9d9d4c31bb v1.93.0.0 wave(browse): #3700 #3773 #3779 #3817 #3842 (#3858)
* feat(loadouts): Opus 5.5 lite loadout with explicit model detection (GSTA-161)

modules/loadouts.json gains the lite loadout (eleven coaching preamble
pieces dropped; upgrade-check and brain-sync kept) and a models map that
sends only claude-opus-5-5 (and its dated snapshots) to it. Detection is
explicit: Capy passes --model to the installer, Claude Code opts in with
./setup --loadout lite; anything unmapped, invalid or unreadable serves full.
The checkout always stays the full render.

Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>

* docs(loadouts): Opus 5.5 lite, detection order, Capy SKILL_ROOT, setup --loadout; troubleshooting anchors; changelog fragment (GSTA-161)

Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>

* fix(loadouts): claude_loadout read through CONFIG_BIN; capy skill context ceiling for the --model line (GSTA-161)

Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>

* fix: pre-landing review fixes (prune keeps a reused loadout folder, GSTACK_LOADOUT_SKILL_ROOT, claude_loadout guard test) (GSTA-161)

Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>

* fix: pre-landing review fixes (Capy skill passes the model as GSTACK_MODEL so an older installer ignores it) (GSTA-161)

Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>

* fix: pre-landing review fixes (prune only owned, unused folders; full root from the canonical Claude install; fallback reason and source) (GSTA-161)

Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>

* docs: loadout source example, install and resolve commands, Capy launch-file note (GSTA-161)

Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>

* feat(autoplan): --ceo-only runs the CEO review alone with zero questions (GST-537)

Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>

* fix: pre-landing review fixes

Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>

* test: classify autoplan-ceo-only in the Codex host metadata

Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>

* fix: select autoplan-ceo-only on the skills it runs; say it needs Claude Code

Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>

* test: run autoplan-ceo-only in the marathon lane

Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>

* docs: changelog entry for /autoplan --ceo-only

Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>

* test: count the autoplan-ceo-only paid file in the shard inventory

Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>

* feat: gstack-autoplan next --ceo-only runs the CEO review alone and ends reviewed (GST-537)

Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>

* fix: budget and close recover a budget-stopped CEO-only run; reviewed is at rest for checkpoints

Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>

* fix: CEO-only finish leaves owner-only cap items undecided, refuses marker phase flags and a plan at the report path, and replaces ceo-review.md atomically

Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>

* fix: the CEO-only report-path guard compares real paths and refuses at the finish when the report file is the plan

Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>

* docs: CEO_ONLY_CONFLICT covers a plan linked to the report path

Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>

* test(browse): failing-first device and touch tests (GSTA-227 F3, #3058)

Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>

* fix: the report-path guard keeps the lexical check, skips safe hard links and compares bigint file ids

Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>

* browse: per-session tabs for parallel agents; tab pins stop moving the shared active tab (#1961)

Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>

* browse: keep the server.ts tab-pin anchor comment for source guards

Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>

* feat(browse): per-tab phone emulation and touch input ($B device, tap, swipe, pinch, touch; #3058, GSTA-227 F3)

Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>

* test: copy lib/result-codes-autoplan.ts into the fixtures that copy the result-code tables, and keep it out of derived closures

Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>

* docs(qa, aside): touch pointer in /qa, Aside touch recipe verified on Aside 1.26, report Phones row (GSTA-227 F3, #3058)

Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>

* browse: drop the sessions paragraph from /browse to stay inside the parity cap

Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>

* changelog: F3 phone emulation and touch entry

Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>

* browse: hide the ps window in the session harness walk

Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>

* docs: the decisions.jsonl status list names auto_kept

Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>

* test(browse): session cap, LRU eviction, headed release and scoped auto-session rows

Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>

* browse: sessions keep tab <id> picks, sidebar claude stays on the user's tab, pinned rebuild and state load rebind the scope

Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>

* fix: pre-landing review fixes (touch cancel with no finger held is a no-op)

Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>

* fix: Step 11 review fixes (headed tab adoption, pinned viewport reset, CLI retry keeps --session, rebuild id remap, --json notice, evicted-session reset, device kept after annotated captures, render opts out of sessions, session: client ids reserved)

Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>

* fix: headed viewport auto keeps the user's active tab; render's closetab opts out of sessions

Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>

* docs: sessions, phone emulation and touch in BROWSER.md; session: client ids reserved; TODOS #1961/#3058 shipped

Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>

* test: copy lib/result-codes-browse.ts into the fixtures that copy the result-code modules

lib/result-codes.ts imports ./result-codes-browse, so fixtures with an explicit
lib list failed with 'Cannot find module'. pr-dependencies lists it with the other
reason-code modules as selection data. GSTA-227

Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>

* feat: gstack-clean reclaims per-repo .gstack scratch and logs, dry run by default (#2627)

Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>

* browse: name the Aside touch recipe's coordinates <tap-x>/<tap-y> with a numeric grammar

The generated-bash lint (INV-3) rejected the free-text <x>/<y> placeholders. GSTA-227

Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>

* fix(clean): leave hard-linked and unreadable logs alone; cut through a no-follow descriptor checked against the evaluated inode

Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>

* test(clean): tests that expect tmp removal see a quiet ps, so a codex elsewhere on the host cannot flake them

Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>

* fix(clean): an unlistable tmp/ skips its group; liveness asked once per group on apply; BROWSER.md says which logs reset at daemon start

Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>

* fix(clean): write output synchronously so --json reaches a pipe whole; test fixes for root, wrapper git and the ps stub

Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>

* fix(clean): every change rechecks the verified folder and the measured file; full writes; EPIPE ends output quietly

Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>

* test(clean): the early-close test checks the bin's own exit status

Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>

* docs: list gstack-clean among the standalone CLIs

Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>

* test(clean): copied-install fixtures and the dependency list carry lib/result-codes-clean.ts

The remote pregate failed 11 cases in gstack-doctor, plan-tune-cathedral and
timeline-stop-hook-slug: each copies an explicit list of lib/result-codes-*.ts
files into a temp install, and lib/result-codes.ts now imports
./result-codes-clean. Same lists GSTA-168 extended for result-codes-eval.

Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>

* chore: regenerate gstack-doctor-components.sh for the gstack-clean result codes

Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>

* qa: fold the touch pointer into the Phases sentence to fit the parity size cap after the main merge

qa's union was 95988 bytes (ratio 1.142 > maxSizeRatio 1.142, train 75). Now 95975 (1.1419). GSTA-227

Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>

* docs: #3058 troubleshooting 'Fixed in' lines name v1.92.10.0 after the main merge

GSTA-227

Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>

* test(clean): classify gstack-tasks rows files (tasks-*) as kept, after main added the producer

Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>

* docs: #3058 troubleshooting 'Fixed in' lines name v1.92.20.0 after the main merge

GSTA-227

Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>

* fix: pre-landing review fixes (restore the lib/result-codes-plan.ts ratchet row the base merge dropped)

Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>

* test(clean): the producer scan ends a name at a colon, so prose after a path names the file (train 78)

Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>

* docs: #3058 troubleshooting 'Fixed in' lines name v1.92.30.0 after the main merge

GSTA-227

Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>

* outside-voice-api: caller receipts, stop_reason, OpenAI max_completion_tokens

A caller can write its own egress receipt (sink, consent, payload class);
the default stays the outside-voice receipt, so existing callers send and
record the same bytes. The result carries the provider's stop reason, and
an OpenAI request gets max_completion_tokens only when the caller sets an
output limit (GST-537 PR 3).

Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>

* autoplan gate: item classes and the owner's answer provenance

Gate items carry item_class, ruleKept() names the keep-direction pick, and
answered decision rows record owner_mode, owner_named, rec_hash and
answered_role parsed from the owner's own words (GST-537 PR 3).

Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>

* autoplan: shadow-mode decider module, frozen prompt and spend accounting

The switch (off by default, fixed at init), the lease, one model call per
gate with a page retry and a truncation retry under one deadline, the
once-per-revision record with its journaled sha256, the 10% audit mark,
and shadow-labeled ledger rows that count against the cap, show as
shadow= on GATE_SPEND and stay out of the per-voice price (GST-537 PR 3).

Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>

* gstack-autoplan: shadow every published gate, the shadow export, config and docs

Every driver command that prints a gate records the shadow first and
checkpoints after it; answer waits for a live lease and records a dead
one retrospectively; gstack-autoplan shadow exports answered revisions
for taste-replay R2 (SHADOW_BEFORE_ANSWER otherwise); gstack-config
autoplan_shadow / autoplan_shadow_model; gstack-artifact validate checks
the record's hash (GST-537 PR 3).

Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>

* changelog: shadow mode for the CEO-only decider (GST-537 PR 3)

Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>

* autoplan shadow: /ship review fixes

- register test/fixtures/shadow-fake-model.ts in FREE_FIXTURES (free-fixtures inventory failed)
- withRunLock: count dead-owner reclaims apart from waitMs iterations, so a long wait can still reclaim
- buildPage: fill the decider template in one pass, so slot names inside the plan stay plan text
- revise regate: the shared publish step prints the GATE and ESCALATE lines once

Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>

* autoplan shadow: reply files never follow a link; the retry reserves against the current cap

- a model reply is written by temp file and rename, so a link at the reply path is replaced, not written through
- each shadow reservation reads the run's current spend cap, so a budget change during the call is not undone

Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>

* outside-voice-api: write the egress receipt under the caller's env

callModel resolved the receipt's state root from process.env even when the caller passed env, so the shadow
tests wrote to the real ~/.gstack (the free-suite home-containment guard caught it). The receipt now uses
req.env; the shadow tests give the fake model a private HOME, GSTACK_HOME and GSTACK_STATE_ROOT, and the receipt
test no longer sets process.env to pass.

Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>

* fix(autoplan): CEO-only runs leave main's open Critical/High finding items to the owner (no decision row)

Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>

* fix(autoplan): shadow pages carry main's finding items; a finding item is never ceo_eligible

After merging #3767 (main 90a170c), open Critical/High findings are gate items
(item_class finding). The shadow records one row per gate item, so its pages now
include them. A CEO-only run leaves those items to the owner, so their rows are
ceo_eligible false (rule_pick was already null). The shadow fixtures follow
main's gate (items, counts, "Do it" tokens); the two merge adapters follow
main's doNext(out, rest, price) and its per-voice reservation (--estimate-usd).

Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>

* docs: #3058 troubleshooting 'Fixed in' lines name v1.92.61.0 after the main merge

GSTA-227

Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>

* fix(investigate): drop the five-phase step procedure (GST-938)

Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>

* changelog: GST-938 fragment wording

Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>

* fix(investigate): keep the web-search sanitize guard; update BROWSER.md and docs/skills.md (review pass 1)

Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>

* changelog: GST-938 wording

Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>

* test(investigate): pin the web-search guard; changelog size (adversarial review)

Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>

* test(clean): classify restamp-staging.json, the restamp producer main gained after #3779's base (wave browse)

Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>

* fix: pre-landing review fixes (wave browse): scoped newtab keeps a concurrent tab switch; shadow validate catches a deleted record; a failed shadow publish is journaled

Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>

* fix: a failed shadow publish journals under the run lock and never throws (wave browse review pass 2)

Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>

* docs(unattended): shadow publish failures and the missing-record validate check (wave browse docs audit)

Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>

* changelog: wave browse integrator fixes fragment

Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>

* fix: shadow reader dedupes by run folder, so two runs started in the same second both count (wave browse CI free-suite 16)

Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>

* chore: regenerate doctor result anchors in generator order (wave browse rebuild without #3818)

Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>

* changelog: name the wave fragment after the rebuilt wave

Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>

---------

Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>
2026-11-02 14:49:46 +00:00

660 lines
27 KiB
TypeScript

/**
* Snapshot command — accessibility tree with ref-based element selection
*
* Architecture (Locator map — no DOM mutation):
* 1. page.locator(scope).ariaSnapshot() → YAML-like accessibility tree
* 2. Parse tree, assign refs @e1, @e2, ...
* 3. Build Playwright Locator for each ref (getByRole + nth)
* 4. Store Map<string, Locator> on BrowserManager
* 5. Return compact text output with refs prepended
*
* Extended features:
* --diff / -D: Compare against last snapshot, return unified diff
* --annotate / -a: Screenshot with overlay boxes at each @ref
* --output / -o: Output path for annotated screenshot
* -C / --cursor-interactive: Scan for cursor:pointer/onclick/tabindex elements
*
* Later: "click @e3" → look up Locator → locator.click()
*/
import type { Page, Frame, Locator } from 'playwright';
import type { TabSession, RefEntry } from './tab-session';
import * as Diff from 'diff';
import { TEMP_DIR, isPathWithin } from './platform';
import { escapeEnvelopeSentinels } from './content-security';
import { stripLoneSurrogates } from './sanitize';
import { guardScreenshotPath } from './screenshot-size-guard';
import { reassertDevice } from './device-emulation';
import { SNAPSHOT_FLAGS, type SnapshotOptions } from './snapshot-flags';
export { SNAPSHOT_FLAGS } from './snapshot-flags';
// Roles considered "interactive" for the -i flag
const INTERACTIVE_ROLES = new Set([
'button', 'link', 'textbox', 'checkbox', 'radio', 'combobox',
'listbox', 'menuitem', 'menuitemcheckbox', 'menuitemradio',
'option', 'searchbox', 'slider', 'spinbutton', 'switch', 'tab',
'treeitem',
]);
interface ParsedNode {
indent: number;
role: string;
name: string | null;
props: string; // e.g., "[level=1]"
children: string; // inline text content after ":"
rawLine: string;
}
/**
* Parse CLI args into SnapshotOptions — driven by SNAPSHOT_FLAGS metadata.
*/
export function parseSnapshotArgs(args: string[]): SnapshotOptions {
const opts: SnapshotOptions = {};
for (let i = 0; i < args.length; i++) {
const flag = SNAPSHOT_FLAGS.find(f => f.short === args[i] || f.long === args[i]);
if (!flag) throw new Error(`Unknown snapshot flag: ${args[i]}`);
if (flag.takesValue) {
const value = args[++i];
if (!value) throw new Error(`Usage: snapshot ${flag.short} <value>`);
if (flag.optionKey === 'depth') {
(opts as any)[flag.optionKey] = parseInt(value, 10);
if (isNaN(opts.depth!)) throw new Error('Usage: snapshot -d <number>');
} else {
(opts as any)[flag.optionKey] = value;
}
} else {
(opts as any)[flag.optionKey] = true;
}
}
return opts;
}
/**
* Parse one line of ariaSnapshot output.
*
* Format examples:
* - heading "Test" [level=1]
* - link "Link A":
* - /url: /a
* - textbox "Name"
* - paragraph: Some text
* - combobox "Role":
*/
function parseLine(line: string): ParsedNode | null {
// Match: (indent)(- )(role)( "name")?( [props])?(: inline)?
const match = line.match(/^(\s*)-\s+(\w+)(?:\s+"([^"]*)")?(?:\s+(\[.*?\]))?\s*(?::\s*(.*))?$/);
if (!match) {
// Skip metadata lines like "- /url: /a"
return null;
}
return {
indent: match[1].length,
role: match[2],
name: match[3] ?? null,
props: match[4] || '',
children: match[5]?.trim() || '',
rawLine: line,
};
}
/**
* Take an accessibility snapshot and build the ref map.
*/
export async function handleSnapshot(
args: string[],
session: TabSession,
securityOpts?: { splitForScoped?: boolean },
): Promise<string> {
const opts = parseSnapshotArgs(args);
const page = session.getPage();
// Frame-aware target for accessibility tree
const target = session.getActiveFrameOrPage();
const inFrame = session.getFrame() !== null;
// Get accessibility tree via ariaSnapshot
let rootLocator: Locator;
if (opts.selector) {
rootLocator = target.locator(opts.selector);
const count = await rootLocator.count();
if (count === 0) throw new Error(`Selector not found: ${opts.selector}`);
} else {
rootLocator = target.locator('body');
}
const ariaText = await rootLocator.ariaSnapshot();
if (!ariaText || ariaText.trim().length === 0) {
session.setRefMap(new Map());
return '(no accessible elements found)';
}
// Parse the ariaSnapshot output
const lines = ariaText.split('\n');
const refMap = new Map<string, RefEntry>();
const output: string[] = [];
let refCounter = 1;
// Track role+name occurrences for nth() disambiguation
const roleNameCounts = new Map<string, number>();
const roleNameSeen = new Map<string, number>();
// First pass: count role+name pairs for disambiguation
for (const line of lines) {
const node = parseLine(line);
if (!node) continue;
const key = `${node.role}:${node.name || ''}`;
roleNameCounts.set(key, (roleNameCounts.get(key) || 0) + 1);
}
// Second pass: assign refs and build locators
for (const line of lines) {
const node = parseLine(line);
if (!node) continue;
const depth = Math.floor(node.indent / 2);
const isInteractive = INTERACTIVE_ROLES.has(node.role);
// Depth filter
if (opts.depth !== undefined && depth > opts.depth) continue;
// Interactive filter: skip non-interactive but still count for locator indices
if (opts.interactive && !isInteractive) {
// Still track for nth() counts
const key = `${node.role}:${node.name || ''}`;
roleNameSeen.set(key, (roleNameSeen.get(key) || 0) + 1);
continue;
}
// Compact filter: skip elements with no name and no inline content that aren't interactive
if (opts.compact && !isInteractive && !node.name && !node.children) continue;
// Assign ref
const ref = `e${refCounter++}`;
const indent = ' '.repeat(depth);
// Build Playwright locator
const key = `${node.role}:${node.name || ''}`;
const seenIndex = roleNameSeen.get(key) || 0;
roleNameSeen.set(key, seenIndex + 1);
const totalCount = roleNameCounts.get(key) || 1;
let locator: Locator;
if (opts.selector) {
locator = target.locator(opts.selector).getByRole(node.role as any, {
name: node.name || undefined,
});
} else {
locator = target.getByRole(node.role as any, {
name: node.name || undefined,
});
}
// Disambiguate with nth() if multiple elements share role+name
if (totalCount > 1) {
locator = locator.nth(seenIndex);
}
refMap.set(ref, { locator, role: node.role, name: node.name || '' });
// Format output line
let outputLine = `${indent}@${ref} [${node.role}]`;
if (node.name) outputLine += ` "${node.name}"`;
if (node.props) outputLine += ` ${node.props}`;
if (node.children) outputLine += `: ${node.children}`;
output.push(outputLine);
}
// ─── Cursor-interactive scan (-C, or auto with -i) ────────
// Auto-enable cursor scan when interactive mode is on — agents asking for
// interactive elements should always see clickable non-ARIA items too.
if (opts.interactive && !opts.cursorInteractive) {
opts.cursorInteractive = true;
}
if (opts.cursorInteractive) {
try {
const cursorElements = await target.evaluate(() => {
const STANDARD_INTERACTIVE = new Set([
'A', 'BUTTON', 'INPUT', 'SELECT', 'TEXTAREA', 'SUMMARY', 'DETAILS',
]);
const results: Array<{ selector: string; text: string; reason: string }> = [];
const allElements = document.querySelectorAll('*');
for (const el of allElements) {
// Skip standard interactive elements (already in ARIA tree)
if (STANDARD_INTERACTIVE.has(el.tagName)) continue;
// Skip hidden elements
if (!(el as HTMLElement).offsetParent && el.tagName !== 'BODY') continue;
const style = getComputedStyle(el);
const hasCursorPointer = style.cursor === 'pointer';
const hasOnclick = el.hasAttribute('onclick');
const hasTabindex = el.hasAttribute('tabindex') && parseInt(el.getAttribute('tabindex')!, 10) >= 0;
const hasRole = el.hasAttribute('role');
// Check if element is inside a floating container (portal/popover/dropdown)
const isInFloating = (() => {
let parent: Element | null = el;
while (parent && parent !== document.documentElement) {
const pStyle = getComputedStyle(parent);
const isFloating = (pStyle.position === 'fixed' || pStyle.position === 'absolute') &&
parseInt(pStyle.zIndex || '0', 10) >= 10;
const hasPortalAttr = parent.hasAttribute('data-floating-ui-portal') ||
parent.hasAttribute('data-radix-popper-content-wrapper') ||
parent.hasAttribute('data-radix-portal') ||
parent.hasAttribute('data-popper-placement') ||
parent.getAttribute('role') === 'listbox' ||
parent.getAttribute('role') === 'menu';
if (isFloating || hasPortalAttr) return true;
parent = parent.parentElement;
}
return false;
})();
if (!hasCursorPointer && !hasOnclick && !hasTabindex) {
// For elements inside floating containers, also check for role="option"/"menuitem"
if (isInFloating && hasRole) {
const role = el.getAttribute('role');
if (role !== 'option' && role !== 'menuitem' && role !== 'menuitemcheckbox' && role !== 'menuitemradio') continue;
} else {
continue;
}
}
// Skip elements with ARIA roles UNLESS they're inside a floating container
// (floating container items may be missed by the accessibility tree)
if (hasRole && !isInFloating) continue;
// Build deterministic nth-child CSS path
const parts: string[] = [];
let current: Element | null = el;
while (current && current !== document.documentElement) {
const parent: Element | null = current.parentElement;
if (!parent) break;
const siblings = [...parent.children];
const index = siblings.indexOf(current) + 1;
parts.unshift(`${current.tagName.toLowerCase()}:nth-child(${index})`);
current = parent;
}
const selector = parts.join(' > ');
const text = (el as HTMLElement).innerText?.trim().slice(0, 80) || el.tagName.toLowerCase();
const reasons: string[] = [];
if (isInFloating) reasons.push('popover-child');
if (hasCursorPointer) reasons.push('cursor:pointer');
if (hasOnclick) reasons.push('onclick');
if (hasTabindex) reasons.push(`tabindex=${el.getAttribute('tabindex')}`);
if (hasRole) reasons.push(`role=${el.getAttribute('role')}`);
results.push({ selector, text, reason: reasons.join(', ') });
}
return results;
});
if (cursorElements.length > 0) {
output.push('');
output.push('── cursor-interactive (not in ARIA tree) ──');
let cRefCounter = 1;
for (const elem of cursorElements) {
const ref = `c${cRefCounter++}`;
const locator = target.locator(elem.selector);
refMap.set(ref, { locator, role: 'cursor-interactive', name: elem.text });
output.push(`@${ref} [${elem.reason}] "${elem.text}"`);
}
}
} catch (err: any) {
// Cursor scan fails on pages with strict CSP or when page has navigated
if (!err?.message?.includes('Execution context') && !err?.message?.includes('closed') && !err?.message?.includes('Target') && !err?.message?.includes('Content Security')) throw err;
output.push('');
output.push('(cursor scan failed — CSP restriction)');
}
}
// Store ref map on BrowserManager
session.setRefMap(refMap);
if (output.length === 0) {
return '(no interactive elements found)';
}
const snapshotText = output.join('\n');
// `-o` only means something to the two modes that PRODUCE an image. Passed
// alone it used to be silently ignored: exit 0, no file, no explanation —
// which reads as "the screenshot feature is broken" rather than "you forgot a
// flag", and cost a real debugging session before anyone noticed. Kept as a
// variable so the diff-mode returns below (which bypass `output`) can carry
// it too — diff mode must not regress to the silent-ignore behavior.
const outputIgnoredWarning = (opts.outputPath && !opts.annotate && !opts.heatmap)
? `[warning] -o/--output was ignored: it names the output file for an annotated (-a/--annotate) or heatmap (-H/--heatmap) screenshot. For a plain screenshot use: browse screenshot ${opts.outputPath}`
: '';
if (outputIgnoredWarning) {
output.push(outputIgnoredWarning);
}
// ─── Annotated screenshot (-a) ────────────────────────────
if (opts.annotate) {
const screenshotPath = opts.outputPath || `${TEMP_DIR}/browse-annotated.png`;
// Validate output path — resolve symlinks to prevent symlink traversal attacks
{
const nodePath = require('path') as typeof import('path');
const nodeFs = require('fs') as typeof import('fs');
const absolute = nodePath.resolve(screenshotPath);
const safeDirs = [TEMP_DIR, process.cwd()].map((d: string) => {
try { return nodeFs.realpathSync(d); } catch (err: any) { if (err?.code !== 'ENOENT') throw err; return d; }
});
let realPath: string;
try {
realPath = nodeFs.realpathSync(absolute);
} catch (err: any) {
if (err.code === 'ENOENT') {
try {
const dir = nodeFs.realpathSync(nodePath.dirname(absolute));
realPath = nodePath.join(dir, nodePath.basename(absolute));
} catch (err2: any) {
if (err2?.code !== 'ENOENT') throw err2;
realPath = absolute;
}
} else {
throw new Error(`Cannot resolve real path: ${screenshotPath} (${err.code})`);
}
}
if (!safeDirs.some((dir: string) => isPathWithin(realPath, dir))) {
throw new Error(`Path must be within: ${safeDirs.join(', ')}`);
}
}
try {
// Inject overlay divs at each ref's bounding box
const boxes: Array<{ ref: string; box: { x: number; y: number; width: number; height: number } }> = [];
const ambiguousRefs: string[] = [];
const skippedRefs: string[] = [];
for (const [ref, entry] of refMap) {
try {
// A ref's locator can resolve to MORE than one element, and Playwright
// strict mode throws on that. It happens whenever a node has no
// accessible name: the locator degrades to `getByRole(role)` with no
// name filter, and the `.nth()` disambiguation above cannot help
// because its count comes from the FILTERED aria snapshot while
// getByRole matches the unfiltered DOM. Measured on a real page: the
// tree surfaced 2 unnamed paragraphs, the DOM had 9. Landmarks
// (banner/main/contentinfo) and paragraphs are correctly unnamed per
// ARIA, so this is the common case, not an edge.
//
// The exact nth-resolved locator stays the primary path; `.first()`
// is the AMBIGUITY FALLBACK only, and every fallback use is counted
// so first-match annotation is never silent. Before this, ONE such
// ref aborted the entire annotated screenshot (see the catch below) —
// which silently cost /qa, /canary and /land-and-deploy the
// screenshots their reports reference.
let locator = entry.locator;
const matchCount = await locator.count();
if (matchCount > 1) {
ambiguousRefs.push(`@${ref}`);
locator = locator.first();
}
const box = await locator.boundingBox({ timeout: 1000 });
if (box) {
boxes.push({ ref: `@${ref}`, box });
} else {
skippedRefs.push(`@${ref}`);
}
} catch (err: any) {
// Element may be offscreen, hidden, or page navigated — skip.
//
// The allowlist is deliberately not exhaustive-by-message any more: a
// box we cannot measure is a box we do not draw, never a reason to
// lose every other annotation on the page. The heatmap path below has
// always used a bare `catch {}` for exactly this reason; annotate was
// the only path that could be killed by a single unmeasurable ref.
skippedRefs.push(`@${ref}`);
if (process.env.BROWSE_DEBUG) console.error(`[annotate] skipped @${ref}: ${err?.message?.split('\n')[0]}`);
}
}
await page.evaluate((boxes) => {
for (const { ref, box } of boxes) {
const overlay = document.createElement('div');
overlay.className = '__browse_annotation__';
overlay.style.cssText = `
position: absolute; top: ${box.y}px; left: ${box.x}px;
width: ${box.width}px; height: ${box.height}px;
border: 2px solid red; background: rgba(255,0,0,0.1);
pointer-events: none; z-index: 99999;
font-size: 10px; color: red; font-weight: bold;
`;
const label = document.createElement('span');
label.textContent = ref;
label.style.cssText = 'position: absolute; top: -14px; left: 0; background: red; color: white; padding: 0 3px; font-size: 10px;';
overlay.appendChild(label);
document.body.appendChild(overlay);
}
}, boxes);
await page.screenshot({ path: screenshotPath, fullPage: true });
await reassertDevice(session);
await guardScreenshotPath(screenshotPath);
// Always remove overlays
await page.evaluate(() => {
document.querySelectorAll('.__browse_annotation__').forEach(el => el.remove());
});
output.push('');
output.push(`[annotated screenshot: ${screenshotPath}]`);
// Ambiguity and skips are visible, not buried behind BROWSE_DEBUG: a
// first-match box or a missing box changes what the screenshot claims.
if (ambiguousRefs.length || skippedRefs.length) {
const cap = (arr: string[]) => arr.slice(0, 8).join(', ') + (arr.length > 8 ? `, +${arr.length - 8} more` : '');
const parts: string[] = [];
if (ambiguousRefs.length) parts.push(`${ambiguousRefs.length} ambiguous (first-match): ${cap(ambiguousRefs)}`);
if (skippedRefs.length) parts.push(`${skippedRefs.length} skipped: ${cap(skippedRefs)}`);
output.push(`[annotated: ${parts.join(' | ')}]`);
}
} catch (err: any) {
// Remove overlays even on screenshot failure — but only swallow page/browser errors
if (!err?.message?.includes('closed') && !err?.message?.includes('Target') && !err?.message?.includes('Execution context') && !err?.message?.includes('screenshot')) throw err;
try {
await page.evaluate(() => {
document.querySelectorAll('.__browse_annotation__').forEach(el => el.remove());
});
} catch (err2: any) {
if (!err2?.message?.includes('closed') && !err2?.message?.includes('Target') && !err2?.message?.includes('Execution context')) throw err2;
}
}
}
// ─── Heatmap mode (-H) ──────────────────────────────────────
if (opts.heatmap) {
const heatmapPath = opts.outputPath || `${TEMP_DIR}/browse-heatmap.png`;
// Validate output path
{
const nodePath = require('path') as typeof import('path');
const nodeFs = require('fs') as typeof import('fs');
const absolute = nodePath.resolve(heatmapPath);
const safeDirs = [TEMP_DIR, process.cwd()].map((d: string) => {
try { return nodeFs.realpathSync(d); } catch (err: any) { if (err?.code !== 'ENOENT') throw err; return d; }
});
let realPath: string;
try {
realPath = nodeFs.realpathSync(absolute);
} catch (err: any) {
if (err.code === 'ENOENT') {
try {
const dir = nodeFs.realpathSync(nodePath.dirname(absolute));
realPath = nodePath.join(dir, nodePath.basename(absolute));
} catch (err2: any) {
if (err2?.code !== 'ENOENT') throw err2;
realPath = absolute;
}
} else {
throw new Error(`Cannot resolve real path: ${heatmapPath} (${err.code})`);
}
}
if (!safeDirs.some((dir: string) => isPathWithin(realPath, dir))) {
throw new Error(`Path must be within: ${safeDirs.join(', ')}`);
}
}
// Parse and validate color map
const VALID_COLORS = new Set(['green', 'yellow', 'red', 'blue', 'orange', 'gray']);
const COLOR_MAP: Record<string, { border: string; bg: string }> = {
green: { border: '#00b400', bg: 'rgba(0,180,0,0.15)' },
yellow: { border: '#ffb400', bg: 'rgba(255,180,0,0.15)' },
red: { border: '#ff0000', bg: 'rgba(255,0,0,0.15)' },
blue: { border: '#0066ff', bg: 'rgba(0,102,255,0.15)' },
orange: { border: '#ff6600', bg: 'rgba(255,102,0,0.15)' },
gray: { border: '#888888', bg: 'rgba(136,136,136,0.15)' },
};
let colorAssignments: Record<string, string>;
try {
const parsed = JSON.parse(opts.heatmap);
if (typeof parsed !== 'object' || parsed === null || Array.isArray(parsed)) {
throw new Error('not an object');
}
colorAssignments = parsed;
} catch {
throw new Error('Invalid heatmap JSON. Expected object: \'{"@e1":"green","@e3":"red"}\'');
}
// Validate colors
for (const [ref, color] of Object.entries(colorAssignments)) {
if (!VALID_COLORS.has(color)) {
throw new Error(`Invalid heatmap color "${color}" for ${ref}. Valid: ${[...VALID_COLORS].join(', ')}`);
}
}
try {
const boxes: Array<{ ref: string; box: { x: number; y: number; width: number; height: number }; color: string }> = [];
for (const [refKey, color] of Object.entries(colorAssignments)) {
const cleanRef = refKey.startsWith('@') ? refKey.slice(1) : refKey;
const entry = refMap.get(cleanRef);
if (!entry) continue; // Skip refs not found on page
try {
const box = await entry.locator.boundingBox({ timeout: 1000 });
if (box) {
const colors = COLOR_MAP[color] || COLOR_MAP.gray;
boxes.push({ ref: `@${cleanRef}`, box, color: JSON.stringify(colors) });
}
} catch {
// Element may be offscreen or hidden — skip
}
}
await page.evaluate((boxes) => {
for (const { ref, box, color } of boxes) {
const colors = JSON.parse(color);
const overlay = document.createElement('div');
overlay.className = '__browse_heatmap__';
overlay.style.cssText = `
position: absolute; top: ${box.y}px; left: ${box.x}px;
width: ${box.width}px; height: ${box.height}px;
border: 2px solid ${colors.border}; background: ${colors.bg};
pointer-events: none; z-index: 99999;
font-size: 10px; color: ${colors.border}; font-weight: bold;
`;
const label = document.createElement('span');
label.textContent = ref;
label.style.cssText = `position: absolute; top: -14px; left: 0; background: ${colors.border}; color: white; padding: 0 3px; font-size: 10px;`;
overlay.appendChild(label);
document.body.appendChild(overlay);
}
}, boxes);
await page.screenshot({ path: heatmapPath, fullPage: true });
await reassertDevice(session);
await guardScreenshotPath(heatmapPath);
// Remove heatmap overlays
await page.evaluate(() => {
document.querySelectorAll('.__browse_heatmap__').forEach(el => el.remove());
});
output.push('');
output.push(`[heatmap screenshot: ${heatmapPath}]`);
} catch (err: any) {
// Cleanup on failure
try {
await page.evaluate(() => {
document.querySelectorAll('.__browse_heatmap__').forEach(el => el.remove());
});
} catch {}
if (!err?.message?.includes('closed') && !err?.message?.includes('Target') && !err?.message?.includes('Execution context') && !err?.message?.includes('screenshot')) throw err;
}
}
// ─── Diff mode (-D) ───────────────────────────────────────
if (opts.diff) {
const lastSnapshot = session.getLastSnapshot();
if (!lastSnapshot) {
session.setLastSnapshot(snapshotText);
return snapshotText + '\n\n(no previous snapshot to diff against — this snapshot stored as baseline)'
+ (outputIgnoredWarning ? '\n' + outputIgnoredWarning : '');
}
const changes = Diff.diffLines(lastSnapshot, snapshotText);
const diffOutput: string[] = ['--- previous snapshot', '+++ current snapshot', ''];
for (const part of changes) {
const prefix = part.added ? '+' : part.removed ? '-' : ' ';
const diffLines = part.value.split('\n').filter(l => l.length > 0);
for (const line of diffLines) {
diffOutput.push(`${prefix} ${line}`);
}
}
session.setLastSnapshot(snapshotText);
return stripLoneSurrogates(diffOutput.join('\n')
+ (outputIgnoredWarning ? '\n' + outputIgnoredWarning : ''));
}
// Store for future diffs
session.setLastSnapshot(snapshotText);
// Add frame context header when operating inside an iframe
if (inFrame) {
const frameUrl = session.getFrame()?.url() ?? 'unknown';
output.unshift(`[Context: iframe src="${frameUrl}"]`);
}
// Split output for scoped tokens: trusted refs + untrusted text
if (securityOpts?.splitForScoped) {
const trustedRefs: string[] = [];
const untrustedLines: string[] = [];
for (const line of output) {
// Lines starting with @ref are interactive elements (trusted metadata)
const refMatch = line.match(/^(\s*)@(e\d+|c\d+)\s+\[([^\]]+)\]\s*(.*)/);
if (refMatch) {
const [, indent, ref, role, rest] = refMatch;
// Truncate element name/content to 50 chars for trusted section
const nameMatch = rest.match(/^"(.+?)"/);
let truncName = nameMatch ? nameMatch[1] : rest.trim();
if (truncName.length > 50) truncName = truncName.slice(0, 47) + '...';
trustedRefs.push(`${indent}@${ref} [${role}] "${truncName}"`);
}
// All lines go to untrusted section (full content)
untrustedLines.push(line);
}
const parts: string[] = [];
if (trustedRefs.length > 0) {
parts.push('INTERACTIVE ELEMENTS (trusted — use these @refs for click/fill):');
parts.push(...trustedRefs);
parts.push('');
}
// Defuse any envelope sentinel that appears inside the page's own
// accessibility text. Without this, a page whose rendered content
// contains the literal `═══ END UNTRUSTED WEB CONTENT ═══` string
// can close the envelope early and forge a fake "trusted" block
// for the LLM. Same escape that wrapUntrustedPageContent applies.
const safeUntrusted = untrustedLines.map(escapeEnvelopeSentinels);
parts.push('═══ BEGIN UNTRUSTED WEB CONTENT ═══');
parts.push(...safeUntrusted);
parts.push('═══ END UNTRUSTED WEB CONTENT ═══');
return stripLoneSurrogates(parts.join('\n'));
}
return stripLoneSurrogates(output.join('\n'));
}