From 542e7836d09c8bae7e0266b39750938a54298f59 Mon Sep 17 00:00:00 2001 From: Garry Tan Date: Sun, 5 Apr 2026 20:25:12 -0700 Subject: [PATCH 1/3] fix: snapshot -i auto-detects dropdown/popover interactive elements (#844) - Auto-enable cursor-interactive scan (-C) when -i flag is used - Add floating container detection (portals, popovers, dropdowns) - Detects position:fixed/absolute with high z-index - Recognizes data-floating-ui-portal, data-radix-* attributes - Recognizes role=listbox, role=menu containers - Elements inside floating containers bypass the hasRole skip - Catches dropdown items missed by the accessibility tree - Role=option/menuitem elements in floating containers captured even without cursor:pointer/onclick - Tag floating container items with 'popover-child' reason - Include role name in @c ref reasons when present - Add dropdown.html test fixture - Add dropdown/popover detection test suite (6 tests) - Add test: -i alone includes cursor-interactive elements Fixes: Bookface autocomplete, Radix UI combobox, React portals, and similar dynamic dropdown patterns where ariaSnapshot() misses the floating content. Co-authored-by: root --- browse/PLAN-snapshot-dropdown-interactive.md | 102 +++++++++++++++++++ browse/src/snapshot.ts | 43 +++++++- browse/test/fixtures/dropdown.html | 61 +++++++++++ browse/test/snapshot.test.ts | 71 +++++++++++++ 4 files changed, 273 insertions(+), 4 deletions(-) create mode 100644 browse/PLAN-snapshot-dropdown-interactive.md create mode 100644 browse/test/fixtures/dropdown.html diff --git a/browse/PLAN-snapshot-dropdown-interactive.md b/browse/PLAN-snapshot-dropdown-interactive.md new file mode 100644 index 00000000..75356911 --- /dev/null +++ b/browse/PLAN-snapshot-dropdown-interactive.md @@ -0,0 +1,102 @@ +# Plan: Snapshot Dropdown/Autocomplete Interactive Element Detection + +## Problem + +`snapshot -i` misses dropdown/autocomplete items on modern web apps. These elements: +1. Are often `
`/`
  • ` with click handlers but no semantic ARIA roles +2. Live inside dynamically-created portals/popovers (floating containers) +3. Don't appear in Playwright's accessibility tree (`ariaSnapshot()`) + +The `-C` flag (cursor-interactive scan) was designed for this but: +- Requires separate flag — agents using `-i` don't get it automatically +- Skips elements that HAVE an ARIA role (even if the ARIA tree missed them) +- Doesn't prioritize popover/portal containers where dropdown items live + +## Root Cause + +Playwright's `ariaSnapshot()` builds from the browser's accessibility tree. Dynamically-rendered popovers (React portals, Radix Popover, etc.) may not be in the accessibility tree if: +- The component doesn't set ARIA roles +- The portal renders outside the scoped `body` locator's subtree timing +- The browser hasn't updated the accessibility tree yet after DOM mutation + +## Changes + +### 1. Auto-enable cursor-interactive scan with `-i` flag + +**File:** `browse/src/snapshot.ts` + +When `-i` (interactive) is passed, automatically include the cursor-interactive scan. This means agents always see clickable non-ARIA elements when they ask for interactive elements. + +The `-C` flag remains as a standalone option for non-interactive snapshots. + +``` +if (opts.interactive) { + opts.cursorInteractive = true; +} +``` + +### 2. Add popover/portal priority scanning + +**File:** `browse/src/snapshot.ts` (inside cursor-interactive evaluate block) + +Before the general cursor:pointer scan, specifically scan for visible floating containers (popovers, dropdowns, menus) and include ALL their direct children as interactive: + +Detection heuristics for floating containers: +- `position: fixed` or `position: absolute` with `z-index >= 10` +- Has `role="listbox"`, `role="menu"`, `role="dialog"`, `role="tooltip"`, `[data-radix-popper-content-wrapper]`, `[data-floating-ui-portal]`, etc. +- Appeared recently in the DOM (not in initial page load) +- Is visible (`offsetParent !== null` or `position: fixed`) + +For each floating container, include child elements that: +- Have text content +- Are visible +- Have cursor:pointer OR onclick OR role="option" OR role="menuitem" +- Tag with reason `popover-child` for clarity + +### 3. Remove the `hasRole` skip in cursor-interactive scan + +**File:** `browse/src/snapshot.ts` + +Currently: `if (hasRole) continue;` — skips any element with an ARIA role, assuming the ARIA tree already captured it. + +Problem: if the ARIA tree MISSED the element (timing, portal, bad DOM structure), it falls through both systems. + +Fix: Only skip if the element's role is in `INTERACTIVE_ROLES` AND it was actually captured in the main refMap. Otherwise include it. + +Since we can't easily check the refMap from inside `page.evaluate()`, the simpler fix: remove the `hasRole` skip entirely for elements inside detected floating containers. For elements outside floating containers, keep the `hasRole` skip as-is (to avoid duplicates in normal page content). + +### 4. Add dropdown test fixture and tests + +**File:** `browse/test/fixtures/dropdown.html` + +HTML page with: +- A combobox input that shows a dropdown on focus/type +- Dropdown items as `
    ` with click handlers (no ARIA roles) +- Dropdown items as `
  • ` with `role="option"` +- A React-portal-style container (`position: fixed`, high z-index) + +**File:** `browse/test/snapshot.test.ts` + +New test cases: +- `snapshot -i` on dropdown page finds dropdown items via cursor scan +- `snapshot -i` on dropdown page includes popover-child elements +- `@c` refs from dropdown scan are clickable +- Elements inside floating containers with ARIA roles are captured even when ARIA tree misses them + +## Rollout Risk + +**Low.** The `-C` scan is additive — it only adds `@c` refs, never removes `@e` refs. The change to auto-enable it with `-i` increases output size but agents already handle mixed ref types. + +**One concern:** The `-C` scan queries ALL elements (`document.querySelectorAll('*')`) which can be slow on heavy pages. For the popover-specific scan, we limit to elements inside detected floating containers, which is fast (small subtree). + +## Testing + +```bash +cd /data/gstack/browse && bun test snapshot +``` + +## Files Changed + +1. `browse/src/snapshot.ts` — auto-enable -C with -i, popover scanning, remove hasRole skip in floating containers +2. `browse/test/fixtures/dropdown.html` — new test fixture +3. `browse/test/snapshot.test.ts` — new dropdown/popover test cases diff --git a/browse/src/snapshot.ts b/browse/src/snapshot.ts index 840cd686..5581fe6e 100644 --- a/browse/src/snapshot.ts +++ b/browse/src/snapshot.ts @@ -233,7 +233,12 @@ export async function handleSnapshot( output.push(outputLine); } - // ─── Cursor-interactive scan (-C) ───────────────────────── + // ─── Cursor-interactive scan (-C, or auto with -i) ──────── + // Auto-enable cursor scan when interactive mode is on — agents asking for + // interactive elements should always see clickable non-ARIA items too. + if (opts.interactive && !opts.cursorInteractive) { + opts.cursorInteractive = true; + } if (opts.cursorInteractive) { try { const cursorElements = await target.evaluate(() => { @@ -256,9 +261,37 @@ export async function handleSnapshot( const hasTabindex = el.hasAttribute('tabindex') && parseInt(el.getAttribute('tabindex')!, 10) >= 0; const hasRole = el.hasAttribute('role'); - if (!hasCursorPointer && !hasOnclick && !hasTabindex) continue; - // Skip if it has an ARIA role (likely already captured) - if (hasRole) continue; + // Check if element is inside a floating container (portal/popover/dropdown) + const isInFloating = (() => { + let parent: Element | null = el; + while (parent && parent !== document.documentElement) { + const pStyle = getComputedStyle(parent); + const isFloating = (pStyle.position === 'fixed' || pStyle.position === 'absolute') && + parseInt(pStyle.zIndex || '0', 10) >= 10; + const hasPortalAttr = parent.hasAttribute('data-floating-ui-portal') || + parent.hasAttribute('data-radix-popper-content-wrapper') || + parent.hasAttribute('data-radix-portal') || + parent.hasAttribute('data-popper-placement') || + parent.getAttribute('role') === 'listbox' || + parent.getAttribute('role') === 'menu'; + if (isFloating || hasPortalAttr) return true; + parent = parent.parentElement; + } + return false; + })(); + + if (!hasCursorPointer && !hasOnclick && !hasTabindex) { + // For elements inside floating containers, also check for role="option"/"menuitem" + if (isInFloating && hasRole) { + const role = el.getAttribute('role'); + if (role !== 'option' && role !== 'menuitem' && role !== 'menuitemcheckbox' && role !== 'menuitemradio') continue; + } else { + continue; + } + } + // Skip elements with ARIA roles UNLESS they're inside a floating container + // (floating container items may be missed by the accessibility tree) + if (hasRole && !isInFloating) continue; // Build deterministic nth-child CSS path const parts: string[] = []; @@ -275,9 +308,11 @@ export async function handleSnapshot( const text = (el as HTMLElement).innerText?.trim().slice(0, 80) || el.tagName.toLowerCase(); const reasons: string[] = []; + if (isInFloating) reasons.push('popover-child'); if (hasCursorPointer) reasons.push('cursor:pointer'); if (hasOnclick) reasons.push('onclick'); if (hasTabindex) reasons.push(`tabindex=${el.getAttribute('tabindex')}`); + if (hasRole) reasons.push(`role=${el.getAttribute('role')}`); results.push({ selector, text, reason: reasons.join(', ') }); } diff --git a/browse/test/fixtures/dropdown.html b/browse/test/fixtures/dropdown.html new file mode 100644 index 00000000..7919bceb --- /dev/null +++ b/browse/test/fixtures/dropdown.html @@ -0,0 +1,61 @@ + + + + + Test Page - Dropdown/Autocomplete + + + +

    Dropdown Test

    + +
    + +
    + + + + + + + Normal Link + + + + diff --git a/browse/test/snapshot.test.ts b/browse/test/snapshot.test.ts index db5e8004..bcbf8cd7 100644 --- a/browse/test/snapshot.test.ts +++ b/browse/test/snapshot.test.ts @@ -386,6 +386,77 @@ describe('Cursor-interactive', () => { // And cursor-interactive section expect(result).toContain('cursor-interactive'); }); + + test('snapshot -i alone also includes cursor-interactive elements', async () => { + await handleWriteCommand('goto', [baseUrl + '/cursor-interactive.html'], bm); + const result = await handleMetaCommand('snapshot', ['-i'], bm, shutdown); + // -i now auto-enables -C + expect(result).toContain('[button]'); + expect(result).toContain('[link]'); + expect(result).toContain('cursor-interactive'); + expect(result).toContain('@c'); + }); +}); + +// ─── Dropdown/Popover Detection ───────────────────────────────── + +describe('Dropdown/popover detection', () => { + test('snapshot -i auto-enables cursor scan and finds dropdown items', async () => { + await handleWriteCommand('goto', [baseUrl + '/dropdown.html'], bm); + const result = await handleMetaCommand('snapshot', ['-i'], bm, shutdown); + // Should find standard interactive elements + expect(result).toContain('[button]'); + expect(result).toContain('[link]'); + expect(result).toContain('[textbox]'); + // Should also find cursor-interactive dropdown items + expect(result).toContain('cursor-interactive'); + expect(result).toContain('@c'); + expect(result).toContain('Alice Johnson'); + expect(result).toContain('Bob Smith'); + }); + + test('dropdown items in floating container are tagged as popover-child', async () => { + await handleWriteCommand('goto', [baseUrl + '/dropdown.html'], bm); + const result = await handleMetaCommand('snapshot', ['-i'], bm, shutdown); + expect(result).toContain('popover-child'); + }); + + test('dropdown items with role="option" in portal are captured', async () => { + await handleWriteCommand('goto', [baseUrl + '/dropdown.html'], bm); + const result = await handleMetaCommand('snapshot', ['-i'], bm, shutdown); + // Dave Wilson has role="option" — should be captured even though it has a role + expect(result).toContain('Dave Wilson'); + }); + + test('static text in dropdown without interactivity is NOT captured', async () => { + await handleWriteCommand('goto', [baseUrl + '/dropdown.html'], bm); + const result = await handleMetaCommand('snapshot', ['-i'], bm, shutdown); + // "No results? Try a different search." has no cursor:pointer, no onclick, no tabindex + expect(result).not.toContain('No results'); + }); + + test('@c ref from dropdown is clickable', async () => { + await handleWriteCommand('goto', [baseUrl + '/dropdown.html'], bm); + const snap = await handleMetaCommand('snapshot', ['-i'], bm, shutdown); + // Find a @c ref for Alice + const aliceLine = snap.split('\n').find(l => l.includes('@c') && l.includes('Alice')); + if (aliceLine) { + const refMatch = aliceLine.match(/@(c\d+)/); + if (refMatch) { + const result = await handleWriteCommand('click', [`@${refMatch[1]}`], bm); + expect(result).toContain('Clicked'); + } + } + }); + + test('snapshot -C still works standalone without -i', async () => { + await handleWriteCommand('goto', [baseUrl + '/dropdown.html'], bm); + const result = await handleMetaCommand('snapshot', ['-C'], bm, shutdown); + expect(result).toContain('cursor-interactive'); + expect(result).toContain('Alice Johnson'); + // Without -i, should include non-interactive ARIA elements too + expect(result).toContain('[heading]'); + }); }); // ─── Snapshot Error Paths ─────────────────────────────────────── From 237ae2abbec3a05ae5ab4ea29b2f977ceb06bd5f Mon Sep 17 00:00:00 2001 From: root Date: Mon, 6 Apr 2026 03:27:13 +0000 Subject: [PATCH 2/3] Revert "fix: snapshot -i auto-detects dropdown/popover interactive elements (#844)" This reverts commit 542e7836d09c8bae7e0266b39750938a54298f59. --- browse/PLAN-snapshot-dropdown-interactive.md | 102 ------------------- browse/src/snapshot.ts | 43 +------- browse/test/fixtures/dropdown.html | 61 ----------- browse/test/snapshot.test.ts | 71 ------------- 4 files changed, 4 insertions(+), 273 deletions(-) delete mode 100644 browse/PLAN-snapshot-dropdown-interactive.md delete mode 100644 browse/test/fixtures/dropdown.html diff --git a/browse/PLAN-snapshot-dropdown-interactive.md b/browse/PLAN-snapshot-dropdown-interactive.md deleted file mode 100644 index 75356911..00000000 --- a/browse/PLAN-snapshot-dropdown-interactive.md +++ /dev/null @@ -1,102 +0,0 @@ -# Plan: Snapshot Dropdown/Autocomplete Interactive Element Detection - -## Problem - -`snapshot -i` misses dropdown/autocomplete items on modern web apps. These elements: -1. Are often `
    `/`
  • ` with click handlers but no semantic ARIA roles -2. Live inside dynamically-created portals/popovers (floating containers) -3. Don't appear in Playwright's accessibility tree (`ariaSnapshot()`) - -The `-C` flag (cursor-interactive scan) was designed for this but: -- Requires separate flag — agents using `-i` don't get it automatically -- Skips elements that HAVE an ARIA role (even if the ARIA tree missed them) -- Doesn't prioritize popover/portal containers where dropdown items live - -## Root Cause - -Playwright's `ariaSnapshot()` builds from the browser's accessibility tree. Dynamically-rendered popovers (React portals, Radix Popover, etc.) may not be in the accessibility tree if: -- The component doesn't set ARIA roles -- The portal renders outside the scoped `body` locator's subtree timing -- The browser hasn't updated the accessibility tree yet after DOM mutation - -## Changes - -### 1. Auto-enable cursor-interactive scan with `-i` flag - -**File:** `browse/src/snapshot.ts` - -When `-i` (interactive) is passed, automatically include the cursor-interactive scan. This means agents always see clickable non-ARIA elements when they ask for interactive elements. - -The `-C` flag remains as a standalone option for non-interactive snapshots. - -``` -if (opts.interactive) { - opts.cursorInteractive = true; -} -``` - -### 2. Add popover/portal priority scanning - -**File:** `browse/src/snapshot.ts` (inside cursor-interactive evaluate block) - -Before the general cursor:pointer scan, specifically scan for visible floating containers (popovers, dropdowns, menus) and include ALL their direct children as interactive: - -Detection heuristics for floating containers: -- `position: fixed` or `position: absolute` with `z-index >= 10` -- Has `role="listbox"`, `role="menu"`, `role="dialog"`, `role="tooltip"`, `[data-radix-popper-content-wrapper]`, `[data-floating-ui-portal]`, etc. -- Appeared recently in the DOM (not in initial page load) -- Is visible (`offsetParent !== null` or `position: fixed`) - -For each floating container, include child elements that: -- Have text content -- Are visible -- Have cursor:pointer OR onclick OR role="option" OR role="menuitem" -- Tag with reason `popover-child` for clarity - -### 3. Remove the `hasRole` skip in cursor-interactive scan - -**File:** `browse/src/snapshot.ts` - -Currently: `if (hasRole) continue;` — skips any element with an ARIA role, assuming the ARIA tree already captured it. - -Problem: if the ARIA tree MISSED the element (timing, portal, bad DOM structure), it falls through both systems. - -Fix: Only skip if the element's role is in `INTERACTIVE_ROLES` AND it was actually captured in the main refMap. Otherwise include it. - -Since we can't easily check the refMap from inside `page.evaluate()`, the simpler fix: remove the `hasRole` skip entirely for elements inside detected floating containers. For elements outside floating containers, keep the `hasRole` skip as-is (to avoid duplicates in normal page content). - -### 4. Add dropdown test fixture and tests - -**File:** `browse/test/fixtures/dropdown.html` - -HTML page with: -- A combobox input that shows a dropdown on focus/type -- Dropdown items as `
    ` with click handlers (no ARIA roles) -- Dropdown items as `
  • ` with `role="option"` -- A React-portal-style container (`position: fixed`, high z-index) - -**File:** `browse/test/snapshot.test.ts` - -New test cases: -- `snapshot -i` on dropdown page finds dropdown items via cursor scan -- `snapshot -i` on dropdown page includes popover-child elements -- `@c` refs from dropdown scan are clickable -- Elements inside floating containers with ARIA roles are captured even when ARIA tree misses them - -## Rollout Risk - -**Low.** The `-C` scan is additive — it only adds `@c` refs, never removes `@e` refs. The change to auto-enable it with `-i` increases output size but agents already handle mixed ref types. - -**One concern:** The `-C` scan queries ALL elements (`document.querySelectorAll('*')`) which can be slow on heavy pages. For the popover-specific scan, we limit to elements inside detected floating containers, which is fast (small subtree). - -## Testing - -```bash -cd /data/gstack/browse && bun test snapshot -``` - -## Files Changed - -1. `browse/src/snapshot.ts` — auto-enable -C with -i, popover scanning, remove hasRole skip in floating containers -2. `browse/test/fixtures/dropdown.html` — new test fixture -3. `browse/test/snapshot.test.ts` — new dropdown/popover test cases diff --git a/browse/src/snapshot.ts b/browse/src/snapshot.ts index 5581fe6e..840cd686 100644 --- a/browse/src/snapshot.ts +++ b/browse/src/snapshot.ts @@ -233,12 +233,7 @@ export async function handleSnapshot( output.push(outputLine); } - // ─── Cursor-interactive scan (-C, or auto with -i) ──────── - // Auto-enable cursor scan when interactive mode is on — agents asking for - // interactive elements should always see clickable non-ARIA items too. - if (opts.interactive && !opts.cursorInteractive) { - opts.cursorInteractive = true; - } + // ─── Cursor-interactive scan (-C) ───────────────────────── if (opts.cursorInteractive) { try { const cursorElements = await target.evaluate(() => { @@ -261,37 +256,9 @@ export async function handleSnapshot( const hasTabindex = el.hasAttribute('tabindex') && parseInt(el.getAttribute('tabindex')!, 10) >= 0; const hasRole = el.hasAttribute('role'); - // Check if element is inside a floating container (portal/popover/dropdown) - const isInFloating = (() => { - let parent: Element | null = el; - while (parent && parent !== document.documentElement) { - const pStyle = getComputedStyle(parent); - const isFloating = (pStyle.position === 'fixed' || pStyle.position === 'absolute') && - parseInt(pStyle.zIndex || '0', 10) >= 10; - const hasPortalAttr = parent.hasAttribute('data-floating-ui-portal') || - parent.hasAttribute('data-radix-popper-content-wrapper') || - parent.hasAttribute('data-radix-portal') || - parent.hasAttribute('data-popper-placement') || - parent.getAttribute('role') === 'listbox' || - parent.getAttribute('role') === 'menu'; - if (isFloating || hasPortalAttr) return true; - parent = parent.parentElement; - } - return false; - })(); - - if (!hasCursorPointer && !hasOnclick && !hasTabindex) { - // For elements inside floating containers, also check for role="option"/"menuitem" - if (isInFloating && hasRole) { - const role = el.getAttribute('role'); - if (role !== 'option' && role !== 'menuitem' && role !== 'menuitemcheckbox' && role !== 'menuitemradio') continue; - } else { - continue; - } - } - // Skip elements with ARIA roles UNLESS they're inside a floating container - // (floating container items may be missed by the accessibility tree) - if (hasRole && !isInFloating) continue; + if (!hasCursorPointer && !hasOnclick && !hasTabindex) continue; + // Skip if it has an ARIA role (likely already captured) + if (hasRole) continue; // Build deterministic nth-child CSS path const parts: string[] = []; @@ -308,11 +275,9 @@ export async function handleSnapshot( const text = (el as HTMLElement).innerText?.trim().slice(0, 80) || el.tagName.toLowerCase(); const reasons: string[] = []; - if (isInFloating) reasons.push('popover-child'); if (hasCursorPointer) reasons.push('cursor:pointer'); if (hasOnclick) reasons.push('onclick'); if (hasTabindex) reasons.push(`tabindex=${el.getAttribute('tabindex')}`); - if (hasRole) reasons.push(`role=${el.getAttribute('role')}`); results.push({ selector, text, reason: reasons.join(', ') }); } diff --git a/browse/test/fixtures/dropdown.html b/browse/test/fixtures/dropdown.html deleted file mode 100644 index 7919bceb..00000000 --- a/browse/test/fixtures/dropdown.html +++ /dev/null @@ -1,61 +0,0 @@ - - - - - Test Page - Dropdown/Autocomplete - - - -

    Dropdown Test

    - -
    - -
    - - - - - - - Normal Link - - - - diff --git a/browse/test/snapshot.test.ts b/browse/test/snapshot.test.ts index bcbf8cd7..db5e8004 100644 --- a/browse/test/snapshot.test.ts +++ b/browse/test/snapshot.test.ts @@ -386,77 +386,6 @@ describe('Cursor-interactive', () => { // And cursor-interactive section expect(result).toContain('cursor-interactive'); }); - - test('snapshot -i alone also includes cursor-interactive elements', async () => { - await handleWriteCommand('goto', [baseUrl + '/cursor-interactive.html'], bm); - const result = await handleMetaCommand('snapshot', ['-i'], bm, shutdown); - // -i now auto-enables -C - expect(result).toContain('[button]'); - expect(result).toContain('[link]'); - expect(result).toContain('cursor-interactive'); - expect(result).toContain('@c'); - }); -}); - -// ─── Dropdown/Popover Detection ───────────────────────────────── - -describe('Dropdown/popover detection', () => { - test('snapshot -i auto-enables cursor scan and finds dropdown items', async () => { - await handleWriteCommand('goto', [baseUrl + '/dropdown.html'], bm); - const result = await handleMetaCommand('snapshot', ['-i'], bm, shutdown); - // Should find standard interactive elements - expect(result).toContain('[button]'); - expect(result).toContain('[link]'); - expect(result).toContain('[textbox]'); - // Should also find cursor-interactive dropdown items - expect(result).toContain('cursor-interactive'); - expect(result).toContain('@c'); - expect(result).toContain('Alice Johnson'); - expect(result).toContain('Bob Smith'); - }); - - test('dropdown items in floating container are tagged as popover-child', async () => { - await handleWriteCommand('goto', [baseUrl + '/dropdown.html'], bm); - const result = await handleMetaCommand('snapshot', ['-i'], bm, shutdown); - expect(result).toContain('popover-child'); - }); - - test('dropdown items with role="option" in portal are captured', async () => { - await handleWriteCommand('goto', [baseUrl + '/dropdown.html'], bm); - const result = await handleMetaCommand('snapshot', ['-i'], bm, shutdown); - // Dave Wilson has role="option" — should be captured even though it has a role - expect(result).toContain('Dave Wilson'); - }); - - test('static text in dropdown without interactivity is NOT captured', async () => { - await handleWriteCommand('goto', [baseUrl + '/dropdown.html'], bm); - const result = await handleMetaCommand('snapshot', ['-i'], bm, shutdown); - // "No results? Try a different search." has no cursor:pointer, no onclick, no tabindex - expect(result).not.toContain('No results'); - }); - - test('@c ref from dropdown is clickable', async () => { - await handleWriteCommand('goto', [baseUrl + '/dropdown.html'], bm); - const snap = await handleMetaCommand('snapshot', ['-i'], bm, shutdown); - // Find a @c ref for Alice - const aliceLine = snap.split('\n').find(l => l.includes('@c') && l.includes('Alice')); - if (aliceLine) { - const refMatch = aliceLine.match(/@(c\d+)/); - if (refMatch) { - const result = await handleWriteCommand('click', [`@${refMatch[1]}`], bm); - expect(result).toContain('Clicked'); - } - } - }); - - test('snapshot -C still works standalone without -i', async () => { - await handleWriteCommand('goto', [baseUrl + '/dropdown.html'], bm); - const result = await handleMetaCommand('snapshot', ['-C'], bm, shutdown); - expect(result).toContain('cursor-interactive'); - expect(result).toContain('Alice Johnson'); - // Without -i, should include non-interactive ARIA elements too - expect(result).toContain('[heading]'); - }); }); // ─── Snapshot Error Paths ─────────────────────────────────────── From a94a64f82155681f4cae7045679bfb036cf11c45 Mon Sep 17 00:00:00 2001 From: Garry Tan Date: Sun, 5 Apr 2026 22:57:45 -0700 Subject: [PATCH 3/3] fix: snapshot -i auto-detects dropdown/popover interactive elements (#845) * fix: snapshot -i auto-detects dropdown/popover interactive elements - Auto-enable cursor-interactive scan (-C) when -i flag is used - Add floating container detection (portals, popovers, dropdowns) - Detects position:fixed/absolute with high z-index - Recognizes data-floating-ui-portal, data-radix-* attributes - Recognizes role=listbox, role=menu containers - Elements inside floating containers bypass the hasRole skip - Catches dropdown items missed by the accessibility tree - Role=option/menuitem elements in floating containers captured even without cursor:pointer/onclick - Tag floating container items with 'popover-child' reason - Include role name in @c ref reasons when present - Add dropdown.html test fixture - Add dropdown/popover detection test suite (6 tests) - Add test: -i alone includes cursor-interactive elements Fixes: Bookface autocomplete, Radix UI combobox, React portals, and similar dynamic dropdown patterns where ariaSnapshot() misses the floating content. * chore: bump version and changelog (v0.15.12.0) Co-Authored-By: Claude Sonnet 4.6 * docs: update snapshot -i/-C flag descriptions to mention auto-enable behavior * test: strengthen clickability test guard assertions The @c ref clickability test previously used if-guards that would silently pass when no Alice line was found in the snapshot output. Both Claude and Codex adversarial review flagged this as a test that could regress without CI noticing. Co-Authored-By: Claude Opus 4.6 * docs: regenerate top-level SKILL.md with updated flag descriptions Co-Authored-By: Claude Opus 4.6 --------- Co-authored-by: root Co-authored-by: gstack Co-authored-by: Claude Sonnet 4.6 --- CHANGELOG.md | 7 ++ SKILL.md | 4 +- VERSION | 2 +- browse/PLAN-snapshot-dropdown-interactive.md | 102 +++++++++++++++++++ browse/SKILL.md | 4 +- browse/src/snapshot.ts | 47 +++++++-- browse/test/fixtures/dropdown.html | 61 +++++++++++ browse/test/snapshot.test.ts | 69 +++++++++++++ 8 files changed, 285 insertions(+), 11 deletions(-) create mode 100644 browse/PLAN-snapshot-dropdown-interactive.md create mode 100644 browse/test/fixtures/dropdown.html diff --git a/CHANGELOG.md b/CHANGELOG.md index 7ae03519..91786cb7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,12 @@ # Changelog +## [0.15.12.0] - 2026-04-06 + +### Fixed +- `snapshot -i` now auto-includes cursor-interactive elements (dropdown items, popover options, custom listboxes). Previously you had to remember to pass `-C` separately — now `-i` alone finds everything clickable on the page. +- Snapshot correctly captures items inside floating containers (React portals, Radix Popover, Floating UI) even when they have ARIA roles. Previously these were silently skipped because the accessibility tree sometimes misses dynamically-rendered portals. +- Dropdown/menu items with `role="option"` or `role="menuitem"` inside popovers are now captured and tagged with `popover-child` in the reason string, making them easy to identify. + ## [0.15.11.0] - 2026-04-05 ### Changed diff --git a/SKILL.md b/SKILL.md index 395ae131..1301aefa 100644 --- a/SKILL.md +++ b/SKILL.md @@ -668,14 +668,14 @@ $B css ".button" "background-color" The snapshot is your primary tool for understanding and interacting with pages. ``` --i --interactive Interactive elements only (buttons, links, inputs) with @e refs +-i --interactive Interactive elements only (buttons, links, inputs) with @e refs. Also auto-enables cursor-interactive scan (-C) to capture dropdowns and popovers. -c --compact Compact (no empty structural nodes) -d --depth Limit tree depth (0 = root only, default: unlimited) -s --selector Scope to CSS selector -D --diff Unified diff against previous snapshot (first call stores baseline) -a --annotate Annotated screenshot with red overlay boxes and ref labels -o --output Output path for annotated screenshot (default: /browse-annotated.png) --C --cursor-interactive Cursor-interactive elements (@c refs — divs with pointer, onclick) +-C --cursor-interactive Cursor-interactive elements (@c refs — divs with pointer, onclick). Auto-enabled when -i is used. ``` All flags can be combined freely. `-o` only applies when `-a` is also used. diff --git a/VERSION b/VERSION index 91b13ea2..7937bb24 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.15.11.0 +0.15.12.0 diff --git a/browse/PLAN-snapshot-dropdown-interactive.md b/browse/PLAN-snapshot-dropdown-interactive.md new file mode 100644 index 00000000..75356911 --- /dev/null +++ b/browse/PLAN-snapshot-dropdown-interactive.md @@ -0,0 +1,102 @@ +# Plan: Snapshot Dropdown/Autocomplete Interactive Element Detection + +## Problem + +`snapshot -i` misses dropdown/autocomplete items on modern web apps. These elements: +1. Are often `
    `/`
  • ` with click handlers but no semantic ARIA roles +2. Live inside dynamically-created portals/popovers (floating containers) +3. Don't appear in Playwright's accessibility tree (`ariaSnapshot()`) + +The `-C` flag (cursor-interactive scan) was designed for this but: +- Requires separate flag — agents using `-i` don't get it automatically +- Skips elements that HAVE an ARIA role (even if the ARIA tree missed them) +- Doesn't prioritize popover/portal containers where dropdown items live + +## Root Cause + +Playwright's `ariaSnapshot()` builds from the browser's accessibility tree. Dynamically-rendered popovers (React portals, Radix Popover, etc.) may not be in the accessibility tree if: +- The component doesn't set ARIA roles +- The portal renders outside the scoped `body` locator's subtree timing +- The browser hasn't updated the accessibility tree yet after DOM mutation + +## Changes + +### 1. Auto-enable cursor-interactive scan with `-i` flag + +**File:** `browse/src/snapshot.ts` + +When `-i` (interactive) is passed, automatically include the cursor-interactive scan. This means agents always see clickable non-ARIA elements when they ask for interactive elements. + +The `-C` flag remains as a standalone option for non-interactive snapshots. + +``` +if (opts.interactive) { + opts.cursorInteractive = true; +} +``` + +### 2. Add popover/portal priority scanning + +**File:** `browse/src/snapshot.ts` (inside cursor-interactive evaluate block) + +Before the general cursor:pointer scan, specifically scan for visible floating containers (popovers, dropdowns, menus) and include ALL their direct children as interactive: + +Detection heuristics for floating containers: +- `position: fixed` or `position: absolute` with `z-index >= 10` +- Has `role="listbox"`, `role="menu"`, `role="dialog"`, `role="tooltip"`, `[data-radix-popper-content-wrapper]`, `[data-floating-ui-portal]`, etc. +- Appeared recently in the DOM (not in initial page load) +- Is visible (`offsetParent !== null` or `position: fixed`) + +For each floating container, include child elements that: +- Have text content +- Are visible +- Have cursor:pointer OR onclick OR role="option" OR role="menuitem" +- Tag with reason `popover-child` for clarity + +### 3. Remove the `hasRole` skip in cursor-interactive scan + +**File:** `browse/src/snapshot.ts` + +Currently: `if (hasRole) continue;` — skips any element with an ARIA role, assuming the ARIA tree already captured it. + +Problem: if the ARIA tree MISSED the element (timing, portal, bad DOM structure), it falls through both systems. + +Fix: Only skip if the element's role is in `INTERACTIVE_ROLES` AND it was actually captured in the main refMap. Otherwise include it. + +Since we can't easily check the refMap from inside `page.evaluate()`, the simpler fix: remove the `hasRole` skip entirely for elements inside detected floating containers. For elements outside floating containers, keep the `hasRole` skip as-is (to avoid duplicates in normal page content). + +### 4. Add dropdown test fixture and tests + +**File:** `browse/test/fixtures/dropdown.html` + +HTML page with: +- A combobox input that shows a dropdown on focus/type +- Dropdown items as `
    ` with click handlers (no ARIA roles) +- Dropdown items as `
  • ` with `role="option"` +- A React-portal-style container (`position: fixed`, high z-index) + +**File:** `browse/test/snapshot.test.ts` + +New test cases: +- `snapshot -i` on dropdown page finds dropdown items via cursor scan +- `snapshot -i` on dropdown page includes popover-child elements +- `@c` refs from dropdown scan are clickable +- Elements inside floating containers with ARIA roles are captured even when ARIA tree misses them + +## Rollout Risk + +**Low.** The `-C` scan is additive — it only adds `@c` refs, never removes `@e` refs. The change to auto-enable it with `-i` increases output size but agents already handle mixed ref types. + +**One concern:** The `-C` scan queries ALL elements (`document.querySelectorAll('*')`) which can be slow on heavy pages. For the popover-specific scan, we limit to elements inside detected floating containers, which is fast (small subtree). + +## Testing + +```bash +cd /data/gstack/browse && bun test snapshot +``` + +## Files Changed + +1. `browse/src/snapshot.ts` — auto-enable -C with -i, popover scanning, remove hasRole skip in floating containers +2. `browse/test/fixtures/dropdown.html` — new test fixture +3. `browse/test/snapshot.test.ts` — new dropdown/popover test cases diff --git a/browse/SKILL.md b/browse/SKILL.md index ed840e69..9ab4b383 100644 --- a/browse/SKILL.md +++ b/browse/SKILL.md @@ -536,14 +536,14 @@ After `resume`, you get a fresh snapshot of wherever the user left off. The snapshot is your primary tool for understanding and interacting with pages. ``` --i --interactive Interactive elements only (buttons, links, inputs) with @e refs +-i --interactive Interactive elements only (buttons, links, inputs) with @e refs. Also auto-enables cursor-interactive scan (-C) to capture dropdowns and popovers. -c --compact Compact (no empty structural nodes) -d --depth Limit tree depth (0 = root only, default: unlimited) -s --selector Scope to CSS selector -D --diff Unified diff against previous snapshot (first call stores baseline) -a --annotate Annotated screenshot with red overlay boxes and ref labels -o --output Output path for annotated screenshot (default: /browse-annotated.png) --C --cursor-interactive Cursor-interactive elements (@c refs — divs with pointer, onclick) +-C --cursor-interactive Cursor-interactive elements (@c refs — divs with pointer, onclick). Auto-enabled when -i is used. ``` All flags can be combined freely. `-o` only applies when `-a` is also used. diff --git a/browse/src/snapshot.ts b/browse/src/snapshot.ts index 840cd686..ae18c3f3 100644 --- a/browse/src/snapshot.ts +++ b/browse/src/snapshot.ts @@ -56,14 +56,14 @@ export const SNAPSHOT_FLAGS: Array<{ valueHint?: string; optionKey: keyof SnapshotOptions; }> = [ - { short: '-i', long: '--interactive', description: 'Interactive elements only (buttons, links, inputs) with @e refs', optionKey: 'interactive' }, + { short: '-i', long: '--interactive', description: 'Interactive elements only (buttons, links, inputs) with @e refs. Also auto-enables cursor-interactive scan (-C) to capture dropdowns and popovers.', optionKey: 'interactive' }, { short: '-c', long: '--compact', description: 'Compact (no empty structural nodes)', optionKey: 'compact' }, { short: '-d', long: '--depth', description: 'Limit tree depth (0 = root only, default: unlimited)', takesValue: true, valueHint: '', optionKey: 'depth' }, { short: '-s', long: '--selector', description: 'Scope to CSS selector', takesValue: true, valueHint: '', optionKey: 'selector' }, { short: '-D', long: '--diff', description: 'Unified diff against previous snapshot (first call stores baseline)', optionKey: 'diff' }, { short: '-a', long: '--annotate', description: 'Annotated screenshot with red overlay boxes and ref labels', optionKey: 'annotate' }, { short: '-o', long: '--output', description: 'Output path for annotated screenshot (default: /browse-annotated.png)', takesValue: true, valueHint: '', optionKey: 'outputPath' }, - { short: '-C', long: '--cursor-interactive', description: 'Cursor-interactive elements (@c refs — divs with pointer, onclick)', optionKey: 'cursorInteractive' }, + { short: '-C', long: '--cursor-interactive', description: 'Cursor-interactive elements (@c refs — divs with pointer, onclick). Auto-enabled when -i is used.', optionKey: 'cursorInteractive' }, ]; interface ParsedNode { @@ -233,7 +233,12 @@ export async function handleSnapshot( output.push(outputLine); } - // ─── Cursor-interactive scan (-C) ───────────────────────── + // ─── Cursor-interactive scan (-C, or auto with -i) ──────── + // Auto-enable cursor scan when interactive mode is on — agents asking for + // interactive elements should always see clickable non-ARIA items too. + if (opts.interactive && !opts.cursorInteractive) { + opts.cursorInteractive = true; + } if (opts.cursorInteractive) { try { const cursorElements = await target.evaluate(() => { @@ -256,9 +261,37 @@ export async function handleSnapshot( const hasTabindex = el.hasAttribute('tabindex') && parseInt(el.getAttribute('tabindex')!, 10) >= 0; const hasRole = el.hasAttribute('role'); - if (!hasCursorPointer && !hasOnclick && !hasTabindex) continue; - // Skip if it has an ARIA role (likely already captured) - if (hasRole) continue; + // Check if element is inside a floating container (portal/popover/dropdown) + const isInFloating = (() => { + let parent: Element | null = el; + while (parent && parent !== document.documentElement) { + const pStyle = getComputedStyle(parent); + const isFloating = (pStyle.position === 'fixed' || pStyle.position === 'absolute') && + parseInt(pStyle.zIndex || '0', 10) >= 10; + const hasPortalAttr = parent.hasAttribute('data-floating-ui-portal') || + parent.hasAttribute('data-radix-popper-content-wrapper') || + parent.hasAttribute('data-radix-portal') || + parent.hasAttribute('data-popper-placement') || + parent.getAttribute('role') === 'listbox' || + parent.getAttribute('role') === 'menu'; + if (isFloating || hasPortalAttr) return true; + parent = parent.parentElement; + } + return false; + })(); + + if (!hasCursorPointer && !hasOnclick && !hasTabindex) { + // For elements inside floating containers, also check for role="option"/"menuitem" + if (isInFloating && hasRole) { + const role = el.getAttribute('role'); + if (role !== 'option' && role !== 'menuitem' && role !== 'menuitemcheckbox' && role !== 'menuitemradio') continue; + } else { + continue; + } + } + // Skip elements with ARIA roles UNLESS they're inside a floating container + // (floating container items may be missed by the accessibility tree) + if (hasRole && !isInFloating) continue; // Build deterministic nth-child CSS path const parts: string[] = []; @@ -275,9 +308,11 @@ export async function handleSnapshot( const text = (el as HTMLElement).innerText?.trim().slice(0, 80) || el.tagName.toLowerCase(); const reasons: string[] = []; + if (isInFloating) reasons.push('popover-child'); if (hasCursorPointer) reasons.push('cursor:pointer'); if (hasOnclick) reasons.push('onclick'); if (hasTabindex) reasons.push(`tabindex=${el.getAttribute('tabindex')}`); + if (hasRole) reasons.push(`role=${el.getAttribute('role')}`); results.push({ selector, text, reason: reasons.join(', ') }); } diff --git a/browse/test/fixtures/dropdown.html b/browse/test/fixtures/dropdown.html new file mode 100644 index 00000000..7919bceb --- /dev/null +++ b/browse/test/fixtures/dropdown.html @@ -0,0 +1,61 @@ + + + + + Test Page - Dropdown/Autocomplete + + + +

    Dropdown Test

    + +
    + +
    + + + + + + + Normal Link + + + + diff --git a/browse/test/snapshot.test.ts b/browse/test/snapshot.test.ts index db5e8004..4b375cda 100644 --- a/browse/test/snapshot.test.ts +++ b/browse/test/snapshot.test.ts @@ -386,6 +386,75 @@ describe('Cursor-interactive', () => { // And cursor-interactive section expect(result).toContain('cursor-interactive'); }); + + test('snapshot -i alone also includes cursor-interactive elements', async () => { + await handleWriteCommand('goto', [baseUrl + '/cursor-interactive.html'], bm); + const result = await handleMetaCommand('snapshot', ['-i'], bm, shutdown); + // -i now auto-enables -C + expect(result).toContain('[button]'); + expect(result).toContain('[link]'); + expect(result).toContain('cursor-interactive'); + expect(result).toContain('@c'); + }); +}); + +// ─── Dropdown/Popover Detection ───────────────────────────────── + +describe('Dropdown/popover detection', () => { + test('snapshot -i auto-enables cursor scan and finds dropdown items', async () => { + await handleWriteCommand('goto', [baseUrl + '/dropdown.html'], bm); + const result = await handleMetaCommand('snapshot', ['-i'], bm, shutdown); + // Should find standard interactive elements + expect(result).toContain('[button]'); + expect(result).toContain('[link]'); + expect(result).toContain('[textbox]'); + // Should also find cursor-interactive dropdown items + expect(result).toContain('cursor-interactive'); + expect(result).toContain('@c'); + expect(result).toContain('Alice Johnson'); + expect(result).toContain('Bob Smith'); + }); + + test('dropdown items in floating container are tagged as popover-child', async () => { + await handleWriteCommand('goto', [baseUrl + '/dropdown.html'], bm); + const result = await handleMetaCommand('snapshot', ['-i'], bm, shutdown); + expect(result).toContain('popover-child'); + }); + + test('dropdown items with role="option" in portal are captured', async () => { + await handleWriteCommand('goto', [baseUrl + '/dropdown.html'], bm); + const result = await handleMetaCommand('snapshot', ['-i'], bm, shutdown); + // Dave Wilson has role="option" — should be captured even though it has a role + expect(result).toContain('Dave Wilson'); + }); + + test('static text in dropdown without interactivity is NOT captured', async () => { + await handleWriteCommand('goto', [baseUrl + '/dropdown.html'], bm); + const result = await handleMetaCommand('snapshot', ['-i'], bm, shutdown); + // "No results? Try a different search." has no cursor:pointer, no onclick, no tabindex + expect(result).not.toContain('No results'); + }); + + test('@c ref from dropdown is clickable', async () => { + await handleWriteCommand('goto', [baseUrl + '/dropdown.html'], bm); + const snap = await handleMetaCommand('snapshot', ['-i'], bm, shutdown); + // Find a @c ref for Alice + const aliceLine = snap.split('\n').find(l => l.includes('@c') && l.includes('Alice')); + expect(aliceLine).toBeTruthy(); + const refMatch = aliceLine!.match(/@(c\d+)/); + expect(refMatch).toBeTruthy(); + const result = await handleWriteCommand('click', [`@${refMatch![1]}`], bm); + expect(result).toContain('Clicked'); + }); + + test('snapshot -C still works standalone without -i', async () => { + await handleWriteCommand('goto', [baseUrl + '/dropdown.html'], bm); + const result = await handleMetaCommand('snapshot', ['-C'], bm, shutdown); + expect(result).toContain('cursor-interactive'); + expect(result).toContain('Alice Johnson'); + // Without -i, should include non-interactive ARIA elements too + expect(result).toContain('[heading]'); + }); }); // ─── Snapshot Error Paths ───────────────────────────────────────