From 48d5f0ce3e07adefdd5583c7e692f19321b41363 Mon Sep 17 00:00:00 2001 From: garrytan Date: Sat, 31 Oct 2026 23:07:37 +0000 Subject: [PATCH 1/3] test(browse): failing-first device and touch tests (GSTA-227 F3, #3058) Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> --- browse/test/fixtures/touch.html | 64 ++++++++ browse/test/touch-commands.test.ts | 229 +++++++++++++++++++++++++++++ 2 files changed, 293 insertions(+) create mode 100644 browse/test/fixtures/touch.html create mode 100644 browse/test/touch-commands.test.ts diff --git a/browse/test/fixtures/touch.html b/browse/test/fixtures/touch.html new file mode 100644 index 000000000..9e5855bc2 --- /dev/null +++ b/browse/test/fixtures/touch.html @@ -0,0 +1,64 @@ + + + + +Touch fixture + + + + +
zoom 1
+
joystick
+ + + + diff --git a/browse/test/touch-commands.test.ts b/browse/test/touch-commands.test.ts new file mode 100644 index 000000000..a8caa41d8 --- /dev/null +++ b/browse/test/touch-commands.test.ts @@ -0,0 +1,229 @@ +/** + * Phone emulation and touch input on gstack's own Chromium (#3058, GSTA-227 F3). + * + * Drives a real in-process BrowserManager (same harness as snapshot.test.ts) + * against browse/test/fixtures/touch.html: a touch logger, a swipe carousel, + * a two-finger zoom surface and a joystick plus fire button. + */ + +import { afterAll, beforeAll, describe, expect, test } from 'bun:test'; +import * as fs from 'fs'; +import * as os from 'os'; +import * as path from 'path'; +import { startTestServer } from './test-server'; +import { BrowserManager } from '../src/browser-manager'; +import { handleWriteCommand } from '../src/write-commands'; +import { handleMetaCommand } from '../src/meta-commands'; +import { WRITE_COMMANDS, COMMAND_DESCRIPTIONS } from '../src/commands'; +import { SCOPE_WRITE } from '../src/token-registry'; +import { TUNNEL_COMMANDS } from '../src/routes/commands'; + +const ORIGINAL_CHROMIUM_PROFILE = process.env.CHROMIUM_PROFILE; +let CHROMIUM_PROFILE_DIR: string | undefined; +let testServer: ReturnType; +let bm: BrowserManager; +let baseUrl: string; +const shutdown = async () => {}; +const VERBS = ['device', 'tap', 'swipe', 'pinch', 'touch']; + +beforeAll(async () => { + CHROMIUM_PROFILE_DIR = fs.mkdtempSync(path.join(os.tmpdir(), 'gstack-test-profile-')); + process.env.CHROMIUM_PROFILE = CHROMIUM_PROFILE_DIR; + testServer = startTestServer(0); + baseUrl = testServer.url; + bm = new BrowserManager(); + await bm.launch(); +}); + +afterAll(async () => { + try { testServer.server.stop(true); } catch {} + try { await Promise.race([bm?.close(), new Promise((resolve) => setTimeout(resolve, 3000))]); } catch {} + if (ORIGINAL_CHROMIUM_PROFILE === undefined) delete process.env.CHROMIUM_PROFILE; + else process.env.CHROMIUM_PROFILE = ORIGINAL_CHROMIUM_PROFILE; + if (CHROMIUM_PROFILE_DIR) { try { fs.rmSync(CHROMIUM_PROFILE_DIR, { recursive: true, force: true }); } catch {} } +}); + +const run = (cmd: string, args: string[] = []) => handleWriteCommand(cmd, args, bm.getActiveSession(), bm); +const page = () => bm.getActiveSession().getPage(); +const probe = () => page().evaluate(() => ({ + coarse: matchMedia('(pointer: coarse)').matches, + maxTouchPoints: navigator.maxTouchPoints, + ua: navigator.userAgent, + width: innerWidth, +})); +const data = (sel: string, key: string) => page().evaluate(([s, k]) => (document.querySelector(s) as HTMLElement).dataset[k], [sel, key]); + +async function freshTouchTab(device: string | null = 'iPhone 15'): Promise { + const id = await bm.newTab(baseUrl + '/touch.html'); + bm.switchTab(id, { bringToFront: false }); + if (device) await run('device', [device]); + await page().reload({ waitUntil: 'domcontentloaded' }); + return id; +} + +describe('device and touch: registration', () => { + test('verbs are write commands with descriptions, in the write scope, never on the tunnel', () => { + for (const verb of VERBS) { + expect(WRITE_COMMANDS.has(verb)).toBe(true); + expect(SCOPE_WRITE.has(verb)).toBe(true); + expect(TUNNEL_COMMANDS.has(verb)).toBe(false); + expect(COMMAND_DESCRIPTIONS[verb]?.usage).toContain(verb); + } + }); +}); + +describe('device', () => { + test('unknown name suggests the nearest names and leaves the tab alone', async () => { + await freshTouchTab(null); + const before = await probe(); + let message = ''; + try { await run('device', ['iPhone 99']); } catch (err: any) { message = err.message; } + expect(message).toContain('BROWSE_DEVICE_UNKNOWN'); + expect(message).toContain('iPhone 99'); + expect(message).toContain('fix: $B device --list'); + expect(message.match(/iPhone \d/g)?.length ?? 0).toBeGreaterThanOrEqual(3); + expect(await probe()).toEqual(before); + expect(await run('device')).toContain('none'); + }); + + test('--list prints Playwright device names', async () => { + const list = await run('device', ['--list']); + expect(list).toContain('iPhone 15'); + expect(list).toContain('Pixel 7'); + }); + + test('emulates the phone on this tab only; device off restores', async () => { + const desktopId = await freshTouchTab(null); + const desktop = await probe(); + await freshTouchTab('iphone 15'); + const phone = await probe(); + expect(phone.coarse).toBe(true); + expect(phone.maxTouchPoints).toBe(5); + expect(phone.ua).toContain('iPhone'); + expect(phone.width).toBe(393); + expect(await run('device')).toContain('iPhone 15'); + + const other = bm.getSession(desktopId).getPage(); + const otherNow = await other.evaluate(() => ({ coarse: matchMedia('(pointer: coarse)').matches, maxTouchPoints: navigator.maxTouchPoints, ua: navigator.userAgent, width: innerWidth })); + expect(otherNow).toEqual(desktop); + expect(otherNow.ua).not.toContain('iPhone'); + + await run('device', ['off']); + await page().reload({ waitUntil: 'domcontentloaded' }); + expect(await probe()).toEqual(desktop); + expect(await run('device')).toContain('none'); + }); + + test('--landscape swaps the orientation', async () => { + await freshTouchTab(null); + await run('device', ['Pixel 7', '--landscape']); + const p = await page().evaluate(() => ({ w: innerWidth, h: innerHeight })); + expect(p.w).toBeGreaterThan(p.h); + await run('device', ['off']); + }); + + test('viewport and responsive refuse while a device is active', async () => { + await freshTouchTab(); + await expect(run('viewport', ['800x600'])).rejects.toThrow('fix: $B device off'); + await expect(handleMetaCommand('responsive', [], bm, shutdown)).rejects.toThrow('fix: $B device off'); + expect((await probe()).width).toBe(393); + await run('device', ['off']); + }); + + test('full-page screenshot keeps the phone metrics', async () => { + await freshTouchTab(); + const out = path.join(os.tmpdir(), `touch-shot-${process.pid}.png`); + await handleMetaCommand('screenshot', [out], bm, shutdown); + fs.rmSync(out, { force: true }); + const p = await probe(); + expect(p.width).toBe(393); + expect(p.coarse).toBe(true); + await run('device', ['off']); + }); + + test('survives a useragent context rebuild; the first restored request carries the phone UA', async () => { + const id = await bm.newTab(baseUrl + '/echo'); + bm.switchTab(id, { bringToFront: false }); + await run('device', ['iPhone 15']); + await run('useragent', ['GstackDesktop/1.0']); + expect(await run('device')).toContain('iPhone 15'); + const echoed = JSON.parse(await page().evaluate(() => document.body.innerText)); + expect(echoed['user-agent']).toContain('iPhone'); + expect((await probe()).coarse).toBe(true); + await run('device', ['off']); + }); +}); + +describe('touch input', () => { + test('touch verbs need an active device', async () => { + await freshTouchTab(null); + await expect(run('tap', ['#fire'])).rejects.toThrow('BROWSE_TOUCH_NO_DEVICE'); + await expect(run('touch', ['down', '0', '10', '10'])).rejects.toThrow('fix: $B device "iPhone 15"'); + }); + + test('tap fires touchstart and touchend', async () => { + await freshTouchTab(); + await run('tap', ['#fire']); + expect(await data('#fire', 'shots')).toBe('1'); + const types = await page().evaluate(() => (window as any).__log.map((e: any) => e.type)); + expect(types).toContain('touchstart'); + expect(types).toContain('touchend'); + await run('tap', ['240', '440', '--hold', '50']); + expect(await data('#fire', 'shots')).toBe('2'); + }); + + test('swipe left advances the carousel; right goes back', async () => { + await freshTouchTab(); + await run('swipe', ['#carousel', 'left', '--distance', '200']); + expect(await data('#carousel', 'index')).toBe('1'); + await run('swipe', ['#carousel', 'right', '--distance', '200', '--ms', '100']); + expect(await data('#carousel', 'index')).toBe('0'); + }); + + test('pinch spreads above 1 and pinches below 1', async () => { + await freshTouchTab(); + await run('pinch', ['#zoom', '2']); + const spread = Number(await data('#zoom', 'scale')); + expect(spread).toBeGreaterThan(1.5); + await run('pinch', ['#zoom', '0.5']); + expect(Number(await data('#zoom', 'scale'))).toBeLessThan(spread * 0.7); + }); + + test('holding finger 0 while tapping finger 1 leaves finger 0 down', async () => { + await freshTouchTab(); + await run('touch', ['down', '0', '70', '440']); + await run('touch', ['down', '1', '240', '440']); + await run('touch', ['up', '1']); + expect(await data('#fire', 'shots')).toBe('1'); + expect(await data('#joystick', 'held')).toBe('true'); + expect(await page().evaluate(() => (window as any).__log.at(-1).touches)).toBe(1); + const held = await run('touch'); + expect(held).toContain('0'); + expect(held).not.toMatch(/\b1\b.*\(/); + await run('touch', ['move', '0', '90', '440']); + await run('touch', ['up', '0']); + expect(await data('#joystick', 'held')).toBe('false'); + expect(await run('touch')).toContain('none'); + }); + + test('usage errors name the problem and give a fix', async () => { + await freshTouchTab(); + await expect(run('touch', ['up', '3'])).rejects.toThrow(/finger 3 is not down[\s\S]*fix:/); + await expect(run('touch', ['down', '12', '1', '1'])).rejects.toThrow(/0-9[\s\S]*fix:/); + await expect(run('swipe', ['#carousel', 'sideways'])).rejects.toThrow(/up\|down\|left\|right[\s\S]*fix:/); + await expect(run('pinch', ['#zoom', '-1'])).rejects.toThrow(/scale[\s\S]*fix:/); + }); + + test('held fingers are cancelled on navigation and by touch cancel', async () => { + await freshTouchTab(); + await run('touch', ['down', '0', '70', '440']); + expect(await run('touch')).toContain('0'); + await page().goto(baseUrl + '/touch.html', { waitUntil: 'domcontentloaded' }); + expect(await run('touch')).toContain('none'); + await run('touch', ['down', '2', '240', '440']); + expect(await data('#fire', 'shots')).toBe('1'); + await run('touch', ['cancel']); + expect(await run('touch')).toContain('none'); + expect(await page().evaluate(() => (window as any).__log.at(-1).type)).toBe('touchcancel'); + }); +}); From 33940a503a4c6985b78e2697d8248a0038cd0639 Mon Sep 17 00:00:00 2001 From: garrytan Date: Sat, 31 Oct 2026 23:18:17 +0000 Subject: [PATCH 2/3] feat(browse): per-tab phone emulation and touch input ($B device, tap, swipe, pinch, touch; #3058, GSTA-227 F3) Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> --- browse/src/browser-manager.ts | 9 +- browse/src/cli.ts | 4 + browse/src/commands.ts | 9 +- browse/src/device-emulation.ts | 225 +++++++++++++++++++++++++ browse/src/meta-commands.ts | 12 +- browse/src/tab-session.ts | 46 +++++ browse/src/token-registry.ts | 1 + browse/src/touch-commands.ts | 184 ++++++++++++++++++++ browse/src/write-commands.ts | 5 + browse/test/touch-commands.test.ts | 25 +-- docs/troubleshooting.md | 35 ++++ lib/result-codes-browse.ts | 25 +++ lib/result-codes.ts | 2 + test/fixtures/module-size-ratchet.json | 3 + 14 files changed, 569 insertions(+), 16 deletions(-) create mode 100644 browse/src/device-emulation.ts create mode 100644 browse/src/touch-commands.ts create mode 100644 lib/result-codes-browse.ts diff --git a/browse/src/browser-manager.ts b/browse/src/browser-manager.ts index a6ff2a7c8..d43509094 100644 --- a/browse/src/browser-manager.ts +++ b/browse/src/browser-manager.ts @@ -21,13 +21,14 @@ import { writeSecureFile, mkdirSecure } from './file-permissions'; import { addConsoleEntry, addNetworkEntry, addDialogEntry, networkBuffer, type DialogEntry } from './buffers'; import { emitActivity } from './activity'; import { validateNavigationUrl, blockedNavigationReason } from './url-validation'; -import { TabSession, type RefEntry } from './tab-session'; +import { TabSession, type DeviceProfile, type RefEntry } from './tab-session'; import { resolveChromiumProfile, cleanSingletonLocks, resolveConfig } from './config'; import { ensureProjectProfile } from './chromium-profiles'; import { launchWithXProtectHeal } from './xprotect-heal'; import { DIRECT_PROXY_ARG, forcesDirect } from './proxy-config'; import { readPidStartTime, shouldSpawnXvfb, spawnFreeXvfb, xvfbInstallHint, type XvfbHandle } from './xvfb'; import { withCdpSession } from './cdp-bridge'; +import { applyDevice } from './device-emulation'; import { resultLine } from '../../lib/result-codes'; import { sanitizeEcho } from '../../lib/sanitize-echo'; import type { MemorySnapshot, MemoryStructureStats, MemoryTabSnapshot, MemoryProcess } from './memory-snapshot'; @@ -352,6 +353,8 @@ export interface BrowserState { * In-memory only. */ owner?: string; + /** `$B device` profile, re-applied before the restored tab's first request. */ + device?: DeviceProfile; }>; } @@ -1831,6 +1834,7 @@ export class BrowserManager { loadedHtml: loaded?.html, loadedHtmlWaitUntil: loaded?.waitUntil, owner, + device: session?.getDevice() ?? undefined, }); } @@ -1893,6 +1897,9 @@ export class BrowserManager { if (saved.owner) { this.tabOwnership.set(id, saved.owner); } + if (saved.device) { + await applyDevice(newSession, saved.device).catch((err: any) => console.warn(`[browse] Failed to re-apply device for tab ${id}: ${err.message}`)); + } if (saved.loadedHtml) { // Replay load-html content via setTabContent — this rehydrates diff --git a/browse/src/cli.ts b/browse/src/cli.ts index 5bea3006e..d741ec177 100644 --- a/browse/src/cli.ts +++ b/browse/src/cli.ts @@ -1856,6 +1856,10 @@ Inspection: js | eval | css | attrs is (visible|hidden|enabled|disabled|checked|editable|focused) Visual: screenshot [--viewport] [--clip x,y,w,h] [@ref|sel] [path] pdf [path] | responsive [prefix] +Touch: device ["" [--landscape] | off | --list] (this tab only) + tap <@ref|sel|x y> [--hold ] | pinch <@ref|sel|x y> + swipe <@ref|sel|x y> [--distance ] [--ms ] + touch [down|move | up | cancel] Snapshot: snapshot [-i] [-c] [-d N] [-s sel] [-D] [-a] [-o path] [-C] -D/--diff: diff against previous snapshot -a/--annotate: annotated screenshot with ref labels diff --git a/browse/src/commands.ts b/browse/src/commands.ts index 2d2a26015..42af45943 100644 --- a/browse/src/commands.ts +++ b/browse/src/commands.ts @@ -27,6 +27,7 @@ export const WRITE_COMMANDS = new Set([ 'upload', 'dialog-accept', 'dialog-dismiss', 'style', 'cleanup', 'prettyscreenshot', 'download', 'scrape', 'archive', + 'device', 'tap', 'swipe', 'pinch', 'touch', ]); export const META_COMMANDS = new Set([ @@ -135,6 +136,12 @@ export const COMMAND_DESCRIPTIONS: Record' }, 'dialog-accept': { category: 'Interaction', description: 'Auto-accept next alert/confirm/prompt. Optional text is sent as the prompt response', usage: 'dialog-accept [text]' }, 'dialog-dismiss': { category: 'Interaction', description: 'Auto-dismiss next dialog' }, + // Phone emulation and touch (this tab only; gstack's own Chromium) + 'device': { category: 'Touch', description: 'Emulate a phone on the current tab only: size, pixel ratio, touch support (pointer: coarse) and user agent from Playwright\'s devices table (names match case-insensitively). Other tabs stay desktop. No name prints the active profile or "none"; --list prints every name; off (or reset, auto) restores. viewport and responsive refuse while a device is active.', usage: 'device [ [--landscape] | off | --list]' }, + 'tap': { category: 'Touch', description: 'Touch and release at an element\'s center (scrolled into view) or at x y. Needs $B device first.', usage: 'tap <@ref|selector|x y> [--hold ]' }, + 'swipe': { category: 'Touch', description: 'One-finger swipe from an element\'s center or x y; the finger moves in the given direction (default 300px over 300ms, 10 moves). Needs $B device first.', usage: 'swipe <@ref|selector|x y> [--distance ] [--ms ]' }, + 'pinch': { category: 'Touch', description: 'Two-finger pinch around an element\'s center or x y: scale above 1 spreads the fingers (zoom in), below 1 brings them together. Needs $B device first.', usage: 'pinch <@ref|selector|x y> ' }, + 'touch': { category: 'Touch', description: 'Hold, move and lift numbered fingers for multi-finger flows (hold a joystick with finger 0, tap fire with finger 1). Lifting one finger leaves the others down. No action lists held fingers; cancel releases all. Navigation and device off cancel held fingers. Needs $B device first.', usage: 'touch [down|move | up | cancel]' }, // Data extraction 'download': { category: 'Extraction', description: 'Download URL or media element to disk using browser cookies. Default path: /browse-download-.. --base64 returns a data:;base64,... string instead of writing, capped at 10MB. Use --navigate for URLs that trigger browser downloads (CDN redirects, Content-Disposition, anti-bot protected sites).', usage: 'download [path] [--base64] [--navigate]' }, 'scrape': { category: 'Extraction', description: 'Bulk download all media from page to --dir (default: /browse-scrape-). Writes files plus manifest.json with source URL, size, type, success/failure counts. --limit defaults to 50 and caps at 200.', usage: 'scrape [--selector sel] [--dir path] [--limit N]' }, @@ -231,7 +238,7 @@ export const NEW_IN_VERSION: Record = { * Levenshtein distance (dynamic programming). * O(a.length * b.length) — fast for command name sizes (<20 chars). */ -function levenshtein(a: string, b: string): number { +export function levenshtein(a: string, b: string): number { if (a === b) return 0; if (a.length === 0) return b.length; if (b.length === 0) return a.length; diff --git a/browse/src/device-emulation.ts b/browse/src/device-emulation.ts new file mode 100644 index 000000000..567222f30 --- /dev/null +++ b/browse/src/device-emulation.ts @@ -0,0 +1,225 @@ +/** + * Phone emulation per tab (`$B device`, #3058). Sizes and user agents come + * from Playwright's own `devices` table. Everything is applied through the + * tab's own CDP session, never through context options or recreateContext(), + * so one agent's phone never resizes another agent's tab. + * + * device "iPhone 15" ─▶ Emulation.setDeviceMetricsOverride (mobile, screen, orientation) + * ─▶ Emulation.setTouchEmulationEnabled (maxTouchPoints 5) + * ─▶ Emulation.setUserAgentOverride (UA, platform, UA-CH metadata) + * device off ─▶ the three clears, then Playwright's viewport re-applied + * + * The profile lives on TabSession; BrowserManager.saveState()/restoreState() + * carry it across context recreation and re-apply it before the first request. + */ + +import * as fs from 'fs'; +import { devices } from 'playwright'; +import type { Page } from 'playwright'; +import type { DeviceProfile, TabSession } from './tab-session'; +import { getOrCreateCdpSession } from './cdp-bridge'; +import { levenshtein } from './commands'; +import { resultLine } from '../../lib/result-codes'; + +const DEVICE_NAMES = Object.keys(devices); +const cdpSessions: WeakMap = new WeakMap(); + +/** The tab's long-lived CDP session; emulation overrides live as long as it does. */ +export function tabCdpSession(page: Page): Promise { + return getOrCreateCdpSession(page, cdpSessions); +} + +export function describeDevice(profile: DeviceProfile | null): string { + if (!profile) return 'none'; + return profile.landscape ? `${profile.name} (landscape)` : profile.name; +} + +/** Usage error for commands that would fight an active phone profile. */ +export function refuseWhileDevice(session: TabSession, command: string): void { + const profile = session.getDevice(); + if (!profile) return; + throw new Error( + `Usage: ${command} cannot change the size of a tab that emulates ${describeDevice(profile)}; the phone profile sets its size.\n` + + 'fix: $B device off', + ); +} + +function findDevice(name: string): string | null { + const lower = name.toLowerCase(); + return DEVICE_NAMES.find((n) => n.toLowerCase() === lower) ?? null; +} + +function nearestDevices(name: string): string[] { + const lower = name.toLowerCase(); + return DEVICE_NAMES + .filter((n) => !n.endsWith(' landscape')) + .map((n) => ({ n, d: levenshtein(lower, n.toLowerCase()) })) + .sort((a, b) => a.d - b.d || a.n.localeCompare(b.n)) + .slice(0, 3) + .map((x) => x.n); +} + +function resolveProfile(profile: DeviceProfile) { + const name = findDevice(profile.name); + if (!name) { + throw new Error(resultLine('browse', 'BROWSE_DEVICE_UNKNOWN', + `"${profile.name}"; nearest: ${nearestDevices(profile.name).map((n) => `"${n}"`).join(', ')}`)); + } + const landscapeName = profile.landscape ? findDevice(`${name} landscape`) : null; + const d = devices[landscapeName ?? name]!; + const landscape = profile.landscape || name.endsWith(' landscape'); + const swap = profile.landscape && !landscapeName; + const viewport = swap ? { width: d.viewport.height, height: d.viewport.width } : d.viewport; + const screen = (d as { screen?: { width: number; height: number } }).screen ?? d.viewport; + return { + profile: { name, landscape: profile.landscape }, + viewport, + screen: swap ? { width: screen.height, height: screen.width } : screen, + deviceScaleFactor: d.deviceScaleFactor, + isMobile: d.isMobile, + hasTouch: d.hasTouch, + userAgent: d.userAgent, + landscape, + }; +} + +function userAgentMetadata(ua: string, mobile: boolean) { + const android = ua.match(/Android (\d+(?:\.\d+)*)/); + const ios = ua.match(/(?:iPhone|CPU) OS (\d+(?:_\d+)*)/); + const chrome = ua.match(/Chrome\/(\d+)\.([\d.]+)/); + const platform = android ? 'Android' : ios ? 'iOS' : /Mac OS X/.test(ua) ? 'macOS' : /Windows/.test(ua) ? 'Windows' : 'Linux'; + return { + brands: chrome ? [{ brand: 'Chromium', version: chrome[1]! }, { brand: 'Not.A/Brand', version: '99' }] : [], + fullVersion: chrome ? `${chrome[1]}.${chrome[2]}` : '', + platform, + platformVersion: android ? android[1]! : ios ? ios[1]!.replace(/_/g, '.') : '', + architecture: android || ios ? 'arm' : 'x86', + model: android ? (ua.match(/Android [\d.]+; ([^;)]+)/)?.[1] ?? '') : '', + mobile, + }; +} + +function navigatorPlatform(ua: string): string { + if (/iPhone/.test(ua)) return 'iPhone'; + if (/iPad/.test(ua)) return 'iPad'; + if (/Android/.test(ua)) return 'Linux armv81'; + if (/Mac OS X/.test(ua)) return 'MacIntel'; + return /Windows/.test(ua) ? 'Win32' : 'Linux x86_64'; +} + +async function sendAll(page: Page, steps: Array<[string, Record]>, partly: string): Promise { + const cdp = await tabCdpSession(page); + let done = 0; + for (const [method, params] of steps) { + try { + await cdp.send(method, params); + done++; + } catch (err: any) { + const failure = new Error(resultLine('browse', 'BROWSE_DEVICE_CDP_FAILED', + `${method}: ${err?.message ?? err}; ${done === 0 ? 'the tab is unchanged' : partly}`)); + throw Object.assign(failure, { partial: done > 0 }); + } + } +} + +/** Validate, then emulate the phone on this tab. Throws before touching the browser on an unknown name. */ +export async function applyDevice(session: TabSession, requested: DeviceProfile): Promise { + const r = resolveProfile(requested); + const page = session.getPage(); + try { + await sendAll(page, [ + ['Emulation.setDeviceMetricsOverride', { + width: r.viewport.width, + height: r.viewport.height, + deviceScaleFactor: r.deviceScaleFactor, + mobile: r.isMobile, + screenWidth: r.screen.width, + screenHeight: r.screen.height, + screenOrientation: r.landscape ? { type: 'landscapePrimary', angle: 90 } : { type: 'portraitPrimary', angle: 0 }, + }], + ['Emulation.setTouchEmulationEnabled', { enabled: r.hasTouch, maxTouchPoints: 5 }], + ['Emulation.setUserAgentOverride', { + userAgent: r.userAgent, + platform: navigatorPlatform(r.userAgent), + userAgentMetadata: userAgentMetadata(r.userAgent, r.isMobile), + }], + ], 'the tab is partly emulated'); + } catch (err: any) { + if (err.partial) session.setDevice(r.profile); + throw err; + } + session.setDevice(r.profile); + return `Emulating ${describeDevice(r.profile)} on this tab: ${r.viewport.width}x${r.viewport.height} @${r.deviceScaleFactor}x, ` + + `${r.hasTouch ? 'touch on' : 'no touch'}, ${r.isMobile ? 'mobile' : 'desktop'} user agent. Reload to rerun page scripts; $B device off restores.`; +} + +/** Drop held fingers and clear the three overrides; Playwright's own viewport comes back. */ +export async function clearDevice(session: TabSession): Promise { + const page = session.getPage(); + const had = session.getDevice(); + session.dropFingers(); + await sendAll(page, [ + ['Emulation.clearDeviceMetricsOverride', {}], + ['Emulation.setTouchEmulationEnabled', { enabled: false }], + ['Emulation.setUserAgentOverride', { userAgent: '' }], + ], 'the tab is partly restored'); + session.setDevice(null); + const viewport = page.viewportSize(); + if (viewport) await page.setViewportSize(viewport); + return had ? `Device emulation off (was ${describeDevice(had)}); desktop size, mouse input and user agent restored` : 'No device was active on this tab'; +} + +/** + * Playwright re-sends its own metrics after a full-page capture, which puts the + * tab back at desktop size. Chromium skips a repeated identical override from + * the same session, so clear ours first, then apply the phone again. + */ +export async function reassertDevice(session: TabSession): Promise { + const profile = session.getDevice(); + if (!profile) return; + await sendAll(session.getPage(), [['Emulation.clearDeviceMetricsOverride', {}]], 'the tab is partly emulated'); + await applyDevice(session, profile); +} + +/** + * Screenshot that keeps a device tab at phone size. Playwright's own capture + * clips to its desktop viewport and then re-sends that viewport, so a device + * tab's viewport and full-page captures go through the tab's CDP session + * (at the phone's pixel ratio); clip captures go through Playwright. Both + * Playwright's capture and a full-page one drop parts of the emulation, so the + * phone is put back afterwards. + */ +export async function screenshotTab(session: TabSession, opts: { path?: string; fullPage?: boolean; clip?: { x: number; y: number; width: number; height: number } }): Promise { + const page = session.getPage(); + if (!session.getDevice()) return page.screenshot(opts); + if (opts.clip) { + const clipped = await page.screenshot(opts); + await reassertDevice(session); + return clipped; + } + const cdp = await tabCdpSession(page); + const params: Record = { format: 'png' }; + if (opts.fullPage) { + const { cssContentSize } = await cdp.send('Page.getLayoutMetrics'); + params.captureBeyondViewport = true; + params.clip = { x: 0, y: 0, width: Math.ceil(cssContentSize.width), height: Math.ceil(cssContentSize.height), scale: 1 }; + } + const { data } = await cdp.send('Page.captureScreenshot', params); + if (opts.fullPage) await reassertDevice(session); + const buffer = Buffer.from(data, 'base64'); + if (opts.path) fs.writeFileSync(opts.path, buffer); + return buffer; +} + +export async function handleDeviceCommand(args: string[], session: TabSession): Promise { + const landscape = args.includes('--landscape'); + const unknownFlag = args.find((a) => a.startsWith('--') && a !== '--landscape' && a !== '--list'); + if (unknownFlag) throw new Error(`Usage: unknown device flag ${unknownFlag}.\nfix: $B device [--landscape] | $B device off | $B device --list`); + if (args.includes('--list')) return DEVICE_NAMES.join('\n'); + const name = args.filter((a) => a !== '--landscape').join(' ').trim(); + if (!name) return describeDevice(session.getDevice()); + if (['off', 'reset', 'auto'].includes(name.toLowerCase())) return clearDevice(session); + const result = await applyDevice(session, { name, landscape }); + session.dropFingers(); + return result; +} diff --git a/browse/src/meta-commands.ts b/browse/src/meta-commands.ts index af49910a9..bb61959f7 100644 --- a/browse/src/meta-commands.ts +++ b/browse/src/meta-commands.ts @@ -17,6 +17,7 @@ import * as fs from 'fs'; import * as path from 'path'; import { writeSecureFile, mkdirSecure } from './file-permissions'; import { TEMP_DIR } from './platform'; +import { reassertDevice, refuseWhileDevice, screenshotTab } from './device-emulation'; import { resolveConfig } from './config'; import { filterSessionCookies } from './session-persist'; import type { Frame } from 'playwright'; @@ -511,10 +512,11 @@ export async function handleMetaCommand( const resolved = await bm.resolveRef(targetSelector); const locator = 'locator' in resolved ? resolved.locator : page.locator(resolved.selector); buffer = await locator.screenshot({ timeout: 5000 }); + await reassertDevice(bm.getActiveSession()); } else if (clipRect) { - buffer = await page.screenshot({ clip: clipRect }); + buffer = await screenshotTab(bm.getActiveSession(), { clip: clipRect }); } else { - buffer = await page.screenshot({ fullPage: !viewportOnly }); + buffer = await screenshotTab(bm.getActiveSession(), { fullPage: !viewportOnly }); // Guard the most common API-bricking case (fullPage). Element / // clip captures usually stay within the cap; we still guard the // path-mode below for fullPage writes. @@ -530,15 +532,16 @@ export async function handleMetaCommand( const resolved = await bm.resolveRef(targetSelector); const locator = 'locator' in resolved ? resolved.locator : page.locator(resolved.selector); await locator.screenshot({ path: outputPath, timeout: 5000 }); + await reassertDevice(bm.getActiveSession()); return `Screenshot saved (element): ${outputPath}`; } if (clipRect) { - await page.screenshot({ path: outputPath, clip: clipRect }); + await screenshotTab(bm.getActiveSession(), { path: outputPath, clip: clipRect }); return `Screenshot saved (clip ${clipRect.x},${clipRect.y},${clipRect.width},${clipRect.height}): ${outputPath}`; } - await page.screenshot({ path: outputPath, fullPage: !viewportOnly }); + await screenshotTab(bm.getActiveSession(), { path: outputPath, fullPage: !viewportOnly }); if (!viewportOnly) await guardScreenshotPath(outputPath); return `Screenshot saved${viewportOnly ? ' (viewport)' : ''}: ${outputPath}`; } @@ -574,6 +577,7 @@ export async function handleMetaCommand( } case 'responsive': { + refuseWhileDevice(bm.getActiveSession(), 'responsive'); const page = bm.getPage(); const prefix = args[0] || `${TEMP_DIR}/browse-responsive`; validateOutputPath(prefix); diff --git a/browse/src/tab-session.ts b/browse/src/tab-session.ts index dccabf28f..0ea85b6ad 100644 --- a/browse/src/tab-session.ts +++ b/browse/src/tab-session.ts @@ -26,6 +26,12 @@ export interface RefEntry { export type SetContentWaitUntil = 'load' | 'domcontentloaded' | 'networkidle'; +/** Phone profile applied by `$B device` (a Playwright `devices` name). */ +export interface DeviceProfile { + name: string; + landscape: boolean; +} + export class TabSession { readonly page: Page; @@ -63,6 +69,12 @@ export class TabSession { private loadedHtml: string | null = null; private loadedHtmlWaitUntil: SetContentWaitUntil | undefined; + // ─── Phone emulation and held touch fingers (device-emulation.ts, touch-commands.ts) ─ + private device: DeviceProfile | null = null; + private fingers: Map = new Map(); + private fingerCancel: (() => void) | null = null; + private gestureChain: Promise = Promise.resolve(); + constructor(page: Page) { this.page = page; } @@ -176,6 +188,40 @@ export class TabSession { this.activeFrame = null; this.loadedHtml = null; this.loadedHtmlWaitUntil = undefined; + this.dropFingers(); + } + + // ─── Phone emulation and touch ───────────────────────────── + getDevice(): DeviceProfile | null { + return this.device; + } + + setDevice(profile: DeviceProfile | null): void { + this.device = profile; + } + + /** Held fingers by finger number; touch-commands.ts mutates this map. */ + heldFingers(): Map { + return this.fingers; + } + + /** How to tell the browser the held fingers are gone (a CDP touchCancel). */ + setFingerCancel(cancel: () => void): void { + this.fingerCancel = cancel; + } + + /** Forget held fingers and cancel them in the page. Navigation and device changes call this. */ + dropFingers(): void { + if (this.fingers.size === 0) return; + this.fingers.clear(); + this.fingerCancel?.(); + } + + /** Run gestures on this tab one at a time. */ + runGesture(fn: () => Promise): Promise { + const next = this.gestureChain.then(fn, fn); + this.gestureChain = next.catch(() => {}); + return next; } // ─── Loaded HTML (load-html replay) ─────────────────────── diff --git a/browse/src/token-registry.ts b/browse/src/token-registry.ts index b241e55e4..4b5546f5c 100644 --- a/browse/src/token-registry.ts +++ b/browse/src/token-registry.ts @@ -56,6 +56,7 @@ export const SCOPE_WRITE = new Set([ 'upload', 'viewport', 'newtab', 'closetab', 'dialog-accept', 'dialog-dismiss', 'download', 'scrape', 'archive', + 'device', 'tap', 'swipe', 'pinch', 'touch', ]); /** Page-level power tools — JS execution, credential access, page mutations */ diff --git a/browse/src/touch-commands.ts b/browse/src/touch-commands.ts new file mode 100644 index 000000000..fedb29090 --- /dev/null +++ b/browse/src/touch-commands.ts @@ -0,0 +1,184 @@ +/** + * Touch input for phone-emulated tabs (`$B tap|swipe|pinch|touch`, #3058). + * Everything goes through Input.dispatchTouchEvent on the tab's own CDP + * session (the one device-emulation.ts set up), so it needs `$B device` first. + * + * held fingers (touch down 0-9) ── TabSession.heldFingers() + * gesture fingers (tap/swipe/pinch) use ids 10+, added beside held ones + * + * Chromium 151 dispatches an event only for the points it lists: touchStart + * with the new finger, touchMove with the moved ones, touchEnd with the lifted + * one (measured: a touchMove that omits a finger does not release it). Fingers + * not listed stay down, which is what multi-finger chords need. Gestures on one + * tab run one at a time (TabSession.runGesture). + */ + +import type { TabSession } from './tab-session'; +import { tabCdpSession } from './device-emulation'; +import { resultLine } from '../../lib/result-codes'; + +type Point = { x: number; y: number }; +type Points = Map; + +const GESTURE_ID = 10; +const STEPS = 10; +const sleep = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms)); + +function usage(problem: string, fix: string): Error { + return new Error(`Usage: ${problem}\nfix: ${fix}`); +} + +async function dispatch(session: TabSession, type: 'touchStart' | 'touchMove' | 'touchEnd', points: Points): Promise { + const cdp = await tabCdpSession(session.getPage()); + session.setFingerCancel(() => { + cdp.send('Input.dispatchTouchEvent', { type: 'touchCancel', touchPoints: [] }).catch(() => {}); + }); + const touchPoints = [...points].map(([id, p]) => ({ id, x: p.x, y: p.y, radiusX: 1, radiusY: 1, force: 1 })); + await cdp.send('Input.dispatchTouchEvent', { type, touchPoints }); +} + +function parseNumber(raw: string | undefined, what: string, fix: string): number { + const n = Number(raw); + if (raw === undefined || raw === '' || !Number.isFinite(n)) throw usage(`${what} must be a number, got "${raw ?? ''}".`, fix); + return n; +} + +function flag(args: string[], name: string, fallback: number, fix: string): number { + const i = args.indexOf(name); + if (i === -1) return fallback; + const n = parseNumber(args[i + 1], name, fix); + if (n < 0 || n > 10000) throw usage(`${name} must be between 0 and 10000, got ${n}.`, fix); + args.splice(i, 2); + return n; +} + +/** `<@ref|selector|x y>` → a viewport point; element targets scroll into view first. */ +async function resolvePoint(session: TabSession, args: string[], fix: string): Promise { + const [a, b] = args; + if (!a) throw usage('missing target (@ref, selector, or x y).', fix); + if (Number.isFinite(Number(a)) && Number.isFinite(Number(b)) && b !== undefined && b !== '') { + args.splice(0, 2); + return { x: Number(a), y: Number(b) }; + } + args.splice(0, 1); + const resolved = await session.resolveRef(a); + const locator = 'locator' in resolved ? resolved.locator : session.getActiveFrameOrPage().locator(resolved.selector).first(); + await locator.scrollIntoViewIfNeeded({ timeout: 5000 }); + const box = await locator.boundingBox(); + if (!box) throw usage(`${a} has no visible box to touch.`, `$B snapshot -i, then target a visible element`); + return { x: box.x + box.width / 2, y: box.y + box.height / 2 }; +} + +async function viewportClamp(session: TabSession): Promise<(p: Point) => Point> { + const [w, h] = await session.getPage().evaluate(() => [innerWidth, innerHeight]); + return (p) => ({ x: Math.min(Math.max(p.x, 0), w - 1), y: Math.min(Math.max(p.y, 0), h - 1) }); +} + +/** Press gesture fingers at `from`, move them to `to` in STEPS moves over `ms`, lift them; held fingers stay down. */ +async function stroke(session: TabSession, from: Points, to: Points, ms: number): Promise { + await dispatch(session, 'touchStart', from); + let at: Points = from; + for (let i = 1; i <= STEPS; i++) { + await sleep(ms / STEPS); + at = new Map([...from].map(([id, a]) => { + const b = to.get(id)!; + return [id, { x: a.x + (b.x - a.x) * i / STEPS, y: a.y + (b.y - a.y) * i / STEPS }]; + })); + await dispatch(session, 'touchMove', at); + } + await dispatch(session, 'touchEnd', at); +} + +const fmt = (p: Point) => `(${Math.round(p.x)}, ${Math.round(p.y)})`; + +async function tap(session: TabSession, args: string[]): Promise { + const fix = '$B tap <@ref|selector|x y> [--hold ]'; + const hold = flag(args, '--hold', 0, fix); + const point = await resolvePoint(session, args, fix); + const finger: Points = new Map([[GESTURE_ID, point]]); + await dispatch(session, 'touchStart', finger); + if (hold) await sleep(hold); + await dispatch(session, 'touchEnd', finger); + return `Tapped ${fmt(point)}${hold ? ` held ${hold}ms` : ''}`; +} + +async function swipe(session: TabSession, args: string[]): Promise { + const fix = '$B swipe <@ref|selector|x y> [--distance ] [--ms ]'; + const distance = flag(args, '--distance', 300, fix); + const ms = flag(args, '--ms', 300, fix); + const start = await resolvePoint(session, args, fix); + const direction = args[0]; + const delta: Record = { up: { x: 0, y: -1 }, down: { x: 0, y: 1 }, left: { x: -1, y: 0 }, right: { x: 1, y: 0 } }; + const unit = direction ? delta[direction] : undefined; + if (!unit || args.length > 1) throw usage(`swipe direction must be up|down|left|right, got "${args.join(' ')}".`, fix); + const clamp = await viewportClamp(session); + const end = clamp({ x: start.x + unit.x * distance, y: start.y + unit.y * distance }); + await stroke(session, new Map([[GESTURE_ID, start]]), new Map([[GESTURE_ID, end]]), ms); + const moved = Math.round(Math.hypot(end.x - start.x, end.y - start.y)); + return `Swiped ${direction} ${moved}px from ${fmt(start)} in ${ms}ms${moved < distance ? ' (stopped at the viewport edge)' : ''}`; +} + +async function pinch(session: TabSession, args: string[]): Promise { + const fix = '$B pinch <@ref|selector|x y> (2 spreads two fingers apart, 0.5 pinches them together)'; + const centre = await resolvePoint(session, args, fix); + const scale = parseNumber(args[0], 'pinch scale', fix); + if (scale <= 0 || scale > 10 || args.length > 1) throw usage(`pinch scale must be one number above 0 and at most 10, got "${args.join(' ')}".`, fix); + const clamp = await viewportClamp(session); + const startHalf = scale >= 1 ? 40 : 40 / scale; + const at = (half: number) => new Map([ + [GESTURE_ID, clamp({ x: centre.x - half, y: centre.y })], + [GESTURE_ID + 1, clamp({ x: centre.x + half, y: centre.y })], + ]); + await stroke(session, at(startHalf), at(startHalf * scale), 300); + return `Pinched at ${fmt(centre)} to scale ${scale}`; +} + +function fingerArg(raw: string | undefined, fix: string): number { + const finger = Number(raw); + if (!Number.isInteger(finger) || finger < 0 || finger > 9) throw usage(`finger must be 0-9, got "${raw ?? ''}".`, fix); + return finger; +} + +function listFingers(held: Points): string { + if (held.size === 0) return 'Held fingers: none'; + return 'Held fingers:\n' + [...held].map(([f, p]) => ` finger ${f} at ${fmt(p)}`).join('\n'); +} + +async function touch(session: TabSession, args: string[]): Promise { + const fix = '$B touch down|move | $B touch up | $B touch cancel | $B touch'; + const held = session.heldFingers(); + const [sub, rawFinger, rawX, rawY] = args; + if (!sub) return listFingers(held); + if (sub === 'cancel') { + const cdp = await tabCdpSession(session.getPage()); + await cdp.send('Input.dispatchTouchEvent', { type: 'touchCancel', touchPoints: [] }); + held.clear(); + return 'Cancelled all held fingers'; + } + if (!['down', 'move', 'up'].includes(sub)) throw usage(`unknown touch action "${sub}".`, fix); + const finger = fingerArg(rawFinger, fix); + if (sub === 'down' && held.has(finger)) throw usage(`finger ${finger} is already down.`, `$B touch move ${finger} or $B touch up ${finger}`); + if (sub !== 'down' && !held.has(finger)) { + throw usage(`finger ${finger} is not down (${listFingers(held).toLowerCase()}).`, `$B touch down ${finger} `); + } + if (sub === 'up') { + await dispatch(session, 'touchEnd', new Map([[finger, held.get(finger)!]])); + held.delete(finger); + return `Finger ${finger} up; ${listFingers(held).toLowerCase()}`; + } + const point = { x: parseNumber(rawX, 'x', fix), y: parseNumber(rawY, 'y', fix) }; + await dispatch(session, sub === 'down' ? 'touchStart' : 'touchMove', new Map([[finger, point]])); + held.set(finger, point); + return `Finger ${finger} ${sub} at ${fmt(point)}`; +} + +export const TOUCH_COMMANDS = new Set(['tap', 'swipe', 'pinch', 'touch']); + +export async function handleTouchCommand(command: string, args: string[], session: TabSession): Promise { + if (!session.getDevice()) { + throw new Error(resultLine('browse', 'BROWSE_TOUCH_NO_DEVICE', `$B ${command} on this tab`)); + } + const rest = [...args]; + const run = { tap, swipe, pinch, touch }[command as 'tap' | 'swipe' | 'pinch' | 'touch']; + return session.runGesture(() => run(session, rest)); +} diff --git a/browse/src/write-commands.ts b/browse/src/write-commands.ts index 7bd23b504..81a683c5b 100644 --- a/browse/src/write-commands.ts +++ b/browse/src/write-commands.ts @@ -21,6 +21,8 @@ import { TEMP_DIR } from './platform'; import { SAFE_DIRECTORIES } from './path-security'; import { modifyStyle, undoModification, resetModifications, getModificationHistory } from './cdp-inspector'; import { withCdpSession } from './cdp-bridge'; +import { handleDeviceCommand, refuseWhileDevice } from './device-emulation'; +import { TOUCH_COMMANDS, handleTouchCommand } from './touch-commands'; /** * Aggressive page cleanup selectors and heuristics. @@ -141,6 +143,8 @@ export async function handleWriteCommand( // Frame-aware target for locator-based operations (click, fill, etc.) const target = session.getActiveFrameOrPage(); const inFrame = session.getFrame() !== null; + if (command === 'device') return handleDeviceCommand(args, session); + if (TOUCH_COMMANDS.has(command)) return handleTouchCommand(command, args, session); switch (command) { case 'goto': { @@ -526,6 +530,7 @@ export async function handleWriteCommand( } case 'viewport': { + refuseWhileDevice(session, 'viewport'); // Parse args: [] [--scale ]. Either may be omitted, but NOT both. let sizeArg: string | undefined; let scaleArg: number | undefined; diff --git a/browse/test/touch-commands.test.ts b/browse/test/touch-commands.test.ts index a8caa41d8..9438917c3 100644 --- a/browse/test/touch-commands.test.ts +++ b/browse/test/touch-commands.test.ts @@ -6,7 +6,7 @@ * a two-finger zoom surface and a joystick plus fire button. */ -import { afterAll, beforeAll, describe, expect, test } from 'bun:test'; +import { afterAll, beforeAll, describe, expect, setDefaultTimeout, test } from 'bun:test'; import * as fs from 'fs'; import * as os from 'os'; import * as path from 'path'; @@ -25,6 +25,7 @@ let bm: BrowserManager; let baseUrl: string; const shutdown = async () => {}; const VERBS = ['device', 'tap', 'swipe', 'pinch', 'touch']; +setDefaultTimeout(20_000); beforeAll(async () => { CHROMIUM_PROFILE_DIR = fs.mkdtempSync(path.join(os.tmpdir(), 'gstack-test-profile-')); @@ -130,14 +131,17 @@ describe('device', () => { await run('device', ['off']); }); - test('full-page screenshot keeps the phone metrics', async () => { + test('screenshots capture the phone and keep its metrics', async () => { await freshTouchTab(); const out = path.join(os.tmpdir(), `touch-shot-${process.pid}.png`); - await handleMetaCommand('screenshot', [out], bm, shutdown); + for (const extra of [[], ['--viewport'], ['#zoom'], ['--clip', '0,0,50,50']]) { + await handleMetaCommand('screenshot', [...extra, out], bm, shutdown); + const png = fs.readFileSync(out); + if (extra.length === 0 || extra[0] === '--viewport') expect(png.readUInt32BE(16)).toBe(393 * 3); + const p = await probe(); + expect({ extra, width: p.width, coarse: p.coarse, touch: p.maxTouchPoints }).toEqual({ extra, width: 393, coarse: true, touch: 5 }); + } fs.rmSync(out, { force: true }); - const p = await probe(); - expect(p.width).toBe(393); - expect(p.coarse).toBe(true); await run('device', ['off']); }); @@ -208,10 +212,11 @@ describe('touch input', () => { test('usage errors name the problem and give a fix', async () => { await freshTouchTab(); - await expect(run('touch', ['up', '3'])).rejects.toThrow(/finger 3 is not down[\s\S]*fix:/); - await expect(run('touch', ['down', '12', '1', '1'])).rejects.toThrow(/0-9[\s\S]*fix:/); - await expect(run('swipe', ['#carousel', 'sideways'])).rejects.toThrow(/up\|down\|left\|right[\s\S]*fix:/); - await expect(run('pinch', ['#zoom', '-1'])).rejects.toThrow(/scale[\s\S]*fix:/); + const message = (cmd: string, args: string[]) => run(cmd, args).then(() => '', (err: Error) => err.message); + expect(await message('touch', ['up', '3'])).toMatch(/finger 3 is not down[\s\S]*fix:/); + expect(await message('touch', ['down', '12', '1', '1'])).toMatch(/0-9[\s\S]*fix:/); + expect(await message('swipe', ['#carousel', 'sideways'])).toMatch(/up\|down\|left\|right[\s\S]*fix:/); + expect(await message('pinch', ['#zoom', '-1'])).toMatch(/scale[\s\S]*fix:/); }); test('held fingers are cancelled on navigation and by touch cancel', async () => { diff --git a/docs/troubleshooting.md b/docs/troubleshooting.md index 74cda9b34..4d996ec6b 100644 --- a/docs/troubleshooting.md +++ b/docs/troubleshooting.md @@ -2057,6 +2057,41 @@ command again. **Fixed in.** The first release after v1.91.101.0 (#3167). + +### ``browse: no Playwright device has that name, so nothing on the tab changed (""; nearest: ...); fix: $B device --list (BROWSE_DEVICE_UNKNOWN)`` + +**Meaning.** `$B device ` matches names from Playwright's `devices` +table, case-insensitively. The name you gave is not in it, so the tab was not +touched. The line lists the three nearest names. + +**Fix.** `$B device --list` prints every name; quote names with spaces +(`$B device "iPhone 15"`). Add `--landscape` for the rotated profile. + +**Fixed in.** The first release after v1.92.2.0 (#3058). + + +### ``browse: Chromium refused a device emulation call on this tab (: ; the tab is partly emulated); fix: $B device off (BROWSE_DEVICE_CDP_FAILED)`` + +**Meaning.** Phone emulation is three DevTools calls on the tab's own session +(size, touch, user agent). One failed; the line names it and says whether the +calls before it already changed the tab. + +**Fix.** `$B device off` clears all three, then try the device again. If it +keeps failing, the tab or browser is gone: `$B restart`. + +**Fixed in.** The first release after v1.92.2.0 (#3058). + + +### ``browse: touch input needs a phone profile on this tab first ($B tap on this tab); fix: $B device "iPhone 15" (BROWSE_TOUCH_NO_DEVICE)`` + +**Meaning.** `tap`, `swipe`, `pinch` and `touch` send real touch events, +which pages only see on a touch device. The current tab has no phone profile. + +**Fix.** `$B device "iPhone 15"` (or any name from `$B device --list`) on +the same tab, then repeat the touch command. + +**Fixed in.** The first release after v1.92.2.0 (#3058). + --- ## Windows diff --git a/lib/result-codes-browse.ts b/lib/result-codes-browse.ts new file mode 100644 index 000000000..35e093d21 --- /dev/null +++ b/lib/result-codes-browse.ts @@ -0,0 +1,25 @@ +/** + * Result codes for the browse daemon's phone emulation and touch verbs + * (`$B device`, `$B tap|swipe|pinch|touch`; #3058). Kept beside + * lib/result-codes.ts, which spreads this table into RESULT_CODES; the same + * rules apply: one stable docs/troubleshooting.md anchor, a summary and a fix. + */ +import type { ResultCode } from './result-codes'; + +export const BROWSE_RESULT_CODES = { + BROWSE_DEVICE_UNKNOWN: { + anchor: 'browse-device-unknown', + summary: 'no Playwright device has that name, so nothing on the tab changed', + fix: '$B device --list', + }, + BROWSE_DEVICE_CDP_FAILED: { + anchor: 'browse-device-cdp-failed', + summary: 'Chromium refused a device emulation call on this tab', + fix: '$B device off', + }, + BROWSE_TOUCH_NO_DEVICE: { + anchor: 'browse-touch-no-device', + summary: 'touch input needs a phone profile on this tab first', + fix: '$B device "iPhone 15"', + }, +} as const satisfies Record; diff --git a/lib/result-codes.ts b/lib/result-codes.ts index beaeb779b..b09c904f8 100644 --- a/lib/result-codes.ts +++ b/lib/result-codes.ts @@ -14,6 +14,7 @@ import { RELEASE_RESULT_CODES } from './result-codes-release'; import { CAPY_RESULT_CODES } from './result-codes-capy'; import { BUDGET_RESULT_CODES } from './result-codes-budget'; import { EVAL_RESULT_CODES } from './result-codes-eval'; +import { BROWSE_RESULT_CODES } from './result-codes-browse'; export interface ResultCode { /** Stable `` in docs/troubleshooting.md. */ @@ -763,6 +764,7 @@ export const RESULT_CODES = { ...CAPY_RESULT_CODES, ...BUDGET_RESULT_CODES, ...EVAL_RESULT_CODES, + ...BROWSE_RESULT_CODES, } as const satisfies Record; export type ResultCodeName = keyof typeof RESULT_CODES; diff --git a/test/fixtures/module-size-ratchet.json b/test/fixtures/module-size-ratchet.json index 8139bdb26..cc5bb27c6 100644 --- a/test/fixtures/module-size-ratchet.json +++ b/test/fixtures/module-size-ratchet.json @@ -102,6 +102,9 @@ { "file": "lib/result-codes-release.ts", "movedFrom": "lib/result-codes.ts (the release-path rows: ship queue, pregate timing, approval classes, state matrix; field list 2 wave B)" }, { "file": "lib/result-codes-capy.ts", "movedFrom": "lib/result-codes.ts (the Capy rows: approval record, Captain brief, PR screenshots; GSTA-61)" }, { "file": "lib/result-codes-budget.ts", "movedFrom": "lib/result-codes.ts (the /ship token budget and handoff rows; GSTA-152)" }, + { "file": "lib/result-codes-browse.ts", "movedFrom": "lib/result-codes.ts (the browse device and touch rows, #3058; GSTA-227 F1 adds its session rows here)" }, + { "file": "browse/src/device-emulation.ts", "movedFrom": "(new: per-tab phone emulation for $B device, #3058, GSTA-227 F3)" }, + { "file": "browse/src/touch-commands.ts", "movedFrom": "(new: $B tap, swipe, pinch and touch through Input.dispatchTouchEvent, #3058, GSTA-227 F3)" }, { "file": "lib/result-codes-eval.ts", "movedFrom": "lib/result-codes.ts (the /eval-plan preregistration rows and the no-gstack arm; GSTA-168)" }, { "file": "lib/ship-budget.ts", "movedFrom": "(new: /ship token budget ledger, meters and admission, GSTA-152)" }, { "file": "lib/ship-budget-cli.ts", "movedFrom": "(new: bin/gstack-ship-budget start/check/settle/close/resume, GSTA-152)" }, From 5d1bb202a0f079a859e299438d03a917ae4470f4 Mon Sep 17 00:00:00 2001 From: garrytan Date: Sat, 31 Oct 2026 23:24:36 +0000 Subject: [PATCH 3/3] docs(qa, aside): touch pointer in /qa, Aside touch recipe verified on Aside 1.26, report Phones row (GSTA-227 F3, #3058) Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> --- bin/gstack-doctor-components.sh | 3 +++ browse/SKILL.md | 11 +++++++++++ browse/sections/command-list.md | 5 +++++ browse/src/commands.ts | 10 +++++----- devex-review/SKILL.md | 11 +++++++++++ gstack/llms.txt | 5 +++++ qa/SKILL.md | 1 + qa/SKILL.md.tmpl | 1 + qa/templates/qa-report-template.md | 1 + scripts/resolvers/aside.ts | 11 +++++++++++ test/aside-driver.test.ts | 3 +++ 11 files changed, 57 insertions(+), 5 deletions(-) diff --git a/bin/gstack-doctor-components.sh b/bin/gstack-doctor-components.sh index 3fa027cb3..d02c08339 100644 --- a/bin/gstack-doctor-components.sh +++ b/bin/gstack-doctor-components.sh @@ -362,6 +362,9 @@ gstack_result_anchor() { PREREG_NO_GSTACK_ARM) printf '%s' 'prereg-no-gstack-arm' ;; EVAL_REPORT_NO_GSTACK_ROW) printf '%s' 'eval-report-no-gstack-row' ;; EVAL_ARM_DUPLICATE) printf '%s' 'eval-arm-duplicate' ;; + BROWSE_DEVICE_UNKNOWN) printf '%s' 'browse-device-unknown' ;; + BROWSE_DEVICE_CDP_FAILED) printf '%s' 'browse-device-cdp-failed' ;; + BROWSE_TOUCH_NO_DEVICE) printf '%s' 'browse-touch-no-device' ;; *) return 1 ;; esac } diff --git a/browse/SKILL.md b/browse/SKILL.md index e5792ca58..469c6970e 100644 --- a/browse/SKILL.md +++ b/browse/SKILL.md @@ -318,6 +318,17 @@ console.log("ASIDE_DIR=" + pwd); await closeTab(pg); console.log("GSTACK_STEP_OK ' ``` +**Touch (verified in Aside 1.26; add the Responsive override for phone size):** + +```bash +aside repl ' +const pg = await openTab(""), t = (m, p) => pg._sendToTarget(m, p); +await t("Emulation.setTouchEmulationEnabled", { enabled: true, maxTouchPoints: 5 }); +for (const [type, touchPoints] of [["touchStart", [{ x: , y: }]], ["touchEnd", []]]) await t("Input.dispatchTouchEvent", { type, touchPoints }); +await closeTab(pg); console.log("GSTACK_STEP_OK"); +' +``` + **Links and their status (same-origin; on a LOCAL target each link is HEAD-checked, on a real site the user's cookies would ride every request so links are listed as `LINK ?` unfetched — consent to LOOK is not consent to hit every URL):** ```bash diff --git a/browse/sections/command-list.md b/browse/sections/command-list.md index 94b7f8ac2..5c125502b 100644 --- a/browse/sections/command-list.md +++ b/browse/sections/command-list.md @@ -95,15 +95,20 @@ Refs are invalidated on navigation — run `snapshot` again after `goto`. | `cookie =` | Set cookie on current page domain | | `cookie-import ` | Import cookies from JSON file | | `cookie-import-browser [browser] [--domain d] [--profile p] [--all] [--clear-storage] [--verify-auth]` | Copy cookies from chrome, chromium, brave, edge, Windows-only opera, opera-gx, or macOS-only comet or arc. Omitted browser retains legacy comet; select the intended browser explicitly. --domain requires a matching current page; no scope flag opens the picker. --profile is the source directory; ambiguous profiles require selection. --all explicitly selects every non-expired cookie and cannot accompany --domain or --clear-storage. Storage is preserved unless --clear-storage resets captured-origin localStorage (shared across context tabs) and target-tab sessionStorage. --verify-auth requires daemon GSTACK_COOKIE_AUTH_SELECTOR and GSTACK_COOKIE_AUTH_EXPECTED_IDENTITY before startup; missing config rejects before mutation. Verified means an exact visible identity match, not cookie counts or HTTP 200. Windows native extraction remains disabled pending qualification. | +| `device [ [--landscape] | off | --list]` | Emulate a phone on the current tab only: size, pixel ratio, touch support (pointer: coarse) and user agent from Playwright's devices table (names match case-insensitively). Other tabs stay desktop. No name prints the active profile or "none"; --list prints every name; off (or reset, auto) restores. viewport and responsive refuse while a device is active. | | `dialog-accept [text]` | Auto-accept next alert/confirm/prompt. Optional text is sent as the prompt response | | `dialog-dismiss` | Auto-dismiss next dialog | | `fill ` | Fill input | | `header :` | Set custom request header (colon-separated, sensitive values auto-redacted) | | `hover ` | Hover element | +| `pinch <@ref|selector|x y> ` | Two-finger pinch around an element's center or x y: scale above 1 spreads the fingers (zoom in), below 1 brings them together. Needs $B device first. | | `press ` | Press a Playwright keyboard key against the focused element. Names are case-sensitive: Enter, Tab, Escape, ArrowUp/Down/Left/Right, Backspace, Delete, Home, End, PageUp, PageDown. Modifiers combine with +: Shift+Enter, Control+A, Meta+K. Single printable chars (a, A, 1) work too. Full key list: https://playwright.dev/docs/api/class-keyboard#keyboard-press | | `scroll [sel|@ref]` | With a selector, smooth-scrolls the element into view. Without a selector, jumps to page bottom. No --by/--to amount option; for pixel-precise scrolling use `js window.scrollTo(0, N)`. | | `select ` | Select dropdown option by value, label, or visible text | | `style | style --undo [N]` | Modify CSS property on element (with undo support) | +| `swipe <@ref|selector|x y> [--distance ] [--ms ]` | One-finger swipe from an element's center or x y; the finger moves in the given direction (default 300px over 300ms, 10 moves). Needs $B device first. | +| `tap <@ref|selector|x y> [--hold ]` | Touch and release at an element's center (scrolled into view) or at x y. Needs $B device first. | +| `touch [down|move | up | cancel]` | Hold, move and lift numbered fingers for multi-finger flows (hold a joystick with finger 0, tap fire with finger 1). Lifting one finger leaves the others down. No action lists held fingers; cancel releases all. Navigation and device off cancel held fingers. Needs $B device first. | | `type [--selector ] [--] ` | Send keystrokes to the focused element, or to with --selector (focuses it first). Text that starts with -- goes after a -- separator. | | `upload [file2...]` | Upload file(s) | | `useragent ` | Set user agent | diff --git a/browse/src/commands.ts b/browse/src/commands.ts index 42af45943..f46bd1394 100644 --- a/browse/src/commands.ts +++ b/browse/src/commands.ts @@ -137,11 +137,11 @@ export const COMMAND_DESCRIPTIONS: Record [--landscape] | off | --list]' }, - 'tap': { category: 'Touch', description: 'Touch and release at an element\'s center (scrolled into view) or at x y. Needs $B device first.', usage: 'tap <@ref|selector|x y> [--hold ]' }, - 'swipe': { category: 'Touch', description: 'One-finger swipe from an element\'s center or x y; the finger moves in the given direction (default 300px over 300ms, 10 moves). Needs $B device first.', usage: 'swipe <@ref|selector|x y> [--distance ] [--ms ]' }, - 'pinch': { category: 'Touch', description: 'Two-finger pinch around an element\'s center or x y: scale above 1 spreads the fingers (zoom in), below 1 brings them together. Needs $B device first.', usage: 'pinch <@ref|selector|x y> ' }, - 'touch': { category: 'Touch', description: 'Hold, move and lift numbered fingers for multi-finger flows (hold a joystick with finger 0, tap fire with finger 1). Lifting one finger leaves the others down. No action lists held fingers; cancel releases all. Navigation and device off cancel held fingers. Needs $B device first.', usage: 'touch [down|move | up | cancel]' }, + 'device': { category: 'Interaction', description: 'Emulate a phone on the current tab only: size, pixel ratio, touch support (pointer: coarse) and user agent from Playwright\'s devices table (names match case-insensitively). Other tabs stay desktop. No name prints the active profile or "none"; --list prints every name; off (or reset, auto) restores. viewport and responsive refuse while a device is active.', usage: 'device [ [--landscape] | off | --list]' }, + 'tap': { category: 'Interaction', description: 'Touch and release at an element\'s center (scrolled into view) or at x y. Needs $B device first.', usage: 'tap <@ref|selector|x y> [--hold ]' }, + 'swipe': { category: 'Interaction', description: 'One-finger swipe from an element\'s center or x y; the finger moves in the given direction (default 300px over 300ms, 10 moves). Needs $B device first.', usage: 'swipe <@ref|selector|x y> [--distance ] [--ms ]' }, + 'pinch': { category: 'Interaction', description: 'Two-finger pinch around an element\'s center or x y: scale above 1 spreads the fingers (zoom in), below 1 brings them together. Needs $B device first.', usage: 'pinch <@ref|selector|x y> ' }, + 'touch': { category: 'Interaction', description: 'Hold, move and lift numbered fingers for multi-finger flows (hold a joystick with finger 0, tap fire with finger 1). Lifting one finger leaves the others down. No action lists held fingers; cancel releases all. Navigation and device off cancel held fingers. Needs $B device first.', usage: 'touch [down|move | up | cancel]' }, // Data extraction 'download': { category: 'Extraction', description: 'Download URL or media element to disk using browser cookies. Default path: /browse-download-.. --base64 returns a data:;base64,... string instead of writing, capped at 10MB. Use --navigate for URLs that trigger browser downloads (CDN redirects, Content-Disposition, anti-bot protected sites).', usage: 'download [path] [--base64] [--navigate]' }, 'scrape': { category: 'Extraction', description: 'Bulk download all media from page to --dir (default: /browse-scrape-). Writes files plus manifest.json with source URL, size, type, success/failure counts. --limit defaults to 50 and caps at 200.', usage: 'scrape [--selector sel] [--dir path] [--limit N]' }, diff --git a/devex-review/SKILL.md b/devex-review/SKILL.md index b479ffeba..f5e02d978 100644 --- a/devex-review/SKILL.md +++ b/devex-review/SKILL.md @@ -331,6 +331,17 @@ console.log("ASIDE_DIR=" + pwd); await closeTab(pg); console.log("GSTACK_STEP_OK ' ``` +**Touch (verified in Aside 1.26; add the Responsive override for phone size):** + +```bash +aside repl ' +const pg = await openTab(""), t = (m, p) => pg._sendToTarget(m, p); +await t("Emulation.setTouchEmulationEnabled", { enabled: true, maxTouchPoints: 5 }); +for (const [type, touchPoints] of [["touchStart", [{ x: , y: }]], ["touchEnd", []]]) await t("Input.dispatchTouchEvent", { type, touchPoints }); +await closeTab(pg); console.log("GSTACK_STEP_OK"); +' +``` + **Links and their status (same-origin; on a LOCAL target each link is HEAD-checked, on a real site the user's cookies would ride every request so links are listed as `LINK ?` unfetched — consent to LOOK is not consent to hit every URL):** ```bash diff --git a/gstack/llms.txt b/gstack/llms.txt index c53668753..66e0c7178 100644 --- a/gstack/llms.txt +++ b/gstack/llms.txt @@ -105,15 +105,20 @@ Run with `browse [args]`. Full reference: `browse/SKILL.md`. - `cookie =`: Set cookie on current page domain - `cookie-import `: Import cookies from JSON file - `cookie-import-browser [browser] [--domain d] [--profile p] [--all] [--clear-storage] [--verify-auth]`: Copy cookies from chrome, chromium, brave, edge, Windows-only opera, opera-gx, or macOS-only comet or arc. +- `device [ [--landscape] | off | --list]`: Emulate a phone on the current tab only: size, pixel ratio, touch support (pointer: coarse) and user agent from Playwright's devices table (names match case-insensitively). - `dialog-accept [text]`: Auto-accept next alert/confirm/prompt. - `dialog-dismiss`: Auto-dismiss next dialog - `fill `: Fill input - `header :`: Set custom request header (colon-separated, sensitive values auto-redacted) - `hover `: Hover element +- `pinch <@ref|selector|x y> `: Two-finger pinch around an element's center or x y: scale above 1 spreads the fingers (zoom in), below 1 brings them together. - `press `: Press a Playwright keyboard key against the focused element. - `scroll [sel|@ref]`: With a selector, smooth-scrolls the element into view. - `select `: Select dropdown option by value, label, or visible text - `style | style --undo [N]`: Modify CSS property on element (with undo support) +- `swipe <@ref|selector|x y> [--distance ] [--ms ]`: One-finger swipe from an element's center or x y; the finger moves in the given direction (default 300px over 300ms, 10 moves). +- `tap <@ref|selector|x y> [--hold ]`: Touch and release at an element's center (scrolled into view) or at x y. +- `touch [down|move | up | cancel]`: Hold, move and lift numbered fingers for multi-finger flows (hold a joystick with finger 0, tap fire with finger 1). - `type [--selector ] [--] `: Send keystrokes to the focused element, or to with --selector (focuses it first). - `upload [file2...]`: Upload file(s) - `useragent `: Set user agent diff --git a/qa/SKILL.md b/qa/SKILL.md index cd3983c92..77bec5905 100644 --- a/qa/SKILL.md +++ b/qa/SKILL.md @@ -317,6 +317,7 @@ Prefer the richer of recent project test plans and plans in conversation over gi Follow the shared section's ordered preparation, then run its probe loop. Inside that loop, browser runs apply qa-patterns.md's numbered Phases 1-6 as techniques, and functional runs apply system-functional.md's contract map; neither is a separate workflow. +Touch: `$B device "iPhone 15"` then tap. > **STOP.** Before running the selected target's QA baseline and exploratory probes, with caller-owned authority, Read `sections/exploratory.md` relative to the installed `qa`/`gstack-qa` SKILL.md directory in full and follow it. > Use this host's installed path, never the product working directory or another host's assets. diff --git a/qa/SKILL.md.tmpl b/qa/SKILL.md.tmpl index 347afed67..a22e3aec1 100644 --- a/qa/SKILL.md.tmpl +++ b/qa/SKILL.md.tmpl @@ -129,6 +129,7 @@ Prefer the richer of recent project test plans and plans in conversation over gi Follow the shared section's ordered preparation, then run its probe loop. Inside that loop, browser runs apply qa-patterns.md's numbered Phases 1-6 as techniques, and functional runs apply system-functional.md's contract map; neither is a separate workflow. +Touch: `$B device "iPhone 15"` then tap. {{SECTION:exploratory}} diff --git a/qa/templates/qa-report-template.md b/qa/templates/qa-report-template.md index 0b44a194d..c3d88b74b 100644 --- a/qa/templates/qa-report-template.md +++ b/qa/templates/qa-report-template.md @@ -13,6 +13,7 @@ | **Pages visited** | {COUNT} | | **Screenshots** | {COUNT} | | **Framework** | {DETECTED or "Unknown"} | +| **Phones** | {Chromium device profiles emulated, or "—"}; real device still needed (/ios-qa): {what, or "—"} | ## Health Score: {SCORE}/100 diff --git a/scripts/resolvers/aside.ts b/scripts/resolvers/aside.ts index 24cb594b7..5f21b54d4 100644 --- a/scripts/resolvers/aside.ts +++ b/scripts/resolvers/aside.ts @@ -240,6 +240,17 @@ console.log("ASIDE_DIR=" + pwd); await closeTab(pg); console.log("GSTACK_STEP_OK ' \`\`\` +**Touch (verified in Aside 1.26; add the Responsive override for phone size):** + +\`\`\`bash +aside repl ' +const pg = await openTab(""), t = (m, p) => pg._sendToTarget(m, p); +await t("Emulation.setTouchEmulationEnabled", { enabled: true, maxTouchPoints: 5 }); +for (const [type, touchPoints] of [["touchStart", [{ x: , y: }]], ["touchEnd", []]]) await t("Input.dispatchTouchEvent", { type, touchPoints }); +await closeTab(pg); console.log("GSTACK_STEP_OK"); +' +\`\`\` + **Links and their status (same-origin; on a LOCAL target each link is HEAD-checked, on a real site the user's cookies would ride every request so links are listed as \`LINK ?\` unfetched — consent to LOOK is not consent to hit every URL):** \`\`\`bash diff --git a/test/aside-driver.test.ts b/test/aside-driver.test.ts index 942ce38f7..1615375c6 100644 --- a/test/aside-driver.test.ts +++ b/test/aside-driver.test.ts @@ -138,6 +138,9 @@ describe('Aside driver contract ({{ASIDE_SETUP}})', () => { test('cookbook uses only the verified Aside APIs', () => { expect(section).toContain('Page.addScriptToEvaluateOnNewDocument'); expect(section).toContain('Emulation.setDeviceMetricsOverride'); + // Touch and phone recipe: verified on Garry's Mac in Aside 1.26 (GSTA-227 F3, #3058). + expect(section).toContain('Emulation.setTouchEmulationEnabled", { enabled: true, maxTouchPoints: 5 }'); + expect(section).toContain('Input.dispatchTouchEvent'); expect(section).toContain('annotatedScreenshot(pg)'); expect(section).toContain('snapshot(pg, { interactive: true })'); // Verified NOT to exist or NOT to persist across CLI calls — must never be recommended.