From 28583accab683000f1e19d7129e700d2bc466dad Mon Sep 17 00:00:00 2001 From: zhom <2717306+zhom@users.noreply.github.com> Date: Tue, 20 Oct 2026 09:54:27 +0400 Subject: [PATCH] feat: anchored sync clicks --- src-tauri/src/synchronizer.rs | 1697 ++++++++++++++++++++++++++++----- 1 file changed, 1465 insertions(+), 232 deletions(-) diff --git a/src-tauri/src/synchronizer.rs b/src-tauri/src/synchronizer.rs index d3b3bc73..137fe680 100644 --- a/src-tauri/src/synchronizer.rs +++ b/src-tauri/src/synchronizer.rs @@ -13,8 +13,9 @@ use crate::profile::types::BrowserProfile; const MAX_CONCURRENT_LAUNCHES: usize = 5; /// Event captured from the leader browser via Wayfern.inputCaptured CDP events. -/// Fields match the Wayfern.inputCaptured event schema directly. -#[derive(Debug, Clone, Serialize, Deserialize)] +/// Fields up to `timestamp` match the Wayfern.inputCaptured event schema; the +/// rest are added on the leader side before the event fans out. +#[derive(Debug, Clone, Default, Serialize, Deserialize)] pub struct CapturedEvent { #[serde(rename = "type")] pub event_type: String, @@ -44,6 +45,582 @@ pub struct CapturedEvent { pub delta_y: Option, #[serde(default)] pub timestamp: Option, + /// Where a pointer event landed on the leader, for followers whose layout + /// differs. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub anchor: Option, + /// "navigated": the leader got there without an input just before it, + /// which is what typing an address looks like. + #[serde(default)] + pub typed: Option, + /// "dialogClosed": how the leader answered the dialog. + #[serde(default)] + pub accept: Option, + #[serde(default, rename = "promptText")] + pub prompt_text: Option, +} + +/// An element and a spot inside it, found on the leader and looked up again on +/// each follower, so a click lands on the same thing when text, scrollbars, +/// window size or a mobile layout put it somewhere else. +#[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize)] +pub struct PointAnchor { + /// Ids, tag names and :nth-of-type steps from the nearest id down. + pub selector: String, + /// Which of the element's content quads held the point (a wrapped link has + /// one per line). + pub quad: usize, + /// Where in that quad, as fractions of its width and height. + pub rx: f64, + pub ry: f64, + /// The quad's top and height on the leader, in viewport CSS pixels. + pub top: f64, + pub height: f64, +} + +/// The bounding box of a CDP quad (eight numbers, four corners). +fn quad_box(quad: &[f64]) -> Option<(f64, f64, f64, f64)> { + if quad.len() < 8 { + return None; + } + let xs = [quad[0], quad[2], quad[4], quad[6]]; + let ys = [quad[1], quad[3], quad[5], quad[7]]; + let min = |v: &[f64]| v.iter().copied().fold(f64::INFINITY, f64::min); + let max = |v: &[f64]| v.iter().copied().fold(f64::NEG_INFINITY, f64::max); + Some((min(&xs), min(&ys), max(&xs), max(&ys))) +} + +fn quads_of(value: &serde_json::Value) -> Vec> { + value + .get("quads") + .and_then(|q| q.as_array()) + .map(|quads| { + quads + .iter() + .filter_map(|q| q.as_array()) + .map(|q| q.iter().filter_map(|n| n.as_f64()).collect()) + .collect() + }) + .unwrap_or_default() +} + +/// The quad nearest to (x, y) and the point's place in it. +fn anchor_in_quads(selector: String, quads: &[Vec], x: f64, y: f64) -> Option { + let mut best: Option<(usize, f64)> = None; + for (index, quad) in quads.iter().enumerate() { + let Some((left, top, right, bottom)) = quad_box(quad) else { + continue; + }; + let dx = (left - x).max(0.0).max(x - right); + let dy = (top - y).max(0.0).max(y - bottom); + let closer = match best { + None => true, + Some((_, distance)) => dx + dy < distance, + }; + if closer { + best = Some((index, dx + dy)); + } + } + let (quad, _) = best?; + let (left, top, right, bottom) = quad_box(&quads[quad])?; + let ratio = |v: f64, lo: f64, hi: f64| { + if hi > lo { + ((v - lo) / (hi - lo)).clamp(0.0, 1.0) + } else { + 0.5 + } + }; + Some(PointAnchor { + selector, + quad, + rx: ratio(x, left, right), + ry: ratio(y, top, bottom), + top, + height: bottom - top, + }) +} + +/// Where an anchor lands among a follower's quads for the same element: +/// the point, and the quad's top and height. +fn point_in_quads(anchor: &PointAnchor, quads: &[Vec]) -> Option<(f64, f64, f64, f64)> { + let quad = quads.get(anchor.quad).or_else(|| quads.last())?; + let (left, top, right, bottom) = quad_box(quad)?; + Some(( + left + anchor.rx * (right - left), + top + anchor.ry * (bottom - top), + top, + bottom - top, + )) +} + +fn css_escape_ident(ident: &str) -> String { + let mut out = String::with_capacity(ident.len()); + for (index, c) in ident.chars().enumerate() { + let plain = c.is_ascii_alphanumeric() || c == '-' || c == '_' || !c.is_ascii(); + if plain && !(index == 0 && c.is_ascii_digit()) { + out.push(c); + } else { + out.push_str(&format!("\\{:x} ", c as u32)); + } + } + out +} + +/// A selector for the element `backend_node_id` names (or the element holding +/// it, for a text node) in a DOM.getDocument tree, and that element's id. +fn selector_for_node(root: &serde_json::Value, backend_node_id: i64) -> Option<(String, i64)> { + fn find<'a>( + node: &'a serde_json::Value, + target: i64, + path: &mut Vec<&'a serde_json::Value>, + ) -> bool { + path.push(node); + if node.get("backendNodeId").and_then(|v| v.as_i64()) == Some(target) { + return true; + } + if let Some(children) = node.get("children").and_then(|c| c.as_array()) { + for child in children { + if find(child, target, path) { + return true; + } + } + } + path.pop(); + false + } + let is_element = + |node: &serde_json::Value| node.get("nodeType").and_then(|t| t.as_i64()) == Some(1); + let attribute = |node: &serde_json::Value, name: &str| -> Option { + let list = node.get("attributes")?.as_array()?; + list + .chunks(2) + .find(|pair| pair.first().and_then(|n| n.as_str()) == Some(name)) + .and_then(|pair| pair.get(1)) + .and_then(|v| v.as_str()) + .map(str::to_string) + }; + let mut path = Vec::new(); + if !find(root, backend_node_id, &mut path) { + return None; + } + // A hit on text belongs to the element around it. + while path.last().is_some_and(|node| !is_element(node)) { + path.pop(); + } + let element_id = path.last()?.get("backendNodeId")?.as_i64()?; + let mut steps: Vec = Vec::new(); + for depth in (0..path.len()).rev() { + let node = path[depth]; + if !is_element(node) { + break; + } + if let Some(id) = attribute(node, "id").filter(|id| !id.is_empty()) { + steps.push(format!("#{}", css_escape_ident(&id))); + break; + } + let name = node.get("nodeName").and_then(|n| n.as_str()).unwrap_or(""); + let tag = node + .get("localName") + .and_then(|n| n.as_str()) + .filter(|n| !n.is_empty()) + .map(str::to_string) + .unwrap_or_else(|| name.to_lowercase()); + if tag == "html" { + steps.push(tag); + break; + } + let position = depth + .checked_sub(1) + .and_then(|parent| path[parent].get("children")) + .and_then(|c| c.as_array()) + .and_then(|siblings| { + siblings + .iter() + .filter(|s| is_element(s) && s.get("nodeName").and_then(|n| n.as_str()) == Some(name)) + .position(|s| std::ptr::eq(s, node)) + }) + .map_or(1, |p| p + 1); + steps.push(format!("{tag}:nth-of-type({position})")); + } + steps.reverse(); + Some((steps.join(" > "), element_id)) +} + +/// Whether two URLs show the same page: same origin and path. Queries and +/// fragments can carry per-session values, so they do not count. +fn same_page(a: &str, b: &str) -> bool { + match (url::Url::parse(a), url::Url::parse(b)) { + (Ok(a), Ok(b)) => a.origin() == b.origin() && a.path() == b.path(), + _ => a == b, + } +} + +/// The editing command a Ctrl/Cmd shortcut runs. A key event sent by CDP does +/// not reach the macOS menu, and the shortcut key differs between macOS and the +/// other systems, so the command travels with the key. +fn shortcut_command(key: &str, shift: bool) -> Option<&'static str> { + match (key.to_ascii_lowercase().as_str(), shift) { + ("a", false) => Some("selectAll"), + ("c", false) => Some("copy"), + ("x", false) => Some("cut"), + ("v", false) => Some("paste"), + ("z", false) => Some("undo"), + ("z", true) | ("y", false) => Some("redo"), + _ => None, + } +} + +/// A wheel burst ends after this much quiet. +const WHEEL_BURST_IDLE: std::time::Duration = std::time::Duration::from_millis(150); +/// How long a follower may take to reach, by replaying the input, the page the +/// leader moved to before it is sent there. +const NAVIGATION_GRACE: std::time::Duration = std::time::Duration::from_secs(4); +/// How long a page that a follower was sent to may take to load. +const LOAD_GRACE: std::time::Duration = std::time::Duration::from_secs(6); +/// How long a click waits for its element, as on a page that is still loading. +const ANCHOR_WAIT: std::time::Duration = std::time::Duration::from_secs(3); +/// How long a wheel waits for its element; a scroll is not worth a long stall. +const WHEEL_ANCHOR_WAIT: std::time::Duration = std::time::Duration::from_millis(300); +/// How long a follower waits for its own copy of the leader's dialog. +const DIALOG_WAIT: std::time::Duration = std::time::Duration::from_secs(2); +/// The longest a reply is awaited. An open dialog holds the reply to an input +/// event until it closes. +const REPLY_TIMEOUT: std::time::Duration = std::time::Duration::from_millis(1500); + +type FollowerSocket = + tokio_tungstenite::WebSocketStream>; +type PendingReplies = + Arc>>>; + +/// What a follower's own page events say about where it is. +#[derive(Debug, Clone, Default, PartialEq)] +struct FollowerPage { + main_frame: String, + url: String, + loaded: bool, + dialog_open: bool, +} + +fn apply_page_event(page: &mut FollowerPage, method: &str, params: &serde_json::Value) { + let text = |path: &str| { + params + .pointer(path) + .and_then(|v| v.as_str()) + .unwrap_or_default() + .to_string() + }; + match method { + "Page.frameNavigated" if params.pointer("/frame/parentId").is_none() => { + page.main_frame = text("/frame/id"); + page.url = text("/frame/url"); + page.loaded = false; + } + "Page.navigatedWithinDocument" if text("/frameId") == page.main_frame => { + page.url = text("/url"); + } + "Page.domContentEventFired" => page.loaded = true, + "Page.javascriptDialogOpening" => page.dialog_open = true, + "Page.javascriptDialogClosed" => page.dialog_open = false, + _ => {} + } +} + +/// Where a follower stands against the page the leader moved to. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum Barrier { + /// On that page and loaded: replay. + Ready, + /// Not there yet, and still in time to get there by itself. + Wait, + /// Out of time on another page: send it there. + Navigate, + /// Out of time on that page while it still loads: replay anyway. + Proceed, +} + +fn barrier(expected: &str, page: &FollowerPage, out_of_time: bool) -> Barrier { + match (same_page(expected, &page.url), page.loaded, out_of_time) { + (true, true, _) => Barrier::Ready, + (_, _, false) => Barrier::Wait, + (false, _, true) => Barrier::Navigate, + (true, false, true) => Barrier::Proceed, + } +} + +/// How far to scroll down so the spot under the leader's pointer comes under +/// the same point here, given where the anchored quad is on this follower. +fn scroll_to_align(anchor: &PointAnchor, top: f64, height: f64) -> f64 { + let here = top + anchor.ry * height; + let there = anchor.top + anchor.ry * anchor.height; + here - there +} + +/// How editing shortcuts travel to a follower. The person at the leader uses +/// the host's shortcut key (Cmd on a Mac, Ctrl elsewhere); a page claiming +/// macOS or iOS expects Cmd; and on a Mac host a key event sent through CDP +/// never reaches the menu that runs the shortcut, so the command is sent with +/// the key there. +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] +struct ShortcutEditing { + follower_apple: bool, + host_mac: bool, +} + +impl ShortcutEditing { + fn for_profile(follower: &BrowserProfile) -> Self { + let follower_apple = follower + .wayfern_config + .as_ref() + .and_then(|config| config.os.as_deref()) + .and_then(crate::wayfern_manager::WayfernManager::normalize_os_name) + .is_some_and(|os| os == "macos" || os == "ios"); + Self { + follower_apple, + host_mac: cfg!(target_os = "macos"), + } + } + + /// The modifiers to send, and the editing command to send with them. + fn map_shortcut(&self, key: &str, modifiers: i32, key_down: bool) -> (i32, Option<&'static str>) { + const CTRL: i32 = 2; + const META: i32 = 4; + const SHIFT: i32 = 8; + if modifiers & (CTRL | META) == 0 { + return (modifiers, None); + } + let Some(command) = shortcut_command(key, modifiers & SHIFT != 0) else { + return (modifiers, None); + }; + let leader_shortcut = if self.host_mac { META } else { CTRL }; + if modifiers & leader_shortcut == 0 { + return (modifiers, None); + } + let follower_shortcut = if self.follower_apple { META } else { CTRL }; + let mapped = (modifiers & !leader_shortcut) | follower_shortcut; + let send_command = key_down && (self.follower_apple || self.host_mac); + (mapped, send_command.then_some(command)) + } +} + +/// The follower end of the replay: commands out, replies and page events in. +struct FollowerLink { + sink: futures_util::stream::SplitSink, + replies: PendingReplies, + page: tokio::sync::watch::Receiver, + page_tx: Arc>, + next_id: u64, +} + +impl FollowerLink { + fn new(ws: FollowerSocket) -> Self { + use futures_util::stream::StreamExt; + let (sink, mut stream) = ws.split(); + let replies: PendingReplies = Arc::default(); + let (page_tx, page) = tokio::sync::watch::channel(FollowerPage::default()); + let page_tx = Arc::new(page_tx); + let reader_replies = replies.clone(); + let reader_page = page_tx.clone(); + tokio::spawn(async move { + while let Some(Ok(message)) = stream.next().await { + let tokio_tungstenite::tungstenite::Message::Text(text) = message else { + continue; + }; + let Ok(value) = serde_json::from_str::(text.as_str()) else { + continue; + }; + if let Some(id) = value.get("id").and_then(|id| id.as_u64()) { + let waiter = reader_replies + .lock() + .ok() + .and_then(|mut replies| replies.remove(&id)); + if let Some(waiter) = waiter { + let _ = waiter.send(value); + } + continue; + } + let method = value.get("method").and_then(|m| m.as_str()).unwrap_or(""); + let params = value.get("params").cloned().unwrap_or_default(); + reader_page.send_modify(|page| apply_page_event(page, method, ¶ms)); + } + // The socket is gone: nobody will answer the calls still waiting. + if let Ok(mut replies) = reader_replies.lock() { + replies.clear(); + } + }); + Self { + sink, + replies, + page, + page_tx, + next_id: 0, + } + } + + /// Send a command without waiting for its reply. Fails only when the socket + /// is gone. + async fn send(&mut self, method: &str, params: serde_json::Value) -> Result<(), String> { + use futures_util::sink::SinkExt; + self.next_id += 1; + let command = serde_json::json!({ "id": self.next_id, "method": method, "params": params }); + self + .sink + .send(tokio_tungstenite::tungstenite::Message::Text( + command.to_string().into(), + )) + .await + .map_err(|e| e.to_string()) + } + + /// Send a command and wait a bounded time for its result: Ok(None) when the + /// browser refused it or did not answer in time, Err when the socket is gone. + async fn call( + &mut self, + method: &str, + params: serde_json::Value, + ) -> Result, String> { + self.call_within(method, params, REPLY_TIMEOUT).await + } + + async fn call_within( + &mut self, + method: &str, + params: serde_json::Value, + timeout: std::time::Duration, + ) -> Result, String> { + let (tx, rx) = tokio::sync::oneshot::channel(); + let id = self.next_id + 1; + if let Ok(mut replies) = self.replies.lock() { + replies.insert(id, tx); + } + self.send(method, params).await?; + let reply = tokio::time::timeout(timeout, rx).await; + if let Ok(mut replies) = self.replies.lock() { + replies.remove(&id); + } + Ok(match reply { + Ok(Ok(reply)) if reply.get("error").is_none() => reply.get("result").cloned(), + _ => None, + }) + } + + /// Where an anchor lands on this follower's page (point, quad top, quad + /// height), waiting up to `wait` for the element to exist. Scrolls the + /// element into view first when the point is outside the viewport. + async fn resolve( + &mut self, + anchor: &PointAnchor, + wait: std::time::Duration, + ) -> Option<(f64, f64, f64, f64)> { + let deadline = tokio::time::Instant::now() + wait; + loop { + if let Some(found) = self.resolve_once(anchor, deadline).await { + return Some(found); + } + if tokio::time::Instant::now() >= deadline { + return None; + } + tokio::time::sleep(std::time::Duration::from_millis(60)).await; + } + } + + async fn resolve_once( + &mut self, + anchor: &PointAnchor, + deadline: tokio::time::Instant, + ) -> Option<(f64, f64, f64, f64)> { + // Each query gets what is left of the wait, so a page that answers slowly + // while it loads cannot stretch it. + let left = || { + deadline + .saturating_duration_since(tokio::time::Instant::now()) + .clamp(std::time::Duration::from_millis(100), REPLY_TIMEOUT) + }; + let document = self + .call_within("DOM.getDocument", serde_json::json!({ "depth": 0 }), left()) + .await + .ok()??; + let root = document.pointer("/root/nodeId")?.as_i64()?; + let found = self + .call_within( + "DOM.querySelector", + serde_json::json!({ "nodeId": root, "selector": anchor.selector }), + left(), + ) + .await + .ok()??; + let node = found.get("nodeId")?.as_i64().filter(|id| *id > 0)?; + let quads = self + .call_within( + "DOM.getContentQuads", + serde_json::json!({ "nodeId": node }), + left(), + ) + .await + .ok()??; + let mut point = point_in_quads(anchor, &quads_of(&quads))?; + let metrics = self + .call_within("Page.getLayoutMetrics", serde_json::json!({}), left()) + .await + .ok()??; + let size = |key: &str| { + metrics + .pointer(&format!("/cssVisualViewport/{key}")) + .and_then(|v| v.as_f64()) + .unwrap_or(f64::MAX) + }; + let (width, height) = (size("clientWidth"), size("clientHeight")); + if point.0 < 0.0 || point.1 < 0.0 || point.0 >= width || point.1 >= height { + let _ = self + .call_within( + "DOM.scrollIntoViewIfNeeded", + serde_json::json!({ "nodeId": node }), + left(), + ) + .await; + let quads = self + .call_within( + "DOM.getContentQuads", + serde_json::json!({ "nodeId": node }), + left(), + ) + .await + .ok()??; + point = point_in_quads(anchor, &quads_of(&quads))?; + } + // Reading the document turned on DOM events, which nothing here uses. + let _ = self.send("DOM.disable", serde_json::json!({})).await; + Some(point) + } + + /// Wait until this follower is on `url`, giving it `grace` to get there by + /// itself; then send it there, unless it already was, and wait for the load. + async fn reach( + &mut self, + url: &str, + grace: std::time::Duration, + already_sent: bool, + ) -> Result<(), String> { + let mut deadline = tokio::time::Instant::now() + grace; + let mut sent = already_sent; + loop { + let page = self.page.borrow().clone(); + match barrier(url, &page, tokio::time::Instant::now() >= deadline) { + Barrier::Ready | Barrier::Proceed => return Ok(()), + Barrier::Navigate if sent => return Ok(()), + Barrier::Navigate => { + self + .send("Page.navigate", serde_json::json!({ "url": url })) + .await?; + sent = true; + deadline = tokio::time::Instant::now() + LOAD_GRACE; + } + Barrier::Wait => { + let left = deadline.saturating_duration_since(tokio::time::Instant::now()); + let _ = tokio::time::timeout(left, self.page.changed()).await; + } + } + } + } } // No JavaScript injection needed — Wayfern.enableInputCapture handles everything natively. @@ -636,11 +1213,96 @@ impl SynchronizerManager { } } + /// What is at (x, y) on the leader's page, as something a follower can + /// look up in its own layout. None when the point holds no element. + async fn leader_anchor( + ws: &mut tokio_tungstenite::WebSocketStream< + tokio_tungstenite::MaybeTlsStream, + >, + cmd_id: &mut u64, + pending_events: &mut Vec, + x: f64, + y: f64, + ) -> Option { + let metrics = send_cmd( + ws, + cmd_id, + pending_events, + "Page.getLayoutMetrics", + serde_json::json!({}), + ) + .await + .ok()?; + let scroll = |axis: &str| { + metrics + .pointer(&format!("/cssVisualViewport/{axis}")) + .and_then(|v| v.as_f64()) + .unwrap_or(0.0) + }; + // The hit test takes document coordinates, the captured point is in the + // viewport. + let hit = send_cmd( + ws, + cmd_id, + pending_events, + "DOM.getNodeForLocation", + serde_json::json!({ + "x": (x + scroll("pageX")).round() as i64, + "y": (y + scroll("pageY")).round() as i64, + "includeUserAgentShadowDOM": false, + "ignorePointerEventsNone": true, + }), + ) + .await + .ok()?; + let hit_node = hit.get("backendNodeId")?.as_i64()?; + let document = send_cmd( + ws, + cmd_id, + pending_events, + "DOM.getDocument", + serde_json::json!({ "depth": -1 }), + ) + .await; + let found = document + .as_ref() + .ok() + .and_then(|document| document.get("root")) + .and_then(|root| selector_for_node(root, hit_node)); + let quads = match &found { + Some((_, element)) => send_cmd( + ws, + cmd_id, + pending_events, + "DOM.getContentQuads", + serde_json::json!({ "backendNodeId": element }), + ) + .await + .ok(), + None => None, + }; + // Reading the document turned on DOM events, which nothing here uses. + let _ = send_cmd( + ws, + cmd_id, + pending_events, + "DOM.disable", + serde_json::json!({}), + ) + .await; + let (selector, _) = found?; + anchor_in_quads(selector, &quads_of(&quads?), x, y) + } + // Use Wayfern's native input capture — no JS injection needed. // Captures all real user input at the browser process level. let setup_commands: Vec<(&str, serde_json::Value)> = vec![ ("Page.enable", serde_json::json!({})), - ("Wayfern.enableInputCapture", serde_json::json!({})), + // Pointer moves are needed for hover menus, drags and selections. + ( + "Wayfern.enableInputCapture", + serde_json::json!({ "trackMouseMove": true }), + ), ]; for (method, params) in setup_commands { @@ -660,6 +1322,24 @@ impl SynchronizerManager { } } + // Same-document navigations are reported per frame; only the main frame's + // move the leader to another page. + let leader_main_frame: Option = send_cmd( + &mut ws_stream, + &mut cmd_id, + &mut pending_events, + "Page.getFrameTree", + serde_json::json!({}), + ) + .await + .ok() + .and_then(|tree| { + tree + .pointer("/frameTree/frame/id") + .and_then(|id| id.as_str()) + .map(str::to_string) + }); + // Get leader window size and resize all followers to match let leader_bounds = send_cmd( &mut ws_stream, @@ -730,8 +1410,9 @@ impl SynchronizerManager { let mgr = manager.clone(); let sid = session_id.clone(); let ah = app_handle.clone(); + let editing = ShortcutEditing::for_profile(&fp); tokio::spawn(async move { - Self::follower_replay_loop(ws, rx, fid_clone, mgr, sid, ah).await; + Self::follower_replay_loop(ws, rx, fid_clone, mgr, sid, ah, editing).await; }); } Err(e) => log::warn!( @@ -767,66 +1448,177 @@ impl SynchronizerManager { // Signal that the session is ready for interaction let _ = ready_tx.send(Ok(())); - // Process any events that were buffered during setup - for event in pending_events.drain(..) { - Self::handle_cdp_event(&event, &gate, &follower_senders, false).await; - } + // Events that arrived while a command waited for its reply, oldest first. + let mut backlog: std::collections::VecDeque = + pending_events.drain(..).collect(); + // Where the last press was, so the release of a click reuses its anchor. + let mut last_press: Option<(f64, f64)> = None; + // The pointer of the wheel burst in progress, and when it counts as over. + let mut wheel_burst: Option<(f64, f64, tokio::time::Instant)> = None; // Main event loop — listen for Wayfern.inputCaptured events let end_reason: String; loop { - tokio::select! { - _ = cancel_rx.changed() => { - if *cancel_rx.borrow() { - end_reason = "cancelled".to_string(); - break; - } + let value = if let Some(value) = backlog.pop_front() { + value + } else { + let wheel_end = wheel_burst.map(|(_, _, at)| at); + tokio::select! { + _ = cancel_rx.changed() => { + if *cancel_rx.borrow() { + end_reason = "cancelled".to_string(); + break; + } + continue; + } + _ = tokio::time::sleep_until(wheel_end.unwrap_or_else(tokio::time::Instant::now)), if wheel_end.is_some() => { + // The scroll stopped: tell the followers what is under the + // pointer now, so they can line the same content up. + if let Some((x, y, _)) = wheel_burst.take() { + let anchor = + leader_anchor(&mut ws_stream, &mut cmd_id, &mut pending_events, x, y).await; + backlog.extend(pending_events.drain(..)); + let ended = CapturedEvent { + event_type: "wheelEnd".to_string(), + x: Some(x), + y: Some(y), + anchor, + ..Default::default() + }; + Self::fan_out(&gate, &follower_senders, &ended); + } + continue; + } + msg = ws_stream.next() => { + match msg { + Some(Ok(Message::Text(text))) => match serde_json::from_str(text.as_str()) { + Ok(value) => value, + Err(_) => continue, + }, + Some(Ok(_)) => continue, // pings, binary, etc. + Some(Err(e)) => { + end_reason = format!("leader_ws_error err=\"{e}\""); + break; + } + None => { + end_reason = "leader_closed".to_string(); + break; + } + } + } + } + }; + + let method = value.get("method").and_then(|m| m.as_str()).unwrap_or(""); + + // Log CDP command response errors + if let (Some(id), Some(error)) = (value.get("id"), value.get("error")) { + log::warn!( + "Synchronizer leader CDP command failed session={} id={id} err={error}", + ShortId(&session_id) + ); + } + + match method { + "Wayfern.inputCaptured" => { + let Some(mut event) = value + .get("params") + .and_then(|params| serde_json::from_value::(params.clone()).ok()) + else { + continue; + }; + // A pointer move is not an action: the pointer crossing the page on + // its way to the address bar must not hide the navigation typed there. + if event.event_type != "mousemove" { + last_user_event_time = std::time::Instant::now(); } - msg = ws_stream.next() => { - match msg { - Some(Ok(Message::Text(text))) => { - let value: serde_json::Value = match serde_json::from_str(text.as_str()) { - Ok(v) => v, - Err(_) => continue, - }; - - let method = value.get("method").and_then(|m| m.as_str()).unwrap_or(""); - - // Log CDP command response errors - if let Some(id) = value.get("id") { - if let Some(error) = value.get("error") { - log::warn!( - "Synchronizer leader CDP command failed session={} id={id} err={error}", - ShortId(&session_id) - ); - } - } - - // Track user interaction timing - if method == "Wayfern.inputCaptured" { - last_user_event_time = std::time::Instant::now(); - } - - let recent_user_event = last_user_event_time.elapsed() < std::time::Duration::from_secs(2); - - Self::handle_cdp_event( - &value, - &gate, - &follower_senders, - recent_user_event, - ).await; - } - Some(Ok(_)) => {} // pings, binary, etc. - Some(Err(e)) => { - end_reason = format!("leader_ws_error err=\"{e}\""); - break; - } - None => { - end_reason = "leader_closed".to_string(); - break; - } - } + let point = (event.x.unwrap_or(0.0), event.y.unwrap_or(0.0)); + let needs_anchor = match event.event_type.as_str() { + "mousedown" => { + last_press = Some(point); + true + } + "mouseup" => !last_press + .take() + .is_some_and(|(x, y)| (x - point.0).hypot(y - point.1) < 4.0), + "wheel" => { + let starts = wheel_burst.is_none(); + wheel_burst = Some(( + point.0, + point.1, + tokio::time::Instant::now() + WHEEL_BURST_IDLE, + )); + starts + } + _ => false, + }; + if needs_anchor { + event.anchor = leader_anchor( + &mut ws_stream, + &mut cmd_id, + &mut pending_events, + point.0, + point.1, + ) + .await; + backlog.extend(pending_events.drain(..)); } + log::trace!("Synchronizer captured event={}", event.event_type); + Self::fan_out(&gate, &follower_senders, &event); + } + // Every top-level navigation tells the followers where the leader is + // now. Those that got there by replaying the input are already there; + // the rest are sent. + "Page.frameNavigated" => { + let frame = value.pointer("/params/frame"); + let is_top = frame.is_some_and(|frame| frame.get("parentId").is_none()); + let url = frame + .and_then(|frame| frame.get("url")) + .and_then(|url| url.as_str()); + if let (true, Some(url)) = (is_top, url) { + if !url.starts_with("about:") && !url.starts_with("chrome://") { + let typed = last_user_event_time.elapsed() >= std::time::Duration::from_secs(2); + let navigated = CapturedEvent { + event_type: "navigated".to_string(), + url: Some(url.to_string()), + typed: Some(typed), + ..Default::default() + }; + Self::fan_out(&gate, &follower_senders, &navigated); + } + } + } + "Page.navigatedWithinDocument" => { + let frame_id = value.pointer("/params/frameId").and_then(|id| id.as_str()); + let url = value.pointer("/params/url").and_then(|url| url.as_str()); + if let (true, Some(url)) = ( + frame_id.is_some() && frame_id == leader_main_frame.as_deref(), + url, + ) { + let navigated = CapturedEvent { + event_type: "navigated".to_string(), + url: Some(url.to_string()), + typed: Some(false), + ..Default::default() + }; + Self::fan_out(&gate, &follower_senders, &navigated); + } + } + // A dialog blocks the follower's page until someone answers it, so the + // leader's answer is passed on. + "Page.javascriptDialogClosed" => { + let closed = CapturedEvent { + event_type: "dialogClosed".to_string(), + accept: value.pointer("/params/result").and_then(|r| r.as_bool()), + prompt_text: value + .pointer("/params/userInput") + .and_then(|text| text.as_str()) + .map(str::to_string), + ..Default::default() + }; + Self::fan_out(&gate, &follower_senders, &closed); + } + _ => {} } } @@ -864,68 +1656,6 @@ impl SynchronizerManager { Ok(()) } - /// Handle a single CDP event from the leader. - /// - /// The gate is read here rather than in the per-follower replay task so a - /// paused session queues nothing at all: an event dropped now cannot arrive - /// late once mirroring resumes. - async fn handle_cdp_event( - value: &serde_json::Value, - gate: &SharedGate, - follower_senders: &HashMap>, - recent_user_event: bool, - ) { - let method = value.get("method").and_then(|m| m.as_str()).unwrap_or(""); - - // Handle Wayfern.inputCaptured — native input events from the browser process - if method == "Wayfern.inputCaptured" { - if let Some(params) = value.get("params") { - let event_type = params.get("type").and_then(|v| v.as_str()).unwrap_or(""); - // Skip mousemove — too noisy and not useful for synchronization - if event_type == "mousemove" { - return; - } - if let Ok(event) = serde_json::from_value::(params.clone()) { - log::trace!("Synchronizer captured event={event_type}"); - Self::fan_out(gate, follower_senders, &event); - } - } - } - - // Handle Page.frameNavigated — replay only for address-bar navigations - if method == "Page.frameNavigated" && !recent_user_event { - if let Some(params) = value.get("params") { - if let Some(frame) = params.get("frame") { - let is_top = frame.get("parentId").is_none(); - if is_top { - if let Some(url) = frame.get("url").and_then(|v| v.as_str()) { - if !url.starts_with("about:") && !url.starts_with("chrome://") { - log::trace!("Synchronizer replaying address-bar navigation"); - let nav_event = CapturedEvent { - event_type: "navigate".to_string(), - url: Some(url.to_string()), - x: None, - y: None, - button: None, - click_count: None, - key: None, - code: None, - key_code: None, - modifiers: None, - text: None, - delta_x: None, - delta_y: None, - timestamp: None, - }; - Self::fan_out(gate, follower_senders, &nav_event); - } - } - } - } - } - } - } - /// Send one captured event to every follower the gate still admits. /// /// The lock is held across the whole fan-out and released before any await, @@ -952,124 +1682,354 @@ impl SynchronizerManager { /// Dedicated replay loop for a single follower with a persistent WebSocket connection. /// Processes events from the channel sequentially — no per-event connection overhead. + /// + /// Pointer events carry an anchor from the leader and are replayed on the + /// same element here, wherever this follower's layout put it. Input waits + /// until the follower is on the page the leader last moved to. async fn follower_replay_loop( - mut ws: tokio_tungstenite::WebSocketStream< - tokio_tungstenite::MaybeTlsStream, - >, + ws: FollowerSocket, mut rx: tokio::sync::mpsc::UnboundedReceiver, follower_id: String, manager: Arc>, session_id: String, app_handle: tauri::AppHandle, + editing: ShortcutEditing, ) { - use futures_util::sink::SinkExt; - use tokio_tungstenite::tungstenite::Message; + let mut link = FollowerLink::new(ws); + // The follower's own page events tell the barrier where it is. + let _ = link.call("Page.enable", serde_json::json!({})).await; + if let Ok(Some(tree)) = link.call("Page.getFrameTree", serde_json::json!({})).await { + let frame = tree.pointer("/frameTree/frame"); + let text = |key: &str| { + frame + .and_then(|frame| frame.get(key)) + .and_then(|v| v.as_str()) + .unwrap_or_default() + .to_string() + }; + let (main_frame, url) = (text("id"), text("url")); + link.page_tx.send_modify(|page| { + page.main_frame = main_frame; + page.url = url; + page.loaded = true; + }); + } - let mut cmd_id: u64 = 0; + // Buttons the leader holds down, as Input.dispatchMouseEvent's bit mask, so + // moves between a press and a release replay as a drag. + let mut buttons_held: i32 = 0; + // An event read ahead while collapsing a run of pointer moves. + let mut queued: Option = None; + // This follower's point minus the leader's at the last anchored event; + // moves and the release of a click follow it. + let mut offset = (0.0f64, 0.0f64); + // The page the leader moved to, and until when the follower may get there + // by itself. + let mut barrier_target: Option<(String, tokio::time::Instant, bool)> = None; - while let Some(event) = rx.recv().await { - cmd_id += 1; + loop { + let mut event = match queued.take() { + Some(event) => event, + None => { + let due = barrier_target.as_ref().map(|(_, at, _)| *at); + tokio::select! { + event = rx.recv() => match event { + Some(event) => event, + None => break, + }, + _ = tokio::time::sleep_until(due.unwrap_or_else(tokio::time::Instant::now)), if due.is_some() => { + if let Some((url, _, sent)) = barrier_target.take() { + if link.reach(&url, std::time::Duration::ZERO, sent).await.is_err() { + Self::mark_follower_desynced(&manager, &session_id, &follower_id, &app_handle).await; + break; + } + } + continue; + } + } + } + }; + // Only the last of a run of queued moves matters: a follower that fell + // behind catches up instead of replaying every intermediate position. + while event.event_type == "mousemove" { + match rx.try_recv() { + Ok(newer) if newer.event_type == "mousemove" => event = newer, + Ok(other) => { + queued = Some(other); + break; + } + Err(_) => break, + } + } + + let is_input = matches!( + event.event_type.as_str(), + "mousedown" + | "mouseup" + | "mousemove" + | "keydown" + | "keyup" + | "char" + | "wheel" + | "insertText" + ); + if is_input { + if let Some((url, until, sent)) = barrier_target.take() { + let grace = until.saturating_duration_since(tokio::time::Instant::now()); + if link.reach(&url, grace, sent).await.is_err() { + Self::mark_follower_desynced(&manager, &session_id, &follower_id, &app_handle).await; + break; + } + } + } + + let leader_point = (event.x.unwrap_or(0.0), event.y.unwrap_or(0.0)); + let modifiers = event.modifiers.unwrap_or(0); let button = event.button.clone().unwrap_or_else(|| "left".to_string()); + let button_bit = match button.as_str() { + "left" => 1, + "right" => 2, + "middle" => 4, + "back" => 8, + "forward" => 16, + _ => 0, + }; - let command = match event.event_type.as_str() { - "navigate" => event - .url - .as_ref() - .map(|url| ("Page.navigate", serde_json::json!({ "url": url }))), - "mousedown" => Some(( - "Input.dispatchMouseEvent", - serde_json::json!({ - "type": "mousePressed", - "x": event.x.unwrap_or(0.0), - "y": event.y.unwrap_or(0.0), - "button": button, - "clickCount": event.click_count.unwrap_or(1), - "modifiers": event.modifiers.unwrap_or(0), - }), - )), - "mouseup" => Some(( - "Input.dispatchMouseEvent", - serde_json::json!({ - "type": "mouseReleased", - "x": event.x.unwrap_or(0.0), - "y": event.y.unwrap_or(0.0), - "button": button, - "modifiers": event.modifiers.unwrap_or(0), - }), - )), - "keydown" => Some(( - "Input.dispatchKeyEvent", - serde_json::json!({ - "type": "keyDown", - "key": event.key.clone().unwrap_or_default(), - "code": event.code.clone().unwrap_or_default(), - "windowsVirtualKeyCode": event.key_code.unwrap_or(0), - "modifiers": event.modifiers.unwrap_or(0), - }), - )), - "keyup" => Some(( - "Input.dispatchKeyEvent", - serde_json::json!({ - "type": "keyUp", - "key": event.key.clone().unwrap_or_default(), - "code": event.code.clone().unwrap_or_default(), - "windowsVirtualKeyCode": event.key_code.unwrap_or(0), - "modifiers": event.modifiers.unwrap_or(0), - }), - )), - "char" => { - let text = event.text.clone().unwrap_or_default(); - if text.is_empty() { - None + let sent: Result<(), String> = match event.event_type.as_str() { + "navigated" => match event.url.clone() { + Some(url) if event.typed == Some(true) => { + // An address the leader typed: nothing replayed brings the + // follower there, so it is sent at once. + barrier_target = Some((url.clone(), tokio::time::Instant::now() + LOAD_GRACE, true)); + link + .send("Page.navigate", serde_json::json!({ "url": url })) + .await + } + Some(url) => { + barrier_target = Some((url, tokio::time::Instant::now() + NAVIGATION_GRACE, false)); + Ok(()) + } + None => Ok(()), + }, + "dialogClosed" => { + let deadline = tokio::time::Instant::now() + DIALOG_WAIT; + loop { + let open = link.page.borrow().dialog_open; + if open || tokio::time::Instant::now() >= deadline { + break; + } + let _ = + tokio::time::timeout(std::time::Duration::from_millis(100), link.page.changed()) + .await; + } + let open = link.page.borrow().dialog_open; + if open { + link + .call( + "Page.handleJavaScriptDialog", + serde_json::json!({ + "accept": event.accept.unwrap_or(false), + "promptText": event.prompt_text.clone().unwrap_or_default(), + }), + ) + .await + .map(|_| ()) } else { - Some(( + Ok(()) + } + } + "mousedown" | "mouseup" => { + if let Some(anchor) = &event.anchor { + offset = match link.resolve(anchor, ANCHOR_WAIT).await { + Some((x, y, _, _)) => (x - leader_point.0, y - leader_point.1), + None => (0.0, 0.0), + }; + } else if event.event_type == "mousedown" { + // Nothing to look up: the leader's point as it is, not the last + // element's offset. + offset = (0.0, 0.0); + } + let (x, y) = (leader_point.0 + offset.0, leader_point.1 + offset.1); + let before = buttons_held; + if event.event_type == "mousedown" { + buttons_held |= button_bit; + } else { + buttons_held &= !button_bit; + } + // The pointer gets there first, so hover and the press land together. + let moved = link + .send( + "Input.dispatchMouseEvent", + serde_json::json!({ + "type": "mouseMoved", + "x": x, + "y": y, + "button": if before & 1 != 0 { "left" } else { "none" }, + "buttons": before, + "modifiers": modifiers, + }), + ) + .await; + match moved { + Err(e) => Err(e), + // Acknowledged, so the next lookup sees what the click did. + Ok(()) => link + .call( + "Input.dispatchMouseEvent", + serde_json::json!({ + "type": if event.event_type == "mousedown" { "mousePressed" } else { "mouseReleased" }, + "x": x, + "y": y, + "button": button, + "buttons": buttons_held, + "clickCount": event.click_count.unwrap_or(1), + "modifiers": modifiers, + }), + ) + .await + .map(|_| ()), + } + } + "mousemove" => { + link + .send( + "Input.dispatchMouseEvent", + serde_json::json!({ + "type": "mouseMoved", + "x": leader_point.0 + offset.0, + "y": leader_point.1 + offset.1, + "button": if buttons_held & 1 != 0 { "left" } else { "none" }, + "buttons": buttons_held, + "modifiers": modifiers, + }), + ) + .await + } + // A wheel event at the pointer scrolls whatever is under it (a list, a + // pane, a modal), as on the leader. The captured deltas have the + // opposite sign of the command's. + "wheel" => { + if let Some(anchor) = &event.anchor { + if let Some((x, y, _, _)) = link.resolve(anchor, WHEEL_ANCHOR_WAIT).await { + offset = (x - leader_point.0, y - leader_point.1); + } + } + link + .call( + "Input.dispatchMouseEvent", + serde_json::json!({ + "type": "mouseWheel", + "x": leader_point.0 + offset.0, + "y": leader_point.1 + offset.1, + "deltaX": -event.delta_x.unwrap_or(0.0), + "deltaY": -event.delta_y.unwrap_or(0.0), + "modifiers": modifiers, + }), + ) + .await + .map(|_| ()) + } + // After a scroll, the content under the leader's pointer is moved under + // the same point here: relative wheel deltas drift with other heights. + "wheelEnd" => match &event.anchor { + Some(anchor) => { + let middle = PointAnchor { + rx: 0.5, + ..anchor.clone() + }; + match link.resolve(&middle, WHEEL_ANCHOR_WAIT).await { + Some((_, _, top, height)) => { + let delta = scroll_to_align(anchor, top, height); + if delta.abs() >= 1.0 { + link + .call( + "Input.dispatchMouseEvent", + serde_json::json!({ + "type": "mouseWheel", + "x": leader_point.0 + offset.0, + "y": leader_point.1, + "deltaX": 0, + "deltaY": delta, + }), + ) + .await + .map(|_| ()) + } else { + Ok(()) + } + } + None => Ok(()), + } + } + None => Ok(()), + }, + "keydown" | "keyup" => { + let key = event.key.clone().unwrap_or_default(); + let (mapped, command) = + editing.map_shortcut(&key, modifiers, event.event_type == "keydown"); + let mut params = serde_json::json!({ + "type": if event.event_type == "keydown" { "keyDown" } else { "keyUp" }, + "key": key, + "code": event.code.clone().unwrap_or_default(), + "windowsVirtualKeyCode": event.key_code.unwrap_or(0), + "modifiers": mapped, + }); + if let Some(command) = command { + params["commands"] = serde_json::json!([command]); + } + link + .call("Input.dispatchKeyEvent", params) + .await + .map(|_| ()) + } + "char" => match event.text.clone().filter(|text| !text.is_empty()) { + Some(text) => link + .call( "Input.dispatchKeyEvent", serde_json::json!({ "type": "char", "text": text, "unmodifiedText": text, - "modifiers": event.modifiers.unwrap_or(0), + "modifiers": modifiers, }), - )) - } - } - // A wheel event at the pointer scrolls whatever is under it (a list, a - // pane, a modal), as on the leader. The captured deltas have the - // opposite sign of the command's. - "wheel" => Some(( - "Input.dispatchMouseEvent", - serde_json::json!({ - "type": "mouseWheel", - "x": event.x.unwrap_or(0.0), - "y": event.y.unwrap_or(0.0), - "deltaX": -event.delta_x.unwrap_or(0.0), - "deltaY": -event.delta_y.unwrap_or(0.0), - "modifiers": event.modifiers.unwrap_or(0), - }), - )), - _ => None, + ) + .await + .map(|_| ()), + None => Ok(()), + }, + "insertText" => match event.text.clone().filter(|text| !text.is_empty()) { + Some(text) => link + .call("Input.insertText", serde_json::json!({ "text": text })) + .await + .map(|_| ()), + None => Ok(()), + }, + _ => Ok(()), }; - if let Some((method, params)) = command { - let cmd = serde_json::json!({ "id": cmd_id, "method": method, "params": params }); - if let Err(e) = ws.send(Message::Text(cmd.to_string().into())).await { - log::warn!( - "Synchronizer follower desynced: send failed session={} profile={follower_id} err=\"{e}\"", - ShortId(&session_id) - ); - // Mark as desynced - let mut inner = manager.lock().await; - if let Some(session) = inner.sessions.get_mut(&session_id) { - if let Some(follower) = session.follower_mut(&follower_id) { - follower.failed_at_url = Some("connection lost".to_string()); - let info = session.info(); - let _ = app_handle.emit("sync-session-changed", &info); - } - } - break; - } - // Don't wait for response — fire and forget for speed. - // CDP commands are processed in order by Chromium. + if let Err(e) = sent { + log::warn!( + "Synchronizer follower desynced: send failed session={} profile={follower_id} err=\"{e}\"", + ShortId(&session_id) + ); + Self::mark_follower_desynced(&manager, &session_id, &follower_id, &app_handle).await; + break; + } + } + } + + async fn mark_follower_desynced( + manager: &Arc>, + session_id: &str, + follower_id: &str, + app_handle: &tauri::AppHandle, + ) { + let mut inner = manager.lock().await; + if let Some(session) = inner.sessions.get_mut(session_id) { + if let Some(follower) = session.follower_mut(follower_id) { + follower.failed_at_url = Some("connection lost".to_string()); + let info = session.info(); + let _ = app_handle.emit("sync-session-changed", &info); } } } @@ -1205,7 +2165,7 @@ impl SynchronizerManager { Ok(info) } - /// Place the session's follower windows on the host display. + /// Place the session's windows on the host display, the leader first. /// /// Held-out followers are placed too: they are still windows on the screen, /// and a person who held one out is usually the one who wants to see it. @@ -1215,7 +2175,10 @@ impl SynchronizerManager { session_id: &str, layout: WindowLayout, ) -> Result { - let (info, follower_ids) = { + // The leader takes the first tile: every window gets the same size, so + // every page lays out as the leader's does and a replayed point finds the + // same content. + let (info, window_ids) = { let inner = self.inner.lock().await; let session = inner .sessions @@ -1223,38 +2186,41 @@ impl SynchronizerManager { .ok_or_else(|| serde_json::json!({ "code": "SYNC_SESSION_NOT_FOUND" }).to_string())?; ( session.info(), - session - .followers - .iter() - .map(|follower| follower.profile_id.clone()) + std::iter::once(session.leader_profile_id.clone()) + .chain( + session + .followers + .iter() + .map(|follower| follower.profile_id.clone()), + ) .collect::>(), ) }; - if follower_ids.is_empty() { + if window_ids.len() < 2 { return Ok(info); } let (screen_width, screen_height) = crate::wayfern_manager::host_screen_size(&app_handle) .ok_or_else(|| serde_json::json!({ "code": "SYNC_DISPLAY_UNAVAILABLE" }).to_string())?; - let rects = layout_windows(layout, follower_ids.len(), screen_width, screen_height); + let rects = layout_windows(layout, window_ids.len(), screen_width, screen_height); if !layout_fits(&rects) { return Err( serde_json::json!({ "code": "SYNC_DISPLAY_TOO_SMALL", - "params": { "count": follower_ids.len().to_string() } + "params": { "count": window_ids.len().to_string() } }) .to_string(), ); } let mut placed = 0usize; - for (follower_id, rect) in follower_ids.iter().zip(rects) { - match Self::place_window(follower_id, rect).await { + for (profile_id, rect) in window_ids.iter().zip(rects) { + match Self::place_window(profile_id, rect).await { Ok(()) => placed += 1, Err(e) => { - log::warn!("Synchronizer window placement failed profile={follower_id} err=\"{e}\"") + log::warn!("Synchronizer window placement failed profile={profile_id} err=\"{e}\"") } } } @@ -1265,12 +2231,12 @@ impl SynchronizerManager { log::info!( "Synchronizer windows arranged session={} placed={placed}/{}", ShortId(session_id), - follower_ids.len() + window_ids.len() ); Ok(info) } - /// Move one follower's browser window through CDP. + /// Move one session window (leader or follower) through CDP. async fn place_window(follower_profile_id: &str, rect: WindowRect) -> Result<(), String> { use futures_util::sink::SinkExt; use futures_util::stream::StreamExt; @@ -1662,6 +2628,273 @@ mod tests { ))); } + /// A DOM.getDocument tree in the shape the browser returns it. + fn node( + backend: i64, + name: &str, + attributes: &[&str], + children: Vec, + ) -> serde_json::Value { + serde_json::json!({ + "nodeId": backend, + "backendNodeId": backend, + "nodeType": 1, + "nodeName": name.to_uppercase(), + "localName": name, + "attributes": attributes, + "children": children, + }) + } + + fn text(backend: i64) -> serde_json::Value { + serde_json::json!({ "nodeId": backend, "backendNodeId": backend, "nodeType": 3, "nodeName": "#text" }) + } + + fn mail_document() -> serde_json::Value { + let rows = (1..=3) + .map(|i| { + node( + 100 + i, + "div", + &["class", "row"], + vec![node(200 + i, "span", &[], vec![text(300 + i)])], + ) + }) + .collect(); + serde_json::json!({ + "nodeId": 1, + "backendNodeId": 1, + "nodeType": 9, + "nodeName": "#document", + "children": [node(2, "html", &[], vec![ + node(3, "head", &[], vec![]), + node(4, "body", &[], vec![ + node(5, "div", &["id", "list"], rows), + node(6, "p", &[], vec![text(7), node(8, "a", &["href", "/x"], vec![text(9)])]), + ]), + ])], + }) + } + + #[test] + fn a_selector_starts_at_the_nearest_id_and_counts_same_tag_siblings() { + let document = mail_document(); + assert_eq!( + selector_for_node(&document, 203), + Some(( + "#list > div:nth-of-type(3) > span:nth-of-type(1)".to_string(), + 203 + )) + ); + // A hit on text names the element around it. + assert_eq!( + selector_for_node(&document, 302), + Some(( + "#list > div:nth-of-type(2) > span:nth-of-type(1)".to_string(), + 202 + )) + ); + // Without an id on the way up the path starts at html. + assert_eq!( + selector_for_node(&document, 9), + Some(( + "html > body:nth-of-type(1) > p:nth-of-type(1) > a:nth-of-type(1)".to_string(), + 8 + )) + ); + assert_eq!(selector_for_node(&document, 999), None); + } + + #[test] + fn an_id_that_is_not_a_plain_identifier_is_escaped() { + assert_eq!(css_escape_ident("list"), "list"); + assert_eq!(css_escape_ident("1st"), "\\31 st"); + assert_eq!(css_escape_ident("a.b"), "a\\2e b"); + } + + fn quad(left: f64, top: f64, width: f64, height: f64) -> Vec { + vec![ + left, + top, + left + width, + top, + left + width, + top + height, + left, + top + height, + ] + } + + #[test] + fn an_anchor_finds_the_same_spot_in_a_follower_layout() { + // A link wrapped over two lines; the leader clicked its second line. + let leader = [ + quad(500.0, 100.0, 120.0, 20.0), + quad(40.0, 120.0, 80.0, 20.0), + ]; + let anchor = anchor_in_quads("a".to_string(), &leader, 60.0, 125.0).unwrap(); + assert_eq!(anchor.quad, 1); + assert!((anchor.rx - 0.25).abs() < 1e-9 && (anchor.ry - 0.25).abs() < 1e-9); + // The follower's text is wider, so the link wraps elsewhere and sits lower. + let follower = [ + quad(520.0, 300.0, 100.0, 22.0), + quad(40.0, 322.0, 120.0, 22.0), + ]; + let (x, y, top, height) = point_in_quads(&anchor, &follower).unwrap(); + assert!((x - 70.0).abs() < 1e-9 && (y - 327.5).abs() < 1e-9); + assert_eq!((top, height), (322.0, 22.0)); + // A follower with fewer boxes uses its last one rather than none. + let single = [quad(10.0, 10.0, 100.0, 20.0)]; + assert!(point_in_quads(&anchor, &single).is_some()); + // A point outside every box takes the nearest one, held inside it. + let outside = anchor_in_quads("a".to_string(), &leader, 700.0, 50.0).unwrap(); + assert_eq!((outside.quad, outside.rx, outside.ry), (0, 1.0, 0.0)); + } + + #[test] + fn alignment_scrolls_the_leader_content_under_the_same_point() { + let anchor = PointAnchor { + selector: "p".to_string(), + quad: 0, + rx: 0.5, + ry: 0.5, + top: 300.0, + height: 100.0, + }; + // The paragraph is 60 px lower here and taller: scroll down by the gap + // between the two midpoints. + assert_eq!(scroll_to_align(&anchor, 360.0, 120.0), 70.0); + assert_eq!(scroll_to_align(&anchor, 300.0, 100.0), 0.0); + } + + #[test] + fn the_same_page_ignores_query_and_fragment() { + assert!(same_page( + "https://a.test/mail?sid=1", + "https://a.test/mail?sid=2#top" + )); + assert!(!same_page("https://a.test/mail", "https://a.test/form")); + assert!(!same_page("https://a.test/mail", "https://b.test/mail")); + } + + #[test] + fn the_barrier_waits_then_sends_the_follower_to_the_leader_page() { + let mut page = FollowerPage { + main_frame: "main".to_string(), + url: "https://a.test/article".to_string(), + loaded: true, + dialog_open: false, + }; + let leader = "https://a.test/mail"; + assert_eq!(barrier(leader, &page, false), Barrier::Wait); + assert_eq!(barrier(leader, &page, true), Barrier::Navigate); + page.url = "https://a.test/mail?from=link".to_string(); + page.loaded = false; + assert_eq!(barrier(leader, &page, false), Barrier::Wait); + assert_eq!(barrier(leader, &page, true), Barrier::Proceed); + page.loaded = true; + assert_eq!(barrier(leader, &page, false), Barrier::Ready); + } + + #[test] + fn page_events_move_the_follower_state() { + let mut page = FollowerPage::default(); + apply_page_event( + &mut page, + "Page.frameNavigated", + &serde_json::json!({ "frame": { "id": "main", "url": "https://a.test/app" } }), + ); + assert_eq!( + (page.url.as_str(), page.loaded), + ("https://a.test/app", false) + ); + // A child frame's navigation is not the page's. + apply_page_event( + &mut page, + "Page.frameNavigated", + &serde_json::json!({ "frame": { "id": "child", "parentId": "main", "url": "https://ads.test/" } }), + ); + assert_eq!(page.url, "https://a.test/app"); + apply_page_event( + &mut page, + "Page.domContentEventFired", + &serde_json::json!({}), + ); + assert!(page.loaded); + apply_page_event( + &mut page, + "Page.navigatedWithinDocument", + &serde_json::json!({ "frameId": "main", "url": "https://a.test/app/products" }), + ); + assert_eq!(page.url, "https://a.test/app/products"); + apply_page_event( + &mut page, + "Page.javascriptDialogOpening", + &serde_json::json!({}), + ); + assert!(page.dialog_open); + apply_page_event( + &mut page, + "Page.javascriptDialogClosed", + &serde_json::json!({}), + ); + assert!(!page.dialog_open); + } + + #[test] + fn shortcuts_follow_the_host_key_and_the_follower_system() { + // Ctrl on a Windows or Linux host, to a follower that claims macOS. + let to_mac = ShortcutEditing { + follower_apple: true, + host_mac: false, + }; + assert_eq!(to_mac.map_shortcut("a", 2, true), (4, Some("selectAll"))); + assert_eq!(to_mac.map_shortcut("z", 2 | 8, true), (4 | 8, Some("redo"))); + // The release carries the mapped modifier and no command. + assert_eq!(to_mac.map_shortcut("a", 2, false), (4, None)); + // The same host, a follower that claims Windows: the key goes as it is. + let same = ShortcutEditing { + follower_apple: false, + host_mac: false, + }; + assert_eq!(same.map_shortcut("c", 2, true), (2, None)); + // Cmd on a Mac host: the command goes with the key to every follower. + let mac_host = ShortcutEditing { + follower_apple: false, + host_mac: true, + }; + assert_eq!(mac_host.map_shortcut("v", 4, true), (2, Some("paste"))); + let mac_to_mac = ShortcutEditing { + follower_apple: true, + host_mac: true, + }; + assert_eq!(mac_to_mac.map_shortcut("x", 4, true), (4, Some("cut"))); + // Ctrl on a Mac host is not the host's shortcut key. + assert_eq!(mac_host.map_shortcut("a", 2, true), (2, None)); + // Keys that are not editing shortcuts pass as they are. + assert_eq!(to_mac.map_shortcut("t", 2, true), (2, None)); + assert_eq!(to_mac.map_shortcut("a", 0, true), (0, None)); + } + + #[test] + fn an_anchor_travels_with_the_event_and_old_events_still_parse() { + let captured: CapturedEvent = serde_json::from_value(serde_json::json!({ + "type": "mousedown", "x": 10.0, "y": 20.0, "button": "left", "clickCount": 1 + })) + .unwrap(); + assert!(captured.anchor.is_none()); + let with_anchor = CapturedEvent { + anchor: Some(PointAnchor { + selector: "#list".to_string(), + ..Default::default() + }), + ..captured + }; + let round_trip: CapturedEvent = + serde_json::from_value(serde_json::to_value(&with_anchor).unwrap()).unwrap(); + assert_eq!(round_trip.anchor, with_anchor.anchor); + } + #[test] fn holding_a_follower_out_stops_only_that_one() { let mut gate = MirrorGate::default();