diff --git a/e2e/lib/fixtures.mjs b/e2e/lib/fixtures.mjs
index 8516c1a5..a28612d7 100644
--- a/e2e/lib/fixtures.mjs
+++ b/e2e/lib/fixtures.mjs
@@ -687,3 +687,137 @@ export function extensionCrx3({
]),
);
}
+
+/**
+ * The form page the MCP locator and typing checks run against. Each part
+ * stands for a kind of field agents reported trouble with: a search input and
+ * a search textarea named by id, a controlled input that keeps only what real
+ * input events gave it, an autocomplete whose input is transparent and one
+ * pixel wide, two fields whose names differ by a suffix, radios drawn by their
+ * labels, a grid of day cells, a field the page empties after every key, and
+ * a textarea far down the page beside a fixed banner that covers the bottom of
+ * the window.
+ */
+export function mcpFormsPage() {
+ const days = Array.from(
+ { length: 31 },
+ (_, i) =>
+ `
+
+
+`;
+}
diff --git a/e2e/run.mjs b/e2e/run.mjs
index 615c23eb..d8da216d 100644
--- a/e2e/run.mjs
+++ b/e2e/run.mjs
@@ -28,7 +28,7 @@ import { pipeline } from "node:stream/promises";
import { fileURLToPath } from "node:url";
import { ensureRclone, serveS3Args } from "../scripts/rclone.mjs";
import { createSafeDiagnostics } from "./lib/diagnostics.mjs";
-import { extensionCrx3 } from "./lib/fixtures.mjs";
+import { extensionCrx3, mcpFormsPage } from "./lib/fixtures.mjs";
import { DRIVER_COMMAND_TIMEOUT_SECONDS } from "./lib/limits.mjs";
const dirname = path.dirname(fileURLToPath(import.meta.url));
@@ -408,6 +408,14 @@ function startFixtureServer(geoIpFixture, geoIpAsnFixture) {
response.end(body);
return;
}
+ if (url.pathname === "/mcp-forms") {
+ response.writeHead(200, {
+ "content-type": "text/html; charset=utf-8",
+ "cache-control": "no-store",
+ });
+ response.end(mcpFormsPage());
+ return;
+ }
if (url.pathname === "/geoip.mmdb" && geoIpFixture) {
response.writeHead(200, {
"content-type": "application/octet-stream",
diff --git a/e2e/tests/browser.test.mjs b/e2e/tests/browser.test.mjs
index 479fba92..b5014d32 100644
--- a/e2e/tests/browser.test.mjs
+++ b/e2e/tests/browser.test.mjs
@@ -150,6 +150,59 @@ const LOCATION_KEYS = new Set([
"accuracy",
]);
+/**
+ * The MCP tools as an agent reaches them over the remote bridge, on one
+ * initialized session. `tool` fails on an error answer; `toolError` expects one
+ * and returns it.
+ */
+async function mcpClient(base, token, profileId) {
+ let rpcId = 0;
+ let session = null;
+ const post = async (method, params) => {
+ const response = await fetch(`${base}/v1/e2e/mcp`, {
+ method: "POST",
+ headers: {
+ authorization: `Bearer ${token}`,
+ "content-type": "application/json",
+ ...(session ? { "mcp-session-id": session } : {}),
+ },
+ body: JSON.stringify({ jsonrpc: "2.0", id: ++rpcId, method, params }),
+ });
+ assert.equal(response.status, 200, await response.clone().text());
+ const answer = await response.json();
+ session = answer.session_id ?? session;
+ return answer.body;
+ };
+ const mcp = async (method, params) => {
+ const body = await post(method, params);
+ assert.equal(body.error, undefined, JSON.stringify(body));
+ return body.result;
+ };
+ const call = (name, args) =>
+ post("tools/call", {
+ name,
+ arguments: { profile_id: profileId, ...args },
+ });
+ const tool = async (name, args = {}) => {
+ const body = await call(name, args);
+ assert.equal(body.error, undefined, JSON.stringify(body));
+ assert.notEqual(body.result.isError, true, JSON.stringify(body.result));
+ return body.result.content[0];
+ };
+ const toolJson = async (name, args) =>
+ JSON.parse((await tool(name, args)).text);
+ const toolError = async (name, args = {}) => {
+ const body = await call(name, args);
+ assert.ok(body.error, JSON.stringify(body));
+ return body.error;
+ };
+ await mcp("initialize", {
+ protocolVersion: "2025-06-18",
+ clientInfo: { name: "Donut E2E", version: "1" },
+ });
+ return { mcp, tool, toolJson, toolError };
+}
+
/** `fingerprint` is the serialised fingerprint STRING, or null for a fresh one. */
async function createRealProfile(app, version, name, fingerprint = null) {
return app.invoke("create_browser_profile_new", {
@@ -750,38 +803,11 @@ test("real Wayfern fingerprinting, terms, API automation, CDP, cookies, and proc
// The MCP browser tools, through the engine an agent reaches over the
// remote bridge. Each action says where the page is afterwards.
- let rpcId = 0;
- let mcpSession = null;
- const mcp = async (method, params) => {
- const response = await fetch(`${base}/v1/e2e/mcp`, {
- method: "POST",
- headers: {
- authorization: `Bearer ${saved.api_token}`,
- "content-type": "application/json",
- ...(mcpSession ? { "mcp-session-id": mcpSession } : {}),
- },
- body: JSON.stringify({ jsonrpc: "2.0", id: ++rpcId, method, params }),
- });
- assert.equal(response.status, 200, await response.clone().text());
- const answer = await response.json();
- mcpSession = answer.session_id ?? mcpSession;
- assert.equal(answer.body.error, undefined, JSON.stringify(answer.body));
- return answer.body.result;
- };
- const tool = async (name, args = {}) => {
- const result = await mcp("tools/call", {
- name,
- arguments: { profile_id: profile.id, ...args },
- });
- assert.notEqual(result.isError, true, JSON.stringify(result));
- return result.content[0];
- };
- const toolJson = async (name, args) =>
- JSON.parse((await tool(name, args)).text);
- await mcp("initialize", {
- protocolVersion: "2025-06-18",
- clientInfo: { name: "Donut E2E", version: "1" },
- });
+ const { mcp, tool, toolJson, toolError } = await mcpClient(
+ base,
+ saved.api_token,
+ profile.id,
+ );
// The device the profile presents, through each way it is read: the
// fingerprint form, REST and MCP. An identity-backed profile stores no
@@ -874,25 +900,6 @@ test("real Wayfern fingerprinting, terms, API automation, CDP, cookies, and proc
);
// Form tools on a native select and a hidden file input added to the page.
- const toolError = async (name, args = {}) => {
- const response = await fetch(`${base}/v1/e2e/mcp`, {
- method: "POST",
- headers: {
- authorization: `Bearer ${saved.api_token}`,
- "content-type": "application/json",
- "mcp-session-id": mcpSession,
- },
- body: JSON.stringify({
- jsonrpc: "2.0",
- id: ++rpcId,
- method: "tools/call",
- params: { name, arguments: { profile_id: profile.id, ...args } },
- }),
- });
- const answer = await response.json();
- assert.ok(answer.body.error, JSON.stringify(answer.body));
- return answer.body.error;
- };
await cdp.evaluate(`(() => {
const form = document.createElement('form');
form.innerHTML = ''
@@ -1492,6 +1499,258 @@ test("real Wayfern fingerprinting, terms, API automation, CDP, cookies, and proc
}
});
+// What MCP agents reported on real forms, on a local page that has each kind
+// of field: a locator by id finds a search input and a search textarea, a click
+// lands on what a person sees (a radio drawn by its label, a day cell, a
+// textarea beside a fixed banner, never the banner), a name that two fields
+// share settles on the exact one, and typing either leaves the text in the
+// field, through the input events a framework listens to, or fails and says so.
+test("MCP locators and typing act on the field a person sees", async () => {
+ assert.ok(process.env.WAYFERN_TEST_TOKEN, "WAYFERN_TEST_TOKEN is required");
+ const localWayfernVersion = cachedFixtureVersion(
+ process.env.DONUT_E2E_PROJECT_ROOT,
+ );
+ const app = appFromEnvironment("browser-mcp-forms", {
+ seedVersionCache: localWayfernVersion ?? false,
+ wayfernTermsAccepted: false,
+ });
+ let cdp;
+ let browserPid;
+ try {
+ const prepared = await prepareWayfern(
+ app,
+ process.env.DONUT_E2E_PROJECT_ROOT,
+ );
+ if (!app.session) await app.start();
+ if (!(await app.invoke("check_wayfern_terms_accepted"))) {
+ await app.invoke("accept_wayfern_terms");
+ }
+ const profile = await createRealProfile(app, prepared.version, "MCP Forms");
+ const settings = await app.invoke("get_app_settings");
+ const saved = await app.invoke("save_app_settings", {
+ settings: {
+ ...settings,
+ api_enabled: true,
+ api_port: 0,
+ api_token: null,
+ onboarding_completed: true,
+ },
+ });
+ const port = await app.invoke("start_api_server", { port: 0 });
+ const base = `http://127.0.0.1:${port}`;
+ const launched = await request(`${base}/v1/profiles/${profile.id}/run`, {
+ method: "POST",
+ token: saved.api_token,
+ body: { url: `${fixtureUrl}/mcp-forms`, headless: true },
+ });
+ assert.equal(launched.response.status, 200, JSON.stringify(launched.value));
+ browserPid = (await app.invoke("list_browser_profiles")).find(
+ (item) => item.id === profile.id,
+ )?.process_id;
+ cdp = await CdpClient.connect(launched.value.remote_debugging_port);
+ await cdp.waitFor("window.__formsReady === true", {
+ description: "the MCP forms fixture",
+ });
+ const { tool, toolJson, toolError } = await mcpClient(
+ base,
+ saved.api_token,
+ profile.id,
+ );
+ const valueOf = (selector) =>
+ cdp.evaluate(`document.querySelector(${JSON.stringify(selector)}).value`);
+ const byId = (id) => ({ attributes: [{ name: "id", value: id }] });
+
+ // An id the browser's own index does not carry: a search input and a
+ // search textarea.
+ const searched = await toolJson("type_locator", {
+ locator: byId("searchInput"),
+ text: "Donut Browser",
+ });
+ assert.equal(searched.typed, true, JSON.stringify(searched));
+ assert.equal(searched.verified, true, JSON.stringify(searched));
+ assert.equal(await valueOf("#searchInput"), "Donut Browser");
+ const asked = await toolJson("type_locator", {
+ locator: byId("sb_form_q"),
+ text: "weather today",
+ });
+ assert.equal(asked.verified, true, JSON.stringify(asked));
+ assert.equal(await valueOf("#sb_form_q"), "weather today");
+
+ // A controlled field keeps only what real input events gave it: a value
+ // assigned by a script is put back on the next render.
+ const controlled = await toolJson("type_locator", {
+ locator: { role: "textbox", name: "Full name" },
+ text: "Jane Doe",
+ });
+ assert.equal(controlled.verified, true, JSON.stringify(controlled));
+ await new Promise((resolve) => setTimeout(resolve, 300));
+ assert.equal(await valueOf("#react-name"), "Jane Doe");
+ assert.equal(
+ await cdp.evaluate("document.querySelector('#react-name').dataset.state"),
+ "Jane Doe",
+ "the page's own state took the text",
+ );
+ // Replaced, not appended, and still through input events.
+ assert.match(
+ (
+ await tool("type_text", {
+ selector: "#react-name",
+ text: "John Roe",
+ })
+ ).text,
+ /the field holds the text/,
+ );
+ await new Promise((resolve) => setTimeout(resolve, 300));
+ assert.equal(await valueOf("#react-name"), "John Roe");
+ assert.equal(
+ await cdp.evaluate("document.querySelector('#react-name').dataset.state"),
+ "John Roe",
+ );
+
+ // An autocomplete whose input is transparent and one pixel wide: the click
+ // goes to its control, and the text reaches the input.
+ const subject = await toolJson("type_locator", {
+ locator: byId("subjectsInput"),
+ text: "Mat",
+ });
+ assert.equal(subject.verified, true, JSON.stringify(subject));
+ assert.equal(await valueOf("#subjectsInput"), "Mat");
+ assert.match(
+ await cdp.evaluate(
+ "document.querySelector('#subjects-menu').textContent",
+ ),
+ /Maths/,
+ );
+ const batch = await toolJson("run_actions", {
+ actions: [
+ {
+ tool: "type_text",
+ arguments: { selector: "#subjectsInput", text: "Phy" },
+ },
+ ],
+ });
+ assert.equal(batch.completed, 1, JSON.stringify(batch));
+ assert.equal(await valueOf("#subjectsInput"), "Phy");
+
+ // Two names that differ by a suffix: the exact one is meant, and a
+ // locator that really is ambiguous says what it matched.
+ const first = await toolJson("type_locator", {
+ locator: { nameContains: "First Name" },
+ text: "Ada",
+ });
+ assert.equal(first.match.name, "First Name", JSON.stringify(first));
+ assert.equal(await valueOf("#first-name"), "Ada");
+ assert.equal(await valueOf("#first-name-optional"), "");
+ const ambiguous = await toolError("resolve_locator", {
+ locator: { role: "textbox", nameContains: "First" },
+ });
+ assert.equal(ambiguous.data.code, "LOCATOR_AMBIGUOUS");
+ assert.deepEqual(
+ ambiguous.data.candidates.map((candidate) => candidate.name).sort(),
+ ["First Name", "First Name (optional)"],
+ );
+ assert.match(ambiguous.message, /First Name \(optional\)/);
+
+ // Radios drawn by their labels: a text locator matches the label, a role
+ // and name locator matches the radio, and both click through the label.
+ const checked = () =>
+ cdp.evaluate("document.querySelector('input[name=gender]:checked')?.id");
+ const female = await toolJson("click_locator", {
+ locator: { text: "Female" },
+ });
+ assert.equal(female.clicked, true, JSON.stringify(female));
+ assert.equal(await checked(), "gender-radio-2");
+ await toolJson("click_locator", {
+ locator: { role: "radio", name: "Male" },
+ });
+ assert.equal(await checked(), "gender-radio-1");
+
+ // A day cell of a date grid.
+ await toolJson("click_locator", { locator: { text: "18" } });
+ assert.equal(
+ await cdp.evaluate("document.querySelector('#days').dataset.picked"),
+ "18",
+ );
+
+ // A field the page will not change: typing fails, and says what it holds.
+ const locked = await toolError("type_locator", {
+ locator: byId("locked"),
+ text: "nothing changes",
+ });
+ assert.equal(locked.data.code, "TYPE_NOT_APPLIED", JSON.stringify(locked));
+ assert.equal(locked.data.reason, "value");
+ assert.match(locked.message, /holds nothing/);
+
+ // A textarea at the bottom of the window, half under a fixed banner: the
+ // click and the typing land on the textarea, never on the banner.
+ const underBanner = `(() => {
+ const box = document.querySelector('#currentAddress').getBoundingClientRect();
+ window.scrollBy(0, box.top - (window.innerHeight - 140));
+ document.activeElement?.blur();
+ return true;
+ })()`;
+ await cdp.evaluate(underBanner);
+ await toolJson("click_locator", { locator: byId("currentAddress") });
+ assert.equal(
+ await cdp.evaluate("document.activeElement?.id"),
+ "currentAddress",
+ );
+ await cdp.evaluate(underBanner);
+ const address = await toolJson("type_locator", {
+ locator: { role: "textbox", name: "Current Address" },
+ text: "221B Baker Street",
+ });
+ assert.equal(address.verified, true, JSON.stringify(address));
+ assert.equal(await valueOf("#currentAddress"), "221B Baker Street");
+ assert.equal(
+ await cdp.evaluate("document.querySelector('#fixedban').dataset.clicks"),
+ undefined,
+ "nothing struck the banner",
+ );
+
+ // By index from a listing, with the same read-back.
+ const listing = (await tool("get_interactive_elements")).text;
+ const line = listing
+ .split("\n")
+ .find((entry) => entry.includes('id="sb_form_q"'));
+ const elementId = /^\[\d+ (e\d+)/.exec(line ?? "")?.[1];
+ assert.ok(elementId, listing);
+ assert.match(
+ (
+ await tool("type_by_index", {
+ element_id: elementId,
+ text: "by index",
+ })
+ ).text,
+ /the field holds the text/,
+ );
+ assert.equal(await valueOf("#sb_form_q"), "by index");
+
+ await request(`${base}/v1/profiles/${profile.id}/kill`, {
+ method: "POST",
+ token: saved.api_token,
+ });
+ if (browserPid) await waitForProcessExit(app, browserPid);
+ await app.invoke("stop_api_server");
+ await app.invoke("delete_profile", { profileId: profile.id });
+ } catch (error) {
+ await app.capture("failure");
+ throw error;
+ } finally {
+ cdp?.close();
+ if (app.session && browserPid && processExists(browserPid)) {
+ const running = (
+ await app.invoke("list_browser_profiles").catch(() => [])
+ ).find((item) => item.process_id === browserPid);
+ if (running)
+ await app
+ .invoke("kill_browser_profile", { profile: running })
+ .catch(() => {});
+ }
+ await app.close();
+ }
+});
+
/// Read every donut-proxy worker config the app has on disk.
async function readWorkerConfigs(app) {
const dir = path.join(app.dataRoot, "cache", "proxy_workers");
diff --git a/sdk/node/src/types.mts b/sdk/node/src/types.mts
index b315f580..f5583646 100644
--- a/sdk/node/src/types.mts
+++ b/sdk/node/src/types.mts
@@ -786,4 +786,10 @@ export interface AgentTyping {
duration_ms: number;
engine: Engine;
match: LocatorCandidate;
+ /**
+ * The field was read back and holds the text. False only for a field whose
+ * value is not the typed text (a date or colour picker); a field that does
+ * not hold the text fails with `TYPE_NOT_APPLIED`.
+ */
+ verified: boolean;
}
diff --git a/sdk/python/src/donutbrowser/models.py b/sdk/python/src/donutbrowser/models.py
index e7e352e7..87b65a96 100644
--- a/sdk/python/src/donutbrowser/models.py
+++ b/sdk/python/src/donutbrowser/models.py
@@ -913,13 +913,17 @@ class _AgentTypingRequired(TypedDict):
duration_ms: float
engine: str
match: LocatorCandidate
+ verified: bool
class AgentTyping(_AgentTypingRequired, total=False):
"""What a typing call did.
``corrections`` is absent on the fallback engine, which does not count its
- own mistypes.
+ own mistypes. ``verified`` is true when the field was read back and holds
+ the text; it is false only for a field whose value is not the typed text
+ (a date or colour picker). A field that does not hold the text fails with
+ ``TYPE_NOT_APPLIED``.
"""
corrections: int
diff --git a/src-tauri/src/api_server.rs b/src-tauri/src/api_server.rs
index 6e868a24..552a06e5 100644
--- a/src-tauri/src/api_server.rs
+++ b/src-tauri/src/api_server.rs
@@ -5505,6 +5505,8 @@ fn agent_error_response(error: crate::mcp_server::AgentError) -> (StatusCode, St
AgentError::AuthorizationUnavailable(_) => StatusCode::SERVICE_UNAVAILABLE,
AgentError::PickerTimedOut { .. } => StatusCode::REQUEST_TIMEOUT,
AgentError::PickerCancelled { .. } => StatusCode::CONFLICT,
+ // The page did not take the text: a conflict with its state, not a fault.
+ AgentError::TypeNotApplied { .. } => StatusCode::CONFLICT,
AgentError::Cdp(CdpError::Unauthorized(_)) => StatusCode::UNAUTHORIZED,
// Nothing answers on the profile's socket: the browser is gone, or is not
// drivable yet. Both read as "start it and try again".
@@ -5628,12 +5630,12 @@ async fn agent_click_api(
),
request_body = crate::mcp_server::AgentTypeRequest,
responses(
- (status = 200, description = "The text was typed", body = crate::mcp_server::AgentTyping),
+ (status = 200, description = "The text was typed, and the field read back holds it (`verified`)", body = crate::mcp_server::AgentTyping),
(status = 400, description = "Malformed request, no element matched, several did, or the text would take longer than the typing budget (`code` TYPING_TOO_LONG)"),
(status = 401, description = "Unauthorized"),
(status = 402, description = "Active paid plan with browser automation required"),
(status = 404, description = "Profile not found"),
- (status = 409, description = "The profile is not running"),
+ (status = 409, description = "The profile is not running, or the field did not take the focus or the text (`code` TYPE_NOT_APPLIED, `reason` not_focused or value)"),
(status = 429, description = "Automation request rate limit exceeded"),
(status = 502, description = "The browser did not answer as documented"),
(status = 500, description = "Internal server error")
diff --git a/src-tauri/src/mcp_server.rs b/src-tauri/src/mcp_server.rs
index 6fd52d16..30be5265 100644
--- a/src-tauri/src/mcp_server.rs
+++ b/src-tauri/src/mcp_server.rs
@@ -699,6 +699,15 @@ pub(crate) enum AgentError {
seconds: f64,
limit: f64,
},
+ /// The field does not hold the typed text: it never took the focus
+ /// (`reason` "not_focused"), or the page kept or reset another value
+ /// ("value").
+ TypeNotApplied {
+ reason: &'static str,
+ /// What the field holds; withheld for a password or card field.
+ value: Option,
+ message: String,
+ },
/// The browser refused the request as malformed (`-32602`), or the page
/// threw at it: what was asked for does not exist or cannot be done.
BadRequest(String),
@@ -724,6 +733,7 @@ impl AgentError {
Self::PickerTimedOut { .. } => "PICKER_TIMEOUT",
Self::PickerCancelled { .. } => "PICKER_CANCELLED",
Self::TypingTooLong { .. } => "TYPING_TOO_LONG",
+ Self::TypeNotApplied { .. } => "TYPE_NOT_APPLIED",
Self::BadRequest(_) => "BAD_REQUEST",
Self::Browser(_) => "BROWSER_ERROR",
Self::Cdp(_) => "CDP_ERROR",
@@ -744,7 +754,9 @@ impl AgentError {
Self::RequiresWayfern152 { version } => format!(
"This tool requires Wayfern 152 or newer; the profile runs {version}. Update the profile's browser, or use the selector and index tools."
),
- Self::AmbiguousLocator { message, .. } | Self::NoMatch { message } => message.clone(),
+ Self::AmbiguousLocator { message, .. }
+ | Self::NoMatch { message }
+ | Self::TypeNotApplied { message, .. } => message.clone(),
Self::PickerTimedOut { timeout_ms } => {
format!("No element was picked within {timeout_ms} ms; the picker has been disarmed")
}
@@ -785,6 +797,12 @@ impl AgentError {
detail["seconds"] = serde_json::Value::from(*seconds);
detail["limit"] = serde_json::Value::from(*limit);
}
+ Self::TypeNotApplied { reason, value, .. } => {
+ detail["reason"] = serde_json::Value::from(*reason);
+ if let Some(value) = value {
+ detail["value"] = serde_json::Value::from(value.as_str());
+ }
+ }
_ => {}
}
detail
@@ -852,6 +870,7 @@ impl From for AgentError {
WayfernError::NoMatch { message } => Self::NoMatch { message },
WayfernError::PickerTimedOut { timeout_ms } => Self::PickerTimedOut { timeout_ms },
WayfernError::PickerCancelled { reason } => Self::PickerCancelled { reason },
+ WayfernError::NotVisible(m) => Self::BadRequest(m),
WayfernError::Malformed(m) => Self::Malformed(m),
}
}
@@ -972,6 +991,10 @@ pub(crate) struct AgentTyping {
pub engine: Engine,
#[serde(rename = "match")]
pub matched: LocatorCandidate,
+ /// The field was read back and holds the text. False only for a field whose
+ /// value is not the typed text (a date or colour picker): a field that does
+ /// not hold the text is refused with `TYPE_NOT_APPLIED`.
+ pub verified: bool,
}
/// The body of `pick_element`.
@@ -1241,9 +1264,17 @@ const AGENT_FALLBACK_JS: &str = r#"(() => {
const attrValue = (el, name) => { if (name === 'value') { if (isProtected(el)) return null; const v = liveValue(el); if (v !== null) return v; } const v = el.getAttribute(name); return v == null ? null : v; };
const urlOf = (el) => { const tag = tagOf(el); if (tag === 'a' || tag === 'area') return el.href || undefined; if (tag === 'img') return el.currentSrc || el.src || undefined; return undefined; };
const visibleRect = (el) => { if (SKIP.has(String(el.tagName).toUpperCase())) return null; const style = window.getComputedStyle(el); if (style.display === 'none' || style.visibility === 'hidden' || style.opacity === '0') return null; const r = el.getBoundingClientRect(); if (!(r.width > 0 && r.height > 0)) return null; return r; };
+ // A form control the page draws itself (a styled radio, an autocomplete's
+ // input) is often transparent or too small to aim at. It is still the control
+ // a person uses, through its label when it has one; `hit` is what to act on.
+ // A transparent one hidden from assistive technology is a trap for bots.
+ const CONTROLS = new Set(['input', 'select', 'textarea']);
+ const presentRect = (el) => { if (SKIP.has(String(el.tagName).toUpperCase())) return null; const style = window.getComputedStyle(el); if (style.display === 'none' || style.visibility === 'hidden') return null; if (style.opacity === '0' && (!CONTROLS.has(tagOf(el)) || el.closest('[aria-hidden="true"]'))) return null; const r = el.getBoundingClientRect(); return r.width > 0 && r.height > 0 ? r : null; };
+ const shownOf = (el) => { const r = presentRect(el); const control = CONTROLS.has(tagOf(el)); if (r && !(control && (r.width < 4 || r.height < 4))) return { r, hit: el }; if (control && el.labels) { for (const l of el.labels) { const lr = visibleRect(l); if (lr) return { r: lr, hit: l }; } } return r ? { r, hit: el } : null; };
+ const signatureOf = (el) => { const role = roleOf(el); return 'js-' + fnv(tagOf(el) + '|' + role + '|' + nameOf(el, role) + '|' + pathOf(el)); };
const describe = (el, r) => {
const role = roleOf(el); const name = nameOf(el, role);
- const out = { role, name, text: contentText(el).slice(0, 2000), signature: 'js-' + fnv(tagOf(el) + '|' + role + '|' + name + '|' + pathOf(el)), attributes: attrsOf(el), bounds: { x: r.left + sx, y: r.top + sy, width: r.width, height: r.height } };
+ const out = { role, name, text: contentText(el).slice(0, 2000), signature: signatureOf(el), attributes: attrsOf(el), bounds: { x: r.left + sx, y: r.top + sy, width: r.width, height: r.height } };
if (!isProtected(el)) { const v = liveValue(el); if (v !== null) out.value = v; }
const u = urlOf(el); if (u) out.url = u;
return out;
@@ -1260,15 +1291,16 @@ const AGENT_FALLBACK_JS: &str = r#"(() => {
const bagOf = (pairs) => pairs.filter((a) => !skip.has(String(a.name).toLowerCase())).map((a) => String(a.name).toLowerCase() + '=' + a.value).sort().join('\n');
const wanted = bagOf(Array.isArray(want.attributes) ? want.attributes : []);
const squash = (s) => norm(s).replace(/ /g, '');
- let found = all.filter((el) => visibleRect(el) && bagOf(Array.from(el.attributes)) === wanted);
+ let found = [];
+ for (const el of all) { if (bagOf(Array.from(el.attributes)) !== wanted) continue; const shown = shownOf(el); if (shown) found.push({ el, hit: shown.hit }); }
// Each step narrows only when something survives it: the role and text the
// browser computed can differ from this script's in spacing or token.
- const narrow = (keep) => { if (found.length > 1) { const kept = found.filter(keep); if (kept.length) found = kept; } };
+ const narrow = (keep) => { if (found.length > 1) { const kept = found.filter((m) => keep(m.el)); if (kept.length) found = kept; } };
if (want.role) narrow((el) => roleMatchKey(roleOf(el)) === roleMatchKey(want.role));
if (want.text) narrow((el) => squash(contentText(el)) === squash(want.text));
if (want.name) narrow((el) => namesOf(el, roleOf(el)).includes(norm(want.name)));
- narrow((el) => !found.some((o) => o !== el && containsDeep(el, o)));
- return answer({ matchCount: found.length }, found.length === 1 ? found[0] : null);
+ narrow((el) => !found.some((o) => o.el !== el && containsDeep(el, o.el)));
+ return answer({ matchCount: found.length }, found.length === 1 ? found[0].hit : null);
}
if (opts.mode === 'resolve') {
@@ -1276,8 +1308,9 @@ const AGENT_FALLBACK_JS: &str = r#"(() => {
const roleWanted = L.role != null ? roleMatchKey(L.role) : null;
const attrs = Array.isArray(L.attributes) ? L.attributes : [];
const matches = [];
+ let hidden = 0;
for (const el of all) {
- const r = visibleRect(el); if (!r) continue;
+ if (SKIP.has(String(el.tagName).toUpperCase())) continue;
const role = roleOf(el);
if (roleWanted !== null && roleMatchKey(role) !== roleWanted) continue;
if (L.name != null || L.nameContains != null) {
@@ -1293,34 +1326,27 @@ const AGENT_FALLBACK_JS: &str = r#"(() => {
let ok = true;
for (const a of attrs) { if (!a || typeof a.name !== 'string' || attrValue(el, a.name) !== String(a.value)) { ok = false; break; } }
if (!ok) continue;
- matches.push({ el, r });
+ const shown = shownOf(el);
+ if (shown) matches.push({ el, r: shown.r, hit: shown.hit }); else hidden++;
}
+ // A label and the control it stands in for are one match: the control.
+ const merged = matches.filter((m) => !(m.el === m.hit && matches.some((o) => o !== m && o.hit === m.el)));
// A wrapper matches a text locator only because its child does; the
// innermost of nested matches is the element the caller meant.
- const kept = matches.filter((m) => !matches.some((o) => o !== m && m.el !== o.el && containsDeep(m.el, o.el)));
+ let kept = merged.filter((m) => !merged.some((o) => o !== m && m.el !== o.el && containsDeep(m.el, o.el)));
+ if (opts.pick) { const chosen = kept.filter((m) => signatureOf(m.el) === opts.pick); if (chosen.length === 1) kept = chosen; }
const limit = Math.max(1, Math.min(100, Number(opts.candidateLimit) || 10));
- const result = { matchCount: kept.length, candidates: kept.slice(0, limit).map((m) => describe(m.el, m.r)) };
+ const result = { matchCount: kept.length, hiddenCount: hidden, candidates: kept.slice(0, limit).map((m) => describe(m.el, m.r)) };
if (kept.length === 1) {
- const el = kept[0].el;
- if (opts.act === 'scroll' || opts.act === 'focus') {
- try { el.scrollIntoView({ block: 'center', inline: 'center', behavior: 'instant' }); } catch (e) { el.scrollIntoView(); }
+ const { el, hit } = kept[0];
+ if (opts.act === 'scroll') {
+ try { hit.scrollIntoView({ block: 'center', inline: 'center', behavior: 'instant' }); } catch (e) { hit.scrollIntoView(); }
}
- if (opts.act === 'focus') {
- el.focus();
- const editable = !!el.isContentEditable; const tag = tagOf(el);
- if (opts.clearFirst) {
- if (editable) { el.textContent = ''; } else if (tag === 'input' || tag === 'textarea') { el.value = ''; }
- el.dispatchEvent(new Event('input', { bubbles: true }));
- } else if (editable) {
- const sel = window.getSelection(); if (sel) { sel.selectAllChildren(el); sel.collapseToEnd(); }
- } else if (typeof el.setSelectionRange === 'function') {
- try { const n = String(el.value || '').length; el.setSelectionRange(n, n); } catch (e) {}
- }
- }
- result.match = describe(el, el.getBoundingClientRect());
- result.center = clampCenter(el.getBoundingClientRect());
+ const box = hit.getBoundingClientRect();
+ result.match = describe(el, box);
+ result.center = clampCenter(box);
}
- return answer(result, kept.length === 1 ? kept[0].el : null);
+ return answer(result, kept.length === 1 ? kept[0].hit : null);
}
const maxNodes = Number(opts.maxNodes) > 0 ? Number(opts.maxNodes) : 100000;
@@ -1402,6 +1428,9 @@ struct FallbackCenter {
#[serde(rename_all = "camelCase")]
struct FallbackResolution {
match_count: u64,
+ /// Elements that match but have no box on screen.
+ #[serde(default)]
+ hidden_count: u64,
#[serde(default)]
candidates: Vec,
#[serde(rename = "match")]
@@ -1409,6 +1438,122 @@ struct FallbackResolution {
center: Option,
}
+impl FallbackResolution {
+ fn from_report(report: serde_json::Value) -> Result {
+ serde_json::from_value(report)
+ .map_err(|e| AgentError::Malformed(format!("fallback resolver: {e}")))
+ }
+
+ /// The signature of the candidate an ambiguous resolution prefers.
+ fn preferred_signature(&self, locator: &LocatorDescription) -> Option {
+ let index = preferred_candidate(locator, self.match_count, &self.candidates)?;
+ self.candidates[index]["signature"]
+ .as_str()
+ .map(str::to_string)
+ }
+}
+
+/// How many candidates a resolver is asked for, so that an ambiguity can be
+/// ranked over all of them. The caller's own limit applies to the error.
+const RANKED_CANDIDATES: u64 = 100;
+
+/// Collapsed whitespace, the way both resolvers compare names and text.
+fn collapsed(text: &str) -> String {
+ text.split_whitespace().collect::>().join(" ")
+}
+
+/// Roles of the words that name a control rather than the control: a label,
+/// and the text inside one.
+const LABEL_ROLES: [&str; 5] = ["labeltext", "label", "statictext", "inlinetextbox", "text"];
+
+/// The one candidate of an ambiguous match that stands out, if one does.
+///
+/// Only a complete list is ranked: a candidate left out of it could be the
+/// better one. A visible candidate comes first; then, for a substring locator,
+/// the one whose name or text is exactly the substring (case first, then any
+/// case); then a control over the label, or the label's text, that names it;
+/// then the innermost, whose box no other candidate's box lies within. Each
+/// step narrows only when something survives it.
+fn preferred_candidate(
+ locator: &LocatorDescription,
+ match_count: u64,
+ candidates: &[serde_json::Value],
+) -> Option {
+ if candidates.len() < 2 || candidates.len() as u64 != match_count {
+ return None;
+ }
+ let narrow = |kept: &mut Vec, keep: &dyn Fn(usize) -> bool| {
+ let next: Vec = kept.iter().copied().filter(|&i| keep(i)).collect();
+ if !next.is_empty() {
+ *kept = next;
+ }
+ };
+ let bounds = |i: usize| {
+ let side = |key: &str| candidates[i]["bounds"][key].as_f64().unwrap_or(0.0);
+ (side("x"), side("y"), side("width"), side("height"))
+ };
+ let mut kept: Vec = (0..candidates.len()).collect();
+ narrow(&mut kept, &|i| {
+ let (_, _, width, height) = bounds(i);
+ width > 0.0 && height > 0.0
+ });
+ for (key, wanted) in [
+ ("name", locator.name_contains.as_deref()),
+ ("text", locator.text_contains.as_deref()),
+ ] {
+ let Some(wanted) = wanted.map(collapsed) else {
+ continue;
+ };
+ let field = |i: usize| collapsed(candidates[i][key].as_str().unwrap_or_default());
+ narrow(&mut kept, &|i| field(i) == wanted);
+ narrow(&mut kept, &|i| {
+ field(i).to_lowercase() == wanted.to_lowercase()
+ });
+ }
+ narrow(&mut kept, &|i| {
+ let role: String = candidates[i]["role"]
+ .as_str()
+ .unwrap_or_default()
+ .chars()
+ .filter(char::is_ascii_alphanumeric)
+ .collect::()
+ .to_ascii_lowercase();
+ !LABEL_ROLES.contains(&role.as_str())
+ });
+ // Innermost, as the page's resolver keeps it: no box around another match.
+ let around = |outer: usize, inner: usize| {
+ let (left, top, width, height) = bounds(outer);
+ let (in_left, in_top, in_width, in_height) = bounds(inner);
+ width * height > in_width * in_height
+ && left <= in_left
+ && top <= in_top
+ && left + width >= in_left + in_width
+ && top + height >= in_top + in_height
+ };
+ let before = kept.clone();
+ narrow(&mut kept, &|i| {
+ !before.iter().any(|&other| other != i && around(i, other))
+ });
+ (kept.len() == 1).then(|| kept[0])
+}
+
+/// The ambiguity refusal, listing at most `limit` candidates.
+fn ambiguity(
+ match_count: u64,
+ mut candidates: Vec,
+ limit: Option,
+) -> AgentError {
+ candidates.truncate(limit.unwrap_or(10).clamp(1, RANKED_CANDIDATES) as usize);
+ AgentError::AmbiguousLocator {
+ match_count,
+ message: format!(
+ "Ambiguous locator: {match_count} nodes match. Refine it with a role, a stable attribute, or more exact text. Candidates: {}",
+ serde_json::Value::Array(candidates.clone())
+ ),
+ candidates,
+ }
+}
+
/// What to do to the one match, once the fallback resolver has found it.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
enum FallbackAct {
@@ -1416,8 +1561,6 @@ enum FallbackAct {
Describe,
/// Scroll it into view, for a click.
Scroll,
- /// Scroll, focus and prepare the field, for typing.
- Focus { clear_first: bool },
}
/// Resolve `locator` in the page with the fallback script.
@@ -1430,45 +1573,57 @@ async fn fallback_resolve(
candidate_limit: Option,
act: FallbackAct,
) -> Result<(LocatorCandidate, Option), AgentError> {
- let (act_name, clear_first) = match act {
- FallbackAct::Describe => ("none", false),
- FallbackAct::Scroll => ("scroll", false),
- FallbackAct::Focus { clear_first } => ("focus", clear_first),
+ let act_name = match act {
+ FallbackAct::Describe => "none",
+ FallbackAct::Scroll => "scroll",
};
- let opts = serde_json::json!({
- "mode": "resolve",
- "locator": locator,
- "candidateLimit": candidate_limit.unwrap_or(10).clamp(1, 100),
- "act": act_name,
- "clearFirst": clear_first,
- });
- let answer = evaluate_json_script(target, fallback_script(&opts)).await?;
- let resolution: FallbackResolution = serde_json::from_value(answer)
- .map_err(|e| AgentError::Malformed(format!("fallback resolver: {e}")))?;
+ let script = |pick: Option<&str>| {
+ fallback_script(&serde_json::json!({
+ "mode": "resolve",
+ "locator": locator,
+ "candidateLimit": RANKED_CANDIDATES,
+ "act": act_name,
+ "pick": pick,
+ }))
+ };
+ let mut resolution =
+ FallbackResolution::from_report(evaluate_json_script(target, script(None)).await?)?;
+ if let Some(signature) = resolution.preferred_signature(locator) {
+ let picked = FallbackResolution::from_report(
+ evaluate_json_script(target, script(Some(&signature))).await?,
+ )?;
+ if picked.match_count == 1 {
+ resolution = picked;
+ }
+ }
let center = resolution.center;
- Ok((settle_fallback(resolution, locator)?, center))
+ Ok((
+ settle_fallback(resolution, locator, candidate_limit)?,
+ center,
+ ))
}
/// The one match a fallback resolution found, or its refusal.
fn settle_fallback(
resolution: FallbackResolution,
locator: &LocatorDescription,
+ candidate_limit: Option,
) -> Result {
match resolution.match_count {
+ 0 if resolution.hidden_count > 0 => Err(AgentError::NoMatch {
+ message: format!(
+ "No visible node matches locator ({}); {} hidden element(s) match it. Open, expand or scroll to the part of the page that shows it, or name what is shown.",
+ describe_locator(locator),
+ resolution.hidden_count
+ ),
+ }),
0 => Err(AgentError::NoMatch {
message: format!("No node matches locator ({}).", describe_locator(locator)),
}),
1 => resolution
.matched
.ok_or_else(|| AgentError::Malformed("fallback resolver answered one match without it".into())),
- count => Err(AgentError::AmbiguousLocator {
- match_count: count,
- message: format!(
- "Ambiguous locator: {count} nodes match. Refine it with a role, a stable attribute, or more exact text. Candidates: {}",
- serde_json::Value::Array(resolution.candidates.clone())
- ),
- candidates: resolution.candidates,
- }),
+ count => Err(ambiguity(count, resolution.candidates, candidate_limit)),
}
}
@@ -1658,46 +1813,59 @@ async fn page_element(
/// when the DOM confirms it; otherwise the page finds the element the browser
/// described. When the browser matches nothing, the page applies the locator
/// itself: the index never carries `id` or `class`, and a large page can outrun
-/// its snapshot budget.
+/// its snapshot budget. Several matches settle on the one
+/// [`preferred_candidate`] picks, or are refused.
async fn wayfern_locate(
session: &mut WayfernSession,
locator: &LocatorDescription,
candidate_limit: Option,
) -> Result {
let options = ResolveOptions {
- candidate_limit,
+ candidate_limit: Some(RANKED_CANDIDATES),
..Default::default()
};
- match wayfern_cdp::resolve_locator(session, locator, options).await {
- Ok(resolution) => {
- let mut matched = resolution.matched;
- let backend_node_id = match confirmed_node_id(session, &matched).await {
- Some(id) => Some(id),
- None => {
- let opts = serde_json::json!({
- "mode": "pin",
- "match": &matched,
- "skipAttributes": UNCOMPARED_ATTRIBUTES,
- "returnElement": true,
- });
- page_element(session, &opts).await?.1
- }
- };
- if let Some(id) = backend_node_id {
- matched.backend_node_id = Some(id);
- return Ok(LocatedNode {
- backend_node_id: id,
- matched,
- engine: Engine::Wayfern,
- });
+ let native = match wayfern_cdp::resolve_locator(session, locator, options).await {
+ Ok(resolution) => Some(resolution.matched),
+ Err(WayfernError::NoMatch { .. }) => None,
+ Err(WayfernError::AmbiguousLocator {
+ match_count,
+ candidates,
+ ..
+ }) => {
+ let preferred = preferred_candidate(locator, match_count, &candidates)
+ .and_then(|index| serde_json::from_value(candidates[index].clone()).ok());
+ match preferred {
+ Some(matched) => Some(matched),
+ None => return Err(ambiguity(match_count, candidates, candidate_limit)),
}
- log::debug!(
- "Locator match not found in the page, applying the locator there node={}",
- matched.backend_node_id.unwrap_or_default()
- );
}
- Err(WayfernError::NoMatch { .. }) => {}
Err(other) => return Err(other.into()),
+ };
+ if let Some(mut matched) = native {
+ let backend_node_id = match confirmed_node_id(session, &matched).await {
+ Some(id) => Some(id),
+ None => {
+ let opts = serde_json::json!({
+ "mode": "pin",
+ "match": &matched,
+ "skipAttributes": UNCOMPARED_ATTRIBUTES,
+ "returnElement": true,
+ });
+ page_element(session, &opts).await?.1
+ }
+ };
+ if let Some(id) = backend_node_id {
+ matched.backend_node_id = Some(id);
+ return Ok(LocatedNode {
+ backend_node_id: id,
+ matched,
+ engine: Engine::Wayfern,
+ });
+ }
+ log::debug!(
+ "Locator match not found in the page, applying the locator there node={}",
+ matched.backend_node_id.unwrap_or_default()
+ );
}
page_locate(session, locator, candidate_limit).await
}
@@ -1708,17 +1876,27 @@ async fn page_locate(
locator: &LocatorDescription,
candidate_limit: Option,
) -> Result {
- let opts = serde_json::json!({
- "mode": "resolve",
- "locator": locator,
- "candidateLimit": candidate_limit.unwrap_or(10).clamp(1, 100),
- "act": "none",
- "returnElement": true,
- });
- let (report, backend_node_id) = page_element(session, &opts).await?;
- let resolution: FallbackResolution = serde_json::from_value(report)
- .map_err(|e| AgentError::Malformed(format!("fallback resolver: {e}")))?;
- let mut matched = settle_fallback(resolution, locator)?;
+ let opts = |pick: Option<&str>| {
+ serde_json::json!({
+ "mode": "resolve",
+ "locator": locator,
+ "candidateLimit": RANKED_CANDIDATES,
+ "act": "none",
+ "returnElement": true,
+ "pick": pick,
+ })
+ };
+ let (report, mut backend_node_id) = page_element(session, &opts(None)).await?;
+ let mut resolution = FallbackResolution::from_report(report)?;
+ if let Some(signature) = resolution.preferred_signature(locator) {
+ let (report, picked_node) = page_element(session, &opts(Some(&signature))).await?;
+ let picked = FallbackResolution::from_report(report)?;
+ if picked.match_count == 1 {
+ resolution = picked;
+ backend_node_id = picked_node;
+ }
+ }
+ let mut matched = settle_fallback(resolution, locator, candidate_limit)?;
let backend_node_id = backend_node_id
.ok_or_else(|| AgentError::Malformed("the page resolved a match it did not return".into()))?;
matched.backend_node_id = Some(backend_node_id);
@@ -1841,137 +2019,514 @@ async fn vellum_click(
outcome
}
-/// What to do to a field between the strike that focuses it and the typing.
-enum FieldPreparation {
- /// Nothing: the field was already prepared by the caller's script.
- None,
- /// Run this script, which must return `true`.
- Script(String),
- /// Clear, or move the caret to the end of, the node behind this id.
- Node { backend_node_id: i64, clear: bool },
-}
-
-/// Script run on a resolved node to empty it, or park the caret at its end.
+/// One page function for every step of typing into a field, run on an element
+/// with `Runtime.callFunctionOn` and named by its first argument:
///
-/// A strike puts the caret where the click landed; without this a text typed
-/// into a field that already holds one would land in the middle of it.
-const PREPARE_FIELD_FN: &str = r#"function(clear) {
+/// - `field`: what to type into: the element, the control a label is for, the
+/// one field inside a container, or the editing host of a rich text editor.
+/// Answered by reference.
+/// - `point`: where a hand would click it, in viewport pixels: on the element,
+/// its label or, for a control too small to aim at, the box around it, and
+/// only where that is what the click would hit. When nothing of it can be hit
+/// where it is, it is scrolled to the centre first. `covered` names what lies
+/// on top when still nothing can be hit, `hidden` means it has no box, and
+/// `frame` that it is in a frame, where the DOM's quads decide.
+/// - `focus`: whether the focus is on the field or inside it, after focusing it
+/// when `arg` is true and it is not.
+/// - `prepare`: what the field holds; then, with `arg`, select all of it, so
+/// the typed text replaces it with the input events a person's typing fires,
+/// or else park the caret at its end.
+/// - `read`: what the field holds, and whether the focus moved on to another
+/// field while the text was typed.
+const TYPING_FN: &str = r#"function (step, arg) {
const el = this;
- const editable = !!el.isContentEditable;
- const tag = String(el.tagName || '').toLowerCase();
- if (clear) {
- if (editable) { el.textContent = ''; } else if (tag === 'input' || tag === 'textarea') { el.value = ''; }
- el.dispatchEvent(new Event('input', { bubbles: true }));
- } else if (editable) {
- const sel = window.getSelection(); if (sel) { sel.selectAllChildren(el); sel.collapseToEnd(); }
- } else if (typeof el.setSelectionRange === 'function') {
- try { const n = String(el.value || '').length; el.setSelectionRange(n, n); } catch (e) {}
+ const tagOf = (n) => String((n && n.tagName) || '').toLowerCase();
+ const NOT_TEXT = new Set(['checkbox', 'radio', 'button', 'submit', 'reset', 'file', 'image', 'range', 'color', 'hidden']);
+ const editable = (n) => !!n && n.nodeType === 1 && ((tagOf(n) === 'input' && !NOT_TEXT.has(String(n.type || 'text').toLowerCase())) || tagOf(n) === 'textarea' || !!n.isContentEditable);
+ const parentOf = (n) => n.parentElement || (n.parentNode && n.parentNode.host) || null;
+ const containsDeep = (outer, inner) => { for (let n = inner; n; n = n.parentNode || n.host) { if (n === outer) return true; } return false; };
+ const deepActive = () => { let a = document.activeElement; while (a && a.shadowRoot && a.shadowRoot.activeElement) a = a.shadowRoot.activeElement; return a; };
+ const nameOf = (n) => { if (!n || n === document.body || n === document.documentElement) return 'the page'; let s = tagOf(n); if (n.id) s += '#' + n.id; const label = n.getAttribute && (n.getAttribute('aria-label') || n.getAttribute('name')); if (label) s += ' "' + String(label).slice(0, 40) + '"'; return s; };
+ const labelsOf = (n) => Array.from(n.labels || []);
+ const typedInto = (n) => { if (editable(n)) return n; const a = deepActive(); return a && containsDeep(n, a) && editable(a) ? a : null; };
+ const valueOf = (n) => {
+ const f = typedInto(n);
+ if (!f) return { kind: 'none' };
+ const tag = tagOf(f);
+ const type = tag === 'input' ? String(f.type || 'text').toLowerCase() : tag;
+ const secret = type === 'password' || /(^|\s)(cc-|password|one-time-code)/i.test(f.getAttribute('autocomplete') || '');
+ if (tag === 'input' || tag === 'textarea') return { kind: 'value', type, value: String(f.value), secret };
+ return { kind: 'editable', type, value: String(f.innerText || ''), secret };
+ };
+ if (step === 'field') {
+ if (el.isContentEditable) { let h = el; while (parentOf(h) && parentOf(h).isContentEditable) h = parentOf(h); return h; }
+ if (editable(el)) return el;
+ if (el.control && editable(el.control)) return el.control;
+ const inner = [];
+ const walk = (root) => { for (const n of root.querySelectorAll('*')) { if (editable(n) && !(n.isContentEditable && parentOf(n) && parentOf(n).isContentEditable)) inner.push(n); if (n.shadowRoot) walk(n.shadowRoot); } };
+ walk(el);
+ if (el.shadowRoot) walk(el.shadowRoot);
+ return inner.length === 1 ? inner[0] : el;
}
- return true;
+ if (step === 'point') {
+ if (window !== window.top) return { frame: true };
+ const vw = window.innerWidth, vh = window.innerHeight;
+ const aimable = (n) => { const r = n.getBoundingClientRect(); if (!(r.width >= 4 && r.height >= 4)) return false; const s = window.getComputedStyle(n); return s.display !== 'none' && s.visibility !== 'hidden'; };
+ const boxes = [];
+ if (aimable(el)) boxes.push(el);
+ for (const l of labelsOf(el)) { if (aimable(l)) boxes.push(l); }
+ if (!boxes.length) { for (let p = parentOf(el), i = 0; p && i < 4; p = parentOf(p), i++) { if (aimable(p)) { boxes.push(p); break; } } }
+ if (!boxes.length) return { hidden: true };
+ const hitAt = (x, y) => { let h = document.elementFromPoint(x, y); while (h && h.shadowRoot) { const inner = h.shadowRoot.elementFromPoint(x, y); if (!inner || inner === h) break; h = inner; } return h; };
+ const takes = (box, h) => !!h && (containsDeep(el, h) || containsDeep(box, h) || labelsOf(el).some((l) => containsDeep(l, h)));
+ const centre = (box) => { const r = box.getBoundingClientRect(); const left = Math.max(r.left, 0), top = Math.max(r.top, 0), right = Math.min(r.right, vw), bottom = Math.min(r.bottom, vh); if (right - left < 1 || bottom - top < 1) return null; return { x: left + (right - left) / 2, y: top + (bottom - top) / 2, width: right - left, height: bottom - top }; };
+ const aim = () => { for (const box of boxes) { const p = centre(box); if (p && takes(box, hitAt(p.x, p.y))) return p; } return null; };
+ let point = aim();
+ if (!point) { try { boxes[0].scrollIntoView({ block: 'center', inline: 'center', behavior: 'instant' }); } catch (e) { boxes[0].scrollIntoView(); } point = aim(); }
+ if (point) return point;
+ const p = centre(boxes[0]);
+ return p ? Object.assign(p, { covered: nameOf(hitAt(p.x, p.y)) }) : { hidden: true };
+ }
+ if (step === 'focus') {
+ const holds = () => { const a = deepActive(); return !!a && (a === el || containsDeep(el, a)); };
+ if (!holds() && arg) { try { el.focus({ preventScroll: true }); } catch (e) {} }
+ return { focused: holds(), active: nameOf(deepActive()) };
+ }
+ if (step === 'prepare') {
+ const before = valueOf(el);
+ const f = typedInto(el);
+ if (f && f.isContentEditable) { const sel = window.getSelection(); if (sel) { sel.selectAllChildren(f); if (!arg) sel.collapseToEnd(); } }
+ else if (f && arg) { try { f.select(); } catch (e) {} }
+ else if (f) { try { const n = String(f.value).length; f.setSelectionRange(n, n); } catch (e) {} }
+ return before;
+ }
+ if (step === 'read') {
+ const now = valueOf(el);
+ const a = deepActive();
+ now.focusMoved = !!a && a !== el && !containsDeep(el, a) && editable(a);
+ return now;
+ }
+ return null;
}"#;
-/// Script run on the focused element to park the caret at its end.
-const CARET_TO_END_JS: &str = r#"(() => {
- const el = document.activeElement;
- if (!el) return true;
- if (el.isContentEditable) { const sel = window.getSelection(); if (sel) { sel.selectAllChildren(el); sel.collapseToEnd(); } }
- else if (typeof el.setSelectionRange === 'function') { try { const n = String(el.value || '').length; el.setSelectionRange(n, n); } catch (e) {} }
- return true;
-})()"#;
+/// Where a hand would click an element, as the `point` step of [`TYPING_FN`]
+/// answers it.
+enum Aim {
+ /// What the click would hit there is the element, or its label.
+ At(ViewportTarget),
+ /// Something else lies on top of the element's centre, named here.
+ Covered(ViewportTarget, String),
+ /// The element has no box to click.
+ Hidden,
+ /// The element is in a frame; its position is the DOM's to tell.
+ Frame,
+}
-async fn prepare_field(
+#[derive(Debug, Default, Deserialize)]
+struct AimReply {
+ #[serde(default)]
+ x: f64,
+ #[serde(default)]
+ y: f64,
+ #[serde(default)]
+ width: f64,
+ #[serde(default)]
+ height: f64,
+ #[serde(default)]
+ covered: Option,
+ #[serde(default)]
+ hidden: bool,
+ #[serde(default)]
+ frame: bool,
+}
+
+/// What a field holds, as the `prepare` and `read` steps answer it.
+#[derive(Debug, Default, Deserialize)]
+#[serde(rename_all = "camelCase")]
+struct FieldValue {
+ /// "value" for an input or textarea, "editable" for rich text, "none" when
+ /// the element is no field and holds none in focus.
+ #[serde(default)]
+ kind: String,
+ #[serde(default, rename = "type")]
+ field_type: String,
+ #[serde(default)]
+ value: String,
+ /// A password or card field: its value is compared, never shown.
+ #[serde(default)]
+ secret: bool,
+ #[serde(default)]
+ focus_moved: bool,
+}
+
+/// Field types whose value is the text typed into them. A date, time or
+/// colour field turns keys into its own value, which a comparison would
+/// misread.
+const TEXT_FIELD_TYPES: [&str; 8] = [
+ "text", "search", "email", "url", "tel", "password", "number", "textarea",
+];
+
+impl FieldValue {
+ fn holds_typed_text(&self) -> bool {
+ self.kind == "editable"
+ || (self.kind == "value" && TEXT_FIELD_TYPES.contains(&self.field_type.as_str()))
+ }
+}
+
+#[derive(Debug, Default, Deserialize)]
+struct FieldFocus {
+ #[serde(default)]
+ focused: bool,
+ #[serde(default)]
+ active: String,
+}
+
+/// Run one step of [`TYPING_FN`] on `object` and read its answer.
+async fn typing_step(
session: &mut WayfernSession,
- preparation: &FieldPreparation,
-) -> Result<(), AgentError> {
- match preparation {
- FieldPreparation::None => Ok(()),
- FieldPreparation::Script(script) => {
- let result = session
- .call(
- "Runtime.evaluate",
- serde_json::json!({ "expression": script, "returnByValue": true }),
- )
- .await?;
- if let Some(exception) = result.get("exceptionDetails") {
- let message = exception
- .get("exception")
- .and_then(|e| e.get("description"))
- .or_else(|| exception.get("text"))
- .and_then(|v| v.as_str())
- .unwrap_or("preparing the field failed");
- return Err(AgentError::BadRequest(message.to_string()));
- }
- Ok(())
+ object: &str,
+ step: &str,
+ arg: serde_json::Value,
+) -> Result {
+ let answer = call_on(
+ session,
+ object,
+ TYPING_FN,
+ serde_json::json!([{ "value": step }, { "value": arg }]),
+ )
+ .await?;
+ serde_json::from_value(answer)
+ .map_err(|e| AgentError::Malformed(format!("the page answered the {step} step with: {e}")))
+}
+
+/// The field behind `element`, by reference in [`FORM_GROUP`].
+async fn typing_field(session: &mut WayfernSession, element: &str) -> Result {
+ let reply = session
+ .call(
+ "Runtime.callFunctionOn",
+ serde_json::json!({
+ "objectId": element,
+ "functionDeclaration": TYPING_FN,
+ "arguments": [{ "value": "field" }, { "value": null }],
+ "returnByValue": false,
+ "objectGroup": FORM_GROUP,
+ }),
+ )
+ .await?;
+ if let Some(thrown) = script_exception(&reply) {
+ return Err(thrown);
+ }
+ reply
+ .pointer("/result/objectId")
+ .and_then(|v| v.as_str())
+ .map(str::to_string)
+ .ok_or_else(|| AgentError::Malformed("the page answered the field step with no element".into()))
+}
+
+/// Where a hand would click `object`.
+async fn aim_at(session: &mut WayfernSession, object: &str) -> Result {
+ let reply: AimReply = typing_step(session, object, "point", serde_json::Value::Null).await?;
+ if reply.frame {
+ return Ok(Aim::Frame);
+ }
+ if reply.hidden || !(reply.width > 0.0 && reply.height > 0.0) {
+ return Ok(Aim::Hidden);
+ }
+ let point = ViewportTarget {
+ x: reply.x,
+ y: reply.y,
+ width: reply.width,
+ height: reply.height,
+ };
+ Ok(match reply.covered {
+ Some(cover) => Aim::Covered(point, cover),
+ None => Aim::At(point),
+ })
+}
+
+/// The refusal for an element with no box to act on.
+fn not_visible() -> AgentError {
+ AgentError::BadRequest(
+ "The element has no visible box to click or type into; it may be hidden or collapsed. Open, expand or scroll to the part of the page that shows it."
+ .to_string(),
+ )
+}
+
+/// Where to strike a field before typing, or `None` when something covers it
+/// and a click would land on that instead.
+async fn typing_point(
+ session: &mut WayfernSession,
+ field: &str,
+ backend_node_id: Option,
+) -> Result, AgentError> {
+ match aim_at(session, field).await? {
+ Aim::At(point) => Ok(Some(point)),
+ Aim::Covered(_, cover) => {
+ log::debug!("Field is covered, focusing it without a click cover={cover}");
+ Ok(None)
}
- FieldPreparation::Node {
- backend_node_id,
- clear,
- } => {
- let resolved = session
- .call(
- "DOM.resolveNode",
- serde_json::json!({ "backendNodeId": backend_node_id }),
- )
- .await?;
- let object_id = resolved
- .get("object")
- .and_then(|o| o.get("objectId"))
- .and_then(|v| v.as_str())
- .ok_or_else(|| AgentError::Malformed("DOM.resolveNode answered no object".into()))?;
- session
- .call(
- "Runtime.callFunctionOn",
- serde_json::json!({
- "objectId": object_id,
- "functionDeclaration": PREPARE_FIELD_FN,
- "arguments": [{ "value": clear }],
- "returnByValue": true,
- }),
- )
- .await?;
- Ok(())
+ Aim::Hidden => Err(not_visible()),
+ Aim::Frame => match backend_node_id {
+ Some(id) => Ok(Some(wayfern_cdp::viewport_target(session, id).await?)),
+ None => Err(AgentError::BadRequest(
+ "The field is inside a frame; name it with a locator (type_locator).".to_string(),
+ )),
+ },
+ }
+}
+
+/// How typed keys reach a field.
+#[derive(Clone, Copy)]
+enum TypingMethod<'a> {
+ /// The browser's own humanized keyboard, Wayfern 152 and newer.
+ Vellum { typos: bool, timeout: Duration },
+ /// An accepted keystroke plan, sent as key events.
+ Keystrokes(&'a [crate::human_typing::TypingEvent]),
+ /// The whole text at once, the way an input method commits it.
+ Insert,
+}
+
+impl<'a> TypingMethod<'a> {
+ /// The method the budgets decided on: Vellum when it was budgeted, the plan
+ /// when one was made, otherwise all at once.
+ fn chosen(
+ vellum_timeout: Option,
+ plan: Option<&'a [crate::human_typing::TypingEvent]>,
+ typos: bool,
+ ) -> Self {
+ match (vellum_timeout, plan) {
+ (Some(timeout), _) => Self::Vellum { typos, timeout },
+ (None, Some(plan)) => Self::Keystrokes(plan),
+ (None, None) => Self::Insert,
}
}
}
-/// Focus `point` with a strike, prepare the field, and type `text`.
-async fn vellum_type_in(
- session: &mut WayfernSession,
- point: ViewportTarget,
- text: &str,
- typos: bool,
- preparation: FieldPreparation,
- timeout: Duration,
-) -> Result {
- let viewport = wayfern_cdp::layout_viewport(session).await?;
- let point = clamp_to_viewport(point, viewport);
- let origin = wayfern_cdp::glide_origin(&point, viewport);
- let width = Some(point.width.min(point.height));
- // Owned by the gesture, for the reason `vellum_click_in` gives.
- let text = text.to_owned();
- let inscription = vellum::with_pointer(session, origin.0, origin.1, |s, p| {
- Box::pin(async move {
- vellum::glide(s, p, point.x, point.y, width).await?;
- vellum::strike(s, p, None, None).await?;
- prepare_field(s, &preparation).await?;
- Ok::<_, AgentError>(vellum::inscribe(s, p, &text, typos, timeout).await?)
- })
- })
- .await?;
- Ok(inscription)
+/// How long the page has to answer the keys (a framework's re-render, a mask)
+/// before the field is read back.
+const TYPING_SETTLE: Duration = Duration::from_millis(150);
+
+/// What typing into a field did.
+struct FieldTyping {
+ inscription: Option,
+ /// The field was read back and holds the text. False when it is not a field
+ /// whose value is the typed text (a date picker, a colour well).
+ verified: bool,
}
-/// Type `text` at `point` on a fresh session.
-async fn vellum_type(
- target: &CdpTarget,
+/// Letters and digits, lower-cased: what survives an input mask.
+fn mask_fold(text: &str) -> String {
+ text
+ .chars()
+ .filter(|c| c.is_alphanumeric())
+ .flat_map(char::to_lowercase)
+ .collect()
+}
+
+/// Whether a field that held `before` and holds `after` took `typed`.
+///
+/// Compared on letters and digits, so a mask that adds separators, a prefix or
+/// a different case still counts. When the field was not emptied, only what
+/// came after `before` counts. A field that moved the focus on while the text
+/// was typed (one box per character) took it when it holds its start. A text
+/// with no letters or digits cannot be told apart from a mask and passes.
+fn field_took(typed: &str, before: &str, after: &str, cleared: bool, focus_moved: bool) -> bool {
+ let wanted = mask_fold(typed);
+ if wanted.is_empty() {
+ return true;
+ }
+ let now = mask_fold(after);
+ let held = mask_fold(before);
+ let added = if cleared {
+ now.as_str()
+ } else {
+ now.strip_prefix(held.as_str()).unwrap_or(now.as_str())
+ };
+ added.contains(&wanted) || (focus_moved && !added.is_empty() && wanted.starts_with(added))
+}
+
+/// The verdict on a field read back after typing: whether it was checked, or
+/// the refusal when it does not hold the text.
+fn typing_verdict(
+ typed: &str,
+ before: &FieldValue,
+ after: &FieldValue,
+ cleared: bool,
+) -> Result {
+ if !after.holds_typed_text() {
+ return Ok(false);
+ }
+ if field_took(
+ typed,
+ &before.value,
+ &after.value,
+ cleared,
+ after.focus_moved,
+ ) {
+ return Ok(true);
+ }
+ let shown: String = after.value.chars().take(200).collect();
+ let holds = if after.secret {
+ format!("{} characters", after.value.chars().count())
+ } else if after.value.is_empty() {
+ "nothing".to_string()
+ } else {
+ format!("{shown:?}")
+ };
+ Err(AgentError::TypeNotApplied {
+ reason: "value",
+ value: (!after.secret).then_some(shown),
+ message: format!(
+ "The field did not take the text: after typing it holds {holds}. The page may have reset it, or the field takes another kind of input (a mask, a length limit, a picker). Read it with perceive_page, then click the field and type again, or use press_key."
+ ),
+ })
+}
+
+/// Make sure the focus is in `field`, then select what it holds or park the
+/// caret at its end. Answers what it held.
+async fn focus_and_prepare(
+ session: &mut WayfernSession,
+ field: &str,
+ clear: bool,
+) -> Result {
+ let focus: FieldFocus = typing_step(session, field, "focus", true.into()).await?;
+ if !focus.focused {
+ return Err(AgentError::TypeNotApplied {
+ reason: "not_focused",
+ value: None,
+ message: format!(
+ "The field did not take the focus, so nothing was typed; the focus is on {}. Close what covers the field, or name the input itself.",
+ focus.active
+ ),
+ });
+ }
+ typing_step(session, field, "prepare", clear.into()).await
+}
+
+/// A trusted click at `point`, for the engines that type without Vellum.
+async fn dispatch_click(
+ session: &mut WayfernSession,
point: ViewportTarget,
+) -> Result<(), AgentError> {
+ session
+ .call(
+ "Input.dispatchMouseEvent",
+ serde_json::json!({ "type": "mouseMoved", "x": point.x, "y": point.y }),
+ )
+ .await?;
+ for kind in ["mousePressed", "mouseReleased"] {
+ session
+ .call(
+ "Input.dispatchMouseEvent",
+ serde_json::json!({
+ "type": kind, "x": point.x, "y": point.y, "button": "left", "clickCount": 1,
+ }),
+ )
+ .await?;
+ }
+ Ok(())
+}
+
+/// Type `text` into the field behind `element`, then read the field back.
+///
+/// With Vellum the field is struck where a hand would land and typed into by
+/// the browser's own keyboard. Otherwise it is focused, and clicked when that
+/// does not take. Either way the focus must be in the field before a key is
+/// sent, and a field that does not hold the text afterwards is a
+/// [`AgentError::TypeNotApplied`], never a success.
+async fn type_into(
+ session: &mut WayfernSession,
+ target: &CdpTarget,
+ element: &str,
+ backend_node_id: Option,
text: &str,
- typos: bool,
- preparation: FieldPreparation,
- timeout: Duration,
-) -> Result {
+ clear: bool,
+ method: TypingMethod<'_>,
+) -> Result {
+ let field = typing_field(session, element).await?;
+ let point = typing_point(session, &field, backend_node_id).await?;
+ let (before, inscription) = if let TypingMethod::Vellum { typos, timeout } = method {
+ let viewport = wayfern_cdp::layout_viewport(session).await?;
+ let point = point.map(|p| clamp_to_viewport(p, viewport));
+ let rest = point.unwrap_or(ViewportTarget {
+ x: viewport.0 / 2.0,
+ y: viewport.1 / 2.0,
+ width: viewport.0,
+ height: viewport.1,
+ });
+ let origin = wayfern_cdp::glide_origin(&rest, viewport);
+ // Owned by the gesture, for the reason `vellum_click_in` gives.
+ let (field, text) = (field.clone(), text.to_owned());
+ vellum::with_pointer(session, origin.0, origin.1, |s, p| {
+ Box::pin(async move {
+ if let Some(point) = point {
+ vellum::glide(s, p, point.x, point.y, Some(point.width.min(point.height))).await?;
+ vellum::strike(s, p, None, None).await?;
+ }
+ let before = focus_and_prepare(s, &field, clear).await?;
+ let inscription = vellum::inscribe(s, p, &text, typos, timeout).await?;
+ Ok::<_, AgentError>((before, Some(inscription)))
+ })
+ })
+ .await?
+ } else {
+ let focus: FieldFocus = typing_step(session, &field, "focus", true.into()).await?;
+ if let (false, Some(point)) = (focus.focused, point) {
+ dispatch_click(session, point).await?;
+ }
+ let before = focus_and_prepare(session, &field, clear).await?;
+ match method {
+ TypingMethod::Keystrokes(plan) => dispatch_keystrokes(target, plan).await?,
+ _ => {
+ session
+ .call("Input.insertText", serde_json::json!({ "text": text }))
+ .await?;
+ }
+ }
+ (before, None)
+ };
+ tokio::time::sleep(TYPING_SETTLE).await;
+ let after: FieldValue = typing_step(session, &field, "read", serde_json::Value::Null).await?;
+ let verified = typing_verdict(text, &before, &after, clear)?;
+ Ok(FieldTyping {
+ inscription,
+ verified,
+ })
+}
+
+/// Type into the element `element` names, on a fresh session: what typing
+/// did, and the locator's match when a locator named it.
+async fn type_element(
+ target: &CdpTarget,
+ engine: Engine,
+ element: &ElementRef,
+ text: &str,
+ clear: bool,
+ method: TypingMethod<'_>,
+) -> Result<(FieldTyping, Option), AgentError> {
let mut session = WayfernSession::open(target).await?;
- let outcome = vellum_type_in(&mut session, point, text, typos, preparation, timeout).await;
+ let outcome = async {
+ let (object, located) = form_element(&mut session, engine, element).await?;
+ let backend_node_id = located.as_ref().map(|l| l.backend_node_id);
+ let typed = type_into(
+ &mut session,
+ target,
+ &object,
+ backend_node_id,
+ text,
+ clear,
+ method,
+ )
+ .await?;
+ Ok::<_, AgentError>((typed, located))
+ }
+ .await;
+ let _ = session
+ .call(
+ "Runtime.releaseObjectGroup",
+ serde_json::json!({ "objectGroup": FORM_GROUP }),
+ )
+ .await;
session.close().await;
outcome
}
@@ -2149,6 +2704,37 @@ pub(crate) async fn agent_resolve_locator(
})
}
+/// Where to strike a located node: where a hand would click it, which for a
+/// control the page draws through its label (a transparent or collapsed radio,
+/// checkbox or file input) is the label, and never a spot where something
+/// else lies on top while a better one is in view. In a frame, or when the
+/// page sees no box, the DOM's quads decide.
+async fn click_point(
+ session: &mut WayfernSession,
+ backend_node_id: i64,
+) -> Result {
+ let node = session
+ .call(
+ "DOM.resolveNode",
+ serde_json::json!({ "backendNodeId": backend_node_id, "objectGroup": FORM_GROUP }),
+ )
+ .await?;
+ let aim = match node.pointer("/object/objectId").and_then(|v| v.as_str()) {
+ Some(object) => aim_at(session, object).await,
+ None => Ok(Aim::Frame),
+ };
+ let _ = session
+ .call(
+ "Runtime.releaseObjectGroup",
+ serde_json::json!({ "objectGroup": FORM_GROUP }),
+ )
+ .await;
+ match aim? {
+ Aim::At(point) | Aim::Covered(point, _) => Ok(point),
+ Aim::Hidden | Aim::Frame => Ok(wayfern_cdp::viewport_target(session, backend_node_id).await?),
+ }
+}
+
/// Resolve a locator and click it.
pub(crate) async fn agent_click_locator(
ctx: &AgentContext,
@@ -2163,7 +2749,7 @@ pub(crate) async fn agent_click_locator(
let outcome = async {
session.call("Page.enable", serde_json::json!({})).await?;
let located = wayfern_locate(&mut session, &locator, None).await?;
- let point = wayfern_cdp::viewport_target(&mut session, located.backend_node_id).await?;
+ let point = click_point(&mut session, located.backend_node_id).await?;
let (_, navigated) =
vellum_click_in(&mut session, point, button, request.click_count).await?;
Ok::<_, AgentError>((located, navigated))
@@ -2242,58 +2828,54 @@ pub(crate) async fn agent_type_locator(
let typos = request.typos.unwrap_or(true);
let locator = canonical_locator(&request.locator);
- if ctx.engine.is_wayfern() {
- let timeout = vellum_typing_budget(&request.text, max_seconds)?;
- let mut session = WayfernSession::open(&ctx.target).await?;
- let outcome = async {
- let located = wayfern_locate(&mut session, &locator, None).await?;
- let backend_node_id = located.backend_node_id;
- let point = wayfern_cdp::viewport_target(&mut session, backend_node_id).await?;
- let inscription = vellum_type_in(
- &mut session,
- point,
- &request.text,
- typos,
- FieldPreparation::Node {
- backend_node_id,
- clear: clear_first,
- },
- timeout,
- )
- .await?;
- Ok::<_, AgentError>((located, inscription))
- }
- .await;
- session.close().await;
- let (located, inscription) = outcome?;
- return Ok(AgentTyping {
- typed: true,
- characters: inscription.characters,
- corrections: Some(inscription.corrections),
- duration_ms: inscription.duration_ms,
- engine: Engine::Wayfern,
- matched: located.matched,
- });
- }
-
- // Planned first, before the field is touched, for the reason the selector
- // tools plan first: a refusal after the clear is a lie.
- let plan = plan_typing(&request.text, request.wpm, max_seconds)?;
- let (matched, _) = fallback_resolve(
+ // Budgeted and planned first, before the field is touched, for the reason
+ // the selector tools plan first: a refusal after the clear is a lie.
+ let wayfern = ctx.engine.is_wayfern();
+ let vellum_timeout = if wayfern {
+ Some(vellum_typing_budget(&request.text, max_seconds)?)
+ } else {
+ None
+ };
+ let plan = if wayfern {
+ None
+ } else {
+ Some(plan_typing(&request.text, request.wpm, max_seconds)?)
+ };
+ let (typed, located) = type_element(
&ctx.target,
- &locator,
- None,
- FallbackAct::Focus { clear_first },
+ ctx.engine,
+ &ElementRef::Locator(locator),
+ &request.text,
+ clear_first,
+ TypingMethod::chosen(vellum_timeout, plan.as_deref(), typos),
)
.await?;
- dispatch_keystrokes(&ctx.target, &plan).await?;
+ let matched = located
+ .map(|l| l.matched)
+ .ok_or_else(|| AgentError::Malformed("the locator settled on no element".into()))?;
+ let (characters, corrections, duration_ms) = match typed.inscription {
+ Some(inscription) => (
+ inscription.characters,
+ Some(inscription.corrections),
+ inscription.duration_ms,
+ ),
+ None => (
+ request.text.chars().count() as u64,
+ None,
+ plan
+ .as_deref()
+ .and_then(<[_]>::last)
+ .map_or(0.0, |event| event.time * 1000.0),
+ ),
+ };
Ok(AgentTyping {
typed: true,
- characters: request.text.chars().count() as u64,
- corrections: None,
- duration_ms: plan.last().map_or(0.0, |event| event.time * 1000.0),
- engine: Engine::Fallback,
+ characters,
+ corrections,
+ duration_ms,
+ engine: ctx.engine,
matched,
+ verified: typed.verified,
})
}
@@ -2346,13 +2928,13 @@ enum ElementRef {
/// tool is done.
const FORM_GROUP: &str = "donut-form";
-/// The page's handle to the element `element` names, and the locator's match
-/// when a locator named it.
+/// The page's handle to the element `element` names, and where the locator
+/// settled when a locator named it.
async fn form_element(
session: &mut WayfernSession,
engine: Engine,
element: &ElementRef,
-) -> Result<(String, Option), AgentError> {
+) -> Result<(String, Option), AgentError> {
let (expression, missing) = match element {
ElementRef::Locator(locator) => {
let located = if engine.is_wayfern() {
@@ -2373,7 +2955,7 @@ async fn form_element(
.pointer("/object/objectId")
.and_then(|v| v.as_str())
.ok_or_else(|| AgentError::Malformed("DOM.resolveNode answered no object".into()))?;
- return Ok((object.to_string(), Some(located.matched)));
+ return Ok((object.to_string(), Some(located)));
}
ElementRef::Selector(selector) => (
format!(
@@ -2524,7 +3106,8 @@ async fn agent_select_option(
let mut session = WayfernSession::open(&ctx.target).await?;
let outcome = async {
session.call("Page.enable", serde_json::json!({})).await?;
- let (object, matched) = form_element(&mut session, ctx.engine, element).await?;
+ let (object, located) = form_element(&mut session, ctx.engine, element).await?;
+ let matched = located.map(|l| l.matched);
let (reply, navigated) = session
.call_then_await_navigation(
"Runtime.callFunctionOn",
@@ -2674,7 +3257,8 @@ async fn agent_file_upload(
let profile_id = ctx.profile.id.to_string();
let mut session = WayfernSession::open(&ctx.target).await?;
let outcome = async {
- let (object, matched) = form_element(&mut session, ctx.engine, element).await?;
+ let (object, located) = form_element(&mut session, ctx.engine, element).await?;
+ let matched = located.map(|l| l.matched);
let reply = session
.call(
"Runtime.callFunctionOn",
@@ -2876,19 +3460,6 @@ async fn locate_by_script(target: &CdpTarget, script: String) -> Result FieldPreparation {
- if clear_first {
- FieldPreparation::None
- } else {
- FieldPreparation::Script(CARET_TO_END_JS.to_string())
- }
-}
-
fn click_report(prefix: &str, navigated: bool) -> String {
if navigated {
format!("{prefix} (a page load followed)")
@@ -3177,11 +3748,18 @@ fn wait_for_script(text: Option<&str>, selector: Option<&str>) -> Option
}
}
-fn typing_report(prefix: &str, inscription: &vellum::Inscription) -> String {
- format!(
- "{prefix} ({} characters, {} corrected, {:.0} ms)",
- inscription.characters, inscription.corrections, inscription.duration_ms
- )
+fn typing_report(prefix: &str, typed: &FieldTyping) -> String {
+ let mut report = prefix.to_string();
+ if let Some(inscription) = &typed.inscription {
+ report.push_str(&format!(
+ " ({} characters, {} corrected, {:.0} ms)",
+ inscription.characters, inscription.corrections, inscription.duration_ms
+ ));
+ }
+ if typed.verified {
+ report.push_str("; the field holds the text");
+ }
+ report
}
impl McpServer {
@@ -4948,7 +5526,7 @@ impl McpServer {
},
McpTool {
name: "type_text".to_string(),
- description: "Focus an element by CSS selector and type text into it. By default uses realistic human-like typing with variable speed, natural errors, and self-corrections. Only set instant=true when you are certain the target does not have bot detection (e.g. browser address bars, developer tools, internal apps) — using instant on public websites risks the profile being flagged as a bot.".to_string(),
+ description: "Focus an element by CSS selector and type text into it. By default uses realistic human-like typing with variable speed, natural errors, and self-corrections. A label, or a container with one field in it, types into that field. The field is read back after typing: when it does not hold the text (the page reset it, a mask or length limit refused it, or it never took the focus) the call fails with TYPE_NOT_APPLIED and says what the field holds. Only set instant=true when you are certain the target does not have bot detection (e.g. browser address bars, developer tools, internal apps) — using instant on public websites risks the profile being flagged as a bot.".to_string(),
input_schema: serde_json::json!({
"type": "object",
"properties": {
@@ -5066,7 +5644,7 @@ impl McpServer {
},
McpTool {
name: "type_by_index".to_string(),
- description: "Focus the element at the given index from the last get_interactive_elements call and type text into it. Same human-like-typing defaults as type_text; only set instant=true when you're sure the target lacks bot detection.".to_string(),
+ description: "Focus the element at the given index from the last get_interactive_elements call and type text into it. Same human-like-typing defaults and the same read-back as type_text (TYPE_NOT_APPLIED when the field does not hold the text); only set instant=true when you're sure the target lacks bot detection.".to_string(),
input_schema: serde_json::json!({
"type": "object",
"properties": {
@@ -5431,7 +6009,7 @@ impl McpServer {
},
McpTool {
name: "resolve_locator".to_string(),
- description: "Resolve a locator (role, accessible name, text, attributes) to EXACTLY ONE element and describe it. Ambiguity is an error whose data.candidates lists what matched, and a locator that matches nothing is an error too, so a click never lands on the wrong element. Use the result's signature or attributes to refine.".to_string(),
+ description: "Resolve a locator (role, accessible name, text, attributes) to EXACTLY ONE element and describe it. When several match, the one that stands out is taken: the only visible one, the one whose name or text is exactly a nameContains or textContains, or a control over its own label. Otherwise ambiguity is an error whose data.candidates lists what matched, and a locator that matches nothing is an error too (its message says when only hidden elements match), so a click never lands on the wrong element. Use the result's signature or attributes to refine.".to_string(),
input_schema: serde_json::json!({
"type": "object",
"properties": {
@@ -5450,7 +6028,7 @@ impl McpServer {
},
McpTool {
name: "click_locator".to_string(),
- description: "Resolve a locator to exactly one element and click it with a humanized pointer: on Wayfern 152 a real pointer glides to the element along a human path and presses with the profile's own timing (the click is never a script); on older builds a trusted mouse event is dispatched at the element's centre. Waits for a page load when the click causes one.".to_string(),
+ description: "Resolve a locator to exactly one element and click it with a humanized pointer: on Wayfern 152 a real pointer glides to the element along a human path and presses with the profile's own timing (the click is never a script); on older builds a trusted mouse event is dispatched at the element's centre. A radio, checkbox or file input that the page hides behind its label is clicked through the label, and text or name matches the label too. Waits for a page load when the click causes one.".to_string(),
input_schema: serde_json::json!({
"type": "object",
"properties": {
@@ -5474,7 +6052,7 @@ impl McpServer {
},
McpTool {
name: "type_locator".to_string(),
- description: "Resolve a locator to exactly one field, focus it with a real click and type text into it one key at a time. On Wayfern 152 the keys are paced by the profile's own typing rhythm, with a few seed-determined typos corrected along the way when typos is on; on older builds the same human-typing model as type_text is used. The field is emptied first unless clear_first is false.".to_string(),
+ description: "Resolve a locator to exactly one field, focus it with a real click and type text into it one key at a time. On Wayfern 152 the keys are paced by the profile's own typing rhythm, with a few seed-determined typos corrected along the way when typos is on; on older builds the same human-typing model as type_text is used. The typed text replaces what the field holds unless clear_first is false. The field is read back afterwards: a field that does not hold the text fails with TYPE_NOT_APPLIED (data.reason not_focused or value, data.value what it holds), never a success. An autocomplete keeps the typed text only until you pick an option.".to_string(),
input_schema: serde_json::json!({
"type": "object",
"properties": {
@@ -9667,14 +10245,6 @@ impl McpServer {
.map_err(cdp_error)
}
- async fn send_planned_keystrokes(
- &self,
- target: &CdpTarget,
- events: &[crate::human_typing::TypingEvent],
- ) -> Result<(), McpError> {
- dispatch_keystrokes(target, events).await.map_err(cdp_error)
- }
-
/// `expect_navigation`: the command is a navigation, so the page's load is
/// always awaited; otherwise only a navigation that starts is.
async fn send_cdp_and_wait_for_load(
@@ -10521,11 +11091,12 @@ impl McpServer {
// browser types through Vellum, which paces the keys itself, so the plan
// below is only built for the fallback engine.
let profile = self.get_wayfern_profile(profile_id)?;
- let humanized = Engine::for_version(&profile.version).is_wayfern() && !instant;
+ let engine = Engine::for_version(&profile.version);
+ let humanized = engine.is_wayfern() && !instant;
// PLANNED FIRST, before anything touches the page. The planner can refuse
- // (TYPING_TOO_LONG), and the focus step below empties the field when
- // `clear_first` is set — so planning after it meant the server answered
+ // (TYPING_TOO_LONG), and the typing below replaces what the field holds
+ // when `clear_first` is set — so planning after it meant the server answered
// "refused, nothing happened" having already wiped the customer's form
// field. The refusal now precedes every mutation on every branch,
// including the Vellum budget, which is decided in the same place.
@@ -10541,96 +11112,20 @@ impl McpServer {
};
let target = resolve_target(&profile).await?;
-
- let selector_escaped = selector.replace('\\', "\\\\").replace('\'', "\\'");
- let focus_js = if clear_first {
- format!(
- r#"(() => {{
- const el = document.querySelector('{}');
- if (!el) throw new Error('Element not found: {}');
- el.scrollIntoView({{block: 'center'}});
- el.focus();
- el.value = '';
- el.dispatchEvent(new Event('input', {{bubbles: true}}));
- {}
- }})()"#,
- selector_escaped, selector_escaped, RETURN_RECT_JS
- )
- } else {
- format!(
- r#"(() => {{
- const el = document.querySelector('{}');
- if (!el) throw new Error('Element not found: {}');
- el.scrollIntoView({{block: 'center'}});
- el.focus();
- {}
- }})()"#,
- selector_escaped, selector_escaped, RETURN_RECT_JS
- )
- };
-
- let focus_result = self
- .send_cdp(
- &target,
- "Runtime.evaluate",
- serde_json::json!({
- "expression": focus_js,
- "returnByValue": true,
- }),
- )
- .await?;
-
- if let Some(exception) = focus_result.get("exceptionDetails") {
- let msg = exception
- .get("exception")
- .and_then(|e| e.get("description"))
- .or_else(|| exception.get("text"))
- .and_then(|v| v.as_str())
- .unwrap_or("Focus failed");
- return Err(McpError {
- code: -32000,
- message: msg.to_string(),
- data: None,
- });
- }
-
- if let Some(timeout) = inscribe_timeout {
- let point = rect_from_script_result(&focus_result)?;
- let inscription = vellum_type(
- &target,
- point,
- text,
- typos,
- caret_preparation(clear_first),
- timeout,
- )
- .await
- .map_err(AgentError::into_mcp)?;
- return Ok(serde_json::json!({
- "content": [{
- "type": "text",
- "text": typing_report(&format!("Typed text into element: {selector}"), &inscription)
- }]
- }));
- }
-
- match &plan {
- Some(events) => self.send_planned_keystrokes(&target, events).await?,
- None => {
- self
- .send_cdp(
- &target,
- "Input.insertText",
- serde_json::json!({ "text": text }),
- )
- .await?;
- }
- }
-
+ let (typed, _) = type_element(
+ &target,
+ engine,
+ &ElementRef::Selector(selector.to_string()),
+ text,
+ clear_first,
+ TypingMethod::chosen(inscribe_timeout, plan.as_deref(), typos),
+ )
+ .await
+ .map_err(AgentError::into_mcp)?;
Ok(serde_json::json!({
"content": [{
"type": "text",
- "text": format!("Typed text into element: {selector}")
+ "text": typing_report(&format!("Typed text into element: {selector}"), &typed)
}]
}))
}
@@ -10996,11 +11491,12 @@ impl McpServer {
// Engine first, then budgets, then the page: see handle_type_text.
let profile = self.get_wayfern_profile(profile_id)?;
- let humanized = Engine::for_version(&profile.version).is_wayfern() && !instant;
+ let engine = Engine::for_version(&profile.version);
+ let humanized = engine.is_wayfern() && !instant;
// PLANNED FIRST, before anything touches the page. The planner can refuse
- // (TYPING_TOO_LONG), and the focus step below empties the field when
- // `clear_first` is set — so planning after it meant the server answered
+ // (TYPING_TOO_LONG), and the typing below replaces what the field holds
+ // when `clear_first` is set — so planning after it meant the server answered
// "refused, nothing happened" having already wiped the customer's form
// field. The refusal now precedes every mutation on every branch.
let plan = if instant || humanized {
@@ -11015,97 +11511,20 @@ impl McpServer {
};
let target = resolve_target(&profile).await?;
-
- // Mirrors handle_type_text's focus step but resolves the element via the
- // cached index instead of a CSS selector.
- let focus_js = if clear_first {
- format!(
- r#"(() => {{
- const arr = window[{cache}];
- if (!arr || !arr[{index}]) throw new Error('No element at index {index}. Call get_interactive_elements first or after navigation.');
- const el = arr[{index}];
- el.scrollIntoView({{block: 'center'}});
- el.focus();
- el.value = '';
- el.dispatchEvent(new Event('input', {{bubbles: true}}));
- {RETURN_RECT_JS}
- }})()"#
- )
- } else {
- format!(
- r#"(() => {{
- const arr = window[{cache}];
- if (!arr || !arr[{index}]) throw new Error('No element at index {index}. Call get_interactive_elements first or after navigation.');
- const el = arr[{index}];
- el.scrollIntoView({{block: 'center'}});
- el.focus();
- {RETURN_RECT_JS}
- }})()"#
- )
- };
-
- let focus_result = self
- .send_cdp(
- &target,
- "Runtime.evaluate",
- serde_json::json!({
- "expression": focus_js,
- "returnByValue": true,
- }),
- )
- .await?;
-
- if let Some(exception) = focus_result.get("exceptionDetails") {
- let msg = exception
- .get("exception")
- .and_then(|e| e.get("description"))
- .or_else(|| exception.get("text"))
- .and_then(|v| v.as_str())
- .unwrap_or("Focus failed");
- return Err(McpError {
- code: -32000,
- message: msg.to_string(),
- data: None,
- });
- }
-
- if let Some(timeout) = inscribe_timeout {
- let point = rect_from_script_result(&focus_result)?;
- let inscription = vellum_type(
- &target,
- point,
- text,
- typos,
- caret_preparation(clear_first),
- timeout,
- )
- .await
- .map_err(AgentError::into_mcp)?;
- return Ok(serde_json::json!({
- "content": [{
- "type": "text",
- "text": typing_report(&format!("Typed text into element at index {index}"), &inscription)
- }]
- }));
- }
-
- match &plan {
- Some(events) => self.send_planned_keystrokes(&target, events).await?,
- None => {
- self
- .send_cdp(
- &target,
- "Input.insertText",
- serde_json::json!({ "text": text }),
- )
- .await?;
- }
- }
-
+ let (typed, _) = type_element(
+ &target,
+ engine,
+ &ElementRef::Index { cache, index },
+ text,
+ clear_first,
+ TypingMethod::chosen(inscribe_timeout, plan.as_deref(), typos),
+ )
+ .await
+ .map_err(AgentError::into_mcp)?;
Ok(serde_json::json!({
"content": [{
"type": "text",
- "text": format!("Typed text into element at index {index}")
+ "text": typing_report(&format!("Typed text into element at index {index}"), &typed)
}]
}))
}
@@ -13387,8 +13806,8 @@ mod tests {
production.matches("window[__CACHE__]").count()
+ production.matches("window[{cache}]").count()
+ production.matches("window[{slot}]").count(),
- 9,
- "the nine cache sites (one write, six reads, two in the teardown) must \
+ 7,
+ "the seven cache sites (one write, four reads, two in the teardown) must \
all be parameterised"
);
}
@@ -13709,10 +14128,10 @@ mod tests {
.split_once("\n#[cfg(test)]")
.map_or(source, |(code, _)| code);
let sender = production
- .split("async fn send_planned_keystrokes(")
+ .split("async fn dispatch_keystrokes(")
.nth(1)
- .expect("send_planned_keystrokes must exist");
- let sender = &sender[..sender.find("\n }").unwrap_or(sender.len())];
+ .expect("dispatch_keystrokes must exist");
+ let sender = &sender[..sender.find("\n}").unwrap_or(sender.len())];
assert!(
!sender.contains("plan_typing"),
"the sender must take the accepted plan, not decide for itself, \
@@ -13728,11 +14147,10 @@ mod tests {
#[test]
fn a_refused_typing_request_has_not_already_emptied_the_field() {
// The ORDER, which the duration bound alone never pinned. Both typing
- // handlers used to focus the element and run `el.value = ''` first and call
+ // handlers used to focus the element and empty it first and call
// `plan_typing` only afterwards, so a TYPING_TOO_LONG was a lie: the server
// answered "refused" with the customer's form field already wiped. Nothing
- // may touch the page before the plan is accepted, on EITHER branch of
- // `clear_first`.
+ // may touch the page before the plan is accepted.
let production = include_str!("mcp_server.rs")
.split_once("\n#[cfg(test)]")
.map_or("", |(code, _)| code);
@@ -13753,29 +14171,14 @@ mod tests {
"{handler} must plan exactly once, before the page is touched"
);
- // Both branches of `clear_first` are present, and BOTH are built after
- // the plan. The clearing branch is the one that destroys data; the other
- // still focuses and scrolls, which is a mutation the caller can see.
- assert_eq!(
- body.matches("el.value = ''").count(),
- 1,
- "{handler} must still have exactly one clearing branch to order"
+ // Everything that touches the page comes after the plan, and the page is
+ // touched in one place: the shared typing path, which replaces a field's
+ // text by selecting it, never by assigning a value.
+ assert!(
+ !body.contains(".value = ''") && !body.contains("let focus_js"),
+ "{handler} must leave the field to type_element"
);
- assert_eq!(
- body.matches("el.focus();").count(),
- 2,
- "{handler} must have both a clearing and a non-clearing focus branch"
- );
-
- for mutation in [
- "el.value = ''",
- "el.focus();",
- "scrollIntoView",
- "let focus_js",
- ".send_cdp(",
- "send_planned_keystrokes",
- "Input.insertText",
- ] {
+ for mutation in ["resolve_target(", "type_element("] {
let at = body
.find(mutation)
.unwrap_or_else(|| panic!("{handler} no longer contains {mutation}"));
@@ -14465,7 +14868,7 @@ mod tests {
);
assert!(sent.iter().any(|m| m == "Wayfern.resolveLocator"));
assert!(sent.iter().any(|m| m == "DOM.describeNode"));
- assert!(sent.iter().any(|m| m == "DOM.getContentQuads"));
+ assert!(sent.iter().any(|m| m == "DOM.resolveNode"));
assert!(
!sent
.iter()
@@ -14512,6 +14915,23 @@ mod tests {
);
}
+ /// What the fake saw, with each step of the typing function named by its
+ /// step and every other frame by its method.
+ fn typing_steps(frames: &crate::wayfern_cdp::test_support::Frames) -> Vec {
+ frames
+ .lock()
+ .unwrap()
+ .iter()
+ .filter_map(|f| {
+ let method = f["method"].as_str()?;
+ match f["params"]["arguments"][0]["value"].as_str() {
+ Some(step) if method == "Runtime.callFunctionOn" => Some(step.to_string()),
+ _ => Some(method.to_string()),
+ }
+ })
+ .collect()
+ }
+
#[tokio::test]
async fn a_152_profile_types_through_inscribe_and_an_older_one_through_key_events() {
use crate::wayfern_cdp::test_support::{fake_browser, methods, Fake};
@@ -14527,30 +14947,31 @@ mod tests {
let ctx = agent_context_for("152.0.7977.64", target);
let typed = agent_type_locator(&ctx, &request, 300.0).await.unwrap();
assert!(typed.typed);
+ assert!(typed.verified, "the field read back holds the text");
assert_eq!(typed.engine, Engine::Wayfern);
assert_eq!(typed.characters, 2);
assert_eq!(typed.corrections, Some(1));
- let sent = methods(&frames);
- let native: Vec<&str> = sent
- .iter()
- .filter(|m| {
- m.starts_with("Vellum.") || m.starts_with("DOM.resolveNode") || m.starts_with("Runtime.")
- })
- .map(String::as_str)
+ let steps: Vec = typing_steps(&frames)
+ .into_iter()
+ .filter(|m| !m.contains('.') || m.starts_with("Vellum.") || m == "DOM.resolveNode")
.collect();
- // Focus by a real strike, THEN the field is emptied on the resolved node,
- // then the keys: clearing before the click would leave the caret wherever
- // the click landed in a field that is no longer empty.
+ // The field is struck where a hand would land, the focus is confirmed in it
+ // before a key is sent, its text is selected so the keys replace it, and it
+ // is read back once the page has answered.
assert_eq!(
- native,
+ steps,
vec![
+ "DOM.resolveNode",
+ "field",
+ "point",
"Vellum.acquire",
"Vellum.glide",
"Vellum.strike",
- "DOM.resolveNode",
- "Runtime.callFunctionOn",
+ "focus",
+ "prepare",
"Vellum.inscribe",
- "Vellum.release"
+ "Vellum.release",
+ "read",
]
);
{
@@ -14561,12 +14982,24 @@ mod tests {
.unwrap();
assert_eq!(inscribe["params"]["text"], "hi");
assert_eq!(inscribe["params"]["typos"], true);
- let clear = sent_frames
+ let glide = sent_frames
.iter()
- .find(|f| f["method"] == "Runtime.callFunctionOn")
+ .find(|f| f["method"] == "Vellum.glide")
.unwrap();
- assert_eq!(clear["params"]["arguments"][0]["value"], true);
- assert_eq!(clear["params"]["objectId"], "obj-7");
+ assert_eq!(glide["params"]["x"], 140.0);
+ let prepare = sent_frames
+ .iter()
+ .find(|f| f["params"]["arguments"][0]["value"] == "prepare")
+ .unwrap();
+ assert_eq!(prepare["params"]["arguments"][1]["value"], true);
+ assert_eq!(prepare["params"]["objectId"], "field-7");
+ assert!(
+ !prepare["params"]["functionDeclaration"]
+ .as_str()
+ .unwrap()
+ .contains(".value ="),
+ "a field is never emptied by assigning its value"
+ );
}
let (target, frames) = fake_browser(Fake::Cooperative).await;
@@ -14575,6 +15008,7 @@ mod tests {
assert_eq!(typed.engine, Engine::Fallback);
assert_eq!(typed.characters, 2);
assert_eq!(typed.corrections, None);
+ assert!(typed.verified);
let sent = methods(&frames);
assert_eq!(sent.first().map(String::as_str), Some("Runtime.evaluate"));
assert!(
@@ -14586,6 +15020,320 @@ mod tests {
"two characters are at least two key downs and two key ups: {sent:?}"
);
assert!(!sent.iter().any(|m| m.starts_with("Vellum.")));
+ // Focused by the page before the keys, so no click was needed.
+ assert!(!sent.iter().any(|m| m == "Input.dispatchMouseEvent"));
+ }
+
+ #[tokio::test]
+ async fn typing_the_page_does_not_keep_is_refused_rather_than_reported() {
+ use crate::wayfern_cdp::test_support::{fake_browser, methods, Fake};
+
+ let request: AgentTypeRequest = serde_json::from_value(serde_json::json!({
+ "locator": { "role": "textbox", "name": "Email" },
+ "text": "hi@example.com"
+ }))
+ .unwrap();
+
+ // Typed, then reset by the page: the field reads back empty.
+ for version in ["152.0.7977.64", "151.0.7922.76"] {
+ let (target, _) = fake_browser(Fake::TypingLost).await;
+ let ctx = agent_context_for(version, target);
+ let refused = agent_type_locator(&ctx, &request, 300.0)
+ .await
+ .expect_err("a field that holds nothing did not take the text");
+ let error = refused.into_mcp();
+ assert_eq!(error.code, -32000);
+ let data = error.data.expect("the verdict travels in data");
+ assert_eq!(data["code"], "TYPE_NOT_APPLIED", "{version}");
+ assert_eq!(data["reason"], "value");
+ assert_eq!(data["value"], "");
+ assert!(error.message.contains("holds nothing"), "{}", error.message);
+ }
+
+ // A field that never takes the focus gets no keys at all: they would land
+ // in whatever has it. The older engine clicks it before it gives up.
+ let (target, frames) = fake_browser(Fake::FieldUnfocusable).await;
+ let ctx = agent_context_for("152.0.7977.64", target);
+ let refused = agent_type_locator(&ctx, &request, 300.0)
+ .await
+ .expect_err("no focus, no typing");
+ assert!(
+ matches!(
+ &refused,
+ AgentError::TypeNotApplied {
+ reason: "not_focused",
+ ..
+ }
+ ),
+ "{refused:?}"
+ );
+ let sent = methods(&frames);
+ assert!(sent.iter().any(|m| m == "Vellum.strike"));
+ assert!(!sent.iter().any(|m| m == "Vellum.inscribe"), "{sent:?}");
+ assert_eq!(
+ sent.last().map(String::as_str),
+ Some("Runtime.releaseObjectGroup")
+ );
+
+ let (target, frames) = fake_browser(Fake::FieldUnfocusable).await;
+ let ctx = agent_context_for("151.0.7922.76", target);
+ let refused = agent_type_locator(&ctx, &request, 300.0)
+ .await
+ .expect_err("no focus, no typing");
+ assert_eq!(refused.code(), "TYPE_NOT_APPLIED");
+ let sent = methods(&frames);
+ assert_eq!(
+ sent
+ .iter()
+ .filter(|m| *m == "Input.dispatchMouseEvent")
+ .count(),
+ 3,
+ "move, press and release before giving up: {sent:?}"
+ );
+ assert!(
+ !sent.iter().any(|m| m == "Input.dispatchKeyEvent"),
+ "{sent:?}"
+ );
+ }
+
+ #[tokio::test]
+ async fn a_click_lands_where_the_page_aims_and_the_dom_decides_in_a_frame() {
+ use crate::wayfern_cdp::test_support::{fake_browser, methods, Fake};
+
+ let request: AgentClickRequest = serde_json::from_value(serde_json::json!({
+ "locator": { "role": "radio", "name": "Female" }
+ }))
+ .unwrap();
+ // The page names the point (a label, an unobstructed spot); no quads.
+ let (target, frames) = fake_browser(Fake::Cooperative).await;
+ let ctx = agent_context_for("154.0.8037.57", target);
+ agent_click_locator(&ctx, &request).await.unwrap();
+ let steps = typing_steps(&frames);
+ let point = steps.iter().position(|m| m == "point").unwrap();
+ let strike = steps.iter().position(|m| m == "Vellum.strike").unwrap();
+ assert!(point < strike, "{steps:?}");
+ assert!(
+ !steps.iter().any(|m| m == "DOM.getContentQuads"),
+ "{steps:?}"
+ );
+
+ // In a frame the page's own coordinates are not the viewport's.
+ let (target, frames) = fake_browser(Fake::FramedNode).await;
+ let ctx = agent_context_for("154.0.8037.57", target);
+ agent_click_locator(&ctx, &request).await.unwrap();
+ assert_eq!(node_ids_sent(&frames, "DOM.getContentQuads"), vec![7]);
+
+ // Nothing to click: refused as the caller's problem, not a broken reply.
+ let (target, frames) = fake_browser(Fake::HiddenControl).await;
+ let ctx = agent_context_for("154.0.8037.57", target);
+ let refused = agent_click_locator(&ctx, &request)
+ .await
+ .expect_err("no box, no click");
+ let error = refused.into_mcp();
+ assert_eq!(error.data.as_ref().unwrap()["code"], "BAD_REQUEST");
+ assert!(
+ error.message.contains("no visible box"),
+ "{}",
+ error.message
+ );
+ assert!(!methods(&frames).iter().any(|m| m == "Vellum.strike"));
+
+ let typing: AgentTypeRequest = serde_json::from_value(serde_json::json!({
+ "locator": { "role": "textbox", "name": "Email" },
+ "text": "hi"
+ }))
+ .unwrap();
+ let (target, frames) = fake_browser(Fake::HiddenControl).await;
+ let ctx = agent_context_for("154.0.8037.57", target);
+ let refused = agent_type_locator(&ctx, &typing, 300.0)
+ .await
+ .expect_err("no box, no typing");
+ assert_eq!(refused.code(), "BAD_REQUEST");
+ assert!(!methods(&frames).iter().any(|m| m.starts_with("Vellum.")));
+ }
+
+ #[test]
+ fn a_field_holds_the_typed_text_through_masks_and_appends() {
+ // Cleared first: the value is the text, whatever a mask added.
+ assert!(field_took("Jane", "", "Jane", true, false));
+ assert!(field_took("5551234567", "", "(555) 123-4567", true, false));
+ assert!(field_took(
+ "5551234567",
+ "",
+ "+1 (555) 123-4567",
+ true,
+ false
+ ));
+ assert!(field_took("ab12cd", "", "AB12CD", true, false));
+ assert!(field_took("New Y", "", "New York", true, false));
+ assert!(field_took(
+ "line one\nline two",
+ "",
+ "line one\r\nline two",
+ true,
+ false
+ ));
+ // Nothing kept, or only part of it.
+ assert!(!field_took("Jane", "", "", true, false));
+ assert!(!field_took("abcdef", "", "abc", true, false));
+ assert!(!field_took("Jane", "Old", "Old", true, false));
+ // A field that already held the text, emptied and typed again.
+ assert!(field_took("Jane", "Jane", "Jane", true, false));
+ // Appended: only what came after the old text counts.
+ assert!(field_took(" Doe", "Jane", "Jane Doe", false, false));
+ assert!(field_took("hello", "hello", "hellohello", false, false));
+ assert!(!field_took("hello", "hello", "hello", false, false));
+ assert!(!field_took("Doe", "Jane Doe", "Jane Doe", false, false));
+ // One box per character: the first box holds the start.
+ assert!(field_took("123456", "", "1", true, true));
+ assert!(!field_took("123456", "", "9", true, true));
+ assert!(!field_took("123456", "", "1", true, false));
+ // Nothing to tell a mask from: no letters or digits.
+ assert!(field_took("--", "", "", true, false));
+ }
+
+ #[test]
+ fn a_field_that_is_not_text_is_not_judged_by_its_value() {
+ let read = |kind: &str, field_type: &str, value: &str| FieldValue {
+ kind: kind.to_string(),
+ field_type: field_type.to_string(),
+ value: value.to_string(),
+ ..Default::default()
+ };
+ let empty = read("value", "text", "");
+ assert!(typing_verdict("Jane", &empty, &read("value", "text", "Jane"), true).unwrap());
+ assert!(typing_verdict("Jane", &empty, &read("editable", "div", "Jane\n"), true).unwrap());
+ // A date field shows its own format, and an element that is no field
+ // shows nothing to compare.
+ assert!(!typing_verdict(
+ "2026-10-19",
+ &empty,
+ &read("value", "date", "2026-10-19"),
+ true
+ )
+ .unwrap());
+ assert!(!typing_verdict("Jane", &empty, &read("none", "", ""), true).unwrap());
+ // A refusal names what the field holds, except a secret.
+ let refused = typing_verdict("Jane", &empty, &read("value", "text", "J"), true).unwrap_err();
+ assert!(
+ refused.message().contains("holds \"J\""),
+ "{}",
+ refused.message()
+ );
+ assert_eq!(refused.detail()["value"], "J");
+ let secret = FieldValue {
+ secret: true,
+ ..read("value", "password", "abc")
+ };
+ let refused = typing_verdict("hunter22", &empty, &secret, true).unwrap_err();
+ assert!(
+ refused.message().contains("3 characters"),
+ "{}",
+ refused.message()
+ );
+ assert!(!refused.message().contains("abc"));
+ assert!(refused.detail().get("value").is_none());
+ }
+
+ #[test]
+ fn the_typing_method_is_the_one_the_budgets_chose() {
+ let plan = [crate::human_typing::TypingEvent {
+ time: 0.1,
+ action: crate::human_typing::TypingAction::Char('a'),
+ }];
+ let timeout = Duration::from_secs(5);
+ assert!(matches!(
+ TypingMethod::chosen(Some(timeout), None, false),
+ TypingMethod::Vellum { typos: false, .. }
+ ));
+ assert!(matches!(
+ TypingMethod::chosen(None, Some(&plan), true),
+ TypingMethod::Keystrokes(events) if events.len() == 1
+ ));
+ assert!(matches!(
+ TypingMethod::chosen(None, None, true),
+ TypingMethod::Insert
+ ));
+ }
+
+ #[test]
+ fn an_ambiguous_match_settles_on_the_candidate_that_stands_out() {
+ let candidate = |name: &str, role: &str, width: f64| {
+ serde_json::json!({
+ "role": role, "name": name, "text": "", "signature": name,
+ "bounds": { "x": 0.0, "y": 0.0, "width": width, "height": 20.0 }
+ })
+ };
+ let locator =
+ |json: serde_json::Value| -> LocatorDescription { serde_json::from_value(json).unwrap() };
+
+ // The exact name over the longer one that contains it.
+ let first_names = [
+ candidate("First Name (optional)", "textField", 200.0),
+ candidate("First Name", "textField", 200.0),
+ ];
+ let contains = locator(serde_json::json!({ "nameContains": "First Name" }));
+ assert_eq!(preferred_candidate(&contains, 2, &first_names), Some(1));
+ let any_case = locator(serde_json::json!({ "nameContains": "first name" }));
+ assert_eq!(preferred_candidate(&any_case, 2, &first_names), Some(1));
+ // The exact form already chose: both carry it, nothing stands out.
+ let exact = locator(serde_json::json!({ "name": "First Name" }));
+ let twins = [
+ candidate("First Name", "textField", 200.0),
+ candidate("First Name", "textField", 200.0),
+ ];
+ assert_eq!(preferred_candidate(&exact, 2, &twins), None);
+ // The visible one over the one with no box.
+ let shown = [
+ candidate("Save", "button", 0.0),
+ candidate("Save", "button", 80.0),
+ ];
+ assert_eq!(preferred_candidate(&exact, 2, &shown), Some(1));
+ // The control over the label, and the label's text, that name it.
+ let labelled = [
+ candidate("Female", "labelText", 60.0),
+ candidate("Female", "staticText", 50.0),
+ candidate("Female", "radioButton", 16.0),
+ ];
+ assert_eq!(preferred_candidate(&exact, 3, &labelled), Some(2));
+ // A label and the text inside it: the text, innermost.
+ let mut text = candidate("Female", "staticText", 40.0);
+ text["bounds"]["x"] = serde_json::json!(10.0);
+ text["bounds"]["height"] = serde_json::json!(16.0);
+ let nested = [candidate("Female", "labelText", 60.0), text];
+ assert_eq!(preferred_candidate(&exact, 2, &nested), Some(1));
+ // A list the resolver cut short is not ranked: the best may be missing.
+ assert_eq!(preferred_candidate(&contains, 12, &first_names), None);
+ assert_eq!(preferred_candidate(&contains, 1, &first_names[..1]), None);
+
+ // The refusal lists what matched, up to the caller's limit.
+ let many: Vec = (0..30)
+ .map(|i| candidate(&format!("Row {i}"), "row", 10.0))
+ .collect();
+ let refused = ambiguity(30, many, Some(5));
+ let detail = refused.detail();
+ assert_eq!(detail["matchCount"], 30);
+ assert_eq!(detail["candidates"].as_array().map(Vec::len), Some(5));
+ assert!(refused
+ .message()
+ .starts_with("Ambiguous locator: 30 nodes match"));
+ assert!(refused.message().contains("Row 4") && !refused.message().contains("Row 5\""));
+ }
+
+ #[tokio::test]
+ async fn a_native_ambiguity_with_one_exact_name_is_not_refused() {
+ use crate::wayfern_cdp::test_support::{fake_browser, Fake};
+
+ let request: AgentResolveRequest = serde_json::from_value(serde_json::json!({
+ "locator": { "nameContains": "First Name" }
+ }))
+ .unwrap();
+ let (target, _) = fake_browser(Fake::AmbiguousByName).await;
+ let ctx = agent_context_for("154.0.8037.57", target);
+ let resolved = agent_resolve_locator(&ctx, &request).await.unwrap();
+ assert_eq!(resolved.matched.name, "First Name");
+ assert_eq!(resolved.matched.signature, "s8");
+ assert_eq!(resolved.engine, Engine::Wayfern);
}
#[tokio::test]
@@ -14765,7 +15513,7 @@ mod tests {
let ctx = agent_context_for("154.0.8037.57", target);
let clicked = agent_click_locator(&ctx, &request).await.unwrap();
assert_eq!(clicked.matched.backend_node_id, Some(353));
- assert_eq!(node_ids_sent(&frames, "DOM.getContentQuads"), vec![353]);
+ assert_eq!(node_ids_sent(&frames, "DOM.resolveNode"), vec![353]);
let (target, frames) = fake_browser(Fake::RenumberedNodeIds).await;
let ctx = agent_context_for("154.0.8037.57", target);
@@ -14773,11 +15521,7 @@ mod tests {
assert_eq!(clicked.engine, Engine::Wayfern);
assert_eq!(clicked.matched.signature, "s7");
assert_eq!(clicked.matched.backend_node_id, Some(353));
- assert_eq!(node_ids_sent(&frames, "DOM.getContentQuads"), vec![353]);
- assert_eq!(
- node_ids_sent(&frames, "DOM.scrollIntoViewIfNeeded"),
- vec![353]
- );
+ assert_eq!(node_ids_sent(&frames, "DOM.resolveNode"), vec![353]);
{
let sent = frames.lock().unwrap();
let pin = sent
@@ -14840,7 +15584,7 @@ mod tests {
let ctx = agent_context_for("154.0.8037.57", target);
let clicked = agent_click_locator(&ctx, &click).await.unwrap();
assert_eq!(clicked.engine, Engine::Wayfern);
- assert_eq!(node_ids_sent(&frames, "DOM.getContentQuads"), vec![353]);
+ assert_eq!(node_ids_sent(&frames, "DOM.resolveNode"), vec![353]);
// Nothing in the page either: the refusal is the usual one.
let nowhere: AgentResolveRequest = serde_json::from_value(serde_json::json!({
@@ -14952,11 +15696,10 @@ mod tests {
#[test]
fn the_typing_tools_plan_only_for_the_engine_that_needs_a_plan() {
- // The selector and index typing tools keep their whole pre-152 body, and
- // the source-scanning test above pins its order. What is pinned here is
- // that the 152 branch is decided from the PROFILE'S version, read at the
- // point of use, and that the Vellum budget is decided in the same place as
- // the plan, before the focus script.
+ // The source-scanning test above pins the order of the selector and index
+ // typing tools. What is pinned here is that the 152 branch is decided from
+ // the PROFILE'S version, read at the point of use, and that the Vellum
+ // budget is decided in the same place as the plan, before the page.
let production = include_str!("mcp_server.rs")
.split_once("\n#[cfg(test)]")
.map_or("", |(code, _)| code);
@@ -14976,18 +15719,16 @@ mod tests {
body.matches("MAX_BRIDGE_TYPING_SECONDS").count() >= 2,
"{handler} must budget both engines on the MCP budget"
);
- let focus = body.find("let focus_js").unwrap();
+ let page = body.find("type_element(").unwrap();
assert!(
- engine < budget && budget < focus,
+ engine < budget && budget < page,
"{handler}: engine, budget, then the page"
);
+ // Vellum when budgeted, the plan when made, otherwise the instant path,
+ // which every engine keeps.
assert!(
- body.contains("vellum_type("),
- "{handler} must type through Vellum on a 152 profile"
- );
- assert!(
- body.contains("Input.insertText"),
- "{handler} must keep the instant path for every engine"
+ body.contains("TypingMethod::chosen(inscribe_timeout, plan.as_deref(), typos)"),
+ "{handler} must type through the method its budgets chose"
);
}
for handler in ["handle_click_element", "handle_click_by_index"] {
diff --git a/src-tauri/src/wayfern_cdp.rs b/src-tauri/src/wayfern_cdp.rs
index 70366c81..d3b89a6f 100644
--- a/src-tauri/src/wayfern_cdp.rs
+++ b/src-tauri/src/wayfern_cdp.rs
@@ -110,6 +110,8 @@ pub enum WayfernError {
PickerTimedOut { timeout_ms: u64 },
/// The picker ended without a pick: "escape", "stopped" or "navigated".
PickerCancelled { reason: String },
+ /// The node has no box on screen to act on.
+ NotVisible(String),
/// A reply did not have the shape the protocol documents.
Malformed(String),
}
@@ -123,6 +125,7 @@ impl std::fmt::Display for WayfernError {
write!(f, "no element was picked within {timeout_ms} ms")
}
Self::PickerCancelled { reason } => write!(f, "the element picker was cancelled ({reason})"),
+ Self::NotVisible(m) => f.write_str(m),
Self::Malformed(m) => write!(f, "unexpected reply from the browser: {m}"),
}
}
@@ -1037,8 +1040,9 @@ pub async fn viewport_target(
.filter_map(|quad| quad_target(quad, viewport_width, viewport_height))
.next()
.ok_or_else(|| {
- WayfernError::Malformed(
- "the node has no visible box to strike; it may be hidden or clipped".into(),
+ WayfernError::NotVisible(
+ "The element has no visible box to click or type into; it may be hidden or clipped. Open, expand or scroll to the part of the page that shows it."
+ .into(),
)
})
}
@@ -1383,6 +1387,16 @@ pub(crate) mod test_support {
NativeNoMatch,
/// Every perception capture reports it ran past the default budget.
SlowPerception,
+ /// The page resets the field after typing: it reads back empty.
+ TypingLost,
+ /// The field never takes the focus.
+ FieldUnfocusable,
+ /// The node has no box: the page aims at nothing and the DOM has no quads.
+ HiddenControl,
+ /// The node is in a frame: the page leaves its position to the DOM.
+ FramedNode,
+ /// `resolveLocator` matches "First Name (optional)" and "First Name".
+ AmbiguousByName,
}
pub(crate) type Frames = Arc>>;
@@ -1448,10 +1462,14 @@ pub(crate) mod test_support {
let port = listener.local_addr().expect("a bound port").port();
let frames: Frames = Arc::new(Mutex::new(Vec::new()));
let recorded = frames.clone();
+ // What the keys put in the field, across every connection: key events
+ // arrive on a socket of their own.
+ let typed: Arc> = Arc::new(Mutex::new(String::new()));
tokio::spawn(async move {
while let Ok((socket, _)) = listener.accept().await {
let recorded = recorded.clone();
+ let typed = typed.clone();
tokio::spawn(async move {
let Ok(mut stream) = tokio_tungstenite::accept_async(socket).await else {
return;
@@ -1489,6 +1507,7 @@ pub(crate) mod test_support {
}
"Vellum.inscribe" => {
let text = params.get("text").and_then(Value::as_str).unwrap_or("");
+ typed.lock().unwrap().push_str(text);
ok(serde_json::json!({
"characters": text.chars().count(), "corrections": 1, "durationMs": 900.0
}))
@@ -1515,13 +1534,30 @@ pub(crate) mod test_support {
};
ok(serde_json::json!({}))
}
+ "Input.insertText" => {
+ let text = params.get("text").and_then(Value::as_str).unwrap_or("");
+ typed.lock().unwrap().push_str(text);
+ ok(serde_json::json!({}))
+ }
+ "Input.dispatchKeyEvent" => {
+ if params["type"] == "keyDown" {
+ let mut field = typed.lock().unwrap();
+ match params.get("text").and_then(Value::as_str) {
+ Some(text) => field.push_str(text),
+ None if params["key"] == "Backspace" => {
+ field.pop();
+ }
+ None => {}
+ }
+ }
+ ok(serde_json::json!({}))
+ }
"Vellum.release"
| "Page.enable"
| "Page.disable"
| "Wayfern.stopElementPicker"
| "DOM.scrollIntoViewIfNeeded"
- | "Input.dispatchMouseEvent"
- | "Input.dispatchKeyEvent" => ok(serde_json::json!({})),
+ | "Input.dispatchMouseEvent" => ok(serde_json::json!({})),
"Wayfern.capturePagePerception" => {
perception_page += 1;
let endless = behaviour == Fake::EndlessPerception;
@@ -1550,6 +1586,19 @@ pub(crate) mod test_support {
"Wayfern.resolveLocator" if behaviour == Fake::AmbiguousLocator => error(
"Ambiguous locator: 2 nodes match. Refine it with a role, a stable attribute, or more exact text. Candidates: [{\"backendNodeId\":7,\"role\":\"button\",\"name\":\"Save\",\"text\":\"Save\",\"signature\":\"s7\",\"attributes\":[]},{\"backendNodeId\":8,\"role\":\"button\",\"name\":\"Save\",\"text\":\"Save\",\"signature\":\"s8\",\"attributes\":[]}]",
),
+ "Wayfern.resolveLocator" if behaviour == Fake::AmbiguousByName => {
+ let named = |name: &str, signature: &str| {
+ let mut candidate = canned_candidate();
+ candidate["role"] = Value::from("textField");
+ candidate["name"] = Value::from(name);
+ candidate["signature"] = Value::from(signature);
+ candidate
+ };
+ error(&format!(
+ "Ambiguous locator: 2 nodes match. Refine it with a role, a stable attribute, or more exact text. Candidates: {}",
+ Value::Array(vec![named("First Name (optional)", "s7"), named("First Name", "s8")])
+ ))
+ }
"Wayfern.resolveLocator" if behaviour == Fake::NativeNoMatch => {
error("No node matches locator (id=save).")
}
@@ -1563,14 +1612,48 @@ pub(crate) mod test_support {
"rowCount": 1, "pageCount": 1, "byteSize": 40, "truncated": false,
"stopReason": "complete"
})),
+ "DOM.getContentQuads" if behaviour == Fake::HiddenControl => {
+ ok(serde_json::json!({ "quads": [] }))
+ }
"DOM.getContentQuads" => ok(serde_json::json!({
"quads": [[100.0, 200.0, 180.0, 200.0, 180.0, 230.0, 100.0, 230.0]]
})),
"DOM.resolveNode" => ok(serde_json::json!({
"object": { "type": "object", "objectId": "obj-7" }
})),
+ "Runtime.callFunctionOn" if params["arguments"][0]["value"] == "field" => {
+ ok(serde_json::json!({
+ "result": { "type": "object", "subtype": "node", "objectId": "field-7" }
+ }))
+ }
"Runtime.callFunctionOn" => {
- ok(serde_json::json!({ "result": { "type": "boolean", "value": true } }))
+ let step = params["arguments"][0]["value"].as_str().unwrap_or_default();
+ let value = match step {
+ "point" if behaviour == Fake::HiddenControl => serde_json::json!({ "hidden": true }),
+ "point" if behaviour == Fake::FramedNode => serde_json::json!({ "frame": true }),
+ "point" => {
+ serde_json::json!({ "x": 140.0, "y": 215.0, "width": 80.0, "height": 30.0 })
+ }
+ "focus" => serde_json::json!({
+ "focused": behaviour != Fake::FieldUnfocusable, "active": "body"
+ }),
+ "prepare" => {
+ serde_json::json!({ "kind": "value", "type": "text", "value": "", "secret": false })
+ }
+ "read" => {
+ let value = if behaviour == Fake::TypingLost {
+ String::new()
+ } else {
+ typed.lock().unwrap().clone()
+ };
+ serde_json::json!({
+ "kind": "value", "type": "text", "value": value, "secret": false, "focusMoved": false
+ })
+ }
+ _ => Value::Bool(true),
+ };
+ let kind = if value.is_boolean() { "boolean" } else { "object" };
+ ok(serde_json::json!({ "result": { "type": kind, "value": value } }))
}
"Page.getLayoutMetrics" => ok(serde_json::json!({
"cssLayoutViewport": { "clientWidth": 1280.0, "clientHeight": 720.0 }