mirror of
https://github.com/zhom/donutbrowser.git
synced 2026-11-14 08:06:58 +01:00
fix: remote dialogs and rest
This commit is contained in:
1 parent
901612c3f6
commit
4fd2d3d3be
18 files changed
+2118
-173
No files matched your search
@@ -65,7 +65,7 @@ donutbrowser/
|
||||
│ │ ├── mcp_toolsets.rs # Optional MCP toolsets: which tools a client lists (core + opt-in sets)
|
||||
│ │ ├── mcp_views.rs # Compact MCP views: profile summaries, filters, paging, size caps
|
||||
│ │ ├── navigation_watch.rs # Whether a CDP action navigated, so a click waits only for a load that comes
|
||||
│ │ ├── page_dialog.rs # Native dialogs on agent-driven pages: one Page-enabled socket per page, DIALOG_OPEN fail fast, answers
|
||||
│ │ ├── page_dialog.rs # Native dialogs on agent-driven pages: a Page-enabled socket per local page, one relay socket per cloud session that pins the page its tools drive, DIALOG_OPEN fail fast, answers
|
||||
│ │ ├── network_capture.rs # MCP network capture: one browser-level socket per profile, in memory, credentials redacted
|
||||
│ │ ├── agent_uploads.rs # Files an agent sends to file_upload, kept until the browser stops
|
||||
│ │ ├── mcp_remote.rs # Remote MCP bridge: outbound websocket to Donut cloud (Enterprise remote control)
|
||||
@@ -174,6 +174,7 @@ suite passes:
|
||||
| Sync client/server, manifests, timestamps, deletion, encryption, password rollover | `pnpm e2e:sync` |
|
||||
| Wayfern download/terms/fingerprint, browser runner, CDP, automation endpoints, process cleanup | `pnpm e2e:browser` |
|
||||
| `donut-sync/` server code (routes, auth, S3 client, presigning, subscriptions) | `cd donut-sync && cargo test` against the compose S3 server |
|
||||
| Cloud (remote) sessions: relay targets, the page a session's tools drive, dialogs and typing there | `pnpm test:fleet` with `DONUT_FLEET_TEST_KEY` (a `dsk_` key of a service account); it starts a real cloud session, so it is not in `pnpm e2e` |
|
||||
| E2E harness, WebDriver plugin/driver, app isolation hooks, or changes spanning multiple rows | Run every affected row; use `pnpm e2e` for cross-cutting changes |
|
||||
|
||||
`e2e:browser` requires `WAYFERN_TEST_TOKEN` in the environment or local `.env`. `e2e:network`
|
||||
|
||||
@@ -1957,6 +1957,55 @@ test("MCP answers native dialogs and types into a background tab", async (t) =>
|
||||
await tool("handle_dialog", { accept: true, prompt_text: "Bob" });
|
||||
assert.equal(await cdp.evaluate("document.body.dataset.renamed"), "Bob");
|
||||
|
||||
// The same over REST: reading is never refused, every other agent call is
|
||||
// refused at once, and the answer reports what it answered.
|
||||
const agentUrl = (action) =>
|
||||
`${base}/v1/profiles/${profile.id}/agent/${action}`;
|
||||
await tool("click_locator", {
|
||||
locator: { role: "button", name: "Delete" },
|
||||
});
|
||||
const restRead = await request(agentUrl("dialog"), {
|
||||
token: saved.api_token,
|
||||
});
|
||||
assert.equal(restRead.response.status, 200, JSON.stringify(restRead.value));
|
||||
assert.equal(restRead.value.open, true);
|
||||
assert.equal(restRead.value.dialog.type, "confirm");
|
||||
const restRefused = await timed(() =>
|
||||
request(agentUrl("type"), {
|
||||
method: "POST",
|
||||
token: saved.api_token,
|
||||
body: { locator: { role: "textbox", name: "Later" }, text: "x" },
|
||||
}),
|
||||
);
|
||||
assert.equal(
|
||||
restRefused.value.response.status,
|
||||
409,
|
||||
JSON.stringify(restRefused.value.value),
|
||||
);
|
||||
assert.equal(restRefused.value.value.code, "DIALOG_OPEN");
|
||||
assert.equal(restRefused.value.value.dialog.message, "Delete it?");
|
||||
assert.ok(restRefused.ms < 5_000, `REST type waited ${restRefused.ms} ms`);
|
||||
const restAnswered = await request(agentUrl("handle-dialog"), {
|
||||
method: "POST",
|
||||
token: saved.api_token,
|
||||
body: { accept: true },
|
||||
});
|
||||
assert.equal(
|
||||
restAnswered.response.status,
|
||||
200,
|
||||
JSON.stringify(restAnswered.value),
|
||||
);
|
||||
assert.equal(restAnswered.value.accepted, true);
|
||||
assert.equal(restAnswered.value.dialog.message, "Delete it?");
|
||||
assert.equal(await cdp.evaluate("document.body.dataset.deleted"), "true");
|
||||
const restNone = await request(agentUrl("handle-dialog"), {
|
||||
method: "POST",
|
||||
token: saved.api_token,
|
||||
body: { accept: true },
|
||||
});
|
||||
assert.equal(restNone.response.status, 409);
|
||||
assert.equal(restNone.value.code, "NO_DIALOG_OPEN");
|
||||
|
||||
// The person brings another tab to the front, and the agent's tab loads
|
||||
// again behind it. The text still reaches the field, and the answer says
|
||||
// how.
|
||||
|
||||
@@ -43,6 +43,7 @@
|
||||
"test:rust": "cd src-tauri && cargo test",
|
||||
"test:donut-sync": "cd donut-sync && cargo test --lib --locked",
|
||||
"test:rust:unit": "cd src-tauri && cargo test --lib && cargo test --test donut_proxy_integration && cargo test --test vpn_integration",
|
||||
"test:fleet": "cd src-tauri && cargo test --lib fleet_ -- --ignored --nocapture",
|
||||
"test:sync-e2e": "node scripts/sync-test-harness.mjs",
|
||||
"e2e": "node e2e/run.mjs --suite=full",
|
||||
"e2e:smoke": "node e2e/run.mjs --suite=smoke",
|
||||
|
||||
+1
-1
@@ -223,7 +223,7 @@ server-side failure.
|
||||
| 403 | `Forbidden` | `Forbidden` | Wayfern terms not accepted, or not signed in |
|
||||
| 404 | `NotFound` | `NotFound` | No entity with that id |
|
||||
| 408 | `RequestTimeout` | `RequestTimeout` | `agent/pick` waited and nothing was picked |
|
||||
| 409 | `Conflict` | `Conflict` | A browser, a teammate or a remote session holds the profile |
|
||||
| 409 | `Conflict` | `Conflict` | A browser, a teammate or a remote session holds the profile; a native dialog holds the page (`DIALOG_OPEN`: read it with `agent_dialog` / `agentDialog`, answer it with `agent_handle_dialog` / `agentHandleDialog`) |
|
||||
| 429 | `RateLimited` | `RateLimited` | Automation quota spent; `retry_after` / `retryAfter` |
|
||||
| 500 | `ServerError` | `ServerError` | Internal failure |
|
||||
| 502 | `BadGateway` | `BadGateway` | The browser or the relay answered wrongly |
|
||||
|
||||
+16
-1
@@ -2,7 +2,7 @@
|
||||
"source": "src-tauri/src/api_server.rs",
|
||||
"regenerate_with": "python3 sdk/tools/extract-api-paths.py",
|
||||
"description": "Every operation the desktop app publishes in its /openapi.json. The SDK test suites assert this list and their own coverage tables match exactly, so an endpoint added to the app fails the SDK tests until it is either wrapped or deliberately listed as omitted.",
|
||||
"operation_count": 85,
|
||||
"operation_count": 88,
|
||||
"operations": [
|
||||
{
|
||||
"operation_id": "download_browser_api",
|
||||
@@ -214,11 +214,26 @@
|
||||
"method": "POST",
|
||||
"path": "/v1/profiles/{id}/agent/click"
|
||||
},
|
||||
{
|
||||
"operation_id": "agent_dialog_api",
|
||||
"method": "GET",
|
||||
"path": "/v1/profiles/{id}/agent/dialog"
|
||||
},
|
||||
{
|
||||
"operation_id": "agent_extract_api",
|
||||
"method": "POST",
|
||||
"path": "/v1/profiles/{id}/agent/extract"
|
||||
},
|
||||
{
|
||||
"operation_id": "agent_handle_dialog_api",
|
||||
"method": "POST",
|
||||
"path": "/v1/profiles/{id}/agent/handle-dialog"
|
||||
},
|
||||
{
|
||||
"operation_id": "agent_hold_api",
|
||||
"method": "POST",
|
||||
"path": "/v1/profiles/{id}/agent/hold"
|
||||
},
|
||||
{
|
||||
"operation_id": "agent_perceive_api",
|
||||
"method": "POST",
|
||||
|
||||
@@ -11,6 +11,8 @@
|
||||
import { DonutConnectionError, DonutError, errorForStatus } from "./errors.mts";
|
||||
import type {
|
||||
AgentClick,
|
||||
AgentDialogAnswer,
|
||||
AgentHold,
|
||||
AgentTyping,
|
||||
ApiBulkCreateResponse,
|
||||
ApiGroupResponse,
|
||||
@@ -47,6 +49,7 @@ import type {
|
||||
ImportProxiesResponse,
|
||||
LocatorDescription,
|
||||
LocatorResolution,
|
||||
PageDialog,
|
||||
PerceptionPage,
|
||||
PickedElement,
|
||||
ProfileImportBatchResult,
|
||||
@@ -607,6 +610,29 @@ export class DonutClient {
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* POST /v1/profiles/{id}/agent/hold
|
||||
*
|
||||
* Name the element with `locator` or `selector`. `duration_ms` is 100 to
|
||||
* 30000, default 3000; `button` is `"left"` (the default), `"middle"` or
|
||||
* `"right"`.
|
||||
*/
|
||||
agentHold(
|
||||
profileId: string,
|
||||
request: {
|
||||
locator?: LocatorDescription;
|
||||
selector?: string;
|
||||
duration_ms?: number;
|
||||
button?: string;
|
||||
},
|
||||
): Promise<AgentHold> {
|
||||
return this.#request(
|
||||
"POST",
|
||||
`/v1/profiles/${segment(profileId)}/agent/hold`,
|
||||
body({ ...request }),
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* POST /v1/profiles/{id}/agent/type
|
||||
*
|
||||
@@ -672,6 +698,34 @@ export class DonutClient {
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* GET /v1/profiles/{id}/agent/dialog
|
||||
*
|
||||
* Whether a native dialog holds the page, and which. Never refused for the
|
||||
* dialog: while one is open, every other agent call throws `DIALOG_OPEN`.
|
||||
*/
|
||||
agentDialog(profileId: string): Promise<PageDialog> {
|
||||
return this.#request("GET", `/v1/profiles/${segment(profileId)}/agent/dialog`);
|
||||
}
|
||||
|
||||
/**
|
||||
* POST /v1/profiles/{id}/agent/handle-dialog
|
||||
*
|
||||
* `accept` true is OK, false is Cancel. An accepted prompt takes
|
||||
* `prompt_text`, or keeps its default text. Throws `NO_DIALOG_OPEN` when
|
||||
* nothing is open.
|
||||
*/
|
||||
agentHandleDialog(
|
||||
profileId: string,
|
||||
request: { accept: boolean; prompt_text?: string },
|
||||
): Promise<AgentDialogAnswer> {
|
||||
return this.#request(
|
||||
"POST",
|
||||
`/v1/profiles/${segment(profileId)}/agent/handle-dialog`,
|
||||
body({ ...request }),
|
||||
);
|
||||
}
|
||||
|
||||
// ------------------------------------------------------------------
|
||||
// Remote sessions
|
||||
// ------------------------------------------------------------------
|
||||
|
||||
@@ -62,7 +62,10 @@ export const OPERATIONS: ReadonlyMap<OperationKey, string> = new Map([
|
||||
["GET /v1/profiles/{id}/fingerprint", "getProfileFingerprint"],
|
||||
["PUT /v1/profiles/{id}", "updateProfile"],
|
||||
["POST /v1/profiles/{id}/agent/click", "agentClick"],
|
||||
["GET /v1/profiles/{id}/agent/dialog", "agentDialog"],
|
||||
["POST /v1/profiles/{id}/agent/extract", "agentExtract"],
|
||||
["POST /v1/profiles/{id}/agent/handle-dialog", "agentHandleDialog"],
|
||||
["POST /v1/profiles/{id}/agent/hold", "agentHold"],
|
||||
["POST /v1/profiles/{id}/agent/perceive", "agentPerceive"],
|
||||
["POST /v1/profiles/{id}/agent/pick", "agentPick"],
|
||||
["POST /v1/profiles/{id}/agent/resolve-locator", "agentResolveLocator"],
|
||||
|
||||
+48
-2
@@ -8,8 +8,8 @@
|
||||
* snake_case; the browser-facing agent types (`LocatorDescription`,
|
||||
* `LocatorCandidate`, `PerceptionPage` and friends) carry the browser's own
|
||||
* camelCase, because they are handed through from the browser rather than
|
||||
* restated. `AgentClick` and `AgentTyping` are the exceptions inside the agent
|
||||
* surface: they are snake_case with a single `match` key. These types follow
|
||||
* restated. `AgentClick`, `AgentHold` and `AgentTyping` are the exceptions
|
||||
* inside the agent surface: they are snake_case with a single `match` key. These types follow
|
||||
* the wire rather than tidying it, so a value read from one call can be passed
|
||||
* straight into the next.
|
||||
*/
|
||||
@@ -800,3 +800,49 @@ export interface AgentTyping {
|
||||
*/
|
||||
typed_with: "browser_keyboard" | "key_events" | "inserted";
|
||||
}
|
||||
|
||||
/** What a press and hold did. Note the snake_case body and the `match` key. */
|
||||
export interface AgentHold {
|
||||
held: boolean;
|
||||
/** The element a locator settled on. Absent when a selector named it. */
|
||||
match?: LocatorCandidate;
|
||||
engine: Engine;
|
||||
/** Whether a page load followed the release. */
|
||||
navigated: boolean;
|
||||
button: string;
|
||||
/** How long the button was down, in milliseconds. */
|
||||
held_ms: number;
|
||||
}
|
||||
|
||||
/** A native dialog (alert, confirm, prompt, "leave this page?") open on a page. */
|
||||
export interface OpenDialog {
|
||||
/** `alert`, `confirm`, `prompt` or `beforeunload`. */
|
||||
type: string;
|
||||
message: string;
|
||||
/** The text a prompt puts in its field. */
|
||||
default_prompt: string;
|
||||
/** The page that opened it. */
|
||||
url: string;
|
||||
}
|
||||
|
||||
/** Whether a native dialog holds the page. */
|
||||
export interface PageDialog {
|
||||
/**
|
||||
* True while a dialog holds the page, false when none does, null when the
|
||||
* page answers nothing and no dialog was seen opening on it.
|
||||
*/
|
||||
open: boolean | null;
|
||||
dialog?: OpenDialog;
|
||||
/** False when the page answers nothing. A navigation ends that. */
|
||||
page_responding?: boolean;
|
||||
note?: string;
|
||||
}
|
||||
|
||||
/** What answering a dialog did. */
|
||||
export interface AgentDialogAnswer {
|
||||
accepted: boolean;
|
||||
/** The dialog that was answered. */
|
||||
dialog: OpenDialog;
|
||||
/** A dialog the page opened in answer. It holds the page now. */
|
||||
next?: OpenDialog;
|
||||
}
|
||||
@@ -349,6 +349,30 @@ const CASES: Case[] = [
|
||||
body: { timeout_ms: 15000 },
|
||||
operation: "POST /v1/profiles/{id}/agent/pick",
|
||||
},
|
||||
{
|
||||
method: "agentHold",
|
||||
args: ["p1", { selector: "#press", duration_ms: 4000 }],
|
||||
verb: "POST",
|
||||
path: "/v1/profiles/p1/agent/hold",
|
||||
body: { selector: "#press", duration_ms: 4000 },
|
||||
operation: "POST /v1/profiles/{id}/agent/hold",
|
||||
},
|
||||
{
|
||||
method: "agentDialog",
|
||||
args: ["p1"],
|
||||
verb: "GET",
|
||||
path: "/v1/profiles/p1/agent/dialog",
|
||||
body: null,
|
||||
operation: "GET /v1/profiles/{id}/agent/dialog",
|
||||
},
|
||||
{
|
||||
method: "agentHandleDialog",
|
||||
args: ["p1", { accept: true, prompt_text: "Bob" }],
|
||||
verb: "POST",
|
||||
path: "/v1/profiles/p1/agent/handle-dialog",
|
||||
body: { accept: true, prompt_text: "Bob" },
|
||||
operation: "POST /v1/profiles/{id}/agent/handle-dialog",
|
||||
},
|
||||
// -- remote sessions -----------------------------------------------------
|
||||
{
|
||||
method: "listRemoteSessions",
|
||||
|
||||
@@ -21,6 +21,8 @@ from urllib.parse import quote, urlencode, urlsplit
|
||||
from .errors import DonutConnectionError, DonutError, error_for_status
|
||||
from .models import (
|
||||
AgentClick,
|
||||
AgentDialogAnswer,
|
||||
AgentHold,
|
||||
AgentTyping,
|
||||
ApiBulkCreateResponse,
|
||||
ApiGroupResponse,
|
||||
@@ -57,6 +59,7 @@ from .models import (
|
||||
ImportProxiesResponse,
|
||||
LocatorDescription,
|
||||
LocatorResolution,
|
||||
PageDialog,
|
||||
PerceptionPage,
|
||||
PickedElement,
|
||||
ProfileImportBatchResult,
|
||||
@@ -666,6 +669,32 @@ class DonutClient:
|
||||
body=_body(locator=dict(locator), button=button, click_count=click_count),
|
||||
)
|
||||
|
||||
def agent_hold(
|
||||
self,
|
||||
profile_id: str,
|
||||
*,
|
||||
locator: Optional[LocatorDescription] = None,
|
||||
selector: Optional[str] = None,
|
||||
duration_ms: Optional[float] = None,
|
||||
button: Optional[str] = None,
|
||||
) -> AgentHold:
|
||||
"""POST /v1/profiles/{id}/agent/hold
|
||||
|
||||
Name the element with ``locator`` or ``selector``. ``duration_ms`` is
|
||||
100 to 30000, default 3000; ``button`` is ``"left"`` (the default),
|
||||
``"middle"`` or ``"right"``.
|
||||
"""
|
||||
return self._request(
|
||||
"POST",
|
||||
f"/v1/profiles/{_segment(profile_id)}/agent/hold",
|
||||
body=_body(
|
||||
locator=dict(locator) if locator is not None else None,
|
||||
selector=selector,
|
||||
duration_ms=duration_ms,
|
||||
button=button,
|
||||
),
|
||||
)
|
||||
|
||||
def agent_type(
|
||||
self,
|
||||
profile_id: str,
|
||||
@@ -739,6 +768,34 @@ class DonutClient:
|
||||
body=_body(timeout_ms=timeout_ms),
|
||||
)
|
||||
|
||||
def agent_dialog(self, profile_id: str) -> PageDialog:
|
||||
"""GET /v1/profiles/{id}/agent/dialog
|
||||
|
||||
Whether a native dialog holds the page, and which. Never refused for
|
||||
the dialog: while one is open, every other agent call raises
|
||||
``DIALOG_OPEN``.
|
||||
"""
|
||||
return self._request("GET", f"/v1/profiles/{_segment(profile_id)}/agent/dialog")
|
||||
|
||||
def agent_handle_dialog(
|
||||
self,
|
||||
profile_id: str,
|
||||
*,
|
||||
accept: bool,
|
||||
prompt_text: Optional[str] = None,
|
||||
) -> AgentDialogAnswer:
|
||||
"""POST /v1/profiles/{id}/agent/handle-dialog
|
||||
|
||||
``accept`` true is OK, false is Cancel. An accepted prompt takes
|
||||
``prompt_text``, or keeps its default text. Raises ``NO_DIALOG_OPEN``
|
||||
when nothing is open.
|
||||
"""
|
||||
return self._request(
|
||||
"POST",
|
||||
f"/v1/profiles/{_segment(profile_id)}/agent/handle-dialog",
|
||||
body=_body(accept=accept, prompt_text=prompt_text),
|
||||
)
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Remote sessions
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
@@ -69,7 +69,10 @@ OPERATIONS: Dict[Operation, str] = {
|
||||
("GET", "/v1/profiles/{id}/fingerprint"): "get_profile_fingerprint",
|
||||
("PUT", "/v1/profiles/{id}"): "update_profile",
|
||||
("POST", "/v1/profiles/{id}/agent/click"): "agent_click",
|
||||
("GET", "/v1/profiles/{id}/agent/dialog"): "agent_dialog",
|
||||
("POST", "/v1/profiles/{id}/agent/extract"): "agent_extract",
|
||||
("POST", "/v1/profiles/{id}/agent/handle-dialog"): "agent_handle_dialog",
|
||||
("POST", "/v1/profiles/{id}/agent/hold"): "agent_hold",
|
||||
("POST", "/v1/profiles/{id}/agent/perceive"): "agent_perceive",
|
||||
("POST", "/v1/profiles/{id}/agent/pick"): "agent_pick",
|
||||
("POST", "/v1/profiles/{id}/agent/resolve-locator"): "agent_resolve_locator",
|
||||
|
||||
@@ -9,8 +9,8 @@ Two spellings live side by side because the app sends both. Most bodies are
|
||||
snake_case; the browser-facing agent types (``LocatorDescription``,
|
||||
``LocatorCandidate``, ``PerceptionPage`` and friends) carry the browser's own
|
||||
camelCase, because they are handed through from the browser rather than
|
||||
restated. ``AgentClick`` and ``AgentTyping`` are the exceptions inside the
|
||||
agent surface: they are snake_case with a single ``match`` key. The types below
|
||||
restated. ``AgentClick``, ``AgentHold`` and ``AgentTyping`` are the exceptions
|
||||
inside the agent surface: they are snake_case with a single ``match`` key. The types below
|
||||
follow the wire rather than tidying it, so a value read from one call can be
|
||||
passed straight into the next.
|
||||
"""
|
||||
@@ -94,7 +94,11 @@ __all__ = [
|
||||
"Extraction",
|
||||
"PickedElement",
|
||||
"AgentClick",
|
||||
"AgentHold",
|
||||
"AgentTyping",
|
||||
"OpenDialog",
|
||||
"PageDialog",
|
||||
"AgentDialogAnswer",
|
||||
]
|
||||
|
||||
# The app's own JSON for a proxy's settings. Declared `Object` in the OpenAPI
|
||||
@@ -931,3 +935,64 @@ class AgentTyping(_AgentTypingRequired, total=False):
|
||||
"""
|
||||
|
||||
corrections: int
|
||||
|
||||
|
||||
class _AgentHoldRequired(TypedDict):
|
||||
held: bool
|
||||
engine: str
|
||||
navigated: bool
|
||||
button: str
|
||||
held_ms: int
|
||||
|
||||
|
||||
class AgentHold(_AgentHoldRequired, total=False):
|
||||
"""What a press and hold did. Note the snake_case body and the ``match`` key.
|
||||
|
||||
``match`` is the element a locator settled on, absent when a selector named
|
||||
it. ``held_ms`` is how long the button was down.
|
||||
"""
|
||||
|
||||
match: LocatorCandidate
|
||||
|
||||
|
||||
class OpenDialog(TypedDict):
|
||||
"""A native dialog (alert, confirm, prompt, "leave this page?") on a page.
|
||||
|
||||
``type`` is ``alert``, ``confirm``, ``prompt`` or ``beforeunload``;
|
||||
``default_prompt`` is the text a prompt puts in its field; ``url`` is the
|
||||
page that opened it.
|
||||
"""
|
||||
|
||||
type: str
|
||||
message: str
|
||||
default_prompt: str
|
||||
url: str
|
||||
|
||||
|
||||
class _PageDialogRequired(TypedDict):
|
||||
open: Optional[bool]
|
||||
|
||||
|
||||
class PageDialog(_PageDialogRequired, total=False):
|
||||
"""Whether a native dialog holds the page.
|
||||
|
||||
``open`` is true while a dialog holds the page, false when none does, and
|
||||
``None`` when the page answers nothing and no dialog was seen opening on
|
||||
it; ``page_responding`` is then false. A navigation ends that.
|
||||
"""
|
||||
|
||||
dialog: OpenDialog
|
||||
page_responding: bool
|
||||
note: str
|
||||
|
||||
|
||||
class _AgentDialogAnswerRequired(TypedDict):
|
||||
accepted: bool
|
||||
dialog: OpenDialog
|
||||
|
||||
|
||||
class AgentDialogAnswer(_AgentDialogAnswerRequired, total=False):
|
||||
"""What answering a dialog did. ``next`` is a dialog the page opened in
|
||||
answer, which holds the page now."""
|
||||
|
||||
next: OpenDialog
|
||||
@@ -365,6 +365,36 @@ CASES: List[Case] = [
|
||||
{},
|
||||
"/v1/profiles/{id}/agent/pick",
|
||||
),
|
||||
(
|
||||
"agent_hold",
|
||||
("p1",),
|
||||
{"selector": "#press", "duration_ms": 4000.0},
|
||||
"POST",
|
||||
"/v1/profiles/p1/agent/hold",
|
||||
{"selector": "#press", "duration_ms": 4000.0},
|
||||
{},
|
||||
"/v1/profiles/{id}/agent/hold",
|
||||
),
|
||||
(
|
||||
"agent_dialog",
|
||||
("p1",),
|
||||
{},
|
||||
"GET",
|
||||
"/v1/profiles/p1/agent/dialog",
|
||||
None,
|
||||
{},
|
||||
"/v1/profiles/{id}/agent/dialog",
|
||||
),
|
||||
(
|
||||
"agent_handle_dialog",
|
||||
("p1",),
|
||||
{"accept": True, "prompt_text": "Bob"},
|
||||
"POST",
|
||||
"/v1/profiles/p1/agent/handle-dialog",
|
||||
{"accept": True, "prompt_text": "Bob"},
|
||||
{},
|
||||
"/v1/profiles/{id}/agent/handle-dialog",
|
||||
),
|
||||
# -- remote sessions ---------------------------------------------------
|
||||
(
|
||||
"list_remote_sessions",
|
||||
|
||||
+160
-9
@@ -885,6 +885,8 @@ struct ImportProxiesResponse {
|
||||
agent_type_api,
|
||||
agent_extract_api,
|
||||
agent_pick_api,
|
||||
agent_dialog_api,
|
||||
agent_handle_dialog_api,
|
||||
),
|
||||
components(schemas(
|
||||
ApiProfile,
|
||||
@@ -1005,6 +1007,10 @@ struct ImportProxiesResponse {
|
||||
crate::mcp_server::AgentTyping,
|
||||
crate::mcp_server::TypedWith,
|
||||
crate::mcp_server::AgentPickRequest,
|
||||
crate::mcp_server::PageDialog,
|
||||
crate::mcp_server::AgentDialogRequest,
|
||||
crate::mcp_server::AgentDialogAnswer,
|
||||
crate::page_dialog::OpenDialog,
|
||||
)),
|
||||
tags(
|
||||
(name = "profiles", description = "Profile management endpoints"),
|
||||
@@ -1265,6 +1271,8 @@ fn build_v1_router() -> Router<ApiServerState> {
|
||||
.routes(routes!(agent_type_api))
|
||||
.routes(routes!(agent_extract_api))
|
||||
.routes(routes!(agent_pick_api))
|
||||
.routes(routes!(agent_dialog_api))
|
||||
.routes(routes!(agent_handle_dialog_api))
|
||||
.split_for_parts();
|
||||
|
||||
// The two paths that carry an extension payload, kept apart so the raised
|
||||
@@ -1455,7 +1463,16 @@ fn is_automation_request(method: &Method, path: &str) -> bool {
|
||||
// exactly as a script one is.
|
||||
Some(_),
|
||||
Some("agent"),
|
||||
Some("perceive" | "resolve-locator" | "click" | "hold" | "type" | "extract" | "pick"),
|
||||
Some(
|
||||
"perceive"
|
||||
| "resolve-locator"
|
||||
| "click"
|
||||
| "hold"
|
||||
| "type"
|
||||
| "extract"
|
||||
| "pick"
|
||||
| "handle-dialog"
|
||||
),
|
||||
None
|
||||
)
|
||||
)
|
||||
@@ -5464,9 +5481,21 @@ async fn check_browser_downloaded(
|
||||
// resolved the way every automation endpoint is: 402 without the plan, 404 for
|
||||
// an unknown profile, 409 when it is not running, 429 from the shared limiter.
|
||||
|
||||
/// The running browser behind `id`, and the engine its version entitles it to.
|
||||
/// The running browser behind `id`, and the engine its version entitles it to,
|
||||
/// refused while a dialog holds its page.
|
||||
async fn agent_context_for(
|
||||
id: &str,
|
||||
) -> Result<crate::mcp_server::AgentContext, (StatusCode, String)> {
|
||||
let ctx = agent_target_for(id).await?;
|
||||
crate::mcp_server::page_ready(&ctx.target, false)
|
||||
.await
|
||||
.map_err(agent_error_response)?;
|
||||
Ok(ctx)
|
||||
}
|
||||
|
||||
/// [`agent_context_for`], whatever the page is doing.
|
||||
async fn agent_target_for(
|
||||
id: &str,
|
||||
) -> Result<crate::mcp_server::AgentContext, (StatusCode, String)> {
|
||||
if !crate::cloud_auth::CLOUD_AUTH
|
||||
.can_use_browser_automation()
|
||||
@@ -5585,7 +5614,7 @@ fn agent_error_response(error: crate::mcp_server::AgentError) -> (StatusCode, St
|
||||
(status = 401, description = "Unauthorized"),
|
||||
(status = 402, description = "Active paid plan with browser automation required"),
|
||||
(status = 404, description = "Profile not found"),
|
||||
(status = 409, description = "The profile is not running"),
|
||||
(status = 409, description = "The profile is not running, or a native dialog holds the page (`code` DIALOG_OPEN; read it with GET agent/dialog, answer it with agent/handle-dialog) or the page answers nothing (`code` PAGE_NOT_RESPONDING)"),
|
||||
(status = 429, description = "Automation request rate limit exceeded"),
|
||||
(status = 502, description = "The browser did not answer as documented"),
|
||||
(status = 500, description = "Internal server error")
|
||||
@@ -5616,7 +5645,7 @@ async fn agent_perceive_api(
|
||||
(status = 401, description = "Unauthorized"),
|
||||
(status = 402, description = "Active paid plan with browser automation required"),
|
||||
(status = 404, description = "Profile not found"),
|
||||
(status = 409, description = "The profile is not running"),
|
||||
(status = 409, description = "The profile is not running, or a native dialog holds the page (`code` DIALOG_OPEN; read it with GET agent/dialog, answer it with agent/handle-dialog) or the page answers nothing (`code` PAGE_NOT_RESPONDING)"),
|
||||
(status = 429, description = "Automation request rate limit exceeded"),
|
||||
(status = 502, description = "The browser did not answer as documented"),
|
||||
(status = 500, description = "Internal server error")
|
||||
@@ -5651,7 +5680,7 @@ async fn agent_resolve_locator_api(
|
||||
(status = 401, description = "Unauthorized"),
|
||||
(status = 402, description = "Active paid plan with browser automation required"),
|
||||
(status = 404, description = "Profile not found"),
|
||||
(status = 409, description = "The profile is not running"),
|
||||
(status = 409, description = "The profile is not running, or a native dialog holds the page (`code` DIALOG_OPEN; read it with GET agent/dialog, answer it with agent/handle-dialog) or the page answers nothing (`code` PAGE_NOT_RESPONDING)"),
|
||||
(status = 429, description = "Automation request rate limit exceeded"),
|
||||
(status = 502, description = "The browser did not answer as documented"),
|
||||
(status = 500, description = "Internal server error")
|
||||
@@ -5686,7 +5715,7 @@ async fn agent_click_api(
|
||||
(status = 401, description = "Unauthorized"),
|
||||
(status = 402, description = "Active paid plan with browser automation required"),
|
||||
(status = 404, description = "Profile not found"),
|
||||
(status = 409, description = "The profile is not running"),
|
||||
(status = 409, description = "The profile is not running, or a native dialog holds the page (`code` DIALOG_OPEN; read it with GET agent/dialog, answer it with agent/handle-dialog) or the page answers nothing (`code` PAGE_NOT_RESPONDING)"),
|
||||
(status = 429, description = "Automation request rate limit exceeded"),
|
||||
(status = 502, description = "The browser did not answer as documented"),
|
||||
(status = 500, description = "Internal server error")
|
||||
@@ -5717,7 +5746,7 @@ async fn agent_hold_api(
|
||||
(status = 401, description = "Unauthorized"),
|
||||
(status = 402, description = "Active paid plan with browser automation required"),
|
||||
(status = 404, description = "Profile not found"),
|
||||
(status = 409, description = "The profile is not running, the field did not take the focus or the text (`code` TYPE_NOT_APPLIED, `reason` not_focused, focus_moved or value), or a dialog is open on the page (`code` DIALOG_OPEN)"),
|
||||
(status = 409, description = "The profile is not running, the field did not take the focus or the text (`code` TYPE_NOT_APPLIED, `reason` not_focused, focus_moved or value), a native dialog holds the page (`code` DIALOG_OPEN), or the page answers nothing (`code` PAGE_NOT_RESPONDING)"),
|
||||
(status = 429, description = "Automation request rate limit exceeded"),
|
||||
(status = 502, description = "The browser did not answer as documented"),
|
||||
(status = 500, description = "Internal server error")
|
||||
@@ -5753,7 +5782,7 @@ async fn agent_type_api(
|
||||
(status = 401, description = "Unauthorized"),
|
||||
(status = 402, description = "Active paid plan with browser automation required"),
|
||||
(status = 404, description = "Profile not found"),
|
||||
(status = 409, description = "The profile is not running"),
|
||||
(status = 409, description = "The profile is not running, or a native dialog holds the page (`code` DIALOG_OPEN; read it with GET agent/dialog, answer it with agent/handle-dialog) or the page answers nothing (`code` PAGE_NOT_RESPONDING)"),
|
||||
(status = 429, description = "Automation request rate limit exceeded"),
|
||||
(status = 502, description = "The browser did not answer as documented"),
|
||||
(status = 500, description = "Internal server error")
|
||||
@@ -5785,7 +5814,7 @@ async fn agent_extract_api(
|
||||
(status = 402, description = "Active paid plan with browser automation required"),
|
||||
(status = 404, description = "Profile not found"),
|
||||
(status = 408, description = "Nothing was picked within timeout_ms; the picker has been disarmed"),
|
||||
(status = 409, description = "The profile is not running, or the picker was cancelled (Escape, or a navigation)"),
|
||||
(status = 409, description = "The profile is not running, the picker was cancelled (Escape, or a navigation), a native dialog holds the page (`code` DIALOG_OPEN), or the page answers nothing (`code` PAGE_NOT_RESPONDING)"),
|
||||
(status = 429, description = "Automation request rate limit exceeded"),
|
||||
(status = 502, description = "The browser did not answer as documented"),
|
||||
(status = 500, description = "Internal server error")
|
||||
@@ -5811,6 +5840,76 @@ async fn agent_pick_api(
|
||||
.await
|
||||
}
|
||||
|
||||
#[utoipa::path(
|
||||
get,
|
||||
path = "/v1/profiles/{id}/agent/dialog",
|
||||
params(
|
||||
("id" = String, Path, description = "Profile ID")
|
||||
),
|
||||
responses(
|
||||
(status = 200, description = "Whether a native dialog (alert, confirm, prompt, \"leave this page?\") holds the page, and which. `open` is null when the page answers nothing and no dialog was seen opening; a navigation ends that", body = crate::mcp_server::PageDialog),
|
||||
(status = 401, description = "Unauthorized"),
|
||||
(status = 402, description = "Active paid plan with browser automation required"),
|
||||
(status = 404, description = "Profile not found"),
|
||||
(status = 409, description = "The profile is not running"),
|
||||
(status = 502, description = "The browser did not answer as documented"),
|
||||
(status = 500, description = "Internal server error")
|
||||
),
|
||||
security(
|
||||
("bearer_auth" = [])
|
||||
),
|
||||
tag = "agent"
|
||||
)]
|
||||
async fn agent_dialog_api(
|
||||
Path(id): Path<String>,
|
||||
) -> Result<Json<crate::mcp_server::PageDialog>, (StatusCode, String)> {
|
||||
let ctx = agent_target_for(&id).await?;
|
||||
agent_reply(&id, crate::mcp_server::agent_dialog(&ctx.target).await).await
|
||||
}
|
||||
|
||||
#[utoipa::path(
|
||||
post,
|
||||
path = "/v1/profiles/{id}/agent/handle-dialog",
|
||||
params(
|
||||
("id" = String, Path, description = "Profile ID")
|
||||
),
|
||||
request_body = crate::mcp_server::AgentDialogRequest,
|
||||
responses(
|
||||
(status = 200, description = "The dialog was accepted or dismissed. `next` is a dialog the page opened in answer, which holds the page now", body = crate::mcp_server::AgentDialogAnswer),
|
||||
(status = 400, description = "Malformed request"),
|
||||
(status = 401, description = "Unauthorized"),
|
||||
(status = 402, description = "Active paid plan with browser automation required"),
|
||||
(status = 404, description = "Profile not found"),
|
||||
(status = 409, description = "The profile is not running, no dialog is open (`code` NO_DIALOG_OPEN), the page answers nothing (`code` PAGE_NOT_RESPONDING), or the person at this computer paused the agents (`code` AGENT_PAUSED) or took over the profile (`code` PROFILE_HELD_BY_HUMAN)"),
|
||||
(status = 429, description = "Automation request rate limit exceeded"),
|
||||
(status = 502, description = "The browser did not answer as documented"),
|
||||
(status = 500, description = "Internal server error")
|
||||
),
|
||||
security(
|
||||
("bearer_auth" = [])
|
||||
),
|
||||
tag = "agent"
|
||||
)]
|
||||
async fn agent_handle_dialog_api(
|
||||
Path(id): Path<String>,
|
||||
Json(request): Json<crate::mcp_server::AgentDialogRequest>,
|
||||
) -> Result<Json<crate::mcp_server::AgentDialogAnswer>, (StatusCode, String)> {
|
||||
if let Err(refusal) =
|
||||
crate::agent_console::check_call(false, &serde_json::json!({ "profile_id": id }))
|
||||
{
|
||||
return Err((
|
||||
StatusCode::CONFLICT,
|
||||
crate::mcp_server::McpServer::console_refusal(refusal).rest_body(),
|
||||
));
|
||||
}
|
||||
let ctx = agent_target_for(&id).await?;
|
||||
agent_reply(
|
||||
&id,
|
||||
crate::mcp_server::agent_handle_dialog(&ctx.target, &request).await,
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod url_guard_tests {
|
||||
//! The REST automation surface takes caller-supplied URLs too.
|
||||
@@ -6207,6 +6306,8 @@ mod tests {
|
||||
"/v1/profiles/profile-id/agent/type",
|
||||
"/v1/profiles/profile-id/agent/extract",
|
||||
"/v1/profiles/profile-id/agent/pick",
|
||||
// Answering a dialog acts on the page; reading one does not.
|
||||
"/v1/profiles/profile-id/agent/handle-dialog",
|
||||
] {
|
||||
assert!(
|
||||
is_automation_request(&Method::POST, path),
|
||||
@@ -6236,6 +6337,7 @@ mod tests {
|
||||
(Method::POST, "/v1/profiles/profile-id/agent/unknown"),
|
||||
(Method::POST, "/v1/profiles/profile-id/agent/click/twice"),
|
||||
(Method::GET, "/v1/profiles/profile-id/agent/perceive"),
|
||||
(Method::GET, "/v1/profiles/profile-id/agent/dialog"),
|
||||
(Method::POST, "/v1/profiles"),
|
||||
(Method::POST, "/v1/profiles/import"),
|
||||
(Method::GET, "/v1/profiles"),
|
||||
@@ -7085,6 +7187,25 @@ mod tests {
|
||||
});
|
||||
assert_eq!(status, StatusCode::CONFLICT);
|
||||
|
||||
// A dialog holds the page: a conflict with its state, with the dialog in
|
||||
// the body, the same envelope MCP answers.
|
||||
let dialog = crate::page_dialog::test_support::dialog("confirm", "Delete it?");
|
||||
let (status, body) = agent_error_response(AgentError::Cdp(CdpError::DialogOpen(dialog)));
|
||||
assert_eq!(status, StatusCode::CONFLICT);
|
||||
let body: serde_json::Value = serde_json::from_str(&body).expect("a JSON body");
|
||||
assert_eq!(body["code"], "DIALOG_OPEN");
|
||||
assert_eq!(body["dialog"]["type"], "confirm");
|
||||
assert_eq!(body["dialog"]["message"], "Delete it?");
|
||||
assert!(body["message"].as_str().unwrap().contains("Delete it?"));
|
||||
for (error, code) in [
|
||||
(AgentError::NoDialog, "NO_DIALOG_OPEN"),
|
||||
(AgentError::PageNotResponding, "PAGE_NOT_RESPONDING"),
|
||||
] {
|
||||
let (status, body) = agent_error_response(error);
|
||||
assert_eq!(status, StatusCode::CONFLICT);
|
||||
assert!(body.contains(code), "{body}");
|
||||
}
|
||||
|
||||
// A browser that is not there is "start it", not "we broke".
|
||||
let (status, _) = agent_error_response(AgentError::Cdp(CdpError::Unreachable("x".into())));
|
||||
assert_eq!(status, StatusCode::CONFLICT);
|
||||
@@ -7692,6 +7813,7 @@ mod tests {
|
||||
"/v1/profiles/{id}/agent/type",
|
||||
"/v1/profiles/{id}/agent/extract",
|
||||
"/v1/profiles/{id}/agent/pick",
|
||||
"/v1/profiles/{id}/agent/handle-dialog",
|
||||
] {
|
||||
assert!(
|
||||
paths[path]["post"]["responses"].get("429").is_some(),
|
||||
@@ -7719,6 +7841,7 @@ mod tests {
|
||||
("/v1/profiles/{id}/agent/type", "AgentTyping"),
|
||||
("/v1/profiles/{id}/agent/extract", "Extraction"),
|
||||
("/v1/profiles/{id}/agent/pick", "PickedElement"),
|
||||
("/v1/profiles/{id}/agent/handle-dialog", "AgentDialogAnswer"),
|
||||
] {
|
||||
let reference =
|
||||
&paths[path]["post"]["responses"]["200"]["content"]["application/json"]["schema"]["$ref"];
|
||||
@@ -7756,6 +7879,10 @@ mod tests {
|
||||
"AgentTypeRequest",
|
||||
"AgentTyping",
|
||||
"AgentPickRequest",
|
||||
"PageDialog",
|
||||
"AgentDialogRequest",
|
||||
"AgentDialogAnswer",
|
||||
"OpenDialog",
|
||||
] {
|
||||
let component = &spec["components"]["schemas"][schema];
|
||||
assert!(
|
||||
@@ -7787,6 +7914,30 @@ mod tests {
|
||||
serde_json::json!(["wayfern", "fallback"])
|
||||
);
|
||||
|
||||
// Reading a dialog is a GET that is never refused for the dialog; it is
|
||||
// gated and resolved like every agent route, and is not metered.
|
||||
let read = &paths["/v1/profiles/{id}/agent/dialog"]["get"];
|
||||
assert_eq!(
|
||||
read["responses"]["200"]["content"]["application/json"]["schema"]["$ref"],
|
||||
"#/components/schemas/PageDialog"
|
||||
);
|
||||
for status in ["402", "404", "409"] {
|
||||
assert!(
|
||||
read["responses"].get(status).is_some(),
|
||||
"GET agent/dialog is missing its {status} response"
|
||||
);
|
||||
}
|
||||
assert!(read["responses"].get("429").is_none());
|
||||
assert_eq!(schema_required(&spec, "AgentDialogRequest"), vec!["accept"]);
|
||||
assert!(schema_required(&spec, "PageDialog").is_empty());
|
||||
let answer = schema_required(&spec, "AgentDialogAnswer");
|
||||
assert!(answer.contains(&"accepted".to_string()) && answer.contains(&"dialog".to_string()));
|
||||
assert!(!answer.contains(&"next".to_string()), "next is optional");
|
||||
assert_eq!(
|
||||
schema_required(&spec, "OpenDialog"),
|
||||
vec!["type", "message", "default_prompt", "url"]
|
||||
);
|
||||
|
||||
assert!(
|
||||
paths["/v1/cookie-bot/runs"]["post"]["responses"]
|
||||
.get("429")
|
||||
|
||||
+77
-16
@@ -401,6 +401,8 @@ pub struct CdpConnection {
|
||||
/// page-level commands reach the page rather than the browser.
|
||||
cdp_session: Option<String>,
|
||||
closed: Option<CloseInfo>,
|
||||
/// The page a relayed connection attached to.
|
||||
page: Option<String>,
|
||||
/// The page's dialog state, when [`crate::page_dialog`] watches the page.
|
||||
dialog: Option<tokio::sync::watch::Receiver<crate::page_dialog::PageState>>,
|
||||
}
|
||||
@@ -427,6 +429,7 @@ impl CdpConnection {
|
||||
stream,
|
||||
cdp_session: None,
|
||||
closed: None,
|
||||
page: None,
|
||||
dialog: None,
|
||||
}
|
||||
}
|
||||
@@ -656,14 +659,23 @@ impl CdpConnection {
|
||||
let _ = self.stream.close(None).await;
|
||||
}
|
||||
|
||||
/// Move a browser-level socket onto a page.
|
||||
/// Move a browser-level socket onto a page: `pinned`, the page the session's
|
||||
/// tools drive, when it is known and still open, else the first page the
|
||||
/// browser lists.
|
||||
///
|
||||
/// The remote endpoint exposes `/devtools/browser/<id>`. Every tool here
|
||||
/// speaks `Page.*`, `Runtime.*` and `Input.*`, which a browser socket answers
|
||||
/// with `'Page.navigate' wasn't found`. Attaching flat, and stamping the
|
||||
/// resulting session id onto everything after it, is what makes the tools
|
||||
/// this app already has work remotely without a single per-tool change.
|
||||
async fn attach_to_page(&mut self) -> Result<(), CdpError> {
|
||||
async fn attach_to_page(&mut self, pinned: Option<String>) -> Result<(), CdpError> {
|
||||
if let Some(page) = pinned {
|
||||
match self.attach(&page).await {
|
||||
// The page closed since it was pinned.
|
||||
Err(CdpError::Protocol(_)) => {}
|
||||
attached => return attached,
|
||||
}
|
||||
}
|
||||
let targets = self
|
||||
.call(
|
||||
HANDSHAKE_GET_TARGETS_ID,
|
||||
@@ -673,7 +685,10 @@ impl CdpConnection {
|
||||
.await?;
|
||||
let target_id = pick_remote_page_target(&targets)
|
||||
.ok_or_else(|| CdpError::NotDrivable("the remote browser has no page open".to_string()))?;
|
||||
self.attach(&target_id).await
|
||||
}
|
||||
|
||||
async fn attach(&mut self, target_id: &str) -> Result<(), CdpError> {
|
||||
let attached = self
|
||||
.call(
|
||||
HANDSHAKE_ATTACH_ID,
|
||||
@@ -689,6 +704,7 @@ impl CdpConnection {
|
||||
CdpError::Protocol("Target.attachToTarget returned no sessionId".to_string())
|
||||
})?;
|
||||
self.cdp_session = Some(session.to_string());
|
||||
self.page = Some(target_id.to_string());
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
@@ -743,8 +759,8 @@ static POOL: std::sync::LazyLock<
|
||||
> = std::sync::LazyLock::new(Default::default);
|
||||
|
||||
impl CdpTarget {
|
||||
/// The page socket this target names: the key of its pooled connection and
|
||||
/// of its dialog watch.
|
||||
/// The page socket this target names, or the cloud session: the key of its
|
||||
/// pooled connection.
|
||||
pub(crate) fn pool_key(&self) -> String {
|
||||
match self {
|
||||
Self::Local { ws_url } => ws_url.clone(),
|
||||
@@ -753,23 +769,40 @@ impl CdpTarget {
|
||||
} => format!("{ws_url}#{session_id}"),
|
||||
}
|
||||
}
|
||||
|
||||
/// The key of the dialog watch of a page: the page socket here, or `page`
|
||||
/// of a cloud session.
|
||||
pub(crate) fn page_key(&self, page: Option<&str>) -> String {
|
||||
match (self, page) {
|
||||
(Self::Remote { .. }, Some(page)) => format!("{}#{page}", self.pool_key()),
|
||||
_ => self.pool_key(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Whether a failure on a pooled socket means the command never reached the
|
||||
/// browser, so a fresh socket may send it again.
|
||||
/// browser, so a fresh socket may send it again. A relayed socket whose page
|
||||
/// closed answers that its page session is gone.
|
||||
fn not_delivered(error: &CdpError) -> bool {
|
||||
matches!(error, CdpError::Transport(message)
|
||||
if message.starts_with("failed to send CDP command")
|
||||
|| message.contains("connection closed"))
|
||||
match error {
|
||||
CdpError::Transport(message) => {
|
||||
message.starts_with("failed to send CDP command") || message.contains("connection closed")
|
||||
}
|
||||
CdpError::Protocol(message) => message.contains(SESSION_GONE),
|
||||
_ => false,
|
||||
}
|
||||
}
|
||||
|
||||
/// How a browser refuses a command to a page session that has ended.
|
||||
const SESSION_GONE: &str = "Session with given id not found";
|
||||
|
||||
/// Whether the socket is still fit for the next command after `result`. A
|
||||
/// late reply to a command cut short by a dialog is skipped by its id.
|
||||
fn reusable(result: &Result<Value, CdpError>) -> bool {
|
||||
matches!(
|
||||
result,
|
||||
Ok(_) | Err(CdpError::Protocol(_)) | Err(CdpError::DialogOpen(_))
|
||||
)
|
||||
match result {
|
||||
Err(CdpError::Protocol(message)) => !message.contains(SESSION_GONE),
|
||||
other => matches!(other, Ok(_) | Err(CdpError::DialogOpen(_))),
|
||||
}
|
||||
}
|
||||
|
||||
async fn give_back(key: String, pooled: PooledConnection) {
|
||||
@@ -995,11 +1028,22 @@ impl CdpTarget {
|
||||
/// Retries a connection that failed for a reason a retry could fix, and never
|
||||
/// one that failed because the answer was no.
|
||||
pub async fn connect(&self) -> Result<CdpConnection, CdpError> {
|
||||
self.connect_with(true).await
|
||||
}
|
||||
|
||||
/// A socket to the browser of a cloud session, on no page. On this computer,
|
||||
/// the page socket.
|
||||
pub(crate) async fn connect_browser(&self) -> Result<CdpConnection, CdpError> {
|
||||
self.connect_with(false).await
|
||||
}
|
||||
|
||||
async fn connect_with(&self, attach: bool) -> Result<CdpConnection, CdpError> {
|
||||
let mut attempt = 0u32;
|
||||
loop {
|
||||
let error = match self.connect_once().await {
|
||||
let error = match self.connect_once(attach).await {
|
||||
Ok(mut connection) => {
|
||||
connection.dialog = crate::page_dialog::state_of(&self.pool_key());
|
||||
connection.dialog =
|
||||
crate::page_dialog::state_of(&self.page_key(connection.page.as_deref()));
|
||||
return Ok(connection);
|
||||
}
|
||||
Err(e) => e,
|
||||
@@ -1018,7 +1062,7 @@ impl CdpTarget {
|
||||
}
|
||||
}
|
||||
|
||||
async fn connect_once(&self) -> Result<CdpConnection, CdpError> {
|
||||
async fn connect_once(&self, attach: bool) -> Result<CdpConnection, CdpError> {
|
||||
match self {
|
||||
Self::Local { ws_url } => {
|
||||
let request = ws_url
|
||||
@@ -1033,7 +1077,11 @@ impl CdpTarget {
|
||||
session_id,
|
||||
} => {
|
||||
let mut connection = dial_relay(ws_url, bearer).await?;
|
||||
if let Err(e) = connection.attach_to_page().await {
|
||||
if !attach {
|
||||
return Ok(connection);
|
||||
}
|
||||
let pinned = crate::page_dialog::remote_page(self);
|
||||
if let Err(e) = connection.attach_to_page(pinned).await {
|
||||
log::warn!(
|
||||
"Remote session page attach failed session={} err=\"{e}\"",
|
||||
ShortId(session_id)
|
||||
@@ -1208,6 +1256,19 @@ pub fn classify_handshake_status(status: u16) -> CdpError {
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn a_kept_socket_whose_page_closed_sends_again_on_a_new_one() {
|
||||
let gone = CdpError::Protocol(
|
||||
r#"{"code":-32001,"message":"Session with given id not found."}"#.to_string(),
|
||||
);
|
||||
assert!(not_delivered(&gone));
|
||||
assert!(!reusable(&Err(gone)));
|
||||
let refused =
|
||||
CdpError::Protocol(r#"{"code":-32000,"message":"Cannot find context"}"#.to_string());
|
||||
assert!(!not_delivered(&refused));
|
||||
assert!(reusable(&Err(refused)));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_page_target_is_preferred_over_the_devtools_frontend() {
|
||||
// Attaching to devtools:// drives the inspector, not the site, and the
|
||||
|
||||
+210
-31
@@ -400,6 +400,16 @@ pub struct McpError {
|
||||
data: Option<serde_json::Value>,
|
||||
}
|
||||
|
||||
impl McpError {
|
||||
/// The body a REST endpoint answers with for the same refusal: the
|
||||
/// structured detail, plus `message`.
|
||||
pub(crate) fn rest_body(&self) -> String {
|
||||
let mut body = self.data.clone().unwrap_or_else(|| serde_json::json!({}));
|
||||
body["message"] = serde_json::Value::from(self.message.as_str());
|
||||
body.to_string()
|
||||
}
|
||||
}
|
||||
|
||||
/// Surface a CDP failure to the agent with the reason intact.
|
||||
///
|
||||
/// The distinction matters to whoever is on the other end: "the session is
|
||||
@@ -4162,6 +4172,10 @@ async fn resolve_target(profile: &BrowserProfile) -> Result<CdpTarget, McpError>
|
||||
|
||||
/// Where a profile's browser is, whatever its page is doing.
|
||||
async fn locate_target(profile: &BrowserProfile) -> Result<CdpTarget, McpError> {
|
||||
#[cfg(test)]
|
||||
if let Some((_, target)) = pinned_profiles::get(&profile.id.to_string()) {
|
||||
return Ok(target);
|
||||
}
|
||||
crate::cdp_target::resolve(profile)
|
||||
.await
|
||||
.map_err(|e| McpError {
|
||||
@@ -4173,14 +4187,15 @@ async fn locate_target(profile: &BrowserProfile) -> Result<CdpTarget, McpError>
|
||||
|
||||
/// Watch the page for dialogs, and refuse it while one holds it: a dialog that
|
||||
/// is open, or a page that answers nothing. `unanswered_ok` lets a navigation
|
||||
/// through, which ends a dialog the watch never saw.
|
||||
async fn watch_page(target: &CdpTarget, unanswered_ok: bool) -> Result<(), McpError> {
|
||||
/// through, which ends a dialog the watch never saw. Every page tool and agent
|
||||
/// endpoint checks this first.
|
||||
pub(crate) async fn page_ready(target: &CdpTarget, unanswered_ok: bool) -> Result<(), AgentError> {
|
||||
use crate::page_dialog::PageState as Page;
|
||||
match crate::page_dialog::watch(target).await {
|
||||
Ok(Page::Ready) => Ok(()),
|
||||
Ok(Page::Dialog(dialog)) => Err(AgentError::Cdp(CdpError::DialogOpen(dialog)).into_mcp()),
|
||||
Ok(Page::Dialog(dialog)) => Err(AgentError::Cdp(CdpError::DialogOpen(dialog))),
|
||||
Ok(Page::Unanswered) if unanswered_ok => Ok(()),
|
||||
Ok(Page::Unanswered) => Err(AgentError::PageNotResponding.into_mcp()),
|
||||
Ok(Page::Unanswered) => Err(AgentError::PageNotResponding),
|
||||
// The tool reaches the page itself and reports what it finds there.
|
||||
Err(error) => {
|
||||
log::debug!("Page watch not opened err=\"{error}\"");
|
||||
@@ -4189,6 +4204,99 @@ async fn watch_page(target: &CdpTarget, unanswered_ok: bool) -> Result<(), McpEr
|
||||
}
|
||||
}
|
||||
|
||||
async fn watch_page(target: &CdpTarget, unanswered_ok: bool) -> Result<(), McpError> {
|
||||
page_ready(target, unanswered_ok)
|
||||
.await
|
||||
.map_err(AgentError::into_mcp)
|
||||
}
|
||||
|
||||
/// What `get_dialog` answers.
|
||||
#[derive(Debug, Clone, PartialEq, Serialize, ToSchema)]
|
||||
pub(crate) struct PageDialog {
|
||||
/// True while a dialog holds the page, false when none does, null when the
|
||||
/// page answers nothing and no dialog was seen opening on it.
|
||||
pub open: Option<bool>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub dialog: Option<crate::page_dialog::OpenDialog>,
|
||||
/// False when the page answers nothing.
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub page_responding: Option<bool>,
|
||||
/// What to do about a page that answers nothing.
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub note: Option<String>,
|
||||
}
|
||||
|
||||
/// The body of `handle_dialog`.
|
||||
#[derive(Debug, Clone, Deserialize, ToSchema)]
|
||||
pub(crate) struct AgentDialogRequest {
|
||||
/// True to accept the dialog (OK), false to dismiss it (Cancel).
|
||||
pub accept: bool,
|
||||
/// The text an accepted prompt dialog takes. Without it, the prompt keeps
|
||||
/// its default text.
|
||||
#[serde(default)]
|
||||
pub prompt_text: Option<String>,
|
||||
}
|
||||
|
||||
/// What `handle_dialog` did.
|
||||
#[derive(Debug, Clone, Serialize, ToSchema)]
|
||||
pub(crate) struct AgentDialogAnswer {
|
||||
pub accepted: bool,
|
||||
/// The dialog that was answered.
|
||||
pub dialog: crate::page_dialog::OpenDialog,
|
||||
/// A dialog the page opened in answer. It holds the page now.
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub next: Option<crate::page_dialog::OpenDialog>,
|
||||
}
|
||||
|
||||
/// Whether a dialog holds the page `target` names, and which. Never refused:
|
||||
/// it is how an agent learns why the page tools are.
|
||||
pub(crate) async fn agent_dialog(target: &CdpTarget) -> Result<PageDialog, AgentError> {
|
||||
use crate::page_dialog::PageState as Page;
|
||||
let state = crate::page_dialog::watch(target)
|
||||
.await
|
||||
.map_err(AgentError::Cdp)?;
|
||||
let closed = PageDialog {
|
||||
open: Some(false),
|
||||
dialog: None,
|
||||
page_responding: None,
|
||||
note: None,
|
||||
};
|
||||
Ok(match state {
|
||||
Page::Dialog(dialog) => PageDialog {
|
||||
open: Some(true),
|
||||
dialog: Some(dialog),
|
||||
..closed
|
||||
},
|
||||
Page::Ready => closed,
|
||||
Page::Unanswered => PageDialog {
|
||||
open: None,
|
||||
page_responding: Some(false),
|
||||
note: Some(AgentError::PageNotResponding.message()),
|
||||
..closed
|
||||
},
|
||||
})
|
||||
}
|
||||
|
||||
/// Accept or dismiss the dialog that holds the page `target` names.
|
||||
pub(crate) async fn agent_handle_dialog(
|
||||
target: &CdpTarget,
|
||||
request: &AgentDialogRequest,
|
||||
) -> Result<AgentDialogAnswer, AgentError> {
|
||||
use crate::page_dialog::AnswerError;
|
||||
let answered = crate::page_dialog::answer(target, request.accept, request.prompt_text.as_deref())
|
||||
.await
|
||||
.map_err(|error| match error {
|
||||
AnswerError::NoDialog => AgentError::NoDialog,
|
||||
AnswerError::Unanswered => AgentError::PageNotResponding,
|
||||
AnswerError::Cdp(error) => AgentError::Cdp(error),
|
||||
})?;
|
||||
Ok(AgentDialogAnswer {
|
||||
accepted: request.accept,
|
||||
dialog: answered.dialog,
|
||||
next: answered.next,
|
||||
})
|
||||
}
|
||||
|
||||
/// The tail of a page script that answers where `el` is, in viewport pixels.
|
||||
const RETURN_RECT_JS: &str = "const r = el.getBoundingClientRect(); return JSON.stringify({x: r.left + r.width / 2, y: r.top + r.height / 2, width: r.width, height: r.height});";
|
||||
|
||||
@@ -11183,6 +11291,10 @@ impl McpServer {
|
||||
/// the session. Whether a browser exists at all is [`resolve_cdp_target`]'s
|
||||
/// answer to give, because only it can see both places one could be.
|
||||
fn get_wayfern_profile(&self, profile_id: &str) -> Result<BrowserProfile, McpError> {
|
||||
#[cfg(test)]
|
||||
if let Some((profile, _)) = pinned_profiles::get(profile_id) {
|
||||
return Ok(profile);
|
||||
}
|
||||
let profiles = ProfileManager::instance()
|
||||
.list_profiles()
|
||||
.map_err(|e| McpError {
|
||||
@@ -12178,21 +12290,11 @@ impl McpServer {
|
||||
) -> Result<serde_json::Value, McpError> {
|
||||
let profile_id = Self::require_str(arguments, "profile_id")?;
|
||||
let target = self.locate_cdp_target(profile_id).await?;
|
||||
let state = crate::page_dialog::watch(&target)
|
||||
.await
|
||||
.map_err(cdp_error)?;
|
||||
let answer = match state {
|
||||
crate::page_dialog::PageState::Dialog(dialog) => {
|
||||
serde_json::json!({ "open": true, "dialog": dialog })
|
||||
}
|
||||
crate::page_dialog::PageState::Ready => serde_json::json!({ "open": false }),
|
||||
crate::page_dialog::PageState::Unanswered => serde_json::json!({
|
||||
"open": serde_json::Value::Null,
|
||||
"page_responding": false,
|
||||
"note": AgentError::PageNotResponding.message(),
|
||||
}),
|
||||
};
|
||||
Self::json_content(&answer)
|
||||
let dialog = agent_dialog(&target).await.map_err(|error| match error {
|
||||
AgentError::Cdp(error) => cdp_error(error),
|
||||
other => other.into_mcp(),
|
||||
})?;
|
||||
Self::json_content(&dialog)
|
||||
}
|
||||
|
||||
async fn handle_handle_dialog(
|
||||
@@ -12200,27 +12302,27 @@ impl McpServer {
|
||||
arguments: &serde_json::Value,
|
||||
) -> Result<serde_json::Value, McpError> {
|
||||
let profile_id = Self::require_str(arguments, "profile_id")?;
|
||||
let accept = arguments
|
||||
if arguments
|
||||
.get("accept")
|
||||
.and_then(serde_json::Value::as_bool)
|
||||
.ok_or_else(|| McpError {
|
||||
.is_none()
|
||||
{
|
||||
return Err(McpError {
|
||||
code: -32602,
|
||||
message: "Missing accept: true to accept the dialog (OK), false to dismiss it (Cancel)"
|
||||
.to_string(),
|
||||
data: None,
|
||||
})?;
|
||||
let prompt_text = arguments
|
||||
.get("prompt_text")
|
||||
.and_then(serde_json::Value::as_str);
|
||||
});
|
||||
}
|
||||
let request: AgentDialogRequest = Self::agent_arguments(arguments)?;
|
||||
let target = self.locate_cdp_target(profile_id).await?;
|
||||
let answered = crate::page_dialog::answer(&target, accept, prompt_text)
|
||||
let answered = agent_handle_dialog(&target, &request)
|
||||
.await
|
||||
.map_err(|error| match error {
|
||||
crate::page_dialog::AnswerError::NoDialog => AgentError::NoDialog.into_mcp(),
|
||||
crate::page_dialog::AnswerError::Unanswered => AgentError::PageNotResponding.into_mcp(),
|
||||
crate::page_dialog::AnswerError::Cdp(error) => cdp_error(error),
|
||||
AgentError::Cdp(error) => cdp_error(error),
|
||||
other => other.into_mcp(),
|
||||
})?;
|
||||
let report = answer_report(&answered.dialog, accept);
|
||||
let report = answer_report(&answered.dialog, request.accept);
|
||||
Ok(match &answered.next {
|
||||
Some(next) => serde_json::json!({
|
||||
"content": [{ "type": "text", "text": format!("{report}\nThe page opened another dialog. {}", dialog_refusal(next)) }]
|
||||
@@ -12865,7 +12967,7 @@ impl McpServer {
|
||||
}
|
||||
}
|
||||
|
||||
fn console_refusal(refusal: crate::agent_console::Refusal) -> McpError {
|
||||
pub(crate) fn console_refusal(refusal: crate::agent_console::Refusal) -> McpError {
|
||||
match refusal {
|
||||
crate::agent_console::Refusal::Paused { note } => McpError {
|
||||
code: -32000,
|
||||
@@ -13876,6 +13978,38 @@ fn cloud_failure_sentence(code: &str) -> Option<&'static str> {
|
||||
#[path = "mcp_server_browser_tests.rs"]
|
||||
mod browser_tests;
|
||||
|
||||
#[cfg(test)]
|
||||
#[path = "mcp_server_fleet_tests.rs"]
|
||||
mod fleet_tests;
|
||||
|
||||
/// Profiles a test drives without a profile on disk: the tools find the
|
||||
/// profile and its browser here first.
|
||||
#[cfg(test)]
|
||||
pub(crate) mod pinned_profiles {
|
||||
use super::{BrowserProfile, CdpTarget};
|
||||
use std::collections::HashMap;
|
||||
use std::sync::Mutex;
|
||||
|
||||
static PINNED: Mutex<Option<HashMap<String, (BrowserProfile, CdpTarget)>>> = Mutex::new(None);
|
||||
|
||||
pub(crate) fn pin(profile: BrowserProfile, target: CdpTarget) {
|
||||
let mut pinned = PINNED.lock().unwrap();
|
||||
pinned
|
||||
.get_or_insert_with(HashMap::new)
|
||||
.insert(profile.id.to_string(), (profile, target));
|
||||
}
|
||||
|
||||
pub(crate) fn unpin(profile_id: &str) {
|
||||
if let Some(pinned) = PINNED.lock().unwrap().as_mut() {
|
||||
pinned.remove(profile_id);
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn get(profile_id: &str) -> Option<(BrowserProfile, CdpTarget)> {
|
||||
PINNED.lock().unwrap().as_ref()?.get(profile_id).cloned()
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
@@ -16524,6 +16658,51 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn a_dialog_is_read_the_same_way_by_every_front_door() {
|
||||
use crate::page_dialog::{test_support, PageState as Page};
|
||||
use crate::wayfern_cdp::test_support::{fake_browser, Fake};
|
||||
|
||||
let (target, _) = fake_browser(Fake::Cooperative).await;
|
||||
let read = |state: Page| {
|
||||
let target = target.clone();
|
||||
async move {
|
||||
test_support::hold(&target, state);
|
||||
let read = serde_json::to_value(agent_dialog(&target).await.unwrap()).unwrap();
|
||||
let ready = page_ready(&target, false).await.err().map(|e| e.code());
|
||||
let navigable = page_ready(&target, true).await.err().map(|e| e.code());
|
||||
test_support::release(&target);
|
||||
(read, ready, navigable)
|
||||
}
|
||||
};
|
||||
assert_eq!(
|
||||
read(Page::Ready).await,
|
||||
(serde_json::json!({ "open": false }), None, None)
|
||||
);
|
||||
let (open, ready, navigable) =
|
||||
read(Page::Dialog(test_support::dialog("prompt", "Your name?"))).await;
|
||||
assert_eq!(open["open"], true);
|
||||
assert_eq!(open["dialog"]["type"], "prompt");
|
||||
assert_eq!(open["dialog"]["message"], "Your name?");
|
||||
assert_eq!(
|
||||
(ready, navigable),
|
||||
(Some("DIALOG_OPEN"), Some("DIALOG_OPEN"))
|
||||
);
|
||||
let (silent, ready, navigable) = read(Page::Unanswered).await;
|
||||
assert_eq!(silent["open"], serde_json::Value::Null);
|
||||
assert_eq!(silent["page_responding"], false);
|
||||
assert!(silent["note"].is_string());
|
||||
assert_eq!((ready, navigable), (Some("PAGE_NOT_RESPONDING"), None));
|
||||
|
||||
let refused = McpServer::console_refusal(crate::agent_console::Refusal::Paused {
|
||||
note: Some("lunch".into()),
|
||||
})
|
||||
.rest_body();
|
||||
let refused: serde_json::Value = serde_json::from_str(&refused).unwrap();
|
||||
assert_eq!(refused["code"], "AGENT_PAUSED");
|
||||
assert!(refused["message"].as_str().unwrap().contains("lunch"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn an_answer_report_says_what_was_answered() {
|
||||
let dialog = crate::page_dialog::test_support::dialog("confirm", "Delete it?");
|
||||
|
||||
@@ -0,0 +1,459 @@
|
||||
//! The dialog and typing tools on a real cloud session, reached through the
|
||||
//! cloud's CDP relay the way a profile run in the cloud is.
|
||||
//!
|
||||
//! `DONUT_FLEET_TEST_KEY` is a `dsk_` key of a Donut service account. The test
|
||||
//! makes a Linux profile named `fleet-dialogs` the first time, starts one
|
||||
//! session on it and stops that session whatever happens. A session costs
|
||||
//! fleet time, so the test is ignored unless asked for:
|
||||
//! `cargo test --lib fleet_ -- --ignored --nocapture`.
|
||||
|
||||
use super::*;
|
||||
use serde_json::{json, Value};
|
||||
use std::time::{Duration, Instant};
|
||||
|
||||
const PROFILE_NAME: &str = "fleet-dialogs";
|
||||
/// The longest the session may live if the test cannot stop it.
|
||||
const SESSION_CAP_SECONDS: u64 = 900;
|
||||
const LIVE_WAIT: Duration = Duration::from_secs(600);
|
||||
|
||||
struct Fleet {
|
||||
client: reqwest::Client,
|
||||
key: String,
|
||||
}
|
||||
|
||||
impl Fleet {
|
||||
fn from_env() -> Self {
|
||||
let key = std::env::var("DONUT_FLEET_TEST_KEY")
|
||||
.expect("DONUT_FLEET_TEST_KEY (a dsk_ key of a Donut service account) is required");
|
||||
Self {
|
||||
client: reqwest::Client::builder()
|
||||
.timeout(Duration::from_secs(120))
|
||||
.build()
|
||||
.unwrap(),
|
||||
key,
|
||||
}
|
||||
}
|
||||
|
||||
async fn call(&self, method: reqwest::Method, path: &str, body: Option<Value>) -> (u16, Value) {
|
||||
let mut request = self
|
||||
.client
|
||||
.request(
|
||||
method,
|
||||
format!("{}{path}", crate::cloud_auth::CLOUD_API_URL),
|
||||
)
|
||||
.bearer_auth(&self.key);
|
||||
if let Some(body) = body {
|
||||
request = request.json(&body);
|
||||
}
|
||||
let response = request.send().await.expect("the cloud API answers");
|
||||
let status = response.status().as_u16();
|
||||
let text = response.text().await.unwrap_or_default();
|
||||
(
|
||||
status,
|
||||
serde_json::from_str(&text).unwrap_or(Value::String(text)),
|
||||
)
|
||||
}
|
||||
|
||||
/// The test profile, made the first time.
|
||||
async fn profile_id(&self) -> String {
|
||||
let (status, listed) = self
|
||||
.call(
|
||||
reqwest::Method::GET,
|
||||
"/api/service/profiles?limit=100",
|
||||
None,
|
||||
)
|
||||
.await;
|
||||
assert_eq!(status, 200, "{listed}");
|
||||
if let Some(id) = listed["profiles"]
|
||||
.as_array()
|
||||
.into_iter()
|
||||
.flatten()
|
||||
.find(|p| p["name"] == PROFILE_NAME)
|
||||
.and_then(|p| p["id"].as_str())
|
||||
{
|
||||
return id.to_string();
|
||||
}
|
||||
let (status, created) = self
|
||||
.call(
|
||||
reqwest::Method::POST,
|
||||
"/api/service/profiles",
|
||||
Some(json!({ "name": PROFILE_NAME, "platform": "linux" })),
|
||||
)
|
||||
.await;
|
||||
assert_eq!(status, 201, "{created}");
|
||||
created["id"].as_str().unwrap().to_string()
|
||||
}
|
||||
|
||||
async fn start(&self, profile_id: &str) -> String {
|
||||
let started = Instant::now();
|
||||
let (status, session) = self
|
||||
.call(
|
||||
reqwest::Method::POST,
|
||||
"/api/remote-sessions",
|
||||
Some(json!({
|
||||
"profile_id": profile_id,
|
||||
"platform": "linux",
|
||||
"idempotency_key": uuid::Uuid::new_v4().to_string(),
|
||||
"max_duration_seconds": SESSION_CAP_SECONDS,
|
||||
})),
|
||||
)
|
||||
.await;
|
||||
assert_eq!(status, 201, "{session}");
|
||||
let id = session["session_id"].as_str().unwrap().to_string();
|
||||
eprintln!("fleet: session {id} started");
|
||||
while started.elapsed() < LIVE_WAIT {
|
||||
let (status, view) = self
|
||||
.call(
|
||||
reqwest::Method::GET,
|
||||
&format!("/api/remote-sessions/{id}"),
|
||||
None,
|
||||
)
|
||||
.await;
|
||||
if status == 200 && view["state"] == "live" && view["cdp_ready"] == true {
|
||||
eprintln!("fleet: live after {} s", started.elapsed().as_secs());
|
||||
return id;
|
||||
}
|
||||
if status != 200 || view["state"] == "closed" || view["state"] == "error" {
|
||||
self.stop(&id).await;
|
||||
panic!("the session did not come up: {status} {view}");
|
||||
}
|
||||
tokio::time::sleep(Duration::from_secs(3)).await;
|
||||
}
|
||||
self.stop(&id).await;
|
||||
panic!("the session was not live within {LIVE_WAIT:?}");
|
||||
}
|
||||
|
||||
async fn target(&self, session_id: &str) -> CdpTarget {
|
||||
let (status, endpoint) = self
|
||||
.call(
|
||||
reqwest::Method::GET,
|
||||
&format!("/api/remote-sessions/{session_id}/cdp"),
|
||||
None,
|
||||
)
|
||||
.await;
|
||||
assert_eq!(status, 200, "{endpoint}");
|
||||
CdpTarget::Remote {
|
||||
ws_url: endpoint["ws_url"].as_str().unwrap().to_string(),
|
||||
bearer: self.key.clone(),
|
||||
session_id: session_id.to_string(),
|
||||
}
|
||||
}
|
||||
|
||||
async fn stop(&self, session_id: &str) {
|
||||
let (status, stopped) = self
|
||||
.call(
|
||||
reqwest::Method::DELETE,
|
||||
&format!("/api/remote-sessions/{session_id}"),
|
||||
None,
|
||||
)
|
||||
.await;
|
||||
eprintln!(
|
||||
"fleet: session {session_id} stopped: {status} billed_seconds={}",
|
||||
stopped["billed_seconds"]
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
/// A button that asks to confirm, and a text field.
|
||||
const FIXTURE: &str = r#"(() => {
|
||||
document.title = 'Fleet dialogs';
|
||||
document.body.innerHTML = '<button id="delete" style="margin:40px;padding:20px" onclick="document.body.dataset.deleted = confirm(\'Delete it?\')">Delete</button><label>Later <input id="later"></label>';
|
||||
return document.visibilityState;
|
||||
})()"#;
|
||||
|
||||
struct Agent<'a> {
|
||||
server: &'a McpServer,
|
||||
profile_id: String,
|
||||
}
|
||||
|
||||
impl Agent<'_> {
|
||||
fn args(&self, mut arguments: Value) -> Value {
|
||||
arguments["profile_id"] = json!(self.profile_id);
|
||||
arguments
|
||||
}
|
||||
|
||||
async fn evaluate(&self, expression: &str) -> Result<Value, McpError> {
|
||||
self
|
||||
.server
|
||||
.handle_evaluate_javascript(&self.args(json!({ "expression": expression })))
|
||||
.await
|
||||
.map(|result| json_text(&result)["value"].clone())
|
||||
}
|
||||
|
||||
async fn navigate_and_build(&self) -> String {
|
||||
self
|
||||
.server
|
||||
.handle_navigate(&self.args(json!({ "url": "about:blank" })))
|
||||
.await
|
||||
.expect("navigate");
|
||||
self
|
||||
.evaluate(FIXTURE)
|
||||
.await
|
||||
.expect("the fixture is built")
|
||||
.as_str()
|
||||
.unwrap_or_default()
|
||||
.to_string()
|
||||
}
|
||||
|
||||
async fn get_dialog(&self) -> Value {
|
||||
json_text(
|
||||
&self
|
||||
.server
|
||||
.handle_get_dialog(&self.args(json!({})))
|
||||
.await
|
||||
.expect("get_dialog"),
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
fn text_of(result: &Value) -> String {
|
||||
result["content"][0]["text"]
|
||||
.as_str()
|
||||
.unwrap_or_default()
|
||||
.to_string()
|
||||
}
|
||||
|
||||
fn json_text(result: &Value) -> Value {
|
||||
serde_json::from_str(&text_of(result)).unwrap_or(Value::Null)
|
||||
}
|
||||
|
||||
fn code_of(error: &McpError) -> String {
|
||||
error
|
||||
.data
|
||||
.as_ref()
|
||||
.and_then(|data| data["code"].as_str())
|
||||
.unwrap_or_default()
|
||||
.to_string()
|
||||
}
|
||||
|
||||
async fn timed<T>(what: &str, call: impl std::future::Future<Output = T>) -> (T, Duration) {
|
||||
let started = Instant::now();
|
||||
let value = call.await;
|
||||
let elapsed = started.elapsed();
|
||||
eprintln!("fleet: {what} took {} ms", elapsed.as_millis());
|
||||
(value, elapsed)
|
||||
}
|
||||
|
||||
/// Run `method` on the browser itself, beside the page the tools drive.
|
||||
async fn on_browser(target: &CdpTarget, method: &str, params: Value) -> Value {
|
||||
let mut connection = target.connect().await.expect("a relay socket");
|
||||
connection
|
||||
.send_to(None, 77, method, params)
|
||||
.await
|
||||
.expect("sent to the browser");
|
||||
let reply = connection
|
||||
.await_reply(77, Duration::from_secs(20))
|
||||
.await
|
||||
.unwrap_or_else(|e| panic!("{method}: {e}"));
|
||||
connection.close().await;
|
||||
reply
|
||||
}
|
||||
|
||||
async fn browser_version(target: &CdpTarget) -> String {
|
||||
let version = on_browser(target, "Browser.getVersion", json!({})).await;
|
||||
version["product"]
|
||||
.as_str()
|
||||
.and_then(|product| product.split('/').nth(1))
|
||||
.unwrap_or("156.0.0.0")
|
||||
.to_string()
|
||||
}
|
||||
|
||||
async fn drive(target: CdpTarget) {
|
||||
let version = browser_version(&target).await;
|
||||
eprintln!("fleet: browser {version}");
|
||||
let profile = BrowserProfile {
|
||||
id: uuid::Uuid::new_v4(),
|
||||
name: "fleet test".to_string(),
|
||||
browser: "wayfern".to_string(),
|
||||
version,
|
||||
..Default::default()
|
||||
};
|
||||
let profile_id = profile.id.to_string();
|
||||
pinned_profiles::pin(profile, target.clone());
|
||||
let server = McpServer::new();
|
||||
let agent = Agent {
|
||||
server: &server,
|
||||
profile_id: profile_id.clone(),
|
||||
};
|
||||
|
||||
// a) A click that opens a confirm, and every page tool after it.
|
||||
agent.navigate_and_build().await;
|
||||
assert_eq!(agent.get_dialog().await, json!({ "open": false }));
|
||||
let (clicked, _) = timed(
|
||||
"click opening a confirm",
|
||||
server.handle_click_element(&agent.args(json!({ "selector": "#delete" }))),
|
||||
)
|
||||
.await;
|
||||
match &clicked {
|
||||
Ok(result) => eprintln!("fleet: click said {:?}", text_of(result)),
|
||||
Err(error) => eprintln!(
|
||||
"fleet: click refused {} {:?}",
|
||||
code_of(error),
|
||||
error.message
|
||||
),
|
||||
}
|
||||
let (refused, took) = timed("evaluate during the dialog", agent.evaluate("1 + 1")).await;
|
||||
let refused = refused.expect_err("a page held by a dialog refuses the tool");
|
||||
assert_eq!(code_of(&refused), "DIALOG_OPEN", "{refused:?}");
|
||||
assert!(took < Duration::from_millis(1_700), "refused in {took:?}");
|
||||
eprintln!("fleet: agent saw {:?}", refused.message);
|
||||
let (info, took) = timed(
|
||||
"get_page_info during the dialog",
|
||||
server.handle_get_page_info(&agent.args(json!({}))),
|
||||
)
|
||||
.await;
|
||||
assert_eq!(code_of(&info.expect_err("refused")), "DIALOG_OPEN");
|
||||
assert!(took < Duration::from_millis(1_700), "refused in {took:?}");
|
||||
let (read, _) = timed("get_dialog", agent.get_dialog()).await;
|
||||
assert_eq!(read["open"], true, "{read}");
|
||||
assert_eq!(read["dialog"]["type"], "confirm");
|
||||
assert_eq!(read["dialog"]["message"], "Delete it?");
|
||||
let (answered, _) = timed(
|
||||
"handle_dialog",
|
||||
server.handle_handle_dialog(&agent.args(json!({ "accept": false }))),
|
||||
)
|
||||
.await;
|
||||
let answered = text_of(&answered.expect("handle_dialog"));
|
||||
eprintln!("fleet: handle_dialog said {answered:?}");
|
||||
assert!(
|
||||
answered.starts_with("Dismissed a confirm dialog that says \"Delete it?\"."),
|
||||
"{answered}"
|
||||
);
|
||||
assert_eq!(
|
||||
agent
|
||||
.evaluate("document.body.dataset.deleted")
|
||||
.await
|
||||
.unwrap(),
|
||||
"false"
|
||||
);
|
||||
let none = server
|
||||
.handle_handle_dialog(&agent.args(json!({ "accept": true })))
|
||||
.await
|
||||
.expect_err("nothing to answer");
|
||||
assert_eq!(code_of(&none), "NO_DIALOG_OPEN");
|
||||
|
||||
// b) Another tab comes to the front and the agent's tab loads again behind
|
||||
// it: the text still reaches the field.
|
||||
let front = on_browser(
|
||||
&target,
|
||||
"Target.createTarget",
|
||||
json!({ "url": "about:blank" }),
|
||||
)
|
||||
.await;
|
||||
let front = front["targetId"].as_str().unwrap().to_string();
|
||||
let visibility = agent.navigate_and_build().await;
|
||||
assert_eq!(
|
||||
visibility, "hidden",
|
||||
"the agent's tab loaded behind the other"
|
||||
);
|
||||
let (typed, _) = timed(
|
||||
"type_locator in the tab behind",
|
||||
server.handle_type_locator(&agent.args(json!({
|
||||
"locator": { "role": "textbox", "name": "Later" },
|
||||
"text": "typed behind",
|
||||
}))),
|
||||
)
|
||||
.await;
|
||||
let typed = json_text(&typed.expect("typed"));
|
||||
eprintln!("fleet: typed_with={} {typed}", typed["typed_with"]);
|
||||
assert_eq!(typed["verified"], true, "{typed}");
|
||||
assert!(
|
||||
typed["typed_with"] == "key_events" || typed["typed_with"] == "browser_keyboard",
|
||||
"{typed}"
|
||||
);
|
||||
assert_eq!(
|
||||
agent
|
||||
.evaluate("document.querySelector('#later').value")
|
||||
.await
|
||||
.unwrap(),
|
||||
"typed behind"
|
||||
);
|
||||
let (typed, _) = timed(
|
||||
"type_text in the tab behind",
|
||||
server.handle_type_text(&agent.args(json!({
|
||||
"selector": "#later",
|
||||
"text": "by selector",
|
||||
}))),
|
||||
)
|
||||
.await;
|
||||
eprintln!(
|
||||
"fleet: type_text said {:?}",
|
||||
text_of(&typed.expect("typed"))
|
||||
);
|
||||
assert_eq!(
|
||||
agent
|
||||
.evaluate("document.querySelector('#later').value")
|
||||
.await
|
||||
.unwrap(),
|
||||
"by selector"
|
||||
);
|
||||
on_browser(&target, "Target.closeTarget", json!({ "targetId": front })).await;
|
||||
|
||||
// c) The agent's tab closes while a dialog holds it: the tab that is left
|
||||
// is driven, no dialog is reported on it, and its own dialogs are seen.
|
||||
agent
|
||||
.evaluate("setTimeout(() => alert('Closing'), 50); 1")
|
||||
.await
|
||||
.unwrap();
|
||||
tokio::time::sleep(Duration::from_millis(800)).await;
|
||||
assert_eq!(agent.get_dialog().await["dialog"]["message"], "Closing");
|
||||
let agents_tab = crate::page_dialog::remote_page(&target).expect("a pinned tab");
|
||||
on_browser(
|
||||
&target,
|
||||
"Target.createTarget",
|
||||
json!({ "url": "about:blank" }),
|
||||
)
|
||||
.await;
|
||||
on_browser(
|
||||
&target,
|
||||
"Target.closeTarget",
|
||||
json!({ "targetId": agents_tab }),
|
||||
)
|
||||
.await;
|
||||
tokio::time::sleep(Duration::from_millis(500)).await;
|
||||
let (after, _) = timed("get_dialog after the tab closed", agent.get_dialog()).await;
|
||||
assert_eq!(after, json!({ "open": false }), "{after}");
|
||||
let (title, _) = timed("evaluate on the tab left", agent.evaluate("1 + 1")).await;
|
||||
assert_eq!(title.expect("the tab left is driven"), 2);
|
||||
assert_ne!(
|
||||
crate::page_dialog::remote_page(&target).as_deref(),
|
||||
Some(agents_tab.as_str())
|
||||
);
|
||||
agent
|
||||
.evaluate("setTimeout(() => confirm('Again?'), 50); 1")
|
||||
.await
|
||||
.unwrap();
|
||||
tokio::time::sleep(Duration::from_millis(800)).await;
|
||||
let (refused, took) = timed(
|
||||
"evaluate during a dialog on the tab left",
|
||||
agent.evaluate("1 + 1"),
|
||||
)
|
||||
.await;
|
||||
assert_eq!(
|
||||
code_of(&refused.expect_err("refused")),
|
||||
"DIALOG_OPEN",
|
||||
"the tab left is watched"
|
||||
);
|
||||
assert!(took < Duration::from_millis(1_700), "refused in {took:?}");
|
||||
server
|
||||
.handle_handle_dialog(&agent.args(json!({ "accept": true })))
|
||||
.await
|
||||
.expect("the dialog on the tab left is answered");
|
||||
assert_eq!(agent.evaluate("1 + 1").await.expect("works again"), 2);
|
||||
|
||||
pinned_profiles::unpin(&profile_id);
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread")]
|
||||
#[ignore = "starts a cloud session: needs DONUT_FLEET_TEST_KEY"]
|
||||
async fn fleet_dialogs_and_typing_behind_another_tab() {
|
||||
let fleet = Fleet::from_env();
|
||||
let profile_id = fleet.profile_id().await;
|
||||
let session = fleet.start(&profile_id).await;
|
||||
let target = fleet.target(&session).await;
|
||||
let outcome = tokio::spawn(drive(target)).await;
|
||||
fleet.stop(&session).await;
|
||||
if let Err(failure) = outcome {
|
||||
std::panic::resume_unwind(failure.into_panic());
|
||||
}
|
||||
}
|
||||
+857
-110
File diff suppressed because it is too large.
Load diff
Reference in new issue
Block a user