From 1109a620eeee1ef455b7f0e49f510fde1879c785 Mon Sep 17 00:00:00 2001 From: RonniSkansing Date: Sat, 12 Sep 2026 19:17:20 +0200 Subject: [PATCH] add / update remote browser state helpers Signed-off-by: RonniSkansing --- backend/embedded/remotebrowser_prelude.js | 163 ++++++++++-------- backend/remotebrowser/prelude_test.go | 41 +++-- .../remote-browser/RemoteBrowserEditor.svelte | 128 ++++++++++---- 3 files changed, 213 insertions(+), 119 deletions(-) diff --git a/backend/embedded/remotebrowser_prelude.js b/backend/embedded/remotebrowser_prelude.js index a7e28d33..3ff51e93 100644 --- a/backend/embedded/remotebrowser_prelude.js +++ b/backend/embedded/remotebrowser_prelude.js @@ -4,50 +4,24 @@ // recognize each page once, then runs a loop that acts on the current page. // Loaded into the script VM before the user script, so these helpers are ready // when newSession() is called. Built only on the public session methods. +// +// Matchers, actions and hooks read and drive the page through the session s in +// their own scope; there is no separate page object. (function () { if (typeof newSession !== "function") { return; } var baseNewSession = newSession; - // pageInspector reads the current page. Passed to matchers and actions as p. - // p.url current URL as a plain string - // p.present(sel) the selector matches at least one node - // p.count(sel) how many nodes match - // p.text(sel) text of the first match - // p.visible(sel) the first match is rendered and not hidden - // p.query(name) value of a URL query parameter, decoded, or null - function pageInspector(s) { - var url = s.location(); - return { - url: url, - present: function (sel) { return s.getNodeCount(sel) > 0; }, - count: function (sel) { return s.getNodeCount(sel); }, - text: function (sel) { return s.getText(sel); }, - visible: function (sel) { - return s.evaluate( - "(function(){var e=document.querySelector(" + JSON.stringify(sel) + ");" + - "if(!e){return false;}var r=e.getBoundingClientRect();var st=getComputedStyle(e);" + - "return (r.width>0||r.height>0)&&st.visibility!=='hidden'&&st.display!=='none';})()" - ) === true; - }, - query: function (name) { - var key = String(name).replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); - var m = url.match(new RegExp("[?&]" + key + "=([^&]*)")); - return m ? decodeURIComponent(m[1]) : null; - } - }; - } - // firstMatch checks each rule in insertion order and returns the first state - // name whose matcher is truthy, or null when none match. - function firstMatch(s, rules) { - var p = pageInspector(s); + // name whose matcher is truthy, or null when none match. A matcher that throws + // counts as no match. + function firstMatch(rules) { var names = Object.keys(rules); for (var i = 0; i < names.length; i++) { var name = names[i]; var hit = false; - try { hit = !!rules[name](p); } catch (e) { hit = false; } + try { hit = !!rules[name](); } catch (e) { hit = false; } if (hit) { return name; } } return null; @@ -59,58 +33,107 @@ if (!timeoutMs) { timeoutMs = 10000; } var deadline = Date.now() + timeoutMs; while (true) { - var name = firstMatch(s, rules); + var name = firstMatch(rules); if (name) { return name; } if (Date.now() >= deadline) { return "timeout"; } s.wait(250); } } + // detectNext polls until a state other than prev matches, or the timeout runs + // out. prev is null on the first cycle, so any state counts. Waiting for a + // different state is what stops run from firing the same action twice while a + // page is still submitting or waiting for approval. + function detectNext(s, rules, prev, timeoutMs) { + if (!timeoutMs) { timeoutMs = 10000; } + var deadline = Date.now() + timeoutMs; + while (true) { + var name = firstMatch(rules); + if (name && name !== prev) { return name; } + if (Date.now() >= deadline) { return "timeout"; } + s.wait(250); + } + } + newSession = function (opts) { var s = baseNewSession(opts); - var declared = null; - // states declares the detection rules once: name -> matcher(p). - s.states = function (rules) { - declared = rules; - return s; + // present is true when the selector matches at least one node. + s.present = function (sel) { return s.getNodeCount(sel) > 0; }; + + // visible is true when the first match is rendered and not hidden. This is + // an instant check, unlike waitVisible which blocks. + s.visible = function (sel) { + return s.evaluate( + "(function(){var e=document.querySelector(" + JSON.stringify(sel) + ");" + + "if(!e){return false;}var r=e.getBoundingClientRect();var st=getComputedStyle(e);" + + "return (r.width>0||r.height>0)&&st.visibility!=='hidden'&&st.display!=='none';})()" + ) === true; }; - // waitForState returns the current state name using the given rules, or the - // rules declared with states(). Returns "timeout" if none match in time. + // query returns a URL query parameter value from the current page, decoded, + // or null when the parameter is absent. + s.query = function (name) { + var key = String(name).replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); + var m = s.location().match(new RegExp("[?&]" + key + "=([^&]*)")); + return m ? decodeURIComponent(m[1]) : null; + }; + + // waitForState returns the current state name for the given rules, or + // "timeout" if none match within timeoutMs (default 10000). Use it directly + // to run your own loop instead of states().run(). s.waitForState = function (rules, timeoutMs) { - return waitForState(s, rules || declared || {}, timeoutMs); + return waitForState(s, rules || {}, timeoutMs); }; - // run drives the loop: detect the current state, run its action, repeat. An - // action ends the loop by returning false or calling loop.stop(); any other - // return re-detects. The built in "timeout" state fires when nothing matched - // within detectTimeout. opts: { detectTimeout, timeout } in milliseconds. - s.run = function (actions, opts) { - if (!declared) { throw new Error("run: call states({...}) before run({...})"); } - opts = opts || {}; - var detectTimeout = opts.detectTimeout || 10000; - var overall = opts.timeout || 0; - var startedAt = Date.now(); - var stopped = false; - var loop = { state: null, stop: function () { stopped = true; } }; - - while (true) { - var state; - if (overall && Date.now() - startedAt > overall) { - state = "timeout"; - } else { - state = waitForState(s, declared, detectTimeout); + // states builds a state machine from the detection rules (name -> matcher). + // The returned machine carries optional before() and after() hooks and a + // run() loop, so a script reads as: s.states({...}).run({...}). + s.states = function (rules) { + var beforeFn = null; + var afterFn = null; + var machine = { + // before registers a callback run just before each detected step's + // action. It receives the state name. + before: function (fn) { beforeFn = fn; return machine; }, + // after registers a callback run just after each step's action. It + // receives the state name. + after: function (fn) { afterFn = fn; return machine; }, + // run drives the loop: detect the state, run its action, then wait for + // the state to change and run the next action. An action ends the loop + // by returning false or calling loop.stop(). The built in "timeout" + // state fires when the state does not change within detectTimeout. + // opts: { detectTimeout, timeout } in milliseconds. + run: function (actions, opts) { + opts = opts || {}; + var detectTimeout = opts.detectTimeout || 10000; + var overall = opts.timeout || 0; + var startedAt = Date.now(); + var stopped = false; + var loop = { state: null, stop: function () { stopped = true; } }; + var prev = null; + while (true) { + var state; + if (overall && Date.now() - startedAt > overall) { + state = "timeout"; + } else { + state = detectNext(s, rules, prev, detectTimeout); + } + loop.state = state; + var action = actions[state]; + if (!action) { + if (typeof log === "function") { log("[run] no action for state", { state: state }); } + return state; + } + if (beforeFn) { beforeFn(state); } + var result = action(loop); + if (afterFn) { afterFn(state); } + if (result === false || stopped) { return state; } + prev = state; + } } - loop.state = state; - var action = actions[state]; - if (!action) { - if (typeof log === "function") { log("[run] no action for state", { state: state }); } - return state; - } - var result = action(pageInspector(s), loop); - if (result === false || stopped) { return state; } - } + }; + return machine; }; return s; diff --git a/backend/remotebrowser/prelude_test.go b/backend/remotebrowser/prelude_test.go index 6cab611c..c96cc182 100644 --- a/backend/remotebrowser/prelude_test.go +++ b/backend/remotebrowser/prelude_test.go @@ -8,9 +8,10 @@ import ( ) // TestPreludeStateMachine runs the real prelude JS in a goja VM against a -// stubbed session and asserts states(), run(), and the loop control work. It -// checks the two mechanics the prelude relies on: reassigning the newSession -// global and adding methods to the session object from JS. +// stubbed session and asserts states() returns a machine whose before/after +// hooks and run loop work, and that present/visible/query were added to the +// session. It checks the mechanic the prelude relies on: reassigning the +// newSession global and augmenting the returned session from JS. func TestPreludeStateMachine(t *testing.T) { vm := goja.New() @@ -57,24 +58,34 @@ func TestPreludeStateMachine(t *testing.T) { script := ` var visited = []; + var beforeSeen = []; + var afterSeen = []; var s = newSession({}); if (typeof s.states !== "function") { throw new Error("s.states missing"); } - if (typeof s.run !== "function") { throw new Error("s.run missing"); } if (typeof s.waitForState !== "function") { throw new Error("s.waitForState missing"); } + if (typeof s.present !== "function") { throw new Error("s.present missing"); } + if (typeof s.visible !== "function") { throw new Error("s.visible missing"); } + if (typeof s.query !== "function") { throw new Error("s.query missing"); } - s.states({ - password: function (p) { return p.present("input[type=password]"); }, - totp: function (p) { return p.present("input[name=otc]"); }, - done: function (p) { return !p.url.includes("microsoftonline.com"); }, + var machine = s.states({ + password: function () { return s.present("input[type=password]"); }, + totp: function () { return s.present("input[name=otc]"); }, + done: function () { return !s.location().includes("microsoftonline.com"); }, }); + if (typeof machine.run !== "function") { throw new Error("machine.run missing"); } + if (typeof machine.before !== "function") { throw new Error("machine.before missing"); } + if (typeof machine.after !== "function") { throw new Error("machine.after missing"); } - s.run({ - password: function (p, loop) { visited.push("password"); advance(); }, - totp: function (p, loop) { visited.push("totp"); advance(); }, - done: function (p, loop) { visited.push("done"); loop.stop(); }, - }, { detectTimeout: 1000 }); + machine + .before(function (state) { beforeSeen.push(state); }) + .after(function (state) { afterSeen.push(state); }) + .run({ + password: function (loop) { visited.push("password"); advance(); }, + totp: function (loop) { visited.push("totp"); advance(); }, + done: function (loop) { visited.push("done"); loop.stop(); }, + }, { detectTimeout: 1000 }); - visited.join(","); + visited.join(",") + "|" + beforeSeen.join(",") + "|" + afterSeen.join(","); ` // advance() bumps the Go stage counter so the stub page moves forward. @@ -88,7 +99,7 @@ func TestPreludeStateMachine(t *testing.T) { t.Fatalf("script failed: %v", err) } got := v.String() - want := "password,totp,done" + want := "password,totp,done|password,totp,done|password,totp,done" if got != want { t.Fatalf("state walk = %q, want %q (logs: %v)", got, want, logs) } diff --git a/frontend/src/lib/components/remote-browser/RemoteBrowserEditor.svelte b/frontend/src/lib/components/remote-browser/RemoteBrowserEditor.svelte index 90c4fff6..e0779a56 100644 --- a/frontend/src/lib/components/remote-browser/RemoteBrowserEditor.svelte +++ b/frontend/src/lib/components/remote-browser/RemoteBrowserEditor.svelte @@ -326,23 +326,7 @@ interface RaceCondition { after?: number; } -/** Reads the current page. Passed to state matchers and run actions as p. */ -interface PageInspector { - /** Current URL as a plain string */ - url: string; - /** The selector matches at least one node */ - present(selector: string): boolean; - /** How many nodes match the selector */ - count(selector: string): number; - /** Text of the first match */ - text(selector: string): string; - /** The first match is rendered and not hidden */ - visible(selector: string): boolean; - /** Value of a URL query parameter, decoded, or null */ - query(name: string): string | null; -} - -/** Controls the loop started by run(). Passed to each action as the 2nd argument. */ +/** Controls the loop started by run(). Passed to each action. */ interface RunLoop { /** Name of the state currently being handled */ state: string; @@ -351,12 +335,27 @@ interface RunLoop { } interface RunOptions { - /** Milliseconds to wait for a state to match each cycle (default 10000) */ + /** Milliseconds to wait each cycle for the state to change (default 10000) */ detectTimeout?: number; /** Overall budget in milliseconds for the whole loop (0 = no limit) */ timeout?: number; } +/** A state machine built by s.states(rules). Add before()/after() hooks, then run(). */ +interface StateMachine { + /** Register a callback run just before each detected step's action. Receives the state name. */ + before(fn: (state: string) => void): StateMachine; + /** Register a callback run just after each step's action. Receives the state name. */ + after(fn: (state: string) => void): StateMachine; + /** + * Run the loop: detect the state, run its action, then wait for the state to + * change and run the next action. Each action receives the loop control. An + * action ends the loop by returning false or calling loop.stop(). The built + * in "timeout" state fires when the state does not change within detectTimeout. + */ + run(actions: { [state: string]: (loop: RunLoop) => any }, options?: RunOptions): string; +} + interface Session { // ── Navigation ──────────────────────────────────────────────────────────── /** Navigate to a URL and wait for the page to load */ @@ -434,6 +433,17 @@ interface Session { */ moveMouse(x: number, y: number, opts?: { duration?: number; jitter?: number }): void; scrollIntoView(selector: string): void; + /** + * Scroll the page by deltaY CSS pixels (positive = down) using eased, + * jittered mouse-wheel steps instead of an instant jump. + */ + humanScroll(deltaY: number, options?: { duration?: number }): void; + /** + * Spend about ms milliseconds producing low-amplitude pointer drift and + * pauses, the way a person rests a hand on the mouse. Builds natural + * behavioural signal on pages that treat inactivity as a bot tell. + */ + humanIdle(ms: number): void; // ── Keyboard ────────────────────────────────────────────────────────────── /** Focus the element and type text character by character */ @@ -466,6 +476,12 @@ interface Session { setJSAttribute(selector: string, prop: string, value: string): void; /** Count elements matching selector */ getNodeCount(selector: string): number; + /** True when the selector matches at least one node */ + present(selector: string): boolean; + /** True when the first match is rendered and not hidden (instant, unlike waitVisible) */ + visible(selector: string): boolean; + /** Value of a URL query parameter from the current page, decoded, or null */ + query(name: string): string | null; // ── JavaScript evaluation ───────────────────────────────────────────────── /** Evaluate a JS expression in the page context and return the result */ @@ -524,6 +540,18 @@ interface Session { withTimeout(ms: number, fn: (s: Session) => void): boolean; close(): void; + // ── State machine ───────────────────────────────────────────────────────── + /** + * Build a state machine from detection rules (state name -> matcher). + * Returns the machine; add before()/after() hooks and call run() on it. + */ + states(rules: { [state: string]: () => boolean }): StateMachine; + /** + * Return the current page state name for the given rules, or "timeout" if none + * match within timeoutMs (default 10000). Use it to run your own loop. + */ + waitForState(rules: { [state: string]: () => boolean }, timeoutMs?: number): string; + // ── Event-driven API ───────────────────────────────────────────────────── /** * Register a handler for a named event. Must be called before s.listen(). @@ -603,6 +631,17 @@ interface FrameSession { clickXY(x: number, y: number): void; moveMouse(x: number, y: number, opts?: { duration?: number; jitter?: number }): void; scrollIntoView(selector: string): void; + /** + * Scroll the page by deltaY CSS pixels (positive = down) using eased, + * jittered mouse-wheel steps instead of an instant jump. + */ + humanScroll(deltaY: number, options?: { duration?: number }): void; + /** + * Spend about ms milliseconds producing low-amplitude pointer drift and + * pauses, the way a person rests a hand on the mouse. Builds natural + * behavioural signal on pages that treat inactivity as a bot tell. + */ + humanIdle(ms: number): void; // ── Keyboard ────────────────────────────────────────────────────────────── sendKeys(selector: string, text: string): void; @@ -665,22 +704,6 @@ interface FrameSession { */ withTimeout(ms: number, fn: (s: FrameSession) => void): boolean; - // ── State machine ───────────────────────────────────────────────────────── - /** Declare how to recognize each page: state name -> matcher. Call before run(). */ - states(rules: { [state: string]: (p: PageInspector) => boolean }): Session; - /** - * Return the current page state name, or "timeout" if none match within - * timeoutMs (default 10000). Uses the given rules, or those set with states(). - */ - waitForState(rules?: { [state: string]: (p: PageInspector) => boolean }, timeoutMs?: number): string; - /** - * Run the state loop: detect the current state, run its action, repeat. - * An action ends the loop by returning false or calling loop.stop(); any - * other return re-detects. The built in "timeout" state fires when nothing - * matched within detectTimeout. - */ - run(actions: { [state: string]: (p: PageInspector, loop: RunLoop) => any }, options?: RunOptions): string; - // ── Nested iframes ──────────────────────────────────────────────────────── /** Scope a sub-session to a nested iframe within this frame. Returns null if not found. */ frame(selector: string): FrameSession | null; @@ -705,6 +728,7 @@ declare function waitForEvent(event: string): any; */ declare function stop(): never; /** Block until any of the listed victim events arrive; returns { event, data } */ +declare function waitForAny(events: string[]): { event: string; data: any }; declare function waitForAny(...events: string[]): { event: string; data: any }; interface RetryContext { @@ -727,6 +751,42 @@ declare function retry(options: { max: number; wait?: number }, fn: (ctx: RetryC // ECMAScript built-ins available in the goja runtime (ES2015+). // (No DOM, no Node.js — those are not available in scripts.) +// Instance methods for the string, number and boolean primitives. The String, +// Number and Boolean call forms are declared further down; these interfaces are +// what give a string value methods like indexOf, includes and split. +interface String { + readonly length: number; + charAt(index: number): string; + charCodeAt(index: number): number; + indexOf(searchValue: string, fromIndex?: number): number; + lastIndexOf(searchValue: string, fromIndex?: number): number; + includes(searchValue: string, fromIndex?: number): boolean; + startsWith(searchValue: string, fromIndex?: number): boolean; + endsWith(searchValue: string, endPosition?: number): boolean; + slice(start?: number, end?: number): string; + substring(start: number, end?: number): string; + toLowerCase(): string; + toUpperCase(): string; + trim(): string; + padStart(targetLength: number, padString?: string): string; + padEnd(targetLength: number, padString?: string): string; + repeat(count: number): string; + split(separator: string | RegExp, limit?: number): string[]; + replace(searchValue: string | RegExp, replaceValue: string): string; + replaceAll(searchValue: string | RegExp, replaceValue: string): string; + match(regexp: string | RegExp): string[] | null; + concat(...strings: string[]): string; + [index: number]: string; +} +interface Number { + toFixed(digits?: number): string; + toString(radix?: number): string; + valueOf(): number; +} +interface Boolean { + valueOf(): boolean; + toString(): string; +} declare var JSON: { parse(text: string): any; stringify(value: any, replacer?: any, space?: string | number): string;