add right click / copy / paste

Signed-off-by: Ronni Skansing <rskansing@gmail.com>
This commit is contained in:
Ronni Skansing
2026-05-25 11:29:19 +02:00
parent 4a90888550
commit b940603e64
4 changed files with 284 additions and 22 deletions
+86 -7
View File
@@ -1170,8 +1170,12 @@ func (m *RemoteBrowserController) StreamLiveSession(g *gin.Context) {
return
}
var header struct {
Type string `json:"type"`
TargetID string `json:"targetID"`
Type string `json:"type"`
TargetID string `json:"targetID"`
X1 float64 `json:"x1"`
Y1 float64 `json:"y1"`
X2 float64 `json:"x2"`
Y2 float64 `json:"y2"`
}
if json.Unmarshal(msg, &header) != nil {
continue
@@ -1196,6 +1200,64 @@ func (m *RemoteBrowserController) StreamLiveSession(g *gin.Context) {
// removes the entry and switches to a fallback tab if needed.
proto.TargetCloseTarget{TargetID: proto.TargetTargetID(header.TargetID)}.Call(entry.page.Browser()) //nolint:errcheck
}
case "select_range":
if controlMode {
x1, y1, x2, y2 := header.X1, header.Y1, header.X2, header.Y2
go func() {
p := getActivePage()
if p == nil {
return
}
// Use caretRangeFromPoint to locate the exact text
// positions at the drag start and end, then build a
// Range and hand it to the Selection API. This is
// far more reliable than synthesised mousemove events
// for triggering Chrome's text selection engine.
js := fmt.Sprintf(`(function(){
var r1=document.caretRangeFromPoint(%f,%f);
var r2=document.caretRangeFromPoint(%f,%f);
if(!r1||!r2)return;
var range=document.createRange();
try{
if(r1.compareBoundaryPoints(Range.START_TO_START,r2)<=0){
range.setStart(r1.startContainer,r1.startOffset);
range.setEnd(r2.startContainer,r2.startOffset);
}else{
range.setStart(r2.startContainer,r2.startOffset);
range.setEnd(r1.startContainer,r1.startOffset);
}
var sel=window.getSelection();
sel.removeAllRanges();
sel.addRange(range);
}catch(e){}
})()`, x1, y1, x2, y2)
p.Eval("() => " + js) //nolint:errcheck
}()
}
case "get_selection":
if controlMode {
go func() {
p := getActivePage()
if p == nil {
return
}
res, err := p.Eval(`() => window.getSelection().toString()`)
if err != nil || res == nil {
return
}
payload, marshalErr := json.Marshal(map[string]string{
"type": "selection_text",
"text": res.Value.Str(),
})
if marshalErr != nil {
return
}
select {
case notifyCh <- payload:
default:
}
}()
}
default:
if controlMode {
m.dispatchInput(getActivePage(), msg)
@@ -1275,6 +1337,7 @@ func (m *RemoteBrowserController) dispatchInput(page *rod.Page, msg []byte) {
X float64 `json:"x"`
Y float64 `json:"y"`
Button string `json:"button"`
Buttons int64 `json:"buttons"` // bitmask of held buttons (left=1, right=2, middle=4)
DeltaX float64 `json:"deltaX"`
DeltaY float64 `json:"deltaY"`
Key string `json:"key"`
@@ -1293,9 +1356,7 @@ func (m *RemoteBrowserController) dispatchInput(page *rod.Page, msg []byte) {
btn = proto.InputMouseButtonRight
}
mods := int(cmd.Modifiers)
// Shared Buttons bitmask values for pointer events.
zeroButtons := 0
oneButton := 1
nowTs := func() proto.TimeSinceEpoch {
return proto.TimeSinceEpoch(float64(time.Now().UnixNano()) / 1e9)
}
@@ -1306,17 +1367,35 @@ func (m *RemoteBrowserController) dispatchInput(page *rod.Page, msg []byte) {
// check), while still adding the ±1 px variation that breaks exact-integer paths.
jx := math.Round(cmd.X + (rand.Float64()*2-1)*0.5)
jy := math.Round(cmd.Y + (rand.Float64()*2-1)*0.5)
// Forward the browser's e.buttons bitmask so CDP sees held buttons during a drag.
// When the frontend does not send buttons (older messages), cmd.Buttons == 0 which
// is the correct "no button held" value for a plain mousemove.
heldButtons := int(cmd.Buttons)
// During a drag the CDP `button` field must name the held button so
// Chrome's text-selection engine recognises it as a drag-select, not a
// plain hover. Bit 1 = left, bit 2 = right.
moveButton := proto.InputMouseButtonNone
if heldButtons&1 != 0 {
moveButton = proto.InputMouseButtonLeft
} else if heldButtons&2 != 0 {
moveButton = proto.InputMouseButtonRight
}
proto.InputDispatchMouseEvent{
Type: proto.InputDispatchMouseEventTypeMouseMoved,
X: jx,
Y: jy,
Modifiers: mods,
Timestamp: nowTs(),
Button: proto.InputMouseButtonNone,
Buttons: &zeroButtons,
Button: moveButton,
Buttons: &heldButtons,
PointerType: proto.InputDispatchMouseEventPointerTypeMouse,
}.Call(page) //nolint:errcheck
case "mousedown":
// Buttons bitmask must match the button being pressed: left=1, right=2.
downButtons := 1
if cmd.Button == "right" {
downButtons = 2
}
proto.InputDispatchMouseEvent{
Type: proto.InputDispatchMouseEventTypeMousePressed,
X: cmd.X,
@@ -1324,7 +1403,7 @@ func (m *RemoteBrowserController) dispatchInput(page *rod.Page, msg []byte) {
Modifiers: mods,
Timestamp: nowTs(),
Button: btn,
Buttons: &oneButton,
Buttons: &downButtons,
ClickCount: 1,
PointerType: proto.InputDispatchMouseEventPointerTypeMouse,
}.Call(page) //nolint:errcheck
+84
View File
@@ -649,6 +649,90 @@ func RegisterBrowserBindings(vm *goja.Runtime, pc *goja.Object, page *rod.Page,
return goja.Undefined()
})
pc.Set("rightClick", func(call goja.FunctionCall) goja.Value {
sel := argStr(call.Argument(0))
dbg("→ rightClick " + sel)
target := findPage(sel)
if target == page {
el, err := page.Element(sel)
must(err)
must(el.ScrollIntoView())
posRes, posErr := el.Eval(`function(){var r=this.getBoundingClientRect();return[r.left+r.width/2,r.top+r.height/2]}`)
must(posErr)
pts := posRes.Value.Arr()
cx := math.Round(pts[0].Num())
cy := math.Round(pts[1].Num())
twoB, zeroB := 2, 0
proto.InputDispatchMouseEvent{
Type: proto.InputDispatchMouseEventTypeMousePressed,
X: cx, Y: cy,
Timestamp: proto.TimeSinceEpoch(float64(time.Now().UnixNano()) / 1e9),
Button: proto.InputMouseButtonRight,
Buttons: &twoB,
ClickCount: 1,
PointerType: proto.InputDispatchMouseEventPointerTypeMouse,
}.Call(page) //nolint:errcheck
time.Sleep(time.Duration(80+rand.Intn(70)) * time.Millisecond)
proto.InputDispatchMouseEvent{
Type: proto.InputDispatchMouseEventTypeMouseReleased,
X: cx, Y: cy,
Timestamp: proto.TimeSinceEpoch(float64(time.Now().UnixNano()) / 1e9),
Button: proto.InputMouseButtonRight,
Buttons: &zeroB,
ClickCount: 1,
PointerType: proto.InputDispatchMouseEventPointerTypeMouse,
}.Call(page) //nolint:errcheck
} else {
must(evalInFrame(target, fmt.Sprintf(
"var el=document.querySelector(%q);if(el)el.dispatchEvent(new MouseEvent('contextmenu',{bubbles:true,cancelable:true,button:2,buttons:2}))",
sel)))
}
dbg("✓ rightClick " + sel)
return goja.Undefined()
})
pc.Set("rightClickXY", func(call goja.FunctionCall) goja.Value {
x := math.Round(call.Argument(0).ToFloat())
y := math.Round(call.Argument(1).ToFloat())
dbg(fmt.Sprintf("→ rightClickXY %.0f,%.0f", x, y))
twoB, zeroB := 2, 0
proto.InputDispatchMouseEvent{
Type: proto.InputDispatchMouseEventTypeMousePressed,
X: x, Y: y,
Timestamp: proto.TimeSinceEpoch(float64(time.Now().UnixNano()) / 1e9),
Button: proto.InputMouseButtonRight,
Buttons: &twoB,
ClickCount: 1,
PointerType: proto.InputDispatchMouseEventPointerTypeMouse,
}.Call(page) //nolint:errcheck
time.Sleep(time.Duration(80+rand.Intn(70)) * time.Millisecond)
proto.InputDispatchMouseEvent{
Type: proto.InputDispatchMouseEventTypeMouseReleased,
X: x, Y: y,
Timestamp: proto.TimeSinceEpoch(float64(time.Now().UnixNano()) / 1e9),
Button: proto.InputMouseButtonRight,
Buttons: &zeroB,
ClickCount: 1,
PointerType: proto.InputDispatchMouseEventPointerTypeMouse,
}.Call(page) //nolint:errcheck
dbg(fmt.Sprintf("✓ rightClickXY %.0f,%.0f", x, y))
return goja.Undefined()
})
pc.Set("selectText", func(call goja.FunctionCall) goja.Value {
sel := argStr(call.Argument(0))
dbg("→ selectText " + sel)
stmt := fmt.Sprintf(
`var el=document.querySelector(%q);if(!el)return;`+
`if(typeof el.select==='function'){el.focus();el.select();}else{`+
`var r=document.createRange();r.selectNodeContents(el);`+
`var s=window.getSelection();s.removeAllRanges();s.addRange(r);}`,
sel)
must(evalInFrame(findPage(sel), stmt))
dbg("✓ selectText " + sel)
return goja.Undefined()
})
// mouseX/Y tracks the last position dispatched by humanMoveTo so that
// consecutive moveMouse and clickXY calls produce a continuous path.
var mouseX, mouseY float64
@@ -381,6 +381,15 @@ interface Session {
// ── Mouse ─────────────────────────────────────────────────────────────────
click(selector: string): void;
doubleClick(selector: string): void;
/** Right-click an element by selector, triggering its context menu. */
rightClick(selector: string): void;
/** Right-click at absolute page coordinates. */
rightClickXY(x: number, y: number): void;
/**
* Select all text content of an element.
* Works on inputs/textareas (uses .select()) and general DOM nodes (uses Selection API).
*/
selectText(selector: string): void;
/**
* Move to absolute coordinates along a curved Bezier path with micro-jitter
* before clicking. Prefer this over bare clickXY when bot detection is a concern.
@@ -540,6 +549,9 @@ interface FrameSession {
// ── Mouse ─────────────────────────────────────────────────────────────────
click(selector: string): void;
doubleClick(selector: string): void;
rightClick(selector: string): void;
rightClickXY(x: number, y: number): void;
selectText(selector: string): void;
clickXY(x: number, y: number): void;
moveMouse(x: number, y: number, opts?: { duration?: number; jitter?: number }): void;
scrollIntoView(selector: string): void;
@@ -31,6 +31,16 @@
let injectEvent = '';
let injectData = '';
/** @type {{ x: number, y: number } | null} */
let contextMenu = null;
// drag-select tracking
let dragStartX = 0;
let dragStartY = 0;
let dragStartCanvasX = 0;
let dragStartCanvasY = 0;
let isDragging = false;
function onLogPanelWheel(e) {
if (e.deltaY < 0) logScrolledUp = true;
}
@@ -151,6 +161,10 @@
if (!urlBarFocused) urlBarValue = msg.value;
} else if (msg.type === 'tabs') {
tabs = msg.tabs || [];
} else if (msg.type === 'selection_text') {
if (msg.text) {
navigator.clipboard.writeText(msg.text).catch(() => {});
}
} else if (msg.type === 'closed') {
status = 'Session ended';
sessionClosed = true;
@@ -226,28 +240,53 @@
const scaleX = remoteWidth / rect.width;
const scaleY = remoteHeight / rect.height;
return {
x: Math.round((e.clientX - rect.left) * scaleX),
y: Math.round((e.clientY - rect.top) * scaleY)
x: Math.max(0, Math.min(remoteWidth, Math.round((e.clientX - rect.left) * scaleX))),
y: Math.max(0, Math.min(remoteHeight, Math.round((e.clientY - rect.top) * scaleY)))
};
}
function onMouseMove(e) {
if (!controlMode) return;
const { x, y } = canvasCoords(e);
sendInput({ type: 'mousemove', x, y });
}
function onMouseDown(e) {
function onPointerDown(e) {
if (!controlMode) return;
if (e.pointerType !== 'mouse') return;
e.preventDefault();
// Capture the pointer so pointermove and pointerup keep firing on this
// element even when the mouse leaves it - required for drag-to-select.
canvas.setPointerCapture(e.pointerId);
const { x, y } = canvasCoords(e);
if (e.button === 0) {
dragStartX = e.clientX;
dragStartY = e.clientY;
dragStartCanvasX = x;
dragStartCanvasY = y;
isDragging = false;
}
sendInput({ type: 'mousedown', x, y, button: e.button === 2 ? 'right' : 'left' });
}
function onMouseUp(e) {
function onPointerMove(e) {
if (!controlMode) return;
if (e.pointerType !== 'mouse') return;
const { x, y } = canvasCoords(e);
if (e.buttons === 1) {
const dx = e.clientX - dragStartX;
const dy = e.clientY - dragStartY;
if (dx * dx + dy * dy > 25) isDragging = true;
}
sendInput({ type: 'mousemove', x, y, buttons: e.buttons });
}
function onPointerUp(e) {
if (!controlMode) return;
if (e.pointerType !== 'mouse') return;
const { x, y } = canvasCoords(e);
sendInput({ type: 'mouseup', x, y, button: e.button === 2 ? 'right' : 'left' });
// Left-button drag ended - use caretRangeFromPoint in the remote browser
// to set the exact selection, since CDP mousemove events alone are
// unreliable for triggering Chrome's text selection engine.
if (e.button === 0 && isDragging) {
sendInput({ type: 'select_range', x1: dragStartCanvasX, y1: dragStartCanvasY, x2: x, y2: y });
}
isDragging = false;
}
function onWheel(e) {
@@ -257,6 +296,33 @@
sendInput({ type: 'scroll', x, y, deltaX: e.deltaX, deltaY: e.deltaY });
}
function onContextMenu(e) {
if (!controlMode) return;
e.preventDefault();
contextMenu = { x: e.clientX, y: e.clientY };
}
function closeContextMenu() {
contextMenu = null;
}
function copyFromRemote() {
closeContextMenu();
sendInput({ type: 'get_selection' });
}
async function pasteToRemote() {
closeContextMenu();
try {
const text = await navigator.clipboard.readText();
if (text) sendInput({ type: 'paste', text });
} catch {
// clipboard access denied - fall back to Ctrl+V
sendInput({ type: 'keydown', key: 'v', code: 'KeyV', keyCode: 86, modifiers: 2, charText: '' });
sendInput({ type: 'keyup', key: 'v', code: 'KeyV', keyCode: 86, modifiers: 2 });
}
}
function mods(e) {
return (e.altKey ? 1 : 0) | (e.ctrlKey ? 2 : 0) | (e.metaKey ? 4 : 0) | (e.shiftKey ? 8 : 0);
}
@@ -457,14 +523,35 @@
<canvas
bind:this={canvas}
class="max-w-full max-h-full object-contain"
style={controlMode ? 'cursor: crosshair;' : ''}
on:mousemove={onMouseMove}
on:mousedown={onMouseDown}
on:mouseup={onMouseUp}
style={controlMode ? 'cursor: crosshair; touch-action: none;' : ''}
on:pointerdown={onPointerDown}
on:pointermove={onPointerMove}
on:pointerup={onPointerUp}
on:wheel|nonpassive={onWheel}
on:contextmenu|preventDefault
on:contextmenu={onContextMenu}
/>
{#if contextMenu}
<!-- svelte-ignore a11y-click-events-have-key-events -->
<!-- svelte-ignore a11y-no-static-element-interactions -->
<div class="fixed inset-0 z-40" on:click={closeContextMenu} on:contextmenu|preventDefault={closeContextMenu}></div>
<div
class="fixed z-50 bg-gray-800 border border-gray-600 rounded shadow-xl py-1 min-w-32 text-sm"
style="left: {contextMenu.x}px; top: {contextMenu.y}px;"
>
<button
type="button"
class="w-full text-left px-3 py-1.5 text-gray-200 hover:bg-gray-700 transition-colors"
on:click={copyFromRemote}
>Copy</button>
<button
type="button"
class="w-full text-left px-3 py-1.5 text-gray-200 hover:bg-gray-700 transition-colors"
on:click={pasteToRemote}
>Paste</button>
</div>
{/if}
{#if logPanelOpen}
<div
class="absolute bottom-0 left-0 right-0 flex flex-col bg-gray-950/95 border-t border-gray-700 rounded-b"