From 596857a75c5cfbdd5ff5d749564f4087f2ffe884 Mon Sep 17 00:00:00 2001 From: RonniSkansing Date: Fri, 24 Jul 2026 02:56:30 +0200 Subject: [PATCH] fix remote browser session view while streaming element to recipient Signed-off-by: RonniSkansing --- backend/controller/remoteBrowser.go | 415 +++++++++++++---------- backend/embedded/remotebrowser_inject.js | 45 ++- 2 files changed, 276 insertions(+), 184 deletions(-) diff --git a/backend/controller/remoteBrowser.go b/backend/controller/remoteBrowser.go index ed40afb..6cb0b6d 100644 --- a/backend/controller/remoteBrowser.go +++ b/backend/controller/remoteBrowser.go @@ -3,7 +3,7 @@ package controller import ( "bytes" "context" - "encoding/base64" + "encoding/binary" "encoding/json" "fmt" "image" @@ -56,6 +56,16 @@ type activeSession struct { // browserPage is set (non-nil) only after newSession() is called. browserPageMu sync.Mutex browserPage *rod.Page + // operator screencast fan-out: when an operator is viewing this session, the + // latest full-page screencast frame is published here so victim element streams + // crop from it instead of running a second capture on the same target — two + // capture mechanisms on one Chrome target corrupt each other (the operator would + // otherwise see only the element's clipped region). + opMu sync.RWMutex + opPageID proto.TargetTargetID + opFrame []byte + opFrameDevW float64 + opFrameDevH float64 } func (a *activeSession) GetCampaignID() uuid.UUID { return a.CampaignID } @@ -74,6 +84,36 @@ func (a *activeSession) setBrowserPage(page *rod.Page) { a.browserPage = page } +// publishOperatorFrame stores the latest operator screencast frame for pageID so +// victim element streams on that page can crop from it. +func (a *activeSession) publishOperatorFrame(pageID proto.TargetTargetID, data []byte, devW, devH float64) { + a.opMu.Lock() + a.opPageID = pageID + a.opFrame = data + a.opFrameDevW, a.opFrameDevH = devW, devH + a.opMu.Unlock() +} + +// clearOperatorFrames stops publishing operator frames when the viewer disconnects. +func (a *activeSession) clearOperatorFrames() { + a.opMu.Lock() + a.opPageID = "" + a.opFrame = nil + a.opMu.Unlock() +} + +// operatorFrameFor returns the latest operator frame and viewport dimensions when an +// operator is currently screencasting pageID, so an element stream can crop from it +// instead of issuing its own capture on that target. +func (a *activeSession) operatorFrameFor(pageID proto.TargetTargetID) ([]byte, float64, float64, bool) { + a.opMu.RLock() + defer a.opMu.RUnlock() + if a.opPageID == "" || a.opPageID != pageID || len(a.opFrame) == 0 { + return nil, 0, 0, false + } + return a.opFrame, a.opFrameDevW, a.opFrameDevH, true +} + // streamInfo tracks a named cropped stream started by s.stream(selector, name). // originX/Y are the element's CSS-pixel top-left corner (for input coord mapping). // scaleX/Y are JPEG pixels per CSS pixel, computed from the first frame received @@ -723,7 +763,7 @@ func (m *RemoteBrowserController) ServeVictim(g *gin.Context) { streamCtx, streamCancel := context.WithCancel(cmd.Page.GetContext()) si := &streamInfo{cancel: streamCancel, maxFps: cmd.MaxFps, quality: cmd.Quality} activeNamedStreams.Store(cmd.Name, si) - go m.runNamedStream(streamCtx, cmd.Page, &connMu, conn, cmd.Selector, cmd.Name, si) + go m.runNamedStream(streamCtx, cmd.Page, sess, &connMu, conn, cmd.Selector, cmd.Name, si) } else if cmd.Op == "stop" { if val, exists := activeNamedStreams.LoadAndDelete(cmd.Name); exists { val.(*streamInfo).cancel() @@ -934,6 +974,9 @@ func (m *RemoteBrowserController) StreamLiveSession(g *gin.Context) { return } defer conn.Close() + // Stop feeding element streams once this operator stops viewing, so they fall + // back to capturing the element region directly. + defer sess.clearOperatorFrames() conn.SetReadLimit(64 * 1024) // Derived from outerCtx so the stream also ends when the victim session ends. @@ -1360,6 +1403,11 @@ func (m *RemoteBrowserController) StreamLiveSession(g *gin.Context) { frameW = frame.Metadata.DeviceWidth frameH = frame.Metadata.DeviceHeight } + // Publish the frame so victim element streams on this page crop from it + // instead of running a competing capture on the same target. + if ap := getActivePage(); ap != nil { + sess.publishOperatorFrame(ap.TargetID, frame.Data, frameW, frameH) + } // The CSS viewport dimensions (used client side for input coordinate // mapping) change rarely, so send them as a small control message only on // change instead of wrapping every frame. The JPEG itself goes as a raw @@ -1735,9 +1783,10 @@ func (m *RemoteBrowserController) saveSubmitEvent( } } -// cropImage crops an already-decoded image and returns base64 JPEG at the given quality (1-100). -// quality 0 means use the default (92). -func cropImage(src image.Image, x, y, w, h, quality int) (string, error) { +// cropImageJPEG crops src to the given region and returns raw JPEG bytes. Used to +// crop the streamed element out of an operator screencast frame so the victim only +// ever receives the element, never the full page. quality 0 uses a sane default. +func cropImageJPEG(src image.Image, x, y, w, h, quality int) ([]byte, error) { b := src.Bounds() if x < b.Min.X { x = b.Min.X @@ -1752,19 +1801,32 @@ func cropImage(src image.Image, x, y, w, h, quality int) (string, error) { h = b.Max.Y - y } if w <= 0 || h <= 0 { - return "", fmt.Errorf("crop region out of bounds") + return nil, fmt.Errorf("crop region out of bounds") } dst := image.NewRGBA(image.Rect(0, 0, w, h)) draw.Draw(dst, dst.Bounds(), src, image.Pt(x, y), draw.Src) - var buf bytes.Buffer q := quality if q <= 0 || q > 100 { - q = 92 + q = 85 } + var buf bytes.Buffer if err := jpeg.Encode(&buf, dst, &jpeg.Options{Quality: q}); err != nil { - return "", err + return nil, err } - return base64.StdEncoding.EncodeToString(buf.Bytes()), nil + return buf.Bytes(), nil +} + +// buildStreamFrameMsg frames a cropped element JPEG for the victim WebSocket as a +// single binary message: [type=1][uint16 name length][name][jpeg bytes]. Sending the +// JPEG raw avoids the base64 inflation and per-frame JSON wrapping of a text message. +func buildStreamFrameMsg(name string, jpegData []byte) []byte { + nameBytes := []byte(name) + buf := make([]byte, 3+len(nameBytes)+len(jpegData)) + buf[0] = 1 + binary.BigEndian.PutUint16(buf[1:3], uint16(len(nameBytes))) + copy(buf[3:], nameBytes) + copy(buf[3+len(nameBytes):], jpegData) + return buf } // runNamedStream queries the element CSS bounding rect, then streams cropped frames to @@ -1776,6 +1838,7 @@ func cropImage(src image.Image, x, y, w, h, quality int) (string, error) { func (m *RemoteBrowserController) runNamedStream( streamCtx context.Context, page *rod.Page, + sess *activeSession, connMu *sync.Mutex, conn *websocket.Conn, selector string, @@ -1802,69 +1865,154 @@ func (m *RemoteBrowserController) runNamedStream( } si.setOrigin(cssRect.X, cssRect.Y) - // displayW/H: CSS pixel size sent to the victim canvas for layout. - // Locked to the element's size at stream-start time; updated only when - // the element itself genuinely resizes (cssRectChanged), NOT when - // EmulateViewport causes responsive-layout reflow that changes cssRect.W/H. - displayW := int(cssRect.W) - displayH := int(cssRect.H) + // displayW/H: element CSS-pixel size sent to the victim canvas for layout. + // imageW/H: the JPEG pixel size Chrome produces for the clip (CSS size times the + // capture scale). Both are refreshed whenever the element resizes. + displayW := int(cssRect.W + 0.5) + displayH := int(cssRect.H + 0.5) streamPage := page.Context(streamCtx) - frameCh := make(chan *proto.PageScreencastFrame, 4) - wait := streamPage.EachEvent(func(e *proto.PageScreencastFrame) (stop bool) { - select { - case frameCh <- e: - default: - } - return - }) - go wait() - nsQ, nsW, nsH, nsN := 85, 3840, 2160, 1 - namedStreamScreencast := proto.PageStartScreencast{ - Format: proto.PageStartScreencastFormatJpeg, - Quality: &nsQ, - MaxWidth: &nsW, - MaxHeight: &nsH, - EveryNthFrame: &nsN, - } - if err := namedStreamScreencast.Call(streamPage); err != nil { - return - } - // page (not streamPage) must be used here: streamCtx is already cancelled when this defer - // runs, so a StopScreencast on streamPage would never reach Chrome. - defer proto.PageStopScreencast{}.Call(page) //nolint:errcheck - - var minInterval time.Duration - if si.maxFps > 0 { - minInterval = time.Second / time.Duration(si.maxFps) - } - var lastFrameSent time.Time - - // cropX/Y/W/H are in JPEG pixels, recomputed whenever the JPEG dimensions or - // the viewport (DeviceWidth/Height) change. The viewport can change mid-stream - // when EmulateViewport is applied after the victim sends their window size. - var cropX, cropY, cropW, cropH int - var lastJpegW, lastJpegH int - var lastDevW, lastDevH float64 // track viewport to detect changes - var lastRectCheck time.Time // throttle for periodic element-size polling - - requeryCSSRect := func(devW, devH float64) { - res, err := page.Eval(fmt.Sprintf(`() => (function(){var el=document.querySelector(%q);if(!el)return null;var r=el.getBoundingClientRect();return JSON.stringify({x:r.left,y:r.top,w:r.width,h:r.height})})()`, selector)) + // scale renders the clip at the remote page's device pixel ratio so the streamed + // element stays crisp on HiDPI or emulated viewports. Clamped to a sane range. + readScale := func() float64 { + res, err := streamPage.Eval(`() => window.devicePixelRatio || 1`) if err != nil { + return 1 + } + s := res.Value.Num() + if s < 1 { + s = 1 + } + if s > 3 { + s = 3 + } + return s + } + scale := readScale() + si.setScale(scale, scale) + + imageW := int(cssRect.W*scale + 0.5) + imageH := int(cssRect.H*scale + 0.5) + + quality := si.quality + if quality <= 0 || quality > 100 { + quality = 85 + } + + // fps caps how often the element is captured. Unbounded (0) would spin the capture + // loop, so default to a smooth but cheap rate for a single element. + fps := si.maxFps + if fps <= 0 { + fps = 30 + } + + // sendStreamStart tells the victim page the current image and CSS dimensions so it + // can size its canvas. Sent up front and again whenever the dimensions change. + sendStreamStart := func() { + startPayload, _ := json.Marshal(map[string]interface{}{ + "type": "stream_start", + "name": name, + "width": imageW, + "height": imageH, + "cssWidth": displayW, + "cssHeight": displayH, + }) + connMu.Lock() + conn.WriteMessage(websocket.TextMessage, startPayload) //nolint:errcheck + connMu.Unlock() + } + sendStreamStart() + + // requeryRect re-reads the element rect and device pixel ratio, updating the crop + // origin and the capture scale used by the direct-capture path. + requeryRect := func() { + res, err := streamPage.Eval(fmt.Sprintf(`() => (function(){var el=document.querySelector(%q);if(!el)return null;var r=el.getBoundingClientRect();return JSON.stringify({x:r.left,y:r.top,w:r.width,h:r.height,dpr:window.devicePixelRatio||1})})()`, selector)) + if err != nil || res.Value.Str() == "" || res.Value.Str() == "null" { return } - if res.Value.Str() == "" || res.Value.Str() == "null" { + var r struct{ X, Y, W, H, Dpr float64 } + if err := json.Unmarshal([]byte(res.Value.Str()), &r); err != nil || r.W <= 0 || r.H <= 0 { return } - var r struct{ X, Y, W, H float64 } - if err := json.Unmarshal([]byte(res.Value.Str()), &r); err != nil || r.W <= 0 { - return + scale = r.Dpr + if scale < 1 { + scale = 1 } - cssRect = r + if scale > 3 { + scale = 3 + } + cssRect = struct{ X, Y, W, H float64 }{r.X, r.Y, r.W, r.H} si.setOrigin(cssRect.X, cssRect.Y) } + // captureElement returns the current element JPEG plus its pixel dimensions. When + // an operator is screencasting this page it crops the element out of the operator's + // frame — running our own capture on the same target would corrupt the operator's + // full-page view. Otherwise it captures just the element region directly, which is + // cheap and leaves the server doing no image codec work. + captureElement := func() ([]byte, int, int, bool) { + if opData, devW, devH, ok := sess.operatorFrameFor(page.TargetID); ok { + src, decErr := jpeg.Decode(bytes.NewReader(opData)) + if decErr != nil { + return nil, 0, 0, false + } + jw, jh := src.Bounds().Dx(), src.Bounds().Dy() + if devW <= 0 { + devW = float64(jw) + } + if devH <= 0 { + devH = float64(jh) + } + sx, sy := float64(jw)/devW, float64(jh)/devH + si.setOrigin(cssRect.X, cssRect.Y) + si.setScale(sx, sy) + cw := int(cssRect.W*sx + 0.5) + ch := int(cssRect.H*sy + 0.5) + cropped, cropErr := cropImageJPEG(src, int(cssRect.X*sx), int(cssRect.Y*sy), cw, ch, quality) + if cropErr != nil { + return nil, 0, 0, false + } + return cropped, cw, ch, true + } + si.setOrigin(cssRect.X, cssRect.Y) + si.setScale(scale, scale) + shot, shotErr := proto.PageCaptureScreenshot{ + Format: proto.PageCaptureScreenshotFormatJpeg, + Quality: &quality, + Clip: &proto.PageViewport{ + X: cssRect.X, + Y: cssRect.Y, + Width: cssRect.W, + Height: cssRect.H, + Scale: scale, + }, + OptimizeForSpeed: true, + }.Call(streamPage) + if shotErr != nil { + return nil, 0, 0, false + } + return shot.Data, int(cssRect.W*scale + 0.5), int(cssRect.H*scale + 0.5), true + } + + // Adaptive polling: capture at fastInterval while the element is changing and back + // off to idleInterval once it has been static for a short streak. Interactive + // elements stay smooth while a static one barely costs anything; the rate snaps + // back to fast the instant a frame differs. + fastInterval := time.Second / time.Duration(fps) + idleInterval := 200 * time.Millisecond + if idleInterval < fastInterval { + idleInterval = fastInterval + } + const backoffAfter = 10 // consecutive unchanged frames before slowing down + + ticker := time.NewTicker(fastInterval) + defer ticker.Stop() + atFastRate := true + unchanged := 0 + var lastRectCheck time.Time + var lastFrame []byte // last JPEG sent, to skip resending unchanged frames + for { select { case <-streamCtx.Done(): @@ -1873,127 +2021,48 @@ func (m *RemoteBrowserController) runNamedStream( conn.WriteMessage(websocket.TextMessage, stopPayload) //nolint:errcheck connMu.Unlock() return - case frame, ok := <-frameCh: - if !ok { - return - } - // Always ack to prevent CDP screencast stalling. - proto.PageScreencastFrameAck{SessionID: frame.SessionID}.Call(page) //nolint:errcheck - // Throttle: drop frames that arrive faster than maxFps. - if minInterval > 0 && !lastFrameSent.IsZero() && time.Since(lastFrameSent) < minInterval { - continue - } - lastFrameSent = time.Now() - - var devW, devH float64 - if frame.Metadata != nil { - devW = frame.Metadata.DeviceWidth - devH = frame.Metadata.DeviceHeight - } - - // Decode JPEG once; reuse for both scale computation and cropping. - src, err := jpeg.Decode(bytes.NewReader(frame.Data)) - if err != nil { - continue - } - jpegW := src.Bounds().Max.X - jpegH := src.Bounds().Max.Y - - if devW <= 0 { - devW = float64(jpegW) - } - if devH <= 0 { - devH = float64(jpegH) - } - - viewportChanged := devW != lastDevW || devH != lastDevH - jpegDimsChanged := jpegW != lastJpegW || jpegH != lastJpegH - - // When the viewport changes (e.g. EmulateViewport applied after victim connects), - // re-query the element's bounding rect — its CSS position and size may have - // changed due to responsive layout reflow. - cssRectChanged := false - if viewportChanged { - lastDevW, lastDevH = devW, devH - oldX, oldY, oldW, oldH := cssRect.X, cssRect.Y, cssRect.W, cssRect.H - requeryCSSRect(devW, devH) - if cssRect.X != oldX || cssRect.Y != oldY || cssRect.W != oldW || cssRect.H != oldH { - cssRectChanged = true - } - } - - // Periodically re-query the element rect to detect size changes caused by - // CSS transitions, popups expanding, or other dynamic layout shifts. - // Skip when a viewport change already triggered a re-query this frame. - if !viewportChanged && cropW > 0 && time.Since(lastRectCheck) >= 250*time.Millisecond { + case <-ticker.C: + // Periodically re-check the element rect for moves or resizes (CSS + // transitions, responsive reflow, popups). + if time.Since(lastRectCheck) >= 250*time.Millisecond { lastRectCheck = time.Now() - oldX, oldY, oldW, oldH := cssRect.X, cssRect.Y, cssRect.W, cssRect.H - requeryCSSRect(devW, devH) - if cssRect.X != oldX || cssRect.Y != oldY || cssRect.W != oldW || cssRect.H != oldH { - cssRectChanged = true - } + requeryRect() } - - // Recompute scale-aware crop rect whenever JPEG dimensions, viewport, or - // the element's own CSS dimensions change. - if jpegDimsChanged || viewportChanged || cssRectChanged { - lastJpegW, lastJpegH = jpegW, jpegH - - scaleX := float64(jpegW) / devW - scaleY := float64(jpegH) / devH - si.setScale(scaleX, scaleY) - - cropX = int(cssRect.X * scaleX) - cropY = int(cssRect.Y * scaleY) - cropW = int(cssRect.W * scaleX) - cropH = int(cssRect.H * scaleY) - - // Update canvas display size only when the element itself resized, - // not when a viewport change triggers responsive-layout reflow. - if cssRectChanged { - displayW = int(cssRect.W) - displayH = int(cssRect.H) - } - - if cropW <= 0 || cropH <= 0 { - continue - } - // cssWidth/cssHeight: stable CSS display size (locked to initial element - // size, updated only on genuine element resize). width/height are the - // JPEG crop buffer dimensions, which can differ on HiDPI displays. - startPayload, _ := json.Marshal(map[string]interface{}{ - "type": "stream_start", - "name": name, - "width": cropW, - "height": cropH, - "cssWidth": displayW, - "cssHeight": displayH, - }) - connMu.Lock() - conn.WriteMessage(websocket.TextMessage, startPayload) //nolint:errcheck - connMu.Unlock() - } - - if cropW <= 0 || cropH <= 0 { + if cssRect.W <= 0 || cssRect.H <= 0 { continue } - - cropped, err := cropImage(src, cropX, cropY, cropW, cropH, si.quality) - if err != nil { + data, iw, ih, ok := captureElement() + if !ok { continue } - payload, err := json.Marshal(map[string]interface{}{ - "type": "stream_frame", - "name": name, - "frame": cropped, - "width": cropW, - "height": cropH, - }) - if err != nil { + // Notify the victim when the image or CSS dimensions change — element + // resize, or a switch between the direct-capture and operator-crop paths. + dispW := int(cssRect.W + 0.5) + dispH := int(cssRect.H + 0.5) + if iw != imageW || ih != imageH || dispW != displayW || dispH != displayH { + imageW, imageH = iw, ih + displayW, displayH = dispW, dispH + sendStreamStart() + } + // Skip the send when the frame is byte-for-byte unchanged (JPEG of identical + // pixels is deterministic) and slow the capture rate after a static streak; + // resume fast capture the instant a frame differs. + if bytes.Equal(data, lastFrame) { + unchanged++ + if unchanged == backoffAfter && atFastRate { + atFastRate = false + ticker.Reset(idleInterval) + } continue } + unchanged = 0 + if !atFastRate { + atFastRate = true + ticker.Reset(fastInterval) + } + lastFrame = data connMu.Lock() - writeErr := conn.WriteMessage(websocket.TextMessage, payload) + writeErr := conn.WriteMessage(websocket.BinaryMessage, buildStreamFrameMsg(name, data)) connMu.Unlock() if writeErr != nil { return diff --git a/backend/embedded/remotebrowser_inject.js b/backend/embedded/remotebrowser_inject.js index 139bc46..7a766db 100644 --- a/backend/embedded/remotebrowser_inject.js +++ b/backend/embedded/remotebrowser_inject.js @@ -1,6 +1,7 @@ (function () { var wsProto = window.location.protocol === 'https:' ? 'wss:' : 'ws:'; var ws = new WebSocket(wsProto + '//' + window.location.host + '/__WS_PATH__/__CR_ID__/__RB_ID__'); + ws.binaryType = 'arraybuffer'; // stream frames arrive as raw binary JPEG messages var h = {}; // event handlers keyed as "e:eventName" or "stream_start:name" etc. var streams = {}; // name → {canvas, w, h, cssW, cssH, autoSize, el} var streamLastStart = {} // name → last stream_start message, so mountStream called late still sizes correctly @@ -28,7 +29,40 @@ } } + // handleStreamFrame draws a binary frame message: [type=1][uint16 nameLen][name][jpeg]. + // Decoding runs off the main thread via createImageBitmap; if newer frames arrive + // while one is decoding they replace the pending one so only the latest is drawn. + function handleStreamFrame(buf) { + var dv = new DataView(buf); + if (dv.getUint8(0) !== 1) return; + var nameLen = dv.getUint16(1); + var name = new TextDecoder().decode(new Uint8Array(buf, 3, nameLen)); + var st = streams[name]; + if (!st) return; + st.pending = new Uint8Array(buf, 3 + nameLen); + if (st.decoding) return; + st.decoding = true; + (function drain() { + var bytes = st.pending; + st.pending = null; + createImageBitmap(new Blob([bytes], { type: 'image/jpeg' })).then(function (bmp) { + if (st.canvas.width !== bmp.width) { st.canvas.width = bmp.width; st.w = bmp.width; } + if (st.canvas.height !== bmp.height) { st.canvas.height = bmp.height; st.h = bmp.height; } + st.canvas.getContext('2d').drawImage(bmp, 0, 0); + bmp.close(); + if (st.pending) { drain(); } else { st.decoding = false; } + }).catch(function () { + if (st.pending) { drain(); } else { st.decoding = false; } + }); + })(); + } + ws.onmessage = function (e) { + // Binary messages are raw JPEG stream frames. + if (typeof e.data !== 'string') { + handleStreamFrame(e.data); + return; + } try { var m = JSON.parse(e.data); @@ -56,17 +90,6 @@ } } - } else if (m.type === 'stream_frame' && m.name) { - var st = streams[m.name]; - if (!st) return; - var img = new Image(); - img.onload = function () { - if (st.canvas.width !== img.naturalWidth) { st.canvas.width = img.naturalWidth; st.w = img.naturalWidth; } - if (st.canvas.height !== img.naturalHeight) { st.canvas.height = img.naturalHeight; st.h = img.naturalHeight; } - st.canvas.getContext('2d').drawImage(img, 0, 0); - }; - img.src = 'data:image/jpeg;base64,' + m.frame; - } else if (m.type === 'done') { (h['e:done'] || []).forEach(function (f) { f(); });