From d7efc3859334945492af728da9b59d77aa5c17f0 Mon Sep 17 00:00:00 2001 From: Adam Shiervani Date: Mon, 24 Mar 2025 11:43:22 +0100 Subject: [PATCH] refactor(WebRTCVideo): improve mouse event handling and video playback logic --- ui/src/components/WebRTCVideo.tsx | 223 ++++++++++++++++++------------ ui/src/routes/devices.$id.tsx | 4 + 2 files changed, 136 insertions(+), 91 deletions(-) diff --git a/ui/src/components/WebRTCVideo.tsx b/ui/src/components/WebRTCVideo.tsx index 29c72d1..de36e37 100644 --- a/ui/src/components/WebRTCVideo.tsx +++ b/ui/src/components/WebRTCVideo.tsx @@ -94,7 +94,7 @@ export default function WebRTCVideo() { ); // Mouse-related - const calcDelta = (pos: number) => Math.abs(pos) < 10 ? pos * 2 : pos; + const calcDelta = (pos: number) => (Math.abs(pos) < 10 ? pos * 2 : pos); const sendRelMouseMovement = useCallback( (x: number, y: number, buttons: number) => { if (settings.mouseMode !== "relative") return; @@ -168,7 +168,14 @@ export default function WebRTCVideo() { const { buttons } = e; sendAbsMouseMovement(x, y, buttons); }, - [sendAbsMouseMovement, videoClientHeight, videoClientWidth, videoWidth, videoHeight, settings.mouseMode], + [ + sendAbsMouseMovement, + videoClientHeight, + videoClientWidth, + videoWidth, + videoHeight, + settings.mouseMode, + ], ); const trackpadSensitivity = useDeviceSettingsStore(state => state.trackpadSensitivity); @@ -355,28 +362,6 @@ export default function WebRTCVideo() { ], ); - // Effect hooks - useEffect( - function setupKeyboardEvents() { - const abortController = new AbortController(); - const signal = abortController.signal; - - document.addEventListener("keydown", keyDownHandler, { signal }); - document.addEventListener("keyup", keyUpHandler, { signal }); - - // eslint-disable-next-line @typescript-eslint/ban-ts-comment - // @ts-expect-error - window.clearKeys = () => sendKeyboardEvent([], []); - window.addEventListener("blur", resetKeyboardState, { signal }); - document.addEventListener("visibilitychange", resetKeyboardState, { signal }); - - return () => { - abortController.abort(); - }; - }, - [keyDownHandler, keyUpHandler, resetKeyboardState, sendKeyboardEvent], - ); - const videoKeyUpHandler = useCallback((e: KeyboardEvent) => { // In fullscreen mode in chrome & safari, the space key is used to pause/play the video // there is no way to prevent this, so we need to simply force play the video when it's paused. @@ -389,71 +374,6 @@ export default function WebRTCVideo() { } }, []); - useEffect( - function setupVideoEventListeners() { - let videoElmRefValue = null; - if (!videoElm.current) return; - videoElmRefValue = videoElm.current; - const abortController = new AbortController(); - const signal = abortController.signal; - - videoElmRefValue.addEventListener("mousemove", absMouseMoveHandler, { signal }); - videoElmRefValue.addEventListener("pointerdown", absMouseMoveHandler, { signal }); - videoElmRefValue.addEventListener("pointerup", absMouseMoveHandler, { signal }); - videoElmRefValue.addEventListener("keyup", videoKeyUpHandler, { signal }); - videoElmRefValue.addEventListener("wheel", mouseWheelHandler, { - signal, - passive: true, - }); - videoElmRefValue.addEventListener( - "contextmenu", - (e: MouseEvent) => e.preventDefault(), - { signal }, - ); - videoElmRefValue.addEventListener("playing", onVideoPlaying, { signal }); - - const local = resetMousePosition; - window.addEventListener("blur", local, { signal }); - document.addEventListener("visibilitychange", local, { signal }); - - return () => { - if (videoElmRefValue) abortController.abort(); - }; - }, - [ - absMouseMoveHandler, - resetMousePosition, - onVideoPlaying, - mouseWheelHandler, - videoKeyUpHandler, - ], - ); - - useEffect( - function setupRelativeMouseEventListeners() { - if (settings.mouseMode !== "relative") return; - - const abortController = new AbortController(); - const signal = abortController.signal; - - // bind to body to capture all mouse events - const body = document.querySelector("body"); - if (!body) return; - - body.addEventListener("mousemove", relMouseMoveHandler, { signal }); - body.addEventListener("pointerdown", relMouseMoveHandler, { signal }); - body.addEventListener("pointerup", relMouseMoveHandler, { signal }); - - return () => { - abortController.abort(); - - body.removeEventListener("mousemove", relMouseMoveHandler); - body.removeEventListener("pointerdown", relMouseMoveHandler); - body.removeEventListener("pointerup", relMouseMoveHandler); - }; - }, [settings.mouseMode, relMouseMoveHandler], - ) - useEffect( function updateVideoStream() { if (!mediaStream) return; @@ -476,6 +396,120 @@ export default function WebRTCVideo() { ], ); + // Setup Keyboard Events + useEffect( + function setupKeyboardEvents() { + const abortController = new AbortController(); + const signal = abortController.signal; + + document.addEventListener("keydown", keyDownHandler, { signal }); + document.addEventListener("keyup", keyUpHandler, { signal }); + + // eslint-disable-next-line @typescript-eslint/ban-ts-comment + // @ts-expect-error + window.clearKeys = () => sendKeyboardEvent([], []); + window.addEventListener("blur", resetKeyboardState, { signal }); + document.addEventListener("visibilitychange", resetKeyboardState, { signal }); + + return () => { + abortController.abort(); + }; + }, + [keyDownHandler, keyUpHandler, resetKeyboardState, sendKeyboardEvent], + ); + + useEffect( + function setupVideoEventListeners() { + const videoElmRefValue = videoElm.current; + if (!videoElmRefValue) return; + + const abortController = new AbortController(); + const signal = abortController.signal; + + // To prevent the video from being paused when the user presses a space in fullscreen mode + videoElmRefValue.addEventListener("keyup", videoKeyUpHandler, { signal }); + + // We need to know when the video is playing to update state and video size + videoElmRefValue.addEventListener("playing", onVideoPlaying, { signal }); + + return () => { + abortController.abort(); + }; + }, + [ + absMouseMoveHandler, + resetMousePosition, + onVideoPlaying, + mouseWheelHandler, + videoKeyUpHandler, + ], + ); + + // Setup Absolute Mouse Events + useEffect( + function setAbsoluteMouseModeEventListeners() { + const videoElmRefValue = videoElm.current; + if (!videoElmRefValue) return; + + if (settings.mouseMode !== "absolute") return; + + const abortController = new AbortController(); + const signal = abortController.signal; + + videoElmRefValue.addEventListener("mousemove", absMouseMoveHandler, { signal }); + videoElmRefValue.addEventListener("pointerdown", absMouseMoveHandler, { signal }); + videoElmRefValue.addEventListener("pointerup", absMouseMoveHandler, { signal }); + videoElmRefValue.addEventListener("wheel", mouseWheelHandler, { + signal, + passive: true, + }); + + // Reset the mouse position when the window is blurred or the document is hidden + const local = resetMousePosition; + window.addEventListener("blur", local, { signal }); + document.addEventListener("visibilitychange", local, { signal }); + + return () => { + abortController.abort(); + }; + }, + [absMouseMoveHandler, mouseWheelHandler, resetMousePosition, settings.mouseMode], + ); + + // Setup Relative Mouse Events + const containerRef = useRef(null); + useEffect( + function setupRelativeMouseEventListeners() { + if (settings.mouseMode !== "relative") return; + + const abortController = new AbortController(); + const signal = abortController.signal; + + // We bind to the larger container in relative mode because of delta between the acceleration of the local + // mouse and the mouse movement of the remote mouse. This simply makes it a bit less painful to use. + // When we get Pointer Lock support, we can remove this. + const containerElm = containerRef.current; + if (!containerElm) return; + + containerElm.addEventListener("mousemove", relMouseMoveHandler, { signal }); + containerElm.addEventListener("pointerdown", relMouseMoveHandler, { signal }); + containerElm.addEventListener("pointerup", relMouseMoveHandler, { signal }); + + containerElm.addEventListener("wheel", mouseWheelHandler, { + signal, + passive: true, + }); + + const preventContextMenu = (e: MouseEvent) => e.preventDefault(); + containerElm.addEventListener("contextmenu", preventContextMenu, { signal }); + + return () => { + abortController.abort(); + }; + }, + [settings.mouseMode, relMouseMoveHandler, mouseWheelHandler], + ); + return (
@@ -490,7 +524,12 @@ export default function WebRTCVideo() {
-
+
e.stopPropagation()} + // onMouseDown={e => e.stopPropagation()} + // onMouseUp={e => e.stopPropagation()} + // onPointerMove={e => e.stopPropagation()} onKeyUp={e => e.stopPropagation()} onKeyDown={e => { e.stopPropagation();