refactor(WebRTCVideo): improve mouse event handling and video playback logic (#282)

ui/src/components/WebRTCVideo.tsx

@@ -94,7 +94,7 @@ export default function WebRTCVideo() {
   );
 
   // Mouse-related
-  const calcDelta = (pos: number) => Math.abs(pos) < 10 ? pos * 2 : pos;
+  const calcDelta = (pos: number) => (Math.abs(pos) < 10 ? pos * 2 : pos);
   const sendRelMouseMovement = useCallback(
     (x: number, y: number, buttons: number) => {
       if (settings.mouseMode !== "relative") return;
@@ -168,7 +168,14 @@ export default function WebRTCVideo() {
       const { buttons } = e;
       sendAbsMouseMovement(x, y, buttons);
     },
-    [sendAbsMouseMovement, videoClientHeight, videoClientWidth, videoWidth, videoHeight, settings.mouseMode],
+    [
+      sendAbsMouseMovement,
+      videoClientHeight,
+      videoClientWidth,
+      videoWidth,
+      videoHeight,
+      settings.mouseMode,
+    ],
   );
 
   const trackpadSensitivity = useDeviceSettingsStore(state => state.trackpadSensitivity);
@@ -355,28 +362,6 @@ export default function WebRTCVideo() {
     ],
   );
 
-  // Effect hooks
-  useEffect(
-    function setupKeyboardEvents() {
-      const abortController = new AbortController();
-      const signal = abortController.signal;
-
-      document.addEventListener("keydown", keyDownHandler, { signal });
-      document.addEventListener("keyup", keyUpHandler, { signal });
-
-      // eslint-disable-next-line @typescript-eslint/ban-ts-comment
-      // @ts-expect-error
-      window.clearKeys = () => sendKeyboardEvent([], []);
-      window.addEventListener("blur", resetKeyboardState, { signal });
-      document.addEventListener("visibilitychange", resetKeyboardState, { signal });
-
-      return () => {
-        abortController.abort();
-      };
-    },
-    [keyDownHandler, keyUpHandler, resetKeyboardState, sendKeyboardEvent],
-  );
-
   const videoKeyUpHandler = useCallback((e: KeyboardEvent) => {
     // In fullscreen mode in chrome & safari, the space key is used to pause/play the video
     // there is no way to prevent this, so we need to simply force play the video when it's paused.
@@ -389,71 +374,6 @@ export default function WebRTCVideo() {
     }
   }, []);
 
-  useEffect(
-    function setupVideoEventListeners() {
-      let videoElmRefValue = null;
-      if (!videoElm.current) return;
-      videoElmRefValue = videoElm.current;
-      const abortController = new AbortController();
-      const signal = abortController.signal;
-
-      videoElmRefValue.addEventListener("mousemove", absMouseMoveHandler, { signal });
-      videoElmRefValue.addEventListener("pointerdown", absMouseMoveHandler, { signal });
-      videoElmRefValue.addEventListener("pointerup", absMouseMoveHandler, { signal });
-      videoElmRefValue.addEventListener("keyup", videoKeyUpHandler, { signal });
-      videoElmRefValue.addEventListener("wheel", mouseWheelHandler, {
-        signal,
-        passive: true,
-      });
-      videoElmRefValue.addEventListener(
-        "contextmenu",
-        (e: MouseEvent) => e.preventDefault(),
-        { signal },
-      );
-      videoElmRefValue.addEventListener("playing", onVideoPlaying, { signal });
-
-      const local = resetMousePosition;
-      window.addEventListener("blur", local, { signal });
-      document.addEventListener("visibilitychange", local, { signal });
-
-      return () => {
-        if (videoElmRefValue) abortController.abort();
-      };
-    },
-    [
-      absMouseMoveHandler,
-      resetMousePosition,
-      onVideoPlaying,
-      mouseWheelHandler,
-      videoKeyUpHandler,
-    ],
-  );
-
-  useEffect(
-    function setupRelativeMouseEventListeners() {
-      if (settings.mouseMode !== "relative") return;
-
-      const abortController = new AbortController();
-      const signal = abortController.signal;
-
-      // bind to body to capture all mouse events
-      const body = document.querySelector("body");
-      if (!body) return;
-
-      body.addEventListener("mousemove", relMouseMoveHandler, { signal });
-      body.addEventListener("pointerdown", relMouseMoveHandler, { signal });
-      body.addEventListener("pointerup", relMouseMoveHandler, { signal });
-
-      return () => {
-        abortController.abort();
-
-        body.removeEventListener("mousemove", relMouseMoveHandler);
-        body.removeEventListener("pointerdown", relMouseMoveHandler);
-        body.removeEventListener("pointerup", relMouseMoveHandler);
-      };
-    }, [settings.mouseMode, relMouseMoveHandler],
-  )
-
   useEffect(
     function updateVideoStream() {
       if (!mediaStream) return;
@@ -476,6 +396,120 @@ export default function WebRTCVideo() {
     ],
   );
 
+  // Setup Keyboard Events
+  useEffect(
+    function setupKeyboardEvents() {
+      const abortController = new AbortController();
+      const signal = abortController.signal;
+
+      document.addEventListener("keydown", keyDownHandler, { signal });
+      document.addEventListener("keyup", keyUpHandler, { signal });
+
+      // eslint-disable-next-line @typescript-eslint/ban-ts-comment
+      // @ts-expect-error
+      window.clearKeys = () => sendKeyboardEvent([], []);
+      window.addEventListener("blur", resetKeyboardState, { signal });
+      document.addEventListener("visibilitychange", resetKeyboardState, { signal });
+
+      return () => {
+        abortController.abort();
+      };
+    },
+    [keyDownHandler, keyUpHandler, resetKeyboardState, sendKeyboardEvent],
+  );
+
+  useEffect(
+    function setupVideoEventListeners() {
+      const videoElmRefValue = videoElm.current;
+      if (!videoElmRefValue) return;
+
+      const abortController = new AbortController();
+      const signal = abortController.signal;
+
+      // To prevent the video from being paused when the user presses a space in fullscreen mode
+      videoElmRefValue.addEventListener("keyup", videoKeyUpHandler, { signal });
+
+      // We need to know when the video is playing to update state and video size
+      videoElmRefValue.addEventListener("playing", onVideoPlaying, { signal });
+
+      return () => {
+        abortController.abort();
+      };
+    },
+    [
+      absMouseMoveHandler,
+      resetMousePosition,
+      onVideoPlaying,
+      mouseWheelHandler,
+      videoKeyUpHandler,
+    ],
+  );
+
+  // Setup Absolute Mouse Events
+  useEffect(
+    function setAbsoluteMouseModeEventListeners() {
+      const videoElmRefValue = videoElm.current;
+      if (!videoElmRefValue) return;
+
+      if (settings.mouseMode !== "absolute") return;
+
+      const abortController = new AbortController();
+      const signal = abortController.signal;
+
+      videoElmRefValue.addEventListener("mousemove", absMouseMoveHandler, { signal });
+      videoElmRefValue.addEventListener("pointerdown", absMouseMoveHandler, { signal });
+      videoElmRefValue.addEventListener("pointerup", absMouseMoveHandler, { signal });
+      videoElmRefValue.addEventListener("wheel", mouseWheelHandler, {
+        signal,
+        passive: true,
+      });
+
+      // Reset the mouse position when the window is blurred or the document is hidden
+      const local = resetMousePosition;
+      window.addEventListener("blur", local, { signal });
+      document.addEventListener("visibilitychange", local, { signal });
+
+      return () => {
+        abortController.abort();
+      };
+    },
+    [absMouseMoveHandler, mouseWheelHandler, resetMousePosition, settings.mouseMode],
+  );
+
+  // Setup Relative Mouse Events
+  const containerRef = useRef<HTMLDivElement>(null);
+  useEffect(
+    function setupRelativeMouseEventListeners() {
+      if (settings.mouseMode !== "relative") return;
+
+      const abortController = new AbortController();
+      const signal = abortController.signal;
+
+      // We bind to the larger container in relative mode because of delta between the acceleration of the local
+      // mouse and the mouse movement of the remote mouse. This simply makes it a bit less painful to use.
+      // When we get Pointer Lock support, we can remove this.
+      const containerElm = containerRef.current;
+      if (!containerElm) return;
+
+      containerElm.addEventListener("mousemove", relMouseMoveHandler, { signal });
+      containerElm.addEventListener("pointerdown", relMouseMoveHandler, { signal });
+      containerElm.addEventListener("pointerup", relMouseMoveHandler, { signal });
+
+      containerElm.addEventListener("wheel", mouseWheelHandler, {
+        signal,
+        passive: true,
+      });
+
+      const preventContextMenu = (e: MouseEvent) => e.preventDefault();
+      containerElm.addEventListener("contextmenu", preventContextMenu, { signal });
+
+      return () => {
+        abortController.abort();
+      };
+    },
+    [settings.mouseMode, relMouseMoveHandler, mouseWheelHandler],
+  );
+
   return (
     <div className="grid h-full w-full grid-rows-layout">
       <div className="min-h-[39.5px]">
@@ -490,7 +524,12 @@ export default function WebRTCVideo() {
         </fieldset>
       </div>
 
-      <div className="h-full overflow-hidden">
+      <div
+        ref={containerRef}
+        className={cx("h-full overflow-hidden", {
+          "cursor-none": settings.mouseMode === "relative" && settings.isCursorHidden,
+        })}
+      >
         <div className="relative h-full">
           <div
             className={cx(
@@ -519,7 +558,9 @@ export default function WebRTCVideo() {
                         className={cx(
                           "outline-50 max-h-full max-w-full object-contain transition-all duration-1000",
                           {
-                            "cursor-none": settings.isCursorHidden,
+                            "cursor-none":
+                              settings.mouseMode === "absolute" &&
+                              settings.isCursorHidden,
                             "opacity-0": isLoading || isConnectionError || hdmiError,
                             "animate-slideUpFade border border-slate-800/30 opacity-0 shadow dark:border-slate-300/20":
                               isPlaying,

ui/src/routes/devices.$id.tsx

@@ -516,6 +516,10 @@ export default function KvmIdRoute() {
 
       <div
         className="isolate"
+        // onMouseMove={e => e.stopPropagation()}
+        // onMouseDown={e => e.stopPropagation()}
+        // onMouseUp={e => e.stopPropagation()}
+        // onPointerMove={e => e.stopPropagation()}
         onKeyUp={e => e.stopPropagation()}
         onKeyDown={e => {
           e.stopPropagation();