add useMouse

ui/src/components/WebRTCVideo.tsx

@@ -7,16 +7,14 @@ import MacroBar from "@/components/MacroBar";
 import InfoBar from "@components/InfoBar";
 import notifications from "@/notifications";
 import useKeyboard from "@/hooks/useKeyboard";
-import { useJsonRpc } from "@/hooks/useJsonRpc";
-import { useHidRpc } from "@/hooks/useHidRpc";
 import { cx } from "@/cva.config";
 import { keys } from "@/keyboardMappings";
 import {
-  useMouseStore,
   useRTCStore,
   useSettingsStore,
   useVideoStore,
 } from "@/hooks/stores";
+import useMouse from "@/hooks/useMouse";
 
 import {
   HDMIErrorOverlay,
@@ -32,10 +30,18 @@ export default function WebRTCVideo() {
   const [isPlaying, setIsPlaying] = useState(false);
   const [isPointerLockActive, setIsPointerLockActive] = useState(false);
   const [isKeyboardLockActive, setIsKeyboardLockActive] = useState(false);
+
+  const isPointerLockPossible = window.location.protocol === "https:" || window.location.hostname === "localhost";
+
   // Store hooks
   const settings = useSettingsStore();
   const { handleKeyPress, resetKeyboardState } = useKeyboard();
-  const { setMousePosition, setMouseMove } = useMouseStore();
+  const {
+    getRelMouseMoveHandler,
+    getAbsMouseMoveHandler,
+    getMouseWheelHandler,
+    resetMousePosition,
+  } = useMouse();
   const {
     setClientSize: setVideoClientSize,
     setSize: setVideoSize,
@@ -56,13 +62,6 @@ export default function WebRTCVideo() {
   const hdmiError = ["no_lock", "no_signal", "out_of_range"].includes(hdmiState);
   const isVideoLoading = !isPlaying;
 
-  // Mouse wheel states
-  const [blockWheelEvent, setBlockWheelEvent] = useState(false);
-
-  // Misc states and hooks
-  const { send } = useJsonRpc();
-  const { reportAbsMouseEvent, reportRelMouseEvent, rpcHidReady } = useHidRpc();
-
   // Video-related
   const handleResize = useCallback(
     ({ width, height }: { width: number | undefined; height: number | undefined }) => {
@@ -101,7 +100,6 @@ export default function WebRTCVideo() {
   );
 
   // Pointer lock and keyboard lock related
-  const isPointerLockPossible = window.location.protocol === "https:" || window.location.hostname === "localhost";
   const isFullscreenEnabled = document.fullscreenEnabled;
 
   const checkNavigatorPermissions = useCallback(async (permissionName: string) => {
@@ -216,150 +214,29 @@ export default function WebRTCVideo() {
     document.addEventListener("fullscreenchange", handleFullscreenChange);
   }, [releaseKeyboardLock]);
 
-  // Mouse-related
+  const absMouseMoveHandler = useMemo(
-  const calcDelta = (pos: number) => (Math.abs(pos) < 10 ? pos * 2 : pos);
+    () => getAbsMouseMoveHandler({
-
+      videoClientWidth,
-  const sendRelMouseMovement = useCallback(
+      videoClientHeight,
-    (x: number, y: number, buttons: number) => {
+      videoWidth,
-      if (settings.mouseMode !== "relative") return;
+      videoHeight,
-      // if we ignore the event, double-click will not work
+    }),
-      // if (x === 0 && y === 0 && buttons === 0) return;
+    [getAbsMouseMoveHandler, videoClientWidth, videoClientHeight, videoWidth, videoHeight],
-      const dx = calcDelta(x);
-      const dy = calcDelta(y);
-      if (rpcHidReady) {
-        reportRelMouseEvent(dx, dy, buttons);
-      } else {
-        // kept for backward compatibility
-        send("relMouseReport", { dx, dy, buttons });
-      }
-      setMouseMove({ x, y, buttons });
-    },
-    [
-      send,
-      reportRelMouseEvent,
-      setMouseMove,
-      settings.mouseMode,
-      rpcHidReady,
-    ],
   );
 
-  const relMouseMoveHandler = useCallback(
+  const relMouseMoveHandler = useMemo(
-    (e: MouseEvent) => {
+    () => getRelMouseMoveHandler({
-      if (settings.mouseMode !== "relative") return;
+      isPointerLockActive,
-      if (isPointerLockActive === false && isPointerLockPossible) return;
+      isPointerLockPossible,
-
+    }),
-      // Send mouse movement
+    [getRelMouseMoveHandler, isPointerLockActive, isPointerLockPossible],
-      const { buttons } = e;
-      sendRelMouseMovement(e.movementX, e.movementY, buttons);
-    },
-    [isPointerLockActive, isPointerLockPossible, sendRelMouseMovement, settings.mouseMode],
   );
 
-  const sendAbsMouseMovement = useCallback(
+  const mouseWheelHandler = useMemo(
-    (x: number, y: number, buttons: number) => {
+    () => getMouseWheelHandler(),
-      if (settings.mouseMode !== "absolute") return;
+    [getMouseWheelHandler],
-      if (rpcHidReady) {
-        reportAbsMouseEvent(x, y, buttons);
-      } else {
-        // kept for backward compatibility
-        send("absMouseReport", { x, y, buttons });
-      }
-      // We set that for the debug info bar
-      setMousePosition(x, y);
-    },
-    [
-      send,
-      reportAbsMouseEvent,
-      setMousePosition,
-      settings.mouseMode,
-      rpcHidReady,
-    ],
   );
 
-  const absMouseMoveHandler = useCallback(
-    (e: MouseEvent) => {
-      if (!videoClientWidth || !videoClientHeight) return;
-      if (settings.mouseMode !== "absolute") return;
-
-      // Get the aspect ratios of the video element and the video stream
-      const videoElementAspectRatio = videoClientWidth / videoClientHeight;
-      const videoStreamAspectRatio = videoWidth / videoHeight;
-
-      // Calculate the effective video display area
-      let effectiveWidth = videoClientWidth;
-      let effectiveHeight = videoClientHeight;
-      let offsetX = 0;
-      let offsetY = 0;
-
-      if (videoElementAspectRatio > videoStreamAspectRatio) {
-        // Pillarboxing: black bars on the left and right
-        effectiveWidth = videoClientHeight * videoStreamAspectRatio;
-        offsetX = (videoClientWidth - effectiveWidth) / 2;
-      } else if (videoElementAspectRatio < videoStreamAspectRatio) {
-        // Letterboxing: black bars on the top and bottom
-        effectiveHeight = videoClientWidth / videoStreamAspectRatio;
-        offsetY = (videoClientHeight - effectiveHeight) / 2;
-      }
-
-      // Clamp mouse position within the effective video boundaries
-      const clampedX = Math.min(Math.max(offsetX, e.offsetX), offsetX + effectiveWidth);
-      const clampedY = Math.min(Math.max(offsetY, e.offsetY), offsetY + effectiveHeight);
-
-      // Map clamped mouse position to the video stream's coordinate system
-      const relativeX = (clampedX - offsetX) / effectiveWidth;
-      const relativeY = (clampedY - offsetY) / effectiveHeight;
-
-      // Convert to HID absolute coordinate system (0-32767 range)
-      const x = Math.round(relativeX * 32767);
-      const y = Math.round(relativeY * 32767);
-
-      // Send mouse movement
-      const { buttons } = e;
-      sendAbsMouseMovement(x, y, buttons);
-    },
-    [settings.mouseMode, videoClientWidth, videoClientHeight, videoWidth, videoHeight, sendAbsMouseMovement],
-  );
-
-  const mouseWheelHandler = useCallback(
-    (e: WheelEvent) => {
-
-      if (settings.scrollThrottling && blockWheelEvent) {
-        return;
-      }
-
-      // Determine if the wheel event is an accel scroll value
-      const isAccel = Math.abs(e.deltaY) >= 100;
-
-      // Calculate the accel scroll value
-      const accelScrollValue = e.deltaY / 100;
-
-      // Calculate the no accel scroll value
-      const noAccelScrollValue = Math.sign(e.deltaY);
-
-      // Get scroll value
-      const scrollValue = isAccel ? accelScrollValue : noAccelScrollValue;
-
-      // Apply clamping (i.e. min and max mouse wheel hardware value)
-      const clampedScrollValue = Math.max(-127, Math.min(127, scrollValue));
-
-      // Invert the clamped scroll value to match expected behavior
-      const invertedScrollValue = -clampedScrollValue;
-
-      send("wheelReport", { wheelY: invertedScrollValue });
-
-      // Apply blocking delay based of throttling settings
-      if (settings.scrollThrottling && !blockWheelEvent) {
-        setBlockWheelEvent(true);
-        setTimeout(() => setBlockWheelEvent(false), settings.scrollThrottling);
-      }
-    },
-    [send, blockWheelEvent, settings],
-  );
-
-  const resetMousePosition = useCallback(() => {
-    sendAbsMouseMovement(0, 0, 0);
-  }, [sendAbsMouseMovement]);
-
   const keyDownHandler = useCallback(
     (e: KeyboardEvent) => {
       e.preventDefault();
@@ -514,14 +391,16 @@ export default function WebRTCVideo() {
     function setMouseModeEventListeners() {
       const videoElmRefValue = videoElm.current;
       if (!videoElmRefValue) return;
+
       const isRelativeMouseMode = (settings.mouseMode === "relative");
+      const mouseHandler = isRelativeMouseMode ? relMouseMoveHandler : absMouseMoveHandler;
 
       const abortController = new AbortController();
       const signal = abortController.signal;
 
-      videoElmRefValue.addEventListener("mousemove", isRelativeMouseMode ? relMouseMoveHandler : absMouseMoveHandler, { signal });
+      videoElmRefValue.addEventListener("mousemove", mouseHandler, { signal });
-      videoElmRefValue.addEventListener("pointerdown", isRelativeMouseMode ? relMouseMoveHandler : absMouseMoveHandler, { signal });
+      videoElmRefValue.addEventListener("pointerdown", mouseHandler, { signal });
-      videoElmRefValue.addEventListener("pointerup", isRelativeMouseMode ? relMouseMoveHandler : absMouseMoveHandler, { signal });
+      videoElmRefValue.addEventListener("pointerup", mouseHandler, { signal });
       videoElmRefValue.addEventListener("wheel", mouseWheelHandler, {
         signal,
         passive: true,
@@ -549,7 +428,16 @@ export default function WebRTCVideo() {
         abortController.abort();
       };
     },
-    [absMouseMoveHandler, isPointerLockActive, isPointerLockPossible, mouseWheelHandler, relMouseMoveHandler, requestPointerLock, resetMousePosition, settings.mouseMode],
+    [
+      isPointerLockActive,
+      isPointerLockPossible,
+      requestPointerLock,
+      absMouseMoveHandler,
+      relMouseMoveHandler,
+      mouseWheelHandler,
+      resetMousePosition,
+      settings.mouseMode,
+    ],
   );
 
   const containerRef = useRef<HTMLDivElement>(null);

ui/src/hooks/useKeyboard.ts

@@ -22,16 +22,22 @@ export default function useKeyboard() {
   // keysDownState when queried since the keyPressReport was introduced together with the 
   // getKeysDownState API.
   const { keyPressReportApiAvailable, setkeyPressReportApiAvailable } = useHidStore();
+  const enableKeyPressReport = useCallback((reason: string) => {
+    if (keyPressReportApiAvailable) return;
+    console.debug(`Enable keyPressReport API because ${reason}`);
+    setkeyPressReportApiAvailable(true);
+  }, [setkeyPressReportApiAvailable, keyPressReportApiAvailable]);
 
   // HidRPC is a binary format for exchanging keyboard and mouse events
   const { reportKeyboardEvent, reportKeypressEvent, rpcHidReady } = useHidRpc((message) => {
     switch (message.constructor) {
       case KeysDownStateMessage:
         setKeysDownState((message as KeysDownStateMessage).keysDownState);
-        setkeyPressReportApiAvailable(true);
+        enableKeyPressReport("HidRPC:KeysDownStateMessage received");
         break;
       case KeyboardLedStateMessage:
         setKeyboardLedState((message as KeyboardLedStateMessage).keyboardLedState);
+        enableKeyPressReport("HidRPC:KeyboardLedStateMessage received");
         break;
       default:
         break;
@@ -51,7 +57,7 @@ export default function useKeyboard() {
       if (rpcHidReady) {
         console.debug("Sending keyboard report via HidRPC");
         reportKeyboardEvent(state.keys, state.modifier);
-        setkeyPressReportApiAvailable(true);
+        enableKeyPressReport("HidRPC:KeyboardReport received");
         return;
       }
 
@@ -66,7 +72,7 @@ export default function useKeyboard() {
       rpcHidReady,
       send,
       reportKeyboardEvent,
-      setkeyPressReportApiAvailable,
+      enableKeyPressReport,
     ],
   );
 

ui/src/hooks/useMouse.ts

@@ -0,0 +1,178 @@
+import { useCallback, useState } from "react";
+
+import { useJsonRpc } from "./useJsonRpc";
+import { useHidRpc } from "./useHidRpc";
+import { useMouseStore, useSettingsStore } from "./stores";
+
+const calcDelta = (pos: number) => (Math.abs(pos) < 10 ? pos * 2 : pos);
+
+export interface AbsMouseMoveHandlerProps {
+  videoClientWidth: number;
+  videoClientHeight: number;
+  videoWidth: number;
+  videoHeight: number;
+}
+
+export interface RelMouseMoveHandlerProps {
+  isPointerLockActive: boolean;
+  isPointerLockPossible: boolean;
+}
+
+export default function useMouse() {
+  // states
+  const { setMousePosition, setMouseMove } = useMouseStore();
+  const [blockWheelEvent, setBlockWheelEvent] = useState(false);
+
+  const { mouseMode, scrollThrottling } = useSettingsStore();
+
+  // RPC hooks
+  const { send } = useJsonRpc();
+  const { reportAbsMouseEvent, reportRelMouseEvent, rpcHidReady } = useHidRpc();
+  // Mouse-related
+
+  const sendRelMouseMovement = useCallback(
+    (x: number, y: number, buttons: number) => {
+      if (mouseMode !== "relative") return;
+      // if we ignore the event, double-click will not work
+      // if (x === 0 && y === 0 && buttons === 0) return;
+      const dx = calcDelta(x);
+      const dy = calcDelta(y);
+      if (rpcHidReady) {
+        reportRelMouseEvent(dx, dy, buttons);
+      } else {
+        // kept for backward compatibility
+        send("relMouseReport", { dx, dy, buttons });
+      }
+      setMouseMove({ x, y, buttons });
+    },
+    [
+      send,
+      reportRelMouseEvent,
+      setMouseMove,
+      mouseMode,
+      rpcHidReady,
+    ],
+  );
+
+  const getRelMouseMoveHandler = useCallback(
+    ({ isPointerLockActive, isPointerLockPossible }: RelMouseMoveHandlerProps) => (e: MouseEvent) => {
+      if (mouseMode !== "relative") return;
+      if (isPointerLockActive === false && isPointerLockPossible) return;
+
+      // Send mouse movement
+      const { buttons } = e;
+      sendRelMouseMovement(e.movementX, e.movementY, buttons);
+    },
+    [sendRelMouseMovement, mouseMode],
+  );
+
+  const sendAbsMouseMovement = useCallback(
+    (x: number, y: number, buttons: number) => {
+      if (mouseMode !== "absolute") return;
+      if (rpcHidReady) {
+        reportAbsMouseEvent(x, y, buttons);
+      } else {
+        // kept for backward compatibility
+        send("absMouseReport", { x, y, buttons });
+      }
+      // We set that for the debug info bar
+      setMousePosition(x, y);
+    },
+    [
+      send,
+      reportAbsMouseEvent,
+      setMousePosition,
+      mouseMode,
+      rpcHidReady,
+    ],
+  );
+
+  const getAbsMouseMoveHandler = useCallback(
+    ({ videoClientWidth, videoClientHeight, videoWidth, videoHeight }: AbsMouseMoveHandlerProps) => (e: MouseEvent) => {
+      if (!videoClientWidth || !videoClientHeight) return;
+      if (mouseMode !== "absolute") return;
+
+      // Get the aspect ratios of the video element and the video stream
+      const videoElementAspectRatio = videoClientWidth / videoClientHeight;
+      const videoStreamAspectRatio = videoWidth / videoHeight;
+
+      // Calculate the effective video display area
+      let effectiveWidth = videoClientWidth;
+      let effectiveHeight = videoClientHeight;
+      let offsetX = 0;
+      let offsetY = 0;
+
+      if (videoElementAspectRatio > videoStreamAspectRatio) {
+        // Pillarboxing: black bars on the left and right
+        effectiveWidth = videoClientHeight * videoStreamAspectRatio;
+        offsetX = (videoClientWidth - effectiveWidth) / 2;
+      } else if (videoElementAspectRatio < videoStreamAspectRatio) {
+        // Letterboxing: black bars on the top and bottom
+        effectiveHeight = videoClientWidth / videoStreamAspectRatio;
+        offsetY = (videoClientHeight - effectiveHeight) / 2;
+      }
+
+      // Clamp mouse position within the effective video boundaries
+      const clampedX = Math.min(Math.max(offsetX, e.offsetX), offsetX + effectiveWidth);
+      const clampedY = Math.min(Math.max(offsetY, e.offsetY), offsetY + effectiveHeight);
+
+      // Map clamped mouse position to the video stream's coordinate system
+      const relativeX = (clampedX - offsetX) / effectiveWidth;
+      const relativeY = (clampedY - offsetY) / effectiveHeight;
+
+      // Convert to HID absolute coordinate system (0-32767 range)
+      const x = Math.round(relativeX * 32767);
+      const y = Math.round(relativeY * 32767);
+
+      // Send mouse movement
+      const { buttons } = e;
+      sendAbsMouseMovement(x, y, buttons);
+    }, [mouseMode, sendAbsMouseMovement],
+  );
+
+  const getMouseWheelHandler = useCallback(
+    () => (e: WheelEvent) => {
+      if (scrollThrottling && blockWheelEvent) {
+        return;
+      }
+
+      // Determine if the wheel event is an accel scroll value
+      const isAccel = Math.abs(e.deltaY) >= 100;
+
+      // Calculate the accel scroll value
+      const accelScrollValue = e.deltaY / 100;
+
+      // Calculate the no accel scroll value
+      const noAccelScrollValue = Math.sign(e.deltaY);
+
+      // Get scroll value
+      const scrollValue = isAccel ? accelScrollValue : noAccelScrollValue;
+
+      // Apply clamping (i.e. min and max mouse wheel hardware value)
+      const clampedScrollValue = Math.max(-127, Math.min(127, scrollValue));
+
+      // Invert the clamped scroll value to match expected behavior
+      const invertedScrollValue = -clampedScrollValue;
+
+      send("wheelReport", { wheelY: invertedScrollValue });
+
+      // Apply blocking delay based of throttling settings
+      if (scrollThrottling && !blockWheelEvent) {
+        setBlockWheelEvent(true);
+        setTimeout(() => setBlockWheelEvent(false), scrollThrottling);
+      }
+    },
+    [send, blockWheelEvent, scrollThrottling],
+  );
+
+  const resetMousePosition = useCallback(() => {
+    sendAbsMouseMovement(0, 0, 0);
+  }, [sendAbsMouseMovement]);
+
+  return {
+    getRelMouseMoveHandler,
+    getAbsMouseMoveHandler,
+    getMouseWheelHandler,
+    resetMousePosition,
+  };
+}