internal/usbgadget/hid_keyboard.go

@@ -148,8 +148,6 @@ func (u *UsbGadget) GetKeysDownState() KeysDownState {
 func (u *UsbGadget) updateKeyDownState(state KeysDownState) {
 	u.log.Trace().Interface("old", u.keysDownState).Interface("new", state).Msg("acquiring keyboardStateLock for updateKeyDownState")
 
-	// this is intentional to unlock keyboard state lock before onKeysDownChange callback
-	{
 	u.keyboardStateLock.Lock()
 	defer u.keyboardStateLock.Unlock()
 
@@ -160,7 +158,6 @@ func (u *UsbGadget) updateKeyDownState(state KeysDownState) {
 
 	u.log.Trace().Interface("old", u.keysDownState).Interface("new", state).Msg("keysDownState updated")
 	u.keysDownState = state
-	}
 
 	if u.onKeysDownChange != nil {
 		u.log.Trace().Interface("state", state).Msg("calling onKeysDownChange")

ui/src/components/WebRTCVideo.tsx

@@ -7,14 +7,16 @@ import MacroBar from "@/components/MacroBar";
 import InfoBar from "@components/InfoBar";
 import notifications from "@/notifications";
 import useKeyboard from "@/hooks/useKeyboard";
+import { useJsonRpc } from "@/hooks/useJsonRpc";
+import { useHidRpc } from "@/hooks/useHidRpc";
 import { cx } from "@/cva.config";
 import { keys } from "@/keyboardMappings";
 import {
+  useMouseStore,
   useRTCStore,
   useSettingsStore,
   useVideoStore,
 } from "@/hooks/stores";
-import useMouse from "@/hooks/useMouse";
 
 import {
   HDMIErrorOverlay,
@@ -30,18 +32,10 @@ export default function WebRTCVideo() {
   const [isPlaying, setIsPlaying] = useState(false);
   const [isPointerLockActive, setIsPointerLockActive] = useState(false);
   const [isKeyboardLockActive, setIsKeyboardLockActive] = useState(false);
-
-  const isPointerLockPossible = window.location.protocol === "https:" || window.location.hostname === "localhost";
-
   // Store hooks
   const settings = useSettingsStore();
   const { handleKeyPress, resetKeyboardState } = useKeyboard();
-  const {
+  const { setMousePosition, setMouseMove } = useMouseStore();
-    getRelMouseMoveHandler,
-    getAbsMouseMoveHandler,
-    getMouseWheelHandler,
-    resetMousePosition,
-  } = useMouse();
   const {
     setClientSize: setVideoClientSize,
     setSize: setVideoSize,
@@ -62,6 +56,13 @@ export default function WebRTCVideo() {
   const hdmiError = ["no_lock", "no_signal", "out_of_range"].includes(hdmiState);
   const isVideoLoading = !isPlaying;
 
+  // Mouse wheel states
+  const [blockWheelEvent, setBlockWheelEvent] = useState(false);
+
+  // Misc states and hooks
+  const { send } = useJsonRpc();
+  const { reportAbsMouseEvent, reportRelMouseEvent, rpcHidReady } = useHidRpc();
+
   // Video-related
   const handleResize = useCallback(
     ({ width, height }: { width: number | undefined; height: number | undefined }) => {
@@ -100,6 +101,7 @@ export default function WebRTCVideo() {
   );
 
   // Pointer lock and keyboard lock related
+  const isPointerLockPossible = window.location.protocol === "https:" || window.location.hostname === "localhost";
   const isFullscreenEnabled = document.fullscreenEnabled;
 
   const checkNavigatorPermissions = useCallback(async (permissionName: string) => {
@@ -211,32 +213,153 @@ export default function WebRTCVideo() {
       }
     };
 
-    document.addEventListener("fullscreenchange", handleFullscreenChange);
+    document.addEventListener("fullscreenchange ", handleFullscreenChange);
   }, [releaseKeyboardLock]);
 
-  const absMouseMoveHandler = useMemo(
+  // Mouse-related
-    () => getAbsMouseMoveHandler({
+  const calcDelta = (pos: number) => (Math.abs(pos) < 10 ? pos * 2 : pos);
-      videoClientWidth,
+
-      videoClientHeight,
+  const sendRelMouseMovement = useCallback(
-      videoWidth,
+    (x: number, y: number, buttons: number) => {
-      videoHeight,
+      if (settings.mouseMode !== "relative") return;
-    }),
+      // if we ignore the event, double-click will not work
-    [getAbsMouseMoveHandler, videoClientWidth, videoClientHeight, videoWidth, videoHeight],
+      // if (x === 0 && y === 0 && buttons === 0) return;
+      const dx = calcDelta(x);
+      const dy = calcDelta(y);
+      if (rpcHidReady) {
+        reportRelMouseEvent(dx, dy, buttons);
+      } else {
+        // kept for backward compatibility
+        send("relMouseReport", { dx, dy, buttons });
+      }
+      setMouseMove({ x, y, buttons });
+    },
+    [
+      send,
+      reportRelMouseEvent,
+      setMouseMove,
+      settings.mouseMode,
+      rpcHidReady,
+    ],
   );
 
-  const relMouseMoveHandler = useMemo(
+  const relMouseMoveHandler = useCallback(
-    () => getRelMouseMoveHandler({
+    (e: MouseEvent) => {
-      isPointerLockActive,
+      if (settings.mouseMode !== "relative") return;
-      isPointerLockPossible,
+      if (isPointerLockActive === false && isPointerLockPossible) return;
-    }),
+
-    [getRelMouseMoveHandler, isPointerLockActive, isPointerLockPossible],
+      // Send mouse movement
+      const { buttons } = e;
+      sendRelMouseMovement(e.movementX, e.movementY, buttons);
+    },
+    [isPointerLockActive, isPointerLockPossible, sendRelMouseMovement, settings.mouseMode],
   );
 
-  const mouseWheelHandler = useMemo(
+  const sendAbsMouseMovement = useCallback(
-    () => getMouseWheelHandler(),
+    (x: number, y: number, buttons: number) => {
-    [getMouseWheelHandler],
+      if (settings.mouseMode !== "absolute") return;
+      if (rpcHidReady) {
+        reportAbsMouseEvent(x, y, buttons);
+      } else {
+        // kept for backward compatibility
+        send("absMouseReport", { x, y, buttons });
+      }
+      // We set that for the debug info bar
+      setMousePosition(x, y);
+    },
+    [
+      send,
+      reportAbsMouseEvent,
+      setMousePosition,
+      settings.mouseMode,
+      rpcHidReady,
+    ],
   );
 
+  const absMouseMoveHandler = useCallback(
+    (e: MouseEvent) => {
+      if (!videoClientWidth || !videoClientHeight) return;
+      if (settings.mouseMode !== "absolute") return;
+
+      // Get the aspect ratios of the video element and the video stream
+      const videoElementAspectRatio = videoClientWidth / videoClientHeight;
+      const videoStreamAspectRatio = videoWidth / videoHeight;
+
+      // Calculate the effective video display area
+      let effectiveWidth = videoClientWidth;
+      let effectiveHeight = videoClientHeight;
+      let offsetX = 0;
+      let offsetY = 0;
+
+      if (videoElementAspectRatio > videoStreamAspectRatio) {
+        // Pillarboxing: black bars on the left and right
+        effectiveWidth = videoClientHeight * videoStreamAspectRatio;
+        offsetX = (videoClientWidth - effectiveWidth) / 2;
+      } else if (videoElementAspectRatio < videoStreamAspectRatio) {
+        // Letterboxing: black bars on the top and bottom
+        effectiveHeight = videoClientWidth / videoStreamAspectRatio;
+        offsetY = (videoClientHeight - effectiveHeight) / 2;
+      }
+
+      // Clamp mouse position within the effective video boundaries
+      const clampedX = Math.min(Math.max(offsetX, e.offsetX), offsetX + effectiveWidth);
+      const clampedY = Math.min(Math.max(offsetY, e.offsetY), offsetY + effectiveHeight);
+
+      // Map clamped mouse position to the video stream's coordinate system
+      const relativeX = (clampedX - offsetX) / effectiveWidth;
+      const relativeY = (clampedY - offsetY) / effectiveHeight;
+
+      // Convert to HID absolute coordinate system (0-32767 range)
+      const x = Math.round(relativeX * 32767);
+      const y = Math.round(relativeY * 32767);
+
+      // Send mouse movement
+      const { buttons } = e;
+      sendAbsMouseMovement(x, y, buttons);
+    },
+    [settings.mouseMode, videoClientWidth, videoClientHeight, videoWidth, videoHeight, sendAbsMouseMovement],
+  );
+
+  const mouseWheelHandler = useCallback(
+    (e: WheelEvent) => {
+
+      if (settings.scrollThrottling && blockWheelEvent) {
+        return;
+      }
+
+      // Determine if the wheel event is an accel scroll value
+      const isAccel = Math.abs(e.deltaY) >= 100;
+
+      // Calculate the accel scroll value
+      const accelScrollValue = e.deltaY / 100;
+
+      // Calculate the no accel scroll value
+      const noAccelScrollValue = Math.sign(e.deltaY);
+
+      // Get scroll value
+      const scrollValue = isAccel ? accelScrollValue : noAccelScrollValue;
+
+      // Apply clamping (i.e. min and max mouse wheel hardware value)
+      const clampedScrollValue = Math.max(-127, Math.min(127, scrollValue));
+
+      // Invert the clamped scroll value to match expected behavior
+      const invertedScrollValue = -clampedScrollValue;
+
+      send("wheelReport", { wheelY: invertedScrollValue });
+
+      // Apply blocking delay based of throttling settings
+      if (settings.scrollThrottling && !blockWheelEvent) {
+        setBlockWheelEvent(true);
+        setTimeout(() => setBlockWheelEvent(false), settings.scrollThrottling);
+      }
+    },
+    [send, blockWheelEvent, settings],
+  );
+
+  const resetMousePosition = useCallback(() => {
+    sendAbsMouseMovement(0, 0, 0);
+  }, [sendAbsMouseMovement]);
+
   const keyDownHandler = useCallback(
     (e: KeyboardEvent) => {
       e.preventDefault();
@@ -391,16 +514,14 @@ export default function WebRTCVideo() {
     function setMouseModeEventListeners() {
       const videoElmRefValue = videoElm.current;
       if (!videoElmRefValue) return;
-
       const isRelativeMouseMode = (settings.mouseMode === "relative");
-      const mouseHandler = isRelativeMouseMode ? relMouseMoveHandler : absMouseMoveHandler;
 
       const abortController = new AbortController();
       const signal = abortController.signal;
 
-      videoElmRefValue.addEventListener("mousemove", mouseHandler, { signal });
+      videoElmRefValue.addEventListener("mousemove", isRelativeMouseMode ? relMouseMoveHandler : absMouseMoveHandler, { signal });
-      videoElmRefValue.addEventListener("pointerdown", mouseHandler, { signal });
+      videoElmRefValue.addEventListener("pointerdown", isRelativeMouseMode ? relMouseMoveHandler : absMouseMoveHandler, { signal });
-      videoElmRefValue.addEventListener("pointerup", mouseHandler, { signal });
+      videoElmRefValue.addEventListener("pointerup", isRelativeMouseMode ? relMouseMoveHandler : absMouseMoveHandler, { signal });
       videoElmRefValue.addEventListener("wheel", mouseWheelHandler, {
         signal,
         passive: true,
@@ -428,16 +549,7 @@ export default function WebRTCVideo() {
         abortController.abort();
       };
     },
-    [
+    [absMouseMoveHandler, isPointerLockActive, isPointerLockPossible, mouseWheelHandler, relMouseMoveHandler, requestPointerLock, resetMousePosition, settings.mouseMode],
-      isPointerLockActive,
-      isPointerLockPossible,
-      requestPointerLock,
-      absMouseMoveHandler,
-      relMouseMoveHandler,
-      mouseWheelHandler,
-      resetMousePosition,
-      settings.mouseMode,
-    ],
   );
 
   const containerRef = useRef<HTMLDivElement>(null);

ui/src/hooks/hidRpc.ts

@@ -60,7 +60,8 @@ export class RpcMessage {
         throw new Error("Not implemented");
     }
 
-    public static unmarshal(_data: Uint8Array): RpcMessage | undefined {
+    // @ts-expect-error: this is a base class, so we don't need to implement it
+    public static unmarshal(data: Uint8Array): RpcMessage | undefined {
         throw new Error("Not implemented");
     }
 }

ui/src/hooks/useKeyboard.ts

@@ -22,22 +22,16 @@ export default function useKeyboard() {
   // keysDownState when queried since the keyPressReport was introduced together with the 
   // getKeysDownState API.
   const { keyPressReportApiAvailable, setkeyPressReportApiAvailable } = useHidStore();
-  const enableKeyPressReport = useCallback((reason: string) => {
-    if (keyPressReportApiAvailable) return;
-    console.debug(`Enable keyPressReport API because ${reason}`);
-    setkeyPressReportApiAvailable(true);
-  }, [setkeyPressReportApiAvailable, keyPressReportApiAvailable]);
 
   // HidRPC is a binary format for exchanging keyboard and mouse events
   const { reportKeyboardEvent, reportKeypressEvent, rpcHidReady } = useHidRpc((message) => {
     switch (message.constructor) {
       case KeysDownStateMessage:
         setKeysDownState((message as KeysDownStateMessage).keysDownState);
-        enableKeyPressReport("HidRPC:KeysDownStateMessage received");
+        setkeyPressReportApiAvailable(true);
         break;
       case KeyboardLedStateMessage:
         setKeyboardLedState((message as KeyboardLedStateMessage).keyboardLedState);
-        enableKeyPressReport("HidRPC:KeyboardLedStateMessage received");
         break;
       default:
         break;
@@ -57,7 +51,7 @@ export default function useKeyboard() {
       if (rpcHidReady) {
         console.debug("Sending keyboard report via HidRPC");
         reportKeyboardEvent(state.keys, state.modifier);
-        enableKeyPressReport("HidRPC:KeyboardReport received");
+        setkeyPressReportApiAvailable(true);
         return;
       }
 
@@ -72,7 +66,7 @@ export default function useKeyboard() {
       rpcHidReady,
       send,
       reportKeyboardEvent,
-      enableKeyPressReport,
+      setkeyPressReportApiAvailable,
     ],
   );
 

ui/src/hooks/useMouse.ts

@@ -1,178 +0,0 @@
-import { useCallback, useState } from "react";
-
-import { useJsonRpc } from "./useJsonRpc";
-import { useHidRpc } from "./useHidRpc";
-import { useMouseStore, useSettingsStore } from "./stores";
-
-const calcDelta = (pos: number) => (Math.abs(pos) < 10 ? pos * 2 : pos);
-
-export interface AbsMouseMoveHandlerProps {
-  videoClientWidth: number;
-  videoClientHeight: number;
-  videoWidth: number;
-  videoHeight: number;
-}
-
-export interface RelMouseMoveHandlerProps {
-  isPointerLockActive: boolean;
-  isPointerLockPossible: boolean;
-}
-
-export default function useMouse() {
-  // states
-  const { setMousePosition, setMouseMove } = useMouseStore();
-  const [blockWheelEvent, setBlockWheelEvent] = useState(false);
-
-  const { mouseMode, scrollThrottling } = useSettingsStore();
-
-  // RPC hooks
-  const { send } = useJsonRpc();
-  const { reportAbsMouseEvent, reportRelMouseEvent, rpcHidReady } = useHidRpc();
-  // Mouse-related
-
-  const sendRelMouseMovement = useCallback(
-    (x: number, y: number, buttons: number) => {
-      if (mouseMode !== "relative") return;
-      // if we ignore the event, double-click will not work
-      // if (x === 0 && y === 0 && buttons === 0) return;
-      const dx = calcDelta(x);
-      const dy = calcDelta(y);
-      if (rpcHidReady) {
-        reportRelMouseEvent(dx, dy, buttons);
-      } else {
-        // kept for backward compatibility
-        send("relMouseReport", { dx, dy, buttons });
-      }
-      setMouseMove({ x, y, buttons });
-    },
-    [
-      send,
-      reportRelMouseEvent,
-      setMouseMove,
-      mouseMode,
-      rpcHidReady,
-    ],
-  );
-
-  const getRelMouseMoveHandler = useCallback(
-    ({ isPointerLockActive, isPointerLockPossible }: RelMouseMoveHandlerProps) => (e: MouseEvent) => {
-      if (mouseMode !== "relative") return;
-      if (isPointerLockActive === false && isPointerLockPossible) return;
-
-      // Send mouse movement
-      const { buttons } = e;
-      sendRelMouseMovement(e.movementX, e.movementY, buttons);
-    },
-    [sendRelMouseMovement, mouseMode],
-  );
-
-  const sendAbsMouseMovement = useCallback(
-    (x: number, y: number, buttons: number) => {
-      if (mouseMode !== "absolute") return;
-      if (rpcHidReady) {
-        reportAbsMouseEvent(x, y, buttons);
-      } else {
-        // kept for backward compatibility
-        send("absMouseReport", { x, y, buttons });
-      }
-      // We set that for the debug info bar
-      setMousePosition(x, y);
-    },
-    [
-      send,
-      reportAbsMouseEvent,
-      setMousePosition,
-      mouseMode,
-      rpcHidReady,
-    ],
-  );
-
-  const getAbsMouseMoveHandler = useCallback(
-    ({ videoClientWidth, videoClientHeight, videoWidth, videoHeight }: AbsMouseMoveHandlerProps) => (e: MouseEvent) => {
-      if (!videoClientWidth || !videoClientHeight) return;
-      if (mouseMode !== "absolute") return;
-
-      // Get the aspect ratios of the video element and the video stream
-      const videoElementAspectRatio = videoClientWidth / videoClientHeight;
-      const videoStreamAspectRatio = videoWidth / videoHeight;
-
-      // Calculate the effective video display area
-      let effectiveWidth = videoClientWidth;
-      let effectiveHeight = videoClientHeight;
-      let offsetX = 0;
-      let offsetY = 0;
-
-      if (videoElementAspectRatio > videoStreamAspectRatio) {
-        // Pillarboxing: black bars on the left and right
-        effectiveWidth = videoClientHeight * videoStreamAspectRatio;
-        offsetX = (videoClientWidth - effectiveWidth) / 2;
-      } else if (videoElementAspectRatio < videoStreamAspectRatio) {
-        // Letterboxing: black bars on the top and bottom
-        effectiveHeight = videoClientWidth / videoStreamAspectRatio;
-        offsetY = (videoClientHeight - effectiveHeight) / 2;
-      }
-
-      // Clamp mouse position within the effective video boundaries
-      const clampedX = Math.min(Math.max(offsetX, e.offsetX), offsetX + effectiveWidth);
-      const clampedY = Math.min(Math.max(offsetY, e.offsetY), offsetY + effectiveHeight);
-
-      // Map clamped mouse position to the video stream's coordinate system
-      const relativeX = (clampedX - offsetX) / effectiveWidth;
-      const relativeY = (clampedY - offsetY) / effectiveHeight;
-
-      // Convert to HID absolute coordinate system (0-32767 range)
-      const x = Math.round(relativeX * 32767);
-      const y = Math.round(relativeY * 32767);
-
-      // Send mouse movement
-      const { buttons } = e;
-      sendAbsMouseMovement(x, y, buttons);
-    }, [mouseMode, sendAbsMouseMovement],
-  );
-
-  const getMouseWheelHandler = useCallback(
-    () => (e: WheelEvent) => {
-      if (scrollThrottling && blockWheelEvent) {
-        return;
-      }
-
-      // Determine if the wheel event is an accel scroll value
-      const isAccel = Math.abs(e.deltaY) >= 100;
-
-      // Calculate the accel scroll value
-      const accelScrollValue = e.deltaY / 100;
-
-      // Calculate the no accel scroll value
-      const noAccelScrollValue = Math.sign(e.deltaY);
-
-      // Get scroll value
-      const scrollValue = isAccel ? accelScrollValue : noAccelScrollValue;
-
-      // Apply clamping (i.e. min and max mouse wheel hardware value)
-      const clampedScrollValue = Math.max(-127, Math.min(127, scrollValue));
-
-      // Invert the clamped scroll value to match expected behavior
-      const invertedScrollValue = -clampedScrollValue;
-
-      send("wheelReport", { wheelY: invertedScrollValue });
-
-      // Apply blocking delay based of throttling settings
-      if (scrollThrottling && !blockWheelEvent) {
-        setBlockWheelEvent(true);
-        setTimeout(() => setBlockWheelEvent(false), scrollThrottling);
-      }
-    },
-    [send, blockWheelEvent, scrollThrottling],
-  );
-
-  const resetMousePosition = useCallback(() => {
-    sendAbsMouseMovement(0, 0, 0);
-  }, [sendAbsMouseMovement]);
-
-  return {
-    getRelMouseMoveHandler,
-    getAbsMouseMoveHandler,
-    getMouseWheelHandler,
-    resetMousePosition,
-  };
-}

webrtc.go

@@ -158,11 +158,7 @@ func newSession(config SessionConfig) (*Session, error) {
 		case "hidrpc":
 			session.HidChannel = d
 			d.OnMessage(func(msg webrtc.DataChannelMessage) {
-				l := scopedLogger.With().Int("length", len(msg.Data)).Logger()
+				l := scopedLogger.With().Str("data", string(msg.Data)).Int("length", len(msg.Data)).Logger()
-				// only log data if the log level is debug or lower
-				if scopedLogger.GetLevel() > zerolog.DebugLevel {
-					l = l.With().Str("data", string(msg.Data)).Logger()
-				}
 
 				if msg.IsString {
 					l.Warn().Msg("received string data in HID RPC message handler")