From a4f0c0d298187a2e33ead9346741f1c13fdc39e4 Mon Sep 17 00:00:00 2001 From: Siyuan Miao Date: Thu, 4 Sep 2025 11:17:34 +0200 Subject: [PATCH] add useMouse --- ui/src/components/WebRTCVideo.tsx | 198 +++++++----------------------- ui/src/hooks/useKeyboard.ts | 12 +- ui/src/hooks/useMouse.ts | 178 +++++++++++++++++++++++++++ 3 files changed, 230 insertions(+), 158 deletions(-) create mode 100644 ui/src/hooks/useMouse.ts diff --git a/ui/src/components/WebRTCVideo.tsx b/ui/src/components/WebRTCVideo.tsx index e503de7a..65267b5c 100644 --- a/ui/src/components/WebRTCVideo.tsx +++ b/ui/src/components/WebRTCVideo.tsx @@ -7,16 +7,14 @@ import MacroBar from "@/components/MacroBar"; import InfoBar from "@components/InfoBar"; import notifications from "@/notifications"; import useKeyboard from "@/hooks/useKeyboard"; -import { useJsonRpc } from "@/hooks/useJsonRpc"; -import { useHidRpc } from "@/hooks/useHidRpc"; import { cx } from "@/cva.config"; import { keys } from "@/keyboardMappings"; import { - useMouseStore, useRTCStore, useSettingsStore, useVideoStore, } from "@/hooks/stores"; +import useMouse from "@/hooks/useMouse"; import { HDMIErrorOverlay, @@ -32,10 +30,18 @@ export default function WebRTCVideo() { const [isPlaying, setIsPlaying] = useState(false); const [isPointerLockActive, setIsPointerLockActive] = useState(false); const [isKeyboardLockActive, setIsKeyboardLockActive] = useState(false); + + const isPointerLockPossible = window.location.protocol === "https:" || window.location.hostname === "localhost"; + // Store hooks const settings = useSettingsStore(); const { handleKeyPress, resetKeyboardState } = useKeyboard(); - const { setMousePosition, setMouseMove } = useMouseStore(); + const { + getRelMouseMoveHandler, + getAbsMouseMoveHandler, + getMouseWheelHandler, + resetMousePosition, + } = useMouse(); const { setClientSize: setVideoClientSize, setSize: setVideoSize, @@ -56,13 +62,6 @@ export default function WebRTCVideo() { const hdmiError = ["no_lock", "no_signal", "out_of_range"].includes(hdmiState); const isVideoLoading = !isPlaying; - // Mouse wheel states - const [blockWheelEvent, setBlockWheelEvent] = useState(false); - - // Misc states and hooks - const { send } = useJsonRpc(); - const { reportAbsMouseEvent, reportRelMouseEvent, rpcHidReady } = useHidRpc(); - // Video-related const handleResize = useCallback( ({ width, height }: { width: number | undefined; height: number | undefined }) => { @@ -101,7 +100,6 @@ export default function WebRTCVideo() { ); // Pointer lock and keyboard lock related - const isPointerLockPossible = window.location.protocol === "https:" || window.location.hostname === "localhost"; const isFullscreenEnabled = document.fullscreenEnabled; const checkNavigatorPermissions = useCallback(async (permissionName: string) => { @@ -213,153 +211,32 @@ export default function WebRTCVideo() { } }; - document.addEventListener("fullscreenchange ", handleFullscreenChange); + document.addEventListener("fullscreenchange", handleFullscreenChange); }, [releaseKeyboardLock]); - // Mouse-related - const calcDelta = (pos: number) => (Math.abs(pos) < 10 ? pos * 2 : pos); - - const sendRelMouseMovement = useCallback( - (x: number, y: number, buttons: number) => { - if (settings.mouseMode !== "relative") return; - // if we ignore the event, double-click will not work - // if (x === 0 && y === 0 && buttons === 0) return; - const dx = calcDelta(x); - const dy = calcDelta(y); - if (rpcHidReady) { - reportRelMouseEvent(dx, dy, buttons); - } else { - // kept for backward compatibility - send("relMouseReport", { dx, dy, buttons }); - } - setMouseMove({ x, y, buttons }); - }, - [ - send, - reportRelMouseEvent, - setMouseMove, - settings.mouseMode, - rpcHidReady, - ], + const absMouseMoveHandler = useMemo( + () => getAbsMouseMoveHandler({ + videoClientWidth, + videoClientHeight, + videoWidth, + videoHeight, + }), + [getAbsMouseMoveHandler, videoClientWidth, videoClientHeight, videoWidth, videoHeight], ); - const relMouseMoveHandler = useCallback( - (e: MouseEvent) => { - if (settings.mouseMode !== "relative") return; - if (isPointerLockActive === false && isPointerLockPossible) return; - - // Send mouse movement - const { buttons } = e; - sendRelMouseMovement(e.movementX, e.movementY, buttons); - }, - [isPointerLockActive, isPointerLockPossible, sendRelMouseMovement, settings.mouseMode], + const relMouseMoveHandler = useMemo( + () => getRelMouseMoveHandler({ + isPointerLockActive, + isPointerLockPossible, + }), + [getRelMouseMoveHandler, isPointerLockActive, isPointerLockPossible], ); - const sendAbsMouseMovement = useCallback( - (x: number, y: number, buttons: number) => { - if (settings.mouseMode !== "absolute") return; - if (rpcHidReady) { - reportAbsMouseEvent(x, y, buttons); - } else { - // kept for backward compatibility - send("absMouseReport", { x, y, buttons }); - } - // We set that for the debug info bar - setMousePosition(x, y); - }, - [ - send, - reportAbsMouseEvent, - setMousePosition, - settings.mouseMode, - rpcHidReady, - ], + const mouseWheelHandler = useMemo( + () => getMouseWheelHandler(), + [getMouseWheelHandler], ); - const absMouseMoveHandler = useCallback( - (e: MouseEvent) => { - if (!videoClientWidth || !videoClientHeight) return; - if (settings.mouseMode !== "absolute") return; - - // Get the aspect ratios of the video element and the video stream - const videoElementAspectRatio = videoClientWidth / videoClientHeight; - const videoStreamAspectRatio = videoWidth / videoHeight; - - // Calculate the effective video display area - let effectiveWidth = videoClientWidth; - let effectiveHeight = videoClientHeight; - let offsetX = 0; - let offsetY = 0; - - if (videoElementAspectRatio > videoStreamAspectRatio) { - // Pillarboxing: black bars on the left and right - effectiveWidth = videoClientHeight * videoStreamAspectRatio; - offsetX = (videoClientWidth - effectiveWidth) / 2; - } else if (videoElementAspectRatio < videoStreamAspectRatio) { - // Letterboxing: black bars on the top and bottom - effectiveHeight = videoClientWidth / videoStreamAspectRatio; - offsetY = (videoClientHeight - effectiveHeight) / 2; - } - - // Clamp mouse position within the effective video boundaries - const clampedX = Math.min(Math.max(offsetX, e.offsetX), offsetX + effectiveWidth); - const clampedY = Math.min(Math.max(offsetY, e.offsetY), offsetY + effectiveHeight); - - // Map clamped mouse position to the video stream's coordinate system - const relativeX = (clampedX - offsetX) / effectiveWidth; - const relativeY = (clampedY - offsetY) / effectiveHeight; - - // Convert to HID absolute coordinate system (0-32767 range) - const x = Math.round(relativeX * 32767); - const y = Math.round(relativeY * 32767); - - // Send mouse movement - const { buttons } = e; - sendAbsMouseMovement(x, y, buttons); - }, - [settings.mouseMode, videoClientWidth, videoClientHeight, videoWidth, videoHeight, sendAbsMouseMovement], - ); - - const mouseWheelHandler = useCallback( - (e: WheelEvent) => { - - if (settings.scrollThrottling && blockWheelEvent) { - return; - } - - // Determine if the wheel event is an accel scroll value - const isAccel = Math.abs(e.deltaY) >= 100; - - // Calculate the accel scroll value - const accelScrollValue = e.deltaY / 100; - - // Calculate the no accel scroll value - const noAccelScrollValue = Math.sign(e.deltaY); - - // Get scroll value - const scrollValue = isAccel ? accelScrollValue : noAccelScrollValue; - - // Apply clamping (i.e. min and max mouse wheel hardware value) - const clampedScrollValue = Math.max(-127, Math.min(127, scrollValue)); - - // Invert the clamped scroll value to match expected behavior - const invertedScrollValue = -clampedScrollValue; - - send("wheelReport", { wheelY: invertedScrollValue }); - - // Apply blocking delay based of throttling settings - if (settings.scrollThrottling && !blockWheelEvent) { - setBlockWheelEvent(true); - setTimeout(() => setBlockWheelEvent(false), settings.scrollThrottling); - } - }, - [send, blockWheelEvent, settings], - ); - - const resetMousePosition = useCallback(() => { - sendAbsMouseMovement(0, 0, 0); - }, [sendAbsMouseMovement]); - const keyDownHandler = useCallback( (e: KeyboardEvent) => { e.preventDefault(); @@ -514,14 +391,16 @@ export default function WebRTCVideo() { function setMouseModeEventListeners() { const videoElmRefValue = videoElm.current; if (!videoElmRefValue) return; + const isRelativeMouseMode = (settings.mouseMode === "relative"); + const mouseHandler = isRelativeMouseMode ? relMouseMoveHandler : absMouseMoveHandler; const abortController = new AbortController(); const signal = abortController.signal; - videoElmRefValue.addEventListener("mousemove", isRelativeMouseMode ? relMouseMoveHandler : absMouseMoveHandler, { signal }); - videoElmRefValue.addEventListener("pointerdown", isRelativeMouseMode ? relMouseMoveHandler : absMouseMoveHandler, { signal }); - videoElmRefValue.addEventListener("pointerup", isRelativeMouseMode ? relMouseMoveHandler : absMouseMoveHandler, { signal }); + videoElmRefValue.addEventListener("mousemove", mouseHandler, { signal }); + videoElmRefValue.addEventListener("pointerdown", mouseHandler, { signal }); + videoElmRefValue.addEventListener("pointerup", mouseHandler, { signal }); videoElmRefValue.addEventListener("wheel", mouseWheelHandler, { signal, passive: true, @@ -549,7 +428,16 @@ export default function WebRTCVideo() { abortController.abort(); }; }, - [absMouseMoveHandler, isPointerLockActive, isPointerLockPossible, mouseWheelHandler, relMouseMoveHandler, requestPointerLock, resetMousePosition, settings.mouseMode], + [ + isPointerLockActive, + isPointerLockPossible, + requestPointerLock, + absMouseMoveHandler, + relMouseMoveHandler, + mouseWheelHandler, + resetMousePosition, + settings.mouseMode, + ], ); const containerRef = useRef(null); diff --git a/ui/src/hooks/useKeyboard.ts b/ui/src/hooks/useKeyboard.ts index 96398f06..7ba89dbf 100644 --- a/ui/src/hooks/useKeyboard.ts +++ b/ui/src/hooks/useKeyboard.ts @@ -22,16 +22,22 @@ export default function useKeyboard() { // keysDownState when queried since the keyPressReport was introduced together with the // getKeysDownState API. const { keyPressReportApiAvailable, setkeyPressReportApiAvailable } = useHidStore(); + const enableKeyPressReport = useCallback((reason: string) => { + if (keyPressReportApiAvailable) return; + console.debug(`Enable keyPressReport API because ${reason}`); + setkeyPressReportApiAvailable(true); + }, [setkeyPressReportApiAvailable, keyPressReportApiAvailable]); // HidRPC is a binary format for exchanging keyboard and mouse events const { reportKeyboardEvent, reportKeypressEvent, rpcHidReady } = useHidRpc((message) => { switch (message.constructor) { case KeysDownStateMessage: setKeysDownState((message as KeysDownStateMessage).keysDownState); - setkeyPressReportApiAvailable(true); + enableKeyPressReport("HidRPC:KeysDownStateMessage received"); break; case KeyboardLedStateMessage: setKeyboardLedState((message as KeyboardLedStateMessage).keyboardLedState); + enableKeyPressReport("HidRPC:KeyboardLedStateMessage received"); break; default: break; @@ -51,7 +57,7 @@ export default function useKeyboard() { if (rpcHidReady) { console.debug("Sending keyboard report via HidRPC"); reportKeyboardEvent(state.keys, state.modifier); - setkeyPressReportApiAvailable(true); + enableKeyPressReport("HidRPC:KeyboardReport received"); return; } @@ -66,7 +72,7 @@ export default function useKeyboard() { rpcHidReady, send, reportKeyboardEvent, - setkeyPressReportApiAvailable, + enableKeyPressReport, ], ); diff --git a/ui/src/hooks/useMouse.ts b/ui/src/hooks/useMouse.ts new file mode 100644 index 00000000..def0d2cd --- /dev/null +++ b/ui/src/hooks/useMouse.ts @@ -0,0 +1,178 @@ +import { useCallback, useState } from "react"; + +import { useJsonRpc } from "./useJsonRpc"; +import { useHidRpc } from "./useHidRpc"; +import { useMouseStore, useSettingsStore } from "./stores"; + +const calcDelta = (pos: number) => (Math.abs(pos) < 10 ? pos * 2 : pos); + +export interface AbsMouseMoveHandlerProps { + videoClientWidth: number; + videoClientHeight: number; + videoWidth: number; + videoHeight: number; +} + +export interface RelMouseMoveHandlerProps { + isPointerLockActive: boolean; + isPointerLockPossible: boolean; +} + +export default function useMouse() { + // states + const { setMousePosition, setMouseMove } = useMouseStore(); + const [blockWheelEvent, setBlockWheelEvent] = useState(false); + + const { mouseMode, scrollThrottling } = useSettingsStore(); + + // RPC hooks + const { send } = useJsonRpc(); + const { reportAbsMouseEvent, reportRelMouseEvent, rpcHidReady } = useHidRpc(); + // Mouse-related + + const sendRelMouseMovement = useCallback( + (x: number, y: number, buttons: number) => { + if (mouseMode !== "relative") return; + // if we ignore the event, double-click will not work + // if (x === 0 && y === 0 && buttons === 0) return; + const dx = calcDelta(x); + const dy = calcDelta(y); + if (rpcHidReady) { + reportRelMouseEvent(dx, dy, buttons); + } else { + // kept for backward compatibility + send("relMouseReport", { dx, dy, buttons }); + } + setMouseMove({ x, y, buttons }); + }, + [ + send, + reportRelMouseEvent, + setMouseMove, + mouseMode, + rpcHidReady, + ], + ); + + const getRelMouseMoveHandler = useCallback( + ({ isPointerLockActive, isPointerLockPossible }: RelMouseMoveHandlerProps) => (e: MouseEvent) => { + if (mouseMode !== "relative") return; + if (isPointerLockActive === false && isPointerLockPossible) return; + + // Send mouse movement + const { buttons } = e; + sendRelMouseMovement(e.movementX, e.movementY, buttons); + }, + [sendRelMouseMovement, mouseMode], + ); + + const sendAbsMouseMovement = useCallback( + (x: number, y: number, buttons: number) => { + if (mouseMode !== "absolute") return; + if (rpcHidReady) { + reportAbsMouseEvent(x, y, buttons); + } else { + // kept for backward compatibility + send("absMouseReport", { x, y, buttons }); + } + // We set that for the debug info bar + setMousePosition(x, y); + }, + [ + send, + reportAbsMouseEvent, + setMousePosition, + mouseMode, + rpcHidReady, + ], + ); + + const getAbsMouseMoveHandler = useCallback( + ({ videoClientWidth, videoClientHeight, videoWidth, videoHeight }: AbsMouseMoveHandlerProps) => (e: MouseEvent) => { + if (!videoClientWidth || !videoClientHeight) return; + if (mouseMode !== "absolute") return; + + // Get the aspect ratios of the video element and the video stream + const videoElementAspectRatio = videoClientWidth / videoClientHeight; + const videoStreamAspectRatio = videoWidth / videoHeight; + + // Calculate the effective video display area + let effectiveWidth = videoClientWidth; + let effectiveHeight = videoClientHeight; + let offsetX = 0; + let offsetY = 0; + + if (videoElementAspectRatio > videoStreamAspectRatio) { + // Pillarboxing: black bars on the left and right + effectiveWidth = videoClientHeight * videoStreamAspectRatio; + offsetX = (videoClientWidth - effectiveWidth) / 2; + } else if (videoElementAspectRatio < videoStreamAspectRatio) { + // Letterboxing: black bars on the top and bottom + effectiveHeight = videoClientWidth / videoStreamAspectRatio; + offsetY = (videoClientHeight - effectiveHeight) / 2; + } + + // Clamp mouse position within the effective video boundaries + const clampedX = Math.min(Math.max(offsetX, e.offsetX), offsetX + effectiveWidth); + const clampedY = Math.min(Math.max(offsetY, e.offsetY), offsetY + effectiveHeight); + + // Map clamped mouse position to the video stream's coordinate system + const relativeX = (clampedX - offsetX) / effectiveWidth; + const relativeY = (clampedY - offsetY) / effectiveHeight; + + // Convert to HID absolute coordinate system (0-32767 range) + const x = Math.round(relativeX * 32767); + const y = Math.round(relativeY * 32767); + + // Send mouse movement + const { buttons } = e; + sendAbsMouseMovement(x, y, buttons); + }, [mouseMode, sendAbsMouseMovement], + ); + + const getMouseWheelHandler = useCallback( + () => (e: WheelEvent) => { + if (scrollThrottling && blockWheelEvent) { + return; + } + + // Determine if the wheel event is an accel scroll value + const isAccel = Math.abs(e.deltaY) >= 100; + + // Calculate the accel scroll value + const accelScrollValue = e.deltaY / 100; + + // Calculate the no accel scroll value + const noAccelScrollValue = Math.sign(e.deltaY); + + // Get scroll value + const scrollValue = isAccel ? accelScrollValue : noAccelScrollValue; + + // Apply clamping (i.e. min and max mouse wheel hardware value) + const clampedScrollValue = Math.max(-127, Math.min(127, scrollValue)); + + // Invert the clamped scroll value to match expected behavior + const invertedScrollValue = -clampedScrollValue; + + send("wheelReport", { wheelY: invertedScrollValue }); + + // Apply blocking delay based of throttling settings + if (scrollThrottling && !blockWheelEvent) { + setBlockWheelEvent(true); + setTimeout(() => setBlockWheelEvent(false), scrollThrottling); + } + }, + [send, blockWheelEvent, scrollThrottling], + ); + + const resetMousePosition = useCallback(() => { + sendAbsMouseMovement(0, 0, 0); + }, [sendAbsMouseMovement]); + + return { + getRelMouseMoveHandler, + getAbsMouseMoveHandler, + getMouseWheelHandler, + resetMousePosition, + }; +} \ No newline at end of file