Superminaren
/
kvm
mirror of https://github.com/jetkvm/kvm.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

add useMouse

This commit is contained in:
Siyuan Miao 2025-09-04 11:17:34 +02:00
parent 7feb92c9c7
commit a4f0c0d298
3 changed files with 230 additions and 158 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

198
ui/src/components/WebRTCVideo.tsx
View File

@ -7,16 +7,14 @@ import MacroBar from "@/components/MacroBar";
import InfoBar from "@components/InfoBar";
import notifications from "@/notifications";
import useKeyboard from "@/hooks/useKeyboard";
import { useJsonRpc } from "@/hooks/useJsonRpc";
import { useHidRpc } from "@/hooks/useHidRpc";
import { cx } from "@/cva.config";
import { keys } from "@/keyboardMappings";
import {
useMouseStore,
useRTCStore,
useSettingsStore,
useVideoStore,
} from "@/hooks/stores";
import useMouse from "@/hooks/useMouse";
import {
HDMIErrorOverlay,
@ -32,10 +30,18 @@ export default function WebRTCVideo() {
const [isPlaying, setIsPlaying] = useState(false);
const [isPointerLockActive, setIsPointerLockActive] = useState(false);
const [isKeyboardLockActive, setIsKeyboardLockActive] = useState(false);
const isPointerLockPossible = window.location.protocol === "https:" || window.location.hostname === "localhost";
// Store hooks
const settings = useSettingsStore();
const { handleKeyPress, resetKeyboardState } = useKeyboard();
const { setMousePosition, setMouseMove } = useMouseStore();
const {
getRelMouseMoveHandler,
getAbsMouseMoveHandler,
getMouseWheelHandler,
resetMousePosition,
} = useMouse();
const {
setClientSize: setVideoClientSize,
setSize: setVideoSize,
@ -56,13 +62,6 @@ export default function WebRTCVideo() {
const hdmiError = ["no_lock", "no_signal", "out_of_range"].includes(hdmiState);
const isVideoLoading = !isPlaying;
// Mouse wheel states
const [blockWheelEvent, setBlockWheelEvent] = useState(false);
// Misc states and hooks
const { send } = useJsonRpc();
const { reportAbsMouseEvent, reportRelMouseEvent, rpcHidReady } = useHidRpc();
// Video-related
const handleResize = useCallback(
({ width, height }: { width: number | undefined; height: number | undefined }) => {
@ -101,7 +100,6 @@ export default function WebRTCVideo() {
);
// Pointer lock and keyboard lock related
const isPointerLockPossible = window.location.protocol === "https:" || window.location.hostname === "localhost";
const isFullscreenEnabled = document.fullscreenEnabled;
const checkNavigatorPermissions = useCallback(async (permissionName: string) => {
@ -213,153 +211,32 @@ export default function WebRTCVideo() {
}
};
document.addEventListener("fullscreenchange ", handleFullscreenChange);
document.addEventListener("fullscreenchange", handleFullscreenChange);
}, [releaseKeyboardLock]);
// Mouse-related
const calcDelta = (pos: number) => (Math.abs(pos) < 10 ? pos * 2 : pos);
const sendRelMouseMovement = useCallback(
(x: number, y: number, buttons: number) => {
if (settings.mouseMode !== "relative") return;
// if we ignore the event, double-click will not work
// if (x === 0 && y === 0 && buttons === 0) return;
const dx = calcDelta(x);
const dy = calcDelta(y);
if (rpcHidReady) {
reportRelMouseEvent(dx, dy, buttons);
} else {
// kept for backward compatibility
send("relMouseReport", { dx, dy, buttons });
}
setMouseMove({ x, y, buttons });
},
[
send,
reportRelMouseEvent,
setMouseMove,
settings.mouseMode,
rpcHidReady,
],
const absMouseMoveHandler = useMemo(
() => getAbsMouseMoveHandler({
videoClientWidth,
videoClientHeight,
videoWidth,
videoHeight,
}),
[getAbsMouseMoveHandler, videoClientWidth, videoClientHeight, videoWidth, videoHeight],
);
const relMouseMoveHandler = useCallback(
(e: MouseEvent) => {
if (settings.mouseMode !== "relative") return;
if (isPointerLockActive === false && isPointerLockPossible) return;
// Send mouse movement
const { buttons } = e;
sendRelMouseMovement(e.movementX, e.movementY, buttons);
},
[isPointerLockActive, isPointerLockPossible, sendRelMouseMovement, settings.mouseMode],
const relMouseMoveHandler = useMemo(
() => getRelMouseMoveHandler({
isPointerLockActive,
isPointerLockPossible,
}),
[getRelMouseMoveHandler, isPointerLockActive, isPointerLockPossible],
);
const sendAbsMouseMovement = useCallback(
(x: number, y: number, buttons: number) => {
if (settings.mouseMode !== "absolute") return;
if (rpcHidReady) {
reportAbsMouseEvent(x, y, buttons);
} else {
// kept for backward compatibility
send("absMouseReport", { x, y, buttons });
}
// We set that for the debug info bar
setMousePosition(x, y);
},
[
send,
reportAbsMouseEvent,
setMousePosition,
settings.mouseMode,
rpcHidReady,
],
const mouseWheelHandler = useMemo(
() => getMouseWheelHandler(),
[getMouseWheelHandler],
);
const absMouseMoveHandler = useCallback(
(e: MouseEvent) => {
if (!videoClientWidth || !videoClientHeight) return;
if (settings.mouseMode !== "absolute") return;
// Get the aspect ratios of the video element and the video stream
const videoElementAspectRatio = videoClientWidth / videoClientHeight;
const videoStreamAspectRatio = videoWidth / videoHeight;
// Calculate the effective video display area
let effectiveWidth = videoClientWidth;
let effectiveHeight = videoClientHeight;
let offsetX = 0;
let offsetY = 0;
if (videoElementAspectRatio > videoStreamAspectRatio) {
// Pillarboxing: black bars on the left and right
effectiveWidth = videoClientHeight * videoStreamAspectRatio;
offsetX = (videoClientWidth - effectiveWidth) / 2;
} else if (videoElementAspectRatio < videoStreamAspectRatio) {
// Letterboxing: black bars on the top and bottom
effectiveHeight = videoClientWidth / videoStreamAspectRatio;
offsetY = (videoClientHeight - effectiveHeight) / 2;
}
// Clamp mouse position within the effective video boundaries
const clampedX = Math.min(Math.max(offsetX, e.offsetX), offsetX + effectiveWidth);
const clampedY = Math.min(Math.max(offsetY, e.offsetY), offsetY + effectiveHeight);
// Map clamped mouse position to the video stream's coordinate system
const relativeX = (clampedX - offsetX) / effectiveWidth;
const relativeY = (clampedY - offsetY) / effectiveHeight;
// Convert to HID absolute coordinate system (0-32767 range)
const x = Math.round(relativeX * 32767);
const y = Math.round(relativeY * 32767);
// Send mouse movement
const { buttons } = e;
sendAbsMouseMovement(x, y, buttons);
},
[settings.mouseMode, videoClientWidth, videoClientHeight, videoWidth, videoHeight, sendAbsMouseMovement],
);
const mouseWheelHandler = useCallback(
(e: WheelEvent) => {
if (settings.scrollThrottling && blockWheelEvent) {
return;
}
// Determine if the wheel event is an accel scroll value
const isAccel = Math.abs(e.deltaY) >= 100;
// Calculate the accel scroll value
const accelScrollValue = e.deltaY / 100;
// Calculate the no accel scroll value
const noAccelScrollValue = Math.sign(e.deltaY);
// Get scroll value
const scrollValue = isAccel ? accelScrollValue : noAccelScrollValue;
// Apply clamping (i.e. min and max mouse wheel hardware value)
const clampedScrollValue = Math.max(-127, Math.min(127, scrollValue));
// Invert the clamped scroll value to match expected behavior
const invertedScrollValue = -clampedScrollValue;
send("wheelReport", { wheelY: invertedScrollValue });
// Apply blocking delay based of throttling settings
if (settings.scrollThrottling && !blockWheelEvent) {
setBlockWheelEvent(true);
setTimeout(() => setBlockWheelEvent(false), settings.scrollThrottling);
}
},
[send, blockWheelEvent, settings],
);
const resetMousePosition = useCallback(() => {
sendAbsMouseMovement(0, 0, 0);
}, [sendAbsMouseMovement]);
const keyDownHandler = useCallback(
(e: KeyboardEvent) => {
e.preventDefault();
@ -514,14 +391,16 @@ export default function WebRTCVideo() {
function setMouseModeEventListeners() {
const videoElmRefValue = videoElm.current;
if (!videoElmRefValue) return;
const isRelativeMouseMode = (settings.mouseMode === "relative");
const mouseHandler = isRelativeMouseMode ? relMouseMoveHandler : absMouseMoveHandler;
const abortController = new AbortController();
const signal = abortController.signal;
videoElmRefValue.addEventListener("mousemove", isRelativeMouseMode ? relMouseMoveHandler : absMouseMoveHandler, { signal });
videoElmRefValue.addEventListener("pointerdown", isRelativeMouseMode ? relMouseMoveHandler : absMouseMoveHandler, { signal });
videoElmRefValue.addEventListener("pointerup", isRelativeMouseMode ? relMouseMoveHandler : absMouseMoveHandler, { signal });
videoElmRefValue.addEventListener("mousemove", mouseHandler, { signal });
videoElmRefValue.addEventListener("pointerdown", mouseHandler, { signal });
videoElmRefValue.addEventListener("pointerup", mouseHandler, { signal });
videoElmRefValue.addEventListener("wheel", mouseWheelHandler, {
signal,
passive: true,
@ -549,7 +428,16 @@ export default function WebRTCVideo() {
abortController.abort();
};
},
[absMouseMoveHandler, isPointerLockActive, isPointerLockPossible, mouseWheelHandler, relMouseMoveHandler, requestPointerLock, resetMousePosition, settings.mouseMode],
[
isPointerLockActive,
isPointerLockPossible,
requestPointerLock,
absMouseMoveHandler,
relMouseMoveHandler,
mouseWheelHandler,
resetMousePosition,
settings.mouseMode,
],
);
const containerRef = useRef<HTMLDivElement>(null);

12
ui/src/hooks/useKeyboard.ts
View File

@ -22,16 +22,22 @@ export default function useKeyboard() {
// keysDownState when queried since the keyPressReport was introduced together with the
// getKeysDownState API.
const { keyPressReportApiAvailable, setkeyPressReportApiAvailable } = useHidStore();
const enableKeyPressReport = useCallback((reason: string) => {
if (keyPressReportApiAvailable) return;
console.debug(`Enable keyPressReport API because ${reason}`);
setkeyPressReportApiAvailable(true);
}, [setkeyPressReportApiAvailable, keyPressReportApiAvailable]);
// HidRPC is a binary format for exchanging keyboard and mouse events
const { reportKeyboardEvent, reportKeypressEvent, rpcHidReady } = useHidRpc((message) => {
switch (message.constructor) {
case KeysDownStateMessage:
setKeysDownState((message as KeysDownStateMessage).keysDownState);
setkeyPressReportApiAvailable(true);
enableKeyPressReport("HidRPC:KeysDownStateMessage received");
break;
case KeyboardLedStateMessage:
setKeyboardLedState((message as KeyboardLedStateMessage).keyboardLedState);
enableKeyPressReport("HidRPC:KeyboardLedStateMessage received");
break;
default:
break;
@ -51,7 +57,7 @@ export default function useKeyboard() {
if (rpcHidReady) {
console.debug("Sending keyboard report via HidRPC");
reportKeyboardEvent(state.keys, state.modifier);
setkeyPressReportApiAvailable(true);
enableKeyPressReport("HidRPC:KeyboardReport received");
return;
}
@ -66,7 +72,7 @@ export default function useKeyboard() {
rpcHidReady,
send,
reportKeyboardEvent,
setkeyPressReportApiAvailable,
enableKeyPressReport,
],
);

178
ui/src/hooks/useMouse.ts Normal file
View File

@ -0,0 +1,178 @@
import { useCallback, useState } from "react";
import { useJsonRpc } from "./useJsonRpc";
import { useHidRpc } from "./useHidRpc";
import { useMouseStore, useSettingsStore } from "./stores";
const calcDelta = (pos: number) => (Math.abs(pos) < 10 ? pos * 2 : pos);
export interface AbsMouseMoveHandlerProps {
videoClientWidth: number;
videoClientHeight: number;
videoWidth: number;
videoHeight: number;
}
export interface RelMouseMoveHandlerProps {
isPointerLockActive: boolean;
isPointerLockPossible: boolean;
}
export default function useMouse() {
// states
const { setMousePosition, setMouseMove } = useMouseStore();
const [blockWheelEvent, setBlockWheelEvent] = useState(false);
const { mouseMode, scrollThrottling } = useSettingsStore();
// RPC hooks
const { send } = useJsonRpc();
const { reportAbsMouseEvent, reportRelMouseEvent, rpcHidReady } = useHidRpc();
// Mouse-related
const sendRelMouseMovement = useCallback(
(x: number, y: number, buttons: number) => {
if (mouseMode !== "relative") return;
// if we ignore the event, double-click will not work
// if (x === 0 && y === 0 && buttons === 0) return;
const dx = calcDelta(x);
const dy = calcDelta(y);
if (rpcHidReady) {
reportRelMouseEvent(dx, dy, buttons);
} else {
// kept for backward compatibility
send("relMouseReport", { dx, dy, buttons });
}
setMouseMove({ x, y, buttons });
},
[
send,
reportRelMouseEvent,
setMouseMove,
mouseMode,
rpcHidReady,
],
);
const getRelMouseMoveHandler = useCallback(
({ isPointerLockActive, isPointerLockPossible }: RelMouseMoveHandlerProps) => (e: MouseEvent) => {
if (mouseMode !== "relative") return;
if (isPointerLockActive === false && isPointerLockPossible) return;
// Send mouse movement
const { buttons } = e;
sendRelMouseMovement(e.movementX, e.movementY, buttons);
},
[sendRelMouseMovement, mouseMode],
);
const sendAbsMouseMovement = useCallback(
(x: number, y: number, buttons: number) => {
if (mouseMode !== "absolute") return;
if (rpcHidReady) {
reportAbsMouseEvent(x, y, buttons);
} else {
// kept for backward compatibility
send("absMouseReport", { x, y, buttons });
}
// We set that for the debug info bar
setMousePosition(x, y);
},
[
send,
reportAbsMouseEvent,
setMousePosition,
mouseMode,
rpcHidReady,
],
);
const getAbsMouseMoveHandler = useCallback(
({ videoClientWidth, videoClientHeight, videoWidth, videoHeight }: AbsMouseMoveHandlerProps) => (e: MouseEvent) => {
if (!videoClientWidth || !videoClientHeight) return;
if (mouseMode !== "absolute") return;
// Get the aspect ratios of the video element and the video stream
const videoElementAspectRatio = videoClientWidth / videoClientHeight;
const videoStreamAspectRatio = videoWidth / videoHeight;
// Calculate the effective video display area
let effectiveWidth = videoClientWidth;
let effectiveHeight = videoClientHeight;
let offsetX = 0;
let offsetY = 0;
if (videoElementAspectRatio > videoStreamAspectRatio) {
// Pillarboxing: black bars on the left and right
effectiveWidth = videoClientHeight * videoStreamAspectRatio;
offsetX = (videoClientWidth - effectiveWidth) / 2;
} else if (videoElementAspectRatio < videoStreamAspectRatio) {
// Letterboxing: black bars on the top and bottom
effectiveHeight = videoClientWidth / videoStreamAspectRatio;
offsetY = (videoClientHeight - effectiveHeight) / 2;
}
// Clamp mouse position within the effective video boundaries
const clampedX = Math.min(Math.max(offsetX, e.offsetX), offsetX + effectiveWidth);
const clampedY = Math.min(Math.max(offsetY, e.offsetY), offsetY + effectiveHeight);
// Map clamped mouse position to the video stream's coordinate system
const relativeX = (clampedX - offsetX) / effectiveWidth;
const relativeY = (clampedY - offsetY) / effectiveHeight;
// Convert to HID absolute coordinate system (0-32767 range)
const x = Math.round(relativeX * 32767);
const y = Math.round(relativeY * 32767);
// Send mouse movement
const { buttons } = e;
sendAbsMouseMovement(x, y, buttons);
}, [mouseMode, sendAbsMouseMovement],
);
const getMouseWheelHandler = useCallback(
() => (e: WheelEvent) => {
if (scrollThrottling && blockWheelEvent) {
return;
}
// Determine if the wheel event is an accel scroll value
const isAccel = Math.abs(e.deltaY) >= 100;
// Calculate the accel scroll value
const accelScrollValue = e.deltaY / 100;
// Calculate the no accel scroll value
const noAccelScrollValue = Math.sign(e.deltaY);
// Get scroll value
const scrollValue = isAccel ? accelScrollValue : noAccelScrollValue;
// Apply clamping (i.e. min and max mouse wheel hardware value)
const clampedScrollValue = Math.max(-127, Math.min(127, scrollValue));
// Invert the clamped scroll value to match expected behavior
const invertedScrollValue = -clampedScrollValue;
send("wheelReport", { wheelY: invertedScrollValue });
// Apply blocking delay based of throttling settings
if (scrollThrottling && !blockWheelEvent) {
setBlockWheelEvent(true);
setTimeout(() => setBlockWheelEvent(false), scrollThrottling);
}
},
[send, blockWheelEvent, scrollThrottling],
);
const resetMousePosition = useCallback(() => {
sendAbsMouseMovement(0, 0, 0);
}, [sendAbsMouseMovement]);
return {
getRelMouseMoveHandler,
getAbsMouseMoveHandler,
getMouseWheelHandler,
resetMousePosition,
};
}