Superminaren
/
kvm
mirror of https://github.com/jetkvm/kvm.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

fix(ui): Fix full-screen keyboard lock issues. (#535) 

When the keyboard lock is supposed to be active (in full-screen mode), hitting the escape key (NOT long-pressing) should NOT dismiss the full-screen mode, and should send the Escape key through to the remote.

- Added awaits to the browser calls that need to complete in order.
- Cleaned up (mostly) duplicate code in the Absolute/Relative mouse handling
- Ensure we don't overrun any existing keyboard lock or pointer lock
- Release the keyboard lock when leaving full-screen
- Per standards, we need to acquire the keyboard and pointer locks before entering full-screen or the user may get multiple messages about exiting.
- Fixed all the missing/excess React dependencies.
- Moved the pointer lock bar up so it is visible.
- Somewhere along the way, the prompt to click the video when in relative-mouse-mode stopped being visible, restored it's visibility
- Fixed all the "should be readonly" warnings.
This commit is contained in:
Marc Brooks 2025-06-02 18:28:35 -05:00 committed by GitHub
parent a7693df92c
commit f4bb47c544
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
2 changed files with 117 additions and 154 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

32
ui/src/components/VideoOverlay.tsx
View File

@ -10,7 +10,7 @@ import LoadingSpinner from "@components/LoadingSpinner";
import Card, { GridCard } from "@components/Card";
interface OverlayContentProps {
children: React.ReactNode;
readonly children: React.ReactNode;
}
function OverlayContent({ children }: OverlayContentProps) {
return (
@ -23,7 +23,7 @@ function OverlayContent({ children }: OverlayContentProps) {
}
interface LoadingOverlayProps {
show: boolean;
readonly show: boolean;
}
export function LoadingVideoOverlay({ show }: LoadingOverlayProps) {
@ -57,8 +57,8 @@ export function LoadingVideoOverlay({ show }: LoadingOverlayProps) {
}
interface LoadingConnectionOverlayProps {
show: boolean;
text: string;
readonly show: boolean;
readonly text: string;
}
export function LoadingConnectionOverlay({ show, text }: LoadingConnectionOverlayProps) {
return (
@ -91,8 +91,8 @@ export function LoadingConnectionOverlay({ show, text }: LoadingConnectionOverla
}
interface ConnectionErrorOverlayProps {
show: boolean;
setupPeerConnection: () => Promise<void>;
readonly show: boolean;
readonly setupPeerConnection: () => Promise<void>;
}
export function ConnectionFailedOverlay({
@ -153,7 +153,7 @@ export function ConnectionFailedOverlay({
}
interface PeerConnectionDisconnectedOverlay {
show: boolean;
readonly show: boolean;
}
export function PeerConnectionDisconnectedOverlay({
@ -207,8 +207,8 @@ export function PeerConnectionDisconnectedOverlay({
}
interface HDMIErrorOverlayProps {
show: boolean;
hdmiState: string;
readonly show: boolean;
readonly hdmiState: string;
}
export function HDMIErrorOverlay({ show, hdmiState }: HDMIErrorOverlayProps) {
@ -310,8 +310,8 @@ export function HDMIErrorOverlay({ show, hdmiState }: HDMIErrorOverlayProps) {
}
interface NoAutoplayPermissionsOverlayProps {
show: boolean;
onPlayClick: () => void;
readonly show: boolean;
readonly onPlayClick: () => void;
}
export function NoAutoplayPermissionsOverlay({
@ -361,7 +361,7 @@ export function NoAutoplayPermissionsOverlay({
}
interface PointerLockBarProps {
show: boolean;
readonly show: boolean;
}
export function PointerLockBar({ show }: PointerLockBarProps) {
@ -369,10 +369,10 @@ export function PointerLockBar({ show }: PointerLockBarProps) {
<AnimatePresence mode="wait">
{show ? (
<motion.div
className="absolute -top-[36px] left-0 right-0 z-20 bg-white"
initial={{ y: 20, opacity: 0, zIndex: 0 }}
animate={{ opacity: 1, y: 0 }}
exit={{ y: 43, zIndex: 0 }}
className="flex w-full items-center justify-between bg-transparent"
initial={{ opacity: 0, zIndex: 0 }}
animate={{ opacity: 1, zIndex: 20 }}
exit={{ opacity: 0, zIndex: 0 }}
transition={{ duration: 0.5, ease: "easeInOut", delay: 0.5 }}
>
<div>

239
ui/src/components/WebRTCVideo.tsx
View File

@ -1,6 +1,15 @@
import { useCallback, useEffect, useMemo, useRef, useState } from "react";
import { useResizeObserver } from "usehooks-ts";
import VirtualKeyboard from "@components/VirtualKeyboard";
import Actionbar from "@components/ActionBar";
import MacroBar from "@/components/MacroBar";
import InfoBar from "@components/InfoBar";
import notifications from "@/notifications";
import useKeyboard from "@/hooks/useKeyboard";
import { useJsonRpc } from "@/hooks/useJsonRpc";
import { cx } from "@/cva.config";
import { keys, modifiers } from "@/keyboardMappings";
import {
useHidStore,
useMouseStore,
@ -9,15 +18,6 @@ import {
useUiStore,
useVideoStore,
} from "@/hooks/stores";
import { keys, modifiers } from "@/keyboardMappings";
import { cx } from "@/cva.config";
import VirtualKeyboard from "@components/VirtualKeyboard";
import Actionbar from "@components/ActionBar";
import MacroBar from "@/components/MacroBar";
import InfoBar from "@components/InfoBar";
import useKeyboard from "@/hooks/useKeyboard";
import { useJsonRpc } from "@/hooks/useJsonRpc";
import notifications from "@/notifications";
import {
HDMIErrorOverlay,
@ -106,8 +106,9 @@ export default function WebRTCVideo() {
);
// Pointer lock and keyboard lock related
const isPointerLockPossible = window.location.protocol === "https:";
const isPointerLockPossible = window.location.protocol === "https:" || window.location.hostname === "localhost";
const isFullscreenEnabled = document.fullscreenEnabled;
const checkNavigatorPermissions = useCallback(async (permissionName: string) => {
const name = permissionName as PermissionName;
const { state } = await navigator.permissions.query({ name });
@ -115,23 +116,47 @@ export default function WebRTCVideo() {
}, []);
const requestPointerLock = useCallback(async () => {
if (document.pointerLockElement) return;
if (!isPointerLockPossible
|| videoElm.current === null
|| document.pointerLockElement) return;
const isPointerLockGranted = await checkNavigatorPermissions("pointer-lock");
if (isPointerLockGranted && settings.mouseMode === "relative") {
videoElm.current?.requestPointerLock();
await videoElm.current.requestPointerLock();
}
}, [checkNavigatorPermissions, settings.mouseMode]);
}, [checkNavigatorPermissions, isPointerLockPossible, settings.mouseMode]);
const requestKeyboardLock = useCallback(async () => {
if (videoElm.current === null) return;
const isKeyboardLockGranted = await checkNavigatorPermissions("keyboard-lock");
if (isKeyboardLockGranted) {
if ("keyboard" in navigator) {
// @ts-expect-error - keyboard lock is not supported in all browsers
await navigator.keyboard.lock();
}
}
}, [checkNavigatorPermissions]);
const releaseKeyboardLock = useCallback(async () => {
if (videoElm.current === null || document.fullscreenElement !== videoElm.current) return;
if ("keyboard" in navigator) {
// @ts-expect-error - keyboard unlock is not supported in all browsers
await navigator.keyboard.unlock();
}
}, []);
useEffect(() => {
if (!isPointerLockPossible || !videoElm.current) return;
const handlePointerLockChange = () => {
if (document.pointerLockElement) {
notifications.success("Pointer lock Enabled, hold escape to exit");
notifications.success("Pointer lock Enabled, press escape to unlock");
setIsPointerLockActive(true);
} else {
notifications.success("Pointer lock disabled");
notifications.success("Pointer lock Disabled");
setIsPointerLockActive(false);
}
};
@ -144,27 +169,39 @@ export default function WebRTCVideo() {
return () => {
abortController.abort();
};
}, [isPointerLockPossible, videoElm]);
}, [isPointerLockPossible]);
const requestFullscreen = useCallback(async () => {
videoElm.current?.requestFullscreen({
navigationUI: "show",
});
if (!isFullscreenEnabled || !videoElm.current) return;
// we do not care about pointer lock if it's for fullscreen
// per https://wicg.github.io/keyboard-lock/#system-key-press-handler
// If keyboard lock is activated after fullscreen is already in effect, then the user my
// see multiple messages about how to exit fullscreen. For this reason, we recommend that
// developers call lock() before they enter fullscreen:
await requestKeyboardLock();
await requestPointerLock();
const isKeyboardLockGranted = await checkNavigatorPermissions("keyboard-lock");
if (isKeyboardLockGranted) {
if ("keyboard" in navigator) {
// @ts-expect-error - keyboard lock is not supported in all browsers
await navigator.keyboard.lock();
await videoElm.current.requestFullscreen({
navigationUI: "show",
});
}, [isFullscreenEnabled, requestKeyboardLock, requestPointerLock]);
// setup to release the keyboard lock anytime the fullscreen ends
useEffect(() => {
if (!videoElm.current) return;
const handleFullscreenChange = () => {
if (!document.fullscreenElement) {
releaseKeyboardLock();
}
}
}, [requestPointerLock, checkNavigatorPermissions]);
};
document.addEventListener("fullscreenchange ", handleFullscreenChange);
}, [releaseKeyboardLock]);
// Mouse-related
const calcDelta = (pos: number) => (Math.abs(pos) < 10 ? pos * 2 : pos);
const sendRelMouseMovement = useCallback(
(x: number, y: number, buttons: number) => {
if (settings.mouseMode !== "relative") return;
@ -179,18 +216,13 @@ export default function WebRTCVideo() {
const relMouseMoveHandler = useCallback(
(e: MouseEvent) => {
if (settings.mouseMode !== "relative") return;
if (isPointerLockActive === false && isPointerLockPossible === true) return;
if (isPointerLockActive === false && isPointerLockPossible) return;
// Send mouse movement
const { buttons } = e;
sendRelMouseMovement(e.movementX, e.movementY, buttons);
},
[
isPointerLockActive,
isPointerLockPossible,
sendRelMouseMovement,
settings.mouseMode,
],
[isPointerLockActive, isPointerLockPossible, sendRelMouseMovement, settings.mouseMode],
);
const sendAbsMouseMovement = useCallback(
@ -244,14 +276,7 @@ export default function WebRTCVideo() {
const { buttons } = e;
sendAbsMouseMovement(x, y, buttons);
},
[
sendAbsMouseMovement,
videoClientHeight,
videoClientWidth,
videoWidth,
videoHeight,
settings.mouseMode,
],
[settings.mouseMode, videoClientWidth, videoClientHeight, videoWidth, videoHeight, sendAbsMouseMovement],
);
const mouseWheelHandler = useCallback(
@ -263,7 +288,7 @@ export default function WebRTCVideo() {
const accelScrollValue = e.deltaY / 100;
// Calculate the no accel scroll value
const noAccelScrollValue = e.deltaY > 0 ? 1 : e.deltaY < 0 ? -1 : 0;
const noAccelScrollValue = Math.sign(e.deltaY);
// Get scroll value
const scrollValue = isAccel ? accelScrollValue : noAccelScrollValue;
@ -356,13 +381,6 @@ export default function WebRTCVideo() {
let code = e.code;
const key = e.key;
// if (document.activeElement?.id !== "videoFocusTrap") {
// console.log("KEYUP: Not focusing on the video", document.activeElement);
// return;
// }
// console.log(document.activeElement);
if (!isKeyboardLedManagedByHost) {
setIsNumLockActive(e.getModifierState("NumLock"));
setIsCapsLockActive(e.getModifierState("CapsLock"));
@ -440,13 +458,15 @@ export default function WebRTCVideo() {
);
const videoKeyUpHandler = useCallback((e: KeyboardEvent) => {
if (!videoElm.current) return;
// In fullscreen mode in chrome & safari, the space key is used to pause/play the video
// there is no way to prevent this, so we need to simply force play the video when it's paused.
// Fix only works in chrome based browsers.
if (e.code === "Space") {
if (videoElm.current?.paused == true) {
if (videoElm.current.paused) {
console.log("Force playing video");
videoElm.current?.play();
videoElm.current.play();
}
}
}, []);
@ -455,7 +475,6 @@ export default function WebRTCVideo() {
(mediaStream: MediaStream) => {
if (!videoElm.current) return;
const videoElmRefValue = videoElm.current;
// console.log("Adding stream to video element", videoElmRefValue);
videoElmRefValue.srcObject = mediaStream;
updateVideoSizeStore(videoElmRefValue);
},
@ -471,7 +490,6 @@ export default function WebRTCVideo() {
peerConnection.addEventListener(
"track",
(e: RTCTrackEvent) => {
// console.log("Adding stream to video element");
addStreamToVideoElm(e.streams[0]);
},
{ signal },
@ -487,7 +505,6 @@ export default function WebRTCVideo() {
useEffect(
function updateVideoStream() {
if (!mediaStream) return;
console.log("Updating video stream from mediaStream");
// We set the as early as possible
addStreamToVideoElm(mediaStream);
},
@ -509,9 +526,6 @@ export default function WebRTCVideo() {
document.addEventListener("keydown", keyDownHandler, { signal });
document.addEventListener("keyup", keyUpHandler, { signal });
// eslint-disable-next-line @typescript-eslint/ban-ts-comment
// @ts-expect-error
window.clearKeys = () => sendKeyboardEvent([], []);
window.addEventListener("blur", resetKeyboardState, { signal });
document.addEventListener("visibilitychange", resetKeyboardState, { signal });
@ -519,7 +533,7 @@ export default function WebRTCVideo() {
abortController.abort();
};
},
[keyDownHandler, keyUpHandler, resetKeyboardState, sendKeyboardEvent],
[keyDownHandler, keyUpHandler, resetKeyboardState],
);
// Setup Video Event Listeners
@ -541,38 +555,42 @@ export default function WebRTCVideo() {
abortController.abort();
};
},
[
absMouseMoveHandler,
resetMousePosition,
onVideoPlaying,
mouseWheelHandler,
videoKeyUpHandler,
],
[onVideoPlaying, videoKeyUpHandler],
);
// Setup Absolute Mouse Events
// Setup Mouse Events
useEffect(
function setAbsoluteMouseModeEventListeners() {
function setMouseModeEventListeners() {
const videoElmRefValue = videoElm.current;
if (!videoElmRefValue) return;
if (settings.mouseMode !== "absolute") return;
const isRelativeMouseMode = (settings.mouseMode === "relative");
const abortController = new AbortController();
const signal = abortController.signal;
videoElmRefValue.addEventListener("mousemove", absMouseMoveHandler, { signal });
videoElmRefValue.addEventListener("pointerdown", absMouseMoveHandler, { signal });
videoElmRefValue.addEventListener("pointerup", absMouseMoveHandler, { signal });
videoElmRefValue.addEventListener("mousemove", isRelativeMouseMode ? relMouseMoveHandler : absMouseMoveHandler, { signal });
videoElmRefValue.addEventListener("pointerdown", isRelativeMouseMode ? relMouseMoveHandler : absMouseMoveHandler, { signal });
videoElmRefValue.addEventListener("pointerup", isRelativeMouseMode ? relMouseMoveHandler :absMouseMoveHandler, { signal });
videoElmRefValue.addEventListener("wheel", mouseWheelHandler, {
signal,
passive: true,
});
// Reset the mouse position when the window is blurred or the document is hidden
const local = resetMousePosition;
window.addEventListener("blur", local, { signal });
document.addEventListener("visibilitychange", local, { signal });
if (isRelativeMouseMode) {
videoElmRefValue.addEventListener("click",
() => {
if (isPointerLockPossible && !isPointerLockActive && !document.pointerLockElement) {
requestPointerLock();
}
},
{ signal },
);
} else {
// Reset the mouse position when the window is blurred or the document is hidden
window.addEventListener("blur", resetMousePosition, { signal });
document.addEventListener("visibilitychange", resetMousePosition, { signal });
}
const preventContextMenu = (e: MouseEvent) => e.preventDefault();
videoElmRefValue.addEventListener("contextmenu", preventContextMenu, { signal });
@ -580,65 +598,18 @@ export default function WebRTCVideo() {
abortController.abort();
};
},
[absMouseMoveHandler, mouseWheelHandler, resetMousePosition, settings.mouseMode],
[absMouseMoveHandler, isPointerLockActive, isPointerLockPossible, mouseWheelHandler, relMouseMoveHandler, requestPointerLock, resetMousePosition, settings.mouseMode],
);
// Setup Relative Mouse Events
const containerRef = useRef<HTMLDivElement>(null);
useEffect(
function setupRelativeMouseEventListeners() {
if (settings.mouseMode !== "relative") return;
// Relative mouse mode should only be active if the pointer lock is active and Pointer Lock is possible
const videoElmRefValue = videoElm.current;
if (!videoElmRefValue) return;
const abortController = new AbortController();
const signal = abortController.signal;
videoElmRefValue.addEventListener("mousemove", relMouseMoveHandler, { signal });
videoElmRefValue.addEventListener("pointerdown", relMouseMoveHandler, { signal });
videoElmRefValue.addEventListener("pointerup", relMouseMoveHandler, { signal });
videoElmRefValue.addEventListener(
"click",
() => {
if (isPointerLockPossible && !document.pointerLockElement) {
requestPointerLock();
}
},
{ signal },
);
videoElmRefValue.addEventListener("wheel", mouseWheelHandler, {
signal,
passive: true,
});
const preventContextMenu = (e: MouseEvent) => e.preventDefault();
videoElmRefValue.addEventListener("contextmenu", preventContextMenu, { signal });
return () => {
abortController.abort();
};
},
[
settings.mouseMode,
relMouseMoveHandler,
mouseWheelHandler,
disableVideoFocusTrap,
requestPointerLock,
isPointerLockPossible,
isPointerLockActive,
],
);
const hasNoAutoPlayPermissions = useMemo(() => {
if (peerConnection?.connectionState !== "connected") return false;
if (isPlaying) return false;
if (hdmiError) return false;
if (videoHeight === 0 || videoWidth === 0) return false;
return true;
}, [peerConnection?.connectionState, isPlaying, hdmiError, videoHeight, videoWidth]);
}, [hdmiError, isPlaying, peerConnection?.connectionState, videoHeight, videoWidth]);
const showPointerLockBar = useMemo(() => {
if (settings.mouseMode !== "relative") return false;
@ -648,15 +619,7 @@ export default function WebRTCVideo() {
if (!isPlaying) return false;
if (videoHeight === 0 || videoWidth === 0) return false;
return true;
}, [
settings.mouseMode,
isPointerLockPossible,
isPointerLockActive,
isVideoLoading,
isPlaying,
videoHeight,
videoWidth,
]);
}, [isPlaying, isPointerLockActive, isPointerLockPossible, isVideoLoading, settings.mouseMode, videoHeight, videoWidth]);
return (
<div className="grid h-full w-full grid-rows-(--grid-layout)">
@ -686,10 +649,10 @@ export default function WebRTCVideo() {
<div className="relative grow overflow-hidden">
<div className="flex h-full flex-col">
<div className="grid grow grid-rows-(--grid-bodyFooter) overflow-hidden">
{/* In relative mouse mode and under https, we enable the pointer lock, and to do so we need a bar to show the user to click on the video to enable mouse control */}
<PointerLockBar show={showPointerLockBar} />
<div className="relative mx-4 my-2 flex items-center justify-center overflow-hidden">
<div className="relative flex h-full w-full items-center justify-center">
{/* In relative mouse mode and under https, we enable the pointer lock, and to do so we need a bar to show the user to click on the video to enable mouse control */}
<PointerLockBar show={showPointerLockBar} />
<video
ref={videoElm}
autoPlay={true}