diff --git a/ui/src/components/VideoOverlay.tsx b/ui/src/components/VideoOverlay.tsx index 49ebd53..1c59e78 100644 --- a/ui/src/components/VideoOverlay.tsx +++ b/ui/src/components/VideoOverlay.tsx @@ -10,7 +10,7 @@ import LoadingSpinner from "@components/LoadingSpinner"; import Card, { GridCard } from "@components/Card"; interface OverlayContentProps { - children: React.ReactNode; + readonly children: React.ReactNode; } function OverlayContent({ children }: OverlayContentProps) { return ( @@ -23,7 +23,7 @@ function OverlayContent({ children }: OverlayContentProps) { } interface LoadingOverlayProps { - show: boolean; + readonly show: boolean; } export function LoadingVideoOverlay({ show }: LoadingOverlayProps) { @@ -57,8 +57,8 @@ export function LoadingVideoOverlay({ show }: LoadingOverlayProps) { } interface LoadingConnectionOverlayProps { - show: boolean; - text: string; + readonly show: boolean; + readonly text: string; } export function LoadingConnectionOverlay({ show, text }: LoadingConnectionOverlayProps) { return ( @@ -91,8 +91,8 @@ export function LoadingConnectionOverlay({ show, text }: LoadingConnectionOverla } interface ConnectionErrorOverlayProps { - show: boolean; - setupPeerConnection: () => Promise; + readonly show: boolean; + readonly setupPeerConnection: () => Promise; } export function ConnectionFailedOverlay({ @@ -153,7 +153,7 @@ export function ConnectionFailedOverlay({ } interface PeerConnectionDisconnectedOverlay { - show: boolean; + readonly show: boolean; } export function PeerConnectionDisconnectedOverlay({ @@ -207,8 +207,8 @@ export function PeerConnectionDisconnectedOverlay({ } interface HDMIErrorOverlayProps { - show: boolean; - hdmiState: string; + readonly show: boolean; + readonly hdmiState: string; } export function HDMIErrorOverlay({ show, hdmiState }: HDMIErrorOverlayProps) { @@ -310,8 +310,8 @@ export function HDMIErrorOverlay({ show, hdmiState }: HDMIErrorOverlayProps) { } interface NoAutoplayPermissionsOverlayProps { - show: boolean; - onPlayClick: () => void; + readonly show: boolean; + readonly onPlayClick: () => void; } export function NoAutoplayPermissionsOverlay({ @@ -361,7 +361,7 @@ export function NoAutoplayPermissionsOverlay({ } interface PointerLockBarProps { - show: boolean; + readonly show: boolean; } export function PointerLockBar({ show }: PointerLockBarProps) { @@ -369,10 +369,10 @@ export function PointerLockBar({ show }: PointerLockBarProps) { {show ? (
diff --git a/ui/src/components/WebRTCVideo.tsx b/ui/src/components/WebRTCVideo.tsx index ca4db08..c3c22f4 100644 --- a/ui/src/components/WebRTCVideo.tsx +++ b/ui/src/components/WebRTCVideo.tsx @@ -1,6 +1,15 @@ import { useCallback, useEffect, useMemo, useRef, useState } from "react"; import { useResizeObserver } from "usehooks-ts"; +import VirtualKeyboard from "@components/VirtualKeyboard"; +import Actionbar from "@components/ActionBar"; +import MacroBar from "@/components/MacroBar"; +import InfoBar from "@components/InfoBar"; +import notifications from "@/notifications"; +import useKeyboard from "@/hooks/useKeyboard"; +import { useJsonRpc } from "@/hooks/useJsonRpc"; +import { cx } from "@/cva.config"; +import { keys, modifiers } from "@/keyboardMappings"; import { useHidStore, useMouseStore, @@ -9,15 +18,6 @@ import { useUiStore, useVideoStore, } from "@/hooks/stores"; -import { keys, modifiers } from "@/keyboardMappings"; -import { cx } from "@/cva.config"; -import VirtualKeyboard from "@components/VirtualKeyboard"; -import Actionbar from "@components/ActionBar"; -import MacroBar from "@/components/MacroBar"; -import InfoBar from "@components/InfoBar"; -import useKeyboard from "@/hooks/useKeyboard"; -import { useJsonRpc } from "@/hooks/useJsonRpc"; -import notifications from "@/notifications"; import { HDMIErrorOverlay, @@ -106,8 +106,9 @@ export default function WebRTCVideo() { ); // Pointer lock and keyboard lock related - const isPointerLockPossible = window.location.protocol === "https:"; - + const isPointerLockPossible = window.location.protocol === "https:" || window.location.hostname === "localhost"; + const isFullscreenEnabled = document.fullscreenEnabled; + const checkNavigatorPermissions = useCallback(async (permissionName: string) => { const name = permissionName as PermissionName; const { state } = await navigator.permissions.query({ name }); @@ -115,23 +116,47 @@ export default function WebRTCVideo() { }, []); const requestPointerLock = useCallback(async () => { - if (document.pointerLockElement) return; + if (!isPointerLockPossible + || videoElm.current === null + || document.pointerLockElement) return; const isPointerLockGranted = await checkNavigatorPermissions("pointer-lock"); + if (isPointerLockGranted && settings.mouseMode === "relative") { - videoElm.current?.requestPointerLock(); + await videoElm.current.requestPointerLock(); } - }, [checkNavigatorPermissions, settings.mouseMode]); + }, [checkNavigatorPermissions, isPointerLockPossible, settings.mouseMode]); + + const requestKeyboardLock = useCallback(async () => { + if (videoElm.current === null) return; + + const isKeyboardLockGranted = await checkNavigatorPermissions("keyboard-lock"); + if (isKeyboardLockGranted) { + if ("keyboard" in navigator) { + // @ts-expect-error - keyboard lock is not supported in all browsers + await navigator.keyboard.lock(); + } + } + }, [checkNavigatorPermissions]); + + const releaseKeyboardLock = useCallback(async () => { + if (videoElm.current === null || document.fullscreenElement !== videoElm.current) return; + + if ("keyboard" in navigator) { + // @ts-expect-error - keyboard unlock is not supported in all browsers + await navigator.keyboard.unlock(); + } + }, []); useEffect(() => { if (!isPointerLockPossible || !videoElm.current) return; const handlePointerLockChange = () => { if (document.pointerLockElement) { - notifications.success("Pointer lock Enabled, hold escape to exit"); + notifications.success("Pointer lock Enabled, press escape to unlock"); setIsPointerLockActive(true); } else { - notifications.success("Pointer lock disabled"); + notifications.success("Pointer lock Disabled"); setIsPointerLockActive(false); } }; @@ -144,27 +169,39 @@ export default function WebRTCVideo() { return () => { abortController.abort(); }; - }, [isPointerLockPossible, videoElm]); + }, [isPointerLockPossible]); const requestFullscreen = useCallback(async () => { - videoElm.current?.requestFullscreen({ - navigationUI: "show", - }); + if (!isFullscreenEnabled || !videoElm.current) return; - // we do not care about pointer lock if it's for fullscreen + // per https://wicg.github.io/keyboard-lock/#system-key-press-handler + // If keyboard lock is activated after fullscreen is already in effect, then the user my + // see multiple messages about how to exit fullscreen. For this reason, we recommend that + // developers call lock() before they enter fullscreen: + await requestKeyboardLock(); await requestPointerLock(); - const isKeyboardLockGranted = await checkNavigatorPermissions("keyboard-lock"); - if (isKeyboardLockGranted) { - if ("keyboard" in navigator) { - // @ts-expect-error - keyboard lock is not supported in all browsers - await navigator.keyboard.lock(); + await videoElm.current.requestFullscreen({ + navigationUI: "show", + }); + }, [isFullscreenEnabled, requestKeyboardLock, requestPointerLock]); + + // setup to release the keyboard lock anytime the fullscreen ends + useEffect(() => { + if (!videoElm.current) return; + + const handleFullscreenChange = () => { + if (!document.fullscreenElement) { + releaseKeyboardLock(); } - } - }, [requestPointerLock, checkNavigatorPermissions]); + }; + + document.addEventListener("fullscreenchange ", handleFullscreenChange); + }, [releaseKeyboardLock]); // Mouse-related const calcDelta = (pos: number) => (Math.abs(pos) < 10 ? pos * 2 : pos); + const sendRelMouseMovement = useCallback( (x: number, y: number, buttons: number) => { if (settings.mouseMode !== "relative") return; @@ -179,18 +216,13 @@ export default function WebRTCVideo() { const relMouseMoveHandler = useCallback( (e: MouseEvent) => { if (settings.mouseMode !== "relative") return; - if (isPointerLockActive === false && isPointerLockPossible === true) return; + if (isPointerLockActive === false && isPointerLockPossible) return; // Send mouse movement const { buttons } = e; sendRelMouseMovement(e.movementX, e.movementY, buttons); }, - [ - isPointerLockActive, - isPointerLockPossible, - sendRelMouseMovement, - settings.mouseMode, - ], + [isPointerLockActive, isPointerLockPossible, sendRelMouseMovement, settings.mouseMode], ); const sendAbsMouseMovement = useCallback( @@ -244,14 +276,7 @@ export default function WebRTCVideo() { const { buttons } = e; sendAbsMouseMovement(x, y, buttons); }, - [ - sendAbsMouseMovement, - videoClientHeight, - videoClientWidth, - videoWidth, - videoHeight, - settings.mouseMode, - ], + [settings.mouseMode, videoClientWidth, videoClientHeight, videoWidth, videoHeight, sendAbsMouseMovement], ); const mouseWheelHandler = useCallback( @@ -263,7 +288,7 @@ export default function WebRTCVideo() { const accelScrollValue = e.deltaY / 100; // Calculate the no accel scroll value - const noAccelScrollValue = e.deltaY > 0 ? 1 : e.deltaY < 0 ? -1 : 0; + const noAccelScrollValue = Math.sign(e.deltaY); // Get scroll value const scrollValue = isAccel ? accelScrollValue : noAccelScrollValue; @@ -356,13 +381,6 @@ export default function WebRTCVideo() { let code = e.code; const key = e.key; - // if (document.activeElement?.id !== "videoFocusTrap") { - // console.log("KEYUP: Not focusing on the video", document.activeElement); - // return; - // } - - // console.log(document.activeElement); - if (!isKeyboardLedManagedByHost) { setIsNumLockActive(e.getModifierState("NumLock")); setIsCapsLockActive(e.getModifierState("CapsLock")); @@ -440,13 +458,15 @@ export default function WebRTCVideo() { ); const videoKeyUpHandler = useCallback((e: KeyboardEvent) => { + if (!videoElm.current) return; + // In fullscreen mode in chrome & safari, the space key is used to pause/play the video // there is no way to prevent this, so we need to simply force play the video when it's paused. // Fix only works in chrome based browsers. if (e.code === "Space") { - if (videoElm.current?.paused == true) { + if (videoElm.current.paused) { console.log("Force playing video"); - videoElm.current?.play(); + videoElm.current.play(); } } }, []); @@ -455,7 +475,6 @@ export default function WebRTCVideo() { (mediaStream: MediaStream) => { if (!videoElm.current) return; const videoElmRefValue = videoElm.current; - // console.log("Adding stream to video element", videoElmRefValue); videoElmRefValue.srcObject = mediaStream; updateVideoSizeStore(videoElmRefValue); }, @@ -471,7 +490,6 @@ export default function WebRTCVideo() { peerConnection.addEventListener( "track", (e: RTCTrackEvent) => { - // console.log("Adding stream to video element"); addStreamToVideoElm(e.streams[0]); }, { signal }, @@ -487,7 +505,6 @@ export default function WebRTCVideo() { useEffect( function updateVideoStream() { if (!mediaStream) return; - console.log("Updating video stream from mediaStream"); // We set the as early as possible addStreamToVideoElm(mediaStream); }, @@ -509,9 +526,6 @@ export default function WebRTCVideo() { document.addEventListener("keydown", keyDownHandler, { signal }); document.addEventListener("keyup", keyUpHandler, { signal }); - // eslint-disable-next-line @typescript-eslint/ban-ts-comment - // @ts-expect-error - window.clearKeys = () => sendKeyboardEvent([], []); window.addEventListener("blur", resetKeyboardState, { signal }); document.addEventListener("visibilitychange", resetKeyboardState, { signal }); @@ -519,7 +533,7 @@ export default function WebRTCVideo() { abortController.abort(); }; }, - [keyDownHandler, keyUpHandler, resetKeyboardState, sendKeyboardEvent], + [keyDownHandler, keyUpHandler, resetKeyboardState], ); // Setup Video Event Listeners @@ -541,38 +555,42 @@ export default function WebRTCVideo() { abortController.abort(); }; }, - [ - absMouseMoveHandler, - resetMousePosition, - onVideoPlaying, - mouseWheelHandler, - videoKeyUpHandler, - ], + [onVideoPlaying, videoKeyUpHandler], ); - // Setup Absolute Mouse Events + // Setup Mouse Events useEffect( - function setAbsoluteMouseModeEventListeners() { + function setMouseModeEventListeners() { const videoElmRefValue = videoElm.current; if (!videoElmRefValue) return; - - if (settings.mouseMode !== "absolute") return; + const isRelativeMouseMode = (settings.mouseMode === "relative"); const abortController = new AbortController(); const signal = abortController.signal; - videoElmRefValue.addEventListener("mousemove", absMouseMoveHandler, { signal }); - videoElmRefValue.addEventListener("pointerdown", absMouseMoveHandler, { signal }); - videoElmRefValue.addEventListener("pointerup", absMouseMoveHandler, { signal }); + videoElmRefValue.addEventListener("mousemove", isRelativeMouseMode ? relMouseMoveHandler : absMouseMoveHandler, { signal }); + videoElmRefValue.addEventListener("pointerdown", isRelativeMouseMode ? relMouseMoveHandler : absMouseMoveHandler, { signal }); + videoElmRefValue.addEventListener("pointerup", isRelativeMouseMode ? relMouseMoveHandler :absMouseMoveHandler, { signal }); videoElmRefValue.addEventListener("wheel", mouseWheelHandler, { signal, passive: true, }); - // Reset the mouse position when the window is blurred or the document is hidden - const local = resetMousePosition; - window.addEventListener("blur", local, { signal }); - document.addEventListener("visibilitychange", local, { signal }); + if (isRelativeMouseMode) { + videoElmRefValue.addEventListener("click", + () => { + if (isPointerLockPossible && !isPointerLockActive && !document.pointerLockElement) { + requestPointerLock(); + } + }, + { signal }, + ); + } else { + // Reset the mouse position when the window is blurred or the document is hidden + window.addEventListener("blur", resetMousePosition, { signal }); + document.addEventListener("visibilitychange", resetMousePosition, { signal }); + } + const preventContextMenu = (e: MouseEvent) => e.preventDefault(); videoElmRefValue.addEventListener("contextmenu", preventContextMenu, { signal }); @@ -580,65 +598,18 @@ export default function WebRTCVideo() { abortController.abort(); }; }, - [absMouseMoveHandler, mouseWheelHandler, resetMousePosition, settings.mouseMode], + [absMouseMoveHandler, isPointerLockActive, isPointerLockPossible, mouseWheelHandler, relMouseMoveHandler, requestPointerLock, resetMousePosition, settings.mouseMode], ); - // Setup Relative Mouse Events const containerRef = useRef(null); - useEffect( - function setupRelativeMouseEventListeners() { - if (settings.mouseMode !== "relative") return; - // Relative mouse mode should only be active if the pointer lock is active and Pointer Lock is possible - - const videoElmRefValue = videoElm.current; - if (!videoElmRefValue) return; - - const abortController = new AbortController(); - const signal = abortController.signal; - - videoElmRefValue.addEventListener("mousemove", relMouseMoveHandler, { signal }); - videoElmRefValue.addEventListener("pointerdown", relMouseMoveHandler, { signal }); - videoElmRefValue.addEventListener("pointerup", relMouseMoveHandler, { signal }); - videoElmRefValue.addEventListener( - "click", - () => { - if (isPointerLockPossible && !document.pointerLockElement) { - requestPointerLock(); - } - }, - { signal }, - ); - videoElmRefValue.addEventListener("wheel", mouseWheelHandler, { - signal, - passive: true, - }); - - const preventContextMenu = (e: MouseEvent) => e.preventDefault(); - videoElmRefValue.addEventListener("contextmenu", preventContextMenu, { signal }); - - return () => { - abortController.abort(); - }; - }, - [ - settings.mouseMode, - relMouseMoveHandler, - mouseWheelHandler, - disableVideoFocusTrap, - requestPointerLock, - isPointerLockPossible, - isPointerLockActive, - ], - ); - const hasNoAutoPlayPermissions = useMemo(() => { if (peerConnection?.connectionState !== "connected") return false; if (isPlaying) return false; if (hdmiError) return false; if (videoHeight === 0 || videoWidth === 0) return false; return true; - }, [peerConnection?.connectionState, isPlaying, hdmiError, videoHeight, videoWidth]); + }, [hdmiError, isPlaying, peerConnection?.connectionState, videoHeight, videoWidth]); const showPointerLockBar = useMemo(() => { if (settings.mouseMode !== "relative") return false; @@ -648,15 +619,7 @@ export default function WebRTCVideo() { if (!isPlaying) return false; if (videoHeight === 0 || videoWidth === 0) return false; return true; - }, [ - settings.mouseMode, - isPointerLockPossible, - isPointerLockActive, - isVideoLoading, - isPlaying, - videoHeight, - videoWidth, - ]); + }, [isPlaying, isPointerLockActive, isPointerLockPossible, isVideoLoading, settings.mouseMode, videoHeight, videoWidth]); return (
@@ -686,10 +649,10 @@ export default function WebRTCVideo() {
+ {/* In relative mouse mode and under https, we enable the pointer lock, and to do so we need a bar to show the user to click on the video to enable mouse control */} +
- {/* In relative mouse mode and under https, we enable the pointer lock, and to do so we need a bar to show the user to click on the video to enable mouse control */} -