import { useCallback, useEffect, useMemo, useRef, useState } from "react"; import { useResizeObserver } from "usehooks-ts"; import VirtualKeyboard from "@components/VirtualKeyboard"; import Actionbar from "@components/ActionBar"; import MacroBar from "@/components/MacroBar"; import InfoBar from "@components/InfoBar"; import notifications from "@/notifications"; import useKeyboard from "@/hooks/useKeyboard"; import { cx } from "@/cva.config"; import { keys } from "@/keyboardMappings"; import { useRTCStore, useSettingsStore, useVideoStore, } from "@/hooks/stores"; import useMouse from "@/hooks/useMouse"; import { HDMIErrorOverlay, LoadingVideoOverlay, NoAutoplayPermissionsOverlay, PointerLockBar, } from "./VideoOverlay"; export default function WebRTCVideo() { // Video and stream related refs and states const videoElm = useRef(null); const { mediaStream, peerConnectionState } = useRTCStore(); const [isPlaying, setIsPlaying] = useState(false); const [isPointerLockActive, setIsPointerLockActive] = useState(false); const [isKeyboardLockActive, setIsKeyboardLockActive] = useState(false); const isPointerLockPossible = window.location.protocol === "https:" || window.location.hostname === "localhost"; // Store hooks const settings = useSettingsStore(); const { handleKeyPress, resetKeyboardState } = useKeyboard(); const { getRelMouseMoveHandler, getAbsMouseMoveHandler, getMouseWheelHandler, resetMousePosition, } = useMouse(); const { setClientSize: setVideoClientSize, setSize: setVideoSize, width: videoWidth, height: videoHeight, clientWidth: videoClientWidth, clientHeight: videoClientHeight, hdmiState, } = useVideoStore(); // Video enhancement settings const { videoSaturation, videoBrightness, videoContrast } = useSettingsStore(); // RTC related states const { peerConnection } = useRTCStore(); // HDMI and UI states const hdmiError = ["no_lock", "no_signal", "out_of_range"].includes(hdmiState); const isVideoLoading = !isPlaying; // Video-related const handleResize = useCallback( ({ width, height }: { width: number | undefined; height: number | undefined }) => { if (!videoElm.current) return; // Do something with width and height, e.g.: setVideoClientSize(width || 0, height || 0); setVideoSize(videoElm.current.videoWidth, videoElm.current.videoHeight); }, [setVideoClientSize, setVideoSize] ); useResizeObserver({ ref: videoElm as React.RefObject, onResize: handleResize, }); const updateVideoSizeStore = useCallback( (videoElm: HTMLVideoElement) => { setVideoClientSize(videoElm.clientWidth, videoElm.clientHeight); setVideoSize(videoElm.videoWidth, videoElm.videoHeight); }, [setVideoClientSize, setVideoSize], ); const onVideoPlaying = useCallback(() => { setIsPlaying(true); if (videoElm.current) updateVideoSizeStore(videoElm.current); }, [updateVideoSizeStore]); // On mount, get the video size useEffect( function updateVideoSizeOnMount() { if (videoElm.current) updateVideoSizeStore(videoElm.current); }, [updateVideoSizeStore], ); // Pointer lock and keyboard lock related const isFullscreenEnabled = document.fullscreenEnabled; const checkNavigatorPermissions = useCallback(async (permissionName: string) => { if (!navigator || !navigator.permissions || !navigator.permissions.query) { return false; // if can't query permissions, assume NOT granted } try { const name = permissionName as PermissionName; const { state } = await navigator.permissions.query({ name }); return state === "granted"; } catch { // ignore errors } return false; // if query fails, assume NOT granted }, []); const requestPointerLock = useCallback(async () => { if (!isPointerLockPossible || videoElm.current === null || document.pointerLockElement) return; const isPointerLockGranted = await checkNavigatorPermissions("pointer-lock"); if (isPointerLockGranted && settings.mouseMode === "relative") { try { await videoElm.current.requestPointerLock(); } catch { // ignore errors } } }, [checkNavigatorPermissions, isPointerLockPossible, settings.mouseMode]); const requestKeyboardLock = useCallback(async () => { if (videoElm.current === null) return; const isKeyboardLockGranted = await checkNavigatorPermissions("keyboard-lock"); if (isKeyboardLockGranted && navigator && "keyboard" in navigator) { try { // @ts-expect-error - keyboard lock is not supported in all browsers await navigator.keyboard.lock(); setIsKeyboardLockActive(true); } catch { // ignore errors } } }, [checkNavigatorPermissions, setIsKeyboardLockActive]); const releaseKeyboardLock = useCallback(async () => { if (videoElm.current === null || document.fullscreenElement !== videoElm.current) return; if (navigator && "keyboard" in navigator) { try { // @ts-expect-error - keyboard unlock is not supported in all browsers await navigator.keyboard.unlock(); } catch { // ignore errors } setIsKeyboardLockActive(false); } }, [setIsKeyboardLockActive]); useEffect(() => { if (!isPointerLockPossible || !videoElm.current) return; const handlePointerLockChange = () => { if (document.pointerLockElement) { notifications.success("Pointer lock Enabled, press escape to unlock"); setIsPointerLockActive(true); } else { notifications.success("Pointer lock Disabled"); setIsPointerLockActive(false); } }; const abortController = new AbortController(); const signal = abortController.signal; document.addEventListener("pointerlockchange", handlePointerLockChange, { signal }); return () => { abortController.abort(); }; }, [isPointerLockPossible]); const requestFullscreen = useCallback(async () => { if (!isFullscreenEnabled || !videoElm.current) return; // per https://wicg.github.io/keyboard-lock/#system-key-press-handler // If keyboard lock is activated after fullscreen is already in effect, then the user my // see multiple messages about how to exit fullscreen. For this reason, we recommend that // developers call lock() before they enter fullscreen: await requestKeyboardLock(); await requestPointerLock(); await videoElm.current.requestFullscreen({ navigationUI: "show", }); }, [isFullscreenEnabled, requestKeyboardLock, requestPointerLock]); // setup to release the keyboard lock anytime the fullscreen ends useEffect(() => { if (!videoElm.current) return; const handleFullscreenChange = () => { if (!document.fullscreenElement) { releaseKeyboardLock(); } }; document.addEventListener("fullscreenchange", handleFullscreenChange); }, [releaseKeyboardLock]); const absMouseMoveHandler = useMemo( () => getAbsMouseMoveHandler({ videoClientWidth, videoClientHeight, videoWidth, videoHeight, }), [getAbsMouseMoveHandler, videoClientWidth, videoClientHeight, videoWidth, videoHeight], ); const relMouseMoveHandler = useMemo( () => getRelMouseMoveHandler(), [getRelMouseMoveHandler], ); const mouseWheelHandler = useMemo( () => getMouseWheelHandler(), [getMouseWheelHandler], ); const keyDownHandler = useCallback( (e: KeyboardEvent) => { e.preventDefault(); const code = getAdjustedKeyCode(e); const hidKey = keys[code]; if (hidKey === undefined) { console.warn(`Key down not mapped: ${code}`); return; } // When pressing the meta key + another key, the key will never trigger a keyup // event, so we need to clear the keys after a short delay // https://bugs.chromium.org/p/chromium/issues/detail?id=28089 // https://bugzilla.mozilla.org/show_bug.cgi?id=1299553 if (e.metaKey && hidKey < 0xE0) { setTimeout(() => { console.debug(`Forcing the meta key release of associated key: ${hidKey}`); handleKeyPress(hidKey, false); }, 10); } console.debug(`Key down: ${hidKey}`); handleKeyPress(hidKey, true); if (!isKeyboardLockActive && hidKey === keys.MetaLeft) { // If the left meta key was just pressed and we're not keyboard locked // we'll never see the keyup event because the browser is going to lose // focus so set a deferred keyup after a short delay setTimeout(() => { console.debug(`Forcing the left meta key release`); handleKeyPress(hidKey, false); }, 100); } }, [handleKeyPress, isKeyboardLockActive], ); const keyUpHandler = useCallback( async (e: KeyboardEvent) => { e.preventDefault(); const code = getAdjustedKeyCode(e); const hidKey = keys[code]; if (hidKey === undefined) { console.warn(`Key up not mapped: ${code}`); return; } console.debug(`Key up: ${hidKey}`); handleKeyPress(hidKey, false); }, [handleKeyPress], ); const videoKeyUpHandler = useCallback((e: KeyboardEvent) => { if (!videoElm.current) return; // In fullscreen mode in chrome & safari, the space key is used to pause/play the video // there is no way to prevent this, so we need to simply force play the video when it's paused. // Fix only works in chrome based browsers. if (e.code === "Space") { if (videoElm.current.paused) { console.debug("Force playing video"); videoElm.current.play(); } } }, []); const addStreamToVideoElm = useCallback( (mediaStream: MediaStream) => { if (!videoElm.current) return; const videoElmRefValue = videoElm.current; videoElmRefValue.srcObject = mediaStream; updateVideoSizeStore(videoElmRefValue); }, [updateVideoSizeStore], ); useEffect( function updateVideoStreamOnNewTrack() { if (!peerConnection) return; const abortController = new AbortController(); const signal = abortController.signal; peerConnection.addEventListener( "track", (e: RTCTrackEvent) => { addStreamToVideoElm(e.streams[0]); }, { signal }, ); return () => { abortController.abort(); }; }, [addStreamToVideoElm, peerConnection], ); useEffect( function updateVideoStream() { if (!mediaStream) return; // We set the as early as possible addStreamToVideoElm(mediaStream); }, [addStreamToVideoElm, mediaStream], ); // Setup Keyboard Events useEffect( function setupKeyboardEvents() { const abortController = new AbortController(); const signal = abortController.signal; document.addEventListener("keydown", keyDownHandler, { signal }); document.addEventListener("keyup", keyUpHandler, { signal }); window.addEventListener("blur", resetKeyboardState, { signal }); document.addEventListener("visibilitychange", resetKeyboardState, { signal }); return () => { abortController.abort(); }; }, [keyDownHandler, keyUpHandler, resetKeyboardState], ); // Setup Video Event Listeners useEffect( function setupVideoEventListeners() { const videoElmRefValue = videoElm.current; if (!videoElmRefValue) return; const abortController = new AbortController(); const signal = abortController.signal; // To prevent the video from being paused when the user presses a space in fullscreen mode videoElmRefValue.addEventListener("keyup", videoKeyUpHandler, { signal }); // We need to know when the video is playing to update state and video size videoElmRefValue.addEventListener("playing", onVideoPlaying, { signal }); return () => { abortController.abort(); }; }, [onVideoPlaying, videoKeyUpHandler], ); // Setup Mouse Events useEffect( function setMouseModeEventListeners() { const videoElmRefValue = videoElm.current; if (!videoElmRefValue) return; const isRelativeMouseMode = (settings.mouseMode === "relative"); const mouseHandler = isRelativeMouseMode ? relMouseMoveHandler : absMouseMoveHandler; const abortController = new AbortController(); const signal = abortController.signal; videoElmRefValue.addEventListener("mousemove", mouseHandler, { signal }); videoElmRefValue.addEventListener("pointerdown", mouseHandler, { signal }); videoElmRefValue.addEventListener("pointerup", mouseHandler, { signal }); videoElmRefValue.addEventListener("wheel", mouseWheelHandler, { signal, passive: true, }); if (isRelativeMouseMode) { videoElmRefValue.addEventListener("click", () => { if (isPointerLockPossible && !isPointerLockActive && !document.pointerLockElement) { requestPointerLock(); } }, { signal }, ); } else { // Reset the mouse position when the window is blurred or the document is hidden window.addEventListener("blur", resetMousePosition, { signal }); document.addEventListener("visibilitychange", resetMousePosition, { signal }); } const preventContextMenu = (e: MouseEvent) => e.preventDefault(); videoElmRefValue.addEventListener("contextmenu", preventContextMenu, { signal }); return () => { abortController.abort(); }; }, [ isPointerLockActive, isPointerLockPossible, requestPointerLock, absMouseMoveHandler, relMouseMoveHandler, mouseWheelHandler, resetMousePosition, settings.mouseMode, ], ); const containerRef = useRef(null); const hasNoAutoPlayPermissions = useMemo(() => { if (peerConnection?.connectionState !== "connected") return false; if (isPlaying) return false; if (hdmiError) return false; if (videoHeight === 0 || videoWidth === 0) return false; return true; }, [hdmiError, isPlaying, peerConnection?.connectionState, videoHeight, videoWidth]); const showPointerLockBar = useMemo(() => { if (settings.mouseMode !== "relative") return false; if (!isPointerLockPossible) return false; if (isPointerLockActive) return false; if (isVideoLoading) return false; if (!isPlaying) return false; if (videoHeight === 0 || videoWidth === 0) return false; return true; }, [isPlaying, isPointerLockActive, isPointerLockPossible, isVideoLoading, settings.mouseMode, videoHeight, videoWidth]); // Conditionally set the filter style so we don't fallback to software rendering if these values are default of 1.0 const videoStyle = useMemo(() => { const isDefault = videoSaturation === 1.0 && videoBrightness === 1.0 && videoContrast === 1.0; return isDefault ? {} // No filter if all settings are default (1.0) : { filter: `saturate(${videoSaturation}) brightness(${videoBrightness}) contrast(${videoContrast})`, }; }, [videoSaturation, videoBrightness, videoContrast]); function getAdjustedKeyCode(e: KeyboardEvent) { const key = e.key; let code = e.code; if (code == "IntlBackslash" && ["`", "~"].includes(key)) { code = "Backquote"; } else if (code == "Backquote" && ["§", "±"].includes(key)) { code = "IntlBackslash"; } return code; } return (
{/* In relative mouse mode and under https, we enable the pointer lock, and to do so we need a bar to show the user to click on the video to enable mouse control */}
); }