fix(ui): Fix full-screen keyboard lock issues. (#535)

When the keyboard lock is supposed to be active (in full-screen mode), hitting the escape key (NOT long-pressing) should NOT dismiss the full-screen mode, and should send the Escape key through to the remote.

- Added awaits to the browser calls that need to complete in order.
- Cleaned up (mostly) duplicate code in the Absolute/Relative mouse handling
- Ensure we don't overrun any existing keyboard lock or pointer lock
- Release the keyboard lock when leaving full-screen
- Per standards, we need to acquire the keyboard and pointer locks before entering full-screen or the user may get multiple messages about exiting.
- Fixed all the missing/excess React dependencies.
- Moved the pointer lock bar up so it is visible.
- Somewhere along the way, the prompt to click the video when in relative-mouse-mode stopped being visible, restored it's visibility
- Fixed all the "should be readonly" warnings.

ui/src/components/VideoOverlay.tsx

@@ -10,7 +10,7 @@ import LoadingSpinner from "@components/LoadingSpinner";
 import Card, { GridCard } from "@components/Card";
 
 interface OverlayContentProps {
-  children: React.ReactNode;
+  readonly children: React.ReactNode;
 }
 function OverlayContent({ children }: OverlayContentProps) {
   return (
@@ -23,7 +23,7 @@ function OverlayContent({ children }: OverlayContentProps) {
 }
 
 interface LoadingOverlayProps {
-  show: boolean;
+  readonly show: boolean;
 }
 
 export function LoadingVideoOverlay({ show }: LoadingOverlayProps) {
@@ -57,8 +57,8 @@ export function LoadingVideoOverlay({ show }: LoadingOverlayProps) {
 }
 
 interface LoadingConnectionOverlayProps {
-  show: boolean;
+  readonly show: boolean;
-  text: string;
+  readonly text: string;
 }
 export function LoadingConnectionOverlay({ show, text }: LoadingConnectionOverlayProps) {
   return (
@@ -91,8 +91,8 @@ export function LoadingConnectionOverlay({ show, text }: LoadingConnectionOverla
 }
 
 interface ConnectionErrorOverlayProps {
-  show: boolean;
+  readonly show: boolean;
-  setupPeerConnection: () => Promise<void>;
+  readonly setupPeerConnection: () => Promise<void>;
 }
 
 export function ConnectionFailedOverlay({
@@ -153,7 +153,7 @@ export function ConnectionFailedOverlay({
 }
 
 interface PeerConnectionDisconnectedOverlay {
-  show: boolean;
+  readonly show: boolean;
 }
 
 export function PeerConnectionDisconnectedOverlay({
@@ -207,8 +207,8 @@ export function PeerConnectionDisconnectedOverlay({
 }
 
 interface HDMIErrorOverlayProps {
-  show: boolean;
+  readonly show: boolean;
-  hdmiState: string;
+  readonly hdmiState: string;
 }
 
 export function HDMIErrorOverlay({ show, hdmiState }: HDMIErrorOverlayProps) {
@@ -310,8 +310,8 @@ export function HDMIErrorOverlay({ show, hdmiState }: HDMIErrorOverlayProps) {
 }
 
 interface NoAutoplayPermissionsOverlayProps {
-  show: boolean;
+  readonly show: boolean;
-  onPlayClick: () => void;
+  readonly onPlayClick: () => void;
 }
 
 export function NoAutoplayPermissionsOverlay({
@@ -361,7 +361,7 @@ export function NoAutoplayPermissionsOverlay({
 }
 
 interface PointerLockBarProps {
-  show: boolean;
+  readonly show: boolean;
 }
 
 export function PointerLockBar({ show }: PointerLockBarProps) {
@@ -369,10 +369,10 @@ export function PointerLockBar({ show }: PointerLockBarProps) {
     <AnimatePresence mode="wait">
       {show ? (
         <motion.div
-          className="absolute -top-[36px] left-0 right-0 z-20 bg-white"
+          className="flex w-full items-center justify-between bg-transparent"
-          initial={{ y: 20, opacity: 0, zIndex: 0 }}
+          initial={{ opacity: 0, zIndex: 0 }}
-          animate={{ opacity: 1, y: 0 }}
+          animate={{ opacity: 1, zIndex: 20 }}
-          exit={{ y: 43, zIndex: 0 }}
+          exit={{ opacity: 0, zIndex: 0 }}
           transition={{ duration: 0.5, ease: "easeInOut", delay: 0.5 }}
         >
           <div>

ui/src/components/WebRTCVideo.tsx

@@ -1,6 +1,15 @@
 import { useCallback, useEffect, useMemo, useRef, useState } from "react";
 import { useResizeObserver } from "usehooks-ts";
 
+import VirtualKeyboard from "@components/VirtualKeyboard";
+import Actionbar from "@components/ActionBar";
+import MacroBar from "@/components/MacroBar";
+import InfoBar from "@components/InfoBar";
+import notifications from "@/notifications";
+import useKeyboard from "@/hooks/useKeyboard";
+import { useJsonRpc } from "@/hooks/useJsonRpc";
+import { cx } from "@/cva.config";
+import { keys, modifiers } from "@/keyboardMappings";
 import {
   useHidStore,
   useMouseStore,
@@ -9,15 +18,6 @@ import {
   useUiStore,
   useVideoStore,
 } from "@/hooks/stores";
-import { keys, modifiers } from "@/keyboardMappings";
-import { cx } from "@/cva.config";
-import VirtualKeyboard from "@components/VirtualKeyboard";
-import Actionbar from "@components/ActionBar";
-import MacroBar from "@/components/MacroBar";
-import InfoBar from "@components/InfoBar";
-import useKeyboard from "@/hooks/useKeyboard";
-import { useJsonRpc } from "@/hooks/useJsonRpc";
-import notifications from "@/notifications";
 
 import {
   HDMIErrorOverlay,
@@ -106,7 +106,8 @@ export default function WebRTCVideo() {
   );
 
   // Pointer lock and keyboard lock related
-  const isPointerLockPossible = window.location.protocol === "https:";
+  const isPointerLockPossible = window.location.protocol === "https:" || window.location.hostname === "localhost";
+  const isFullscreenEnabled = document.fullscreenEnabled;
  
   const checkNavigatorPermissions = useCallback(async (permissionName: string) => {
     const name = permissionName as PermissionName;
@@ -115,23 +116,47 @@ export default function WebRTCVideo() {
   }, []);
 
   const requestPointerLock = useCallback(async () => {
-    if (document.pointerLockElement) return;
+    if (!isPointerLockPossible
+      || videoElm.current === null
+      || document.pointerLockElement) return;
 
     const isPointerLockGranted = await checkNavigatorPermissions("pointer-lock");
+
     if (isPointerLockGranted && settings.mouseMode === "relative") {
-      videoElm.current?.requestPointerLock();
+      await videoElm.current.requestPointerLock();
     }
-  }, [checkNavigatorPermissions, settings.mouseMode]);
+  }, [checkNavigatorPermissions, isPointerLockPossible, settings.mouseMode]);
+
+  const requestKeyboardLock = useCallback(async () => {
+    if (videoElm.current === null) return;
+
+    const isKeyboardLockGranted = await checkNavigatorPermissions("keyboard-lock");
+    if (isKeyboardLockGranted) {
+      if ("keyboard" in navigator) {
+        // @ts-expect-error - keyboard lock is not supported in all browsers
+        await navigator.keyboard.lock();
+      }
+    }
+  }, [checkNavigatorPermissions]);
+
+  const releaseKeyboardLock = useCallback(async () => {
+    if (videoElm.current === null || document.fullscreenElement !== videoElm.current) return;
+
+    if ("keyboard" in navigator) {
+        // @ts-expect-error - keyboard unlock is not supported in all browsers
+        await navigator.keyboard.unlock();
+    }
+  }, []);
 
   useEffect(() => {
     if (!isPointerLockPossible || !videoElm.current) return;
 
     const handlePointerLockChange = () => {
       if (document.pointerLockElement) {
-        notifications.success("Pointer lock Enabled, hold escape to exit");
+        notifications.success("Pointer lock Enabled, press escape to unlock");
         setIsPointerLockActive(true);
       } else {
-        notifications.success("Pointer lock disabled");
+        notifications.success("Pointer lock Disabled");
         setIsPointerLockActive(false);
       }
     };
@@ -144,27 +169,39 @@ export default function WebRTCVideo() {
     return () => {
       abortController.abort();
     };
-  }, [isPointerLockPossible, videoElm]);
+  }, [isPointerLockPossible]);
 
   const requestFullscreen = useCallback(async () => {
-    videoElm.current?.requestFullscreen({
+     if (!isFullscreenEnabled || !videoElm.current) return;
-      navigationUI: "show",
-    });
 
-    // we do not care about pointer lock if it's for fullscreen
+    // per https://wicg.github.io/keyboard-lock/#system-key-press-handler
+    // If keyboard lock is activated after fullscreen is already in effect, then the user my 
+    // see multiple messages about how to exit fullscreen. For this reason, we recommend that
+    // developers call lock() before they enter fullscreen:
+    await requestKeyboardLock();
     await requestPointerLock();
 
-    const isKeyboardLockGranted = await checkNavigatorPermissions("keyboard-lock");
+    await videoElm.current.requestFullscreen({
-    if (isKeyboardLockGranted) {
+      navigationUI: "show",
-      if ("keyboard" in navigator) {
+    });
-        // @ts-expect-error - keyboard lock is not supported in all browsers
+  }, [isFullscreenEnabled, requestKeyboardLock, requestPointerLock]);
-        await navigator.keyboard.lock();
+
+  // setup to release the keyboard lock anytime the fullscreen ends
+  useEffect(() => {
+    if (!videoElm.current) return;
+
+    const handleFullscreenChange = () => {
+      if (!document.fullscreenElement) {
+        releaseKeyboardLock();
       }
-    }
+    };
-  }, [requestPointerLock, checkNavigatorPermissions]);
+
+    document.addEventListener("fullscreenchange ", handleFullscreenChange);
+  }, [releaseKeyboardLock]);
 
   // Mouse-related
   const calcDelta = (pos: number) => (Math.abs(pos) < 10 ? pos * 2 : pos);
+
   const sendRelMouseMovement = useCallback(
     (x: number, y: number, buttons: number) => {
       if (settings.mouseMode !== "relative") return;
@@ -179,18 +216,13 @@ export default function WebRTCVideo() {
   const relMouseMoveHandler = useCallback(
     (e: MouseEvent) => {
       if (settings.mouseMode !== "relative") return;
-      if (isPointerLockActive === false && isPointerLockPossible === true) return;
+      if (isPointerLockActive === false && isPointerLockPossible) return;
 
       // Send mouse movement
       const { buttons } = e;
       sendRelMouseMovement(e.movementX, e.movementY, buttons);
     },
-    [
+    [isPointerLockActive, isPointerLockPossible, sendRelMouseMovement, settings.mouseMode],
-      isPointerLockActive,
-      isPointerLockPossible,
-      sendRelMouseMovement,
-      settings.mouseMode,
-    ],
   );
 
   const sendAbsMouseMovement = useCallback(
@@ -244,14 +276,7 @@ export default function WebRTCVideo() {
       const { buttons } = e;
       sendAbsMouseMovement(x, y, buttons);
     },
-    [
+    [settings.mouseMode, videoClientWidth, videoClientHeight, videoWidth, videoHeight, sendAbsMouseMovement],
-      sendAbsMouseMovement,
-      videoClientHeight,
-      videoClientWidth,
-      videoWidth,
-      videoHeight,
-      settings.mouseMode,
-    ],
   );
 
   const mouseWheelHandler = useCallback(
@@ -263,7 +288,7 @@ export default function WebRTCVideo() {
       const accelScrollValue = e.deltaY / 100;
 
       // Calculate the no accel scroll value
-      const noAccelScrollValue = e.deltaY > 0 ? 1 : e.deltaY < 0 ? -1 : 0;
+      const noAccelScrollValue = Math.sign(e.deltaY);
 
       // Get scroll value
       const scrollValue = isAccel ? accelScrollValue : noAccelScrollValue;
@@ -356,13 +381,6 @@ export default function WebRTCVideo() {
       let code = e.code;
       const key = e.key;
 
-      // if (document.activeElement?.id !== "videoFocusTrap") {
-      //   console.log("KEYUP: Not focusing on the video", document.activeElement);
-      //   return;
-      // }
-
-      // console.log(document.activeElement);
-
       if (!isKeyboardLedManagedByHost) {
         setIsNumLockActive(e.getModifierState("NumLock"));
         setIsCapsLockActive(e.getModifierState("CapsLock"));
@@ -440,13 +458,15 @@ export default function WebRTCVideo() {
   );
 
   const videoKeyUpHandler = useCallback((e: KeyboardEvent) => {
+    if (!videoElm.current) return;
+
     // In fullscreen mode in chrome & safari, the space key is used to pause/play the video
     // there is no way to prevent this, so we need to simply force play the video when it's paused.
     // Fix only works in chrome based browsers.
     if (e.code === "Space") {
-      if (videoElm.current?.paused == true) {
+      if (videoElm.current.paused) {
         console.log("Force playing video");
-        videoElm.current?.play();
+        videoElm.current.play();
       }
     }
   }, []);
@@ -455,7 +475,6 @@ export default function WebRTCVideo() {
     (mediaStream: MediaStream) => {
       if (!videoElm.current) return;
       const videoElmRefValue = videoElm.current;
-      // console.log("Adding stream to video element", videoElmRefValue);
       videoElmRefValue.srcObject = mediaStream;
       updateVideoSizeStore(videoElmRefValue);
     },
@@ -471,7 +490,6 @@ export default function WebRTCVideo() {
       peerConnection.addEventListener(
         "track",
         (e: RTCTrackEvent) => {
-          // console.log("Adding stream to video element");
           addStreamToVideoElm(e.streams[0]);
         },
         { signal },
@@ -487,7 +505,6 @@ export default function WebRTCVideo() {
   useEffect(
     function updateVideoStream() {
       if (!mediaStream) return;
-      console.log("Updating video stream from mediaStream");
       // We set the as early as possible
       addStreamToVideoElm(mediaStream);
     },
@@ -509,9 +526,6 @@ export default function WebRTCVideo() {
       document.addEventListener("keydown", keyDownHandler, { signal });
       document.addEventListener("keyup", keyUpHandler, { signal });
 
-      // eslint-disable-next-line @typescript-eslint/ban-ts-comment
-      // @ts-expect-error
-      window.clearKeys = () => sendKeyboardEvent([], []);
       window.addEventListener("blur", resetKeyboardState, { signal });
       document.addEventListener("visibilitychange", resetKeyboardState, { signal });
 
@@ -519,7 +533,7 @@ export default function WebRTCVideo() {
         abortController.abort();
       };
     },
-    [keyDownHandler, keyUpHandler, resetKeyboardState, sendKeyboardEvent],
+    [keyDownHandler, keyUpHandler, resetKeyboardState],
   );
 
   // Setup Video Event Listeners
@@ -541,38 +555,42 @@ export default function WebRTCVideo() {
         abortController.abort();
       };
     },
-    [
+    [onVideoPlaying, videoKeyUpHandler],
-      absMouseMoveHandler,
-      resetMousePosition,
-      onVideoPlaying,
-      mouseWheelHandler,
-      videoKeyUpHandler,
-    ],
   );
 
-  // Setup Absolute Mouse Events
+  // Setup Mouse Events
   useEffect(
-    function setAbsoluteMouseModeEventListeners() {
+    function setMouseModeEventListeners() {
       const videoElmRefValue = videoElm.current;
       if (!videoElmRefValue) return;
-
+      const isRelativeMouseMode = (settings.mouseMode === "relative");
-      if (settings.mouseMode !== "absolute") return;
 
       const abortController = new AbortController();
       const signal = abortController.signal;
 
-      videoElmRefValue.addEventListener("mousemove", absMouseMoveHandler, { signal });
+      videoElmRefValue.addEventListener("mousemove", isRelativeMouseMode ? relMouseMoveHandler : absMouseMoveHandler, { signal });
-      videoElmRefValue.addEventListener("pointerdown", absMouseMoveHandler, { signal });
+      videoElmRefValue.addEventListener("pointerdown", isRelativeMouseMode ? relMouseMoveHandler : absMouseMoveHandler, { signal });
-      videoElmRefValue.addEventListener("pointerup", absMouseMoveHandler, { signal });
+      videoElmRefValue.addEventListener("pointerup", isRelativeMouseMode ? relMouseMoveHandler :absMouseMoveHandler, { signal });
       videoElmRefValue.addEventListener("wheel", mouseWheelHandler, {
         signal,
         passive: true,
       });
 
-      // Reset the mouse position when the window is blurred or the document is hidden
+      if (isRelativeMouseMode) {
-      const local = resetMousePosition;
+        videoElmRefValue.addEventListener("click",
-      window.addEventListener("blur", local, { signal });
+          () => {
-      document.addEventListener("visibilitychange", local, { signal });
+            if (isPointerLockPossible && !isPointerLockActive && !document.pointerLockElement) {
+              requestPointerLock();
+            }
+          },
+          { signal },
+        );
+      } else {
+        // Reset the mouse position when the window is blurred or the document is hidden
+        window.addEventListener("blur", resetMousePosition, { signal });
+        document.addEventListener("visibilitychange", resetMousePosition, { signal });
+      }
+
       const preventContextMenu = (e: MouseEvent) => e.preventDefault();
       videoElmRefValue.addEventListener("contextmenu", preventContextMenu, { signal });
 
@@ -580,65 +598,18 @@ export default function WebRTCVideo() {
         abortController.abort();
       };
     },
-    [absMouseMoveHandler, mouseWheelHandler, resetMousePosition, settings.mouseMode],
+    [absMouseMoveHandler, isPointerLockActive, isPointerLockPossible, mouseWheelHandler, relMouseMoveHandler, requestPointerLock, resetMousePosition, settings.mouseMode],
   );
 
-  // Setup Relative Mouse Events
   const containerRef = useRef<HTMLDivElement>(null);
 
-  useEffect(
-    function setupRelativeMouseEventListeners() {
-      if (settings.mouseMode !== "relative") return;
-      // Relative mouse mode should only be active if the pointer lock is active and Pointer Lock is possible
-
-      const videoElmRefValue = videoElm.current;
-      if (!videoElmRefValue) return;
-
-      const abortController = new AbortController();
-      const signal = abortController.signal;
-
-      videoElmRefValue.addEventListener("mousemove", relMouseMoveHandler, { signal });
-      videoElmRefValue.addEventListener("pointerdown", relMouseMoveHandler, { signal });
-      videoElmRefValue.addEventListener("pointerup", relMouseMoveHandler, { signal });
-      videoElmRefValue.addEventListener(
-        "click",
-        () => {
-          if (isPointerLockPossible && !document.pointerLockElement) {
-            requestPointerLock();
-          }
-        },
-        { signal },
-      );
-      videoElmRefValue.addEventListener("wheel", mouseWheelHandler, {
-        signal,
-        passive: true,
-      });
-
-      const preventContextMenu = (e: MouseEvent) => e.preventDefault();
-      videoElmRefValue.addEventListener("contextmenu", preventContextMenu, { signal });
-
-      return () => {
-        abortController.abort();
-      };
-    },
-    [
-      settings.mouseMode,
-      relMouseMoveHandler,
-      mouseWheelHandler,
-      disableVideoFocusTrap,
-      requestPointerLock,
-      isPointerLockPossible,
-      isPointerLockActive,
-    ],
-  );
-
   const hasNoAutoPlayPermissions = useMemo(() => {
     if (peerConnection?.connectionState !== "connected") return false;
     if (isPlaying) return false;
     if (hdmiError) return false;
     if (videoHeight === 0 || videoWidth === 0) return false;
     return true;
-  }, [peerConnection?.connectionState, isPlaying, hdmiError, videoHeight, videoWidth]);
+  }, [hdmiError, isPlaying, peerConnection?.connectionState, videoHeight, videoWidth]);
 
   const showPointerLockBar = useMemo(() => {
     if (settings.mouseMode !== "relative") return false;
@@ -648,15 +619,7 @@ export default function WebRTCVideo() {
     if (!isPlaying) return false;
     if (videoHeight === 0 || videoWidth === 0) return false;
     return true;
-  }, [
+  }, [isPlaying, isPointerLockActive, isPointerLockPossible, isVideoLoading, settings.mouseMode, videoHeight, videoWidth]);
-    settings.mouseMode,
-    isPointerLockPossible,
-    isPointerLockActive,
-    isVideoLoading,
-    isPlaying,
-    videoHeight,
-    videoWidth,
-  ]);
 
   return (
     <div className="grid h-full w-full grid-rows-(--grid-layout)">
@@ -686,10 +649,10 @@ export default function WebRTCVideo() {
             <div className="relative grow overflow-hidden">
               <div className="flex h-full flex-col">
                 <div className="grid grow grid-rows-(--grid-bodyFooter) overflow-hidden">
+                  {/* In relative mouse mode and under https, we enable the pointer lock, and to do so we need a bar to show the user to click on the video to enable mouse control */}
+                  <PointerLockBar show={showPointerLockBar} />
                   <div className="relative mx-4 my-2 flex items-center justify-center overflow-hidden">
                     <div className="relative flex h-full w-full items-center justify-center">
-                        {/* In relative mouse mode and under https, we enable the pointer lock, and to do so we need a bar to show the user to click on the video to enable mouse control */}
-                        <PointerLockBar show={showPointerLockBar} />
                         <video
                           ref={videoElm}
                           autoPlay={true}