Merge branch 'dev' of github.com:jackislanding/jack-kvm into feature/jiggler-scheduler

.github/workflows/golangci-lint.yml

@@ -27,6 +27,9 @@ jobs:
         uses: actions/setup-go@41dfa10bad2bb2ae585af6ee5bb4d7d973ad74ed # v5.1.0
         with:
           go-version: 1.23.x
+      - name: Create empty resource directory
+        run: |
+          mkdir -p static && touch static/.gitkeep
       - name: Lint
         uses: golangci/golangci-lint-action@971e284b6050e8a5849b72094c50ab08da042db8 # v6.1.1
         with:

ui/src/components/UsbDeviceSetting.tsx

@@ -9,7 +9,6 @@ import { Button } from "./Button";
 import { SelectMenuBasic } from "./SelectMenuBasic";
 import { SettingsSectionHeader } from "./SettingsSectionHeader";
 import Fieldset from "./Fieldset";
-
 export interface USBConfig {
   vendor_id: string;
   product_id: string;
@@ -119,13 +118,12 @@ export function UsbDeviceSetting() {
 
   const onUsbConfigItemChange = useCallback(
     (key: keyof UsbDeviceConfig) => (e: React.ChangeEvent<HTMLInputElement>) => {
-      setUsbDeviceConfig(val => {
+      setUsbDeviceConfig(prev => ({
-        val[key] = e.target.checked;
+        ...prev,
-        handleUsbConfigChange(val);
+        [key]: e.target.checked,
-        return val;
+      }));
-      });
     },
-    [handleUsbConfigChange],
+    [],
   );
 
   const handlePresetChange = useCallback(

ui/src/components/VideoOverlay.tsx

@@ -1,10 +1,11 @@
 import React from "react";
 import { ExclamationTriangleIcon } from "@heroicons/react/24/solid";
 import { ArrowRightIcon } from "@heroicons/react/16/solid";
-import { LinkButton } from "@components/Button";
+import { Button, LinkButton } from "@components/Button";
 import LoadingSpinner from "@components/LoadingSpinner";
 import { GridCard } from "@components/Card";
 import { motion, AnimatePresence } from "motion/react";
+import { LuPlay } from "react-icons/lu";
 
 interface OverlayContentProps {
   children: React.ReactNode;
@@ -34,7 +35,7 @@ export function LoadingOverlay({ show }: LoadingOverlayProps) {
           exit={{ opacity: 0 }}
           transition={{
             duration: show ? 0.3 : 0.1,
-            ease: "easeInOut"
+            ease: "easeInOut",
           }}
         >
           <OverlayContent>
@@ -68,7 +69,7 @@ export function ConnectionErrorOverlay({ show }: ConnectionErrorOverlayProps) {
           exit={{ opacity: 0 }}
           transition={{
             duration: 0.3,
-            ease: "easeInOut"
+            ease: "easeInOut",
           }}
         >
           <OverlayContent>
@@ -118,25 +119,27 @@ export function HDMIErrorOverlay({ show, hdmiState }: HDMIErrorOverlayProps) {
       <AnimatePresence>
         {show && isNoSignal && (
           <motion.div
-            className="absolute inset-0 w-full h-full aspect-video"
+            className="absolute inset-0 aspect-video h-full w-full"
             initial={{ opacity: 0 }}
             animate={{ opacity: 1 }}
             exit={{ opacity: 0 }}
             transition={{
               duration: 0.3,
-              ease: "easeInOut"
+              ease: "easeInOut",
             }}
           >
             <OverlayContent>
               <div className="flex flex-col items-start gap-y-1">
-                <ExclamationTriangleIcon className="w-12 h-12 text-yellow-500" />
+                <ExclamationTriangleIcon className="h-12 w-12 text-yellow-500" />
-                <div className="text-sm text-left text-slate-700 dark:text-slate-300">
+                <div className="text-left text-sm text-slate-700 dark:text-slate-300">
                   <div className="space-y-4">
                     <div className="space-y-2 text-black dark:text-white">
                       <h2 className="text-xl font-bold">No HDMI signal detected.</h2>
                       <ul className="list-disc space-y-2 pl-4 text-left">
                         <li>Ensure the HDMI cable securely connected at both ends</li>
-                        <li>Ensure source device is powered on and outputting a signal</li>
+                        <li>
+                          Ensure source device is powered on and outputting a signal
+                        </li>
                         <li>
                           If using an adapter, it&apos;s compatible and functioning
                           correctly
@@ -169,7 +172,7 @@ export function HDMIErrorOverlay({ show, hdmiState }: HDMIErrorOverlayProps) {
             exit={{ opacity: 0 }}
             transition={{
               duration: 0.3,
-              ease: "easeInOut"
+              ease: "easeInOut",
             }}
           >
             <OverlayContent>
@@ -187,7 +190,7 @@ export function HDMIErrorOverlay({ show, hdmiState }: HDMIErrorOverlayProps) {
                     </div>
                     <div>
                       <LinkButton
-                        to={"/help/hdmi-error"}
+                        to={"https://jetkvm.com/docs/getting-started/troubleshooting"}
                         theme="light"
                         text="Learn more"
                         TrailingIcon={ArrowRightIcon}
@@ -204,3 +207,54 @@ export function HDMIErrorOverlay({ show, hdmiState }: HDMIErrorOverlayProps) {
     </>
   );
 }
+
+interface NoAutoplayPermissionsOverlayProps {
+  show: boolean;
+  onPlayClick: () => void;
+}
+
+export function NoAutoplayPermissionsOverlay({
+  show,
+  onPlayClick,
+}: NoAutoplayPermissionsOverlayProps) {
+  return (
+    <AnimatePresence>
+      {show && (
+        <motion.div
+          className="absolute inset-0 z-10 aspect-video h-full w-full"
+          initial={{ opacity: 0 }}
+          animate={{ opacity: 1 }}
+          exit={{ opacity: 0 }}
+          transition={{
+            duration: 0.3,
+            ease: "easeInOut",
+          }}
+        >
+          <OverlayContent>
+            <div className="space-y-4">
+              <h2 className="text-2xl font-extrabold text-black dark:text-white">
+                Autoplay permissions required
+              </h2>
+
+              <div className="space-y-2 text-center">
+                <div>
+                  <Button
+                    size="MD"
+                    theme="primary"
+                    LeadingIcon={LuPlay}
+                    text="Manually start stream"
+                    onClick={onPlayClick}
+                  />
+                </div>
+
+                <div className="text-xs text-slate-600 dark:text-slate-400">
+                  Please adjust browser settings to enable autoplay
+                </div>
+              </div>
+            </div>
+          </OverlayContent>
+        </motion.div>
+      )}
+    </AnimatePresence>
+  );
+}

ui/src/components/WebRTCVideo.tsx

@@ -1,4 +1,4 @@
-import { useCallback, useEffect, useRef, useState } from "react";
+import { useCallback, useEffect, useMemo, useRef, useState } from "react";
 import {
   useDeviceSettingsStore,
   useHidStore,
@@ -15,7 +15,7 @@ import Actionbar from "@components/ActionBar";
 import InfoBar from "@components/InfoBar";
 import useKeyboard from "@/hooks/useKeyboard";
 import { useJsonRpc } from "@/hooks/useJsonRpc";
-import { HDMIErrorOverlay } from "./VideoOverlay";
+import { HDMIErrorOverlay, NoAutoplayPermissionsOverlay } from "./VideoOverlay";
 import { ConnectionErrorOverlay } from "./VideoOverlay";
 import { LoadingOverlay } from "./VideoOverlay";
 
@@ -47,7 +47,7 @@ export default function WebRTCVideo() {
   const hdmiState = useVideoStore(state => state.hdmiState);
   const hdmiError = ["no_lock", "no_signal", "out_of_range"].includes(hdmiState);
   const isLoading = !hdmiError && !isPlaying;
-  const isConnectionError = ["error", "failed", "disconnected"].includes(
+  const isConnectionError = ["error", "failed", "disconnected", "closed"].includes(
     peerConnectionState || "",
   );
 
@@ -94,7 +94,7 @@ export default function WebRTCVideo() {
   );
 
   // Mouse-related
-  const calcDelta = (pos: number) => Math.abs(pos) < 10 ? pos * 2 : pos;
+  const calcDelta = (pos: number) => (Math.abs(pos) < 10 ? pos * 2 : pos);
   const sendRelMouseMovement = useCallback(
     (x: number, y: number, buttons: number) => {
       if (settings.mouseMode !== "relative") return;
@@ -168,7 +168,14 @@ export default function WebRTCVideo() {
       const { buttons } = e;
       sendAbsMouseMovement(x, y, buttons);
     },
-    [sendAbsMouseMovement, videoClientHeight, videoClientWidth, videoWidth, videoHeight, settings.mouseMode],
+    [
+      sendAbsMouseMovement,
+      videoClientHeight,
+      videoClientWidth,
+      videoWidth,
+      videoHeight,
+      settings.mouseMode,
+    ],
   );
 
   const trackpadSensitivity = useDeviceSettingsStore(state => state.trackpadSensitivity);
@@ -355,28 +362,6 @@ export default function WebRTCVideo() {
     ],
   );
 
-  // Effect hooks
-  useEffect(
-    function setupKeyboardEvents() {
-      const abortController = new AbortController();
-      const signal = abortController.signal;
-
-      document.addEventListener("keydown", keyDownHandler, { signal });
-      document.addEventListener("keyup", keyUpHandler, { signal });
-
-      // eslint-disable-next-line @typescript-eslint/ban-ts-comment
-      // @ts-expect-error
-      window.clearKeys = () => sendKeyboardEvent([], []);
-      window.addEventListener("blur", resetKeyboardState, { signal });
-      document.addEventListener("visibilitychange", resetKeyboardState, { signal });
-
-      return () => {
-        abortController.abort();
-      };
-    },
-    [keyDownHandler, keyUpHandler, resetKeyboardState, sendKeyboardEvent],
-  );
-
   const videoKeyUpHandler = useCallback((e: KeyboardEvent) => {
     // In fullscreen mode in chrome & safari, the space key is used to pause/play the video
     // there is no way to prevent this, so we need to simply force play the video when it's paused.
@@ -389,71 +374,6 @@ export default function WebRTCVideo() {
     }
   }, []);
 
-  useEffect(
-    function setupVideoEventListeners() {
-      let videoElmRefValue = null;
-      if (!videoElm.current) return;
-      videoElmRefValue = videoElm.current;
-      const abortController = new AbortController();
-      const signal = abortController.signal;
-
-      videoElmRefValue.addEventListener("mousemove", absMouseMoveHandler, { signal });
-      videoElmRefValue.addEventListener("pointerdown", absMouseMoveHandler, { signal });
-      videoElmRefValue.addEventListener("pointerup", absMouseMoveHandler, { signal });
-      videoElmRefValue.addEventListener("keyup", videoKeyUpHandler, { signal });
-      videoElmRefValue.addEventListener("wheel", mouseWheelHandler, {
-        signal,
-        passive: true,
-      });
-      videoElmRefValue.addEventListener(
-        "contextmenu",
-        (e: MouseEvent) => e.preventDefault(),
-        { signal },
-      );
-      videoElmRefValue.addEventListener("playing", onVideoPlaying, { signal });
-
-      const local = resetMousePosition;
-      window.addEventListener("blur", local, { signal });
-      document.addEventListener("visibilitychange", local, { signal });
-
-      return () => {
-        if (videoElmRefValue) abortController.abort();
-      };
-    },
-    [
-      absMouseMoveHandler,
-      resetMousePosition,
-      onVideoPlaying,
-      mouseWheelHandler,
-      videoKeyUpHandler,
-    ],
-  );
-
-  useEffect(
-    function setupRelativeMouseEventListeners() {
-      if (settings.mouseMode !== "relative") return;
-
-      const abortController = new AbortController();
-      const signal = abortController.signal;
-
-      // bind to body to capture all mouse events
-      const body = document.querySelector("body");
-      if (!body) return;
-
-      body.addEventListener("mousemove", relMouseMoveHandler, { signal });
-      body.addEventListener("pointerdown", relMouseMoveHandler, { signal });
-      body.addEventListener("pointerup", relMouseMoveHandler, { signal });
-
-      return () => {
-        abortController.abort();
-
-        body.removeEventListener("mousemove", relMouseMoveHandler);
-        body.removeEventListener("pointerdown", relMouseMoveHandler);
-        body.removeEventListener("pointerup", relMouseMoveHandler);
-      };
-    }, [settings.mouseMode, relMouseMoveHandler],
-  )
-
   useEffect(
     function updateVideoStream() {
       if (!mediaStream) return;
@@ -476,6 +396,129 @@ export default function WebRTCVideo() {
     ],
   );
 
+  // Setup Keyboard Events
+  useEffect(
+    function setupKeyboardEvents() {
+      const abortController = new AbortController();
+      const signal = abortController.signal;
+
+      document.addEventListener("keydown", keyDownHandler, { signal });
+      document.addEventListener("keyup", keyUpHandler, { signal });
+
+      // eslint-disable-next-line @typescript-eslint/ban-ts-comment
+      // @ts-expect-error
+      window.clearKeys = () => sendKeyboardEvent([], []);
+      window.addEventListener("blur", resetKeyboardState, { signal });
+      document.addEventListener("visibilitychange", resetKeyboardState, { signal });
+
+      return () => {
+        abortController.abort();
+      };
+    },
+    [keyDownHandler, keyUpHandler, resetKeyboardState, sendKeyboardEvent],
+  );
+
+  // Setup Video Event Listeners
+  useEffect(
+    function setupVideoEventListeners() {
+      const videoElmRefValue = videoElm.current;
+      if (!videoElmRefValue) return;
+
+      const abortController = new AbortController();
+      const signal = abortController.signal;
+
+      // To prevent the video from being paused when the user presses a space in fullscreen mode
+      videoElmRefValue.addEventListener("keyup", videoKeyUpHandler, { signal });
+
+      // We need to know when the video is playing to update state and video size
+      videoElmRefValue.addEventListener("playing", onVideoPlaying, { signal });
+
+      return () => {
+        abortController.abort();
+      };
+    },
+    [
+      absMouseMoveHandler,
+      resetMousePosition,
+      onVideoPlaying,
+      mouseWheelHandler,
+      videoKeyUpHandler,
+    ],
+  );
+
+  // Setup Absolute Mouse Events
+  useEffect(
+    function setAbsoluteMouseModeEventListeners() {
+      const videoElmRefValue = videoElm.current;
+      if (!videoElmRefValue) return;
+
+      if (settings.mouseMode !== "absolute") return;
+
+      const abortController = new AbortController();
+      const signal = abortController.signal;
+
+      videoElmRefValue.addEventListener("mousemove", absMouseMoveHandler, { signal });
+      videoElmRefValue.addEventListener("pointerdown", absMouseMoveHandler, { signal });
+      videoElmRefValue.addEventListener("pointerup", absMouseMoveHandler, { signal });
+      videoElmRefValue.addEventListener("wheel", mouseWheelHandler, {
+        signal,
+        passive: true,
+      });
+
+      // Reset the mouse position when the window is blurred or the document is hidden
+      const local = resetMousePosition;
+      window.addEventListener("blur", local, { signal });
+      document.addEventListener("visibilitychange", local, { signal });
+
+      return () => {
+        abortController.abort();
+      };
+    },
+    [absMouseMoveHandler, mouseWheelHandler, resetMousePosition, settings.mouseMode],
+  );
+
+  // Setup Relative Mouse Events
+  const containerRef = useRef<HTMLDivElement>(null);
+  useEffect(
+    function setupRelativeMouseEventListeners() {
+      if (settings.mouseMode !== "relative") return;
+
+      const abortController = new AbortController();
+      const signal = abortController.signal;
+
+      // We bind to the larger container in relative mode because of delta between the acceleration of the local
+      // mouse and the mouse movement of the remote mouse. This simply makes it a bit less painful to use.
+      // When we get Pointer Lock support, we can remove this.
+      const containerElm = containerRef.current;
+      if (!containerElm) return;
+
+      containerElm.addEventListener("mousemove", relMouseMoveHandler, { signal });
+      containerElm.addEventListener("pointerdown", relMouseMoveHandler, { signal });
+      containerElm.addEventListener("pointerup", relMouseMoveHandler, { signal });
+
+      containerElm.addEventListener("wheel", mouseWheelHandler, {
+        signal,
+        passive: true,
+      });
+
+      const preventContextMenu = (e: MouseEvent) => e.preventDefault();
+      containerElm.addEventListener("contextmenu", preventContextMenu, { signal });
+
+      return () => {
+        abortController.abort();
+      };
+    },
+    [settings.mouseMode, relMouseMoveHandler, mouseWheelHandler],
+  );
+
+  const hasNoAutoPlayPermissions = useMemo(() => {
+    if (peerConnectionState !== "connected") return false;
+    if (isPlaying) return false;
+    if (hdmiError) return false;
+    if (videoHeight === 0 || videoWidth === 0) return false;
+    return true;
+  }, [peerConnectionState, isPlaying, hdmiError, videoHeight, videoWidth]);
+
   return (
     <div className="grid h-full w-full grid-rows-layout">
       <div className="min-h-[39.5px]">
@@ -490,7 +533,12 @@ export default function WebRTCVideo() {
         </fieldset>
       </div>
 
-      <div className="h-full overflow-hidden">
+      <div
+        ref={containerRef}
+        className={cx("h-full overflow-hidden", {
+          "cursor-none": settings.mouseMode === "relative" && settings.isCursorHidden,
+        })}
+      >
         <div className="relative h-full">
           <div
             className={cx(
@@ -519,7 +567,9 @@ export default function WebRTCVideo() {
                         className={cx(
                           "outline-50 max-h-full max-w-full object-contain transition-all duration-1000",
                           {
-                            "cursor-none": settings.isCursorHidden,
+                            "cursor-none":
+                              settings.mouseMode === "absolute" &&
+                              settings.isCursorHidden,
                             "opacity-0": isLoading || isConnectionError || hdmiError,
                             "animate-slideUpFade border border-slate-800/30 opacity-0 shadow dark:border-slate-300/20":
                               isPlaying,
@@ -534,6 +584,12 @@ export default function WebRTCVideo() {
                           <LoadingOverlay show={isLoading} />
                           <ConnectionErrorOverlay show={isConnectionError} />
                           <HDMIErrorOverlay show={hdmiError} hdmiState={hdmiState} />
+                          <NoAutoplayPermissionsOverlay
+                            show={hasNoAutoPlayPermissions}
+                            onPlayClick={() => {
+                              videoElm.current?.play();
+                            }}
+                          />
                         </div>
                       </div>
                     </div>

ui/src/routes/devices.$id.tsx

@@ -130,10 +130,68 @@ export default function KvmIdRoute() {
   const setDiskChannel = useRTCStore(state => state.setDiskChannel);
   const setRpcDataChannel = useRTCStore(state => state.setRpcDataChannel);
   const setTransceiver = useRTCStore(state => state.setTransceiver);
+  const location = useLocation();
+
+  const [connectionAttempts, setConnectionAttempts] = useState(0);
+
+  const [startedConnectingAt, setStartedConnectingAt] = useState<Date | null>(null);
+  const [connectedAt, setConnectedAt] = useState<Date | null>(null);
+
+  const [connectionFailed, setConnectionFailed] = useState(false);
 
   const navigate = useNavigate();
   const { otaState, setOtaState, setModalView } = useUpdateStore();
 
+  const closePeerConnection = useCallback(
+    function closePeerConnection() {
+      peerConnection?.close();
+      // "closed" is a valid RTCPeerConnection state according to the WebRTC spec
+      // https://developer.mozilla.org/en-US/docs/Web/API/RTCPeerConnection/connectionState#closed
+      // However, the onconnectionstatechange event doesn't fire when close() is called manually
+      // So we need to explicitly update our state to maintain consistency
+      // I don't know why this is happening, but this is the best way I can think of to handle it
+      setPeerConnectionState("closed");
+    },
+    [peerConnection, setPeerConnectionState],
+  );
+
+  useEffect(() => {
+    const connectionAttemptsThreshold = 30;
+    if (connectionAttempts > connectionAttemptsThreshold) {
+      console.log(`Connection failed after ${connectionAttempts} attempts.`);
+      setConnectionFailed(true);
+      closePeerConnection();
+    }
+  }, [connectionAttempts, closePeerConnection]);
+
+  useEffect(() => {
+    // Skip if already connected
+    if (connectedAt) return;
+
+    // Skip if connection is declared as failed
+    if (connectionFailed) return;
+
+    const interval = setInterval(() => {
+      console.log("Checking connection status");
+
+      // Skip if connection hasn't started
+      if (!startedConnectingAt) return;
+
+      const elapsedTime = Math.floor(
+        new Date().getTime() - startedConnectingAt.getTime(),
+      );
+
+      // Fail connection if it's been over X seconds since we started connecting
+      if (elapsedTime > 60 * 1000) {
+        console.error(`Connection failed after ${elapsedTime} ms.`);
+        setConnectionFailed(true);
+        closePeerConnection();
+      }
+    }, 1000);
+
+    return () => clearInterval(interval);
+  }, [closePeerConnection, connectedAt, connectionFailed, startedConnectingAt]);
+
   const sdp = useCallback(
     async (event: RTCPeerConnectionIceEvent, pc: RTCPeerConnection) => {
       if (!pc) return;
@@ -169,7 +227,7 @@ export default function KvmIdRoute() {
           // - In device mode, the device api would timeout, the fetch would throw an error, therefore the catch block would be hit
           // Regardless, we should close the peer connection and let the useInterval handle reconnecting
           if (!res.ok) {
-            pc?.close();
+            closePeerConnection();
             console.error(`Error setting SDP - Status: ${res.status}}`, json);
             return;
           }
@@ -180,14 +238,20 @@ export default function KvmIdRoute() {
         ).catch(e => console.log(`Error setting remote description: ${e}`));
       } catch (error) {
         console.error(`Error setting SDP: ${error}`);
-        pc?.close();
+        closePeerConnection();
       }
     },
-    [navigate, params.id],
+    [closePeerConnection, navigate, params.id],
   );
 
   const connectWebRTC = useCallback(async () => {
     console.log("Attempting to connect WebRTC");
+
+    // Track connection status to detect failures and show error overlay
+    setConnectionAttempts(x => x + 1);
+    setStartedConnectingAt(new Date());
+    setConnectedAt(null);
+
     const pc = new RTCPeerConnection({
       // We only use STUN or TURN servers if we're in the cloud
       ...(isInCloud && iceConfig?.iceServers
@@ -197,6 +261,11 @@ export default function KvmIdRoute() {
 
     // Set up event listeners and data channels
     pc.onconnectionstatechange = () => {
+      // If the connection state is connected, we reset the connection attempts.
+      if (pc.connectionState === "connected") {
+        setConnectionAttempts(0);
+        setConnectedAt(new Date());
+      }
       setPeerConnectionState(pc.connectionState);
     };
 
@@ -236,16 +305,35 @@ export default function KvmIdRoute() {
     setTransceiver,
   ]);
 
-  // WebRTC connection management
+  useEffect(() => {
-  useInterval(() => {
+    console.log("Attempting to connect WebRTC");
+
+    // If we're in an other session, we don't need to connect
+    if (location.pathname.includes("other-session")) return;
+
+    // If we're already connected or connecting, we don't need to connect
     if (
       ["connected", "connecting", "new"].includes(peerConnection?.connectionState ?? "")
     ) {
       return;
     }
-    if (location.pathname.includes("other-session")) return;
+
+    // In certain cases, we want to never connect again. This happens when we've tried for a long time and failed
+    if (connectionFailed) {
+      console.log("Connection failed. We won't attempt to connect again.");
+      return;
+    }
+
+    const interval = setInterval(() => {
       connectWebRTC();
     }, 3000);
+    return () => clearInterval(interval);
+  }, [
+    connectWebRTC,
+    connectionFailed,
+    location.pathname,
+    peerConnection?.connectionState,
+  ]);
 
   // On boot, if the connection state is undefined, we connect to the WebRTC
   useEffect(() => {
@@ -431,7 +519,6 @@ export default function KvmIdRoute() {
   }, [kvmTerminal, peerConnection, serialConsole]);
 
   const outlet = useOutlet();
-  const location = useLocation();
   const onModalClose = useCallback(() => {
     if (location.pathname !== "/other-session") navigateTo("/");
   }, [navigateTo, location.pathname]);
@@ -523,6 +610,7 @@ export default function KvmIdRoute() {
         }}
       >
         <Modal open={outlet !== null} onClose={onModalClose}>
+          {/* The 'used by other session' modal needs to have access to the connectWebRTC function */}
           <Outlet context={{ connectWebRTC }} />
         </Modal>
       </div>