From c13782d23464ca7b0506ba6874bd906d09c2740e Mon Sep 17 00:00:00 2001 From: Adam Shiervani Date: Wed, 16 Apr 2025 00:27:46 +0200 Subject: [PATCH] feat: Add Pointer lock functionality and SSL support in dev mode - Introduced @vitejs/plugin-basic-ssl for enabling SSL in development. - Added a new script `dev:ssl` to run the development server with SSL. - Implemented pointer lock feature in the WebRTCVideo component, enhancing user interaction. - Added a PointerLockBar component to guide users on enabling mouse control. - Cleaned up the VideoOverlay and WebRTCVideo components for better readability and functionality. --- ui/package-lock.json | 56 ++------ ui/package.json | 2 + ui/src/components/VideoOverlay.tsx | 34 +++++ ui/src/components/WebRTCVideo.tsx | 218 ++++++++++++++++------------- ui/vite.config.ts | 13 +- 5 files changed, 182 insertions(+), 141 deletions(-) diff --git a/ui/package-lock.json b/ui/package-lock.json index ebce148..b51a2ea 100644 --- a/ui/package-lock.json +++ b/ui/package-lock.json @@ -11,6 +11,7 @@ "@headlessui/react": "^2.2.0", "@headlessui/tailwindcss": "^0.2.1", "@heroicons/react": "^2.2.0", + "@vitejs/plugin-basic-ssl": "^1.2.0", "@xterm/addon-clipboard": "^0.1.0", "@xterm/addon-fit": "^0.10.0", "@xterm/addon-unicode11": "^0.8.0", @@ -105,7 +106,6 @@ "cpu": [ "ppc64" ], - "dev": true, "optional": true, "os": [ "aix" @@ -121,7 +121,6 @@ "cpu": [ "arm" ], - "dev": true, "optional": true, "os": [ "android" @@ -137,7 +136,6 @@ "cpu": [ "arm64" ], - "dev": true, "optional": true, "os": [ "android" @@ -153,7 +151,6 @@ "cpu": [ "x64" ], - "dev": true, "optional": true, "os": [ "android" @@ -169,7 +166,6 @@ "cpu": [ "arm64" ], - "dev": true, "optional": true, "os": [ "darwin" @@ -185,7 +181,6 @@ "cpu": [ "x64" ], - "dev": true, "optional": true, "os": [ "darwin" @@ -201,7 +196,6 @@ "cpu": [ "arm64" ], - "dev": true, "optional": true, "os": [ "freebsd" @@ -217,7 +211,6 @@ "cpu": [ "x64" ], - "dev": true, "optional": true, "os": [ "freebsd" @@ -233,7 +226,6 @@ "cpu": [ "arm" ], - "dev": true, "optional": true, "os": [ "linux" @@ -249,7 +241,6 @@ "cpu": [ "arm64" ], - "dev": true, "optional": true, "os": [ "linux" @@ -265,7 +256,6 @@ "cpu": [ "ia32" ], - "dev": true, "optional": true, "os": [ "linux" @@ -281,7 +271,6 @@ "cpu": [ "loong64" ], - "dev": true, "optional": true, "os": [ "linux" @@ -297,7 +286,6 @@ "cpu": [ "mips64el" ], - "dev": true, "optional": true, "os": [ "linux" @@ -313,7 +301,6 @@ "cpu": [ "ppc64" ], - "dev": true, "optional": true, "os": [ "linux" @@ -329,7 +316,6 @@ "cpu": [ "riscv64" ], - "dev": true, "optional": true, "os": [ "linux" @@ -345,7 +331,6 @@ "cpu": [ "s390x" ], - "dev": true, "optional": true, "os": [ "linux" @@ -361,7 +346,6 @@ "cpu": [ "x64" ], - "dev": true, "optional": true, "os": [ "linux" @@ -377,7 +361,6 @@ "cpu": [ "x64" ], - "dev": true, "optional": true, "os": [ "netbsd" @@ -393,7 +376,6 @@ "cpu": [ "x64" ], - "dev": true, "optional": true, "os": [ "openbsd" @@ -409,7 +391,6 @@ "cpu": [ "x64" ], - "dev": true, "optional": true, "os": [ "sunos" @@ -425,7 +406,6 @@ "cpu": [ "arm64" ], - "dev": true, "optional": true, "os": [ "win32" @@ -441,7 +421,6 @@ "cpu": [ "ia32" ], - "dev": true, "optional": true, "os": [ "win32" @@ -457,7 +436,6 @@ "cpu": [ "x64" ], - "dev": true, "optional": true, "os": [ "win32" @@ -890,7 +868,6 @@ "cpu": [ "arm" ], - "dev": true, "optional": true, "os": [ "android" @@ -903,7 +880,6 @@ "cpu": [ "arm64" ], - "dev": true, "optional": true, "os": [ "android" @@ -916,7 +892,6 @@ "cpu": [ "arm64" ], - "dev": true, "optional": true, "os": [ "darwin" @@ -929,7 +904,6 @@ "cpu": [ "x64" ], - "dev": true, "optional": true, "os": [ "darwin" @@ -942,7 +916,6 @@ "cpu": [ "arm" ], - "dev": true, "optional": true, "os": [ "linux" @@ -955,7 +928,6 @@ "cpu": [ "arm64" ], - "dev": true, "optional": true, "os": [ "linux" @@ -968,7 +940,6 @@ "cpu": [ "arm64" ], - "dev": true, "optional": true, "os": [ "linux" @@ -981,7 +952,6 @@ "cpu": [ "ppc64le" ], - "dev": true, "optional": true, "os": [ "linux" @@ -994,7 +964,6 @@ "cpu": [ "riscv64" ], - "dev": true, "optional": true, "os": [ "linux" @@ -1007,7 +976,6 @@ "cpu": [ "s390x" ], - "dev": true, "optional": true, "os": [ "linux" @@ -1020,7 +988,6 @@ "cpu": [ "x64" ], - "dev": true, "optional": true, "os": [ "linux" @@ -1033,7 +1000,6 @@ "cpu": [ "x64" ], - "dev": true, "optional": true, "os": [ "linux" @@ -1046,7 +1012,6 @@ "cpu": [ "arm64" ], - "dev": true, "optional": true, "os": [ "win32" @@ -1059,7 +1024,6 @@ "cpu": [ "ia32" ], - "dev": true, "optional": true, "os": [ "win32" @@ -1072,7 +1036,6 @@ "cpu": [ "x64" ], - "dev": true, "optional": true, "os": [ "win32" @@ -1426,8 +1389,7 @@ "node_modules/@types/estree": { "version": "1.0.5", "resolved": "https://registry.npmjs.org/@types/estree/-/estree-1.0.5.tgz", - "integrity": "sha512-/kYRxGDLWzHOB7q+wtSUQlFrtcdUccpfy+X+9iMBpHK8QLLhx2wIPYuS5DYtR9Wa/YlZAbIovy7qVdB1Aq6Lyw==", - "dev": true + "integrity": "sha512-/kYRxGDLWzHOB7q+wtSUQlFrtcdUccpfy+X+9iMBpHK8QLLhx2wIPYuS5DYtR9Wa/YlZAbIovy7qVdB1Aq6Lyw==" }, "node_modules/@types/json5": { "version": "0.0.29", @@ -1675,6 +1637,17 @@ "resolved": "https://registry.npmjs.org/@ungap/structured-clone/-/structured-clone-1.2.0.tgz", "integrity": "sha512-zuVdFrMJiuCDQUMCzQaD6KL28MjnqqN8XnAqiEq9PNm/hCPTSGfrXCOfwj1ow4LFb/tNymJPwsNbVePc1xFqrQ==" }, + "node_modules/@vitejs/plugin-basic-ssl": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/@vitejs/plugin-basic-ssl/-/plugin-basic-ssl-1.2.0.tgz", + "integrity": "sha512-mkQnxTkcldAzIsomk1UuLfAu9n+kpQ3JbHcpCp7d2Oo6ITtji8pHS3QToOWjhPFvNQSnhlkAjmGbhv2QvwO/7Q==", + "engines": { + "node": ">=14.21.3" + }, + "peerDependencies": { + "vite": "^3.0.0 || ^4.0.0 || ^5.0.0 || ^6.0.0" + } + }, "node_modules/@vitejs/plugin-react-swc": { "version": "3.8.0", "resolved": "https://registry.npmjs.org/@vitejs/plugin-react-swc/-/plugin-react-swc-3.8.0.tgz", @@ -2740,7 +2713,6 @@ "version": "0.20.2", "resolved": "https://registry.npmjs.org/esbuild/-/esbuild-0.20.2.tgz", "integrity": "sha512-WdOOppmUNU+IbZ0PaDiTst80zjnrOkyJNHoKupIcVyU8Lvla3Ugx94VzkQ32Ijqd7UhHJy75gNWDMUekcrSJ6g==", - "dev": true, "hasInstallScript": true, "bin": { "esbuild": "bin/esbuild" @@ -5345,7 +5317,6 @@ "version": "4.14.1", "resolved": "https://registry.npmjs.org/rollup/-/rollup-4.14.1.tgz", "integrity": "sha512-4LnHSdd3QK2pa1J6dFbfm1HN0D7vSK/ZuZTsdyUAlA6Rr1yTouUTL13HaDOGJVgby461AhrNGBS7sCGXXtT+SA==", - "dev": true, "dependencies": { "@types/estree": "1.0.5" }, @@ -6250,7 +6221,6 @@ "version": "5.2.8", "resolved": "https://registry.npmjs.org/vite/-/vite-5.2.8.tgz", "integrity": "sha512-OyZR+c1CE8yeHw5V5t59aXsUPPVTHMDjEZz8MgguLL/Q7NblxhZUlTu9xSPqlsUO/y+X7dlU05jdhvyycD55DA==", - "dev": true, "dependencies": { "esbuild": "^0.20.1", "postcss": "^8.4.38", diff --git a/ui/package.json b/ui/package.json index a248616..3160297 100644 --- a/ui/package.json +++ b/ui/package.json @@ -8,6 +8,7 @@ }, "scripts": { "dev": "./dev_device.sh", + "dev:ssl": "USE_SSL=true ./dev_device.sh", "dev:cloud": "vite dev --mode=cloud-development", "build": "npm run build:prod", "build:device": "tsc && vite build --mode=device --emptyOutDir", @@ -21,6 +22,7 @@ "@headlessui/react": "^2.2.0", "@headlessui/tailwindcss": "^0.2.1", "@heroicons/react": "^2.2.0", + "@vitejs/plugin-basic-ssl": "^1.2.0", "@xterm/addon-clipboard": "^0.1.0", "@xterm/addon-fit": "^0.10.0", "@xterm/addon-unicode11": "^0.8.0", diff --git a/ui/src/components/VideoOverlay.tsx b/ui/src/components/VideoOverlay.tsx index e34cf10..f6baeec 100644 --- a/ui/src/components/VideoOverlay.tsx +++ b/ui/src/components/VideoOverlay.tsx @@ -7,6 +7,7 @@ import { LuPlay } from "react-icons/lu"; import { Button, LinkButton } from "@components/Button"; import LoadingSpinner from "@components/LoadingSpinner"; import Card, { GridCard } from "@components/Card"; +import { BsMouseFill } from "react-icons/bs"; interface OverlayContentProps { children: React.ReactNode; @@ -358,3 +359,36 @@ export function NoAutoplayPermissionsOverlay({ ); } + +interface PointerLockBarProps { + show: boolean; +} + +export function PointerLockBar({ show }: PointerLockBarProps) { + return ( + + {show ? ( + +
+ +
+
+ + + Click on the video to enable mouse control + +
+
+ +
+ + ) : null} + + ); +} diff --git a/ui/src/components/WebRTCVideo.tsx b/ui/src/components/WebRTCVideo.tsx index c0324e0..b73135b 100644 --- a/ui/src/components/WebRTCVideo.tsx +++ b/ui/src/components/WebRTCVideo.tsx @@ -18,13 +18,14 @@ import MacroBar from "@/components/MacroBar"; import InfoBar from "@components/InfoBar"; import useKeyboard from "@/hooks/useKeyboard"; import { useJsonRpc } from "@/hooks/useJsonRpc"; +import notifications from "@/notifications"; import { HDMIErrorOverlay, LoadingVideoOverlay, NoAutoplayPermissionsOverlay, + PointerLockBar, } from "./VideoOverlay"; -import notifications from "@/notifications"; export default function WebRTCVideo() { // Video and stream related refs and states @@ -32,7 +33,7 @@ export default function WebRTCVideo() { const mediaStream = useRTCStore(state => state.mediaStream); const [isPlaying, setIsPlaying] = useState(false); const peerConnectionState = useRTCStore(state => state.peerConnectionState); - + const [isPointerLockActive, setIsPointerLockActive] = useState(false); // Store hooks const settings = useSettingsStore(); const { sendKeyboardEvent, resetKeyboardState } = useKeyboard(); @@ -55,8 +56,6 @@ export default function WebRTCVideo() { const hdmiError = ["no_lock", "no_signal", "out_of_range"].includes(hdmiState); const isVideoLoading = !isPlaying; - // console.log("peerConnection?.connectionState", peerConnection?.connectionState); - // Keyboard related states const { setIsNumLockActive, setIsCapsLockActive, setIsScrollLockActive } = useHidStore(); @@ -101,15 +100,13 @@ export default function WebRTCVideo() { ); // Pointer lock and keyboard lock related - const isPointerLockPossible = useMemo(() => { - return window.location.protocol === "https:"; - }, [window.location.protocol]); + const isPointerLockPossible = window.location.protocol === "https:"; - const checkNavigatorPermissions = async (permissionName: string) => { + const checkNavigatorPermissions = useCallback(async (permissionName: string) => { const name = permissionName as PermissionName; const { state } = await navigator.permissions.query({ name }); return state === "granted"; - }; + }, []); const requestPointerLock = useCallback(async () => { if (document.pointerLockElement) return; @@ -118,23 +115,28 @@ export default function WebRTCVideo() { if (isPointerLockGranted && settings.mouseMode === "relative") { videoElm.current?.requestPointerLock(); } - }, [settings.mouseMode, videoElm]); + }, [checkNavigatorPermissions, settings.mouseMode]); useEffect(() => { if (!isPointerLockPossible || !videoElm.current) return; const handlePointerLockChange = () => { if (document.pointerLockElement) { - notifications.success("Pointer lock enabled, to exit it, press the escape key for a few seconds"); + notifications.success("Pointer lock Enabled, hold escape to exit"); + setIsPointerLockActive(true); } else { notifications.success("Pointer lock disabled"); + setIsPointerLockActive(false); } }; - document.addEventListener("pointerlockchange", handlePointerLockChange); + const abortController = new AbortController(); + const signal = abortController.signal; + + document.addEventListener("pointerlockchange", handlePointerLockChange, { signal }); return () => { - document.removeEventListener("pointerlockchange", handlePointerLockChange); + abortController.abort(); }; }, [isPointerLockPossible, videoElm]); @@ -148,13 +150,12 @@ export default function WebRTCVideo() { const isKeyboardLockGranted = await checkNavigatorPermissions("keyboard-lock"); if (isKeyboardLockGranted) { - if ('keyboard' in navigator) { - // @ts-ignore + if ("keyboard" in navigator) { + // @ts-ignore await navigator.keyboard.lock(); } } - - }, [disableVideoFocusTrap, requestPointerLock, checkNavigatorPermissions]); + }, [requestPointerLock, checkNavigatorPermissions]); // Mouse-related const calcDelta = (pos: number) => (Math.abs(pos) < 10 ? pos * 2 : pos); @@ -172,12 +173,18 @@ export default function WebRTCVideo() { const relMouseMoveHandler = useCallback( (e: MouseEvent) => { if (settings.mouseMode !== "relative") return; + if (isPointerLockActive === false && isPointerLockPossible === true) return; // Send mouse movement const { buttons } = e; sendRelMouseMovement(e.movementX, e.movementY, buttons); }, - [sendRelMouseMovement, settings.mouseMode], + [ + isPointerLockActive, + isPointerLockPossible, + sendRelMouseMovement, + settings.mouseMode, + ], ); const sendAbsMouseMovement = useCallback( @@ -353,7 +360,7 @@ export default function WebRTCVideo() { // console.log("KEYUP: Not focusing on the video", document.activeElement); // return; // } - + // console.log(document.activeElement); setIsNumLockActive(e.getModifierState("NumLock")); @@ -572,48 +579,50 @@ export default function WebRTCVideo() { // Setup Relative Mouse Events const containerRef = useRef(null); + useEffect( function setupRelativeMouseEventListeners() { if (settings.mouseMode !== "relative") return; + // Relative mouse mode should only be active if the pointer lock is active and Pointer Lock is possible + + const videoElmRefValue = videoElm.current; + if (!videoElmRefValue) return; const abortController = new AbortController(); const signal = abortController.signal; - // We bind to the larger container in relative mode because of delta between the acceleration of the local - // mouse and the mouse movement of the remote mouse. This simply makes it a bit less painful to use. - // When we get Pointer Lock support, we can remove this. - const containerElm = containerRef.current; - if (!containerElm) return; - - containerElm.addEventListener("mousemove", relMouseMoveHandler, { signal }); - containerElm.addEventListener("pointerdown", relMouseMoveHandler, { signal }); - containerElm.addEventListener("pointerup", relMouseMoveHandler, { signal }); - - containerElm.addEventListener("wheel", mouseWheelHandler, { + videoElmRefValue.addEventListener("mousemove", relMouseMoveHandler, { signal }); + videoElmRefValue.addEventListener("pointerdown", relMouseMoveHandler, { signal }); + videoElmRefValue.addEventListener("pointerup", relMouseMoveHandler, { signal }); + videoElmRefValue.addEventListener( + "click", + () => { + if (isPointerLockPossible && !document.pointerLockElement) { + requestPointerLock(); + } + }, + { signal }, + ); + videoElmRefValue.addEventListener("wheel", mouseWheelHandler, { signal, passive: true, }); const preventContextMenu = (e: MouseEvent) => e.preventDefault(); - containerElm.addEventListener("contextmenu", preventContextMenu, { signal }); - - // Request pointer lock when the container is clicked - if (isPointerLockPossible) { - const videoElmRefValue = videoElm.current; - if (videoElmRefValue) containerElm.addEventListener("click", () => { - if (disableVideoFocusTrap) return; - console.log("Requesting pointer lock"); - requestPointerLock(); - }, { signal }); - } + videoElmRefValue.addEventListener("contextmenu", preventContextMenu, { signal }); return () => { abortController.abort(); }; }, [ - settings.mouseMode, relMouseMoveHandler, mouseWheelHandler, - disableVideoFocusTrap, requestPointerLock, isPointerLockPossible + settings.mouseMode, + relMouseMoveHandler, + mouseWheelHandler, + disableVideoFocusTrap, + requestPointerLock, + isPointerLockPossible, + isPointerLockActive, ], ); @@ -625,29 +634,43 @@ export default function WebRTCVideo() { return true; }, [peerConnection?.connectionState, isPlaying, hdmiError, videoHeight, videoWidth]); + const showPointerLockBar = useMemo(() => { + if (settings.mouseMode !== "relative") return false; + if (!isPointerLockPossible) return false; + if (isPointerLockActive) return false; + if (isVideoLoading) return false; + if (!isPlaying) return false; + if (videoHeight === 0 || videoWidth === 0) return false; + return true; + }, [ + settings.mouseMode, + isPointerLockPossible, + isPointerLockActive, + isVideoLoading, + isPlaying, + videoHeight, + videoWidth, + ]); + return (
-
+
-
- +
+
-
+
-
diff --git a/ui/vite.config.ts b/ui/vite.config.ts index f6aae50..f8459cd 100644 --- a/ui/vite.config.ts +++ b/ui/vite.config.ts @@ -1,23 +1,32 @@ import { defineConfig } from "vite"; import react from "@vitejs/plugin-react-swc"; import tsconfigPaths from "vite-tsconfig-paths"; +import basicSsl from "@vitejs/plugin-basic-ssl"; declare const process: { env: { JETKVM_PROXY_URL: string; + USE_SSL: string; }; }; export default defineConfig(({ mode, command }) => { const isCloud = mode.indexOf("cloud") !== -1; const onDevice = mode === "device"; - const { JETKVM_PROXY_URL } = process.env; + const { JETKVM_PROXY_URL, USE_SSL } = process.env; + const useSSL = USE_SSL === "true"; + + const plugins = [tsconfigPaths(), react()]; + if (useSSL) { + plugins.push(basicSsl()); + } return { - plugins: [tsconfigPaths(), react()], + plugins, build: { outDir: isCloud ? "dist" : "../static" }, server: { host: "0.0.0.0", + https: useSSL, proxy: JETKVM_PROXY_URL ? { "/me": JETKVM_PROXY_URL,