diff --git a/ui/src/components/InfoBar.tsx b/ui/src/components/InfoBar.tsx index 29f159d..36f6e95 100644 --- a/ui/src/components/InfoBar.tsx +++ b/ui/src/components/InfoBar.tsx @@ -10,11 +10,13 @@ import { VideoState } from "@/hooks/stores"; import { keys, modifiers } from "@/keyboardMappings"; +import { useHidRpc } from "@/hooks/useHidRpc"; export default function InfoBar() { const { keysDownState } = useHidStore(); const { mouseX, mouseY, mouseMove } = useMouseStore(); - + const { rpcHidReady } = useHidRpc(); + const videoClientSize = useVideoStore( (state: VideoState) => `${Math.round(state.clientWidth)}x${Math.round(state.clientHeight)}`, ); @@ -100,6 +102,12 @@ export default function InfoBar() { {hdmiState} )} + {debugMode && ( +
+ HidRPC State: + {rpcHidReady ? "Ready" : "Not Ready"} +
+ )} {showPressedKeys && (
diff --git a/ui/src/components/WebRTCVideo.tsx b/ui/src/components/WebRTCVideo.tsx index b879a70..ba6ee5c 100644 --- a/ui/src/components/WebRTCVideo.tsx +++ b/ui/src/components/WebRTCVideo.tsx @@ -61,7 +61,7 @@ export default function WebRTCVideo() { // Misc states and hooks const { send } = useJsonRpc(); - const { reportAbsMouseEvent, reportRelMouseEvent, handshakeCompleted } = useHidRpc(); + const { reportAbsMouseEvent, reportRelMouseEvent, rpcHidReady } = useHidRpc(); // Video-related const handleResize = useCallback( @@ -226,7 +226,7 @@ export default function WebRTCVideo() { // if (x === 0 && y === 0 && buttons === 0) return; const dx = calcDelta(x); const dy = calcDelta(y); - if (handshakeCompleted) { + if (rpcHidReady) { reportRelMouseEvent(dx, dy, buttons); } else { send("relMouseReport", { dx, dy, buttons }); @@ -238,7 +238,7 @@ export default function WebRTCVideo() { reportRelMouseEvent, setMouseMove, settings.mouseMode, - handshakeCompleted, + rpcHidReady, ], ); @@ -257,7 +257,7 @@ export default function WebRTCVideo() { const sendAbsMouseMovement = useCallback( (x: number, y: number, buttons: number) => { if (settings.mouseMode !== "absolute") return; - if (handshakeCompleted) { + if (rpcHidReady) { reportAbsMouseEvent(x, y, buttons); } else { send("absMouseReport", { x, y, buttons }); @@ -270,7 +270,7 @@ export default function WebRTCVideo() { reportAbsMouseEvent, setMousePosition, settings.mouseMode, - handshakeCompleted, + rpcHidReady, ], ); diff --git a/ui/src/hooks/stores.ts b/ui/src/hooks/stores.ts index 1b5afcc..21cd7ed 100644 --- a/ui/src/hooks/stores.ts +++ b/ui/src/hooks/stores.ts @@ -105,6 +105,9 @@ export interface RTCState { setRpcDataChannel: (channel: RTCDataChannel) => void; rpcDataChannel: RTCDataChannel | null; + rpcHidProtocolVersion: number | null; + setRpcHidProtocolVersion: (version: number) => void; + setRpcHidChannel: (channel: RTCDataChannel) => void; rpcHidChannel: RTCDataChannel | null; @@ -154,6 +157,9 @@ export const useRTCStore = create(set => ({ rpcDataChannel: null, setRpcDataChannel: (channel: RTCDataChannel) => set({ rpcDataChannel: channel }), + rpcHidProtocolVersion: null, + setRpcHidProtocolVersion: (version: number) => set({ rpcHidProtocolVersion: version }), + rpcHidChannel: null, setRpcHidChannel: (channel: RTCDataChannel) => set({ rpcHidChannel: channel }), diff --git a/ui/src/hooks/useHidRpc.ts b/ui/src/hooks/useHidRpc.ts index 9bda4c4..8dce581 100644 --- a/ui/src/hooks/useHidRpc.ts +++ b/ui/src/hooks/useHidRpc.ts @@ -1,6 +1,6 @@ -import { useCallback, useEffect, useState } from "react"; +import { useCallback, useEffect, useMemo } from "react"; -import { KeysDownState, useRTCStore } from "@/hooks/stores"; +import { KeyboardLedState, KeysDownState, useRTCStore } from "@/hooks/stores"; export const HID_RPC_MESSAGE_TYPES = { Handshake: 0x01, @@ -40,6 +40,30 @@ const fromInt8ToUint8 = (n: number) => { return (n >> 0) & 0xFF; }; +const keyboardLedStateMasks = { + num_lock: 1 << 0, + caps_lock: 1 << 1, + scroll_lock: 1 << 2, + compose: 1 << 3, + kana: 1 << 4, + shift: 1 << 6, +} + +export const toKeyboardLedState = (s: number): KeyboardLedState => { + if (!withinUint8Range(s)) { + throw new Error(`State ${s} is not within the uint8 range`); + } + + return { + num_lock: (s & keyboardLedStateMasks.num_lock) !== 0, + caps_lock: (s & keyboardLedStateMasks.caps_lock) !== 0, + scroll_lock: (s & keyboardLedStateMasks.scroll_lock) !== 0, + compose: (s & keyboardLedStateMasks.compose) !== 0, // TODO: check if this is correct + kana: (s & keyboardLedStateMasks.kana) !== 0, + shift: (s & keyboardLedStateMasks.shift) !== 0, + } as KeyboardLedState; +}; + const toPointerReportEvent = (x: number, y: number, buttons: number) => { if (!withinUint8Range(buttons)) { throw new Error(`Buttons ${buttons} is not within the uint8 range`); @@ -130,47 +154,49 @@ const unmarshalHidRpcMessage = (data: Uint8Array): HidRpcMessage | undefined => }; export function useHidRpc(onHidRpcMessage?: (payload: HidRpcMessage) => void) { - const { rpcHidChannel } = useRTCStore(); - const [handshakeCompleted, setHandshakeCompleted] = useState(false); + const { rpcHidChannel, setRpcHidProtocolVersion, rpcHidProtocolVersion } = useRTCStore(); + const rpcHidReady = useMemo(() => { + return rpcHidChannel?.readyState === "open" && rpcHidProtocolVersion !== null; + }, [rpcHidChannel, rpcHidProtocolVersion]); const reportKeyboardEvent = useCallback( (keys: number[], modifier: number) => { - if (rpcHidChannel?.readyState !== "open") return; - rpcHidChannel.send(toKeyboardReportEvent(keys, modifier)); + if (!rpcHidReady) return; + rpcHidChannel?.send(toKeyboardReportEvent(keys, modifier)); }, - [rpcHidChannel], + [rpcHidChannel, rpcHidReady], ); const reportKeypressEvent = useCallback( (key: number, press: boolean) => { - if (rpcHidChannel?.readyState !== "open") return; - rpcHidChannel.send(toKeypressReportEvent(key, press)); + if (!rpcHidReady) return; + rpcHidChannel?.send(toKeypressReportEvent(key, press)); }, - [rpcHidChannel], + [rpcHidChannel, rpcHidReady], ); const reportAbsMouseEvent = useCallback( (x: number, y: number, buttons: number) => { - if (rpcHidChannel?.readyState !== "open") return; - rpcHidChannel.send(toPointerReportEvent(x, y, buttons)); + if (!rpcHidReady) return; + rpcHidChannel?.send(toPointerReportEvent(x, y, buttons)); }, - [rpcHidChannel], + [rpcHidChannel, rpcHidReady], ); const reportRelMouseEvent = useCallback( (dx: number, dy: number, buttons: number) => { - if (rpcHidChannel?.readyState !== "open") return; - rpcHidChannel.send(toMouseReportEvent(dx, dy, buttons)); + if (!rpcHidReady) return; + rpcHidChannel?.send(toMouseReportEvent(dx, dy, buttons)); }, - [rpcHidChannel], + [rpcHidChannel, rpcHidReady], ); const doHandshake = useCallback(() => { - if (handshakeCompleted) return; + if (rpcHidProtocolVersion) return; if (!rpcHidChannel) return; - rpcHidChannel.send(toHandshakeMessage()); - }, [rpcHidChannel, handshakeCompleted]); + rpcHidChannel?.send(toHandshakeMessage()); + }, [rpcHidChannel, rpcHidProtocolVersion]); useEffect(() => { if (!rpcHidChannel) return; @@ -184,7 +210,7 @@ export function useHidRpc(onHidRpcMessage?: (payload: HidRpcMessage) => void) { return; } - console.log("Received HID RPC message", e.data); + console.debug("Received HID RPC message", e.data); const message = unmarshalHidRpcMessage(new Uint8Array(e.data)); if (!message) { @@ -193,7 +219,7 @@ export function useHidRpc(onHidRpcMessage?: (payload: HidRpcMessage) => void) { } if (message.type === HID_RPC_MESSAGE_TYPES.Handshake) { - setHandshakeCompleted(true); + setRpcHidProtocolVersion(1); } onHidRpcMessage?.(message); @@ -205,13 +231,21 @@ export function useHidRpc(onHidRpcMessage?: (payload: HidRpcMessage) => void) { rpcHidChannel.removeEventListener("message", messageHandler); }; }, - [rpcHidChannel, onHidRpcMessage, setHandshakeCompleted, doHandshake]); + [ + rpcHidChannel, + onHidRpcMessage, + setRpcHidProtocolVersion, + doHandshake, + rpcHidReady, + ], + ); return { reportKeyboardEvent, reportKeypressEvent, reportAbsMouseEvent, reportRelMouseEvent, - handshakeCompleted, + rpcHidProtocolVersion, + rpcHidReady, }; } diff --git a/ui/src/hooks/useKeyboard.ts b/ui/src/hooks/useKeyboard.ts index 1d733da..ae790bb 100644 --- a/ui/src/hooks/useKeyboard.ts +++ b/ui/src/hooks/useKeyboard.ts @@ -7,7 +7,7 @@ import { hidKeyToModifierMask, keys, modifiers } from "@/keyboardMappings"; export default function useKeyboard() { const { send } = useJsonRpc(); - const { rpcDataChannel, rpcHidChannel } = useRTCStore(); + const { rpcDataChannel } = useRTCStore(); const { keysDownState, setKeysDownState } = useHidStore(); // INTRODUCTION: The earlier version of the JetKVM device shipped with all keyboard state @@ -23,7 +23,7 @@ export default function useKeyboard() { const { keyPressReportApiAvailable, setkeyPressReportApiAvailable } = useHidStore(); // HidRPC is a binary format for exchanging keyboard and mouse events - const { reportKeyboardEvent, reportKeypressEvent } = useHidRpc((message) => { + const { reportKeyboardEvent, reportKeypressEvent, rpcHidReady } = useHidRpc((message) => { if (message.type === HID_RPC_MESSAGE_TYPES.KeysDownState) { if (!message.keysDownState) { return; @@ -40,11 +40,11 @@ export default function useKeyboard() { // or just accept the state if it does not support (returning no result) const sendKeyboardEvent = useCallback( async (state: KeysDownState) => { - if (rpcDataChannel?.readyState !== "open" && rpcHidChannel?.readyState !== "open") return; + if (rpcDataChannel?.readyState !== "open" && !rpcHidReady) return; console.debug(`Send keyboardReport keys: ${state.keys}, modifier: ${state.modifier}`); - if (rpcHidChannel?.readyState === "open") { + if (rpcHidReady) { console.debug("Sending keyboard report via HidRPC"); reportKeyboardEvent(state.keys, state.modifier); return; @@ -72,7 +72,7 @@ export default function useKeyboard() { }, [ rpcDataChannel?.readyState, - rpcHidChannel?.readyState, + rpcHidReady, send, reportKeyboardEvent, setKeysDownState, @@ -88,11 +88,11 @@ export default function useKeyboard() { // in client/browser-side code using simulateDeviceSideKeyHandlingForLegacyDevices. const sendKeypressEvent = useCallback( async (key: number, press: boolean) => { - if (rpcDataChannel?.readyState !== "open" && rpcHidChannel?.readyState !== "open") return; + if (rpcDataChannel?.readyState !== "open" && !rpcHidReady) return; console.debug(`Send keypressEvent key: ${key}, press: ${press}`); - if (rpcHidChannel?.readyState === "open") { + if (rpcHidReady) { console.debug("Sending keypress event via HidRPC"); reportKeypressEvent(key, press); return; @@ -119,7 +119,7 @@ export default function useKeyboard() { }, [ rpcDataChannel?.readyState, - rpcHidChannel?.readyState, + rpcHidReady, send, setkeyPressReportApiAvailable, setKeysDownState, @@ -176,10 +176,10 @@ export default function useKeyboard() { // It then sends the full keyboard state to the device. const handleKeyPress = useCallback( async (key: number, press: boolean) => { - if (rpcDataChannel?.readyState !== "open" && rpcHidChannel?.readyState !== "open") return; + if (rpcDataChannel?.readyState !== "open" && !rpcHidReady) return; if ((key || 0) === 0) return; // ignore zero key presses (they are bad mappings) - if (rpcHidChannel?.readyState === "open") { + if (rpcHidReady) { console.debug("Sending keypress event via HidRPC"); reportKeypressEvent(key, press); return; @@ -204,7 +204,7 @@ export default function useKeyboard() { keysDownState, resetKeyboardState, rpcDataChannel?.readyState, - rpcHidChannel?.readyState, + rpcHidReady, sendKeyboardEvent, sendKeypressEvent, reportKeypressEvent, diff --git a/ui/vite.config.ts b/ui/vite.config.ts index 5871c4b..44eec3a 100644 --- a/ui/vite.config.ts +++ b/ui/vite.config.ts @@ -28,6 +28,9 @@ export default defineConfig(({ mode, command }) => { return { plugins, + esbuild: { + pure: ["console.debug"], + }, build: { outDir: isCloud ? "dist" : "../static" }, server: { host: "0.0.0.0",