Merge 894e66efaa into 608f69db13

ui/src/components/InfoBar.tsx

@@ -133,7 +133,7 @@ export default function InfoBar() {
           <div
             className={cx(
               "shrink-0 p-1 px-1.5 text-xs",
-              keyboardLedState?.caps_lock
+              keyboardLedState.caps_lock
                 ? "text-black dark:text-white"
                 : "text-slate-800/20 dark:text-slate-300/20",
             )}
@@ -143,7 +143,7 @@ export default function InfoBar() {
           <div
             className={cx(
               "shrink-0 p-1 px-1.5 text-xs",
-              keyboardLedState?.num_lock
+              keyboardLedState.num_lock
                 ? "text-black dark:text-white"
                 : "text-slate-800/20 dark:text-slate-300/20",
             )}
@@ -153,24 +153,24 @@ export default function InfoBar() {
           <div
             className={cx(
               "shrink-0 p-1 px-1.5 text-xs",
-              keyboardLedState?.scroll_lock
+              keyboardLedState.scroll_lock
                 ? "text-black dark:text-white"
                 : "text-slate-800/20 dark:text-slate-300/20",
             )}
           >
             Scroll Lock
           </div>
-          {keyboardLedState?.compose ? (
+          {keyboardLedState.compose ? (
             <div className="shrink-0 p-1 px-1.5 text-xs">
               Compose
             </div>
           ) : null}
-          {keyboardLedState?.kana ? (
+          {keyboardLedState.kana ? (
             <div className="shrink-0 p-1 px-1.5 text-xs">
               Kana
             </div>
           ) : null}
-          {keyboardLedState?.shift ? (
+          {keyboardLedState.shift ? (
             <div className="shrink-0 p-1 px-1.5 text-xs">
               Shift
             </div>

ui/src/components/VirtualKeyboard.tsx

@@ -1,4 +1,3 @@
-import { useShallow } from "zustand/react/shallow";
 import { ChevronDownIcon } from "@heroicons/react/16/solid";
 import { AnimatePresence, motion } from "framer-motion";
 import { useCallback, useEffect, useRef, useState } from "react";
@@ -13,7 +12,7 @@ import "react-simple-keyboard/build/css/index.css";
 import AttachIconRaw from "@/assets/attach-icon.svg";
 import DetachIconRaw from "@/assets/detach-icon.svg";
 import { cx } from "@/cva.config";
-import { useHidStore, useUiStore } from "@/hooks/stores";
+import { HidState, useHidStore, useUiStore } from "@/hooks/stores";
 import useKeyboard from "@/hooks/useKeyboard";
 import { keyDisplayMap, keys, modifiers } from "@/keyboardMappings";
 
@@ -26,7 +25,7 @@ const AttachIcon = ({ className }: { className?: string }) => {
 };
 
 function KeyboardWrapper() {
-  const [layoutName, setLayoutName] = useState("default");
+  const [layoutName] = useState("default");
 
   const keyboardRef = useRef<HTMLDivElement>(null);
   const showAttachedVirtualKeyboard = useUiStore(
@@ -36,14 +35,16 @@ function KeyboardWrapper() {
     state => state.setAttachedVirtualKeyboardVisibility,
   );
 
-  const { sendKeyboardEvent, resetKeyboardState } = useKeyboard();
+  const virtualKeyboard = useHidStore(state => state.isVirtualKeyboardEnabled);
+  const setVirtualKeyboard = useHidStore(state => state.setVirtualKeyboardEnabled);
+
+  const keysDownState = useHidStore((state: HidState) => state.keysDownState);
+  const { handleKeyPress, sendKeyboardEvent, resetKeyboardState } = useKeyboard();
 
   const [isDragging, setIsDragging] = useState(false);
   const [position, setPosition] = useState({ x: 0, y: 0 });
   const [newPosition, setNewPosition] = useState({ x: 0, y: 0 });
 
-  const isCapsLockActive = useHidStore(useShallow(state => state.keyboardLedState?.caps_lock));
-
   /*
   // These will be used to display the currently pressed keys and modifiers on the virtual keyboard
 
@@ -129,74 +130,55 @@ function KeyboardWrapper() {
 
   const onKeyDown = useCallback(
     (key: string) => {
-      const isKeyShift = key === "{shift}" || key === "ShiftLeft" || key === "ShiftRight";
-      const isKeyCaps = key === "CapsLock";
-      const cleanKey = key.replace(/[()]/g, "");
-      const keyHasShiftModifier = key.includes("(");
-
-      // Handle toggle of layout for shift or caps lock
-      const toggleLayout = () => {
-        setLayoutName(prevLayout => (prevLayout === "default" ? "shift" : "default"));
-      };
+      const latchingKeys = ["CapsLock", "ScrollLock", "NumLock", "Meta", "Compose", "Kana"];
+      const dynamicKeys = ["ShiftLeft", "ShiftRight", "ControlLeft", "ControlRight", "AltLeft", "AltRight", "MetaLeft", "MetaRight"];
 
       if (key === "CtrlAltDelete") {
-        sendKeyboardEvent(
-          [keys["Delete"]],
-          [modifiers["ControlLeft"], modifiers["AltLeft"]],
-        );
+        sendKeyboardEvent({ keys: [keys.Delete], modifier: modifiers.ControlLeft | modifiers.AltLeft });
         setTimeout(resetKeyboardState, 100);
         return;
       }
 
       if (key === "AltMetaEscape") {
-        sendKeyboardEvent(
-          [keys["Escape"]],
-          [modifiers["MetaLeft"], modifiers["AltLeft"]],
-        );
-
+        sendKeyboardEvent({ keys: [keys.Escape], modifier: modifiers.AltLeft | modifiers.MetaLeft });
         setTimeout(resetKeyboardState, 100);
         return;
       }
 
       if (key === "CtrlAltBackspace") {
-        sendKeyboardEvent(
-          [keys["Backspace"]],
-          [modifiers["ControlLeft"], modifiers["AltLeft"]],
-        );
-
+        sendKeyboardEvent({ keys: [keys.Backspace], modifier: modifiers.ControlLeft | modifiers.AltLeft });
         setTimeout(resetKeyboardState, 100);
         return;
       }
 
-      if (isKeyShift || isKeyCaps) {
-        toggleLayout();
-
-        if (isCapsLockActive) {
-          sendKeyboardEvent([keys["CapsLock"]], []);
-          return;
-        }
+      // if they press any of the latching keys, we send a keypress down event and the release it automatically (on timer)
+      if (latchingKeys.includes(key)) {
+        handleKeyPress(keys[key], true)
+        setTimeout(() => handleKeyPress(keys[key], false), 100);
+        return;
       }
 
-      // Collect new active keys and modifiers
-      const newKeys = keys[cleanKey] ? [keys[cleanKey]] : [];
-      const newModifiers =
-        keyHasShiftModifier && !isCapsLockActive ? [modifiers["ShiftLeft"]] : [];
-
-      // Update current keys and modifiers
-      sendKeyboardEvent(newKeys, newModifiers);
-
-      // If shift was used as a modifier and caps lock is not active, revert to default layout
-      if (keyHasShiftModifier && !isCapsLockActive) {
-        setLayoutName("default");
+      // if they press any of the dynamic keys, we send a keypress down event but we don't release it until they click it again
+      if (dynamicKeys.includes(key)) {
+        const currentlyDown = keysDownState.keys.includes(keys[key]);
+        handleKeyPress(keys[key], !currentlyDown)
+        return;
       }
 
-      setTimeout(resetKeyboardState, 100);
+      // otherwise, just treat it as a down+up pair
+      const cleanKey = key.replace(/[()]/g, "");
+      handleKeyPress(keys[cleanKey], true);
+      setTimeout(() => handleKeyPress(keys[cleanKey], false), 50);
     },
-    [isCapsLockActive, sendKeyboardEvent, resetKeyboardState],
+    [handleKeyPress, sendKeyboardEvent, resetKeyboardState, keysDownState],
   );
 
-  const virtualKeyboard = useHidStore(state => state.isVirtualKeyboardEnabled);
-  const setVirtualKeyboard = useHidStore(state => state.setVirtualKeyboardEnabled);
+  // TODO handle the display of down keys and the layout change for shift/caps lock
+  //  const isCapsLockActive = useHidStore(useShallow(state => state.keyboardLedState.caps_lock));
+  // // Handle toggle of layout for shift or caps lock
+  // const toggleLayout = () => {
+  //  setLayoutName(prevLayout => (prevLayout === "default" ? "shift" : "default"));
+  // };
 
   return (
     <div

ui/src/components/WebRTCVideo.tsx

@@ -37,7 +37,7 @@ export default function WebRTCVideo() {
   const [isPointerLockActive, setIsPointerLockActive] = useState(false);
   // Store hooks
   const settings = useSettingsStore();
-  const { sendKeypressEvent, resetKeyboardState } = useKeyboard();
+  const { handleKeyPress, resetKeyboardState } = useKeyboard();
   const setMousePosition = useMouseStore((state: MouseState) => state.setMousePosition);
   const setMouseMove = useMouseStore((state: MouseState) => state.setMouseMove);
   const {
@@ -55,13 +55,14 @@ export default function WebRTCVideo() {
   const videoContrast = useSettingsStore((state: SettingsState) => state.videoContrast);
 
   // RTC related states
-  const peerConnection = useRTCStore((state: RTCState ) => state.peerConnection);
+  const peerConnection = useRTCStore((state: RTCState) => state.peerConnection);
 
   // HDMI and UI states
   const hdmiState = useVideoStore((state: VideoState) => state.hdmiState);
   const hdmiError = ["no_lock", "no_signal", "out_of_range"].includes(hdmiState);
   const isVideoLoading = !isPlaying;
 
+  // Mouse wheel states
   const [blockWheelEvent, setBlockWheelEvent] = useState(false);
 
   // Misc states and hooks
@@ -104,7 +105,7 @@ export default function WebRTCVideo() {
   // Pointer lock and keyboard lock related
   const isPointerLockPossible = window.location.protocol === "https:" || window.location.hostname === "localhost";
   const isFullscreenEnabled = document.fullscreenEnabled;
- 
+
   const checkNavigatorPermissions = useCallback(async (permissionName: string) => {
     if (!navigator.permissions || !navigator.permissions.query) {
       return false; // if can't query permissions, assume NOT granted
@@ -140,11 +141,11 @@ export default function WebRTCVideo() {
     if (videoElm.current === null) return;
 
     const isKeyboardLockGranted = await checkNavigatorPermissions("keyboard-lock");
-  
+
     if (isKeyboardLockGranted && "keyboard" in navigator) {
       try {
         // @ts-expect-error - keyboard lock is not supported in all browsers
-         await navigator.keyboard.lock();
+        await navigator.keyboard.lock();
       } catch {
         // ignore errors
       }
@@ -155,12 +156,12 @@ export default function WebRTCVideo() {
     if (videoElm.current === null || document.fullscreenElement !== videoElm.current) return;
 
     if ("keyboard" in navigator) {
-        try {
-          // @ts-expect-error - keyboard unlock is not supported in all browsers
-          await navigator.keyboard.unlock();
-        } catch {
-          // ignore errors
-       }
+      try {
+        // @ts-expect-error - keyboard unlock is not supported in all browsers
+        await navigator.keyboard.unlock();
+      } catch {
+        // ignore errors
+      }
     }
   }, []);
 
@@ -188,7 +189,7 @@ export default function WebRTCVideo() {
   }, [isPointerLockPossible]);
 
   const requestFullscreen = useCallback(async () => {
-     if (!isFullscreenEnabled || !videoElm.current) return;
+    if (!isFullscreenEnabled || !videoElm.current) return;
 
     // per https://wicg.github.io/keyboard-lock/#system-key-press-handler
     // If keyboard lock is activated after fullscreen is already in effect, then the user my 
@@ -352,12 +353,12 @@ export default function WebRTCVideo() {
       // https://bugzilla.mozilla.org/show_bug.cgi?id=1299553
       if (e.metaKey && hidKey < 0xE0) {
         setTimeout(() => {
-          sendKeypressEvent(hidKey, false);
+          handleKeyPress(hidKey, false);
         }, 10);
       }
-      sendKeypressEvent(hidKey, true);
+      handleKeyPress(hidKey, true);
     },
-    [sendKeypressEvent],
+    [handleKeyPress],
   );
 
   const keyUpHandler = useCallback(
@@ -365,15 +366,15 @@ export default function WebRTCVideo() {
       e.preventDefault();
       const code = getAdjustedKeyCode(e);
       const hidKey = keys[code];
-      
+
       if (hidKey === undefined) {
         console.warn(`Key up not mapped: ${code}`);
         return;
       }
 
-      sendKeypressEvent(hidKey, false);
+      handleKeyPress(hidKey, false);
     },
-    [sendKeypressEvent],
+    [handleKeyPress],
   );
 
   const videoKeyUpHandler = useCallback((e: KeyboardEvent) => {
@@ -489,7 +490,7 @@ export default function WebRTCVideo() {
 
       videoElmRefValue.addEventListener("mousemove", isRelativeMouseMode ? relMouseMoveHandler : absMouseMoveHandler, { signal });
       videoElmRefValue.addEventListener("pointerdown", isRelativeMouseMode ? relMouseMoveHandler : absMouseMoveHandler, { signal });
-      videoElmRefValue.addEventListener("pointerup", isRelativeMouseMode ? relMouseMoveHandler :absMouseMoveHandler, { signal });
+      videoElmRefValue.addEventListener("pointerup", isRelativeMouseMode ? relMouseMoveHandler : absMouseMoveHandler, { signal });
       videoElmRefValue.addEventListener("wheel", mouseWheelHandler, {
         signal,
         passive: true,
@@ -546,8 +547,8 @@ export default function WebRTCVideo() {
     return isDefault
       ? {} // No filter if all settings are default (1.0)
       : {
-          filter: `saturate(${videoSaturation}) brightness(${videoBrightness}) contrast(${videoContrast})`,
-        };
+        filter: `saturate(${videoSaturation}) brightness(${videoBrightness}) contrast(${videoContrast})`,
+      };
   }, [videoSaturation, videoBrightness, videoContrast]);
 
   function getAdjustedKeyCode(e: KeyboardEvent) {
@@ -594,48 +595,48 @@ export default function WebRTCVideo() {
                   <PointerLockBar show={showPointerLockBar} />
                   <div className="relative mx-4 my-2 flex items-center justify-center overflow-hidden">
                     <div className="relative flex h-full w-full items-center justify-center">
-                        <video
-                          ref={videoElm}
-                          autoPlay
-                          controls={false}
-                          onPlaying={onVideoPlaying}
-                          onPlay={onVideoPlaying}
-                          muted
-                          playsInline
-                          disablePictureInPicture
-                          controlsList="nofullscreen"
-                          style={videoStyle}
-                          className={cx(
-                            "max-h-full min-h-[384px] max-w-full min-w-[512px] bg-black/50 object-contain transition-all duration-1000",
-                            {
-                              "cursor-none": settings.isCursorHidden,
-                              "opacity-0":
-                                isVideoLoading ||
-                                hdmiError ||
-                                peerConnectionState !== "connected",
-                              "opacity-60!": showPointerLockBar,
-                              "animate-slideUpFade border border-slate-800/30 shadow-xs dark:border-slate-300/20":
-                                isPlaying,
-                            },
-                          )}
-                        />
-                        {peerConnection?.connectionState == "connected" && (
-                          <div
-                            style={{ animationDuration: "500ms" }}
-                            className="animate-slideUpFade pointer-events-none absolute inset-0 flex items-center justify-center"
-                          >
-                            <div className="relative h-full w-full rounded-md">
-                              <LoadingVideoOverlay show={isVideoLoading} />
-                              <HDMIErrorOverlay show={hdmiError} hdmiState={hdmiState} />
-                              <NoAutoplayPermissionsOverlay
-                                show={hasNoAutoPlayPermissions}
-                                onPlayClick={() => {
-                                  videoElm.current?.play();
-                                }}
-                              />
-                            </div>
-                          </div>
+                      <video
+                        ref={videoElm}
+                        autoPlay
+                        controls={false}
+                        onPlaying={onVideoPlaying}
+                        onPlay={onVideoPlaying}
+                        muted
+                        playsInline
+                        disablePictureInPicture
+                        controlsList="nofullscreen"
+                        style={videoStyle}
+                        className={cx(
+                          "max-h-full min-h-[384px] max-w-full min-w-[512px] bg-black/50 object-contain transition-all duration-1000",
+                          {
+                            "cursor-none": settings.isCursorHidden,
+                            "opacity-0":
+                              isVideoLoading ||
+                              hdmiError ||
+                              peerConnectionState !== "connected",
+                            "opacity-60!": showPointerLockBar,
+                            "animate-slideUpFade border border-slate-800/30 shadow-xs dark:border-slate-300/20":
+                              isPlaying,
+                          },
                         )}
+                      />
+                      {peerConnection?.connectionState == "connected" && (
+                        <div
+                          style={{ animationDuration: "500ms" }}
+                          className="animate-slideUpFade pointer-events-none absolute inset-0 flex items-center justify-center"
+                        >
+                          <div className="relative h-full w-full rounded-md">
+                            <LoadingVideoOverlay show={isVideoLoading} />
+                            <HDMIErrorOverlay show={hdmiError} hdmiState={hdmiState} />
+                            <NoAutoplayPermissionsOverlay
+                              show={hasNoAutoPlayPermissions}
+                              onPlayClick={() => {
+                                videoElm.current?.play();
+                              }}
+                            />
+                          </div>
+                        </div>
+                      )}
                     </div>
                   </div>
                   <VirtualKeyboard />

ui/src/hooks/stores.ts

@@ -436,27 +436,24 @@ export interface KeyboardLedState {
   shift: boolean; // Optional, as not all keyboards have a shift LED
 };
 
+export const hidKeyBufferSize = 6;
+export const hidErrorRollOver = 0x01;
+
 export interface KeysDownState {
 	modifier: number;
 	keys: number[];
 }
 
 export interface HidState {
-  altGrArmed: boolean;
-  setAltGrArmed: (armed: boolean) => void;
-
-  altGrTimer: number | null; // _altGrCtrlTime
-  setAltGrTimer: (timeout: number | null) => void;
-
-  altGrCtrlTime: number; // _altGrCtrlTime
-  setAltGrCtrlTime: (time: number) => void;
-
-  keyboardLedState?: KeyboardLedState;
+  keyboardLedState: KeyboardLedState;
   setKeyboardLedState: (state: KeyboardLedState) => void;
 
-  keysDownState?: KeysDownState;
+  keysDownState: KeysDownState;
   setKeysDownState: (state: KeysDownState) => void;
 
+  keyPressAvailable: boolean;
+  setKeyPressAvailable: (available: boolean) => void;
+
   isVirtualKeyboardEnabled: boolean;
   setVirtualKeyboardEnabled: (enabled: boolean) => void;
 
@@ -468,21 +465,15 @@ export interface HidState {
 }
 
 export const useHidStore = create<HidState>(set => ({
-  altGrArmed: false,
-  setAltGrArmed: (armed: boolean): void => set({ altGrArmed: armed }),
-
-  altGrTimer: 0,
-  setAltGrTimer: (timeout: number | null): void => set({ altGrTimer: timeout }),
-
-  altGrCtrlTime: 0,
-  setAltGrCtrlTime: (time: number): void => set({ altGrCtrlTime: time }),
-
-  keyboardLedState: undefined,
+  keyboardLedState: {} as KeyboardLedState,
   setKeyboardLedState: (ledState: KeyboardLedState): void => set({ keyboardLedState: ledState }),
 
-  keysDownState: undefined,
+  keysDownState: { modifier: 0, keys: [0,0,0,0,0,0] } as KeysDownState,
   setKeysDownState: (state: KeysDownState): void => set({ keysDownState: state }),
 
+  keyPressAvailable: true,
+  setKeyPressAvailable: (available: boolean) => set({ keyPressAvailable: available }),
+
   isVirtualKeyboardEnabled: false,
   setVirtualKeyboardEnabled: (enabled: boolean): void => set({ isVirtualKeyboardEnabled: enabled }),
 

ui/src/hooks/useKeyboard.ts

@@ -1,24 +1,29 @@
 import { useCallback } from "react";
 
-import { KeysDownState, HidState, useHidStore, RTCState, useRTCStore } from "@/hooks/stores";
+import { KeysDownState, HidState, useHidStore, RTCState, useRTCStore, hidKeyBufferSize, hidErrorRollOver } from "@/hooks/stores";
 import { JsonRpcResponse, useJsonRpc } from "@/hooks/useJsonRpc";
-import { keys, modifiers } from "@/keyboardMappings";
+import { hidKeyToModifierMask, keys, modifiers } from "@/keyboardMappings";
 
 export default function useKeyboard() {
   const [send] = useJsonRpc();
 
   const rpcDataChannel = useRTCStore((state: RTCState) => state.rpcDataChannel);
+
+  const keysDownState = useHidStore((state: HidState) => state.keysDownState);
   const setKeysDownState = useHidStore((state: HidState) => state.setKeysDownState);
 
-  const sendKeyboardEvent = useCallback(
-    (keys: number[], modifiers: number[]) => {
-      if (rpcDataChannel?.readyState !== "open") return;
-      const accModifier = modifiers.reduce((acc, val) => acc + val, 0);
+  const keyPressAvailable = useHidStore((state: HidState) => state.keyPressAvailable);
+  const setKeyPressAvailable = useHidStore((state: HidState) => state.setKeyPressAvailable);
+
+  const sendKeyboardEvent = useCallback(
+    (state: KeysDownState) => {
+      if (rpcDataChannel?.readyState !== "open") return;
 
-      send("keyboardReport", { keys, modifier: accModifier });
       //TODO would be nice if the keyboardReport rpc call returned the current state like keypressReport does
+      send("keyboardReport", { keys: state.keys, modifier: state.modifier });
+
       // We do this for the info bar to display the currently pressed keys for the user
-      setKeysDownState({ keys: keys, modifier: accModifier });
+      setKeysDownState(state);
     },
     [rpcDataChannel?.readyState, send, setKeysDownState],
   );
@@ -28,30 +33,37 @@ export default function useKeyboard() {
       if (rpcDataChannel?.readyState !== "open") return;
 
       send("keypressReport", { key, press }, (resp: JsonRpcResponse) => {
-            if ("error" in resp) {
-              console.error("Failed to send keypress:", resp.error);
-            } else {
-              const keyDownState = resp.result as KeysDownState;
-              // We do this for the info bar to display the currently pressed keys for the user
-              setKeysDownState(keyDownState);
-            }
-          });
+        if ("error" in resp) {
+          // -32601 means the method is not supported
+          if (resp.error.code === -32601) {
+            // if we don't support key press report, we need to disable all that handling
+            console.error("Failed calling keypressReport, switching to local handling", resp.error);
+            setKeyPressAvailable(false);
+          } else {
+            console.error(`Failed to send key ${key} press: ${press}`, resp.error);
+          }
+        } else {
+          const keyDownState = resp.result as KeysDownState;
+          // We do this for the info bar to display the currently pressed keys for the user
+          setKeysDownState(keyDownState);
+        }
+      });
     },
-    [rpcDataChannel?.readyState, send, setKeysDownState],
+    [rpcDataChannel?.readyState, send, setKeyPressAvailable, setKeysDownState],
   );
 
   const resetKeyboardState = useCallback(() => {
-    sendKeyboardEvent([], []);
+    sendKeyboardEvent({ keys: [], modifier: 0 });
   }, [sendKeyboardEvent]);
 
   const executeMacro = async (steps: { keys: string[] | null; modifiers: string[] | null; delay: number }[]) => {
     for (const [index, step] of steps.entries()) {
-      const keyValues = step.keys?.map(key => keys[key]).filter(Boolean) || [];
-      const modifierValues = step.modifiers?.map(mod => modifiers[mod]).filter(Boolean) || [];
+      const keyValues = (step.keys || []).map(key => keys[key]).filter(Boolean);
+      const modifierMask: number = (step.modifiers || []).map(mod => modifiers[mod]).reduce((acc, val) => acc + val, 0);
 
       // If the step has keys and/or modifiers, press them and hold for the delay
-      if (keyValues.length > 0 || modifierValues.length > 0) {
-        sendKeyboardEvent(keyValues, modifierValues);
+      if (keyValues.length > 0 || modifierMask > 0) {
+        sendKeyboardEvent({ keys: keyValues, modifier: modifierMask });
         await new Promise(resolve => setTimeout(resolve, step.delay || 50));
 
         resetKeyboardState();
@@ -67,5 +79,75 @@ export default function useKeyboard() {
     }
   };
 
-  return { sendKeyboardEvent, sendKeypressEvent, resetKeyboardState, executeMacro };
+  // this code exists because we have devices that don't support the keysPress api yet (not current)
+  // so we mirror the device-side code here to keep track of the keyboard state
+  function handleKeyLocally(state: KeysDownState, key: number, press: boolean): KeysDownState {
+    const keys = state.keys;
+    let modifiers = state.modifier;
+    const modifierMask = hidKeyToModifierMask[key] || 0;
+
+    if (modifierMask !== 0) {
+      if (press) {
+        modifiers |= modifierMask;
+      } else {
+        modifiers &= ~modifierMask;
+      }
+    } else {
+      // handle other keys that are not modifier keys by placing or removing them
+      // from the key buffer since the buffer tracks currently pressed keys
+      let overrun = true;
+      for (let i = 0; i < hidKeyBufferSize && overrun; i++) {
+        // If we find the key in the buffer the buffer, we either remove it (if press is false)
+        // or do nothing (if down is true) because the buffer tracks currently pressed keys
+        // and if we find a zero byte, we can place the key there (if press is true)
+        if (keys[i] == key || keys[i] == 0) {
+          if (press) {
+            keys[i] = key // overwrites the zero byte or the same key if already pressed
+          } else {
+            // we are releasing the key, remove it from the buffer
+            if (keys[i] != 0) {
+              keys.splice(i, 1);
+              keys.push(0); // add a zero at the end
+            }
+          }
+          overrun = false // We found a slot for the key
+        }
+
+        // If we reach here it means we didn't find an empty slot or the key in the buffer
+        if (overrun) {
+          if (press) {
+            console.warn(`keyboard buffer overflow, key: ${key} not added`);
+            // Fill all key slots with ErrorRollOver (0x01) to indicate overflow
+            keys.length = 6;
+            keys.fill(hidErrorRollOver);
+          } else {
+            // If we are releasing a key, and we didn't find it in a slot, who cares?
+            console.debug(`key ${key} not found in buffer, nothing to release`)
+          }
+        }
+      }
+    }
+    return { modifier: modifiers, keys };
+  }
+
+  const handleKeyPress = useCallback(
+    (key: number, press: boolean) => {
+      if (keyPressAvailable) {
+        // if the keyPress api is available, we can just send the key press event
+        sendKeypressEvent(key, press);
+
+        // TODO handle the case where the keyPress api is not available and we need to handle the key locally now...
+      } else {
+        // if the keyPress api is not available, we need to handle the key locally
+        const newKeysDownState = handleKeyLocally(keysDownState, key, press);
+        setKeysDownState(newKeysDownState);
+
+        // then we send the full state
+        sendKeyboardEvent(newKeysDownState);
+      }
+    },
+    [keyPressAvailable, keysDownState, sendKeyboardEvent, sendKeypressEvent, setKeysDownState],
+  );
+
+  return { handleKeyPress, sendKeyboardEvent, sendKeypressEvent, resetKeyboardState, executeMacro };
 }

ui/src/keyboardMappings.ts

@@ -142,6 +142,17 @@ export const modifiers = {
   MetaRight: 0x80,
 } as Record<string, number>;
 
+export const hidKeyToModifierMask = {
+  0xe0: modifiers.ControlLeft,
+  0xe1: modifiers.ShiftLeft,
+  0xe2: modifiers.AltLeft,
+  0xe3: modifiers.MetaLeft,
+  0xe4: modifiers.ControlRight,
+  0xe5: modifiers.ShiftRight,
+  0xe6: modifiers.AltRight,
+  0xe7: modifiers.MetaRight,
+} as Record<number, number>;
+
 export const modifierDisplayMap: Record<string, string> = {
   ControlLeft: "Left Ctrl",
   ControlRight: "Right Ctrl",

ui/src/routes/devices.$id.settings.tsx

@@ -29,7 +29,7 @@ import { cx } from "../cva.config";
 export default function SettingsRoute() {
   const location = useLocation();
   const setDisableVideoFocusTrap = useUiStore(state => state.setDisableVideoFocusTrap);
-  const { sendKeyboardEvent } = useKeyboard();
+  const { resetKeyboardState } = useKeyboard();
   const scrollContainerRef = useRef<HTMLDivElement>(null);
   const [showLeftGradient, setShowLeftGradient] = useState(false);
   const [showRightGradient, setShowRightGradient] = useState(false);
@@ -67,8 +67,8 @@ export default function SettingsRoute() {
   useEffect(() => {
     // disable focus trap
     setTimeout(() => {
-      // Reset keyboard state. Incase the user is pressing a key while enabling the sidebar
-      sendKeyboardEvent([], []);
+      // Reset keyboard state. In case the user is pressing a key while enabling the sidebar
+      resetKeyboardState();
       setDisableVideoFocusTrap(true);
       // For some reason, the focus trap is not disabled immediately
       // so we need to blur the active element
@@ -79,7 +79,7 @@ export default function SettingsRoute() {
     return () => {
       setDisableVideoFocusTrap(false);
     };
-  }, [sendKeyboardEvent, setDisableVideoFocusTrap]);
+  }, [resetKeyboardState, setDisableVideoFocusTrap]);
 
   return (
     <div className="pointer-events-auto relative mx-auto max-w-4xl translate-x-0 transform text-left dark:text-white">

ui/src/routes/devices.$id.tsx

@@ -519,7 +519,7 @@ export default function KvmIdRoute() {
   // Cleanup effect
   const clearInboundRtpStats = useRTCStore((state: RTCState) => state.clearInboundRtpStats);
   const clearCandidatePairStats = useRTCStore((state: RTCState) => state.clearCandidatePairStats);
-  const setSidebarView = useUiStore((state: UIState)  => state.setSidebarView);
+  const setSidebarView = useUiStore((state: UIState) => state.setSidebarView);
 
   useEffect(() => {
     return () => {
@@ -597,6 +597,7 @@ export default function KvmIdRoute() {
 
   const keysDownState = useHidStore((state: HidState) => state.keysDownState);
   const setKeysDownState = useHidStore((state: HidState) => state.setKeysDownState);
+  const setKeyPressAvailable = useHidStore((state: HidState) => state.setKeyPressAvailable);
 
   const [hasUpdated, setHasUpdated] = useState(false);
   const { navigateTo } = useDeviceUiNavigation();
@@ -624,7 +625,7 @@ export default function KvmIdRoute() {
       console.log("Setting keyboard led state", ledState);
       setKeyboardLedState(ledState);
     }
-    
+
     if (resp.method === "keysDownState") {
       const downState = resp.params as KeysDownState;
       console.log("Setting key down state", downState);
@@ -667,10 +668,12 @@ export default function KvmIdRoute() {
     });
   }, [rpcDataChannel?.readyState, send, setHdmiState]);
 
+  const [needLedState, setNeedLedState] = useState(true);
+
   // request keyboard led state from the device
   useEffect(() => {
     if (rpcDataChannel?.readyState !== "open") return;
-    if (keyboardLedState !== undefined) return;
+    if (!needLedState) return;
     console.log("Requesting keyboard led state");
 
     send("getKeyboardLedState", {}, (resp: JsonRpcResponse) => {
@@ -680,24 +683,35 @@ export default function KvmIdRoute() {
       }
       console.log("Keyboard led state", resp.result);
       setKeyboardLedState(resp.result as KeyboardLedState);
+      setNeedLedState(false);
     });
-  }, [rpcDataChannel?.readyState, send, setKeyboardLedState, keyboardLedState]);
+  }, [rpcDataChannel?.readyState, send, setKeyboardLedState, keyboardLedState, needLedState]);
+
+  const [needKeyDownState, setNeedKeyDownState] = useState(true);
 
   // request keyboard key down state from the device
   useEffect(() => {
     if (rpcDataChannel?.readyState !== "open") return;
-    if (keysDownState !== undefined) return;
+    if (!needKeyDownState) return;
     console.log("Requesting keys down state");
 
     send("getKeyDownState", {}, (resp: JsonRpcResponse) => {
       if ("error" in resp) {
-        console.error("Failed to get key down state", resp.error);
+        // -32601 means the method is not supported
+        if (resp.error.code === -32601) {
+          // if we don't support key down state, we know key press is also not available
+          console.error("Failed to get key down state, switching to old-school", resp.error);
+          setKeyPressAvailable(false);
+        } else {
+          console.error("Failed to get key down state", resp.error);
+        }
         return;
       }
       console.log("Keyboard key down state", resp.result);
       setKeysDownState(resp.result as KeysDownState);
+      setNeedKeyDownState(false);
     });
-  }, [keysDownState, rpcDataChannel?.readyState, send, setKeysDownState]);
+  }, [keysDownState, needKeyDownState, rpcDataChannel?.readyState, send, setKeyPressAvailable, setKeysDownState]);
 
   // When the update is successful, we need to refresh the client javascript and show a success modal
   useEffect(() => {
@@ -758,7 +772,7 @@ export default function KvmIdRoute() {
     send("getUpdateStatus", {}, async (resp: JsonRpcResponse) => {
       if ("error" in resp) {
         notifications.error(`Failed to get device version: ${resp.error}`);
-        return 
+        return
       }
 
       const result = resp.result as SystemVersionInfo;
@@ -895,7 +909,7 @@ interface SidebarContainerProps {
 }
 
 function SidebarContainer(props: SidebarContainerProps) {
-  const { sidebarView }= props;
+  const { sidebarView } = props;
   return (
     <div
       className={cx(