Merge 84e4b44df0 into 0d7f47c109

Made the operation keys look more like a keyboard
Add persistent state management for the Shift/Ctrl/Alt/Meta
Add infoBar indicators for Shift/Ctr/Alt/Meta/AltGr
Remove redundant Break it's actually on keyboard as (Pause)
Add a style for the virtual keyboard "depressed" buttons.
Delete the no-longer-needed button variants
Added missing keycodes
Added modifier tracking for all the other keys
Ensures the InfoBar tracks for physical and virtual keyboard
Shows what buttons are depressed for sticky keys
Now treats all the Shift/Control/Alt/Meta/AltGr keys as if they were sticky keys so users can click the button and hit the next key,

config.go

@@ -111,7 +111,7 @@ var defaultConfig = &Config{
 	ActiveExtension:      "",
 	KeyboardMacros:       []KeyboardMacro{},
 	DisplayRotation:      "270",
-	KeyboardLayout:       "en-US",
+	KeyboardLayout:       "en_US",
 	DisplayMaxBrightness: 64,
 	DisplayDimAfterSec:   120,  // 2 minutes
 	DisplayOffAfterSec:   1800, // 30 minutes

native.go

@@ -8,6 +8,7 @@ import (
 	"io"
 	"net"
 	"os"
+	"os/exec"
 	"sync"
 	"time"
 
@@ -41,6 +42,11 @@ var ongoingRequests = make(map[int32]chan *CtrlResponse)
 
 var lock = &sync.Mutex{}
 
+var (
+	nativeCmd     *exec.Cmd
+	nativeCmdLock = &sync.Mutex{}
+)
+
 func CallCtrlAction(action string, params map[string]interface{}) (*CtrlResponse, error) {
 	lock.Lock()
 	defer lock.Unlock()
@@ -129,16 +135,26 @@ func StartNativeSocketServer(socketPath string, handleClient func(net.Conn), isC
 	scopedLogger.Info().Msg("server listening")
 
 	go func() {
-		conn, err := listener.Accept()
-		listener.Close()
-		if err != nil {
-			scopedLogger.Warn().Err(err).Msg("failed to accept socket")
+		for {
+			conn, err := listener.Accept()
+
+			if err != nil {
+				scopedLogger.Warn().Err(err).Msg("failed to accept socket")
+				continue
+			}
+			if isCtrl {
+				// check if the channel is closed
+				select {
+				case <-ctrlClientConnected:
+					scopedLogger.Debug().Msg("ctrl client reconnected")
+				default:
+					close(ctrlClientConnected)
+					scopedLogger.Debug().Msg("first native ctrl socket client connected")
+				}
+			}
+
+			go handleClient(conn)
 		}
-		if isCtrl {
-			close(ctrlClientConnected)
-			scopedLogger.Debug().Msg("first native ctrl socket client connected")
-		}
-		handleClient(conn)
 	}()
 
 	return listener
@@ -235,6 +251,51 @@ func handleVideoClient(conn net.Conn) {
 	}
 }
 
+func startNativeBinaryWithLock(binaryPath string) (*exec.Cmd, error) {
+	nativeCmdLock.Lock()
+	defer nativeCmdLock.Unlock()
+
+	cmd, err := startNativeBinary(binaryPath)
+	if err != nil {
+		return nil, err
+	}
+	nativeCmd = cmd
+	return cmd, nil
+}
+
+func restartNativeBinary(binaryPath string) error {
+	time.Sleep(10 * time.Second)
+	// restart the binary
+	nativeLogger.Info().Msg("restarting jetkvm_native binary")
+	cmd, err := startNativeBinary(binaryPath)
+	if err != nil {
+		nativeLogger.Warn().Err(err).Msg("failed to restart binary")
+	}
+	nativeCmd = cmd
+	return err
+}
+
+func superviseNativeBinary(binaryPath string) error {
+	nativeCmdLock.Lock()
+	defer nativeCmdLock.Unlock()
+
+	if nativeCmd == nil || nativeCmd.Process == nil {
+		return restartNativeBinary(binaryPath)
+	}
+
+	err := nativeCmd.Wait()
+
+	if err == nil {
+		nativeLogger.Info().Err(err).Msg("jetkvm_native binary exited with no error")
+	} else if exiterr, ok := err.(*exec.ExitError); ok {
+		nativeLogger.Warn().Int("exit_code", exiterr.ExitCode()).Msg("jetkvm_native binary exited with error")
+	} else {
+		nativeLogger.Warn().Err(err).Msg("jetkvm_native binary exited with unknown error")
+	}
+
+	return restartNativeBinary(binaryPath)
+}
+
 func ExtractAndRunNativeBin() error {
 	binaryPath := "/userdata/jetkvm/bin/jetkvm_native"
 	if err := ensureBinaryUpdated(binaryPath); err != nil {
@@ -246,12 +307,28 @@ func ExtractAndRunNativeBin() error {
 		return fmt.Errorf("failed to make binary executable: %w", err)
 	}
 	// Run the binary in the background
-	cmd, err := startNativeBinary(binaryPath)
+	cmd, err := startNativeBinaryWithLock(binaryPath)
 	if err != nil {
 		return fmt.Errorf("failed to start binary: %w", err)
 	}
 
-	//TODO: add auto restart
+	// check if the binary is still running every 10 seconds
+	go func() {
+		for {
+			select {
+			case <-appCtx.Done():
+				nativeLogger.Info().Msg("stopping native binary supervisor")
+				return
+			default:
+				err := superviseNativeBinary(binaryPath)
+				if err != nil {
+					nativeLogger.Warn().Err(err).Msg("failed to supervise native binary")
+					time.Sleep(1 * time.Second) // Add a short delay to prevent rapid successive calls
+				}
+			}
+		}
+	}()
+
 	go func() {
 		<-appCtx.Done()
 		nativeLogger.Info().Int("pid", cmd.Process.Pid).Msg("killing process")

ui/src/components/InfoBar.tsx

@@ -41,6 +41,12 @@ export default function InfoBar() {
   const keyboardLedStateSyncAvailable = useHidStore(state => state.keyboardLedStateSyncAvailable);
   const keyboardLedSync = useSettingsStore(state => state.keyboardLedSync);
   
+  const isShiftActive = useHidStore(state => state.isShiftActive);
+  const isCtrlActive = useHidStore(state => state.isCtrlActive);
+  const isAltActive = useHidStore(state => state.isAltActive);
+  const isMetaActive = useHidStore(state => state.isMetaActive);
+  const isAltGrActive = useHidStore(state => state.isAltGrActive);
+
   const isTurnServerInUse = useRTCStore(state => state.isTurnServerInUse);
 
   const usbState = useHidStore(state => state.usbState);
@@ -135,6 +141,56 @@ export default function InfoBar() {
               {keyboardLedSync === "browser" ? "Browser" : "Host"}
             </div>
           ) : null}
+          <div
+            className={cx(
+              "shrink-0 p-1 px-1.5 text-xs",
+              isShiftActive
+                ? "text-black dark:text-white"
+                : "text-slate-800/20 dark:text-slate-300/20",
+            )}
+          >
+            Shift
+          </div>
+          <div
+            className={cx(
+              "shrink-0 p-1 px-1.5 text-xs",
+              isCtrlActive
+                ? "text-black dark:text-white"
+                : "text-slate-800/20 dark:text-slate-300/20",
+            )}
+          >
+            Ctrl
+          </div>
+          <div
+            className={cx(
+              "shrink-0 p-1 px-1.5 text-xs",
+              isAltActive
+                ? "text-black dark:text-white"
+                : "text-slate-800/20 dark:text-slate-300/20",
+            )}
+          >
+            Alt
+          </div>
+          <div
+            className={cx(
+              "shrink-0 p-1 px-1.5 text-xs",
+              isMetaActive
+                ? "text-black dark:text-white"
+                : "text-slate-800/20 dark:text-slate-300/20",
+            )}
+          >
+            Meta
+          </div>
+          <div
+            className={cx(
+              "shrink-0 p-1 px-1.5 text-xs",
+              isAltGrActive
+                ? "text-black dark:text-white"
+                : "text-slate-800/20 dark:text-slate-300/20",
+            )}
+          >
+            AltGr
+          </div>
           <div
             className={cx(
               "shrink-0 p-1 px-1.5 text-xs",

ui/src/components/VirtualKeyboard.tsx

@@ -27,6 +27,7 @@ const AttachIcon = ({ className }: { className?: string }) => {
 
 function KeyboardWrapper() {
   const [layoutName, setLayoutName] = useState("default");
+  const [depressedButtons, setDepressedButtons] = useState("");
 
   const keyboardRef = useRef<HTMLDivElement>(null);
   const showAttachedVirtualKeyboard = useUiStore(
@@ -54,6 +55,21 @@ function KeyboardWrapper() {
 
   const setIsCapsLockActive = useHidStore(state => state.setIsCapsLockActive);
 
+  const isShiftActive = useHidStore(state => state.isShiftActive);
+  const setIsShiftActive = useHidStore(state => state.setIsShiftActive);
+
+  const isCtrlActive = useHidStore(state => state.isCtrlActive);
+  const setIsCtrlActive = useHidStore(state => state.setIsCtrlActive);
+
+  const isAltActive = useHidStore(state => state.isAltActive);
+  const setIsAltActive = useHidStore(state => state.setIsAltActive);
+
+  const isMetaActive = useHidStore(state => state.isMetaActive);
+  const setIsMetaActive = useHidStore(state => state.setIsMetaActive);
+
+  const isAltGrActive = useHidStore(state => state.isAltGrActive);
+  const setIsAltGrActive = useHidStore(state => state.setIsAltGrActive);
+
   const startDrag = useCallback((e: MouseEvent | TouchEvent) => {
     if (!keyboardRef.current) return;
     if (e instanceof TouchEvent && e.touches.length > 1) return;
@@ -123,80 +139,123 @@ function KeyboardWrapper() {
     };
   }, [endDrag, onDrag, startDrag]);
 
-  const onKeyDown = useCallback(
-    (key: string) => {
-      const isKeyShift = key === "{shift}" || key === "ShiftLeft" || key === "ShiftRight";
-      const isKeyCaps = key === "CapsLock";
-      const cleanKey = key.replace(/[()]/g, "");
-      const keyHasShiftModifier = key.includes("(");
+  useEffect(() => {
+    // if you have the CapsLock "down", then the shift state is inverted
+    const effectiveShift = isCapsLockActive ? false === isShiftActive : isShiftActive;
+    setLayoutName(effectiveShift ? "shift" : "default");
+  },
+    [setLayoutName, isCapsLockActive, isShiftActive]
+  );
 
-      // Handle toggle of layout for shift or caps lock
-      const toggleLayout = () => {
-        setLayoutName(prevLayout => (prevLayout === "default" ? "shift" : "default"));
-      };
+  // this causes the buttons to look depressed/clicked depending on the sticky state
+  useEffect(() => {
+    let buttons = "None ";  // make sure we name at least one (fake) button
+    if (isCapsLockActive) buttons += "CapsLock ";
+    if (isShiftActive) buttons += "ShiftLeft ShiftRight ";
+    if (isCtrlActive) buttons += "ControlLeft ControlRight ";
+    if (isAltActive) buttons += "AltLeft AltRight ";
+    if (isMetaActive) buttons += "MetaLeft MetaRight ";
+    setDepressedButtons(buttons.trimEnd());
+  },
+    [setDepressedButtons, isCapsLockActive, isShiftActive, isCtrlActive, isAltActive, isMetaActive, isAltGrActive]
+  );
 
-      if (key === "CtrlAltDelete") {
-        sendKeyboardEvent(
-          [keys["Delete"]],
-          [modifiers["ControlLeft"], modifiers["AltLeft"]],
-        );
-        setTimeout(resetKeyboardState, 100);
-        return;
-      }
+  const onKeyPress = useCallback((key: string) => {
+    // handle the fake combo keys first
+    if (key === "CtrlAltDelete") {
+      sendKeyboardEvent(
+        [keys["Delete"]],
+        [modifiers["ControlLeft"], modifiers["AltLeft"]],
+      );
+      setTimeout(resetKeyboardState, 100);
+      return;
+    }
 
-      if (key === "AltMetaEscape") {
-        sendKeyboardEvent(
-          [keys["Escape"]],
-          [modifiers["MetaLeft"], modifiers["AltLeft"]],
-        );
-
-        setTimeout(resetKeyboardState, 100);
-        return;
-      }
-
-      if (key === "CtrlAltBackspace") {
-        sendKeyboardEvent(
-          [keys["Backspace"]],
-          [modifiers["ControlLeft"], modifiers["AltLeft"]],
-        );
-
-        setTimeout(resetKeyboardState, 100);
-        return;
-      }
-
-      if (isKeyShift || isKeyCaps) {
-        toggleLayout();
-
-        if (isCapsLockActive) {
-          if (!isKeyboardLedManagedByHost) {
-            setIsCapsLockActive(false);
-          }
-          sendKeyboardEvent([keys["CapsLock"]], []);
-          return;
-        }
-      }
-
-      // Handle caps lock state change
-      if (isKeyCaps && !isKeyboardLedManagedByHost) {
-        setIsCapsLockActive(!isCapsLockActive);
-      }
-
-      // Collect new active keys and modifiers
-      const newKeys = keys[cleanKey] ? [keys[cleanKey]] : [];
-      const newModifiers =
-        keyHasShiftModifier && !isCapsLockActive ? [modifiers["ShiftLeft"]] : [];
-
-      // Update current keys and modifiers
-      sendKeyboardEvent(newKeys, newModifiers);
-
-      // If shift was used as a modifier and caps lock is not active, revert to default layout
-      if (keyHasShiftModifier && !isCapsLockActive) {
-        setLayoutName("default");
-      }
+    if (key === "AltMetaEscape") {
+      sendKeyboardEvent(
+        [keys["Escape"]],
+        [modifiers["MetaLeft"], modifiers["AltLeft"]],
+      );
 
       setTimeout(resetKeyboardState, 100);
-    },
-    [isCapsLockActive, isKeyboardLedManagedByHost, sendKeyboardEvent, resetKeyboardState, setIsCapsLockActive],
+      return;
+    }
+
+    if (key === "CtrlAltBackspace") {
+      sendKeyboardEvent(
+        [keys["Backspace"]],
+        [modifiers["ControlLeft"], modifiers["AltLeft"]],
+      );
+
+      setTimeout(resetKeyboardState, 100);
+      return;
+    }
+
+    // strip away the parens for shifted characters
+    const cleanKey = key.replace(/[()]/g, "");
+
+    const passthrough = ["PrintScreen", "SystemRequest", "Pause", "Break", "ScrollLock", "Enter", "Space"].find((value) => value === cleanKey);
+
+    if (passthrough) {
+      emitkeycode(cleanKey);
+      return;
+    }
+
+    // adjust the sticky state of the Shift/Ctrl/Alt/Meta/AltGr
+    if (key === "CapsLock" && !isKeyboardLedManagedByHost)
+      setIsCapsLockActive(!isCapsLockActive);
+    else if (key === "ShiftLeft" || key === "ShiftRight")
+      setIsShiftActive(!isShiftActive);
+    else if (key === "ControlLeft" || key === "ControlRight")
+      setIsCtrlActive(!isCtrlActive);
+    else if (key === "AltLeft" || key === "AltRight")
+      setIsAltActive(!isAltActive);
+    else if (key === "MetaLeft" || key === "MetaRight")
+      setIsMetaActive(!isMetaActive);
+    else if (key === "AltGr")
+      setIsAltGrActive(!isAltGrActive);
+
+    emitkeycode(cleanKey);
+
+    function emitkeycode(key: string) {
+      const effectiveMods: number[] = [];
+
+      if (isShiftActive)
+        effectiveMods.push(modifiers["ShiftLeft"]);
+
+      if (isCtrlActive)
+        effectiveMods.push(modifiers["ControlLeft"]);
+
+      if (isAltActive)
+        effectiveMods.push(modifiers["AltLeft"]);
+
+      if (isMetaActive)
+        effectiveMods.push(modifiers["MetaLeft"]);
+
+      if (isAltGrActive) {
+        effectiveMods.push(modifiers["MetaRight"]);
+        effectiveMods.push(modifiers["CtrlLeft"]);
+      }
+
+      const keycode = keys[key];
+      if (keycode) {
+        // send the keycode with modifiers
+        sendKeyboardEvent([keycode], effectiveMods);
+      }
+
+      // release the key (if one pressed), but retain the modifiers
+      setTimeout(() => sendKeyboardEvent([], effectiveMods), 50);
+    }
+  },
+    [isKeyboardLedManagedByHost,
+      setIsCapsLockActive, isCapsLockActive,
+      setIsShiftActive, isShiftActive,
+      setIsCtrlActive, isCtrlActive,
+      setIsAltActive, isAltActive,
+      setIsMetaActive, isMetaActive,
+      setIsAltGrActive, isAltGrActive,
+      sendKeyboardEvent, resetKeyboardState
+    ],
   );
 
   const virtualKeyboard = useHidStore(state => state.isVirtualKeyboardEnabled);
@@ -276,12 +335,16 @@ function KeyboardWrapper() {
                     <Keyboard
                       baseClass="simple-keyboard-main"
                       layoutName={layoutName}
-                      onKeyPress={onKeyDown}
+                      onKeyPress={onKeyPress}
                       buttonTheme={[
                         {
                           class: "combination-key",
                           buttons: "CtrlAltDelete AltMetaEscape CtrlAltBackspace",
                         },
+                        {
+                          class: "depressed-key",
+                          buttons: depressedButtons
+                        },
                       ]}
                       display={keyDisplayMap}
                       layout={{
@@ -305,8 +368,6 @@ function KeyboardWrapper() {
                         ],
                       }}
                       disableButtonHold={true}
-                      syncInstanceInputs={true}
-                      debug={false}
                     />
 
                     <div className="controlArrows">
@@ -314,25 +375,24 @@ function KeyboardWrapper() {
                         baseClass="simple-keyboard-control"
                         theme="simple-keyboard hg-theme-default hg-layout-default"
                         layoutName={layoutName}
-                        onKeyPress={onKeyDown}
+                        onKeyPress={onKeyPress}
                         display={keyDisplayMap}
                         layout={{
-                          default: ["PrintScreen ScrollLock Pause", "Insert Home Pageup", "Delete End Pagedown"],
-                          shift: ["(PrintScreen) ScrollLock (Pause)", "Insert Home Pageup", "Delete End Pagedown"],
+                          default: ["PrintScreen ScrollLock Pause", "Insert Home PageUp", "Delete End PageDown"],
+                          shift: ["(PrintScreen) ScrollLock (Pause)", "Insert Home PageUp", "Delete End PageDown"],
                         }}
-                        syncInstanceInputs={true}
-                        debug={false}
+                        disableButtonHold={true}
                       />
                       <Keyboard
                         baseClass="simple-keyboard-arrows"
                         theme="simple-keyboard hg-theme-default hg-layout-default"
-                        onKeyPress={onKeyDown}
+                        onKeyPress={onKeyPress}
                         display={keyDisplayMap}
                         layout={{
                           default: ["ArrowUp", "ArrowLeft ArrowDown ArrowRight"],
+                          shift: ["ArrowUp", "ArrowLeft ArrowDown ArrowRight"],
                         }}
-                        syncInstanceInputs={true}
-                        debug={false}
+                        disableButtonHold={true}
                       />
                     </div>
                   </div>

ui/src/components/WebRTCVideo.tsx

@@ -71,6 +71,15 @@ export default function WebRTCVideo() {
   const hdmiError = ["no_lock", "no_signal", "out_of_range"].includes(hdmiState);
   const isVideoLoading = !isPlaying;
 
+  // Keyboard related states
+  const { 
+    setIsShiftActive,
+    setIsCtrlActive,
+    setIsAltActive,
+    setIsMetaActive,
+    setIsAltGrActive
+  } = useHidStore();
+
   const [blockWheelEvent, setBlockWheelEvent] = useState(false);
 
   // Misc states and hooks
@@ -115,9 +124,18 @@ export default function WebRTCVideo() {
   const isFullscreenEnabled = document.fullscreenEnabled;
  
   const checkNavigatorPermissions = useCallback(async (permissionName: string) => {
-    const name = permissionName as PermissionName;
-    const { state } = await navigator.permissions.query({ name });
-    return state === "granted";
+    if (!navigator.permissions || !navigator.permissions.query) {
+      return false; // if can't query permissions, assume NOT granted
+    }
+
+    try {
+      const name = permissionName as PermissionName;
+      const { state } = await navigator.permissions.query({ name });
+      return state === "granted";
+    } catch {
+      // ignore errors
+    }
+    return false; // if query fails, assume NOT granted
   }, []);
 
   const requestPointerLock = useCallback(async () => {
@@ -128,7 +146,11 @@ export default function WebRTCVideo() {
     const isPointerLockGranted = await checkNavigatorPermissions("pointer-lock");
 
     if (isPointerLockGranted && settings.mouseMode === "relative") {
-      await videoElm.current.requestPointerLock();
+      try {
+        await videoElm.current.requestPointerLock();
+      } catch {
+        // ignore errors
+      }
     }
   }, [checkNavigatorPermissions, isPointerLockPossible, settings.mouseMode]);
 
@@ -136,10 +158,13 @@ export default function WebRTCVideo() {
     if (videoElm.current === null) return;
 
     const isKeyboardLockGranted = await checkNavigatorPermissions("keyboard-lock");
-    if (isKeyboardLockGranted) {
-      if ("keyboard" in navigator) {
+  
+    if (isKeyboardLockGranted && "keyboard" in navigator) {
+      try {
         // @ts-expect-error - keyboard lock is not supported in all browsers
-        await navigator.keyboard.lock();
+         await navigator.keyboard.lock();
+      } catch {
+        // ignore errors
       }
     }
   }, [checkNavigatorPermissions]);
@@ -148,8 +173,12 @@ export default function WebRTCVideo() {
     if (videoElm.current === null || document.fullscreenElement !== videoElm.current) return;
 
     if ("keyboard" in navigator) {
-        // @ts-expect-error - keyboard unlock is not supported in all browsers
-        await navigator.keyboard.unlock();
+        try {
+          // @ts-expect-error - keyboard unlock is not supported in all browsers
+          await navigator.keyboard.unlock();
+        } catch {
+          // ignore errors
+       }
     }
   }, []);
 
@@ -403,6 +432,12 @@ export default function WebRTCVideo() {
         setIsScrollLockActive(e.getModifierState("ScrollLock"));
       }
 
+      setIsShiftActive(e.getModifierState("Shift"))
+      setIsCtrlActive(e.getModifierState("Control"))
+      setIsAltActive(e.getModifierState("Alt"))
+      setIsMetaActive(e.getModifierState("Meta"))
+      setIsAltGrActive(e.getModifierState("AltGraph"))
+
       if (code == "IntlBackslash" && ["`", "~"].includes(key)) {
         code = "Backquote";
       } else if (code == "Backquote" && ["§", "±"].includes(key)) {
@@ -432,12 +467,17 @@ export default function WebRTCVideo() {
       sendKeyboardEvent([...new Set(newKeys)], [...new Set(newModifiers)]);
     },
     [
-      handleModifierKeys,
-      sendKeyboardEvent,
       isKeyboardLedManagedByHost,
       setIsNumLockActive,
       setIsCapsLockActive,
       setIsScrollLockActive,
+      setIsShiftActive,
+      setIsCtrlActive,
+      setIsAltActive,
+      setIsMetaActive,
+      setIsAltGrActive,
+      handleModifierKeys,
+      sendKeyboardEvent
     ],
   );
 
@@ -452,6 +492,12 @@ export default function WebRTCVideo() {
         setIsScrollLockActive(e.getModifierState("ScrollLock"));
       }
 
+      setIsShiftActive(e.getModifierState("Shift"))
+      setIsCtrlActive(e.getModifierState("Control"))
+      setIsAltActive(e.getModifierState("Alt"))
+      setIsMetaActive(e.getModifierState("Meta"))
+      setIsAltGrActive(e.getModifierState("AltGraph"))
+
       // Filtering out the key that was just released (keys[e.code])
       const newKeys = prev.activeKeys.filter(k => k !== keys[e.code]).filter(Boolean);
 
@@ -464,12 +510,17 @@ export default function WebRTCVideo() {
       sendKeyboardEvent([...new Set(newKeys)], [...new Set(newModifiers)]);
     },
     [
-      handleModifierKeys,
-      sendKeyboardEvent,
       isKeyboardLedManagedByHost,
       setIsNumLockActive,
       setIsCapsLockActive,
       setIsScrollLockActive,
+      setIsShiftActive,
+      setIsCtrlActive,
+      setIsAltActive,
+      setIsMetaActive,
+      setIsAltGrActive,
+      handleModifierKeys,
+      sendKeyboardEvent
     ],
   );
 

ui/src/components/popovers/PasteModal.tsx

@@ -39,11 +39,11 @@ export default function PasteModal() {
     state => state.setKeyboardLayout,
   );
 
-  // this ensures we always get the original en-US if it hasn't been set yet
+  // this ensures we always get the original en_US if it hasn't been set yet
   const safeKeyboardLayout = useMemo(() => {
     if (keyboardLayout && keyboardLayout.length > 0)
       return keyboardLayout;
-    return "en-US";
+    return "en_US";
   }, [keyboardLayout]);
 
   useEffect(() => {

ui/src/hooks/stores.ts

@@ -481,6 +481,21 @@ export interface HidState {
   isVirtualKeyboardEnabled: boolean;
   setVirtualKeyboardEnabled: (enabled: boolean) => void;
 
+  isShiftActive: boolean;
+  setIsShiftActive: (enabled: boolean) => void;
+
+  isCtrlActive: boolean;
+  setIsCtrlActive: (enabled: boolean) => void;
+
+  isAltActive: boolean;
+  setIsAltActive: (enabled: boolean) => void;
+
+  isMetaActive: boolean;
+  setIsMetaActive: (enabled: boolean) => void;
+
+  isAltGrActive: boolean;
+  setIsAltGrActive: (enabled: boolean) => void;
+
   isPasteModeEnabled: boolean;
   setPasteModeEnabled: (enabled: boolean) => void;
 
@@ -527,6 +542,21 @@ export const useHidStore = create<HidState>((set, get) => ({
   isVirtualKeyboardEnabled: false,
   setVirtualKeyboardEnabled: enabled => set({ isVirtualKeyboardEnabled: enabled }),
 
+  isShiftActive: false,
+  setIsShiftActive: enabled => set({ isShiftActive: enabled }),
+
+  isCtrlActive: false,
+  setIsCtrlActive: enabled => set({ isCtrlActive: enabled }),
+
+  isAltActive: false,
+  setIsAltActive: enabled => set({ isAltActive: enabled }),
+
+  isMetaActive: false,
+  setIsMetaActive: enabled => set({ isMetaActive: enabled }),
+
+  isAltGrActive: false,
+  setIsAltGrActive: enabled => set({ isAltGrActive: enabled }),
+
   isPasteModeEnabled: false,
   setPasteModeEnabled: enabled => set({ isPasteModeEnabled: enabled }),
 

ui/src/index.css

@@ -222,6 +222,10 @@ video::-webkit-media-controls {
   background: none;
 }
 
+.simple-keyboard .hg-button.depressed-key {
+  @apply border-gray-800! border-b-gray-600! border-t-gray-900! bg-gray-700!;
+}
+
 .simple-keyboard .hg-button.selectedButton {
   background: rgba(5, 25, 70, 0.53);
   @apply text-white;

ui/src/keyboardMappings.ts

@@ -5,14 +5,14 @@ export const keys = {
   ArrowLeft: 0x50,
   ArrowRight: 0x4f,
   ArrowUp: 0x52,
-  Backquote: 0x35,
+  Backquote: 0x35,    // aka Grave
   Backslash: 0x31,
   Backspace: 0x2a,
-  BracketLeft: 0x2f,
-  BracketRight: 0x30,
+  BracketLeft: 0x2f,  // aka LeftBrace
+  BracketRight: 0x30, // aka RightBrace
   CapsLock: 0x39,
   Comma: 0x36,
-  ContextMenu: 0,
+  Compose: 0x65,
   Delete: 0x4c,
   Digit0: 0x27,
   Digit1: 0x1e,
@@ -41,6 +41,18 @@ export const keys = {
   F11: 0x44,
   F12: 0x45,
   F13: 0x68,
+  F14: 0x69,
+  F15: 0x6a,
+  F16: 0x6b,
+  F17: 0x6c,
+  F18: 0x6d,
+  F19: 0x6e,
+  F20: 0x6f,
+  F21: 0x70,
+  F22: 0x71,
+  F23: 0x72,
+  F24: 0x73,
+  HashTilde: 0x32,
   Home: 0x4a,
   Insert: 0x49,
   IntlBackslash: 0x64,
@@ -70,37 +82,58 @@ export const keys = {
   KeyX: 0x1b,
   KeyY: 0x1c,
   KeyZ: 0x1d,
+  KeypadComma: 0x85,
+  KeypadEqual: 0x86,
+  KeyRO: 0x87,
+  KatakanaHiragana: 0x88,
+  Yen: 0x89,
+  Henkan: 0x8a,
+  Muhenkan: 0x8b,
+  KPJPComma: 0x8c,
+  International7: 0x8d,
+  International8: 0x8e,
+  International9: 0x8f,
+  Hangeul: 0x90,
+  Hanja: 0x91,
+  Katakana: 0x92,
+  Hiragana: 0x93,
+  Zenkakuhankaku:0x94,
+  KeypadLeftParen: 0xb6,
+  KeypadRightParen: 0xb7,
   KeypadExclamation: 0xcf,
   Minus: 0x2d,
-  NumLock: 0x53,
-  Numpad0: 0x62,
-  Numpad1: 0x59,
-  Numpad2: 0x5a,
-  Numpad3: 0x5b,
-  Numpad4: 0x5c,
+  None: 0x00,
+  NumLock: 0x53,  // and Clear
+  Numpad0: 0x62,  // and Insert
+  Numpad1: 0x59,  // and End
+  Numpad2: 0x5a,  // and Down Arrow
+  Numpad3: 0x5b,  // and Page Down
+  Numpad4: 0x5c,  // and Left Arrow
   Numpad5: 0x5d,
-  Numpad6: 0x5e,
-  Numpad7: 0x5f,
-  Numpad8: 0x60,
-  Numpad9: 0x61,
-  NumpadAdd: 0x57,
-  NumpadDivide: 0x54,
+  Numpad6: 0x5e,  // and Right Arrow
+  Numpad7: 0x5f,  // and Home
+  Numpad8: 0x60,  // and Up Arrow
+  Numpad9: 0x61,  // and Page Up
+  NumpadPlus: 0x57,
+  NumpadSlash: 0x54,
   NumpadEnter: 0x58,
   NumpadEqual: 0x67,
-  NumpadMultiply: 0x55,
-  NumpadSubtract: 0x56,
-  NumpadDecimal: 0x63,
+  NumpadAsterisk: 0x55,
+  NumpadMinus: 0x56,
+  NumpadDecimal: 0x63,  // aka NumpadDot and Delete
+  Overflow: 0x01,
   PageDown: 0x4e,
   PageUp: 0x4b,
-  Period: 0x37,
+  Period: 0x37,     // aka Dot
   PrintScreen: 0x46,
   Pause: 0x48,
-  Quote: 0x34,
+  Power: 0x66,
+  Quote: 0x34,      // aka Apostrophe
   ScrollLock: 0x47,
   Semicolon: 0x33,
   Slash: 0x38,
   Space: 0x2c,
-  SystemRequest: 0x9a,
+  SystemRequest: 0x9a,  // aka Attention
   Tab: 0x2b,
 } as Record<string, number>;
 
@@ -131,23 +164,23 @@ export const keyDisplayMap: Record<string, string> = {
   AltMetaEscape: "Alt + Meta + Escape",
   CtrlAltBackspace: "Ctrl + Alt + Backspace",
   Escape: "esc",
-  Tab: "tab",
-  Backspace: "backspace",
-  "(Backspace)": "backspace",
+  Tab: "tab ⇥",
+  Backspace: "backspace ⌫",
   Enter: "enter",
-  CapsLock: "caps lock",
-  ShiftLeft: "shift",
-  ShiftRight: "shift",
-  ControlLeft: "ctrl",
-  AltLeft: "alt",
-  AltRight: "alt",
-  MetaLeft: "meta",
-  MetaRight: "meta",
+  CapsLock: "caps lock ⇪",
+  ShiftLeft: "shift ⇧",
+  ShiftRight: "⇧ shift",
+  ControlLeft: "ctrl ⌃",
+  ControlRight: "⌃ ctrl",
+  AltLeft: "alt ⌥",
+  AltRight: "⌥ alt",
+  MetaLeft: "meta ⌘", // "meta ⊞" for windows
+  MetaRight: "⌘ meta",// "≣ meta" for windows
   Space: " ",
-  Insert: "insert",
+  Insert: "ins",
   Home: "home",
   PageUp: "page up",
-  Delete: "delete",
+  Delete: "del ⌦",
   End: "end",
   PageDown: "page down",
   ArrowLeft: "←",
@@ -213,13 +246,12 @@ export const keyDisplayMap: Record<string, string> = {
   Numpad0: "Num 0", Numpad1: "Num 1", Numpad2: "Num 2",
   Numpad3: "Num 3", Numpad4: "Num 4", Numpad5: "Num 5",
   Numpad6: "Num 6", Numpad7: "Num 7", Numpad8: "Num 8",
-  Numpad9: "Num 9", NumpadAdd: "Num +", NumpadSubtract: "Num -",
-  NumpadMultiply: "Num *", NumpadDivide: "Num /", NumpadDecimal: "Num .",
+  Numpad9: "Num 9", NumpadPlus: "Num +", NumpadMinus: "Num -",
+  NumpadAsterisk: "Num *", NumpadSlash: "Num /", NumpadDecimal: "Num .",
   NumpadEqual: "Num =", NumpadEnter: "Num Enter",
   NumLock: "Num Lock",
 
   // Modals
   PrintScreen: "prt sc", ScrollLock: "scr lk", Pause: "pause",
-  "(PrintScreen)": "sys rq", "(Pause)": "break",
-  SystemRequest: "sys rq", Break: "break"
+  "(PrintScreen)": "sys rq", "(Pause)": "break"
 };

ui/src/routes/devices.$id.settings.keyboard.tsx

@@ -25,11 +25,11 @@ export default function SettingsKeyboardRoute() {
     state => state.setShowPressedKeys,
   );
 
-  // this ensures we always get the original en-US if it hasn't been set yet
+  // this ensures we always get the original en_US if it hasn't been set yet
   const safeKeyboardLayout = useMemo(() => {
       if (keyboardLayout && keyboardLayout.length > 0)
         return keyboardLayout;
-      return "en-US";
+      return "en_US";
   }, [keyboardLayout]);
 
   const layoutOptions = Object.entries(layouts).map(([code, language]) => { return { value: code, label: language } })