feat: sync keyboard led status (#502)

internal/usbgadget/hid_keyboard.go

@@ -1,8 +1,11 @@
 package usbgadget
 
 import (
+	"context"
 	"fmt"
 	"os"
+	"reflect"
+	"time"
 )
 
 var keyboardConfig = gadgetConfigItem{
@@ -36,6 +39,7 @@ var keyboardReportDesc = []byte{
 	0x81, 0x03, /*   INPUT (Cnst,Var,Abs)                 */
 	0x95, 0x05, /*   REPORT_COUNT (5)                     */
 	0x75, 0x01, /*   REPORT_SIZE (1)                      */
+
 	0x05, 0x08, /*   USAGE_PAGE (LEDs)                    */
 	0x19, 0x01, /*   USAGE_MINIMUM (Num Lock)             */
 	0x29, 0x05, /*   USAGE_MAXIMUM (Kana)                 */
@@ -54,13 +58,139 @@ var keyboardReportDesc = []byte{
 	0xc0, /* END_COLLECTION                         */
 }
 
-func (u *UsbGadget) keyboardWriteHidFile(data []byte) error {
+const (
-	if u.keyboardHidFile == nil {
+	hidReadBufferSize = 8
-		var err error
+	// https://www.usb.org/sites/default/files/documents/hid1_11.pdf
-		u.keyboardHidFile, err = os.OpenFile("/dev/hidg0", os.O_RDWR, 0666)
+	// https://www.usb.org/sites/default/files/hut1_2.pdf
-		if err != nil {
+	KeyboardLedMaskNumLock    = 1 << 0
-			return fmt.Errorf("failed to open hidg0: %w", err)
+	KeyboardLedMaskCapsLock   = 1 << 1
+	KeyboardLedMaskScrollLock = 1 << 2
+	KeyboardLedMaskCompose    = 1 << 3
+	KeyboardLedMaskKana       = 1 << 4
+	ValidKeyboardLedMasks     = KeyboardLedMaskNumLock | KeyboardLedMaskCapsLock | KeyboardLedMaskScrollLock | KeyboardLedMaskCompose | KeyboardLedMaskKana
+)
+
+// Synchronization between LED states and CAPS LOCK, NUM LOCK, SCROLL LOCK,
+// COMPOSE, and KANA events is maintained by the host and NOT the keyboard. If
+// using the keyboard descriptor in Appendix B, LED states are set by sending a
+// 5-bit absolute report to the keyboard via a Set_Report(Output) request.
+type KeyboardState struct {
+	NumLock    bool `json:"num_lock"`
+	CapsLock   bool `json:"caps_lock"`
+	ScrollLock bool `json:"scroll_lock"`
+	Compose    bool `json:"compose"`
+	Kana       bool `json:"kana"`
+}
+
+func getKeyboardState(b byte) KeyboardState {
+	// should we check if it's the correct usage page?
+	return KeyboardState{
+		NumLock:    b&KeyboardLedMaskNumLock != 0,
+		CapsLock:   b&KeyboardLedMaskCapsLock != 0,
+		ScrollLock: b&KeyboardLedMaskScrollLock != 0,
+		Compose:    b&KeyboardLedMaskCompose != 0,
+		Kana:       b&KeyboardLedMaskKana != 0,
+	}
+}
+
+func (u *UsbGadget) updateKeyboardState(b byte) {
+	u.keyboardStateLock.Lock()
+	defer u.keyboardStateLock.Unlock()
+
+	if b&^ValidKeyboardLedMasks != 0 {
+		u.log.Trace().Uint8("b", b).Msg("contains invalid bits, ignoring")
+		return
+	}
+
+	newState := getKeyboardState(b)
+	if reflect.DeepEqual(u.keyboardState, newState) {
+		return
+	}
+	u.log.Info().Interface("old", u.keyboardState).Interface("new", newState).Msg("keyboardState updated")
+	u.keyboardState = newState
+
+	if u.onKeyboardStateChange != nil {
+		(*u.onKeyboardStateChange)(newState)
+	}
+}
+
+func (u *UsbGadget) SetOnKeyboardStateChange(f func(state KeyboardState)) {
+	u.onKeyboardStateChange = &f
+}
+
+func (u *UsbGadget) GetKeyboardState() KeyboardState {
+	u.keyboardStateLock.Lock()
+	defer u.keyboardStateLock.Unlock()
+
+	return u.keyboardState
+}
+
+func (u *UsbGadget) listenKeyboardEvents() {
+	var path string
+	if u.keyboardHidFile != nil {
+		path = u.keyboardHidFile.Name()
+	}
+	l := u.log.With().Str("listener", "keyboardEvents").Str("path", path).Logger()
+	l.Trace().Msg("starting")
+
+	go func() {
+		buf := make([]byte, hidReadBufferSize)
+		for {
+			select {
+			case <-u.keyboardStateCtx.Done():
+				l.Info().Msg("context done")
+				return
+			default:
+				l.Trace().Msg("reading from keyboard")
+				if u.keyboardHidFile == nil {
+					l.Error().Msg("keyboardHidFile is nil")
+					time.Sleep(time.Second)
+					continue
+				}
+				n, err := u.keyboardHidFile.Read(buf)
+				if err != nil {
+					l.Error().Err(err).Msg("failed to read")
+					continue
+				}
+				l.Trace().Int("n", n).Bytes("buf", buf).Msg("got data from keyboard")
+				if n != 1 {
+					l.Trace().Int("n", n).Msg("expected 1 byte, got")
+					continue
+				}
+				u.updateKeyboardState(buf[0])
+			}
 		}
+	}()
+}
+
+func (u *UsbGadget) openKeyboardHidFile() error {
+	if u.keyboardHidFile != nil {
+		return nil
+	}
+
+	var err error
+	u.keyboardHidFile, err = os.OpenFile("/dev/hidg0", os.O_RDWR, 0666)
+	if err != nil {
+		return fmt.Errorf("failed to open hidg0: %w", err)
+	}
+
+	if u.keyboardStateCancel != nil {
+		u.keyboardStateCancel()
+	}
+
+	u.keyboardStateCtx, u.keyboardStateCancel = context.WithCancel(context.Background())
+	u.listenKeyboardEvents()
+
+	return nil
+}
+
+func (u *UsbGadget) OpenKeyboardHidFile() error {
+	return u.openKeyboardHidFile()
+}
+
+func (u *UsbGadget) keyboardWriteHidFile(data []byte) error {
+	if err := u.openKeyboardHidFile(); err != nil {
+		return err
 	}
 
 	_, err := u.keyboardHidFile.Write(data)

internal/usbgadget/usbgadget.go

@@ -3,6 +3,7 @@
 package usbgadget
 
 import (
+	"context"
 	"os"
 	"path"
 	"sync"
@@ -59,6 +60,11 @@ type UsbGadget struct {
 	relMouseHidFile *os.File
 	relMouseLock    sync.Mutex
 
+	keyboardState       KeyboardState
+	keyboardStateLock   sync.Mutex
+	keyboardStateCtx    context.Context
+	keyboardStateCancel context.CancelFunc
+
 	enabledDevices Devices
 
 	strictMode bool // only intended for testing for now
@@ -70,6 +76,8 @@ type UsbGadget struct {
 	tx     *UsbGadgetTransaction
 	txLock sync.Mutex
 
+	onKeyboardStateChange *func(state KeyboardState)
+
 	log *zerolog.Logger
 }
 
@@ -96,20 +104,25 @@ func newUsbGadget(name string, configMap map[string]gadgetConfigItem, enabledDev
 		config = &Config{isEmpty: true}
 	}
 
+	keyboardCtx, keyboardCancel := context.WithCancel(context.Background())
+
 	g := &UsbGadget{
-		name:           name,
+		name:                name,
-		kvmGadgetPath:  path.Join(gadgetPath, name),
+		kvmGadgetPath:       path.Join(gadgetPath, name),
-		configC1Path:   path.Join(gadgetPath, name, "configs/c.1"),
+		configC1Path:        path.Join(gadgetPath, name, "configs/c.1"),
-		configMap:      configMap,
+		configMap:           configMap,
-		customConfig:   *config,
+		customConfig:        *config,
-		configLock:     sync.Mutex{},
+		configLock:          sync.Mutex{},
-		keyboardLock:   sync.Mutex{},
+		keyboardLock:        sync.Mutex{},
-		absMouseLock:   sync.Mutex{},
+		absMouseLock:        sync.Mutex{},
-		relMouseLock:   sync.Mutex{},
+		relMouseLock:        sync.Mutex{},
-		txLock:         sync.Mutex{},
+		txLock:              sync.Mutex{},
-		enabledDevices: *enabledDevices,
+		keyboardStateCtx:    keyboardCtx,
-		lastUserInput:  time.Now(),
+		keyboardStateCancel: keyboardCancel,
-		log:            logger,
+		keyboardState:       KeyboardState{},
+		enabledDevices:      *enabledDevices,
+		lastUserInput:       time.Now(),
+		log:                 logger,
 
 		strictMode: config.strictMode,
 

jsonrpc.go

@@ -1017,6 +1017,7 @@ var rpcHandlers = map[string]RPCHandler{
 	"setNetworkSettings":     {Func: rpcSetNetworkSettings, Params: []string{"settings"}},
 	"renewDHCPLease":         {Func: rpcRenewDHCPLease},
 	"keyboardReport":         {Func: rpcKeyboardReport, Params: []string{"modifier", "keys"}},
+	"getKeyboardLedState":    {Func: rpcGetKeyboardLedState},
 	"absMouseReport":         {Func: rpcAbsMouseReport, Params: []string{"x", "y", "buttons"}},
 	"relMouseReport":         {Func: rpcRelMouseReport, Params: []string{"dx", "dy", "buttons"}},
 	"wheelReport":            {Func: rpcWheelReport, Params: []string{"wheelY"}},

ui/src/components/InfoBar.tsx

@@ -36,9 +36,7 @@ export default function InfoBar() {
       console.log(`Error on DataChannel '${rpcDataChannel.label}': ${e}`);
   }, [rpcDataChannel]);
 
-  const isCapsLockActive = useHidStore(state => state.isCapsLockActive);
+  const keyboardLedState = useHidStore(state => state.keyboardLedState);
-  const isNumLockActive = useHidStore(state => state.isNumLockActive);
-  const isScrollLockActive = useHidStore(state => state.isScrollLockActive);
 
   const isTurnServerInUse = useRTCStore(state => state.isTurnServerInUse);
 
@@ -121,7 +119,7 @@ export default function InfoBar() {
           <div
             className={cx(
               "shrink-0 p-1 px-1.5 text-xs",
-              isCapsLockActive
+              keyboardLedState?.caps_lock
                 ? "text-black dark:text-white"
                 : "text-slate-800/20 dark:text-slate-300/20",
             )}
@@ -131,7 +129,7 @@ export default function InfoBar() {
           <div
             className={cx(
               "shrink-0 p-1 px-1.5 text-xs",
-              isNumLockActive
+              keyboardLedState?.num_lock
                 ? "text-black dark:text-white"
                 : "text-slate-800/20 dark:text-slate-300/20",
             )}
@@ -141,13 +139,23 @@ export default function InfoBar() {
           <div
             className={cx(
               "shrink-0 p-1 px-1.5 text-xs",
-              isScrollLockActive
+              keyboardLedState?.scroll_lock
                 ? "text-black dark:text-white"
                 : "text-slate-800/20 dark:text-slate-300/20",
             )}
           >
             Scroll Lock
           </div>
+          {keyboardLedState?.compose ? (
+            <div className="shrink-0 p-1 px-1.5 text-xs">
+              Compose
+            </div>
+          ) : null}
+          {keyboardLedState?.kana ? (
+            <div className="shrink-0 p-1 px-1.5 text-xs">
+              Kana
+            </div>
+          ) : null}
         </div>
       </div>
     </div>

ui/src/components/VirtualKeyboard.tsx

@@ -1,7 +1,8 @@
+import { useShallow } from "zustand/react/shallow";
+import { ChevronDownIcon } from "@heroicons/react/16/solid";
+import { AnimatePresence, motion } from "framer-motion";
 import { useCallback, useEffect, useRef, useState } from "react";
 import Keyboard from "react-simple-keyboard";
-import { ChevronDownIcon } from "@heroicons/react/16/solid";
-import { motion, AnimatePresence } from "framer-motion";
 
 import Card from "@components/Card";
 // eslint-disable-next-line import/order
@@ -9,12 +10,12 @@ import { Button } from "@components/Button";
 
 import "react-simple-keyboard/build/css/index.css";
 
-import { useHidStore, useUiStore } from "@/hooks/stores";
-import { cx } from "@/cva.config";
-import { keys, modifiers, keyDisplayMap } from "@/keyboardMappings";
-import useKeyboard from "@/hooks/useKeyboard";
-import DetachIconRaw from "@/assets/detach-icon.svg";
 import AttachIconRaw from "@/assets/attach-icon.svg";
+import DetachIconRaw from "@/assets/detach-icon.svg";
+import { cx } from "@/cva.config";
+import { useHidStore, useUiStore } from "@/hooks/stores";
+import useKeyboard from "@/hooks/useKeyboard";
+import { keyDisplayMap, keys, modifiers } from "@/keyboardMappings";
 
 export const DetachIcon = ({ className }: { className?: string }) => {
   return <img src={DetachIconRaw} alt="Detach Icon" className={className} />;
@@ -40,8 +41,8 @@ function KeyboardWrapper() {
   const [isDragging, setIsDragging] = useState(false);
   const [position, setPosition] = useState({ x: 0, y: 0 });
   const [newPosition, setNewPosition] = useState({ x: 0, y: 0 });
-  const isCapsLockActive = useHidStore(state => state.isCapsLockActive);
+
-  const setIsCapsLockActive = useHidStore(state => state.setIsCapsLockActive);
+  const isCapsLockActive = useHidStore(useShallow(state => state.keyboardLedState?.caps_lock));
 
   const startDrag = useCallback((e: MouseEvent | TouchEvent) => {
     if (!keyboardRef.current) return;
@@ -157,17 +158,11 @@ function KeyboardWrapper() {
         toggleLayout();
 
         if (isCapsLockActive) {
-          setIsCapsLockActive(false);
           sendKeyboardEvent([keys["CapsLock"]], []);
           return;
         }
       }
 
-      // Handle caps lock state change
-      if (isKeyCaps) {
-        setIsCapsLockActive(!isCapsLockActive);
-      }
-
       // Collect new active keys and modifiers
       const newKeys = keys[cleanKey] ? [keys[cleanKey]] : [];
       const newModifiers =
@@ -183,7 +178,7 @@ function KeyboardWrapper() {
 
       setTimeout(resetKeyboardState, 100);
     },
-    [isCapsLockActive, sendKeyboardEvent, resetKeyboardState, setIsCapsLockActive],
+    [isCapsLockActive, sendKeyboardEvent, resetKeyboardState],
   );
 
   const virtualKeyboard = useHidStore(state => state.isVirtualKeyboardEnabled);

ui/src/components/WebRTCVideo.tsx

@@ -55,10 +55,6 @@ export default function WebRTCVideo() {
   const hdmiError = ["no_lock", "no_signal", "out_of_range"].includes(hdmiState);
   const isVideoLoading = !isPlaying;
 
-  // Keyboard related states
-  const { setIsNumLockActive, setIsCapsLockActive, setIsScrollLockActive } =
-    useHidStore();
-
   // Misc states and hooks
   const disableVideoFocusTrap = useUiStore(state => state.disableVideoFocusTrap);
   const [send] = useJsonRpc();
@@ -355,10 +351,6 @@ export default function WebRTCVideo() {
 
       // console.log(document.activeElement);
 
-      setIsNumLockActive(e.getModifierState("NumLock"));
-      setIsCapsLockActive(e.getModifierState("CapsLock"));
-      setIsScrollLockActive(e.getModifierState("ScrollLock"));
-
       if (code == "IntlBackslash" && ["`", "~"].includes(key)) {
         code = "Backquote";
       } else if (code == "Backquote" && ["§", "±"].includes(key)) {
@@ -388,9 +380,6 @@ export default function WebRTCVideo() {
       sendKeyboardEvent([...new Set(newKeys)], [...new Set(newModifiers)]);
     },
     [
-      setIsNumLockActive,
-      setIsCapsLockActive,
-      setIsScrollLockActive,
       handleModifierKeys,
       sendKeyboardEvent,
     ],
@@ -401,10 +390,6 @@ export default function WebRTCVideo() {
       e.preventDefault();
       const prev = useHidStore.getState();
 
-      setIsNumLockActive(e.getModifierState("NumLock"));
-      setIsCapsLockActive(e.getModifierState("CapsLock"));
-      setIsScrollLockActive(e.getModifierState("ScrollLock"));
-
       // Filtering out the key that was just released (keys[e.code])
       const newKeys = prev.activeKeys.filter(k => k !== keys[e.code]).filter(Boolean);
 
@@ -417,9 +402,6 @@ export default function WebRTCVideo() {
       sendKeyboardEvent([...new Set(newKeys)], [...new Set(newModifiers)]);
     },
     [
-      setIsNumLockActive,
-      setIsCapsLockActive,
-      setIsScrollLockActive,
       handleModifierKeys,
       sendKeyboardEvent,
     ],

ui/src/hooks/stores.ts

@@ -405,6 +405,14 @@ export const useMountMediaStore = create<MountMediaState>(set => ({
   setErrorMessage: message => set({ errorMessage: message }),
 }));
 
+export interface KeyboardLedState {
+  num_lock: boolean;
+  caps_lock: boolean;
+  scroll_lock: boolean;
+  compose: boolean;
+  kana: boolean;
+}
+
 export interface HidState {
   activeKeys: number[];
   activeModifiers: number[];
@@ -423,18 +431,12 @@ export interface HidState {
   altGrCtrlTime: number; // _altGrCtrlTime
   setAltGrCtrlTime: (time: number) => void;
 
-  isNumLockActive: boolean;
+  keyboardLedState?: KeyboardLedState;
-  setIsNumLockActive: (enabled: boolean) => void;
+  setKeyboardLedState: (state: KeyboardLedState) => void;
-
-  isScrollLockActive: boolean;
-  setIsScrollLockActive: (enabled: boolean) => void;
 
   isVirtualKeyboardEnabled: boolean;
   setVirtualKeyboardEnabled: (enabled: boolean) => void;
 
-  isCapsLockActive: boolean;
-  setIsCapsLockActive: (enabled: boolean) => void;
-
   isPasteModeEnabled: boolean;
   setPasteModeEnabled: (enabled: boolean) => void;
 
@@ -458,18 +460,11 @@ export const useHidStore = create<HidState>(set => ({
   altGrCtrlTime: 0,
   setAltGrCtrlTime: time => set({ altGrCtrlTime: time }),
 
-  isNumLockActive: false,
+  setKeyboardLedState: ledState => set({ keyboardLedState: ledState }),
-  setIsNumLockActive: enabled => set({ isNumLockActive: enabled }),
-
-  isScrollLockActive: false,
-  setIsScrollLockActive: enabled => set({ isScrollLockActive: enabled }),
 
   isVirtualKeyboardEnabled: false,
   setVirtualKeyboardEnabled: enabled => set({ isVirtualKeyboardEnabled: enabled }),
 
-  isCapsLockActive: false,
-  setIsCapsLockActive: enabled => set({ isCapsLockActive: enabled }),
-
   isPasteModeEnabled: false,
   setPasteModeEnabled: enabled => set({ isPasteModeEnabled: enabled }),
 

ui/src/routes/devices.$id.tsx

@@ -19,6 +19,7 @@ import useWebSocket from "react-use-websocket";
 import { cx } from "@/cva.config";
 import {
   HidState,
+  KeyboardLedState,
   NetworkState,
   UpdateState,
   useDeviceStore,
@@ -586,6 +587,9 @@ export default function KvmIdRoute() {
   const setUsbState = useHidStore(state => state.setUsbState);
   const setHdmiState = useVideoStore(state => state.setHdmiState);
 
+  const keyboardLedState = useHidStore(state => state.keyboardLedState);
+  const setKeyboardLedState = useHidStore(state => state.setKeyboardLedState);
+
   const [hasUpdated, setHasUpdated] = useState(false);
   const { navigateTo } = useDeviceUiNavigation();
 
@@ -607,6 +611,12 @@ export default function KvmIdRoute() {
       setNetworkState(resp.params as NetworkState);
     }
 
+    if (resp.method === "keyboardLedState") {
+      const ledState = resp.params as KeyboardLedState;
+      console.log("Setting keyboard led state", ledState);
+      setKeyboardLedState(ledState);
+    }
+
     if (resp.method === "otaState") {
       const otaState = resp.params as UpdateState["otaState"];
       setOtaState(otaState);
@@ -643,6 +653,18 @@ export default function KvmIdRoute() {
     });
   }, [rpcDataChannel?.readyState, send, setHdmiState]);
 
+  // request keyboard led state from the device
+  useEffect(() => {
+    if (rpcDataChannel?.readyState !== "open") return;
+    if (keyboardLedState !== undefined) return;
+    console.log("Requesting keyboard led state");
+    send("getKeyboardLedState", {}, resp => {
+      if ("error" in resp) return;
+      console.log("Keyboard led state", resp.result);
+      setKeyboardLedState(resp.result as KeyboardLedState);
+    });
+  }, [rpcDataChannel?.readyState, send, setKeyboardLedState, keyboardLedState]);
+
   // When the update is successful, we need to refresh the client javascript and show a success modal
   useEffect(() => {
     if (queryParams.get("updateSuccess")) {

usb.go

@@ -24,6 +24,17 @@ func initUsbGadget() {
 			time.Sleep(500 * time.Millisecond)
 		}
 	}()
+
+	gadget.SetOnKeyboardStateChange(func(state usbgadget.KeyboardState) {
+		if currentSession != nil {
+			writeJSONRPCEvent("keyboardLedState", state, currentSession)
+		}
+	})
+
+	// open the keyboard hid file to listen for keyboard events
+	if err := gadget.OpenKeyboardHidFile(); err != nil {
+		usbLogger.Error().Err(err).Msg("failed to open keyboard hid file")
+	}
 }
 
 func rpcKeyboardReport(modifier uint8, keys []uint8) error {
@@ -42,6 +53,10 @@ func rpcWheelReport(wheelY int8) error {
 	return gadget.AbsMouseWheelReport(wheelY)
 }
 
+func rpcGetKeyboardLedState() (state usbgadget.KeyboardState) {
+	return gadget.GetKeyboardState()
+}
+
 var usbState = "unknown"
 
 func rpcGetUSBState() (state string) {