From 0c5c69f2d39681c92741ec8551c4dbf0f82e2709 Mon Sep 17 00:00:00 2001 From: Aveline <352441+ym@users.noreply.github.com> Date: Fri, 23 May 2025 00:12:18 +0200 Subject: [PATCH] feat: sync keyboard led status (#502) --- internal/usbgadget/hid_keyboard.go | 142 ++++++++++++++++++++++++-- internal/usbgadget/usbgadget.go | 39 ++++--- jsonrpc.go | 1 + ui/src/components/InfoBar.tsx | 20 ++-- ui/src/components/VirtualKeyboard.tsx | 27 ++--- ui/src/components/WebRTCVideo.tsx | 18 ---- ui/src/hooks/stores.ts | 27 ++--- ui/src/routes/devices.$id.tsx | 22 ++++ usb.go | 15 +++ 9 files changed, 236 insertions(+), 75 deletions(-) diff --git a/internal/usbgadget/hid_keyboard.go b/internal/usbgadget/hid_keyboard.go index de007e4..12b0de9 100644 --- a/internal/usbgadget/hid_keyboard.go +++ b/internal/usbgadget/hid_keyboard.go @@ -1,8 +1,11 @@ package usbgadget import ( + "context" "fmt" "os" + "reflect" + "time" ) var keyboardConfig = gadgetConfigItem{ @@ -36,6 +39,7 @@ var keyboardReportDesc = []byte{ 0x81, 0x03, /* INPUT (Cnst,Var,Abs) */ 0x95, 0x05, /* REPORT_COUNT (5) */ 0x75, 0x01, /* REPORT_SIZE (1) */ + 0x05, 0x08, /* USAGE_PAGE (LEDs) */ 0x19, 0x01, /* USAGE_MINIMUM (Num Lock) */ 0x29, 0x05, /* USAGE_MAXIMUM (Kana) */ @@ -54,13 +58,139 @@ var keyboardReportDesc = []byte{ 0xc0, /* END_COLLECTION */ } -func (u *UsbGadget) keyboardWriteHidFile(data []byte) error { - if u.keyboardHidFile == nil { - var err error - u.keyboardHidFile, err = os.OpenFile("/dev/hidg0", os.O_RDWR, 0666) - if err != nil { - return fmt.Errorf("failed to open hidg0: %w", err) +const ( + hidReadBufferSize = 8 + // https://www.usb.org/sites/default/files/documents/hid1_11.pdf + // https://www.usb.org/sites/default/files/hut1_2.pdf + KeyboardLedMaskNumLock = 1 << 0 + KeyboardLedMaskCapsLock = 1 << 1 + KeyboardLedMaskScrollLock = 1 << 2 + KeyboardLedMaskCompose = 1 << 3 + KeyboardLedMaskKana = 1 << 4 + ValidKeyboardLedMasks = KeyboardLedMaskNumLock | KeyboardLedMaskCapsLock | KeyboardLedMaskScrollLock | KeyboardLedMaskCompose | KeyboardLedMaskKana +) + +// Synchronization between LED states and CAPS LOCK, NUM LOCK, SCROLL LOCK, +// COMPOSE, and KANA events is maintained by the host and NOT the keyboard. If +// using the keyboard descriptor in Appendix B, LED states are set by sending a +// 5-bit absolute report to the keyboard via a Set_Report(Output) request. +type KeyboardState struct { + NumLock bool `json:"num_lock"` + CapsLock bool `json:"caps_lock"` + ScrollLock bool `json:"scroll_lock"` + Compose bool `json:"compose"` + Kana bool `json:"kana"` +} + +func getKeyboardState(b byte) KeyboardState { + // should we check if it's the correct usage page? + return KeyboardState{ + NumLock: b&KeyboardLedMaskNumLock != 0, + CapsLock: b&KeyboardLedMaskCapsLock != 0, + ScrollLock: b&KeyboardLedMaskScrollLock != 0, + Compose: b&KeyboardLedMaskCompose != 0, + Kana: b&KeyboardLedMaskKana != 0, + } +} + +func (u *UsbGadget) updateKeyboardState(b byte) { + u.keyboardStateLock.Lock() + defer u.keyboardStateLock.Unlock() + + if b&^ValidKeyboardLedMasks != 0 { + u.log.Trace().Uint8("b", b).Msg("contains invalid bits, ignoring") + return + } + + newState := getKeyboardState(b) + if reflect.DeepEqual(u.keyboardState, newState) { + return + } + u.log.Info().Interface("old", u.keyboardState).Interface("new", newState).Msg("keyboardState updated") + u.keyboardState = newState + + if u.onKeyboardStateChange != nil { + (*u.onKeyboardStateChange)(newState) + } +} + +func (u *UsbGadget) SetOnKeyboardStateChange(f func(state KeyboardState)) { + u.onKeyboardStateChange = &f +} + +func (u *UsbGadget) GetKeyboardState() KeyboardState { + u.keyboardStateLock.Lock() + defer u.keyboardStateLock.Unlock() + + return u.keyboardState +} + +func (u *UsbGadget) listenKeyboardEvents() { + var path string + if u.keyboardHidFile != nil { + path = u.keyboardHidFile.Name() + } + l := u.log.With().Str("listener", "keyboardEvents").Str("path", path).Logger() + l.Trace().Msg("starting") + + go func() { + buf := make([]byte, hidReadBufferSize) + for { + select { + case <-u.keyboardStateCtx.Done(): + l.Info().Msg("context done") + return + default: + l.Trace().Msg("reading from keyboard") + if u.keyboardHidFile == nil { + l.Error().Msg("keyboardHidFile is nil") + time.Sleep(time.Second) + continue + } + n, err := u.keyboardHidFile.Read(buf) + if err != nil { + l.Error().Err(err).Msg("failed to read") + continue + } + l.Trace().Int("n", n).Bytes("buf", buf).Msg("got data from keyboard") + if n != 1 { + l.Trace().Int("n", n).Msg("expected 1 byte, got") + continue + } + u.updateKeyboardState(buf[0]) + } } + }() +} + +func (u *UsbGadget) openKeyboardHidFile() error { + if u.keyboardHidFile != nil { + return nil + } + + var err error + u.keyboardHidFile, err = os.OpenFile("/dev/hidg0", os.O_RDWR, 0666) + if err != nil { + return fmt.Errorf("failed to open hidg0: %w", err) + } + + if u.keyboardStateCancel != nil { + u.keyboardStateCancel() + } + + u.keyboardStateCtx, u.keyboardStateCancel = context.WithCancel(context.Background()) + u.listenKeyboardEvents() + + return nil +} + +func (u *UsbGadget) OpenKeyboardHidFile() error { + return u.openKeyboardHidFile() +} + +func (u *UsbGadget) keyboardWriteHidFile(data []byte) error { + if err := u.openKeyboardHidFile(); err != nil { + return err } _, err := u.keyboardHidFile.Write(data) diff --git a/internal/usbgadget/usbgadget.go b/internal/usbgadget/usbgadget.go index f8b2b3e..2eab822 100644 --- a/internal/usbgadget/usbgadget.go +++ b/internal/usbgadget/usbgadget.go @@ -3,6 +3,7 @@ package usbgadget import ( + "context" "os" "path" "sync" @@ -59,6 +60,11 @@ type UsbGadget struct { relMouseHidFile *os.File relMouseLock sync.Mutex + keyboardState KeyboardState + keyboardStateLock sync.Mutex + keyboardStateCtx context.Context + keyboardStateCancel context.CancelFunc + enabledDevices Devices strictMode bool // only intended for testing for now @@ -70,6 +76,8 @@ type UsbGadget struct { tx *UsbGadgetTransaction txLock sync.Mutex + onKeyboardStateChange *func(state KeyboardState) + log *zerolog.Logger } @@ -96,20 +104,25 @@ func newUsbGadget(name string, configMap map[string]gadgetConfigItem, enabledDev config = &Config{isEmpty: true} } + keyboardCtx, keyboardCancel := context.WithCancel(context.Background()) + g := &UsbGadget{ - name: name, - kvmGadgetPath: path.Join(gadgetPath, name), - configC1Path: path.Join(gadgetPath, name, "configs/c.1"), - configMap: configMap, - customConfig: *config, - configLock: sync.Mutex{}, - keyboardLock: sync.Mutex{}, - absMouseLock: sync.Mutex{}, - relMouseLock: sync.Mutex{}, - txLock: sync.Mutex{}, - enabledDevices: *enabledDevices, - lastUserInput: time.Now(), - log: logger, + name: name, + kvmGadgetPath: path.Join(gadgetPath, name), + configC1Path: path.Join(gadgetPath, name, "configs/c.1"), + configMap: configMap, + customConfig: *config, + configLock: sync.Mutex{}, + keyboardLock: sync.Mutex{}, + absMouseLock: sync.Mutex{}, + relMouseLock: sync.Mutex{}, + txLock: sync.Mutex{}, + keyboardStateCtx: keyboardCtx, + keyboardStateCancel: keyboardCancel, + keyboardState: KeyboardState{}, + enabledDevices: *enabledDevices, + lastUserInput: time.Now(), + log: logger, strictMode: config.strictMode, diff --git a/jsonrpc.go b/jsonrpc.go index db25c6d..a32cab2 100644 --- a/jsonrpc.go +++ b/jsonrpc.go @@ -1017,6 +1017,7 @@ var rpcHandlers = map[string]RPCHandler{ "setNetworkSettings": {Func: rpcSetNetworkSettings, Params: []string{"settings"}}, "renewDHCPLease": {Func: rpcRenewDHCPLease}, "keyboardReport": {Func: rpcKeyboardReport, Params: []string{"modifier", "keys"}}, + "getKeyboardLedState": {Func: rpcGetKeyboardLedState}, "absMouseReport": {Func: rpcAbsMouseReport, Params: []string{"x", "y", "buttons"}}, "relMouseReport": {Func: rpcRelMouseReport, Params: []string{"dx", "dy", "buttons"}}, "wheelReport": {Func: rpcWheelReport, Params: []string{"wheelY"}}, diff --git a/ui/src/components/InfoBar.tsx b/ui/src/components/InfoBar.tsx index aa00da7..4ee4149 100644 --- a/ui/src/components/InfoBar.tsx +++ b/ui/src/components/InfoBar.tsx @@ -36,9 +36,7 @@ export default function InfoBar() { console.log(`Error on DataChannel '${rpcDataChannel.label}': ${e}`); }, [rpcDataChannel]); - const isCapsLockActive = useHidStore(state => state.isCapsLockActive); - const isNumLockActive = useHidStore(state => state.isNumLockActive); - const isScrollLockActive = useHidStore(state => state.isScrollLockActive); + const keyboardLedState = useHidStore(state => state.keyboardLedState); const isTurnServerInUse = useRTCStore(state => state.isTurnServerInUse); @@ -121,7 +119,7 @@ export default function InfoBar() {
Scroll Lock
+ {keyboardLedState?.compose ? ( +
+ Compose +
+ ) : null} + {keyboardLedState?.kana ? ( +
+ Kana +
+ ) : null} diff --git a/ui/src/components/VirtualKeyboard.tsx b/ui/src/components/VirtualKeyboard.tsx index 03cb331..9566bca 100644 --- a/ui/src/components/VirtualKeyboard.tsx +++ b/ui/src/components/VirtualKeyboard.tsx @@ -1,7 +1,8 @@ +import { useShallow } from "zustand/react/shallow"; +import { ChevronDownIcon } from "@heroicons/react/16/solid"; +import { AnimatePresence, motion } from "framer-motion"; import { useCallback, useEffect, useRef, useState } from "react"; import Keyboard from "react-simple-keyboard"; -import { ChevronDownIcon } from "@heroicons/react/16/solid"; -import { motion, AnimatePresence } from "framer-motion"; import Card from "@components/Card"; // eslint-disable-next-line import/order @@ -9,12 +10,12 @@ import { Button } from "@components/Button"; import "react-simple-keyboard/build/css/index.css"; -import { useHidStore, useUiStore } from "@/hooks/stores"; -import { cx } from "@/cva.config"; -import { keys, modifiers, keyDisplayMap } from "@/keyboardMappings"; -import useKeyboard from "@/hooks/useKeyboard"; -import DetachIconRaw from "@/assets/detach-icon.svg"; import AttachIconRaw from "@/assets/attach-icon.svg"; +import DetachIconRaw from "@/assets/detach-icon.svg"; +import { cx } from "@/cva.config"; +import { useHidStore, useUiStore } from "@/hooks/stores"; +import useKeyboard from "@/hooks/useKeyboard"; +import { keyDisplayMap, keys, modifiers } from "@/keyboardMappings"; export const DetachIcon = ({ className }: { className?: string }) => { return Detach Icon; @@ -40,8 +41,8 @@ function KeyboardWrapper() { const [isDragging, setIsDragging] = useState(false); const [position, setPosition] = useState({ x: 0, y: 0 }); const [newPosition, setNewPosition] = useState({ x: 0, y: 0 }); - const isCapsLockActive = useHidStore(state => state.isCapsLockActive); - const setIsCapsLockActive = useHidStore(state => state.setIsCapsLockActive); + + const isCapsLockActive = useHidStore(useShallow(state => state.keyboardLedState?.caps_lock)); const startDrag = useCallback((e: MouseEvent | TouchEvent) => { if (!keyboardRef.current) return; @@ -157,17 +158,11 @@ function KeyboardWrapper() { toggleLayout(); if (isCapsLockActive) { - setIsCapsLockActive(false); sendKeyboardEvent([keys["CapsLock"]], []); return; } } - // Handle caps lock state change - if (isKeyCaps) { - setIsCapsLockActive(!isCapsLockActive); - } - // Collect new active keys and modifiers const newKeys = keys[cleanKey] ? [keys[cleanKey]] : []; const newModifiers = @@ -183,7 +178,7 @@ function KeyboardWrapper() { setTimeout(resetKeyboardState, 100); }, - [isCapsLockActive, sendKeyboardEvent, resetKeyboardState, setIsCapsLockActive], + [isCapsLockActive, sendKeyboardEvent, resetKeyboardState], ); const virtualKeyboard = useHidStore(state => state.isVirtualKeyboardEnabled); diff --git a/ui/src/components/WebRTCVideo.tsx b/ui/src/components/WebRTCVideo.tsx index 4fd4290..87f2d4e 100644 --- a/ui/src/components/WebRTCVideo.tsx +++ b/ui/src/components/WebRTCVideo.tsx @@ -55,10 +55,6 @@ export default function WebRTCVideo() { const hdmiError = ["no_lock", "no_signal", "out_of_range"].includes(hdmiState); const isVideoLoading = !isPlaying; - // Keyboard related states - const { setIsNumLockActive, setIsCapsLockActive, setIsScrollLockActive } = - useHidStore(); - // Misc states and hooks const disableVideoFocusTrap = useUiStore(state => state.disableVideoFocusTrap); const [send] = useJsonRpc(); @@ -355,10 +351,6 @@ export default function WebRTCVideo() { // console.log(document.activeElement); - setIsNumLockActive(e.getModifierState("NumLock")); - setIsCapsLockActive(e.getModifierState("CapsLock")); - setIsScrollLockActive(e.getModifierState("ScrollLock")); - if (code == "IntlBackslash" && ["`", "~"].includes(key)) { code = "Backquote"; } else if (code == "Backquote" && ["§", "±"].includes(key)) { @@ -388,9 +380,6 @@ export default function WebRTCVideo() { sendKeyboardEvent([...new Set(newKeys)], [...new Set(newModifiers)]); }, [ - setIsNumLockActive, - setIsCapsLockActive, - setIsScrollLockActive, handleModifierKeys, sendKeyboardEvent, ], @@ -401,10 +390,6 @@ export default function WebRTCVideo() { e.preventDefault(); const prev = useHidStore.getState(); - setIsNumLockActive(e.getModifierState("NumLock")); - setIsCapsLockActive(e.getModifierState("CapsLock")); - setIsScrollLockActive(e.getModifierState("ScrollLock")); - // Filtering out the key that was just released (keys[e.code]) const newKeys = prev.activeKeys.filter(k => k !== keys[e.code]).filter(Boolean); @@ -417,9 +402,6 @@ export default function WebRTCVideo() { sendKeyboardEvent([...new Set(newKeys)], [...new Set(newModifiers)]); }, [ - setIsNumLockActive, - setIsCapsLockActive, - setIsScrollLockActive, handleModifierKeys, sendKeyboardEvent, ], diff --git a/ui/src/hooks/stores.ts b/ui/src/hooks/stores.ts index 01d1257..1754a87 100644 --- a/ui/src/hooks/stores.ts +++ b/ui/src/hooks/stores.ts @@ -405,6 +405,14 @@ export const useMountMediaStore = create(set => ({ setErrorMessage: message => set({ errorMessage: message }), })); +export interface KeyboardLedState { + num_lock: boolean; + caps_lock: boolean; + scroll_lock: boolean; + compose: boolean; + kana: boolean; +} + export interface HidState { activeKeys: number[]; activeModifiers: number[]; @@ -423,18 +431,12 @@ export interface HidState { altGrCtrlTime: number; // _altGrCtrlTime setAltGrCtrlTime: (time: number) => void; - isNumLockActive: boolean; - setIsNumLockActive: (enabled: boolean) => void; - - isScrollLockActive: boolean; - setIsScrollLockActive: (enabled: boolean) => void; + keyboardLedState?: KeyboardLedState; + setKeyboardLedState: (state: KeyboardLedState) => void; isVirtualKeyboardEnabled: boolean; setVirtualKeyboardEnabled: (enabled: boolean) => void; - isCapsLockActive: boolean; - setIsCapsLockActive: (enabled: boolean) => void; - isPasteModeEnabled: boolean; setPasteModeEnabled: (enabled: boolean) => void; @@ -458,18 +460,11 @@ export const useHidStore = create(set => ({ altGrCtrlTime: 0, setAltGrCtrlTime: time => set({ altGrCtrlTime: time }), - isNumLockActive: false, - setIsNumLockActive: enabled => set({ isNumLockActive: enabled }), - - isScrollLockActive: false, - setIsScrollLockActive: enabled => set({ isScrollLockActive: enabled }), + setKeyboardLedState: ledState => set({ keyboardLedState: ledState }), isVirtualKeyboardEnabled: false, setVirtualKeyboardEnabled: enabled => set({ isVirtualKeyboardEnabled: enabled }), - isCapsLockActive: false, - setIsCapsLockActive: enabled => set({ isCapsLockActive: enabled }), - isPasteModeEnabled: false, setPasteModeEnabled: enabled => set({ isPasteModeEnabled: enabled }), diff --git a/ui/src/routes/devices.$id.tsx b/ui/src/routes/devices.$id.tsx index d35915b..a6be368 100644 --- a/ui/src/routes/devices.$id.tsx +++ b/ui/src/routes/devices.$id.tsx @@ -19,6 +19,7 @@ import useWebSocket from "react-use-websocket"; import { cx } from "@/cva.config"; import { HidState, + KeyboardLedState, NetworkState, UpdateState, useDeviceStore, @@ -586,6 +587,9 @@ export default function KvmIdRoute() { const setUsbState = useHidStore(state => state.setUsbState); const setHdmiState = useVideoStore(state => state.setHdmiState); + const keyboardLedState = useHidStore(state => state.keyboardLedState); + const setKeyboardLedState = useHidStore(state => state.setKeyboardLedState); + const [hasUpdated, setHasUpdated] = useState(false); const { navigateTo } = useDeviceUiNavigation(); @@ -607,6 +611,12 @@ export default function KvmIdRoute() { setNetworkState(resp.params as NetworkState); } + if (resp.method === "keyboardLedState") { + const ledState = resp.params as KeyboardLedState; + console.log("Setting keyboard led state", ledState); + setKeyboardLedState(ledState); + } + if (resp.method === "otaState") { const otaState = resp.params as UpdateState["otaState"]; setOtaState(otaState); @@ -643,6 +653,18 @@ export default function KvmIdRoute() { }); }, [rpcDataChannel?.readyState, send, setHdmiState]); + // request keyboard led state from the device + useEffect(() => { + if (rpcDataChannel?.readyState !== "open") return; + if (keyboardLedState !== undefined) return; + console.log("Requesting keyboard led state"); + send("getKeyboardLedState", {}, resp => { + if ("error" in resp) return; + console.log("Keyboard led state", resp.result); + setKeyboardLedState(resp.result as KeyboardLedState); + }); + }, [rpcDataChannel?.readyState, send, setKeyboardLedState, keyboardLedState]); + // When the update is successful, we need to refresh the client javascript and show a success modal useEffect(() => { if (queryParams.get("updateSuccess")) { diff --git a/usb.go b/usb.go index 91674c9..f777f89 100644 --- a/usb.go +++ b/usb.go @@ -24,6 +24,17 @@ func initUsbGadget() { time.Sleep(500 * time.Millisecond) } }() + + gadget.SetOnKeyboardStateChange(func(state usbgadget.KeyboardState) { + if currentSession != nil { + writeJSONRPCEvent("keyboardLedState", state, currentSession) + } + }) + + // open the keyboard hid file to listen for keyboard events + if err := gadget.OpenKeyboardHidFile(); err != nil { + usbLogger.Error().Err(err).Msg("failed to open keyboard hid file") + } } func rpcKeyboardReport(modifier uint8, keys []uint8) error { @@ -42,6 +53,10 @@ func rpcWheelReport(wheelY int8) error { return gadget.AbsMouseWheelReport(wheelY) } +func rpcGetKeyboardLedState() (state usbgadget.KeyboardState) { + return gadget.GetKeyboardState() +} + var usbState = "unknown" func rpcGetUSBState() (state string) {