From 2f0aa18d1d2b7f43729b576b79437b72ffcea901 Mon Sep 17 00:00:00 2001 From: Siyuan Miao Date: Thu, 11 Sep 2025 13:05:20 +0200 Subject: [PATCH 01/20] feat: release keyPress automatically --- internal/usbgadget/hid_keyboard.go | 50 ++++++++++++++++++++++++++++++ internal/usbgadget/usbgadget.go | 35 +++++++++++++++++++++ 2 files changed, 85 insertions(+) diff --git a/internal/usbgadget/hid_keyboard.go b/internal/usbgadget/hid_keyboard.go index fb710c20..8b7080a7 100644 --- a/internal/usbgadget/hid_keyboard.go +++ b/internal/usbgadget/hid_keyboard.go @@ -173,6 +173,50 @@ func (u *UsbGadget) SetOnKeysDownChange(f func(state KeysDownState)) { u.onKeysDownChange = &f } +const autoReleaseKeyboardInterval = time.Millisecond * 300 + +func (u *UsbGadget) scheduleAutoRelease(key byte) { + u.keysAutoReleaseLock.Lock() + defer u.keysAutoReleaseLock.Unlock() + + if u.keysAutoReleaseTimer != nil { + u.keysAutoReleaseTimer.Stop() + } + + u.keysAutoReleaseTimer = time.AfterFunc(autoReleaseKeyboardInterval, func() { + u.performAutoRelease(key) + }) +} + +func (u *UsbGadget) cancelAutoRelease() { + u.keysAutoReleaseLock.Lock() + defer u.keysAutoReleaseLock.Unlock() + + if u.keysAutoReleaseTimer != nil { + u.keysAutoReleaseTimer.Stop() + } +} + +func (u *UsbGadget) performAutoRelease(key byte) { + u.keysAutoReleaseLock.Lock() + defer u.keysAutoReleaseLock.Unlock() + + select { + case <-u.keyboardStateCtx.Done(): + return + default: + } + + _, err := u.KeypressReport(key, false) + if err != nil { + u.log.Warn().Uint8("key", key).Msg("failed to auto-release keyboard key") + } + + u.keysAutoReleaseTimer = nil + + u.log.Trace().Uint8("key", key).Msg("auto release performed") +} + func (u *UsbGadget) listenKeyboardEvents() { var path string if u.keyboardHidFile != nil { @@ -398,5 +442,11 @@ func (u *UsbGadget) KeypressReport(key byte, press bool) (KeysDownState, error) u.log.Warn().Uint8("modifier", modifier).Uints8("keys", keys).Msg("Could not write keypress report to hidg0") } + if press { + u.scheduleAutoRelease(key) + } else { + u.cancelAutoRelease() + } + return u.UpdateKeysDown(modifier, keys), err } diff --git a/internal/usbgadget/usbgadget.go b/internal/usbgadget/usbgadget.go index 3a01a447..7485fc20 100644 --- a/internal/usbgadget/usbgadget.go +++ b/internal/usbgadget/usbgadget.go @@ -68,6 +68,9 @@ type UsbGadget struct { keyboardState byte // keyboard latched state (NumLock, CapsLock, ScrollLock, Compose, Kana) keysDownState KeysDownState // keyboard dynamic state (modifier keys and pressed keys) + keysAutoReleaseLock sync.Mutex + keysAutoReleaseTimer *time.Timer + keyboardStateLock sync.Mutex keyboardStateCtx context.Context keyboardStateCancel context.CancelFunc @@ -149,3 +152,35 @@ func newUsbGadget(name string, configMap map[string]gadgetConfigItem, enabledDev return g } + +// Close cleans up resources used by the USB gadget +func (u *UsbGadget) Close() error { + // Cancel keyboard state context + if u.keyboardStateCancel != nil { + u.keyboardStateCancel() + } + + // Stop auto-release timer + u.keysAutoReleaseLock.Lock() + if u.keysAutoReleaseTimer != nil { + u.keysAutoReleaseTimer.Stop() + u.keysAutoReleaseTimer = nil + } + u.keysAutoReleaseLock.Unlock() + + // Close HID files + if u.keyboardHidFile != nil { + u.keyboardHidFile.Close() + u.keyboardHidFile = nil + } + if u.absMouseHidFile != nil { + u.absMouseHidFile.Close() + u.absMouseHidFile = nil + } + if u.relMouseHidFile != nil { + u.relMouseHidFile.Close() + u.relMouseHidFile = nil + } + + return nil +} From 4b42c7e7e34eaf3499cdd8bb1b8643ca9d31d62f Mon Sep 17 00:00:00 2001 From: Siyuan Miao Date: Thu, 11 Sep 2025 14:10:18 +0200 Subject: [PATCH 02/20] send keepalive when pressing the key --- hidrpc.go | 2 + internal/hidrpc/hidrpc.go | 25 +++++--- internal/usbgadget/hid_keyboard.go | 98 ++++++++++++++++++++++++------ internal/usbgadget/usbgadget.go | 15 ++--- ui/src/hooks/hidRpc.ts | 12 ++++ ui/src/hooks/useHidRpc.ts | 8 +++ ui/src/hooks/useKeyboard.ts | 77 ++++++++++++++++++----- 7 files changed, 189 insertions(+), 48 deletions(-) diff --git a/hidrpc.go b/hidrpc.go index 74fe687f..52022ba6 100644 --- a/hidrpc.go +++ b/hidrpc.go @@ -29,6 +29,8 @@ func handleHidRPCMessage(message hidrpc.Message, session *Session) { session.reportHidRPCKeysDownState(*keysDownState) } rpcErr = err + case hidrpc.TypeKeypressKeepAliveReport: + gadget.DelayAutoRelease() case hidrpc.TypePointerReport: pointerReport, err := message.PointerReport() if err != nil { diff --git a/internal/hidrpc/hidrpc.go b/internal/hidrpc/hidrpc.go index e9c8c24d..7161dc8f 100644 --- a/internal/hidrpc/hidrpc.go +++ b/internal/hidrpc/hidrpc.go @@ -10,14 +10,15 @@ import ( type MessageType byte const ( - TypeHandshake MessageType = 0x01 - TypeKeyboardReport MessageType = 0x02 - TypePointerReport MessageType = 0x03 - TypeWheelReport MessageType = 0x04 - TypeKeypressReport MessageType = 0x05 - TypeMouseReport MessageType = 0x06 - TypeKeyboardLedState MessageType = 0x32 - TypeKeydownState MessageType = 0x33 + TypeHandshake MessageType = 0x01 + TypeKeyboardReport MessageType = 0x02 + TypePointerReport MessageType = 0x03 + TypeWheelReport MessageType = 0x04 + TypeKeypressReport MessageType = 0x05 + TypeKeypressKeepAliveReport MessageType = 0x09 + TypeMouseReport MessageType = 0x06 + TypeKeyboardLedState MessageType = 0x32 + TypeKeydownState MessageType = 0x33 ) const ( @@ -98,3 +99,11 @@ func NewKeydownStateMessage(state usbgadget.KeysDownState) *Message { d: data, } } + +// NewKeypressKeepAliveMessage creates a new keypress keep alive message. +func NewKeypressKeepAliveMessage() *Message { + return &Message{ + t: TypeKeypressKeepAliveReport, + d: []byte{}, + } +} diff --git a/internal/usbgadget/hid_keyboard.go b/internal/usbgadget/hid_keyboard.go index 8b7080a7..f9bd20db 100644 --- a/internal/usbgadget/hid_keyboard.go +++ b/internal/usbgadget/hid_keyboard.go @@ -173,33 +173,47 @@ func (u *UsbGadget) SetOnKeysDownChange(f func(state KeysDownState)) { u.onKeysDownChange = &f } -const autoReleaseKeyboardInterval = time.Millisecond * 300 +const autoReleaseKeyboardInterval = time.Millisecond * 450 func (u *UsbGadget) scheduleAutoRelease(key byte) { - u.keysAutoReleaseLock.Lock() - defer u.keysAutoReleaseLock.Unlock() + u.kbdAutoReleaseLock.Lock() + defer u.kbdAutoReleaseLock.Unlock() - if u.keysAutoReleaseTimer != nil { - u.keysAutoReleaseTimer.Stop() + if u.kbdAutoReleaseTimer != nil { + u.kbdAutoReleaseTimer.Stop() } - u.keysAutoReleaseTimer = time.AfterFunc(autoReleaseKeyboardInterval, func() { + u.kbdAutoReleaseTimer = time.AfterFunc(autoReleaseKeyboardInterval, func() { u.performAutoRelease(key) }) } func (u *UsbGadget) cancelAutoRelease() { - u.keysAutoReleaseLock.Lock() - defer u.keysAutoReleaseLock.Unlock() + u.kbdAutoReleaseLock.Lock() + defer u.kbdAutoReleaseLock.Unlock() - if u.keysAutoReleaseTimer != nil { - u.keysAutoReleaseTimer.Stop() + if u.kbdAutoReleaseTimer != nil { + u.kbdAutoReleaseTimer.Stop() } } +func (u *UsbGadget) DelayAutoRelease() { + u.kbdAutoReleaseLock.Lock() + defer u.kbdAutoReleaseLock.Unlock() + + u.log.Info().Msg("delaying auto-release") + + if u.kbdAutoReleaseTimer == nil { + return + } + + u.log.Info().Msg("resetting auto-release timer") + u.kbdAutoReleaseTimer.Reset(autoReleaseKeyboardInterval) +} + func (u *UsbGadget) performAutoRelease(key byte) { - u.keysAutoReleaseLock.Lock() - defer u.keysAutoReleaseLock.Unlock() + u.kbdAutoReleaseLock.Lock() + defer u.kbdAutoReleaseLock.Unlock() select { case <-u.keyboardStateCtx.Done(): @@ -207,12 +221,12 @@ func (u *UsbGadget) performAutoRelease(key byte) { default: } - _, err := u.KeypressReport(key, false) + _, err := u.keypressReport(key, false, false) if err != nil { u.log.Warn().Uint8("key", key).Msg("failed to auto-release keyboard key") } - u.keysAutoReleaseTimer = nil + u.kbdAutoReleaseTimer = nil u.log.Trace().Uint8("key", key).Msg("auto release performed") } @@ -375,11 +389,21 @@ var KeyCodeToMaskMap = map[byte]byte{ RightSuper: ModifierMaskRightSuper, } -func (u *UsbGadget) KeypressReport(key byte, press bool) (KeysDownState, error) { +func (u *UsbGadget) keypressReport(key byte, press bool, autoRelease bool) (KeysDownState, error) { + ll := u.log.Info().Str("component", "kbd") + ll.Uint8("key", key).Msg("locking keyboardLock") + u.keyboardLock.Lock() - defer u.keyboardLock.Unlock() + defer func() { + u.keyboardLock.Unlock() + ll.Uint8("key", key).Msg("unlocked keyboardLock") + }() + + ll.Uint8("key", key).Msg("resetting user input time") defer u.resetUserInputTime() + ll.Uint8("key", key).Msg("locked keyboardLock") + // IMPORTANT: This code parallels the logic in the kernel's hid-gadget driver // for handling key presses and releases. It ensures that the USB gadget // behaves similarly to a real USB HID keyboard. This logic is paralleled @@ -437,16 +461,54 @@ func (u *UsbGadget) KeypressReport(key byte, press bool) (KeysDownState, error) } } + ll.Uint8("key", key).Msg("checking if auto-release is enabled") + + // if autoRelease { + // u.kbdAutoReleaseLock.Lock() + // u.kbdAutoReleaseLock.Unlock() + // ll.Uint8("key", key).Msg("locking kbdAutoReleaseLock, autoReleasLastKey reset") + + // defer func() { + // ll.Uint8("key", key).Msg("unlocked kbdAutoReleaseLock, autoReleasLastKey reset") + // }() + + // if u.kbdAutoReleaseLastKey == key { + // ll.Uint8("key", key).Msg("key already released by auto-release, skipping") + // u.kbdAutoReleaseLastKey = 0 + + // return u.UpdateKeysDown(modifier, keys), nil + // } + // } + + ll.Uint8("key", key).Msg("writing keypress report to hidg0") + err := u.keyboardWriteHidFile(modifier, keys) if err != nil { u.log.Warn().Uint8("modifier", modifier).Uints8("keys", keys).Msg("Could not write keypress report to hidg0") } if press { - u.scheduleAutoRelease(key) + { + u.kbdAutoReleaseLock.Lock() + u.kbdAutoReleaseLastKey = key + u.kbdAutoReleaseLock.Unlock() + } + + if autoRelease { + ll.Uint8("key", key).Msg("scheduling auto-release") + u.scheduleAutoRelease(key) + } } else { - u.cancelAutoRelease() + if autoRelease { + ll.Uint8("key", key).Msg("canceling auto-release") + u.cancelAutoRelease() + ll.Uint8("key", key).Msg("auto-release canceled") + } } return u.UpdateKeysDown(modifier, keys), err } + +func (u *UsbGadget) KeypressReport(key byte, press bool) (KeysDownState, error) { + return u.keypressReport(key, press, true) +} diff --git a/internal/usbgadget/usbgadget.go b/internal/usbgadget/usbgadget.go index 7485fc20..e491a78c 100644 --- a/internal/usbgadget/usbgadget.go +++ b/internal/usbgadget/usbgadget.go @@ -68,8 +68,9 @@ type UsbGadget struct { keyboardState byte // keyboard latched state (NumLock, CapsLock, ScrollLock, Compose, Kana) keysDownState KeysDownState // keyboard dynamic state (modifier keys and pressed keys) - keysAutoReleaseLock sync.Mutex - keysAutoReleaseTimer *time.Timer + kbdAutoReleaseLock sync.Mutex + kbdAutoReleaseTimer *time.Timer + kbdAutoReleaseLastKey byte keyboardStateLock sync.Mutex keyboardStateCtx context.Context @@ -161,12 +162,12 @@ func (u *UsbGadget) Close() error { } // Stop auto-release timer - u.keysAutoReleaseLock.Lock() - if u.keysAutoReleaseTimer != nil { - u.keysAutoReleaseTimer.Stop() - u.keysAutoReleaseTimer = nil + u.kbdAutoReleaseLock.Lock() + if u.kbdAutoReleaseTimer != nil { + u.kbdAutoReleaseTimer.Stop() + u.kbdAutoReleaseTimer = nil } - u.keysAutoReleaseLock.Unlock() + u.kbdAutoReleaseLock.Unlock() // Close HID files if u.keyboardHidFile != nil { diff --git a/ui/src/hooks/hidRpc.ts b/ui/src/hooks/hidRpc.ts index 20b8a108..d6b4ad96 100644 --- a/ui/src/hooks/hidRpc.ts +++ b/ui/src/hooks/hidRpc.ts @@ -6,6 +6,7 @@ export const HID_RPC_MESSAGE_TYPES = { PointerReport: 0x03, WheelReport: 0x04, KeypressReport: 0x05, + KeypressKeepAliveReport: 0x09, MouseReport: 0x06, KeyboardLedState: 0x32, KeysDownState: 0x33, @@ -278,12 +279,23 @@ export class MouseReportMessage extends RpcMessage { } } +export class KeypressKeepAliveMessage extends RpcMessage { + constructor() { + super(HID_RPC_MESSAGE_TYPES.KeypressKeepAliveReport); + } + + marshal(): Uint8Array { + return new Uint8Array([this.messageType]); + } +} + export const messageRegistry = { [HID_RPC_MESSAGE_TYPES.Handshake]: HandshakeMessage, [HID_RPC_MESSAGE_TYPES.KeysDownState]: KeysDownStateMessage, [HID_RPC_MESSAGE_TYPES.KeyboardLedState]: KeyboardLedStateMessage, [HID_RPC_MESSAGE_TYPES.KeyboardReport]: KeyboardReportMessage, [HID_RPC_MESSAGE_TYPES.KeypressReport]: KeypressReportMessage, + [HID_RPC_MESSAGE_TYPES.KeypressKeepAliveReport]: KeypressKeepAliveMessage, } export const unmarshalHidRpcMessage = (data: Uint8Array): RpcMessage | undefined => { diff --git a/ui/src/hooks/useHidRpc.ts b/ui/src/hooks/useHidRpc.ts index ea0c7112..d5690849 100644 --- a/ui/src/hooks/useHidRpc.ts +++ b/ui/src/hooks/useHidRpc.ts @@ -6,6 +6,7 @@ import { HID_RPC_VERSION, HandshakeMessage, KeyboardReportMessage, + KeypressKeepAliveMessage, KeypressReportMessage, MouseReportMessage, PointerReportMessage, @@ -13,6 +14,8 @@ import { unmarshalHidRpcMessage, } from "./hidRpc"; +const KEEPALIVE_MESSAGE = new KeypressKeepAliveMessage(); + export function useHidRpc(onHidRpcMessage?: (payload: RpcMessage) => void) { const { rpcHidChannel, setRpcHidProtocolVersion, rpcHidProtocolVersion } = useRTCStore(); const rpcHidReady = useMemo(() => { @@ -68,6 +71,10 @@ export function useHidRpc(onHidRpcMessage?: (payload: RpcMessage) => void) { [sendMessage], ); + const reportKeypressKeepAlive = useCallback(() => { + sendMessage(KEEPALIVE_MESSAGE); + }, [sendMessage]); + const sendHandshake = useCallback(() => { if (rpcHidProtocolVersion) return; if (!rpcHidChannel) return; @@ -143,6 +150,7 @@ export function useHidRpc(onHidRpcMessage?: (payload: RpcMessage) => void) { reportKeypressEvent, reportAbsMouseEvent, reportRelMouseEvent, + reportKeypressKeepAlive, rpcHidProtocolVersion, rpcHidReady, rpcHidStatus, diff --git a/ui/src/hooks/useKeyboard.ts b/ui/src/hooks/useKeyboard.ts index 787df9a9..fb9fa73a 100644 --- a/ui/src/hooks/useKeyboard.ts +++ b/ui/src/hooks/useKeyboard.ts @@ -1,4 +1,4 @@ -import { useCallback } from "react"; +import { useCallback, useRef } from "react"; import { hidErrorRollOver, hidKeyBufferSize, KeysDownState, useHidStore, useRTCStore } from "@/hooks/stores"; import { JsonRpcResponse, useJsonRpc } from "@/hooks/useJsonRpc"; @@ -11,6 +11,9 @@ export default function useKeyboard() { const { rpcDataChannel } = useRTCStore(); const { keysDownState, setKeysDownState, setKeyboardLedState } = useHidStore(); + // Keepalive timer management + const keepAliveTimerRef = useRef(null); + // INTRODUCTION: The earlier version of the JetKVM device shipped with all keyboard state // being tracked on the browser/client-side. When adding the keyPressReport API to the // device-side code, we have to still support the situation where the browser/client-side code @@ -26,6 +29,7 @@ export default function useKeyboard() { const { reportKeyboardEvent: sendKeyboardEventHidRpc, reportKeypressEvent: sendKeypressEventHidRpc, + reportKeypressKeepAlive: sendKeypressKeepAliveHidRpc, rpcHidReady, } = useHidRpc((message) => { switch (message.constructor) { @@ -70,17 +74,6 @@ export default function useKeyboard() { ], ); - // resetKeyboardState is used to reset the keyboard state to no keys pressed and no modifiers. - // This is useful for macros and when the browser loses focus to ensure that the keyboard state - // is clean. - const resetKeyboardState = useCallback( - async () => { - // Reset the keys buffer to zeros and the modifier state to zero - keysDownState.keys.length = hidKeyBufferSize; - keysDownState.keys.fill(0); - keysDownState.modifier = 0; - sendKeyboardEvent(keysDownState); - }, [keysDownState, sendKeyboardEvent]); // executeMacro is used to execute a macro consisting of multiple steps. // Each step can have multiple keys, multiple modifiers and a delay. @@ -111,12 +104,61 @@ export default function useKeyboard() { } }; + const KEEPALIVE_INTERVAL = 200; // 200ms interval + + const cancelKeepAlive = useCallback(() => { + if (keepAliveTimerRef.current) { + clearInterval(keepAliveTimerRef.current); + keepAliveTimerRef.current = null; + } + }, []); + + const scheduleKeepAlive = useCallback(() => { + // Clear existing timer if it exists + if (keepAliveTimerRef.current) { + clearInterval(keepAliveTimerRef.current); + } + + // Create new interval timer + keepAliveTimerRef.current = setInterval(() => { + sendKeypressKeepAliveHidRpc(); + }, KEEPALIVE_INTERVAL); + }, [sendKeypressKeepAliveHidRpc]); + + // resetKeyboardState is used to reset the keyboard state to no keys pressed and no modifiers. + // This is useful for macros and when the browser loses focus to ensure that the keyboard state + // is clean. + const resetKeyboardState = useCallback(async () => { + // Cancel keepalive since we're resetting the keyboard state + cancelKeepAlive(); + + // Reset the keys buffer to zeros and the modifier state to zero + keysDownState.keys.length = hidKeyBufferSize; + keysDownState.keys.fill(0); + keysDownState.modifier = 0; + sendKeyboardEvent(keysDownState); + }, [keysDownState, sendKeyboardEvent, cancelKeepAlive]); + // handleKeyPress is used to handle a key press or release event. // This function handle both key press and key release events. // It checks if the keyPressReport API is available and sends the key press event. // If the keyPressReport API is not available, it simulates the device-side key // handling for legacy devices and updates the keysDownState accordingly. // It then sends the full keyboard state to the device. + + const sendKeypress = useCallback( + (key: number, press: boolean) => { + cancelKeepAlive(); + + sendKeypressEventHidRpc(key, press); + + if (press) { + scheduleKeepAlive(); + } + }, + [sendKeypressEventHidRpc, scheduleKeepAlive, cancelKeepAlive], + ); + const handleKeyPress = useCallback( async (key: number, press: boolean) => { if (rpcDataChannel?.readyState !== "open" && !rpcHidReady) return; @@ -129,7 +171,7 @@ export default function useKeyboard() { // Older device version doesn't support this API, so we will switch to local key handling // In that case we will switch to local key handling and update the keysDownState // in client/browser-side code using simulateDeviceSideKeyHandlingForLegacyDevices. - sendKeypressEventHidRpc(key, press); + sendKeypress(key, press); } else { // if the keyPress api is not available, we need to handle the key locally const downState = simulateDeviceSideKeyHandlingForLegacyDevices(keysDownState, key, press); @@ -147,7 +189,7 @@ export default function useKeyboard() { resetKeyboardState, rpcDataChannel?.readyState, sendKeyboardEvent, - sendKeypressEventHidRpc, + sendKeypress, ], ); @@ -210,5 +252,10 @@ export default function useKeyboard() { return { modifier: modifiers, keys }; } - return { handleKeyPress, resetKeyboardState, executeMacro }; + // Cleanup function to cancel keepalive timer + const cleanup = useCallback(() => { + cancelKeepAlive(); + }, [cancelKeepAlive]); + + return { handleKeyPress, resetKeyboardState, executeMacro, cleanup }; } From 8b079033887bcc0e47b239722a5e0b491433bfca Mon Sep 17 00:00:00 2001 From: Siyuan Miao Date: Thu, 11 Sep 2025 15:00:03 +0200 Subject: [PATCH 03/20] remove logging --- internal/usbgadget/hid_keyboard.go | 18 +----------------- 1 file changed, 1 insertion(+), 17 deletions(-) diff --git a/internal/usbgadget/hid_keyboard.go b/internal/usbgadget/hid_keyboard.go index f9bd20db..0cb978b4 100644 --- a/internal/usbgadget/hid_keyboard.go +++ b/internal/usbgadget/hid_keyboard.go @@ -390,20 +390,11 @@ var KeyCodeToMaskMap = map[byte]byte{ } func (u *UsbGadget) keypressReport(key byte, press bool, autoRelease bool) (KeysDownState, error) { - ll := u.log.Info().Str("component", "kbd") - ll.Uint8("key", key).Msg("locking keyboardLock") - u.keyboardLock.Lock() - defer func() { - u.keyboardLock.Unlock() - ll.Uint8("key", key).Msg("unlocked keyboardLock") - }() + defer u.keyboardLock.Unlock() - ll.Uint8("key", key).Msg("resetting user input time") defer u.resetUserInputTime() - ll.Uint8("key", key).Msg("locked keyboardLock") - // IMPORTANT: This code parallels the logic in the kernel's hid-gadget driver // for handling key presses and releases. It ensures that the USB gadget // behaves similarly to a real USB HID keyboard. This logic is paralleled @@ -461,8 +452,6 @@ func (u *UsbGadget) keypressReport(key byte, press bool, autoRelease bool) (Keys } } - ll.Uint8("key", key).Msg("checking if auto-release is enabled") - // if autoRelease { // u.kbdAutoReleaseLock.Lock() // u.kbdAutoReleaseLock.Unlock() @@ -480,8 +469,6 @@ func (u *UsbGadget) keypressReport(key byte, press bool, autoRelease bool) (Keys // } // } - ll.Uint8("key", key).Msg("writing keypress report to hidg0") - err := u.keyboardWriteHidFile(modifier, keys) if err != nil { u.log.Warn().Uint8("modifier", modifier).Uints8("keys", keys).Msg("Could not write keypress report to hidg0") @@ -495,14 +482,11 @@ func (u *UsbGadget) keypressReport(key byte, press bool, autoRelease bool) (Keys } if autoRelease { - ll.Uint8("key", key).Msg("scheduling auto-release") u.scheduleAutoRelease(key) } } else { if autoRelease { - ll.Uint8("key", key).Msg("canceling auto-release") u.cancelAutoRelease() - ll.Uint8("key", key).Msg("auto-release canceled") } } From 46aee61565e9aa7e5ade1413c633661836b734a2 Mon Sep 17 00:00:00 2001 From: Siyuan Miao Date: Thu, 11 Sep 2025 15:04:35 +0200 Subject: [PATCH 04/20] clean up logging --- internal/hidrpc/message.go | 2 ++ internal/usbgadget/hid_keyboard.go | 5 +++-- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/internal/hidrpc/message.go b/internal/hidrpc/message.go index 84bbda7c..e0f4493e 100644 --- a/internal/hidrpc/message.go +++ b/internal/hidrpc/message.go @@ -43,6 +43,8 @@ func (m *Message) String() string { return fmt.Sprintf("MouseReport{Malformed: %v}", m.d) } return fmt.Sprintf("MouseReport{DX: %d, DY: %d, Button: %d}", m.d[0], m.d[1], m.d[2]) + case TypeKeypressKeepAliveReport: + return "KeypressKeepAliveReport" default: return fmt.Sprintf("Unknown{Type: %d, Data: %v}", m.t, m.d) } diff --git a/internal/usbgadget/hid_keyboard.go b/internal/usbgadget/hid_keyboard.go index 0cb978b4..40dff0be 100644 --- a/internal/usbgadget/hid_keyboard.go +++ b/internal/usbgadget/hid_keyboard.go @@ -201,14 +201,15 @@ func (u *UsbGadget) DelayAutoRelease() { u.kbdAutoReleaseLock.Lock() defer u.kbdAutoReleaseLock.Unlock() - u.log.Info().Msg("delaying auto-release") + u.log.Trace().Msg("delaying auto-release") if u.kbdAutoReleaseTimer == nil { return } - u.log.Info().Msg("resetting auto-release timer") u.kbdAutoReleaseTimer.Reset(autoReleaseKeyboardInterval) + + u.log.Trace().Msg("auto-release timer reset") } func (u *UsbGadget) performAutoRelease(key byte) { From 8e9554f4d60bd5fb7faba19f84ad003ce86464e3 Mon Sep 17 00:00:00 2001 From: Siyuan Miao Date: Thu, 11 Sep 2025 16:16:47 +0200 Subject: [PATCH 05/20] chore: use unreliable channel to send keepalive events --- ui/src/hooks/stores.ts | 6 +++ ui/src/hooks/useHidRpc.ts | 35 +++++++++++++---- ui/src/routes/devices.$id.tsx | 12 ++++++ webrtc.go | 73 +++++++++++++++++++---------------- 4 files changed, 84 insertions(+), 42 deletions(-) diff --git a/ui/src/hooks/stores.ts b/ui/src/hooks/stores.ts index f99fd07d..7007f4e6 100644 --- a/ui/src/hooks/stores.ts +++ b/ui/src/hooks/stores.ts @@ -111,6 +111,9 @@ export interface RTCState { rpcHidChannel: RTCDataChannel | null; setRpcHidChannel: (channel: RTCDataChannel) => void; + rpcHidUnreliableChannel: RTCDataChannel | null; + setRpcHidUnreliableChannel: (channel: RTCDataChannel) => void; + peerConnectionState: RTCPeerConnectionState | null; setPeerConnectionState: (state: RTCPeerConnectionState) => void; @@ -163,6 +166,9 @@ export const useRTCStore = create(set => ({ rpcHidChannel: null, setRpcHidChannel: (channel: RTCDataChannel) => set({ rpcHidChannel: channel }), + rpcHidUnreliableChannel: null, + setRpcHidUnreliableChannel: (channel: RTCDataChannel) => set({ rpcHidUnreliableChannel: channel }), + transceiver: null, setTransceiver: (transceiver: RTCRtpTransceiver) => set({ transceiver }), diff --git a/ui/src/hooks/useHidRpc.ts b/ui/src/hooks/useHidRpc.ts index d5690849..ff22885c 100644 --- a/ui/src/hooks/useHidRpc.ts +++ b/ui/src/hooks/useHidRpc.ts @@ -16,20 +16,35 @@ import { const KEEPALIVE_MESSAGE = new KeypressKeepAliveMessage(); +interface sendMessageParams { + ignoreHandshakeState?: boolean; + useUnreliableChannel?: boolean; +} + export function useHidRpc(onHidRpcMessage?: (payload: RpcMessage) => void) { - const { rpcHidChannel, setRpcHidProtocolVersion, rpcHidProtocolVersion } = useRTCStore(); + const { + rpcHidChannel, + rpcHidUnreliableChannel, + setRpcHidProtocolVersion, + rpcHidProtocolVersion, + } = useRTCStore(); + const rpcHidReady = useMemo(() => { return rpcHidChannel?.readyState === "open" && rpcHidProtocolVersion !== null; }, [rpcHidChannel, rpcHidProtocolVersion]); + const rpcHidUnreliableReady = useMemo(() => { + return rpcHidUnreliableChannel?.readyState === "open" && rpcHidProtocolVersion !== null; + }, [rpcHidUnreliableChannel, rpcHidProtocolVersion]); + const rpcHidStatus = useMemo(() => { if (!rpcHidChannel) return "N/A"; if (rpcHidChannel.readyState !== "open") return rpcHidChannel.readyState; if (!rpcHidProtocolVersion) return "handshaking"; - return `ready (v${rpcHidProtocolVersion})`; - }, [rpcHidChannel, rpcHidProtocolVersion]); + return `ready (v${rpcHidProtocolVersion}${rpcHidUnreliableReady ? "+u" : ""})`; + }, [rpcHidChannel, rpcHidUnreliableReady, rpcHidProtocolVersion]); - const sendMessage = useCallback((message: RpcMessage, ignoreHandshakeState = false) => { + const sendMessage = useCallback((message: RpcMessage, { ignoreHandshakeState, useUnreliableChannel }: sendMessageParams = {}) => { if (rpcHidChannel?.readyState !== "open") return; if (!rpcHidReady && !ignoreHandshakeState) return; @@ -41,8 +56,12 @@ export function useHidRpc(onHidRpcMessage?: (payload: RpcMessage) => void) { } if (!data) return; - rpcHidChannel?.send(data as unknown as ArrayBuffer); - }, [rpcHidChannel, rpcHidReady]); + if (useUnreliableChannel && rpcHidUnreliableReady) { + rpcHidUnreliableChannel?.send(data as unknown as ArrayBuffer); + } else { + rpcHidChannel?.send(data as unknown as ArrayBuffer); + } + }, [rpcHidChannel, rpcHidReady, rpcHidUnreliableChannel, rpcHidUnreliableReady]); const reportKeyboardEvent = useCallback( (keys: number[], modifier: number) => { @@ -72,14 +91,14 @@ export function useHidRpc(onHidRpcMessage?: (payload: RpcMessage) => void) { ); const reportKeypressKeepAlive = useCallback(() => { - sendMessage(KEEPALIVE_MESSAGE); + sendMessage(KEEPALIVE_MESSAGE, { useUnreliableChannel: true }); }, [sendMessage]); const sendHandshake = useCallback(() => { if (rpcHidProtocolVersion) return; if (!rpcHidChannel) return; - sendMessage(new HandshakeMessage(HID_RPC_VERSION), true); + sendMessage(new HandshakeMessage(HID_RPC_VERSION), { ignoreHandshakeState: true }); }, [rpcHidChannel, rpcHidProtocolVersion, sendMessage]); const handleHandshake = useCallback((message: HandshakeMessage) => { diff --git a/ui/src/routes/devices.$id.tsx b/ui/src/routes/devices.$id.tsx index 4318447e..444e347d 100644 --- a/ui/src/routes/devices.$id.tsx +++ b/ui/src/routes/devices.$id.tsx @@ -136,6 +136,7 @@ export default function KvmIdRoute() { rpcDataChannel, setTransceiver, setRpcHidChannel, + setRpcHidUnreliableChannel, } = useRTCStore(); const location = useLocation(); @@ -488,6 +489,16 @@ export default function KvmIdRoute() { setRpcHidChannel(rpcHidChannel); }; + const rpcHidUnreliableChannel = pc.createDataChannel("hidrpc-unreliable", { + // We don't need to be ordered, as we're using the unreliable channel for keepalive messages + ordered: false, + maxRetransmits: 0, + }); + rpcHidUnreliableChannel.binaryType = "arraybuffer"; + rpcHidUnreliableChannel.onopen = () => { + setRpcHidUnreliableChannel(rpcHidUnreliableChannel); + }; + setPeerConnection(pc); }, [ cleanupAndStopReconnecting, @@ -499,6 +510,7 @@ export default function KvmIdRoute() { setPeerConnectionState, setRpcDataChannel, setRpcHidChannel, + setRpcHidUnreliableChannel, setTransceiver, ]); diff --git a/webrtc.go b/webrtc.go index c3d0dc1b..e43c2512 100644 --- a/webrtc.go +++ b/webrtc.go @@ -145,6 +145,41 @@ func newSession(config SessionConfig) (*Session, error) { go session.handleQueues(i) } + onHidMessage := func(msg webrtc.DataChannelMessage) { + l := scopedLogger.With().Int("length", len(msg.Data)).Logger() + // only log data if the log level is debug or lower + if scopedLogger.GetLevel() > zerolog.DebugLevel { + l = l.With().Str("data", string(msg.Data)).Logger() + } + + if msg.IsString { + l.Warn().Msg("received string data in HID RPC message handler") + return + } + + if len(msg.Data) < 1 { + l.Warn().Msg("received empty data in HID RPC message handler") + return + } + + l.Trace().Msg("received data in HID RPC message handler") + + // Enqueue to ensure ordered processing + queueIndex := hidrpc.GetQueueIndex(hidrpc.MessageType(msg.Data[0])) + if queueIndex >= len(session.hidQueue) || queueIndex < 0 { + l.Warn().Int("queueIndex", queueIndex).Msg("received data in HID RPC message handler, but queue index not found") + queueIndex = 3 + } + + queue := session.hidQueue[queueIndex] + if queue != nil { + queue <- msg + } else { + l.Warn().Int("queueIndex", queueIndex).Msg("received data in HID RPC message handler, but queue is nil") + return + } + } + peerConnection.OnDataChannel(func(d *webrtc.DataChannel) { defer func() { if r := recover(); r != nil { @@ -157,40 +192,10 @@ func newSession(config SessionConfig) (*Session, error) { switch d.Label() { case "hidrpc": session.HidChannel = d - d.OnMessage(func(msg webrtc.DataChannelMessage) { - l := scopedLogger.With().Int("length", len(msg.Data)).Logger() - // only log data if the log level is debug or lower - if scopedLogger.GetLevel() > zerolog.DebugLevel { - l = l.With().Str("data", string(msg.Data)).Logger() - } - - if msg.IsString { - l.Warn().Msg("received string data in HID RPC message handler") - return - } - - if len(msg.Data) < 1 { - l.Warn().Msg("received empty data in HID RPC message handler") - return - } - - l.Trace().Msg("received data in HID RPC message handler") - - // Enqueue to ensure ordered processing - queueIndex := hidrpc.GetQueueIndex(hidrpc.MessageType(msg.Data[0])) - if queueIndex >= len(session.hidQueue) || queueIndex < 0 { - l.Warn().Int("queueIndex", queueIndex).Msg("received data in HID RPC message handler, but queue index not found") - queueIndex = 3 - } - - queue := session.hidQueue[queueIndex] - if queue != nil { - queue <- msg - } else { - l.Warn().Int("queueIndex", queueIndex).Msg("received data in HID RPC message handler, but queue is nil") - return - } - }) + d.OnMessage(onHidMessage) + // we won't send anything over the unreliable channel + case "hidrpc-unreliable": + d.OnMessage(onHidMessage) case "rpc": session.RPCChannel = d d.OnMessage(func(msg webrtc.DataChannelMessage) { From 3901ac91e0a4274a07c6fc11ecc21157a636588f Mon Sep 17 00:00:00 2001 From: Siyuan Miao Date: Thu, 11 Sep 2025 17:24:10 +0200 Subject: [PATCH 06/20] chore: use ordered unreliable channel for pointer events --- hidrpc.go | 9 ++- ui/src/hooks/stores.ts | 6 ++ ui/src/hooks/useHidRpc.ts | 34 +++++++++--- ui/src/routes/devices.$id.tsx | 16 +++++- webrtc.go | 101 +++++++++++++++++++--------------- 5 files changed, 110 insertions(+), 56 deletions(-) diff --git a/hidrpc.go b/hidrpc.go index 52022ba6..2c3ca27b 100644 --- a/hidrpc.go +++ b/hidrpc.go @@ -54,8 +54,13 @@ func handleHidRPCMessage(message hidrpc.Message, session *Session) { } } -func onHidMessage(data []byte, session *Session) { - scopedLogger := hidRPCLogger.With().Bytes("data", data).Logger() +func onHidMessage(msg hidQueueMessage, session *Session) { + data := msg.Data + + scopedLogger := hidRPCLogger.With(). + Str("channel", msg.channel). + Bytes("data", data). + Logger() scopedLogger.Debug().Msg("HID RPC message received") if len(data) < 1 { diff --git a/ui/src/hooks/stores.ts b/ui/src/hooks/stores.ts index 7007f4e6..b849fa55 100644 --- a/ui/src/hooks/stores.ts +++ b/ui/src/hooks/stores.ts @@ -114,6 +114,9 @@ export interface RTCState { rpcHidUnreliableChannel: RTCDataChannel | null; setRpcHidUnreliableChannel: (channel: RTCDataChannel) => void; + rpcHidUnreliableNonOrderedChannel: RTCDataChannel | null; + setRpcHidUnreliableNonOrderedChannel: (channel: RTCDataChannel) => void; + peerConnectionState: RTCPeerConnectionState | null; setPeerConnectionState: (state: RTCPeerConnectionState) => void; @@ -169,6 +172,9 @@ export const useRTCStore = create(set => ({ rpcHidUnreliableChannel: null, setRpcHidUnreliableChannel: (channel: RTCDataChannel) => set({ rpcHidUnreliableChannel: channel }), + rpcHidUnreliableNonOrderedChannel: null, + setRpcHidUnreliableNonOrderedChannel: (channel: RTCDataChannel) => set({ rpcHidUnreliableNonOrderedChannel: channel }), + transceiver: null, setTransceiver: (transceiver: RTCRtpTransceiver) => set({ transceiver }), diff --git a/ui/src/hooks/useHidRpc.ts b/ui/src/hooks/useHidRpc.ts index ff22885c..58c04f42 100644 --- a/ui/src/hooks/useHidRpc.ts +++ b/ui/src/hooks/useHidRpc.ts @@ -19,12 +19,14 @@ const KEEPALIVE_MESSAGE = new KeypressKeepAliveMessage(); interface sendMessageParams { ignoreHandshakeState?: boolean; useUnreliableChannel?: boolean; + requireOrdered?: boolean; } export function useHidRpc(onHidRpcMessage?: (payload: RpcMessage) => void) { const { rpcHidChannel, rpcHidUnreliableChannel, + rpcHidUnreliableNonOrderedChannel, setRpcHidProtocolVersion, rpcHidProtocolVersion, } = useRTCStore(); @@ -37,6 +39,10 @@ export function useHidRpc(onHidRpcMessage?: (payload: RpcMessage) => void) { return rpcHidUnreliableChannel?.readyState === "open" && rpcHidProtocolVersion !== null; }, [rpcHidUnreliableChannel, rpcHidProtocolVersion]); + const rpcHidUnreliableNonOrderedReady = useMemo(() => { + return rpcHidUnreliableNonOrderedChannel?.readyState === "open" && rpcHidProtocolVersion !== null; + }, [rpcHidUnreliableNonOrderedChannel, rpcHidProtocolVersion]); + const rpcHidStatus = useMemo(() => { if (!rpcHidChannel) return "N/A"; if (rpcHidChannel.readyState !== "open") return rpcHidChannel.readyState; @@ -44,7 +50,7 @@ export function useHidRpc(onHidRpcMessage?: (payload: RpcMessage) => void) { return `ready (v${rpcHidProtocolVersion}${rpcHidUnreliableReady ? "+u" : ""})`; }, [rpcHidChannel, rpcHidUnreliableReady, rpcHidProtocolVersion]); - const sendMessage = useCallback((message: RpcMessage, { ignoreHandshakeState, useUnreliableChannel }: sendMessageParams = {}) => { + const sendMessage = useCallback((message: RpcMessage, { ignoreHandshakeState, useUnreliableChannel, requireOrdered = true }: sendMessageParams = {}) => { if (rpcHidChannel?.readyState !== "open") return; if (!rpcHidReady && !ignoreHandshakeState) return; @@ -56,12 +62,24 @@ export function useHidRpc(onHidRpcMessage?: (payload: RpcMessage) => void) { } if (!data) return; - if (useUnreliableChannel && rpcHidUnreliableReady) { - rpcHidUnreliableChannel?.send(data as unknown as ArrayBuffer); - } else { - rpcHidChannel?.send(data as unknown as ArrayBuffer); + if (useUnreliableChannel) { + if (requireOrdered && rpcHidUnreliableReady) { + rpcHidUnreliableChannel?.send(data as unknown as ArrayBuffer); + } else if (!requireOrdered && rpcHidUnreliableNonOrderedReady) { + rpcHidUnreliableNonOrderedChannel?.send(data as unknown as ArrayBuffer); + } + return; } - }, [rpcHidChannel, rpcHidReady, rpcHidUnreliableChannel, rpcHidUnreliableReady]); + + rpcHidChannel?.send(data as unknown as ArrayBuffer); + }, [ + rpcHidChannel, + rpcHidUnreliableChannel, + rpcHidUnreliableNonOrderedChannel, + rpcHidReady, + rpcHidUnreliableReady, + rpcHidUnreliableNonOrderedReady, + ]); const reportKeyboardEvent = useCallback( (keys: number[], modifier: number) => { @@ -78,7 +96,7 @@ export function useHidRpc(onHidRpcMessage?: (payload: RpcMessage) => void) { const reportAbsMouseEvent = useCallback( (x: number, y: number, buttons: number) => { - sendMessage(new PointerReportMessage(x, y, buttons)); + sendMessage(new PointerReportMessage(x, y, buttons), { useUnreliableChannel: true }); }, [sendMessage], ); @@ -91,7 +109,7 @@ export function useHidRpc(onHidRpcMessage?: (payload: RpcMessage) => void) { ); const reportKeypressKeepAlive = useCallback(() => { - sendMessage(KEEPALIVE_MESSAGE, { useUnreliableChannel: true }); + sendMessage(KEEPALIVE_MESSAGE, { useUnreliableChannel: true, requireOrdered: false }); }, [sendMessage]); const sendHandshake = useCallback(() => { diff --git a/ui/src/routes/devices.$id.tsx b/ui/src/routes/devices.$id.tsx index 444e347d..8a363199 100644 --- a/ui/src/routes/devices.$id.tsx +++ b/ui/src/routes/devices.$id.tsx @@ -136,6 +136,7 @@ export default function KvmIdRoute() { rpcDataChannel, setTransceiver, setRpcHidChannel, + setRpcHidUnreliableNonOrderedChannel, setRpcHidUnreliableChannel, } = useRTCStore(); @@ -489,9 +490,8 @@ export default function KvmIdRoute() { setRpcHidChannel(rpcHidChannel); }; - const rpcHidUnreliableChannel = pc.createDataChannel("hidrpc-unreliable", { - // We don't need to be ordered, as we're using the unreliable channel for keepalive messages - ordered: false, + const rpcHidUnreliableChannel = pc.createDataChannel("hidrpc-unreliable-ordered", { + ordered: true, maxRetransmits: 0, }); rpcHidUnreliableChannel.binaryType = "arraybuffer"; @@ -499,6 +499,15 @@ export default function KvmIdRoute() { setRpcHidUnreliableChannel(rpcHidUnreliableChannel); }; + const rpcHidUnreliableNonOrderedChannel = pc.createDataChannel("hidrpc-unreliable-nonordered", { + ordered: false, + maxRetransmits: 0, + }); + rpcHidUnreliableNonOrderedChannel.binaryType = "arraybuffer"; + rpcHidUnreliableNonOrderedChannel.onopen = () => { + setRpcHidUnreliableNonOrderedChannel(rpcHidUnreliableNonOrderedChannel); + }; + setPeerConnection(pc); }, [ cleanupAndStopReconnecting, @@ -510,6 +519,7 @@ export default function KvmIdRoute() { setPeerConnectionState, setRpcDataChannel, setRpcHidChannel, + setRpcHidUnreliableNonOrderedChannel, setRpcHidUnreliableChannel, setTransceiver, ]); diff --git a/webrtc.go b/webrtc.go index e43c2512..333a58b8 100644 --- a/webrtc.go +++ b/webrtc.go @@ -29,7 +29,12 @@ type Session struct { hidRPCAvailable bool hidQueueLock sync.Mutex - hidQueue []chan webrtc.DataChannelMessage + hidQueue []chan hidQueueMessage +} + +type hidQueueMessage struct { + webrtc.DataChannelMessage + channel string } type SessionConfig struct { @@ -78,16 +83,59 @@ func (s *Session) initQueues() { s.hidQueueLock.Lock() defer s.hidQueueLock.Unlock() - s.hidQueue = make([]chan webrtc.DataChannelMessage, 0) + s.hidQueue = make([]chan hidQueueMessage, 0) for i := 0; i < 4; i++ { - q := make(chan webrtc.DataChannelMessage, 256) + q := make(chan hidQueueMessage, 256) s.hidQueue = append(s.hidQueue, q) } } func (s *Session) handleQueues(index int) { for msg := range s.hidQueue[index] { - onHidMessage(msg.Data, s) + onHidMessage(msg, s) + } +} + +func getOnHidMessageHandler(session *Session, scopedLogger *zerolog.Logger, channel string) func(msg webrtc.DataChannelMessage) { + return func(msg webrtc.DataChannelMessage) { + l := scopedLogger.With(). + Str("channel", channel). + Int("length", len(msg.Data)). + Logger() + // only log data if the log level is debug or lower + if scopedLogger.GetLevel() > zerolog.DebugLevel { + l = l.With().Str("data", string(msg.Data)).Logger() + } + + if msg.IsString { + l.Warn().Msg("received string data in HID RPC message handler") + return + } + + if len(msg.Data) < 1 { + l.Warn().Msg("received empty data in HID RPC message handler") + return + } + + l.Trace().Msg("received data in HID RPC message handler") + + // Enqueue to ensure ordered processing + queueIndex := hidrpc.GetQueueIndex(hidrpc.MessageType(msg.Data[0])) + if queueIndex >= len(session.hidQueue) || queueIndex < 0 { + l.Warn().Int("queueIndex", queueIndex).Msg("received data in HID RPC message handler, but queue index not found") + queueIndex = 3 + } + + queue := session.hidQueue[queueIndex] + if queue != nil { + queue <- hidQueueMessage{ + DataChannelMessage: msg, + channel: channel, + } + } else { + l.Warn().Int("queueIndex", queueIndex).Msg("received data in HID RPC message handler, but queue is nil") + return + } } } @@ -145,41 +193,6 @@ func newSession(config SessionConfig) (*Session, error) { go session.handleQueues(i) } - onHidMessage := func(msg webrtc.DataChannelMessage) { - l := scopedLogger.With().Int("length", len(msg.Data)).Logger() - // only log data if the log level is debug or lower - if scopedLogger.GetLevel() > zerolog.DebugLevel { - l = l.With().Str("data", string(msg.Data)).Logger() - } - - if msg.IsString { - l.Warn().Msg("received string data in HID RPC message handler") - return - } - - if len(msg.Data) < 1 { - l.Warn().Msg("received empty data in HID RPC message handler") - return - } - - l.Trace().Msg("received data in HID RPC message handler") - - // Enqueue to ensure ordered processing - queueIndex := hidrpc.GetQueueIndex(hidrpc.MessageType(msg.Data[0])) - if queueIndex >= len(session.hidQueue) || queueIndex < 0 { - l.Warn().Int("queueIndex", queueIndex).Msg("received data in HID RPC message handler, but queue index not found") - queueIndex = 3 - } - - queue := session.hidQueue[queueIndex] - if queue != nil { - queue <- msg - } else { - l.Warn().Int("queueIndex", queueIndex).Msg("received data in HID RPC message handler, but queue is nil") - return - } - } - peerConnection.OnDataChannel(func(d *webrtc.DataChannel) { defer func() { if r := recover(); r != nil { @@ -192,10 +205,12 @@ func newSession(config SessionConfig) (*Session, error) { switch d.Label() { case "hidrpc": session.HidChannel = d - d.OnMessage(onHidMessage) - // we won't send anything over the unreliable channel - case "hidrpc-unreliable": - d.OnMessage(onHidMessage) + d.OnMessage(getOnHidMessageHandler(session, scopedLogger, "hidrpc")) + // we won't send anything over the unreliable channels + case "hidrpc-unreliable-ordered": + d.OnMessage(getOnHidMessageHandler(session, scopedLogger, "hidrpc-unreliable-ordered")) + case "hidrpc-unreliable-nonordered": + d.OnMessage(getOnHidMessageHandler(session, scopedLogger, "hidrpc-unreliable-nonordered")) case "rpc": session.RPCChannel = d d.OnMessage(func(msg webrtc.DataChannelMessage) { From cbf1c7fba68dc9c4a0a083537444bc85d3a28020 Mon Sep 17 00:00:00 2001 From: Siyuan Miao Date: Thu, 11 Sep 2025 17:46:50 +0200 Subject: [PATCH 07/20] chore: adjust auto release key interval --- internal/usbgadget/hid_keyboard.go | 27 +++++++-------------------- ui/src/hooks/useKeyboard.ts | 2 +- 2 files changed, 8 insertions(+), 21 deletions(-) diff --git a/internal/usbgadget/hid_keyboard.go b/internal/usbgadget/hid_keyboard.go index 40dff0be..e3997ea6 100644 --- a/internal/usbgadget/hid_keyboard.go +++ b/internal/usbgadget/hid_keyboard.go @@ -22,6 +22,11 @@ var keyboardConfig = gadgetConfigItem{ reportDesc: keyboardReportDesc, } +// macOS default: 15 * 15 = 225ms https://discussions.apple.com/thread/1316947?sortBy=rank +// Linux default: 250ms https://man.archlinux.org/man/kbdrate.8.en +// Windows default: 1ms `HKEY_CURRENT_USER\Control Panel\Accessibility\Keyboard Response\AutoRepeatDelay` +const autoReleaseKeyboardInterval = time.Millisecond * 225 + // Source: https://www.kernel.org/doc/Documentation/usb/gadget_hid.txt var keyboardReportDesc = []byte{ 0x05, 0x01, /* USAGE_PAGE (Generic Desktop) */ @@ -173,8 +178,6 @@ func (u *UsbGadget) SetOnKeysDownChange(f func(state KeysDownState)) { u.onKeysDownChange = &f } -const autoReleaseKeyboardInterval = time.Millisecond * 450 - func (u *UsbGadget) scheduleAutoRelease(key byte) { u.kbdAutoReleaseLock.Lock() defer u.kbdAutoReleaseLock.Unlock() @@ -222,7 +225,8 @@ func (u *UsbGadget) performAutoRelease(key byte) { default: } - _, err := u.keypressReport(key, false, false) + // we just reset the keyboard state to 0 no matter what + _, err := u.keypressReport(0, false, false) if err != nil { u.log.Warn().Uint8("key", key).Msg("failed to auto-release keyboard key") } @@ -453,23 +457,6 @@ func (u *UsbGadget) keypressReport(key byte, press bool, autoRelease bool) (Keys } } - // if autoRelease { - // u.kbdAutoReleaseLock.Lock() - // u.kbdAutoReleaseLock.Unlock() - // ll.Uint8("key", key).Msg("locking kbdAutoReleaseLock, autoReleasLastKey reset") - - // defer func() { - // ll.Uint8("key", key).Msg("unlocked kbdAutoReleaseLock, autoReleasLastKey reset") - // }() - - // if u.kbdAutoReleaseLastKey == key { - // ll.Uint8("key", key).Msg("key already released by auto-release, skipping") - // u.kbdAutoReleaseLastKey = 0 - - // return u.UpdateKeysDown(modifier, keys), nil - // } - // } - err := u.keyboardWriteHidFile(modifier, keys) if err != nil { u.log.Warn().Uint8("modifier", modifier).Uints8("keys", keys).Msg("Could not write keypress report to hidg0") diff --git a/ui/src/hooks/useKeyboard.ts b/ui/src/hooks/useKeyboard.ts index fb9fa73a..cc9796f3 100644 --- a/ui/src/hooks/useKeyboard.ts +++ b/ui/src/hooks/useKeyboard.ts @@ -104,7 +104,7 @@ export default function useKeyboard() { } }; - const KEEPALIVE_INTERVAL = 200; // 200ms interval + const KEEPALIVE_INTERVAL = 75; // 200ms interval const cancelKeepAlive = useCallback(() => { if (keepAliveTimerRef.current) { From 67ea3c04322a26df2b3035a971f3f3241122804e Mon Sep 17 00:00:00 2001 From: Siyuan Miao Date: Fri, 12 Sep 2025 10:41:23 +0200 Subject: [PATCH 08/20] chore: update logging for kbdAutoReleaseLock --- internal/usbgadget/hid_keyboard.go | 17 ++++++++++------- internal/usbgadget/utils.go | 6 ++++++ 2 files changed, 16 insertions(+), 7 deletions(-) diff --git a/internal/usbgadget/hid_keyboard.go b/internal/usbgadget/hid_keyboard.go index e3997ea6..0c17355d 100644 --- a/internal/usbgadget/hid_keyboard.go +++ b/internal/usbgadget/hid_keyboard.go @@ -179,8 +179,9 @@ func (u *UsbGadget) SetOnKeysDownChange(f func(state KeysDownState)) { } func (u *UsbGadget) scheduleAutoRelease(key byte) { + u.log.Trace().Msg("scheduling autoRelease") u.kbdAutoReleaseLock.Lock() - defer u.kbdAutoReleaseLock.Unlock() + defer unlockWithLog(&u.kbdAutoReleaseLock, u.log, "autoRelease scheduled") if u.kbdAutoReleaseTimer != nil { u.kbdAutoReleaseTimer.Stop() @@ -192,8 +193,9 @@ func (u *UsbGadget) scheduleAutoRelease(key byte) { } func (u *UsbGadget) cancelAutoRelease() { + u.log.Trace().Msg("cancelling autoRelease") u.kbdAutoReleaseLock.Lock() - defer u.kbdAutoReleaseLock.Unlock() + defer unlockWithLog(&u.kbdAutoReleaseLock, u.log, "autoRelease cancelled") if u.kbdAutoReleaseTimer != nil { u.kbdAutoReleaseTimer.Stop() @@ -201,10 +203,9 @@ func (u *UsbGadget) cancelAutoRelease() { } func (u *UsbGadget) DelayAutoRelease() { + u.log.Trace().Msg("delaying autoRelease") u.kbdAutoReleaseLock.Lock() - defer u.kbdAutoReleaseLock.Unlock() - - u.log.Trace().Msg("delaying auto-release") + defer unlockWithLog(&u.kbdAutoReleaseLock, u.log, "autoRelease delayed") if u.kbdAutoReleaseTimer == nil { return @@ -216,8 +217,9 @@ func (u *UsbGadget) DelayAutoRelease() { } func (u *UsbGadget) performAutoRelease(key byte) { + u.log.Trace().Msg("performing autoRelease") u.kbdAutoReleaseLock.Lock() - defer u.kbdAutoReleaseLock.Unlock() + defer unlockWithLog(&u.kbdAutoReleaseLock, u.log, "autoRelease performed") select { case <-u.keyboardStateCtx.Done(): @@ -464,9 +466,10 @@ func (u *UsbGadget) keypressReport(key byte, press bool, autoRelease bool) (Keys if press { { + u.log.Trace().Msg("acquiring kbdAutoReleaseLock to update last key") u.kbdAutoReleaseLock.Lock() u.kbdAutoReleaseLastKey = key - u.kbdAutoReleaseLock.Unlock() + unlockWithLog(&u.kbdAutoReleaseLock, u.log, "last key updated") } if autoRelease { diff --git a/internal/usbgadget/utils.go b/internal/usbgadget/utils.go index d51f9e40..56f9c308 100644 --- a/internal/usbgadget/utils.go +++ b/internal/usbgadget/utils.go @@ -9,6 +9,7 @@ import ( "path/filepath" "strconv" "strings" + "sync" "time" "github.com/rs/zerolog" @@ -164,3 +165,8 @@ func (u *UsbGadget) resetLogSuppressionCounter(counterName string) { u.logSuppressionCounter[counterName] = 0 } } + +func unlockWithLog(lock *sync.Mutex, logger *zerolog.Logger, msg string, args ...any) { + logger.Trace().Msgf(msg, args...) + lock.Unlock() +} From 0bb3a6f3c29442d90f360ffd7df09c7f60c1b6d8 Mon Sep 17 00:00:00 2001 From: Siyuan Miao Date: Fri, 12 Sep 2025 10:41:40 +0200 Subject: [PATCH 09/20] chore: update comment for KEEPALIVE_INTERVAL --- ui/src/hooks/useKeyboard.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ui/src/hooks/useKeyboard.ts b/ui/src/hooks/useKeyboard.ts index cc9796f3..561485fb 100644 --- a/ui/src/hooks/useKeyboard.ts +++ b/ui/src/hooks/useKeyboard.ts @@ -104,7 +104,7 @@ export default function useKeyboard() { } }; - const KEEPALIVE_INTERVAL = 75; // 200ms interval + const KEEPALIVE_INTERVAL = 75; // TODO: use an adaptive interval based on RTT later const cancelKeepAlive = useCallback(() => { if (keepAliveTimerRef.current) { From 39d885c973bcb868a8257aa9fcb868f2b9bf44f7 Mon Sep 17 00:00:00 2001 From: Siyuan Miao Date: Fri, 12 Sep 2025 12:33:43 +0200 Subject: [PATCH 10/20] fix: should cancelAutorelease when pressed is true --- internal/usbgadget/hid_keyboard.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/internal/usbgadget/hid_keyboard.go b/internal/usbgadget/hid_keyboard.go index 0c17355d..786cb9c8 100644 --- a/internal/usbgadget/hid_keyboard.go +++ b/internal/usbgadget/hid_keyboard.go @@ -25,7 +25,7 @@ var keyboardConfig = gadgetConfigItem{ // macOS default: 15 * 15 = 225ms https://discussions.apple.com/thread/1316947?sortBy=rank // Linux default: 250ms https://man.archlinux.org/man/kbdrate.8.en // Windows default: 1ms `HKEY_CURRENT_USER\Control Panel\Accessibility\Keyboard Response\AutoRepeatDelay` -const autoReleaseKeyboardInterval = time.Millisecond * 225 +const autoReleaseKeyboardInterval = time.Millisecond * 100 // Source: https://www.kernel.org/doc/Documentation/usb/gadget_hid.txt var keyboardReportDesc = []byte{ @@ -473,11 +473,11 @@ func (u *UsbGadget) keypressReport(key byte, press bool, autoRelease bool) (Keys } if autoRelease { - u.scheduleAutoRelease(key) + u.cancelAutoRelease() } } else { if autoRelease { - u.cancelAutoRelease() + u.scheduleAutoRelease(key) } } From cc71e3273cdab3980a951c0002b429d77c3f6788 Mon Sep 17 00:00:00 2001 From: Siyuan Miao Date: Fri, 12 Sep 2025 12:48:34 +0200 Subject: [PATCH 11/20] fix: handshake won't happen if webrtc reconnects --- hidrpc.go | 5 ++++- ui/src/hooks/stores.ts | 4 ++-- ui/src/hooks/useHidRpc.ts | 14 ++++++++++++++ 3 files changed, 20 insertions(+), 3 deletions(-) diff --git a/hidrpc.go b/hidrpc.go index 2c3ca27b..64018a0e 100644 --- a/hidrpc.go +++ b/hidrpc.go @@ -122,7 +122,10 @@ func reportHidRPC(params any, session *Session) { } if !session.hidRPCAvailable || session.HidChannel == nil { - logger.Warn().Msg("HID RPC is not available, skipping reportHidRPC") + logger.Warn(). + Bool("hidRPCAvailable", session.hidRPCAvailable). + Bool("HidChannel", session.HidChannel != nil). + Msg("HID RPC is not available, skipping reportHidRPC") return } diff --git a/ui/src/hooks/stores.ts b/ui/src/hooks/stores.ts index b849fa55..9d5117c8 100644 --- a/ui/src/hooks/stores.ts +++ b/ui/src/hooks/stores.ts @@ -106,7 +106,7 @@ export interface RTCState { rpcDataChannel: RTCDataChannel | null; rpcHidProtocolVersion: number | null; - setRpcHidProtocolVersion: (version: number) => void; + setRpcHidProtocolVersion: (version: number | null) => void; rpcHidChannel: RTCDataChannel | null; setRpcHidChannel: (channel: RTCDataChannel) => void; @@ -164,7 +164,7 @@ export const useRTCStore = create(set => ({ setRpcDataChannel: (channel: RTCDataChannel) => set({ rpcDataChannel: channel }), rpcHidProtocolVersion: null, - setRpcHidProtocolVersion: (version: number) => set({ rpcHidProtocolVersion: version }), + setRpcHidProtocolVersion: (version: number | null) => set({ rpcHidProtocolVersion: version }), rpcHidChannel: null, setRpcHidChannel: (channel: RTCDataChannel) => set({ rpcHidChannel: channel }), diff --git a/ui/src/hooks/useHidRpc.ts b/ui/src/hooks/useHidRpc.ts index 58c04f42..0670acb8 100644 --- a/ui/src/hooks/useHidRpc.ts +++ b/ui/src/hooks/useHidRpc.ts @@ -167,10 +167,24 @@ export function useHidRpc(onHidRpcMessage?: (payload: RpcMessage) => void) { onHidRpcMessage?.(message); }; + const openHandler = () => { + console.warn("HID RPC channel opened"); + sendHandshake(); + }; + + const closeHandler = () => { + console.warn("HID RPC channel closed"); + setRpcHidProtocolVersion(null); + }; + rpcHidChannel.addEventListener("message", messageHandler); + rpcHidChannel.addEventListener("close", closeHandler); + rpcHidChannel.addEventListener("open", openHandler); return () => { rpcHidChannel.removeEventListener("message", messageHandler); + rpcHidChannel.removeEventListener("close", closeHandler); + rpcHidChannel.removeEventListener("open", openHandler); }; }, [ From f056d8af457bd31e83249f3779ff569ee63445b9 Mon Sep 17 00:00:00 2001 From: Siyuan Miao Date: Fri, 12 Sep 2025 12:50:46 +0200 Subject: [PATCH 12/20] chore: add trace log for writeWithTimeout --- internal/usbgadget/utils.go | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/internal/usbgadget/utils.go b/internal/usbgadget/utils.go index 56f9c308..85bf1579 100644 --- a/internal/usbgadget/utils.go +++ b/internal/usbgadget/utils.go @@ -121,6 +121,12 @@ func (u *UsbGadget) writeWithTimeout(file *os.File, data []byte) (n int, err err return } + u.log.Trace(). + Str("file", file.Name()). + Bytes("data", data). + Err(err). + Msg("write failed") + if errors.Is(err, os.ErrDeadlineExceeded) { u.logWithSuppression( fmt.Sprintf("writeWithTimeout_%s", file.Name()), From 8d0c2c24ad4bde78b6bcd03acf19bb2bf44d7855 Mon Sep 17 00:00:00 2001 From: Siyuan Miao Date: Fri, 12 Sep 2025 13:32:02 +0200 Subject: [PATCH 13/20] chore: add timeout for KeypressReport --- internal/usbgadget/hid_keyboard.go | 30 ++++++++++++++++++++++++++---- 1 file changed, 26 insertions(+), 4 deletions(-) diff --git a/internal/usbgadget/hid_keyboard.go b/internal/usbgadget/hid_keyboard.go index 786cb9c8..76e5a125 100644 --- a/internal/usbgadget/hid_keyboard.go +++ b/internal/usbgadget/hid_keyboard.go @@ -396,10 +396,7 @@ var KeyCodeToMaskMap = map[byte]byte{ RightSuper: ModifierMaskRightSuper, } -func (u *UsbGadget) keypressReport(key byte, press bool, autoRelease bool) (KeysDownState, error) { - u.keyboardLock.Lock() - defer u.keyboardLock.Unlock() - +func (u *UsbGadget) keypressReportNonThreadSafe(key byte, press bool, autoRelease bool) (KeysDownState, error) { defer u.resetUserInputTime() // IMPORTANT: This code parallels the logic in the kernel's hid-gadget driver @@ -484,6 +481,31 @@ func (u *UsbGadget) keypressReport(key byte, press bool, autoRelease bool) (Keys return u.UpdateKeysDown(modifier, keys), err } +type keypressReportResult struct { + KeysDownState KeysDownState + Error error +} + +func (u *UsbGadget) keypressReport(key byte, press bool, autoRelease bool) (KeysDownState, error) { + u.keyboardLock.Lock() + defer u.keyboardLock.Unlock() + + r := make(chan keypressReportResult) + go func() { + state, err := u.keypressReportNonThreadSafe(key, press, autoRelease) + r <- keypressReportResult{KeysDownState: state, Error: err} + }() + + select { + case <-time.After(1 * time.Second): + u.log.Warn().Msg("keypressReport timed out, possibly stuck") + return u.keysDownState, fmt.Errorf("keypressReport timed out, possibly stuck") + case ret := <-r: + u.log.Debug().Msg("keypressReport handled") + return ret.KeysDownState, ret.Error + } +} + func (u *UsbGadget) KeypressReport(key byte, press bool) (KeysDownState, error) { return u.keypressReport(key, press, true) } From 12210fac96ef7b9ddc4d0bdf4c7e476e7455505a Mon Sep 17 00:00:00 2001 From: Siyuan Miao Date: Fri, 12 Sep 2025 14:51:11 +0200 Subject: [PATCH 14/20] chore: use the proper key to send release command --- go.mod | 2 ++ go.sum | 3 ++ hidrpc.go | 7 ++++- internal/usbgadget/hid_keyboard.go | 44 ++++++++++++++++++++++-------- 4 files changed, 44 insertions(+), 12 deletions(-) diff --git a/go.mod b/go.mod index 72e57cd2..3a1b2543 100644 --- a/go.mod +++ b/go.mod @@ -12,6 +12,7 @@ require ( github.com/gin-contrib/logger v1.2.6 github.com/gin-gonic/gin v1.10.1 github.com/go-co-op/gocron/v2 v2.16.5 + github.com/google/flatbuffers v25.2.10+incompatible github.com/google/uuid v1.6.0 github.com/guregu/null/v6 v6.0.0 github.com/gwatts/rootcerts v0.0.0-20250901182336-dc5ae18bd79f @@ -23,6 +24,7 @@ require ( github.com/prometheus/common v0.66.0 github.com/prometheus/procfs v0.17.0 github.com/psanford/httpreadat v0.1.0 + github.com/rs/xid v1.6.0 github.com/rs/zerolog v1.34.0 github.com/sourcegraph/tf-dag v0.2.2-0.20250131204052-3e8ff1477b4f github.com/stretchr/testify v1.11.1 diff --git a/go.sum b/go.sum index 36087a28..7d42b802 100644 --- a/go.sum +++ b/go.sum @@ -53,6 +53,8 @@ github.com/go-playground/validator/v10 v10.26.0/go.mod h1:I5QpIEbmr8On7W0TktmJAu github.com/goccy/go-json v0.10.5 h1:Fq85nIqj+gXn/S5ahsiTlK3TmC85qgirsdTP/+DeaC4= github.com/goccy/go-json v0.10.5/go.mod h1:oq7eo15ShAhp70Anwd5lgX2pLfOS3QCiwU/PULtXL6M= github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= +github.com/google/flatbuffers v25.2.10+incompatible h1:F3vclr7C3HpB1k9mxCGRMXq6FdUalZ6H/pNX4FP1v0Q= +github.com/google/flatbuffers v25.2.10+incompatible/go.mod h1:1AeVuKshWv4vARoZatz6mlQ0JxURH0Kv5+zNeJKJCa8= github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= @@ -152,6 +154,7 @@ github.com/robfig/cron/v3 v3.0.1 h1:WdRxkvbJztn8LMz/QEvLN5sBU+xKpSqwwUO1Pjr4qDs= github.com/robfig/cron/v3 v3.0.1/go.mod h1:eQICP3HwyT7UooqI/z+Ov+PtYAWygg1TEWWzGIFLtro= github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ= github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc= +github.com/rs/xid v1.6.0 h1:fV591PaemRlL6JfRxGDEPl69wICngIQ3shQtzfy2gxU= github.com/rs/xid v1.6.0/go.mod h1:7XoLgs4eV+QndskICGsho+ADou8ySMSjJKDIan90Nz0= github.com/rs/zerolog v1.34.0 h1:k43nTLIwcTVQAncfCw4KZ2VY6ukYoZaBPNOE8txlOeY= github.com/rs/zerolog v1.34.0/go.mod h1:bJsvje4Z08ROH4Nhs5iH600c3IkWhwp44iRc54W6wYQ= diff --git a/hidrpc.go b/hidrpc.go index 64018a0e..12824ac6 100644 --- a/hidrpc.go +++ b/hidrpc.go @@ -6,6 +6,7 @@ import ( "github.com/jetkvm/kvm/internal/hidrpc" "github.com/jetkvm/kvm/internal/usbgadget" + "github.com/rs/zerolog" ) func handleHidRPCMessage(message hidrpc.Message, session *Session) { @@ -75,7 +76,9 @@ func onHidMessage(msg hidQueueMessage, session *Session) { return } - scopedLogger = scopedLogger.With().Str("descr", message.String()).Logger() + if scopedLogger.GetLevel() <= zerolog.DebugLevel { + scopedLogger = scopedLogger.With().Str("descr", message.String()).Logger() + } t := time.Now() @@ -166,7 +169,9 @@ func (s *Session) reportHidRPCKeyboardLedState(state usbgadget.KeyboardState) { func (s *Session) reportHidRPCKeysDownState(state usbgadget.KeysDownState) { if !s.hidRPCAvailable { + usbLogger.Debug().Interface("state", state).Msg("reporting keys down state") writeJSONRPCEvent("keysDownState", state, s) } + usbLogger.Debug().Interface("state", state).Msg("reporting keys down state, calling reportHidRPC") reportHidRPC(state, s) } diff --git a/internal/usbgadget/hid_keyboard.go b/internal/usbgadget/hid_keyboard.go index 76e5a125..53e232c0 100644 --- a/internal/usbgadget/hid_keyboard.go +++ b/internal/usbgadget/hid_keyboard.go @@ -6,6 +6,9 @@ import ( "fmt" "os" "time" + + "github.com/rs/xid" + "github.com/rs/zerolog" ) var keyboardConfig = gadgetConfigItem{ @@ -188,7 +191,7 @@ func (u *UsbGadget) scheduleAutoRelease(key byte) { } u.kbdAutoReleaseTimer = time.AfterFunc(autoReleaseKeyboardInterval, func() { - u.performAutoRelease(key) + u.performAutoRelease() }) } @@ -216,10 +219,12 @@ func (u *UsbGadget) DelayAutoRelease() { u.log.Trace().Msg("auto-release timer reset") } -func (u *UsbGadget) performAutoRelease(key byte) { +func (u *UsbGadget) performAutoRelease() { u.log.Trace().Msg("performing autoRelease") u.kbdAutoReleaseLock.Lock() - defer unlockWithLog(&u.kbdAutoReleaseLock, u.log, "autoRelease performed") + defer unlockWithLog(&u.kbdAutoReleaseLock, u.log, "autoRelease unlocked") + + key := u.kbdAutoReleaseLastKey select { case <-u.keyboardStateCtx.Done(): @@ -228,7 +233,8 @@ func (u *UsbGadget) performAutoRelease(key byte) { } // we just reset the keyboard state to 0 no matter what - _, err := u.keypressReport(0, false, false) + u.log.Trace().Uint8("key", key).Msg("auto-releasing keyboard key") + _, err := u.keypressReport(key, false, false) if err != nil { u.log.Warn().Uint8("key", key).Msg("failed to auto-release keyboard key") } @@ -399,12 +405,20 @@ var KeyCodeToMaskMap = map[byte]byte{ func (u *UsbGadget) keypressReportNonThreadSafe(key byte, press bool, autoRelease bool) (KeysDownState, error) { defer u.resetUserInputTime() + l := u.log.With().Uint8("key", key).Bool("press", press).Bool("autoRelease", autoRelease).Logger() + if l.GetLevel() <= zerolog.DebugLevel { + requestID := xid.New() + l = l.With().Str("requestID", requestID.String()).Logger() + } + // IMPORTANT: This code parallels the logic in the kernel's hid-gadget driver // for handling key presses and releases. It ensures that the USB gadget // behaves similarly to a real USB HID keyboard. This logic is paralleled // in the client/browser-side code in useKeyboard.ts so make sure to keep // them in sync. - var state = u.keysDownState + var state = u.GetKeysDownState() + l.Trace().Interface("state", state).Msg("got keys down state") + modifier := state.Modifier keys := append([]byte(nil), state.Keys...) @@ -444,37 +458,45 @@ func (u *UsbGadget) keypressReportNonThreadSafe(key byte, press bool, autoReleas // If we reach here it means we didn't find an empty slot or the key in the buffer if overrun { if press { - u.log.Error().Uint8("key", key).Msg("keyboard buffer overflow, key not added") + l.Error().Msg("keyboard buffer overflow, key not added") // Fill all key slots with ErrorRollOver (0x01) to indicate overflow for i := range keys { keys[i] = hidErrorRollOver } } else { // If we are releasing a key, and we didn't find it in a slot, who cares? - u.log.Warn().Uint8("key", key).Msg("key not found in buffer, nothing to release") + l.Warn().Msg("key not found in buffer, nothing to release") } } } + if l.GetLevel() <= zerolog.DebugLevel { + l = l.With().Uint8("modifier", modifier).Uints8("keys", keys).Logger() + } + + l.Trace().Msg("writing keypress report to hidg0") + err := u.keyboardWriteHidFile(modifier, keys) if err != nil { - u.log.Warn().Uint8("modifier", modifier).Uints8("keys", keys).Msg("Could not write keypress report to hidg0") + l.Warn().Msg("Could not write keypress report to hidg0") } + l.Trace().Msg("keypress report written to hidg0") + if press { { - u.log.Trace().Msg("acquiring kbdAutoReleaseLock to update last key") + l.Trace().Msg("acquiring kbdAutoReleaseLock to update last key") u.kbdAutoReleaseLock.Lock() u.kbdAutoReleaseLastKey = key unlockWithLog(&u.kbdAutoReleaseLock, u.log, "last key updated") } if autoRelease { - u.cancelAutoRelease() + u.scheduleAutoRelease(key) } } else { if autoRelease { - u.scheduleAutoRelease(key) + u.cancelAutoRelease() } } From 5cac4c46048bd7ccbd778bc4b8f42c020ad5af95 Mon Sep 17 00:00:00 2001 From: Adam Shiervani Date: Tue, 16 Sep 2025 01:48:13 +0200 Subject: [PATCH 15/20] refactor: simplify HID RPC keyboard input handling and improve key state management - Updated `handleHidRPCKeyboardInput` to return errors directly instead of keys down state. - Refactored `rpcKeyboardReport` and `rpcKeypressReport` to return errors instead of states. - Introduced a queue for managing key down state updates in the `Session` struct to prevent input handling stalls. - Adjusted the `UpdateKeysDown` method to handle state changes more efficiently. - Removed unnecessary logging and commented-out code for clarity. --- hidrpc.go | 20 +-- internal/usbgadget/hid_keyboard.go | 163 ++++++++-------------- jsonrpc.go | 2 - ui/src/components/WebRTCVideo.tsx | 3 +- ui/src/components/popovers/PasteModal.tsx | 78 ++++------- ui/src/hooks/useHidRpc.ts | 129 +++++++++-------- usb.go | 6 +- webrtc.go | 30 ++++ 8 files changed, 198 insertions(+), 233 deletions(-) diff --git a/hidrpc.go b/hidrpc.go index 12824ac6..53d758d8 100644 --- a/hidrpc.go +++ b/hidrpc.go @@ -25,11 +25,7 @@ func handleHidRPCMessage(message hidrpc.Message, session *Session) { } session.hidRPCAvailable = true case hidrpc.TypeKeypressReport, hidrpc.TypeKeyboardReport: - keysDownState, err := handleHidRPCKeyboardInput(message) - if keysDownState != nil { - session.reportHidRPCKeysDownState(*keysDownState) - } - rpcErr = err + rpcErr = handleHidRPCKeyboardInput(message) case hidrpc.TypeKeypressKeepAliveReport: gadget.DelayAutoRelease() case hidrpc.TypePointerReport: @@ -95,27 +91,25 @@ func onHidMessage(msg hidQueueMessage, session *Session) { } } -func handleHidRPCKeyboardInput(message hidrpc.Message) (*usbgadget.KeysDownState, error) { +func handleHidRPCKeyboardInput(message hidrpc.Message) error { switch message.Type() { case hidrpc.TypeKeypressReport: keypressReport, err := message.KeypressReport() if err != nil { logger.Warn().Err(err).Msg("failed to get keypress report") - return nil, err + return err } - keysDownState, rpcError := rpcKeypressReport(keypressReport.Key, keypressReport.Press) - return &keysDownState, rpcError + return rpcKeypressReport(keypressReport.Key, keypressReport.Press) case hidrpc.TypeKeyboardReport: keyboardReport, err := message.KeyboardReport() if err != nil { logger.Warn().Err(err).Msg("failed to get keyboard report") - return nil, err + return err } - keysDownState, rpcError := rpcKeyboardReport(keyboardReport.Modifier, keyboardReport.Keys) - return &keysDownState, rpcError + return rpcKeyboardReport(keyboardReport.Modifier, keyboardReport.Keys) } - return nil, fmt.Errorf("unknown HID RPC message type: %d", message.Type()) + return fmt.Errorf("unknown HID RPC message type: %d", message.Type()) } func reportHidRPC(params any, session *Session) { diff --git a/internal/usbgadget/hid_keyboard.go b/internal/usbgadget/hid_keyboard.go index 53e232c0..fa3b2a0c 100644 --- a/internal/usbgadget/hid_keyboard.go +++ b/internal/usbgadget/hid_keyboard.go @@ -5,6 +5,7 @@ import ( "context" "fmt" "os" + "sync" "time" "github.com/rs/xid" @@ -153,36 +154,11 @@ func (u *UsbGadget) GetKeysDownState() KeysDownState { return u.keysDownState } -func (u *UsbGadget) updateKeyDownState(state KeysDownState) { - u.log.Trace().Interface("old", u.keysDownState).Interface("new", state).Msg("acquiring keyboardStateLock for updateKeyDownState") - - // this is intentional to unlock keyboard state lock before onKeysDownChange callback - { - u.keyboardStateLock.Lock() - defer u.keyboardStateLock.Unlock() - - if u.keysDownState.Modifier == state.Modifier && - bytes.Equal(u.keysDownState.Keys, state.Keys) { - return // No change in key down state - } - - u.log.Trace().Interface("old", u.keysDownState).Interface("new", state).Msg("keysDownState updated") - u.keysDownState = state - } - - if u.onKeysDownChange != nil { - u.log.Trace().Interface("state", state).Msg("calling onKeysDownChange") - (*u.onKeysDownChange)(state) - u.log.Trace().Interface("state", state).Msg("onKeysDownChange called") - } -} - func (u *UsbGadget) SetOnKeysDownChange(f func(state KeysDownState)) { u.onKeysDownChange = &f } -func (u *UsbGadget) scheduleAutoRelease(key byte) { - u.log.Trace().Msg("scheduling autoRelease") +func (u *UsbGadget) scheduleAutoRelease() { u.kbdAutoReleaseLock.Lock() defer unlockWithLog(&u.kbdAutoReleaseLock, u.log, "autoRelease scheduled") @@ -196,7 +172,6 @@ func (u *UsbGadget) scheduleAutoRelease(key byte) { } func (u *UsbGadget) cancelAutoRelease() { - u.log.Trace().Msg("cancelling autoRelease") u.kbdAutoReleaseLock.Lock() defer unlockWithLog(&u.kbdAutoReleaseLock, u.log, "autoRelease cancelled") @@ -206,7 +181,6 @@ func (u *UsbGadget) cancelAutoRelease() { } func (u *UsbGadget) DelayAutoRelease() { - u.log.Trace().Msg("delaying autoRelease") u.kbdAutoReleaseLock.Lock() defer unlockWithLog(&u.kbdAutoReleaseLock, u.log, "autoRelease delayed") @@ -215,33 +189,37 @@ func (u *UsbGadget) DelayAutoRelease() { } u.kbdAutoReleaseTimer.Reset(autoReleaseKeyboardInterval) - - u.log.Trace().Msg("auto-release timer reset") } func (u *UsbGadget) performAutoRelease() { - u.log.Trace().Msg("performing autoRelease") - u.kbdAutoReleaseLock.Lock() - defer unlockWithLog(&u.kbdAutoReleaseLock, u.log, "autoRelease unlocked") - - key := u.kbdAutoReleaseLastKey - select { case <-u.keyboardStateCtx.Done(): return default: } - // we just reset the keyboard state to 0 no matter what - u.log.Trace().Uint8("key", key).Msg("auto-releasing keyboard key") - _, err := u.keypressReport(key, false, false) - if err != nil { - u.log.Warn().Uint8("key", key).Msg("failed to auto-release keyboard key") + u.kbdAutoReleaseLock.Lock() + + key := u.kbdAutoReleaseLastKey + + // Skip if already released + state := u.GetKeysDownState() + alreadyReleased := true + for i := range state.Keys { + if state.Keys[i] == key { + alreadyReleased = false + break + } + } + + if alreadyReleased { + return } u.kbdAutoReleaseTimer = nil + u.kbdAutoReleaseLock.Unlock() - u.log.Trace().Uint8("key", key).Msg("auto release performed") + u.keypressReport(key, false) // autoRelease the ket } func (u *UsbGadget) listenKeyboardEvents() { @@ -313,7 +291,11 @@ func (u *UsbGadget) OpenKeyboardHidFile() error { return u.openKeyboardHidFile() } +var keyboardWriteHidFileLock sync.Mutex + func (u *UsbGadget) keyboardWriteHidFile(modifier byte, keys []byte) error { + keyboardWriteHidFileLock.Lock() + defer keyboardWriteHidFileLock.Unlock() if err := u.openKeyboardHidFile(); err != nil { return err } @@ -329,7 +311,7 @@ func (u *UsbGadget) keyboardWriteHidFile(modifier byte, keys []byte) error { return nil } -func (u *UsbGadget) UpdateKeysDown(modifier byte, keys []byte) KeysDownState { +func (u *UsbGadget) UpdateKeysDown(modifier byte, keys []byte) { // if we just reported an error roll over, we should clear the keys if keys[0] == hidErrorRollOver { for i := range keys { @@ -337,17 +319,29 @@ func (u *UsbGadget) UpdateKeysDown(modifier byte, keys []byte) KeysDownState { } } - downState := KeysDownState{ + state := KeysDownState{ Modifier: modifier, Keys: []byte(keys[:]), } - u.updateKeyDownState(downState) - return downState + + u.keyboardStateLock.Lock() + + if u.keysDownState.Modifier == state.Modifier && + bytes.Equal(u.keysDownState.Keys, state.Keys) { + u.keyboardStateLock.Unlock() + return // No change in key down state + } + + u.keysDownState = state + u.keyboardStateLock.Unlock() + + if u.onKeysDownChange != nil { + (*u.onKeysDownChange)(state) // this enques to the outgoing hidrpc queue via usb.go → currentSession.enqueueKeysDownState(...) + } + return } -func (u *UsbGadget) KeyboardReport(modifier byte, keys []byte) (KeysDownState, error) { - u.keyboardLock.Lock() - defer u.keyboardLock.Unlock() +func (u *UsbGadget) KeyboardReport(modifier byte, keys []byte) error { defer u.resetUserInputTime() if len(keys) > hidKeyBufferSize { @@ -362,7 +356,8 @@ func (u *UsbGadget) KeyboardReport(modifier byte, keys []byte) (KeysDownState, e u.log.Warn().Uint8("modifier", modifier).Uints8("keys", keys).Msg("Could not write keyboard report to hidg0") } - return u.UpdateKeysDown(modifier, keys), err + u.UpdateKeysDown(modifier, keys) + return err } const ( @@ -402,10 +397,10 @@ var KeyCodeToMaskMap = map[byte]byte{ RightSuper: ModifierMaskRightSuper, } -func (u *UsbGadget) keypressReportNonThreadSafe(key byte, press bool, autoRelease bool) (KeysDownState, error) { +func (u *UsbGadget) keypressReport(key byte, press bool) error { defer u.resetUserInputTime() - l := u.log.With().Uint8("key", key).Bool("press", press).Bool("autoRelease", autoRelease).Logger() + l := u.log.With().Uint8("key", key).Bool("press", press).Logger() if l.GetLevel() <= zerolog.DebugLevel { requestID := xid.New() l = l.With().Str("requestID", requestID.String()).Logger() @@ -470,64 +465,22 @@ func (u *UsbGadget) keypressReportNonThreadSafe(key byte, press bool, autoReleas } } - if l.GetLevel() <= zerolog.DebugLevel { - l = l.With().Uint8("modifier", modifier).Uints8("keys", keys).Logger() - } - - l.Trace().Msg("writing keypress report to hidg0") - err := u.keyboardWriteHidFile(modifier, keys) - if err != nil { - l.Warn().Msg("Could not write keypress report to hidg0") - } + u.UpdateKeysDown(modifier, keys) + return err +} - l.Trace().Msg("keypress report written to hidg0") +func (u *UsbGadget) KeypressReport(key byte, press bool) error { + u.kbdAutoReleaseLock.Lock() + u.kbdAutoReleaseLastKey = key + u.kbdAutoReleaseLock.Unlock() if press { - { - l.Trace().Msg("acquiring kbdAutoReleaseLock to update last key") - u.kbdAutoReleaseLock.Lock() - u.kbdAutoReleaseLastKey = key - unlockWithLog(&u.kbdAutoReleaseLock, u.log, "last key updated") - } - - if autoRelease { - u.scheduleAutoRelease(key) - } + u.scheduleAutoRelease() } else { - if autoRelease { - u.cancelAutoRelease() - } + u.cancelAutoRelease() } - return u.UpdateKeysDown(modifier, keys), err -} - -type keypressReportResult struct { - KeysDownState KeysDownState - Error error -} - -func (u *UsbGadget) keypressReport(key byte, press bool, autoRelease bool) (KeysDownState, error) { - u.keyboardLock.Lock() - defer u.keyboardLock.Unlock() - - r := make(chan keypressReportResult) - go func() { - state, err := u.keypressReportNonThreadSafe(key, press, autoRelease) - r <- keypressReportResult{KeysDownState: state, Error: err} - }() - - select { - case <-time.After(1 * time.Second): - u.log.Warn().Msg("keypressReport timed out, possibly stuck") - return u.keysDownState, fmt.Errorf("keypressReport timed out, possibly stuck") - case ret := <-r: - u.log.Debug().Msg("keypressReport handled") - return ret.KeysDownState, ret.Error - } -} - -func (u *UsbGadget) KeypressReport(key byte, press bool) (KeysDownState, error) { - return u.keypressReport(key, press, true) + err := u.keypressReport(key, press) + return err } diff --git a/jsonrpc.go b/jsonrpc.go index ff3a4b12..a4705a37 100644 --- a/jsonrpc.go +++ b/jsonrpc.go @@ -1059,9 +1059,7 @@ var rpcHandlers = map[string]RPCHandler{ "getNetworkSettings": {Func: rpcGetNetworkSettings}, "setNetworkSettings": {Func: rpcSetNetworkSettings, Params: []string{"settings"}}, "renewDHCPLease": {Func: rpcRenewDHCPLease}, - "keyboardReport": {Func: rpcKeyboardReport, Params: []string{"modifier", "keys"}}, "getKeyboardLedState": {Func: rpcGetKeyboardLedState}, - "keypressReport": {Func: rpcKeypressReport, Params: []string{"key", "press"}}, "getKeyDownState": {Func: rpcGetKeysDownState}, "absMouseReport": {Func: rpcAbsMouseReport, Params: []string{"x", "y", "buttons"}}, "relMouseReport": {Func: rpcRelMouseReport, Params: []string{"dx", "dy", "buttons"}}, diff --git a/ui/src/components/WebRTCVideo.tsx b/ui/src/components/WebRTCVideo.tsx index bc6897ea..64452bf8 100644 --- a/ui/src/components/WebRTCVideo.tsx +++ b/ui/src/components/WebRTCVideo.tsx @@ -190,7 +190,7 @@ export default function WebRTCVideo() { if (!isFullscreenEnabled || !videoElm.current) return; // per https://wicg.github.io/keyboard-lock/#system-key-press-handler - // If keyboard lock is activated after fullscreen is already in effect, then the user my + // If keyboard lock is activated after fullscreen is already in effect, then the user my // see multiple messages about how to exit fullscreen. For this reason, we recommend that // developers call lock() before they enter fullscreen: await requestKeyboardLock(); @@ -237,6 +237,7 @@ export default function WebRTCVideo() { const keyDownHandler = useCallback( (e: KeyboardEvent) => { e.preventDefault(); + if (e.repeat) return; const code = getAdjustedKeyCode(e); const hidKey = keys[code]; diff --git a/ui/src/components/popovers/PasteModal.tsx b/ui/src/components/popovers/PasteModal.tsx index 077759b7..fc11476f 100644 --- a/ui/src/components/popovers/PasteModal.tsx +++ b/ui/src/components/popovers/PasteModal.tsx @@ -9,20 +9,12 @@ import { TextAreaWithLabel } from "@components/TextArea"; import { SettingsPageHeader } from "@components/SettingsPageheader"; import { JsonRpcResponse, useJsonRpc } from "@/hooks/useJsonRpc"; import { useHidStore, useRTCStore, useUiStore, useSettingsStore } from "@/hooks/stores"; -import { keys, modifiers } from "@/keyboardMappings"; -import { KeyStroke } from "@/keyboardLayouts"; +import { keys } from "@/keyboardMappings"; import useKeyboardLayout from "@/hooks/useKeyboardLayout"; -import notifications from "@/notifications"; -const hidKeyboardPayload = (modifier: number, keys: number[]) => { - return { modifier, keys }; -}; +import useKeyboard from "../../hooks/useKeyboard"; + -const modifierCode = (shift?: boolean, altRight?: boolean) => { - return (shift ? modifiers.ShiftLeft : 0) - | (altRight ? modifiers.AltRight : 0) -} -const noModifier = 0 export default function PasteModal() { const TextAreaRef = useRef(null); @@ -34,9 +26,9 @@ export default function PasteModal() { const [invalidChars, setInvalidChars] = useState([]); const close = useClose(); - + const { handleKeyPress } = useKeyboard(); const { setKeyboardLayout } = useSettingsStore(); - const { selectedKeyboard } = useKeyboardLayout(); + const { selectedKeyboard } = useKeyboardLayout(); useEffect(() => { send("getKeyboardLayout", {}, (resp: JsonRpcResponse) => { @@ -58,51 +50,29 @@ export default function PasteModal() { if (rpcDataChannel?.readyState !== "open" || !TextAreaRef.current) return; if (!selectedKeyboard) return; - const text = TextAreaRef.current.value; - try { - for (const char of text) { - const keyprops = selectedKeyboard.chars[char]; - if (!keyprops) continue; - - const { key, shift, altRight, deadKey, accentKey } = keyprops; - if (!key) continue; - - // if this is an accented character, we need to send that accent FIRST - if (accentKey) { - await sendKeystroke({modifier: modifierCode(accentKey.shift, accentKey.altRight), keys: [ keys[accentKey.key] ] }) - } - - // now send the actual key - await sendKeystroke({ modifier: modifierCode(shift, altRight), keys: [ keys[key] ]}); - - // if what was requested was a dead key, we need to send an unmodified space to emit - // just the accent character - if (deadKey) { - await sendKeystroke({ modifier: noModifier, keys: [ keys["Space"] ] }); - } - - // now send a message with no keys down to "release" the keys - await sendKeystroke({ modifier: 0, keys: [] }); + for (let i = 0; i < 5; i++) { + for (let i = 0; i < 26; i++) { + handleKeyPress(keys[`Key${String.fromCharCode(65 + i)}`], true); + await new Promise(resolve => setTimeout(resolve, 50)); + handleKeyPress(keys[`Key${String.fromCharCode(65 + i)}`], false); + await new Promise(resolve => setTimeout(resolve, 50)); } - } catch (error) { - console.error("Failed to paste text:", error); - notifications.error("Failed to paste text"); + await new Promise(resolve => setTimeout(resolve, 50)); + handleKeyPress(keys.Enter, true); + await new Promise(resolve => setTimeout(resolve, 50)); + handleKeyPress(keys.Enter, false); + await new Promise(resolve => setTimeout(resolve, 50)); } - async function sendKeystroke(stroke: KeyStroke) { - await new Promise((resolve, reject) => { - send( - "keyboardReport", - hidKeyboardPayload(stroke.modifier, stroke.keys), - params => { - if ("error" in params) return reject(params.error); - resolve(); - } - ); - }); - } - }, [selectedKeyboard, rpcDataChannel?.readyState, send, setDisableVideoFocusTrap, setPasteModeEnabled]); + + // for (let index = 0; index < 2; index++) { + // handleKeyPress(keys.KeyA, true); + // await new Promise(resolve => setTimeout(resolve, 3000)); + // handleKeyPress(keys.KeyA, false); + // } + + }, [setPasteModeEnabled, setDisableVideoFocusTrap, rpcDataChannel?.readyState, selectedKeyboard, handleKeyPress]); useEffect(() => { if (TextAreaRef.current) { diff --git a/ui/src/hooks/useHidRpc.ts b/ui/src/hooks/useHidRpc.ts index 0670acb8..5480148f 100644 --- a/ui/src/hooks/useHidRpc.ts +++ b/ui/src/hooks/useHidRpc.ts @@ -36,11 +36,16 @@ export function useHidRpc(onHidRpcMessage?: (payload: RpcMessage) => void) { }, [rpcHidChannel, rpcHidProtocolVersion]); const rpcHidUnreliableReady = useMemo(() => { - return rpcHidUnreliableChannel?.readyState === "open" && rpcHidProtocolVersion !== null; + return ( + rpcHidUnreliableChannel?.readyState === "open" && rpcHidProtocolVersion !== null + ); }, [rpcHidUnreliableChannel, rpcHidProtocolVersion]); const rpcHidUnreliableNonOrderedReady = useMemo(() => { - return rpcHidUnreliableNonOrderedChannel?.readyState === "open" && rpcHidProtocolVersion !== null; + return ( + rpcHidUnreliableNonOrderedChannel?.readyState === "open" && + rpcHidProtocolVersion !== null + ); }, [rpcHidUnreliableNonOrderedChannel, rpcHidProtocolVersion]); const rpcHidStatus = useMemo(() => { @@ -50,41 +55,52 @@ export function useHidRpc(onHidRpcMessage?: (payload: RpcMessage) => void) { return `ready (v${rpcHidProtocolVersion}${rpcHidUnreliableReady ? "+u" : ""})`; }, [rpcHidChannel, rpcHidUnreliableReady, rpcHidProtocolVersion]); - const sendMessage = useCallback((message: RpcMessage, { ignoreHandshakeState, useUnreliableChannel, requireOrdered = true }: sendMessageParams = {}) => { - if (rpcHidChannel?.readyState !== "open") return; - if (!rpcHidReady && !ignoreHandshakeState) return; + const sendMessage = useCallback( + ( + message: RpcMessage, + { + ignoreHandshakeState, + useUnreliableChannel, + requireOrdered = true, + }: sendMessageParams = {}, + ) => { + if (rpcHidChannel?.readyState !== "open") return; + if (!rpcHidReady && !ignoreHandshakeState) return; - let data: Uint8Array | undefined; - try { - data = message.marshal(); - } catch (e) { - console.error("Failed to send HID RPC message", e); - } - if (!data) return; - - if (useUnreliableChannel) { - if (requireOrdered && rpcHidUnreliableReady) { - rpcHidUnreliableChannel?.send(data as unknown as ArrayBuffer); - } else if (!requireOrdered && rpcHidUnreliableNonOrderedReady) { - rpcHidUnreliableNonOrderedChannel?.send(data as unknown as ArrayBuffer); + let data: Uint8Array | undefined; + try { + data = message.marshal(); + } catch (e) { + console.error("Failed to send HID RPC message", e); } - return; - } + if (!data) return; - rpcHidChannel?.send(data as unknown as ArrayBuffer); - }, [ - rpcHidChannel, - rpcHidUnreliableChannel, - rpcHidUnreliableNonOrderedChannel, - rpcHidReady, - rpcHidUnreliableReady, - rpcHidUnreliableNonOrderedReady, - ]); + if (useUnreliableChannel) { + if (requireOrdered && rpcHidUnreliableReady) { + rpcHidUnreliableChannel?.send(data as unknown as ArrayBuffer); + } else if (!requireOrdered && rpcHidUnreliableNonOrderedReady) { + rpcHidUnreliableNonOrderedChannel?.send(data as unknown as ArrayBuffer); + } + return; + } + + rpcHidChannel?.send(data as unknown as ArrayBuffer); + }, + [ + rpcHidChannel, + rpcHidUnreliableChannel, + rpcHidUnreliableNonOrderedChannel, + rpcHidReady, + rpcHidUnreliableReady, + rpcHidUnreliableNonOrderedReady, + ], + ); const reportKeyboardEvent = useCallback( (keys: number[], modifier: number) => { sendMessage(new KeyboardReportMessage(keys, modifier)); - }, [sendMessage], + }, + [sendMessage], ); const reportKeypressEvent = useCallback( @@ -96,7 +112,9 @@ export function useHidRpc(onHidRpcMessage?: (payload: RpcMessage) => void) { const reportAbsMouseEvent = useCallback( (x: number, y: number, buttons: number) => { - sendMessage(new PointerReportMessage(x, y, buttons), { useUnreliableChannel: true }); + sendMessage(new PointerReportMessage(x, y, buttons), { + useUnreliableChannel: true, + }); }, [sendMessage], ); @@ -109,7 +127,7 @@ export function useHidRpc(onHidRpcMessage?: (payload: RpcMessage) => void) { ); const reportKeypressKeepAlive = useCallback(() => { - sendMessage(KEEPALIVE_MESSAGE, { useUnreliableChannel: true, requireOrdered: false }); + sendMessage(KEEPALIVE_MESSAGE); }, [sendMessage]); const sendHandshake = useCallback(() => { @@ -119,22 +137,25 @@ export function useHidRpc(onHidRpcMessage?: (payload: RpcMessage) => void) { sendMessage(new HandshakeMessage(HID_RPC_VERSION), { ignoreHandshakeState: true }); }, [rpcHidChannel, rpcHidProtocolVersion, sendMessage]); - const handleHandshake = useCallback((message: HandshakeMessage) => { - if (!message.version) { - console.error("Received handshake message without version", message); - return; - } + const handleHandshake = useCallback( + (message: HandshakeMessage) => { + if (!message.version) { + console.error("Received handshake message without version", message); + return; + } - if (message.version > HID_RPC_VERSION) { - // we assume that the UI is always using the latest version of the HID RPC protocol - // so we can't support this - // TODO: use capabilities to determine rather than version number - console.error("Server is using a newer HID RPC version than the client", message); - return; - } + if (message.version > HID_RPC_VERSION) { + // we assume that the UI is always using the latest version of the HID RPC protocol + // so we can't support this + // TODO: use capabilities to determine rather than version number + console.error("Server is using a newer HID RPC version than the client", message); + return; + } - setRpcHidProtocolVersion(message.version); - }, [setRpcHidProtocolVersion]); + setRpcHidProtocolVersion(message.version); + }, + [setRpcHidProtocolVersion], + ); useEffect(() => { if (!rpcHidChannel) return; @@ -186,15 +207,13 @@ export function useHidRpc(onHidRpcMessage?: (payload: RpcMessage) => void) { rpcHidChannel.removeEventListener("close", closeHandler); rpcHidChannel.removeEventListener("open", openHandler); }; - }, - [ - rpcHidChannel, - onHidRpcMessage, - setRpcHidProtocolVersion, - sendHandshake, - handleHandshake, - ], - ); + }, [ + rpcHidChannel, + onHidRpcMessage, + setRpcHidProtocolVersion, + sendHandshake, + handleHandshake, + ]); return { reportKeyboardEvent, diff --git a/usb.go b/usb.go index 131cd517..f63a6aab 100644 --- a/usb.go +++ b/usb.go @@ -33,7 +33,7 @@ func initUsbGadget() { gadget.SetOnKeysDownChange(func(state usbgadget.KeysDownState) { if currentSession != nil { - currentSession.reportHidRPCKeysDownState(state) + currentSession.enqueueKeysDownState(state) } }) @@ -43,11 +43,11 @@ func initUsbGadget() { } } -func rpcKeyboardReport(modifier byte, keys []byte) (usbgadget.KeysDownState, error) { +func rpcKeyboardReport(modifier byte, keys []byte) error { return gadget.KeyboardReport(modifier, keys) } -func rpcKeypressReport(key byte, press bool) (usbgadget.KeysDownState, error) { +func rpcKeypressReport(key byte, press bool) error { return gadget.KeypressReport(key, press) } diff --git a/webrtc.go b/webrtc.go index 333a58b8..9593e627 100644 --- a/webrtc.go +++ b/webrtc.go @@ -13,6 +13,7 @@ import ( "github.com/gin-gonic/gin" "github.com/jetkvm/kvm/internal/hidrpc" "github.com/jetkvm/kvm/internal/logging" + "github.com/jetkvm/kvm/internal/usbgadget" "github.com/pion/webrtc/v4" "github.com/rs/zerolog" ) @@ -30,6 +31,8 @@ type Session struct { hidRPCAvailable bool hidQueueLock sync.Mutex hidQueue []chan hidQueueMessage + + keysDownStateQueue chan usbgadget.KeysDownState } type hidQueueMessage struct { @@ -96,6 +99,32 @@ func (s *Session) handleQueues(index int) { } } +const keysDownStateQueueSize = 256 + +func (s *Session) initKeysDownStateQueue() { + // serialise outbound key state reports so unreliable links can't stall input handling + s.keysDownStateQueue = make(chan usbgadget.KeysDownState, keysDownStateQueueSize) + go s.handleKeysDownStateQueue() +} + +func (s *Session) handleKeysDownStateQueue() { + for state := range s.keysDownStateQueue { + s.reportHidRPCKeysDownState(state) + } +} + +func (s *Session) enqueueKeysDownState(state usbgadget.KeysDownState) { + if s == nil || s.keysDownStateQueue == nil { + return + } + + select { + case s.keysDownStateQueue <- state: + default: + hidRPCLogger.Warn().Msg("dropping keys down state update; queue full") + } +} + func getOnHidMessageHandler(session *Session, scopedLogger *zerolog.Logger, channel string) func(msg webrtc.DataChannelMessage) { return func(msg webrtc.DataChannelMessage) { l := scopedLogger.With(). @@ -181,6 +210,7 @@ func newSession(config SessionConfig) (*Session, error) { session := &Session{peerConnection: peerConnection} session.rpcQueue = make(chan webrtc.DataChannelMessage, 256) session.initQueues() + session.initKeysDownStateQueue() go func() { for msg := range session.rpcQueue { From 9593a81cb840fba074fdca0b8a122ac98826c657 Mon Sep 17 00:00:00 2001 From: Adam Shiervani Date: Tue, 16 Sep 2025 12:28:57 +0200 Subject: [PATCH 16/20] refactor: enhance keyboard auto-release functionality and key state management --- internal/usbgadget/hid_keyboard.go | 76 ++++++++++++---------- internal/usbgadget/usbgadget.go | 46 ++++++------- ui/src/components/popovers/PasteModal.tsx | 78 ++++++++++++++++------- 3 files changed, 118 insertions(+), 82 deletions(-) diff --git a/internal/usbgadget/hid_keyboard.go b/internal/usbgadget/hid_keyboard.go index fa3b2a0c..5b6aa91e 100644 --- a/internal/usbgadget/hid_keyboard.go +++ b/internal/usbgadget/hid_keyboard.go @@ -158,25 +158,27 @@ func (u *UsbGadget) SetOnKeysDownChange(f func(state KeysDownState)) { u.onKeysDownChange = &f } -func (u *UsbGadget) scheduleAutoRelease() { +func (u *UsbGadget) scheduleAutoRelease(key byte) { u.kbdAutoReleaseLock.Lock() defer unlockWithLog(&u.kbdAutoReleaseLock, u.log, "autoRelease scheduled") - if u.kbdAutoReleaseTimer != nil { - u.kbdAutoReleaseTimer.Stop() + if u.kbdAutoReleaseTimers[key] != nil { + u.kbdAutoReleaseTimers[key].Stop() } - u.kbdAutoReleaseTimer = time.AfterFunc(autoReleaseKeyboardInterval, func() { - u.performAutoRelease() + u.kbdAutoReleaseTimers[key] = time.AfterFunc(autoReleaseKeyboardInterval, func() { + u.performAutoRelease(key) }) } -func (u *UsbGadget) cancelAutoRelease() { +func (u *UsbGadget) cancelAutoRelease(key byte) { u.kbdAutoReleaseLock.Lock() defer unlockWithLog(&u.kbdAutoReleaseLock, u.log, "autoRelease cancelled") - if u.kbdAutoReleaseTimer != nil { - u.kbdAutoReleaseTimer.Stop() + if timer := u.kbdAutoReleaseTimers[key]; timer != nil { + timer.Stop() + u.kbdAutoReleaseTimers[key] = nil + delete(u.kbdAutoReleaseTimers, key) } } @@ -184,27 +186,35 @@ func (u *UsbGadget) DelayAutoRelease() { u.kbdAutoReleaseLock.Lock() defer unlockWithLog(&u.kbdAutoReleaseLock, u.log, "autoRelease delayed") - if u.kbdAutoReleaseTimer == nil { + if u.kbdAutoReleaseTimers == nil { return } - u.kbdAutoReleaseTimer.Reset(autoReleaseKeyboardInterval) + for _, timer := range u.kbdAutoReleaseTimers { + if timer != nil { + timer.Reset(autoReleaseKeyboardInterval) + } + } } -func (u *UsbGadget) performAutoRelease() { - select { - case <-u.keyboardStateCtx.Done(): - return - default: - } - +func (u *UsbGadget) performAutoRelease(key byte) { u.kbdAutoReleaseLock.Lock() - key := u.kbdAutoReleaseLastKey + if u.kbdAutoReleaseTimers[key] == nil { + u.log.Warn().Uint8("key", key).Msg("autoRelease timer not found") + u.kbdAutoReleaseLock.Unlock() + return + } + + u.kbdAutoReleaseTimers[key].Stop() + u.kbdAutoReleaseTimers[key] = nil + delete(u.kbdAutoReleaseTimers, key) + u.kbdAutoReleaseLock.Unlock() // Skip if already released state := u.GetKeysDownState() alreadyReleased := true + for i := range state.Keys { if state.Keys[i] == key { alreadyReleased = false @@ -216,10 +226,7 @@ func (u *UsbGadget) performAutoRelease() { return } - u.kbdAutoReleaseTimer = nil - u.kbdAutoReleaseLock.Unlock() - - u.keypressReport(key, false) // autoRelease the ket + u.keypressReport(key, false) } func (u *UsbGadget) listenKeyboardEvents() { @@ -311,7 +318,7 @@ func (u *UsbGadget) keyboardWriteHidFile(modifier byte, keys []byte) error { return nil } -func (u *UsbGadget) UpdateKeysDown(modifier byte, keys []byte) { +func (u *UsbGadget) UpdateKeysDown(modifier byte, keys []byte) KeysDownState { // if we just reported an error roll over, we should clear the keys if keys[0] == hidErrorRollOver { for i := range keys { @@ -329,7 +336,7 @@ func (u *UsbGadget) UpdateKeysDown(modifier byte, keys []byte) { if u.keysDownState.Modifier == state.Modifier && bytes.Equal(u.keysDownState.Keys, state.Keys) { u.keyboardStateLock.Unlock() - return // No change in key down state + return state // No change in key down state } u.keysDownState = state @@ -338,7 +345,7 @@ func (u *UsbGadget) UpdateKeysDown(modifier byte, keys []byte) { if u.onKeysDownChange != nil { (*u.onKeysDownChange)(state) // this enques to the outgoing hidrpc queue via usb.go → currentSession.enqueueKeysDownState(...) } - return + return state } func (u *UsbGadget) KeyboardReport(modifier byte, keys []byte) error { @@ -397,7 +404,7 @@ var KeyCodeToMaskMap = map[byte]byte{ RightSuper: ModifierMaskRightSuper, } -func (u *UsbGadget) keypressReport(key byte, press bool) error { +func (u *UsbGadget) keypressReport(key byte, press bool) (KeysDownState, error) { defer u.resetUserInputTime() l := u.log.With().Uint8("key", key).Bool("press", press).Logger() @@ -466,21 +473,20 @@ func (u *UsbGadget) keypressReport(key byte, press bool) error { } err := u.keyboardWriteHidFile(modifier, keys) - u.UpdateKeysDown(modifier, keys) - return err + return u.UpdateKeysDown(modifier, keys), err } func (u *UsbGadget) KeypressReport(key byte, press bool) error { - u.kbdAutoReleaseLock.Lock() - u.kbdAutoReleaseLastKey = key - u.kbdAutoReleaseLock.Unlock() + state, err := u.keypressReport(key, press) + isRolledOver := state.Keys[0] == hidErrorRollOver - if press { - u.scheduleAutoRelease() + if isRolledOver { + u.cancelAutoRelease(key) + } else if press { + u.scheduleAutoRelease(key) } else { - u.cancelAutoRelease() + u.cancelAutoRelease(key) } - err := u.keypressReport(key, press) return err } diff --git a/internal/usbgadget/usbgadget.go b/internal/usbgadget/usbgadget.go index e491a78c..3db872cc 100644 --- a/internal/usbgadget/usbgadget.go +++ b/internal/usbgadget/usbgadget.go @@ -68,9 +68,8 @@ type UsbGadget struct { keyboardState byte // keyboard latched state (NumLock, CapsLock, ScrollLock, Compose, Kana) keysDownState KeysDownState // keyboard dynamic state (modifier keys and pressed keys) - kbdAutoReleaseLock sync.Mutex - kbdAutoReleaseTimer *time.Timer - kbdAutoReleaseLastKey byte + kbdAutoReleaseLock sync.Mutex + kbdAutoReleaseTimers map[byte]*time.Timer keyboardStateLock sync.Mutex keyboardStateCtx context.Context @@ -122,23 +121,24 @@ func newUsbGadget(name string, configMap map[string]gadgetConfigItem, enabledDev keyboardCtx, keyboardCancel := context.WithCancel(context.Background()) g := &UsbGadget{ - name: name, - kvmGadgetPath: path.Join(gadgetPath, name), - configC1Path: path.Join(gadgetPath, name, "configs/c.1"), - configMap: configMap, - customConfig: *config, - configLock: sync.Mutex{}, - keyboardLock: sync.Mutex{}, - absMouseLock: sync.Mutex{}, - relMouseLock: sync.Mutex{}, - txLock: sync.Mutex{}, - keyboardStateCtx: keyboardCtx, - keyboardStateCancel: keyboardCancel, - keyboardState: 0, - keysDownState: KeysDownState{Modifier: 0, Keys: []byte{0, 0, 0, 0, 0, 0}}, // must be initialized to hidKeyBufferSize (6) zero bytes - enabledDevices: *enabledDevices, - lastUserInput: time.Now(), - log: logger, + name: name, + kvmGadgetPath: path.Join(gadgetPath, name), + configC1Path: path.Join(gadgetPath, name, "configs/c.1"), + configMap: configMap, + customConfig: *config, + configLock: sync.Mutex{}, + keyboardLock: sync.Mutex{}, + absMouseLock: sync.Mutex{}, + relMouseLock: sync.Mutex{}, + txLock: sync.Mutex{}, + keyboardStateCtx: keyboardCtx, + keyboardStateCancel: keyboardCancel, + keyboardState: 0, + keysDownState: KeysDownState{Modifier: 0, Keys: []byte{0, 0, 0, 0, 0, 0}}, // must be initialized to hidKeyBufferSize (6) zero bytes + kbdAutoReleaseTimers: make(map[byte]*time.Timer), + enabledDevices: *enabledDevices, + lastUserInput: time.Now(), + log: logger, strictMode: config.strictMode, @@ -163,10 +163,10 @@ func (u *UsbGadget) Close() error { // Stop auto-release timer u.kbdAutoReleaseLock.Lock() - if u.kbdAutoReleaseTimer != nil { - u.kbdAutoReleaseTimer.Stop() - u.kbdAutoReleaseTimer = nil + for _, timer := range u.kbdAutoReleaseTimers { + timer.Stop() } + u.kbdAutoReleaseTimers = make(map[byte]*time.Timer) u.kbdAutoReleaseLock.Unlock() // Close HID files diff --git a/ui/src/components/popovers/PasteModal.tsx b/ui/src/components/popovers/PasteModal.tsx index fc11476f..077759b7 100644 --- a/ui/src/components/popovers/PasteModal.tsx +++ b/ui/src/components/popovers/PasteModal.tsx @@ -9,12 +9,20 @@ import { TextAreaWithLabel } from "@components/TextArea"; import { SettingsPageHeader } from "@components/SettingsPageheader"; import { JsonRpcResponse, useJsonRpc } from "@/hooks/useJsonRpc"; import { useHidStore, useRTCStore, useUiStore, useSettingsStore } from "@/hooks/stores"; -import { keys } from "@/keyboardMappings"; +import { keys, modifiers } from "@/keyboardMappings"; +import { KeyStroke } from "@/keyboardLayouts"; import useKeyboardLayout from "@/hooks/useKeyboardLayout"; +import notifications from "@/notifications"; -import useKeyboard from "../../hooks/useKeyboard"; - +const hidKeyboardPayload = (modifier: number, keys: number[]) => { + return { modifier, keys }; +}; +const modifierCode = (shift?: boolean, altRight?: boolean) => { + return (shift ? modifiers.ShiftLeft : 0) + | (altRight ? modifiers.AltRight : 0) +} +const noModifier = 0 export default function PasteModal() { const TextAreaRef = useRef(null); @@ -26,9 +34,9 @@ export default function PasteModal() { const [invalidChars, setInvalidChars] = useState([]); const close = useClose(); - const { handleKeyPress } = useKeyboard(); + const { setKeyboardLayout } = useSettingsStore(); - const { selectedKeyboard } = useKeyboardLayout(); + const { selectedKeyboard } = useKeyboardLayout(); useEffect(() => { send("getKeyboardLayout", {}, (resp: JsonRpcResponse) => { @@ -50,29 +58,51 @@ export default function PasteModal() { if (rpcDataChannel?.readyState !== "open" || !TextAreaRef.current) return; if (!selectedKeyboard) return; + const text = TextAreaRef.current.value; - for (let i = 0; i < 5; i++) { - for (let i = 0; i < 26; i++) { - handleKeyPress(keys[`Key${String.fromCharCode(65 + i)}`], true); - await new Promise(resolve => setTimeout(resolve, 50)); - handleKeyPress(keys[`Key${String.fromCharCode(65 + i)}`], false); - await new Promise(resolve => setTimeout(resolve, 50)); + try { + for (const char of text) { + const keyprops = selectedKeyboard.chars[char]; + if (!keyprops) continue; + + const { key, shift, altRight, deadKey, accentKey } = keyprops; + if (!key) continue; + + // if this is an accented character, we need to send that accent FIRST + if (accentKey) { + await sendKeystroke({modifier: modifierCode(accentKey.shift, accentKey.altRight), keys: [ keys[accentKey.key] ] }) + } + + // now send the actual key + await sendKeystroke({ modifier: modifierCode(shift, altRight), keys: [ keys[key] ]}); + + // if what was requested was a dead key, we need to send an unmodified space to emit + // just the accent character + if (deadKey) { + await sendKeystroke({ modifier: noModifier, keys: [ keys["Space"] ] }); + } + + // now send a message with no keys down to "release" the keys + await sendKeystroke({ modifier: 0, keys: [] }); } - await new Promise(resolve => setTimeout(resolve, 50)); - handleKeyPress(keys.Enter, true); - await new Promise(resolve => setTimeout(resolve, 50)); - handleKeyPress(keys.Enter, false); - await new Promise(resolve => setTimeout(resolve, 50)); + } catch (error) { + console.error("Failed to paste text:", error); + notifications.error("Failed to paste text"); } - - // for (let index = 0; index < 2; index++) { - // handleKeyPress(keys.KeyA, true); - // await new Promise(resolve => setTimeout(resolve, 3000)); - // handleKeyPress(keys.KeyA, false); - // } - - }, [setPasteModeEnabled, setDisableVideoFocusTrap, rpcDataChannel?.readyState, selectedKeyboard, handleKeyPress]); + async function sendKeystroke(stroke: KeyStroke) { + await new Promise((resolve, reject) => { + send( + "keyboardReport", + hidKeyboardPayload(stroke.modifier, stroke.keys), + params => { + if ("error" in params) return reject(params.error); + resolve(); + } + ); + }); + } + }, [selectedKeyboard, rpcDataChannel?.readyState, send, setDisableVideoFocusTrap, setPasteModeEnabled]); useEffect(() => { if (TextAreaRef.current) { From 11b3e8935f33f9e47cfac9ccb89ab11f51093854 Mon Sep 17 00:00:00 2001 From: Adam Shiervani Date: Tue, 16 Sep 2025 12:29:35 +0200 Subject: [PATCH 17/20] fix: correct Windows default auto-repeat delay comment from 1ms to 1s --- internal/usbgadget/hid_keyboard.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/usbgadget/hid_keyboard.go b/internal/usbgadget/hid_keyboard.go index 5b6aa91e..a325977e 100644 --- a/internal/usbgadget/hid_keyboard.go +++ b/internal/usbgadget/hid_keyboard.go @@ -28,7 +28,7 @@ var keyboardConfig = gadgetConfigItem{ // macOS default: 15 * 15 = 225ms https://discussions.apple.com/thread/1316947?sortBy=rank // Linux default: 250ms https://man.archlinux.org/man/kbdrate.8.en -// Windows default: 1ms `HKEY_CURRENT_USER\Control Panel\Accessibility\Keyboard Response\AutoRepeatDelay` +// Windows default: 1s `HKEY_CURRENT_USER\Control Panel\Accessibility\Keyboard Response\AutoRepeatDelay` const autoReleaseKeyboardInterval = time.Millisecond * 100 // Source: https://www.kernel.org/doc/Documentation/usb/gadget_hid.txt From 421666ccdc1756a3902a1d366523c00748034f95 Mon Sep 17 00:00:00 2001 From: Adam Shiervani Date: Tue, 16 Sep 2025 12:34:06 +0200 Subject: [PATCH 18/20] refactor: send keypress as early as possible --- ui/src/hooks/useKeyboard.ts | 72 ++++++++++++++++++++++++------------- 1 file changed, 47 insertions(+), 25 deletions(-) diff --git a/ui/src/hooks/useKeyboard.ts b/ui/src/hooks/useKeyboard.ts index 561485fb..aba4dd50 100644 --- a/ui/src/hooks/useKeyboard.ts +++ b/ui/src/hooks/useKeyboard.ts @@ -1,6 +1,12 @@ import { useCallback, useRef } from "react"; -import { hidErrorRollOver, hidKeyBufferSize, KeysDownState, useHidStore, useRTCStore } from "@/hooks/stores"; +import { + hidErrorRollOver, + hidKeyBufferSize, + KeysDownState, + useHidStore, + useRTCStore, +} from "@/hooks/stores"; import { JsonRpcResponse, useJsonRpc } from "@/hooks/useJsonRpc"; import { useHidRpc } from "@/hooks/useHidRpc"; import { KeyboardLedStateMessage, KeysDownStateMessage } from "@/hooks/hidRpc"; @@ -20,9 +26,9 @@ export default function useKeyboard() { // is running on the cloud against a device that has not been updated yet and thus does not // support the keyPressReport API. In that case, we need to handle the key presses locally // and send the full state to the device, so it can behave like a real USB HID keyboard. - // This flag indicates whether the keyPressReport API is available on the device which is + // This flag indicates whether the keyPressReport API is available on the device which is // dynamically set when the device responds to the first key press event or reports its - // keysDownState when queried since the keyPressReport was introduced together with the + // keysDownState when queried since the keyPressReport was introduced together with the // getKeysDownState API. // HidRPC is a binary format for exchanging keyboard and mouse events @@ -31,7 +37,7 @@ export default function useKeyboard() { reportKeypressEvent: sendKeypressEventHidRpc, reportKeypressKeepAlive: sendKeypressKeepAliveHidRpc, rpcHidReady, - } = useHidRpc((message) => { + } = useHidRpc(message => { switch (message.constructor) { case KeysDownStateMessage: setKeysDownState((message as KeysDownStateMessage).keysDownState); @@ -52,7 +58,9 @@ export default function useKeyboard() { async (state: KeysDownState) => { if (rpcDataChannel?.readyState !== "open" && !rpcHidReady) return; - console.debug(`Send keyboardReport keys: ${state.keys}, modifier: ${state.modifier}`); + console.debug( + `Send keyboardReport keys: ${state.keys}, modifier: ${state.modifier}`, + ); if (rpcHidReady) { console.debug("Sending keyboard report via HidRPC"); @@ -60,31 +68,33 @@ export default function useKeyboard() { return; } - send("keyboardReport", { keys: state.keys, modifier: state.modifier }, (resp: JsonRpcResponse) => { - if ("error" in resp) { - console.error(`Failed to send keyboard report ${state}`, resp.error); - } - }); + send( + "keyboardReport", + { keys: state.keys, modifier: state.modifier }, + (resp: JsonRpcResponse) => { + if ("error" in resp) { + console.error(`Failed to send keyboard report ${state}`, resp.error); + } + }, + ); }, - [ - rpcDataChannel?.readyState, - rpcHidReady, - send, - sendKeyboardEventHidRpc, - ], + [rpcDataChannel?.readyState, rpcHidReady, send, sendKeyboardEventHidRpc], ); - // executeMacro is used to execute a macro consisting of multiple steps. // Each step can have multiple keys, multiple modifiers and a delay. // The keys and modifiers are pressed together and held for the delay duration. // After the delay, the keys and modifiers are released and the next step is executed. // If a step has no keys or modifiers, it is treated as a delay-only step. // A small pause is added between steps to ensure that the device can process the events. - const executeMacro = async (steps: { keys: string[] | null; modifiers: string[] | null; delay: number }[]) => { + const executeMacro = async ( + steps: { keys: string[] | null; modifiers: string[] | null; delay: number }[], + ) => { for (const [index, step] of steps.entries()) { const keyValues = (step.keys || []).map(key => keys[key]).filter(Boolean); - const modifierMask: number = (step.modifiers || []).map(mod => modifiers[mod]).reduce((acc, val) => acc + val, 0); + const modifierMask: number = (step.modifiers || []) + .map(mod => modifiers[mod]) + .reduce((acc, val) => acc + val, 0); // If the step has keys and/or modifiers, press them and hold for the delay if (keyValues.length > 0 || modifierMask > 0) { @@ -119,6 +129,8 @@ export default function useKeyboard() { clearInterval(keepAliveTimerRef.current); } + sendKeypressKeepAliveHidRpc(); + // Create new interval timer keepAliveTimerRef.current = setInterval(() => { sendKeypressKeepAliveHidRpc(); @@ -174,7 +186,11 @@ export default function useKeyboard() { sendKeypress(key, press); } else { // if the keyPress api is not available, we need to handle the key locally - const downState = simulateDeviceSideKeyHandlingForLegacyDevices(keysDownState, key, press); + const downState = simulateDeviceSideKeyHandlingForLegacyDevices( + keysDownState, + key, + press, + ); sendKeyboardEvent(downState); // then we send the full state // if we just sent ErrorRollOver, reset to empty state @@ -194,7 +210,11 @@ export default function useKeyboard() { ); // IMPORTANT: See the keyPressReportApiAvailable comment above for the reason this exists - function simulateDeviceSideKeyHandlingForLegacyDevices(state: KeysDownState, key: number, press: boolean): KeysDownState { + function simulateDeviceSideKeyHandlingForLegacyDevices( + state: KeysDownState, + key: number, + press: boolean, + ): KeysDownState { // IMPORTANT: This code parallels the logic in the kernel's hid-gadget driver // for handling key presses and releases. It ensures that the USB gadget // behaves similarly to a real USB HID keyboard. This logic is paralleled @@ -206,7 +226,7 @@ export default function useKeyboard() { if (modifierMask !== 0) { // If the key is a modifier key, we update the keyboardModifier state // by setting or clearing the corresponding bit in the modifier byte. - // This allows us to track the state of dynamic modifier keys like + // This allows us to track the state of dynamic modifier keys like // Shift, Control, Alt, and Super. if (press) { modifiers |= modifierMask; @@ -223,7 +243,7 @@ export default function useKeyboard() { // and if we find a zero byte, we can place the key there (if press is true) if (keys[i] === key || keys[i] === 0) { if (press) { - keys[i] = key // overwrites the zero byte or the same key if already pressed + keys[i] = key; // overwrites the zero byte or the same key if already pressed } else { // we are releasing the key, remove it from the buffer if (keys[i] !== 0) { @@ -239,13 +259,15 @@ export default function useKeyboard() { // If we reach here it means we didn't find an empty slot or the key in the buffer if (overrun) { if (press) { - console.warn(`keyboard buffer overflow current keys ${keys}, key: ${key} not added`); + console.warn( + `keyboard buffer overflow current keys ${keys}, key: ${key} not added`, + ); // Fill all key slots with ErrorRollOver (0x01) to indicate overflow keys.length = hidKeyBufferSize; keys.fill(hidErrorRollOver); } else { // If we are releasing a key, and we didn't find it in a slot, who cares? - console.debug(`key ${key} not found in buffer, nothing to release`) + console.debug(`key ${key} not found in buffer, nothing to release`); } } } From 4e5ae37bf409e14c2d36e35d456951df14134c57 Mon Sep 17 00:00:00 2001 From: Adam Shiervani Date: Tue, 16 Sep 2025 12:35:25 +0200 Subject: [PATCH 19/20] refactor: replace console.warn with console.info for HID RPC channel events --- ui/src/hooks/useHidRpc.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ui/src/hooks/useHidRpc.ts b/ui/src/hooks/useHidRpc.ts index 5480148f..4546b625 100644 --- a/ui/src/hooks/useHidRpc.ts +++ b/ui/src/hooks/useHidRpc.ts @@ -189,12 +189,12 @@ export function useHidRpc(onHidRpcMessage?: (payload: RpcMessage) => void) { }; const openHandler = () => { - console.warn("HID RPC channel opened"); + console.info("HID RPC channel opened"); sendHandshake(); }; const closeHandler = () => { - console.warn("HID RPC channel closed"); + console.info("HID RPC channel closed"); setRpcHidProtocolVersion(null); }; From 20a23de227525bb72a3c72824bc9b228355162ad Mon Sep 17 00:00:00 2001 From: Adam Shiervani Date: Tue, 16 Sep 2025 13:09:32 +0200 Subject: [PATCH 20/20] refactor: remove unused NewKeypressKeepAliveMessage function from HID RPC --- internal/hidrpc/hidrpc.go | 8 -------- 1 file changed, 8 deletions(-) diff --git a/internal/hidrpc/hidrpc.go b/internal/hidrpc/hidrpc.go index 7161dc8f..1bc9807b 100644 --- a/internal/hidrpc/hidrpc.go +++ b/internal/hidrpc/hidrpc.go @@ -99,11 +99,3 @@ func NewKeydownStateMessage(state usbgadget.KeysDownState) *Message { d: data, } } - -// NewKeypressKeepAliveMessage creates a new keypress keep alive message. -func NewKeypressKeepAliveMessage() *Message { - return &Message{ - t: TypeKeypressKeepAliveReport, - d: []byte{}, - } -}