From 2f0aa18d1d2b7f43729b576b79437b72ffcea901 Mon Sep 17 00:00:00 2001 From: Siyuan Miao Date: Thu, 11 Sep 2025 13:05:20 +0200 Subject: [PATCH 01/14] feat: release keyPress automatically --- internal/usbgadget/hid_keyboard.go | 50 ++++++++++++++++++++++++++++++ internal/usbgadget/usbgadget.go | 35 +++++++++++++++++++++ 2 files changed, 85 insertions(+) diff --git a/internal/usbgadget/hid_keyboard.go b/internal/usbgadget/hid_keyboard.go index fb710c20..8b7080a7 100644 --- a/internal/usbgadget/hid_keyboard.go +++ b/internal/usbgadget/hid_keyboard.go @@ -173,6 +173,50 @@ func (u *UsbGadget) SetOnKeysDownChange(f func(state KeysDownState)) { u.onKeysDownChange = &f } +const autoReleaseKeyboardInterval = time.Millisecond * 300 + +func (u *UsbGadget) scheduleAutoRelease(key byte) { + u.keysAutoReleaseLock.Lock() + defer u.keysAutoReleaseLock.Unlock() + + if u.keysAutoReleaseTimer != nil { + u.keysAutoReleaseTimer.Stop() + } + + u.keysAutoReleaseTimer = time.AfterFunc(autoReleaseKeyboardInterval, func() { + u.performAutoRelease(key) + }) +} + +func (u *UsbGadget) cancelAutoRelease() { + u.keysAutoReleaseLock.Lock() + defer u.keysAutoReleaseLock.Unlock() + + if u.keysAutoReleaseTimer != nil { + u.keysAutoReleaseTimer.Stop() + } +} + +func (u *UsbGadget) performAutoRelease(key byte) { + u.keysAutoReleaseLock.Lock() + defer u.keysAutoReleaseLock.Unlock() + + select { + case <-u.keyboardStateCtx.Done(): + return + default: + } + + _, err := u.KeypressReport(key, false) + if err != nil { + u.log.Warn().Uint8("key", key).Msg("failed to auto-release keyboard key") + } + + u.keysAutoReleaseTimer = nil + + u.log.Trace().Uint8("key", key).Msg("auto release performed") +} + func (u *UsbGadget) listenKeyboardEvents() { var path string if u.keyboardHidFile != nil { @@ -398,5 +442,11 @@ func (u *UsbGadget) KeypressReport(key byte, press bool) (KeysDownState, error) u.log.Warn().Uint8("modifier", modifier).Uints8("keys", keys).Msg("Could not write keypress report to hidg0") } + if press { + u.scheduleAutoRelease(key) + } else { + u.cancelAutoRelease() + } + return u.UpdateKeysDown(modifier, keys), err } diff --git a/internal/usbgadget/usbgadget.go b/internal/usbgadget/usbgadget.go index 3a01a447..7485fc20 100644 --- a/internal/usbgadget/usbgadget.go +++ b/internal/usbgadget/usbgadget.go @@ -68,6 +68,9 @@ type UsbGadget struct { keyboardState byte // keyboard latched state (NumLock, CapsLock, ScrollLock, Compose, Kana) keysDownState KeysDownState // keyboard dynamic state (modifier keys and pressed keys) + keysAutoReleaseLock sync.Mutex + keysAutoReleaseTimer *time.Timer + keyboardStateLock sync.Mutex keyboardStateCtx context.Context keyboardStateCancel context.CancelFunc @@ -149,3 +152,35 @@ func newUsbGadget(name string, configMap map[string]gadgetConfigItem, enabledDev return g } + +// Close cleans up resources used by the USB gadget +func (u *UsbGadget) Close() error { + // Cancel keyboard state context + if u.keyboardStateCancel != nil { + u.keyboardStateCancel() + } + + // Stop auto-release timer + u.keysAutoReleaseLock.Lock() + if u.keysAutoReleaseTimer != nil { + u.keysAutoReleaseTimer.Stop() + u.keysAutoReleaseTimer = nil + } + u.keysAutoReleaseLock.Unlock() + + // Close HID files + if u.keyboardHidFile != nil { + u.keyboardHidFile.Close() + u.keyboardHidFile = nil + } + if u.absMouseHidFile != nil { + u.absMouseHidFile.Close() + u.absMouseHidFile = nil + } + if u.relMouseHidFile != nil { + u.relMouseHidFile.Close() + u.relMouseHidFile = nil + } + + return nil +} From 4b42c7e7e34eaf3499cdd8bb1b8643ca9d31d62f Mon Sep 17 00:00:00 2001 From: Siyuan Miao Date: Thu, 11 Sep 2025 14:10:18 +0200 Subject: [PATCH 02/14] send keepalive when pressing the key --- hidrpc.go | 2 + internal/hidrpc/hidrpc.go | 25 +++++--- internal/usbgadget/hid_keyboard.go | 98 ++++++++++++++++++++++++------ internal/usbgadget/usbgadget.go | 15 ++--- ui/src/hooks/hidRpc.ts | 12 ++++ ui/src/hooks/useHidRpc.ts | 8 +++ ui/src/hooks/useKeyboard.ts | 77 ++++++++++++++++++----- 7 files changed, 189 insertions(+), 48 deletions(-) diff --git a/hidrpc.go b/hidrpc.go index 74fe687f..52022ba6 100644 --- a/hidrpc.go +++ b/hidrpc.go @@ -29,6 +29,8 @@ func handleHidRPCMessage(message hidrpc.Message, session *Session) { session.reportHidRPCKeysDownState(*keysDownState) } rpcErr = err + case hidrpc.TypeKeypressKeepAliveReport: + gadget.DelayAutoRelease() case hidrpc.TypePointerReport: pointerReport, err := message.PointerReport() if err != nil { diff --git a/internal/hidrpc/hidrpc.go b/internal/hidrpc/hidrpc.go index e9c8c24d..7161dc8f 100644 --- a/internal/hidrpc/hidrpc.go +++ b/internal/hidrpc/hidrpc.go @@ -10,14 +10,15 @@ import ( type MessageType byte const ( - TypeHandshake MessageType = 0x01 - TypeKeyboardReport MessageType = 0x02 - TypePointerReport MessageType = 0x03 - TypeWheelReport MessageType = 0x04 - TypeKeypressReport MessageType = 0x05 - TypeMouseReport MessageType = 0x06 - TypeKeyboardLedState MessageType = 0x32 - TypeKeydownState MessageType = 0x33 + TypeHandshake MessageType = 0x01 + TypeKeyboardReport MessageType = 0x02 + TypePointerReport MessageType = 0x03 + TypeWheelReport MessageType = 0x04 + TypeKeypressReport MessageType = 0x05 + TypeKeypressKeepAliveReport MessageType = 0x09 + TypeMouseReport MessageType = 0x06 + TypeKeyboardLedState MessageType = 0x32 + TypeKeydownState MessageType = 0x33 ) const ( @@ -98,3 +99,11 @@ func NewKeydownStateMessage(state usbgadget.KeysDownState) *Message { d: data, } } + +// NewKeypressKeepAliveMessage creates a new keypress keep alive message. +func NewKeypressKeepAliveMessage() *Message { + return &Message{ + t: TypeKeypressKeepAliveReport, + d: []byte{}, + } +} diff --git a/internal/usbgadget/hid_keyboard.go b/internal/usbgadget/hid_keyboard.go index 8b7080a7..f9bd20db 100644 --- a/internal/usbgadget/hid_keyboard.go +++ b/internal/usbgadget/hid_keyboard.go @@ -173,33 +173,47 @@ func (u *UsbGadget) SetOnKeysDownChange(f func(state KeysDownState)) { u.onKeysDownChange = &f } -const autoReleaseKeyboardInterval = time.Millisecond * 300 +const autoReleaseKeyboardInterval = time.Millisecond * 450 func (u *UsbGadget) scheduleAutoRelease(key byte) { - u.keysAutoReleaseLock.Lock() - defer u.keysAutoReleaseLock.Unlock() + u.kbdAutoReleaseLock.Lock() + defer u.kbdAutoReleaseLock.Unlock() - if u.keysAutoReleaseTimer != nil { - u.keysAutoReleaseTimer.Stop() + if u.kbdAutoReleaseTimer != nil { + u.kbdAutoReleaseTimer.Stop() } - u.keysAutoReleaseTimer = time.AfterFunc(autoReleaseKeyboardInterval, func() { + u.kbdAutoReleaseTimer = time.AfterFunc(autoReleaseKeyboardInterval, func() { u.performAutoRelease(key) }) } func (u *UsbGadget) cancelAutoRelease() { - u.keysAutoReleaseLock.Lock() - defer u.keysAutoReleaseLock.Unlock() + u.kbdAutoReleaseLock.Lock() + defer u.kbdAutoReleaseLock.Unlock() - if u.keysAutoReleaseTimer != nil { - u.keysAutoReleaseTimer.Stop() + if u.kbdAutoReleaseTimer != nil { + u.kbdAutoReleaseTimer.Stop() } } +func (u *UsbGadget) DelayAutoRelease() { + u.kbdAutoReleaseLock.Lock() + defer u.kbdAutoReleaseLock.Unlock() + + u.log.Info().Msg("delaying auto-release") + + if u.kbdAutoReleaseTimer == nil { + return + } + + u.log.Info().Msg("resetting auto-release timer") + u.kbdAutoReleaseTimer.Reset(autoReleaseKeyboardInterval) +} + func (u *UsbGadget) performAutoRelease(key byte) { - u.keysAutoReleaseLock.Lock() - defer u.keysAutoReleaseLock.Unlock() + u.kbdAutoReleaseLock.Lock() + defer u.kbdAutoReleaseLock.Unlock() select { case <-u.keyboardStateCtx.Done(): @@ -207,12 +221,12 @@ func (u *UsbGadget) performAutoRelease(key byte) { default: } - _, err := u.KeypressReport(key, false) + _, err := u.keypressReport(key, false, false) if err != nil { u.log.Warn().Uint8("key", key).Msg("failed to auto-release keyboard key") } - u.keysAutoReleaseTimer = nil + u.kbdAutoReleaseTimer = nil u.log.Trace().Uint8("key", key).Msg("auto release performed") } @@ -375,11 +389,21 @@ var KeyCodeToMaskMap = map[byte]byte{ RightSuper: ModifierMaskRightSuper, } -func (u *UsbGadget) KeypressReport(key byte, press bool) (KeysDownState, error) { +func (u *UsbGadget) keypressReport(key byte, press bool, autoRelease bool) (KeysDownState, error) { + ll := u.log.Info().Str("component", "kbd") + ll.Uint8("key", key).Msg("locking keyboardLock") + u.keyboardLock.Lock() - defer u.keyboardLock.Unlock() + defer func() { + u.keyboardLock.Unlock() + ll.Uint8("key", key).Msg("unlocked keyboardLock") + }() + + ll.Uint8("key", key).Msg("resetting user input time") defer u.resetUserInputTime() + ll.Uint8("key", key).Msg("locked keyboardLock") + // IMPORTANT: This code parallels the logic in the kernel's hid-gadget driver // for handling key presses and releases. It ensures that the USB gadget // behaves similarly to a real USB HID keyboard. This logic is paralleled @@ -437,16 +461,54 @@ func (u *UsbGadget) KeypressReport(key byte, press bool) (KeysDownState, error) } } + ll.Uint8("key", key).Msg("checking if auto-release is enabled") + + // if autoRelease { + // u.kbdAutoReleaseLock.Lock() + // u.kbdAutoReleaseLock.Unlock() + // ll.Uint8("key", key).Msg("locking kbdAutoReleaseLock, autoReleasLastKey reset") + + // defer func() { + // ll.Uint8("key", key).Msg("unlocked kbdAutoReleaseLock, autoReleasLastKey reset") + // }() + + // if u.kbdAutoReleaseLastKey == key { + // ll.Uint8("key", key).Msg("key already released by auto-release, skipping") + // u.kbdAutoReleaseLastKey = 0 + + // return u.UpdateKeysDown(modifier, keys), nil + // } + // } + + ll.Uint8("key", key).Msg("writing keypress report to hidg0") + err := u.keyboardWriteHidFile(modifier, keys) if err != nil { u.log.Warn().Uint8("modifier", modifier).Uints8("keys", keys).Msg("Could not write keypress report to hidg0") } if press { - u.scheduleAutoRelease(key) + { + u.kbdAutoReleaseLock.Lock() + u.kbdAutoReleaseLastKey = key + u.kbdAutoReleaseLock.Unlock() + } + + if autoRelease { + ll.Uint8("key", key).Msg("scheduling auto-release") + u.scheduleAutoRelease(key) + } } else { - u.cancelAutoRelease() + if autoRelease { + ll.Uint8("key", key).Msg("canceling auto-release") + u.cancelAutoRelease() + ll.Uint8("key", key).Msg("auto-release canceled") + } } return u.UpdateKeysDown(modifier, keys), err } + +func (u *UsbGadget) KeypressReport(key byte, press bool) (KeysDownState, error) { + return u.keypressReport(key, press, true) +} diff --git a/internal/usbgadget/usbgadget.go b/internal/usbgadget/usbgadget.go index 7485fc20..e491a78c 100644 --- a/internal/usbgadget/usbgadget.go +++ b/internal/usbgadget/usbgadget.go @@ -68,8 +68,9 @@ type UsbGadget struct { keyboardState byte // keyboard latched state (NumLock, CapsLock, ScrollLock, Compose, Kana) keysDownState KeysDownState // keyboard dynamic state (modifier keys and pressed keys) - keysAutoReleaseLock sync.Mutex - keysAutoReleaseTimer *time.Timer + kbdAutoReleaseLock sync.Mutex + kbdAutoReleaseTimer *time.Timer + kbdAutoReleaseLastKey byte keyboardStateLock sync.Mutex keyboardStateCtx context.Context @@ -161,12 +162,12 @@ func (u *UsbGadget) Close() error { } // Stop auto-release timer - u.keysAutoReleaseLock.Lock() - if u.keysAutoReleaseTimer != nil { - u.keysAutoReleaseTimer.Stop() - u.keysAutoReleaseTimer = nil + u.kbdAutoReleaseLock.Lock() + if u.kbdAutoReleaseTimer != nil { + u.kbdAutoReleaseTimer.Stop() + u.kbdAutoReleaseTimer = nil } - u.keysAutoReleaseLock.Unlock() + u.kbdAutoReleaseLock.Unlock() // Close HID files if u.keyboardHidFile != nil { diff --git a/ui/src/hooks/hidRpc.ts b/ui/src/hooks/hidRpc.ts index 20b8a108..d6b4ad96 100644 --- a/ui/src/hooks/hidRpc.ts +++ b/ui/src/hooks/hidRpc.ts @@ -6,6 +6,7 @@ export const HID_RPC_MESSAGE_TYPES = { PointerReport: 0x03, WheelReport: 0x04, KeypressReport: 0x05, + KeypressKeepAliveReport: 0x09, MouseReport: 0x06, KeyboardLedState: 0x32, KeysDownState: 0x33, @@ -278,12 +279,23 @@ export class MouseReportMessage extends RpcMessage { } } +export class KeypressKeepAliveMessage extends RpcMessage { + constructor() { + super(HID_RPC_MESSAGE_TYPES.KeypressKeepAliveReport); + } + + marshal(): Uint8Array { + return new Uint8Array([this.messageType]); + } +} + export const messageRegistry = { [HID_RPC_MESSAGE_TYPES.Handshake]: HandshakeMessage, [HID_RPC_MESSAGE_TYPES.KeysDownState]: KeysDownStateMessage, [HID_RPC_MESSAGE_TYPES.KeyboardLedState]: KeyboardLedStateMessage, [HID_RPC_MESSAGE_TYPES.KeyboardReport]: KeyboardReportMessage, [HID_RPC_MESSAGE_TYPES.KeypressReport]: KeypressReportMessage, + [HID_RPC_MESSAGE_TYPES.KeypressKeepAliveReport]: KeypressKeepAliveMessage, } export const unmarshalHidRpcMessage = (data: Uint8Array): RpcMessage | undefined => { diff --git a/ui/src/hooks/useHidRpc.ts b/ui/src/hooks/useHidRpc.ts index ea0c7112..d5690849 100644 --- a/ui/src/hooks/useHidRpc.ts +++ b/ui/src/hooks/useHidRpc.ts @@ -6,6 +6,7 @@ import { HID_RPC_VERSION, HandshakeMessage, KeyboardReportMessage, + KeypressKeepAliveMessage, KeypressReportMessage, MouseReportMessage, PointerReportMessage, @@ -13,6 +14,8 @@ import { unmarshalHidRpcMessage, } from "./hidRpc"; +const KEEPALIVE_MESSAGE = new KeypressKeepAliveMessage(); + export function useHidRpc(onHidRpcMessage?: (payload: RpcMessage) => void) { const { rpcHidChannel, setRpcHidProtocolVersion, rpcHidProtocolVersion } = useRTCStore(); const rpcHidReady = useMemo(() => { @@ -68,6 +71,10 @@ export function useHidRpc(onHidRpcMessage?: (payload: RpcMessage) => void) { [sendMessage], ); + const reportKeypressKeepAlive = useCallback(() => { + sendMessage(KEEPALIVE_MESSAGE); + }, [sendMessage]); + const sendHandshake = useCallback(() => { if (rpcHidProtocolVersion) return; if (!rpcHidChannel) return; @@ -143,6 +150,7 @@ export function useHidRpc(onHidRpcMessage?: (payload: RpcMessage) => void) { reportKeypressEvent, reportAbsMouseEvent, reportRelMouseEvent, + reportKeypressKeepAlive, rpcHidProtocolVersion, rpcHidReady, rpcHidStatus, diff --git a/ui/src/hooks/useKeyboard.ts b/ui/src/hooks/useKeyboard.ts index 787df9a9..fb9fa73a 100644 --- a/ui/src/hooks/useKeyboard.ts +++ b/ui/src/hooks/useKeyboard.ts @@ -1,4 +1,4 @@ -import { useCallback } from "react"; +import { useCallback, useRef } from "react"; import { hidErrorRollOver, hidKeyBufferSize, KeysDownState, useHidStore, useRTCStore } from "@/hooks/stores"; import { JsonRpcResponse, useJsonRpc } from "@/hooks/useJsonRpc"; @@ -11,6 +11,9 @@ export default function useKeyboard() { const { rpcDataChannel } = useRTCStore(); const { keysDownState, setKeysDownState, setKeyboardLedState } = useHidStore(); + // Keepalive timer management + const keepAliveTimerRef = useRef(null); + // INTRODUCTION: The earlier version of the JetKVM device shipped with all keyboard state // being tracked on the browser/client-side. When adding the keyPressReport API to the // device-side code, we have to still support the situation where the browser/client-side code @@ -26,6 +29,7 @@ export default function useKeyboard() { const { reportKeyboardEvent: sendKeyboardEventHidRpc, reportKeypressEvent: sendKeypressEventHidRpc, + reportKeypressKeepAlive: sendKeypressKeepAliveHidRpc, rpcHidReady, } = useHidRpc((message) => { switch (message.constructor) { @@ -70,17 +74,6 @@ export default function useKeyboard() { ], ); - // resetKeyboardState is used to reset the keyboard state to no keys pressed and no modifiers. - // This is useful for macros and when the browser loses focus to ensure that the keyboard state - // is clean. - const resetKeyboardState = useCallback( - async () => { - // Reset the keys buffer to zeros and the modifier state to zero - keysDownState.keys.length = hidKeyBufferSize; - keysDownState.keys.fill(0); - keysDownState.modifier = 0; - sendKeyboardEvent(keysDownState); - }, [keysDownState, sendKeyboardEvent]); // executeMacro is used to execute a macro consisting of multiple steps. // Each step can have multiple keys, multiple modifiers and a delay. @@ -111,12 +104,61 @@ export default function useKeyboard() { } }; + const KEEPALIVE_INTERVAL = 200; // 200ms interval + + const cancelKeepAlive = useCallback(() => { + if (keepAliveTimerRef.current) { + clearInterval(keepAliveTimerRef.current); + keepAliveTimerRef.current = null; + } + }, []); + + const scheduleKeepAlive = useCallback(() => { + // Clear existing timer if it exists + if (keepAliveTimerRef.current) { + clearInterval(keepAliveTimerRef.current); + } + + // Create new interval timer + keepAliveTimerRef.current = setInterval(() => { + sendKeypressKeepAliveHidRpc(); + }, KEEPALIVE_INTERVAL); + }, [sendKeypressKeepAliveHidRpc]); + + // resetKeyboardState is used to reset the keyboard state to no keys pressed and no modifiers. + // This is useful for macros and when the browser loses focus to ensure that the keyboard state + // is clean. + const resetKeyboardState = useCallback(async () => { + // Cancel keepalive since we're resetting the keyboard state + cancelKeepAlive(); + + // Reset the keys buffer to zeros and the modifier state to zero + keysDownState.keys.length = hidKeyBufferSize; + keysDownState.keys.fill(0); + keysDownState.modifier = 0; + sendKeyboardEvent(keysDownState); + }, [keysDownState, sendKeyboardEvent, cancelKeepAlive]); + // handleKeyPress is used to handle a key press or release event. // This function handle both key press and key release events. // It checks if the keyPressReport API is available and sends the key press event. // If the keyPressReport API is not available, it simulates the device-side key // handling for legacy devices and updates the keysDownState accordingly. // It then sends the full keyboard state to the device. + + const sendKeypress = useCallback( + (key: number, press: boolean) => { + cancelKeepAlive(); + + sendKeypressEventHidRpc(key, press); + + if (press) { + scheduleKeepAlive(); + } + }, + [sendKeypressEventHidRpc, scheduleKeepAlive, cancelKeepAlive], + ); + const handleKeyPress = useCallback( async (key: number, press: boolean) => { if (rpcDataChannel?.readyState !== "open" && !rpcHidReady) return; @@ -129,7 +171,7 @@ export default function useKeyboard() { // Older device version doesn't support this API, so we will switch to local key handling // In that case we will switch to local key handling and update the keysDownState // in client/browser-side code using simulateDeviceSideKeyHandlingForLegacyDevices. - sendKeypressEventHidRpc(key, press); + sendKeypress(key, press); } else { // if the keyPress api is not available, we need to handle the key locally const downState = simulateDeviceSideKeyHandlingForLegacyDevices(keysDownState, key, press); @@ -147,7 +189,7 @@ export default function useKeyboard() { resetKeyboardState, rpcDataChannel?.readyState, sendKeyboardEvent, - sendKeypressEventHidRpc, + sendKeypress, ], ); @@ -210,5 +252,10 @@ export default function useKeyboard() { return { modifier: modifiers, keys }; } - return { handleKeyPress, resetKeyboardState, executeMacro }; + // Cleanup function to cancel keepalive timer + const cleanup = useCallback(() => { + cancelKeepAlive(); + }, [cancelKeepAlive]); + + return { handleKeyPress, resetKeyboardState, executeMacro, cleanup }; } From 8b079033887bcc0e47b239722a5e0b491433bfca Mon Sep 17 00:00:00 2001 From: Siyuan Miao Date: Thu, 11 Sep 2025 15:00:03 +0200 Subject: [PATCH 03/14] remove logging --- internal/usbgadget/hid_keyboard.go | 18 +----------------- 1 file changed, 1 insertion(+), 17 deletions(-) diff --git a/internal/usbgadget/hid_keyboard.go b/internal/usbgadget/hid_keyboard.go index f9bd20db..0cb978b4 100644 --- a/internal/usbgadget/hid_keyboard.go +++ b/internal/usbgadget/hid_keyboard.go @@ -390,20 +390,11 @@ var KeyCodeToMaskMap = map[byte]byte{ } func (u *UsbGadget) keypressReport(key byte, press bool, autoRelease bool) (KeysDownState, error) { - ll := u.log.Info().Str("component", "kbd") - ll.Uint8("key", key).Msg("locking keyboardLock") - u.keyboardLock.Lock() - defer func() { - u.keyboardLock.Unlock() - ll.Uint8("key", key).Msg("unlocked keyboardLock") - }() + defer u.keyboardLock.Unlock() - ll.Uint8("key", key).Msg("resetting user input time") defer u.resetUserInputTime() - ll.Uint8("key", key).Msg("locked keyboardLock") - // IMPORTANT: This code parallels the logic in the kernel's hid-gadget driver // for handling key presses and releases. It ensures that the USB gadget // behaves similarly to a real USB HID keyboard. This logic is paralleled @@ -461,8 +452,6 @@ func (u *UsbGadget) keypressReport(key byte, press bool, autoRelease bool) (Keys } } - ll.Uint8("key", key).Msg("checking if auto-release is enabled") - // if autoRelease { // u.kbdAutoReleaseLock.Lock() // u.kbdAutoReleaseLock.Unlock() @@ -480,8 +469,6 @@ func (u *UsbGadget) keypressReport(key byte, press bool, autoRelease bool) (Keys // } // } - ll.Uint8("key", key).Msg("writing keypress report to hidg0") - err := u.keyboardWriteHidFile(modifier, keys) if err != nil { u.log.Warn().Uint8("modifier", modifier).Uints8("keys", keys).Msg("Could not write keypress report to hidg0") @@ -495,14 +482,11 @@ func (u *UsbGadget) keypressReport(key byte, press bool, autoRelease bool) (Keys } if autoRelease { - ll.Uint8("key", key).Msg("scheduling auto-release") u.scheduleAutoRelease(key) } } else { if autoRelease { - ll.Uint8("key", key).Msg("canceling auto-release") u.cancelAutoRelease() - ll.Uint8("key", key).Msg("auto-release canceled") } } From 46aee61565e9aa7e5ade1413c633661836b734a2 Mon Sep 17 00:00:00 2001 From: Siyuan Miao Date: Thu, 11 Sep 2025 15:04:35 +0200 Subject: [PATCH 04/14] clean up logging --- internal/hidrpc/message.go | 2 ++ internal/usbgadget/hid_keyboard.go | 5 +++-- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/internal/hidrpc/message.go b/internal/hidrpc/message.go index 84bbda7c..e0f4493e 100644 --- a/internal/hidrpc/message.go +++ b/internal/hidrpc/message.go @@ -43,6 +43,8 @@ func (m *Message) String() string { return fmt.Sprintf("MouseReport{Malformed: %v}", m.d) } return fmt.Sprintf("MouseReport{DX: %d, DY: %d, Button: %d}", m.d[0], m.d[1], m.d[2]) + case TypeKeypressKeepAliveReport: + return "KeypressKeepAliveReport" default: return fmt.Sprintf("Unknown{Type: %d, Data: %v}", m.t, m.d) } diff --git a/internal/usbgadget/hid_keyboard.go b/internal/usbgadget/hid_keyboard.go index 0cb978b4..40dff0be 100644 --- a/internal/usbgadget/hid_keyboard.go +++ b/internal/usbgadget/hid_keyboard.go @@ -201,14 +201,15 @@ func (u *UsbGadget) DelayAutoRelease() { u.kbdAutoReleaseLock.Lock() defer u.kbdAutoReleaseLock.Unlock() - u.log.Info().Msg("delaying auto-release") + u.log.Trace().Msg("delaying auto-release") if u.kbdAutoReleaseTimer == nil { return } - u.log.Info().Msg("resetting auto-release timer") u.kbdAutoReleaseTimer.Reset(autoReleaseKeyboardInterval) + + u.log.Trace().Msg("auto-release timer reset") } func (u *UsbGadget) performAutoRelease(key byte) { From 8e9554f4d60bd5fb7faba19f84ad003ce86464e3 Mon Sep 17 00:00:00 2001 From: Siyuan Miao Date: Thu, 11 Sep 2025 16:16:47 +0200 Subject: [PATCH 05/14] chore: use unreliable channel to send keepalive events --- ui/src/hooks/stores.ts | 6 +++ ui/src/hooks/useHidRpc.ts | 35 +++++++++++++---- ui/src/routes/devices.$id.tsx | 12 ++++++ webrtc.go | 73 +++++++++++++++++++---------------- 4 files changed, 84 insertions(+), 42 deletions(-) diff --git a/ui/src/hooks/stores.ts b/ui/src/hooks/stores.ts index f99fd07d..7007f4e6 100644 --- a/ui/src/hooks/stores.ts +++ b/ui/src/hooks/stores.ts @@ -111,6 +111,9 @@ export interface RTCState { rpcHidChannel: RTCDataChannel | null; setRpcHidChannel: (channel: RTCDataChannel) => void; + rpcHidUnreliableChannel: RTCDataChannel | null; + setRpcHidUnreliableChannel: (channel: RTCDataChannel) => void; + peerConnectionState: RTCPeerConnectionState | null; setPeerConnectionState: (state: RTCPeerConnectionState) => void; @@ -163,6 +166,9 @@ export const useRTCStore = create(set => ({ rpcHidChannel: null, setRpcHidChannel: (channel: RTCDataChannel) => set({ rpcHidChannel: channel }), + rpcHidUnreliableChannel: null, + setRpcHidUnreliableChannel: (channel: RTCDataChannel) => set({ rpcHidUnreliableChannel: channel }), + transceiver: null, setTransceiver: (transceiver: RTCRtpTransceiver) => set({ transceiver }), diff --git a/ui/src/hooks/useHidRpc.ts b/ui/src/hooks/useHidRpc.ts index d5690849..ff22885c 100644 --- a/ui/src/hooks/useHidRpc.ts +++ b/ui/src/hooks/useHidRpc.ts @@ -16,20 +16,35 @@ import { const KEEPALIVE_MESSAGE = new KeypressKeepAliveMessage(); +interface sendMessageParams { + ignoreHandshakeState?: boolean; + useUnreliableChannel?: boolean; +} + export function useHidRpc(onHidRpcMessage?: (payload: RpcMessage) => void) { - const { rpcHidChannel, setRpcHidProtocolVersion, rpcHidProtocolVersion } = useRTCStore(); + const { + rpcHidChannel, + rpcHidUnreliableChannel, + setRpcHidProtocolVersion, + rpcHidProtocolVersion, + } = useRTCStore(); + const rpcHidReady = useMemo(() => { return rpcHidChannel?.readyState === "open" && rpcHidProtocolVersion !== null; }, [rpcHidChannel, rpcHidProtocolVersion]); + const rpcHidUnreliableReady = useMemo(() => { + return rpcHidUnreliableChannel?.readyState === "open" && rpcHidProtocolVersion !== null; + }, [rpcHidUnreliableChannel, rpcHidProtocolVersion]); + const rpcHidStatus = useMemo(() => { if (!rpcHidChannel) return "N/A"; if (rpcHidChannel.readyState !== "open") return rpcHidChannel.readyState; if (!rpcHidProtocolVersion) return "handshaking"; - return `ready (v${rpcHidProtocolVersion})`; - }, [rpcHidChannel, rpcHidProtocolVersion]); + return `ready (v${rpcHidProtocolVersion}${rpcHidUnreliableReady ? "+u" : ""})`; + }, [rpcHidChannel, rpcHidUnreliableReady, rpcHidProtocolVersion]); - const sendMessage = useCallback((message: RpcMessage, ignoreHandshakeState = false) => { + const sendMessage = useCallback((message: RpcMessage, { ignoreHandshakeState, useUnreliableChannel }: sendMessageParams = {}) => { if (rpcHidChannel?.readyState !== "open") return; if (!rpcHidReady && !ignoreHandshakeState) return; @@ -41,8 +56,12 @@ export function useHidRpc(onHidRpcMessage?: (payload: RpcMessage) => void) { } if (!data) return; - rpcHidChannel?.send(data as unknown as ArrayBuffer); - }, [rpcHidChannel, rpcHidReady]); + if (useUnreliableChannel && rpcHidUnreliableReady) { + rpcHidUnreliableChannel?.send(data as unknown as ArrayBuffer); + } else { + rpcHidChannel?.send(data as unknown as ArrayBuffer); + } + }, [rpcHidChannel, rpcHidReady, rpcHidUnreliableChannel, rpcHidUnreliableReady]); const reportKeyboardEvent = useCallback( (keys: number[], modifier: number) => { @@ -72,14 +91,14 @@ export function useHidRpc(onHidRpcMessage?: (payload: RpcMessage) => void) { ); const reportKeypressKeepAlive = useCallback(() => { - sendMessage(KEEPALIVE_MESSAGE); + sendMessage(KEEPALIVE_MESSAGE, { useUnreliableChannel: true }); }, [sendMessage]); const sendHandshake = useCallback(() => { if (rpcHidProtocolVersion) return; if (!rpcHidChannel) return; - sendMessage(new HandshakeMessage(HID_RPC_VERSION), true); + sendMessage(new HandshakeMessage(HID_RPC_VERSION), { ignoreHandshakeState: true }); }, [rpcHidChannel, rpcHidProtocolVersion, sendMessage]); const handleHandshake = useCallback((message: HandshakeMessage) => { diff --git a/ui/src/routes/devices.$id.tsx b/ui/src/routes/devices.$id.tsx index 4318447e..444e347d 100644 --- a/ui/src/routes/devices.$id.tsx +++ b/ui/src/routes/devices.$id.tsx @@ -136,6 +136,7 @@ export default function KvmIdRoute() { rpcDataChannel, setTransceiver, setRpcHidChannel, + setRpcHidUnreliableChannel, } = useRTCStore(); const location = useLocation(); @@ -488,6 +489,16 @@ export default function KvmIdRoute() { setRpcHidChannel(rpcHidChannel); }; + const rpcHidUnreliableChannel = pc.createDataChannel("hidrpc-unreliable", { + // We don't need to be ordered, as we're using the unreliable channel for keepalive messages + ordered: false, + maxRetransmits: 0, + }); + rpcHidUnreliableChannel.binaryType = "arraybuffer"; + rpcHidUnreliableChannel.onopen = () => { + setRpcHidUnreliableChannel(rpcHidUnreliableChannel); + }; + setPeerConnection(pc); }, [ cleanupAndStopReconnecting, @@ -499,6 +510,7 @@ export default function KvmIdRoute() { setPeerConnectionState, setRpcDataChannel, setRpcHidChannel, + setRpcHidUnreliableChannel, setTransceiver, ]); diff --git a/webrtc.go b/webrtc.go index c3d0dc1b..e43c2512 100644 --- a/webrtc.go +++ b/webrtc.go @@ -145,6 +145,41 @@ func newSession(config SessionConfig) (*Session, error) { go session.handleQueues(i) } + onHidMessage := func(msg webrtc.DataChannelMessage) { + l := scopedLogger.With().Int("length", len(msg.Data)).Logger() + // only log data if the log level is debug or lower + if scopedLogger.GetLevel() > zerolog.DebugLevel { + l = l.With().Str("data", string(msg.Data)).Logger() + } + + if msg.IsString { + l.Warn().Msg("received string data in HID RPC message handler") + return + } + + if len(msg.Data) < 1 { + l.Warn().Msg("received empty data in HID RPC message handler") + return + } + + l.Trace().Msg("received data in HID RPC message handler") + + // Enqueue to ensure ordered processing + queueIndex := hidrpc.GetQueueIndex(hidrpc.MessageType(msg.Data[0])) + if queueIndex >= len(session.hidQueue) || queueIndex < 0 { + l.Warn().Int("queueIndex", queueIndex).Msg("received data in HID RPC message handler, but queue index not found") + queueIndex = 3 + } + + queue := session.hidQueue[queueIndex] + if queue != nil { + queue <- msg + } else { + l.Warn().Int("queueIndex", queueIndex).Msg("received data in HID RPC message handler, but queue is nil") + return + } + } + peerConnection.OnDataChannel(func(d *webrtc.DataChannel) { defer func() { if r := recover(); r != nil { @@ -157,40 +192,10 @@ func newSession(config SessionConfig) (*Session, error) { switch d.Label() { case "hidrpc": session.HidChannel = d - d.OnMessage(func(msg webrtc.DataChannelMessage) { - l := scopedLogger.With().Int("length", len(msg.Data)).Logger() - // only log data if the log level is debug or lower - if scopedLogger.GetLevel() > zerolog.DebugLevel { - l = l.With().Str("data", string(msg.Data)).Logger() - } - - if msg.IsString { - l.Warn().Msg("received string data in HID RPC message handler") - return - } - - if len(msg.Data) < 1 { - l.Warn().Msg("received empty data in HID RPC message handler") - return - } - - l.Trace().Msg("received data in HID RPC message handler") - - // Enqueue to ensure ordered processing - queueIndex := hidrpc.GetQueueIndex(hidrpc.MessageType(msg.Data[0])) - if queueIndex >= len(session.hidQueue) || queueIndex < 0 { - l.Warn().Int("queueIndex", queueIndex).Msg("received data in HID RPC message handler, but queue index not found") - queueIndex = 3 - } - - queue := session.hidQueue[queueIndex] - if queue != nil { - queue <- msg - } else { - l.Warn().Int("queueIndex", queueIndex).Msg("received data in HID RPC message handler, but queue is nil") - return - } - }) + d.OnMessage(onHidMessage) + // we won't send anything over the unreliable channel + case "hidrpc-unreliable": + d.OnMessage(onHidMessage) case "rpc": session.RPCChannel = d d.OnMessage(func(msg webrtc.DataChannelMessage) { From 3901ac91e0a4274a07c6fc11ecc21157a636588f Mon Sep 17 00:00:00 2001 From: Siyuan Miao Date: Thu, 11 Sep 2025 17:24:10 +0200 Subject: [PATCH 06/14] chore: use ordered unreliable channel for pointer events --- hidrpc.go | 9 ++- ui/src/hooks/stores.ts | 6 ++ ui/src/hooks/useHidRpc.ts | 34 +++++++++--- ui/src/routes/devices.$id.tsx | 16 +++++- webrtc.go | 101 +++++++++++++++++++--------------- 5 files changed, 110 insertions(+), 56 deletions(-) diff --git a/hidrpc.go b/hidrpc.go index 52022ba6..2c3ca27b 100644 --- a/hidrpc.go +++ b/hidrpc.go @@ -54,8 +54,13 @@ func handleHidRPCMessage(message hidrpc.Message, session *Session) { } } -func onHidMessage(data []byte, session *Session) { - scopedLogger := hidRPCLogger.With().Bytes("data", data).Logger() +func onHidMessage(msg hidQueueMessage, session *Session) { + data := msg.Data + + scopedLogger := hidRPCLogger.With(). + Str("channel", msg.channel). + Bytes("data", data). + Logger() scopedLogger.Debug().Msg("HID RPC message received") if len(data) < 1 { diff --git a/ui/src/hooks/stores.ts b/ui/src/hooks/stores.ts index 7007f4e6..b849fa55 100644 --- a/ui/src/hooks/stores.ts +++ b/ui/src/hooks/stores.ts @@ -114,6 +114,9 @@ export interface RTCState { rpcHidUnreliableChannel: RTCDataChannel | null; setRpcHidUnreliableChannel: (channel: RTCDataChannel) => void; + rpcHidUnreliableNonOrderedChannel: RTCDataChannel | null; + setRpcHidUnreliableNonOrderedChannel: (channel: RTCDataChannel) => void; + peerConnectionState: RTCPeerConnectionState | null; setPeerConnectionState: (state: RTCPeerConnectionState) => void; @@ -169,6 +172,9 @@ export const useRTCStore = create(set => ({ rpcHidUnreliableChannel: null, setRpcHidUnreliableChannel: (channel: RTCDataChannel) => set({ rpcHidUnreliableChannel: channel }), + rpcHidUnreliableNonOrderedChannel: null, + setRpcHidUnreliableNonOrderedChannel: (channel: RTCDataChannel) => set({ rpcHidUnreliableNonOrderedChannel: channel }), + transceiver: null, setTransceiver: (transceiver: RTCRtpTransceiver) => set({ transceiver }), diff --git a/ui/src/hooks/useHidRpc.ts b/ui/src/hooks/useHidRpc.ts index ff22885c..58c04f42 100644 --- a/ui/src/hooks/useHidRpc.ts +++ b/ui/src/hooks/useHidRpc.ts @@ -19,12 +19,14 @@ const KEEPALIVE_MESSAGE = new KeypressKeepAliveMessage(); interface sendMessageParams { ignoreHandshakeState?: boolean; useUnreliableChannel?: boolean; + requireOrdered?: boolean; } export function useHidRpc(onHidRpcMessage?: (payload: RpcMessage) => void) { const { rpcHidChannel, rpcHidUnreliableChannel, + rpcHidUnreliableNonOrderedChannel, setRpcHidProtocolVersion, rpcHidProtocolVersion, } = useRTCStore(); @@ -37,6 +39,10 @@ export function useHidRpc(onHidRpcMessage?: (payload: RpcMessage) => void) { return rpcHidUnreliableChannel?.readyState === "open" && rpcHidProtocolVersion !== null; }, [rpcHidUnreliableChannel, rpcHidProtocolVersion]); + const rpcHidUnreliableNonOrderedReady = useMemo(() => { + return rpcHidUnreliableNonOrderedChannel?.readyState === "open" && rpcHidProtocolVersion !== null; + }, [rpcHidUnreliableNonOrderedChannel, rpcHidProtocolVersion]); + const rpcHidStatus = useMemo(() => { if (!rpcHidChannel) return "N/A"; if (rpcHidChannel.readyState !== "open") return rpcHidChannel.readyState; @@ -44,7 +50,7 @@ export function useHidRpc(onHidRpcMessage?: (payload: RpcMessage) => void) { return `ready (v${rpcHidProtocolVersion}${rpcHidUnreliableReady ? "+u" : ""})`; }, [rpcHidChannel, rpcHidUnreliableReady, rpcHidProtocolVersion]); - const sendMessage = useCallback((message: RpcMessage, { ignoreHandshakeState, useUnreliableChannel }: sendMessageParams = {}) => { + const sendMessage = useCallback((message: RpcMessage, { ignoreHandshakeState, useUnreliableChannel, requireOrdered = true }: sendMessageParams = {}) => { if (rpcHidChannel?.readyState !== "open") return; if (!rpcHidReady && !ignoreHandshakeState) return; @@ -56,12 +62,24 @@ export function useHidRpc(onHidRpcMessage?: (payload: RpcMessage) => void) { } if (!data) return; - if (useUnreliableChannel && rpcHidUnreliableReady) { - rpcHidUnreliableChannel?.send(data as unknown as ArrayBuffer); - } else { - rpcHidChannel?.send(data as unknown as ArrayBuffer); + if (useUnreliableChannel) { + if (requireOrdered && rpcHidUnreliableReady) { + rpcHidUnreliableChannel?.send(data as unknown as ArrayBuffer); + } else if (!requireOrdered && rpcHidUnreliableNonOrderedReady) { + rpcHidUnreliableNonOrderedChannel?.send(data as unknown as ArrayBuffer); + } + return; } - }, [rpcHidChannel, rpcHidReady, rpcHidUnreliableChannel, rpcHidUnreliableReady]); + + rpcHidChannel?.send(data as unknown as ArrayBuffer); + }, [ + rpcHidChannel, + rpcHidUnreliableChannel, + rpcHidUnreliableNonOrderedChannel, + rpcHidReady, + rpcHidUnreliableReady, + rpcHidUnreliableNonOrderedReady, + ]); const reportKeyboardEvent = useCallback( (keys: number[], modifier: number) => { @@ -78,7 +96,7 @@ export function useHidRpc(onHidRpcMessage?: (payload: RpcMessage) => void) { const reportAbsMouseEvent = useCallback( (x: number, y: number, buttons: number) => { - sendMessage(new PointerReportMessage(x, y, buttons)); + sendMessage(new PointerReportMessage(x, y, buttons), { useUnreliableChannel: true }); }, [sendMessage], ); @@ -91,7 +109,7 @@ export function useHidRpc(onHidRpcMessage?: (payload: RpcMessage) => void) { ); const reportKeypressKeepAlive = useCallback(() => { - sendMessage(KEEPALIVE_MESSAGE, { useUnreliableChannel: true }); + sendMessage(KEEPALIVE_MESSAGE, { useUnreliableChannel: true, requireOrdered: false }); }, [sendMessage]); const sendHandshake = useCallback(() => { diff --git a/ui/src/routes/devices.$id.tsx b/ui/src/routes/devices.$id.tsx index 444e347d..8a363199 100644 --- a/ui/src/routes/devices.$id.tsx +++ b/ui/src/routes/devices.$id.tsx @@ -136,6 +136,7 @@ export default function KvmIdRoute() { rpcDataChannel, setTransceiver, setRpcHidChannel, + setRpcHidUnreliableNonOrderedChannel, setRpcHidUnreliableChannel, } = useRTCStore(); @@ -489,9 +490,8 @@ export default function KvmIdRoute() { setRpcHidChannel(rpcHidChannel); }; - const rpcHidUnreliableChannel = pc.createDataChannel("hidrpc-unreliable", { - // We don't need to be ordered, as we're using the unreliable channel for keepalive messages - ordered: false, + const rpcHidUnreliableChannel = pc.createDataChannel("hidrpc-unreliable-ordered", { + ordered: true, maxRetransmits: 0, }); rpcHidUnreliableChannel.binaryType = "arraybuffer"; @@ -499,6 +499,15 @@ export default function KvmIdRoute() { setRpcHidUnreliableChannel(rpcHidUnreliableChannel); }; + const rpcHidUnreliableNonOrderedChannel = pc.createDataChannel("hidrpc-unreliable-nonordered", { + ordered: false, + maxRetransmits: 0, + }); + rpcHidUnreliableNonOrderedChannel.binaryType = "arraybuffer"; + rpcHidUnreliableNonOrderedChannel.onopen = () => { + setRpcHidUnreliableNonOrderedChannel(rpcHidUnreliableNonOrderedChannel); + }; + setPeerConnection(pc); }, [ cleanupAndStopReconnecting, @@ -510,6 +519,7 @@ export default function KvmIdRoute() { setPeerConnectionState, setRpcDataChannel, setRpcHidChannel, + setRpcHidUnreliableNonOrderedChannel, setRpcHidUnreliableChannel, setTransceiver, ]); diff --git a/webrtc.go b/webrtc.go index e43c2512..333a58b8 100644 --- a/webrtc.go +++ b/webrtc.go @@ -29,7 +29,12 @@ type Session struct { hidRPCAvailable bool hidQueueLock sync.Mutex - hidQueue []chan webrtc.DataChannelMessage + hidQueue []chan hidQueueMessage +} + +type hidQueueMessage struct { + webrtc.DataChannelMessage + channel string } type SessionConfig struct { @@ -78,16 +83,59 @@ func (s *Session) initQueues() { s.hidQueueLock.Lock() defer s.hidQueueLock.Unlock() - s.hidQueue = make([]chan webrtc.DataChannelMessage, 0) + s.hidQueue = make([]chan hidQueueMessage, 0) for i := 0; i < 4; i++ { - q := make(chan webrtc.DataChannelMessage, 256) + q := make(chan hidQueueMessage, 256) s.hidQueue = append(s.hidQueue, q) } } func (s *Session) handleQueues(index int) { for msg := range s.hidQueue[index] { - onHidMessage(msg.Data, s) + onHidMessage(msg, s) + } +} + +func getOnHidMessageHandler(session *Session, scopedLogger *zerolog.Logger, channel string) func(msg webrtc.DataChannelMessage) { + return func(msg webrtc.DataChannelMessage) { + l := scopedLogger.With(). + Str("channel", channel). + Int("length", len(msg.Data)). + Logger() + // only log data if the log level is debug or lower + if scopedLogger.GetLevel() > zerolog.DebugLevel { + l = l.With().Str("data", string(msg.Data)).Logger() + } + + if msg.IsString { + l.Warn().Msg("received string data in HID RPC message handler") + return + } + + if len(msg.Data) < 1 { + l.Warn().Msg("received empty data in HID RPC message handler") + return + } + + l.Trace().Msg("received data in HID RPC message handler") + + // Enqueue to ensure ordered processing + queueIndex := hidrpc.GetQueueIndex(hidrpc.MessageType(msg.Data[0])) + if queueIndex >= len(session.hidQueue) || queueIndex < 0 { + l.Warn().Int("queueIndex", queueIndex).Msg("received data in HID RPC message handler, but queue index not found") + queueIndex = 3 + } + + queue := session.hidQueue[queueIndex] + if queue != nil { + queue <- hidQueueMessage{ + DataChannelMessage: msg, + channel: channel, + } + } else { + l.Warn().Int("queueIndex", queueIndex).Msg("received data in HID RPC message handler, but queue is nil") + return + } } } @@ -145,41 +193,6 @@ func newSession(config SessionConfig) (*Session, error) { go session.handleQueues(i) } - onHidMessage := func(msg webrtc.DataChannelMessage) { - l := scopedLogger.With().Int("length", len(msg.Data)).Logger() - // only log data if the log level is debug or lower - if scopedLogger.GetLevel() > zerolog.DebugLevel { - l = l.With().Str("data", string(msg.Data)).Logger() - } - - if msg.IsString { - l.Warn().Msg("received string data in HID RPC message handler") - return - } - - if len(msg.Data) < 1 { - l.Warn().Msg("received empty data in HID RPC message handler") - return - } - - l.Trace().Msg("received data in HID RPC message handler") - - // Enqueue to ensure ordered processing - queueIndex := hidrpc.GetQueueIndex(hidrpc.MessageType(msg.Data[0])) - if queueIndex >= len(session.hidQueue) || queueIndex < 0 { - l.Warn().Int("queueIndex", queueIndex).Msg("received data in HID RPC message handler, but queue index not found") - queueIndex = 3 - } - - queue := session.hidQueue[queueIndex] - if queue != nil { - queue <- msg - } else { - l.Warn().Int("queueIndex", queueIndex).Msg("received data in HID RPC message handler, but queue is nil") - return - } - } - peerConnection.OnDataChannel(func(d *webrtc.DataChannel) { defer func() { if r := recover(); r != nil { @@ -192,10 +205,12 @@ func newSession(config SessionConfig) (*Session, error) { switch d.Label() { case "hidrpc": session.HidChannel = d - d.OnMessage(onHidMessage) - // we won't send anything over the unreliable channel - case "hidrpc-unreliable": - d.OnMessage(onHidMessage) + d.OnMessage(getOnHidMessageHandler(session, scopedLogger, "hidrpc")) + // we won't send anything over the unreliable channels + case "hidrpc-unreliable-ordered": + d.OnMessage(getOnHidMessageHandler(session, scopedLogger, "hidrpc-unreliable-ordered")) + case "hidrpc-unreliable-nonordered": + d.OnMessage(getOnHidMessageHandler(session, scopedLogger, "hidrpc-unreliable-nonordered")) case "rpc": session.RPCChannel = d d.OnMessage(func(msg webrtc.DataChannelMessage) { From cbf1c7fba68dc9c4a0a083537444bc85d3a28020 Mon Sep 17 00:00:00 2001 From: Siyuan Miao Date: Thu, 11 Sep 2025 17:46:50 +0200 Subject: [PATCH 07/14] chore: adjust auto release key interval --- internal/usbgadget/hid_keyboard.go | 27 +++++++-------------------- ui/src/hooks/useKeyboard.ts | 2 +- 2 files changed, 8 insertions(+), 21 deletions(-) diff --git a/internal/usbgadget/hid_keyboard.go b/internal/usbgadget/hid_keyboard.go index 40dff0be..e3997ea6 100644 --- a/internal/usbgadget/hid_keyboard.go +++ b/internal/usbgadget/hid_keyboard.go @@ -22,6 +22,11 @@ var keyboardConfig = gadgetConfigItem{ reportDesc: keyboardReportDesc, } +// macOS default: 15 * 15 = 225ms https://discussions.apple.com/thread/1316947?sortBy=rank +// Linux default: 250ms https://man.archlinux.org/man/kbdrate.8.en +// Windows default: 1ms `HKEY_CURRENT_USER\Control Panel\Accessibility\Keyboard Response\AutoRepeatDelay` +const autoReleaseKeyboardInterval = time.Millisecond * 225 + // Source: https://www.kernel.org/doc/Documentation/usb/gadget_hid.txt var keyboardReportDesc = []byte{ 0x05, 0x01, /* USAGE_PAGE (Generic Desktop) */ @@ -173,8 +178,6 @@ func (u *UsbGadget) SetOnKeysDownChange(f func(state KeysDownState)) { u.onKeysDownChange = &f } -const autoReleaseKeyboardInterval = time.Millisecond * 450 - func (u *UsbGadget) scheduleAutoRelease(key byte) { u.kbdAutoReleaseLock.Lock() defer u.kbdAutoReleaseLock.Unlock() @@ -222,7 +225,8 @@ func (u *UsbGadget) performAutoRelease(key byte) { default: } - _, err := u.keypressReport(key, false, false) + // we just reset the keyboard state to 0 no matter what + _, err := u.keypressReport(0, false, false) if err != nil { u.log.Warn().Uint8("key", key).Msg("failed to auto-release keyboard key") } @@ -453,23 +457,6 @@ func (u *UsbGadget) keypressReport(key byte, press bool, autoRelease bool) (Keys } } - // if autoRelease { - // u.kbdAutoReleaseLock.Lock() - // u.kbdAutoReleaseLock.Unlock() - // ll.Uint8("key", key).Msg("locking kbdAutoReleaseLock, autoReleasLastKey reset") - - // defer func() { - // ll.Uint8("key", key).Msg("unlocked kbdAutoReleaseLock, autoReleasLastKey reset") - // }() - - // if u.kbdAutoReleaseLastKey == key { - // ll.Uint8("key", key).Msg("key already released by auto-release, skipping") - // u.kbdAutoReleaseLastKey = 0 - - // return u.UpdateKeysDown(modifier, keys), nil - // } - // } - err := u.keyboardWriteHidFile(modifier, keys) if err != nil { u.log.Warn().Uint8("modifier", modifier).Uints8("keys", keys).Msg("Could not write keypress report to hidg0") diff --git a/ui/src/hooks/useKeyboard.ts b/ui/src/hooks/useKeyboard.ts index fb9fa73a..cc9796f3 100644 --- a/ui/src/hooks/useKeyboard.ts +++ b/ui/src/hooks/useKeyboard.ts @@ -104,7 +104,7 @@ export default function useKeyboard() { } }; - const KEEPALIVE_INTERVAL = 200; // 200ms interval + const KEEPALIVE_INTERVAL = 75; // 200ms interval const cancelKeepAlive = useCallback(() => { if (keepAliveTimerRef.current) { From 67ea3c04322a26df2b3035a971f3f3241122804e Mon Sep 17 00:00:00 2001 From: Siyuan Miao Date: Fri, 12 Sep 2025 10:41:23 +0200 Subject: [PATCH 08/14] chore: update logging for kbdAutoReleaseLock --- internal/usbgadget/hid_keyboard.go | 17 ++++++++++------- internal/usbgadget/utils.go | 6 ++++++ 2 files changed, 16 insertions(+), 7 deletions(-) diff --git a/internal/usbgadget/hid_keyboard.go b/internal/usbgadget/hid_keyboard.go index e3997ea6..0c17355d 100644 --- a/internal/usbgadget/hid_keyboard.go +++ b/internal/usbgadget/hid_keyboard.go @@ -179,8 +179,9 @@ func (u *UsbGadget) SetOnKeysDownChange(f func(state KeysDownState)) { } func (u *UsbGadget) scheduleAutoRelease(key byte) { + u.log.Trace().Msg("scheduling autoRelease") u.kbdAutoReleaseLock.Lock() - defer u.kbdAutoReleaseLock.Unlock() + defer unlockWithLog(&u.kbdAutoReleaseLock, u.log, "autoRelease scheduled") if u.kbdAutoReleaseTimer != nil { u.kbdAutoReleaseTimer.Stop() @@ -192,8 +193,9 @@ func (u *UsbGadget) scheduleAutoRelease(key byte) { } func (u *UsbGadget) cancelAutoRelease() { + u.log.Trace().Msg("cancelling autoRelease") u.kbdAutoReleaseLock.Lock() - defer u.kbdAutoReleaseLock.Unlock() + defer unlockWithLog(&u.kbdAutoReleaseLock, u.log, "autoRelease cancelled") if u.kbdAutoReleaseTimer != nil { u.kbdAutoReleaseTimer.Stop() @@ -201,10 +203,9 @@ func (u *UsbGadget) cancelAutoRelease() { } func (u *UsbGadget) DelayAutoRelease() { + u.log.Trace().Msg("delaying autoRelease") u.kbdAutoReleaseLock.Lock() - defer u.kbdAutoReleaseLock.Unlock() - - u.log.Trace().Msg("delaying auto-release") + defer unlockWithLog(&u.kbdAutoReleaseLock, u.log, "autoRelease delayed") if u.kbdAutoReleaseTimer == nil { return @@ -216,8 +217,9 @@ func (u *UsbGadget) DelayAutoRelease() { } func (u *UsbGadget) performAutoRelease(key byte) { + u.log.Trace().Msg("performing autoRelease") u.kbdAutoReleaseLock.Lock() - defer u.kbdAutoReleaseLock.Unlock() + defer unlockWithLog(&u.kbdAutoReleaseLock, u.log, "autoRelease performed") select { case <-u.keyboardStateCtx.Done(): @@ -464,9 +466,10 @@ func (u *UsbGadget) keypressReport(key byte, press bool, autoRelease bool) (Keys if press { { + u.log.Trace().Msg("acquiring kbdAutoReleaseLock to update last key") u.kbdAutoReleaseLock.Lock() u.kbdAutoReleaseLastKey = key - u.kbdAutoReleaseLock.Unlock() + unlockWithLog(&u.kbdAutoReleaseLock, u.log, "last key updated") } if autoRelease { diff --git a/internal/usbgadget/utils.go b/internal/usbgadget/utils.go index d51f9e40..56f9c308 100644 --- a/internal/usbgadget/utils.go +++ b/internal/usbgadget/utils.go @@ -9,6 +9,7 @@ import ( "path/filepath" "strconv" "strings" + "sync" "time" "github.com/rs/zerolog" @@ -164,3 +165,8 @@ func (u *UsbGadget) resetLogSuppressionCounter(counterName string) { u.logSuppressionCounter[counterName] = 0 } } + +func unlockWithLog(lock *sync.Mutex, logger *zerolog.Logger, msg string, args ...any) { + logger.Trace().Msgf(msg, args...) + lock.Unlock() +} From 0bb3a6f3c29442d90f360ffd7df09c7f60c1b6d8 Mon Sep 17 00:00:00 2001 From: Siyuan Miao Date: Fri, 12 Sep 2025 10:41:40 +0200 Subject: [PATCH 09/14] chore: update comment for KEEPALIVE_INTERVAL --- ui/src/hooks/useKeyboard.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ui/src/hooks/useKeyboard.ts b/ui/src/hooks/useKeyboard.ts index cc9796f3..561485fb 100644 --- a/ui/src/hooks/useKeyboard.ts +++ b/ui/src/hooks/useKeyboard.ts @@ -104,7 +104,7 @@ export default function useKeyboard() { } }; - const KEEPALIVE_INTERVAL = 75; // 200ms interval + const KEEPALIVE_INTERVAL = 75; // TODO: use an adaptive interval based on RTT later const cancelKeepAlive = useCallback(() => { if (keepAliveTimerRef.current) { From 39d885c973bcb868a8257aa9fcb868f2b9bf44f7 Mon Sep 17 00:00:00 2001 From: Siyuan Miao Date: Fri, 12 Sep 2025 12:33:43 +0200 Subject: [PATCH 10/14] fix: should cancelAutorelease when pressed is true --- internal/usbgadget/hid_keyboard.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/internal/usbgadget/hid_keyboard.go b/internal/usbgadget/hid_keyboard.go index 0c17355d..786cb9c8 100644 --- a/internal/usbgadget/hid_keyboard.go +++ b/internal/usbgadget/hid_keyboard.go @@ -25,7 +25,7 @@ var keyboardConfig = gadgetConfigItem{ // macOS default: 15 * 15 = 225ms https://discussions.apple.com/thread/1316947?sortBy=rank // Linux default: 250ms https://man.archlinux.org/man/kbdrate.8.en // Windows default: 1ms `HKEY_CURRENT_USER\Control Panel\Accessibility\Keyboard Response\AutoRepeatDelay` -const autoReleaseKeyboardInterval = time.Millisecond * 225 +const autoReleaseKeyboardInterval = time.Millisecond * 100 // Source: https://www.kernel.org/doc/Documentation/usb/gadget_hid.txt var keyboardReportDesc = []byte{ @@ -473,11 +473,11 @@ func (u *UsbGadget) keypressReport(key byte, press bool, autoRelease bool) (Keys } if autoRelease { - u.scheduleAutoRelease(key) + u.cancelAutoRelease() } } else { if autoRelease { - u.cancelAutoRelease() + u.scheduleAutoRelease(key) } } From cc71e3273cdab3980a951c0002b429d77c3f6788 Mon Sep 17 00:00:00 2001 From: Siyuan Miao Date: Fri, 12 Sep 2025 12:48:34 +0200 Subject: [PATCH 11/14] fix: handshake won't happen if webrtc reconnects --- hidrpc.go | 5 ++++- ui/src/hooks/stores.ts | 4 ++-- ui/src/hooks/useHidRpc.ts | 14 ++++++++++++++ 3 files changed, 20 insertions(+), 3 deletions(-) diff --git a/hidrpc.go b/hidrpc.go index 2c3ca27b..64018a0e 100644 --- a/hidrpc.go +++ b/hidrpc.go @@ -122,7 +122,10 @@ func reportHidRPC(params any, session *Session) { } if !session.hidRPCAvailable || session.HidChannel == nil { - logger.Warn().Msg("HID RPC is not available, skipping reportHidRPC") + logger.Warn(). + Bool("hidRPCAvailable", session.hidRPCAvailable). + Bool("HidChannel", session.HidChannel != nil). + Msg("HID RPC is not available, skipping reportHidRPC") return } diff --git a/ui/src/hooks/stores.ts b/ui/src/hooks/stores.ts index b849fa55..9d5117c8 100644 --- a/ui/src/hooks/stores.ts +++ b/ui/src/hooks/stores.ts @@ -106,7 +106,7 @@ export interface RTCState { rpcDataChannel: RTCDataChannel | null; rpcHidProtocolVersion: number | null; - setRpcHidProtocolVersion: (version: number) => void; + setRpcHidProtocolVersion: (version: number | null) => void; rpcHidChannel: RTCDataChannel | null; setRpcHidChannel: (channel: RTCDataChannel) => void; @@ -164,7 +164,7 @@ export const useRTCStore = create(set => ({ setRpcDataChannel: (channel: RTCDataChannel) => set({ rpcDataChannel: channel }), rpcHidProtocolVersion: null, - setRpcHidProtocolVersion: (version: number) => set({ rpcHidProtocolVersion: version }), + setRpcHidProtocolVersion: (version: number | null) => set({ rpcHidProtocolVersion: version }), rpcHidChannel: null, setRpcHidChannel: (channel: RTCDataChannel) => set({ rpcHidChannel: channel }), diff --git a/ui/src/hooks/useHidRpc.ts b/ui/src/hooks/useHidRpc.ts index 58c04f42..0670acb8 100644 --- a/ui/src/hooks/useHidRpc.ts +++ b/ui/src/hooks/useHidRpc.ts @@ -167,10 +167,24 @@ export function useHidRpc(onHidRpcMessage?: (payload: RpcMessage) => void) { onHidRpcMessage?.(message); }; + const openHandler = () => { + console.warn("HID RPC channel opened"); + sendHandshake(); + }; + + const closeHandler = () => { + console.warn("HID RPC channel closed"); + setRpcHidProtocolVersion(null); + }; + rpcHidChannel.addEventListener("message", messageHandler); + rpcHidChannel.addEventListener("close", closeHandler); + rpcHidChannel.addEventListener("open", openHandler); return () => { rpcHidChannel.removeEventListener("message", messageHandler); + rpcHidChannel.removeEventListener("close", closeHandler); + rpcHidChannel.removeEventListener("open", openHandler); }; }, [ From f056d8af457bd31e83249f3779ff569ee63445b9 Mon Sep 17 00:00:00 2001 From: Siyuan Miao Date: Fri, 12 Sep 2025 12:50:46 +0200 Subject: [PATCH 12/14] chore: add trace log for writeWithTimeout --- internal/usbgadget/utils.go | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/internal/usbgadget/utils.go b/internal/usbgadget/utils.go index 56f9c308..85bf1579 100644 --- a/internal/usbgadget/utils.go +++ b/internal/usbgadget/utils.go @@ -121,6 +121,12 @@ func (u *UsbGadget) writeWithTimeout(file *os.File, data []byte) (n int, err err return } + u.log.Trace(). + Str("file", file.Name()). + Bytes("data", data). + Err(err). + Msg("write failed") + if errors.Is(err, os.ErrDeadlineExceeded) { u.logWithSuppression( fmt.Sprintf("writeWithTimeout_%s", file.Name()), From 8d0c2c24ad4bde78b6bcd03acf19bb2bf44d7855 Mon Sep 17 00:00:00 2001 From: Siyuan Miao Date: Fri, 12 Sep 2025 13:32:02 +0200 Subject: [PATCH 13/14] chore: add timeout for KeypressReport --- internal/usbgadget/hid_keyboard.go | 30 ++++++++++++++++++++++++++---- 1 file changed, 26 insertions(+), 4 deletions(-) diff --git a/internal/usbgadget/hid_keyboard.go b/internal/usbgadget/hid_keyboard.go index 786cb9c8..76e5a125 100644 --- a/internal/usbgadget/hid_keyboard.go +++ b/internal/usbgadget/hid_keyboard.go @@ -396,10 +396,7 @@ var KeyCodeToMaskMap = map[byte]byte{ RightSuper: ModifierMaskRightSuper, } -func (u *UsbGadget) keypressReport(key byte, press bool, autoRelease bool) (KeysDownState, error) { - u.keyboardLock.Lock() - defer u.keyboardLock.Unlock() - +func (u *UsbGadget) keypressReportNonThreadSafe(key byte, press bool, autoRelease bool) (KeysDownState, error) { defer u.resetUserInputTime() // IMPORTANT: This code parallels the logic in the kernel's hid-gadget driver @@ -484,6 +481,31 @@ func (u *UsbGadget) keypressReport(key byte, press bool, autoRelease bool) (Keys return u.UpdateKeysDown(modifier, keys), err } +type keypressReportResult struct { + KeysDownState KeysDownState + Error error +} + +func (u *UsbGadget) keypressReport(key byte, press bool, autoRelease bool) (KeysDownState, error) { + u.keyboardLock.Lock() + defer u.keyboardLock.Unlock() + + r := make(chan keypressReportResult) + go func() { + state, err := u.keypressReportNonThreadSafe(key, press, autoRelease) + r <- keypressReportResult{KeysDownState: state, Error: err} + }() + + select { + case <-time.After(1 * time.Second): + u.log.Warn().Msg("keypressReport timed out, possibly stuck") + return u.keysDownState, fmt.Errorf("keypressReport timed out, possibly stuck") + case ret := <-r: + u.log.Debug().Msg("keypressReport handled") + return ret.KeysDownState, ret.Error + } +} + func (u *UsbGadget) KeypressReport(key byte, press bool) (KeysDownState, error) { return u.keypressReport(key, press, true) } From 12210fac96ef7b9ddc4d0bdf4c7e476e7455505a Mon Sep 17 00:00:00 2001 From: Siyuan Miao Date: Fri, 12 Sep 2025 14:51:11 +0200 Subject: [PATCH 14/14] chore: use the proper key to send release command --- go.mod | 2 ++ go.sum | 3 ++ hidrpc.go | 7 ++++- internal/usbgadget/hid_keyboard.go | 44 ++++++++++++++++++++++-------- 4 files changed, 44 insertions(+), 12 deletions(-) diff --git a/go.mod b/go.mod index 72e57cd2..3a1b2543 100644 --- a/go.mod +++ b/go.mod @@ -12,6 +12,7 @@ require ( github.com/gin-contrib/logger v1.2.6 github.com/gin-gonic/gin v1.10.1 github.com/go-co-op/gocron/v2 v2.16.5 + github.com/google/flatbuffers v25.2.10+incompatible github.com/google/uuid v1.6.0 github.com/guregu/null/v6 v6.0.0 github.com/gwatts/rootcerts v0.0.0-20250901182336-dc5ae18bd79f @@ -23,6 +24,7 @@ require ( github.com/prometheus/common v0.66.0 github.com/prometheus/procfs v0.17.0 github.com/psanford/httpreadat v0.1.0 + github.com/rs/xid v1.6.0 github.com/rs/zerolog v1.34.0 github.com/sourcegraph/tf-dag v0.2.2-0.20250131204052-3e8ff1477b4f github.com/stretchr/testify v1.11.1 diff --git a/go.sum b/go.sum index 36087a28..7d42b802 100644 --- a/go.sum +++ b/go.sum @@ -53,6 +53,8 @@ github.com/go-playground/validator/v10 v10.26.0/go.mod h1:I5QpIEbmr8On7W0TktmJAu github.com/goccy/go-json v0.10.5 h1:Fq85nIqj+gXn/S5ahsiTlK3TmC85qgirsdTP/+DeaC4= github.com/goccy/go-json v0.10.5/go.mod h1:oq7eo15ShAhp70Anwd5lgX2pLfOS3QCiwU/PULtXL6M= github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= +github.com/google/flatbuffers v25.2.10+incompatible h1:F3vclr7C3HpB1k9mxCGRMXq6FdUalZ6H/pNX4FP1v0Q= +github.com/google/flatbuffers v25.2.10+incompatible/go.mod h1:1AeVuKshWv4vARoZatz6mlQ0JxURH0Kv5+zNeJKJCa8= github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= @@ -152,6 +154,7 @@ github.com/robfig/cron/v3 v3.0.1 h1:WdRxkvbJztn8LMz/QEvLN5sBU+xKpSqwwUO1Pjr4qDs= github.com/robfig/cron/v3 v3.0.1/go.mod h1:eQICP3HwyT7UooqI/z+Ov+PtYAWygg1TEWWzGIFLtro= github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ= github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc= +github.com/rs/xid v1.6.0 h1:fV591PaemRlL6JfRxGDEPl69wICngIQ3shQtzfy2gxU= github.com/rs/xid v1.6.0/go.mod h1:7XoLgs4eV+QndskICGsho+ADou8ySMSjJKDIan90Nz0= github.com/rs/zerolog v1.34.0 h1:k43nTLIwcTVQAncfCw4KZ2VY6ukYoZaBPNOE8txlOeY= github.com/rs/zerolog v1.34.0/go.mod h1:bJsvje4Z08ROH4Nhs5iH600c3IkWhwp44iRc54W6wYQ= diff --git a/hidrpc.go b/hidrpc.go index 64018a0e..12824ac6 100644 --- a/hidrpc.go +++ b/hidrpc.go @@ -6,6 +6,7 @@ import ( "github.com/jetkvm/kvm/internal/hidrpc" "github.com/jetkvm/kvm/internal/usbgadget" + "github.com/rs/zerolog" ) func handleHidRPCMessage(message hidrpc.Message, session *Session) { @@ -75,7 +76,9 @@ func onHidMessage(msg hidQueueMessage, session *Session) { return } - scopedLogger = scopedLogger.With().Str("descr", message.String()).Logger() + if scopedLogger.GetLevel() <= zerolog.DebugLevel { + scopedLogger = scopedLogger.With().Str("descr", message.String()).Logger() + } t := time.Now() @@ -166,7 +169,9 @@ func (s *Session) reportHidRPCKeyboardLedState(state usbgadget.KeyboardState) { func (s *Session) reportHidRPCKeysDownState(state usbgadget.KeysDownState) { if !s.hidRPCAvailable { + usbLogger.Debug().Interface("state", state).Msg("reporting keys down state") writeJSONRPCEvent("keysDownState", state, s) } + usbLogger.Debug().Interface("state", state).Msg("reporting keys down state, calling reportHidRPC") reportHidRPC(state, s) } diff --git a/internal/usbgadget/hid_keyboard.go b/internal/usbgadget/hid_keyboard.go index 76e5a125..53e232c0 100644 --- a/internal/usbgadget/hid_keyboard.go +++ b/internal/usbgadget/hid_keyboard.go @@ -6,6 +6,9 @@ import ( "fmt" "os" "time" + + "github.com/rs/xid" + "github.com/rs/zerolog" ) var keyboardConfig = gadgetConfigItem{ @@ -188,7 +191,7 @@ func (u *UsbGadget) scheduleAutoRelease(key byte) { } u.kbdAutoReleaseTimer = time.AfterFunc(autoReleaseKeyboardInterval, func() { - u.performAutoRelease(key) + u.performAutoRelease() }) } @@ -216,10 +219,12 @@ func (u *UsbGadget) DelayAutoRelease() { u.log.Trace().Msg("auto-release timer reset") } -func (u *UsbGadget) performAutoRelease(key byte) { +func (u *UsbGadget) performAutoRelease() { u.log.Trace().Msg("performing autoRelease") u.kbdAutoReleaseLock.Lock() - defer unlockWithLog(&u.kbdAutoReleaseLock, u.log, "autoRelease performed") + defer unlockWithLog(&u.kbdAutoReleaseLock, u.log, "autoRelease unlocked") + + key := u.kbdAutoReleaseLastKey select { case <-u.keyboardStateCtx.Done(): @@ -228,7 +233,8 @@ func (u *UsbGadget) performAutoRelease(key byte) { } // we just reset the keyboard state to 0 no matter what - _, err := u.keypressReport(0, false, false) + u.log.Trace().Uint8("key", key).Msg("auto-releasing keyboard key") + _, err := u.keypressReport(key, false, false) if err != nil { u.log.Warn().Uint8("key", key).Msg("failed to auto-release keyboard key") } @@ -399,12 +405,20 @@ var KeyCodeToMaskMap = map[byte]byte{ func (u *UsbGadget) keypressReportNonThreadSafe(key byte, press bool, autoRelease bool) (KeysDownState, error) { defer u.resetUserInputTime() + l := u.log.With().Uint8("key", key).Bool("press", press).Bool("autoRelease", autoRelease).Logger() + if l.GetLevel() <= zerolog.DebugLevel { + requestID := xid.New() + l = l.With().Str("requestID", requestID.String()).Logger() + } + // IMPORTANT: This code parallels the logic in the kernel's hid-gadget driver // for handling key presses and releases. It ensures that the USB gadget // behaves similarly to a real USB HID keyboard. This logic is paralleled // in the client/browser-side code in useKeyboard.ts so make sure to keep // them in sync. - var state = u.keysDownState + var state = u.GetKeysDownState() + l.Trace().Interface("state", state).Msg("got keys down state") + modifier := state.Modifier keys := append([]byte(nil), state.Keys...) @@ -444,37 +458,45 @@ func (u *UsbGadget) keypressReportNonThreadSafe(key byte, press bool, autoReleas // If we reach here it means we didn't find an empty slot or the key in the buffer if overrun { if press { - u.log.Error().Uint8("key", key).Msg("keyboard buffer overflow, key not added") + l.Error().Msg("keyboard buffer overflow, key not added") // Fill all key slots with ErrorRollOver (0x01) to indicate overflow for i := range keys { keys[i] = hidErrorRollOver } } else { // If we are releasing a key, and we didn't find it in a slot, who cares? - u.log.Warn().Uint8("key", key).Msg("key not found in buffer, nothing to release") + l.Warn().Msg("key not found in buffer, nothing to release") } } } + if l.GetLevel() <= zerolog.DebugLevel { + l = l.With().Uint8("modifier", modifier).Uints8("keys", keys).Logger() + } + + l.Trace().Msg("writing keypress report to hidg0") + err := u.keyboardWriteHidFile(modifier, keys) if err != nil { - u.log.Warn().Uint8("modifier", modifier).Uints8("keys", keys).Msg("Could not write keypress report to hidg0") + l.Warn().Msg("Could not write keypress report to hidg0") } + l.Trace().Msg("keypress report written to hidg0") + if press { { - u.log.Trace().Msg("acquiring kbdAutoReleaseLock to update last key") + l.Trace().Msg("acquiring kbdAutoReleaseLock to update last key") u.kbdAutoReleaseLock.Lock() u.kbdAutoReleaseLastKey = key unlockWithLog(&u.kbdAutoReleaseLock, u.log, "last key updated") } if autoRelease { - u.cancelAutoRelease() + u.scheduleAutoRelease(key) } } else { if autoRelease { - u.scheduleAutoRelease(key) + u.cancelAutoRelease() } }