From 2f0aa18d1d2b7f43729b576b79437b72ffcea901 Mon Sep 17 00:00:00 2001 From: Siyuan Miao Date: Thu, 11 Sep 2025 13:05:20 +0200 Subject: [PATCH 1/7] feat: release keyPress automatically --- internal/usbgadget/hid_keyboard.go | 50 ++++++++++++++++++++++++++++++ internal/usbgadget/usbgadget.go | 35 +++++++++++++++++++++ 2 files changed, 85 insertions(+) diff --git a/internal/usbgadget/hid_keyboard.go b/internal/usbgadget/hid_keyboard.go index fb710c20..8b7080a7 100644 --- a/internal/usbgadget/hid_keyboard.go +++ b/internal/usbgadget/hid_keyboard.go @@ -173,6 +173,50 @@ func (u *UsbGadget) SetOnKeysDownChange(f func(state KeysDownState)) { u.onKeysDownChange = &f } +const autoReleaseKeyboardInterval = time.Millisecond * 300 + +func (u *UsbGadget) scheduleAutoRelease(key byte) { + u.keysAutoReleaseLock.Lock() + defer u.keysAutoReleaseLock.Unlock() + + if u.keysAutoReleaseTimer != nil { + u.keysAutoReleaseTimer.Stop() + } + + u.keysAutoReleaseTimer = time.AfterFunc(autoReleaseKeyboardInterval, func() { + u.performAutoRelease(key) + }) +} + +func (u *UsbGadget) cancelAutoRelease() { + u.keysAutoReleaseLock.Lock() + defer u.keysAutoReleaseLock.Unlock() + + if u.keysAutoReleaseTimer != nil { + u.keysAutoReleaseTimer.Stop() + } +} + +func (u *UsbGadget) performAutoRelease(key byte) { + u.keysAutoReleaseLock.Lock() + defer u.keysAutoReleaseLock.Unlock() + + select { + case <-u.keyboardStateCtx.Done(): + return + default: + } + + _, err := u.KeypressReport(key, false) + if err != nil { + u.log.Warn().Uint8("key", key).Msg("failed to auto-release keyboard key") + } + + u.keysAutoReleaseTimer = nil + + u.log.Trace().Uint8("key", key).Msg("auto release performed") +} + func (u *UsbGadget) listenKeyboardEvents() { var path string if u.keyboardHidFile != nil { @@ -398,5 +442,11 @@ func (u *UsbGadget) KeypressReport(key byte, press bool) (KeysDownState, error) u.log.Warn().Uint8("modifier", modifier).Uints8("keys", keys).Msg("Could not write keypress report to hidg0") } + if press { + u.scheduleAutoRelease(key) + } else { + u.cancelAutoRelease() + } + return u.UpdateKeysDown(modifier, keys), err } diff --git a/internal/usbgadget/usbgadget.go b/internal/usbgadget/usbgadget.go index 3a01a447..7485fc20 100644 --- a/internal/usbgadget/usbgadget.go +++ b/internal/usbgadget/usbgadget.go @@ -68,6 +68,9 @@ type UsbGadget struct { keyboardState byte // keyboard latched state (NumLock, CapsLock, ScrollLock, Compose, Kana) keysDownState KeysDownState // keyboard dynamic state (modifier keys and pressed keys) + keysAutoReleaseLock sync.Mutex + keysAutoReleaseTimer *time.Timer + keyboardStateLock sync.Mutex keyboardStateCtx context.Context keyboardStateCancel context.CancelFunc @@ -149,3 +152,35 @@ func newUsbGadget(name string, configMap map[string]gadgetConfigItem, enabledDev return g } + +// Close cleans up resources used by the USB gadget +func (u *UsbGadget) Close() error { + // Cancel keyboard state context + if u.keyboardStateCancel != nil { + u.keyboardStateCancel() + } + + // Stop auto-release timer + u.keysAutoReleaseLock.Lock() + if u.keysAutoReleaseTimer != nil { + u.keysAutoReleaseTimer.Stop() + u.keysAutoReleaseTimer = nil + } + u.keysAutoReleaseLock.Unlock() + + // Close HID files + if u.keyboardHidFile != nil { + u.keyboardHidFile.Close() + u.keyboardHidFile = nil + } + if u.absMouseHidFile != nil { + u.absMouseHidFile.Close() + u.absMouseHidFile = nil + } + if u.relMouseHidFile != nil { + u.relMouseHidFile.Close() + u.relMouseHidFile = nil + } + + return nil +} From 4b42c7e7e34eaf3499cdd8bb1b8643ca9d31d62f Mon Sep 17 00:00:00 2001 From: Siyuan Miao Date: Thu, 11 Sep 2025 14:10:18 +0200 Subject: [PATCH 2/7] send keepalive when pressing the key --- hidrpc.go | 2 + internal/hidrpc/hidrpc.go | 25 +++++--- internal/usbgadget/hid_keyboard.go | 98 ++++++++++++++++++++++++------ internal/usbgadget/usbgadget.go | 15 ++--- ui/src/hooks/hidRpc.ts | 12 ++++ ui/src/hooks/useHidRpc.ts | 8 +++ ui/src/hooks/useKeyboard.ts | 77 ++++++++++++++++++----- 7 files changed, 189 insertions(+), 48 deletions(-) diff --git a/hidrpc.go b/hidrpc.go index 74fe687f..52022ba6 100644 --- a/hidrpc.go +++ b/hidrpc.go @@ -29,6 +29,8 @@ func handleHidRPCMessage(message hidrpc.Message, session *Session) { session.reportHidRPCKeysDownState(*keysDownState) } rpcErr = err + case hidrpc.TypeKeypressKeepAliveReport: + gadget.DelayAutoRelease() case hidrpc.TypePointerReport: pointerReport, err := message.PointerReport() if err != nil { diff --git a/internal/hidrpc/hidrpc.go b/internal/hidrpc/hidrpc.go index e9c8c24d..7161dc8f 100644 --- a/internal/hidrpc/hidrpc.go +++ b/internal/hidrpc/hidrpc.go @@ -10,14 +10,15 @@ import ( type MessageType byte const ( - TypeHandshake MessageType = 0x01 - TypeKeyboardReport MessageType = 0x02 - TypePointerReport MessageType = 0x03 - TypeWheelReport MessageType = 0x04 - TypeKeypressReport MessageType = 0x05 - TypeMouseReport MessageType = 0x06 - TypeKeyboardLedState MessageType = 0x32 - TypeKeydownState MessageType = 0x33 + TypeHandshake MessageType = 0x01 + TypeKeyboardReport MessageType = 0x02 + TypePointerReport MessageType = 0x03 + TypeWheelReport MessageType = 0x04 + TypeKeypressReport MessageType = 0x05 + TypeKeypressKeepAliveReport MessageType = 0x09 + TypeMouseReport MessageType = 0x06 + TypeKeyboardLedState MessageType = 0x32 + TypeKeydownState MessageType = 0x33 ) const ( @@ -98,3 +99,11 @@ func NewKeydownStateMessage(state usbgadget.KeysDownState) *Message { d: data, } } + +// NewKeypressKeepAliveMessage creates a new keypress keep alive message. +func NewKeypressKeepAliveMessage() *Message { + return &Message{ + t: TypeKeypressKeepAliveReport, + d: []byte{}, + } +} diff --git a/internal/usbgadget/hid_keyboard.go b/internal/usbgadget/hid_keyboard.go index 8b7080a7..f9bd20db 100644 --- a/internal/usbgadget/hid_keyboard.go +++ b/internal/usbgadget/hid_keyboard.go @@ -173,33 +173,47 @@ func (u *UsbGadget) SetOnKeysDownChange(f func(state KeysDownState)) { u.onKeysDownChange = &f } -const autoReleaseKeyboardInterval = time.Millisecond * 300 +const autoReleaseKeyboardInterval = time.Millisecond * 450 func (u *UsbGadget) scheduleAutoRelease(key byte) { - u.keysAutoReleaseLock.Lock() - defer u.keysAutoReleaseLock.Unlock() + u.kbdAutoReleaseLock.Lock() + defer u.kbdAutoReleaseLock.Unlock() - if u.keysAutoReleaseTimer != nil { - u.keysAutoReleaseTimer.Stop() + if u.kbdAutoReleaseTimer != nil { + u.kbdAutoReleaseTimer.Stop() } - u.keysAutoReleaseTimer = time.AfterFunc(autoReleaseKeyboardInterval, func() { + u.kbdAutoReleaseTimer = time.AfterFunc(autoReleaseKeyboardInterval, func() { u.performAutoRelease(key) }) } func (u *UsbGadget) cancelAutoRelease() { - u.keysAutoReleaseLock.Lock() - defer u.keysAutoReleaseLock.Unlock() + u.kbdAutoReleaseLock.Lock() + defer u.kbdAutoReleaseLock.Unlock() - if u.keysAutoReleaseTimer != nil { - u.keysAutoReleaseTimer.Stop() + if u.kbdAutoReleaseTimer != nil { + u.kbdAutoReleaseTimer.Stop() } } +func (u *UsbGadget) DelayAutoRelease() { + u.kbdAutoReleaseLock.Lock() + defer u.kbdAutoReleaseLock.Unlock() + + u.log.Info().Msg("delaying auto-release") + + if u.kbdAutoReleaseTimer == nil { + return + } + + u.log.Info().Msg("resetting auto-release timer") + u.kbdAutoReleaseTimer.Reset(autoReleaseKeyboardInterval) +} + func (u *UsbGadget) performAutoRelease(key byte) { - u.keysAutoReleaseLock.Lock() - defer u.keysAutoReleaseLock.Unlock() + u.kbdAutoReleaseLock.Lock() + defer u.kbdAutoReleaseLock.Unlock() select { case <-u.keyboardStateCtx.Done(): @@ -207,12 +221,12 @@ func (u *UsbGadget) performAutoRelease(key byte) { default: } - _, err := u.KeypressReport(key, false) + _, err := u.keypressReport(key, false, false) if err != nil { u.log.Warn().Uint8("key", key).Msg("failed to auto-release keyboard key") } - u.keysAutoReleaseTimer = nil + u.kbdAutoReleaseTimer = nil u.log.Trace().Uint8("key", key).Msg("auto release performed") } @@ -375,11 +389,21 @@ var KeyCodeToMaskMap = map[byte]byte{ RightSuper: ModifierMaskRightSuper, } -func (u *UsbGadget) KeypressReport(key byte, press bool) (KeysDownState, error) { +func (u *UsbGadget) keypressReport(key byte, press bool, autoRelease bool) (KeysDownState, error) { + ll := u.log.Info().Str("component", "kbd") + ll.Uint8("key", key).Msg("locking keyboardLock") + u.keyboardLock.Lock() - defer u.keyboardLock.Unlock() + defer func() { + u.keyboardLock.Unlock() + ll.Uint8("key", key).Msg("unlocked keyboardLock") + }() + + ll.Uint8("key", key).Msg("resetting user input time") defer u.resetUserInputTime() + ll.Uint8("key", key).Msg("locked keyboardLock") + // IMPORTANT: This code parallels the logic in the kernel's hid-gadget driver // for handling key presses and releases. It ensures that the USB gadget // behaves similarly to a real USB HID keyboard. This logic is paralleled @@ -437,16 +461,54 @@ func (u *UsbGadget) KeypressReport(key byte, press bool) (KeysDownState, error) } } + ll.Uint8("key", key).Msg("checking if auto-release is enabled") + + // if autoRelease { + // u.kbdAutoReleaseLock.Lock() + // u.kbdAutoReleaseLock.Unlock() + // ll.Uint8("key", key).Msg("locking kbdAutoReleaseLock, autoReleasLastKey reset") + + // defer func() { + // ll.Uint8("key", key).Msg("unlocked kbdAutoReleaseLock, autoReleasLastKey reset") + // }() + + // if u.kbdAutoReleaseLastKey == key { + // ll.Uint8("key", key).Msg("key already released by auto-release, skipping") + // u.kbdAutoReleaseLastKey = 0 + + // return u.UpdateKeysDown(modifier, keys), nil + // } + // } + + ll.Uint8("key", key).Msg("writing keypress report to hidg0") + err := u.keyboardWriteHidFile(modifier, keys) if err != nil { u.log.Warn().Uint8("modifier", modifier).Uints8("keys", keys).Msg("Could not write keypress report to hidg0") } if press { - u.scheduleAutoRelease(key) + { + u.kbdAutoReleaseLock.Lock() + u.kbdAutoReleaseLastKey = key + u.kbdAutoReleaseLock.Unlock() + } + + if autoRelease { + ll.Uint8("key", key).Msg("scheduling auto-release") + u.scheduleAutoRelease(key) + } } else { - u.cancelAutoRelease() + if autoRelease { + ll.Uint8("key", key).Msg("canceling auto-release") + u.cancelAutoRelease() + ll.Uint8("key", key).Msg("auto-release canceled") + } } return u.UpdateKeysDown(modifier, keys), err } + +func (u *UsbGadget) KeypressReport(key byte, press bool) (KeysDownState, error) { + return u.keypressReport(key, press, true) +} diff --git a/internal/usbgadget/usbgadget.go b/internal/usbgadget/usbgadget.go index 7485fc20..e491a78c 100644 --- a/internal/usbgadget/usbgadget.go +++ b/internal/usbgadget/usbgadget.go @@ -68,8 +68,9 @@ type UsbGadget struct { keyboardState byte // keyboard latched state (NumLock, CapsLock, ScrollLock, Compose, Kana) keysDownState KeysDownState // keyboard dynamic state (modifier keys and pressed keys) - keysAutoReleaseLock sync.Mutex - keysAutoReleaseTimer *time.Timer + kbdAutoReleaseLock sync.Mutex + kbdAutoReleaseTimer *time.Timer + kbdAutoReleaseLastKey byte keyboardStateLock sync.Mutex keyboardStateCtx context.Context @@ -161,12 +162,12 @@ func (u *UsbGadget) Close() error { } // Stop auto-release timer - u.keysAutoReleaseLock.Lock() - if u.keysAutoReleaseTimer != nil { - u.keysAutoReleaseTimer.Stop() - u.keysAutoReleaseTimer = nil + u.kbdAutoReleaseLock.Lock() + if u.kbdAutoReleaseTimer != nil { + u.kbdAutoReleaseTimer.Stop() + u.kbdAutoReleaseTimer = nil } - u.keysAutoReleaseLock.Unlock() + u.kbdAutoReleaseLock.Unlock() // Close HID files if u.keyboardHidFile != nil { diff --git a/ui/src/hooks/hidRpc.ts b/ui/src/hooks/hidRpc.ts index 20b8a108..d6b4ad96 100644 --- a/ui/src/hooks/hidRpc.ts +++ b/ui/src/hooks/hidRpc.ts @@ -6,6 +6,7 @@ export const HID_RPC_MESSAGE_TYPES = { PointerReport: 0x03, WheelReport: 0x04, KeypressReport: 0x05, + KeypressKeepAliveReport: 0x09, MouseReport: 0x06, KeyboardLedState: 0x32, KeysDownState: 0x33, @@ -278,12 +279,23 @@ export class MouseReportMessage extends RpcMessage { } } +export class KeypressKeepAliveMessage extends RpcMessage { + constructor() { + super(HID_RPC_MESSAGE_TYPES.KeypressKeepAliveReport); + } + + marshal(): Uint8Array { + return new Uint8Array([this.messageType]); + } +} + export const messageRegistry = { [HID_RPC_MESSAGE_TYPES.Handshake]: HandshakeMessage, [HID_RPC_MESSAGE_TYPES.KeysDownState]: KeysDownStateMessage, [HID_RPC_MESSAGE_TYPES.KeyboardLedState]: KeyboardLedStateMessage, [HID_RPC_MESSAGE_TYPES.KeyboardReport]: KeyboardReportMessage, [HID_RPC_MESSAGE_TYPES.KeypressReport]: KeypressReportMessage, + [HID_RPC_MESSAGE_TYPES.KeypressKeepAliveReport]: KeypressKeepAliveMessage, } export const unmarshalHidRpcMessage = (data: Uint8Array): RpcMessage | undefined => { diff --git a/ui/src/hooks/useHidRpc.ts b/ui/src/hooks/useHidRpc.ts index ea0c7112..d5690849 100644 --- a/ui/src/hooks/useHidRpc.ts +++ b/ui/src/hooks/useHidRpc.ts @@ -6,6 +6,7 @@ import { HID_RPC_VERSION, HandshakeMessage, KeyboardReportMessage, + KeypressKeepAliveMessage, KeypressReportMessage, MouseReportMessage, PointerReportMessage, @@ -13,6 +14,8 @@ import { unmarshalHidRpcMessage, } from "./hidRpc"; +const KEEPALIVE_MESSAGE = new KeypressKeepAliveMessage(); + export function useHidRpc(onHidRpcMessage?: (payload: RpcMessage) => void) { const { rpcHidChannel, setRpcHidProtocolVersion, rpcHidProtocolVersion } = useRTCStore(); const rpcHidReady = useMemo(() => { @@ -68,6 +71,10 @@ export function useHidRpc(onHidRpcMessage?: (payload: RpcMessage) => void) { [sendMessage], ); + const reportKeypressKeepAlive = useCallback(() => { + sendMessage(KEEPALIVE_MESSAGE); + }, [sendMessage]); + const sendHandshake = useCallback(() => { if (rpcHidProtocolVersion) return; if (!rpcHidChannel) return; @@ -143,6 +150,7 @@ export function useHidRpc(onHidRpcMessage?: (payload: RpcMessage) => void) { reportKeypressEvent, reportAbsMouseEvent, reportRelMouseEvent, + reportKeypressKeepAlive, rpcHidProtocolVersion, rpcHidReady, rpcHidStatus, diff --git a/ui/src/hooks/useKeyboard.ts b/ui/src/hooks/useKeyboard.ts index 787df9a9..fb9fa73a 100644 --- a/ui/src/hooks/useKeyboard.ts +++ b/ui/src/hooks/useKeyboard.ts @@ -1,4 +1,4 @@ -import { useCallback } from "react"; +import { useCallback, useRef } from "react"; import { hidErrorRollOver, hidKeyBufferSize, KeysDownState, useHidStore, useRTCStore } from "@/hooks/stores"; import { JsonRpcResponse, useJsonRpc } from "@/hooks/useJsonRpc"; @@ -11,6 +11,9 @@ export default function useKeyboard() { const { rpcDataChannel } = useRTCStore(); const { keysDownState, setKeysDownState, setKeyboardLedState } = useHidStore(); + // Keepalive timer management + const keepAliveTimerRef = useRef(null); + // INTRODUCTION: The earlier version of the JetKVM device shipped with all keyboard state // being tracked on the browser/client-side. When adding the keyPressReport API to the // device-side code, we have to still support the situation where the browser/client-side code @@ -26,6 +29,7 @@ export default function useKeyboard() { const { reportKeyboardEvent: sendKeyboardEventHidRpc, reportKeypressEvent: sendKeypressEventHidRpc, + reportKeypressKeepAlive: sendKeypressKeepAliveHidRpc, rpcHidReady, } = useHidRpc((message) => { switch (message.constructor) { @@ -70,17 +74,6 @@ export default function useKeyboard() { ], ); - // resetKeyboardState is used to reset the keyboard state to no keys pressed and no modifiers. - // This is useful for macros and when the browser loses focus to ensure that the keyboard state - // is clean. - const resetKeyboardState = useCallback( - async () => { - // Reset the keys buffer to zeros and the modifier state to zero - keysDownState.keys.length = hidKeyBufferSize; - keysDownState.keys.fill(0); - keysDownState.modifier = 0; - sendKeyboardEvent(keysDownState); - }, [keysDownState, sendKeyboardEvent]); // executeMacro is used to execute a macro consisting of multiple steps. // Each step can have multiple keys, multiple modifiers and a delay. @@ -111,12 +104,61 @@ export default function useKeyboard() { } }; + const KEEPALIVE_INTERVAL = 200; // 200ms interval + + const cancelKeepAlive = useCallback(() => { + if (keepAliveTimerRef.current) { + clearInterval(keepAliveTimerRef.current); + keepAliveTimerRef.current = null; + } + }, []); + + const scheduleKeepAlive = useCallback(() => { + // Clear existing timer if it exists + if (keepAliveTimerRef.current) { + clearInterval(keepAliveTimerRef.current); + } + + // Create new interval timer + keepAliveTimerRef.current = setInterval(() => { + sendKeypressKeepAliveHidRpc(); + }, KEEPALIVE_INTERVAL); + }, [sendKeypressKeepAliveHidRpc]); + + // resetKeyboardState is used to reset the keyboard state to no keys pressed and no modifiers. + // This is useful for macros and when the browser loses focus to ensure that the keyboard state + // is clean. + const resetKeyboardState = useCallback(async () => { + // Cancel keepalive since we're resetting the keyboard state + cancelKeepAlive(); + + // Reset the keys buffer to zeros and the modifier state to zero + keysDownState.keys.length = hidKeyBufferSize; + keysDownState.keys.fill(0); + keysDownState.modifier = 0; + sendKeyboardEvent(keysDownState); + }, [keysDownState, sendKeyboardEvent, cancelKeepAlive]); + // handleKeyPress is used to handle a key press or release event. // This function handle both key press and key release events. // It checks if the keyPressReport API is available and sends the key press event. // If the keyPressReport API is not available, it simulates the device-side key // handling for legacy devices and updates the keysDownState accordingly. // It then sends the full keyboard state to the device. + + const sendKeypress = useCallback( + (key: number, press: boolean) => { + cancelKeepAlive(); + + sendKeypressEventHidRpc(key, press); + + if (press) { + scheduleKeepAlive(); + } + }, + [sendKeypressEventHidRpc, scheduleKeepAlive, cancelKeepAlive], + ); + const handleKeyPress = useCallback( async (key: number, press: boolean) => { if (rpcDataChannel?.readyState !== "open" && !rpcHidReady) return; @@ -129,7 +171,7 @@ export default function useKeyboard() { // Older device version doesn't support this API, so we will switch to local key handling // In that case we will switch to local key handling and update the keysDownState // in client/browser-side code using simulateDeviceSideKeyHandlingForLegacyDevices. - sendKeypressEventHidRpc(key, press); + sendKeypress(key, press); } else { // if the keyPress api is not available, we need to handle the key locally const downState = simulateDeviceSideKeyHandlingForLegacyDevices(keysDownState, key, press); @@ -147,7 +189,7 @@ export default function useKeyboard() { resetKeyboardState, rpcDataChannel?.readyState, sendKeyboardEvent, - sendKeypressEventHidRpc, + sendKeypress, ], ); @@ -210,5 +252,10 @@ export default function useKeyboard() { return { modifier: modifiers, keys }; } - return { handleKeyPress, resetKeyboardState, executeMacro }; + // Cleanup function to cancel keepalive timer + const cleanup = useCallback(() => { + cancelKeepAlive(); + }, [cancelKeepAlive]); + + return { handleKeyPress, resetKeyboardState, executeMacro, cleanup }; } From 8b079033887bcc0e47b239722a5e0b491433bfca Mon Sep 17 00:00:00 2001 From: Siyuan Miao Date: Thu, 11 Sep 2025 15:00:03 +0200 Subject: [PATCH 3/7] remove logging --- internal/usbgadget/hid_keyboard.go | 18 +----------------- 1 file changed, 1 insertion(+), 17 deletions(-) diff --git a/internal/usbgadget/hid_keyboard.go b/internal/usbgadget/hid_keyboard.go index f9bd20db..0cb978b4 100644 --- a/internal/usbgadget/hid_keyboard.go +++ b/internal/usbgadget/hid_keyboard.go @@ -390,20 +390,11 @@ var KeyCodeToMaskMap = map[byte]byte{ } func (u *UsbGadget) keypressReport(key byte, press bool, autoRelease bool) (KeysDownState, error) { - ll := u.log.Info().Str("component", "kbd") - ll.Uint8("key", key).Msg("locking keyboardLock") - u.keyboardLock.Lock() - defer func() { - u.keyboardLock.Unlock() - ll.Uint8("key", key).Msg("unlocked keyboardLock") - }() + defer u.keyboardLock.Unlock() - ll.Uint8("key", key).Msg("resetting user input time") defer u.resetUserInputTime() - ll.Uint8("key", key).Msg("locked keyboardLock") - // IMPORTANT: This code parallels the logic in the kernel's hid-gadget driver // for handling key presses and releases. It ensures that the USB gadget // behaves similarly to a real USB HID keyboard. This logic is paralleled @@ -461,8 +452,6 @@ func (u *UsbGadget) keypressReport(key byte, press bool, autoRelease bool) (Keys } } - ll.Uint8("key", key).Msg("checking if auto-release is enabled") - // if autoRelease { // u.kbdAutoReleaseLock.Lock() // u.kbdAutoReleaseLock.Unlock() @@ -480,8 +469,6 @@ func (u *UsbGadget) keypressReport(key byte, press bool, autoRelease bool) (Keys // } // } - ll.Uint8("key", key).Msg("writing keypress report to hidg0") - err := u.keyboardWriteHidFile(modifier, keys) if err != nil { u.log.Warn().Uint8("modifier", modifier).Uints8("keys", keys).Msg("Could not write keypress report to hidg0") @@ -495,14 +482,11 @@ func (u *UsbGadget) keypressReport(key byte, press bool, autoRelease bool) (Keys } if autoRelease { - ll.Uint8("key", key).Msg("scheduling auto-release") u.scheduleAutoRelease(key) } } else { if autoRelease { - ll.Uint8("key", key).Msg("canceling auto-release") u.cancelAutoRelease() - ll.Uint8("key", key).Msg("auto-release canceled") } } From 46aee61565e9aa7e5ade1413c633661836b734a2 Mon Sep 17 00:00:00 2001 From: Siyuan Miao Date: Thu, 11 Sep 2025 15:04:35 +0200 Subject: [PATCH 4/7] clean up logging --- internal/hidrpc/message.go | 2 ++ internal/usbgadget/hid_keyboard.go | 5 +++-- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/internal/hidrpc/message.go b/internal/hidrpc/message.go index 84bbda7c..e0f4493e 100644 --- a/internal/hidrpc/message.go +++ b/internal/hidrpc/message.go @@ -43,6 +43,8 @@ func (m *Message) String() string { return fmt.Sprintf("MouseReport{Malformed: %v}", m.d) } return fmt.Sprintf("MouseReport{DX: %d, DY: %d, Button: %d}", m.d[0], m.d[1], m.d[2]) + case TypeKeypressKeepAliveReport: + return "KeypressKeepAliveReport" default: return fmt.Sprintf("Unknown{Type: %d, Data: %v}", m.t, m.d) } diff --git a/internal/usbgadget/hid_keyboard.go b/internal/usbgadget/hid_keyboard.go index 0cb978b4..40dff0be 100644 --- a/internal/usbgadget/hid_keyboard.go +++ b/internal/usbgadget/hid_keyboard.go @@ -201,14 +201,15 @@ func (u *UsbGadget) DelayAutoRelease() { u.kbdAutoReleaseLock.Lock() defer u.kbdAutoReleaseLock.Unlock() - u.log.Info().Msg("delaying auto-release") + u.log.Trace().Msg("delaying auto-release") if u.kbdAutoReleaseTimer == nil { return } - u.log.Info().Msg("resetting auto-release timer") u.kbdAutoReleaseTimer.Reset(autoReleaseKeyboardInterval) + + u.log.Trace().Msg("auto-release timer reset") } func (u *UsbGadget) performAutoRelease(key byte) { From 8e9554f4d60bd5fb7faba19f84ad003ce86464e3 Mon Sep 17 00:00:00 2001 From: Siyuan Miao Date: Thu, 11 Sep 2025 16:16:47 +0200 Subject: [PATCH 5/7] chore: use unreliable channel to send keepalive events --- ui/src/hooks/stores.ts | 6 +++ ui/src/hooks/useHidRpc.ts | 35 +++++++++++++---- ui/src/routes/devices.$id.tsx | 12 ++++++ webrtc.go | 73 +++++++++++++++++++---------------- 4 files changed, 84 insertions(+), 42 deletions(-) diff --git a/ui/src/hooks/stores.ts b/ui/src/hooks/stores.ts index f99fd07d..7007f4e6 100644 --- a/ui/src/hooks/stores.ts +++ b/ui/src/hooks/stores.ts @@ -111,6 +111,9 @@ export interface RTCState { rpcHidChannel: RTCDataChannel | null; setRpcHidChannel: (channel: RTCDataChannel) => void; + rpcHidUnreliableChannel: RTCDataChannel | null; + setRpcHidUnreliableChannel: (channel: RTCDataChannel) => void; + peerConnectionState: RTCPeerConnectionState | null; setPeerConnectionState: (state: RTCPeerConnectionState) => void; @@ -163,6 +166,9 @@ export const useRTCStore = create(set => ({ rpcHidChannel: null, setRpcHidChannel: (channel: RTCDataChannel) => set({ rpcHidChannel: channel }), + rpcHidUnreliableChannel: null, + setRpcHidUnreliableChannel: (channel: RTCDataChannel) => set({ rpcHidUnreliableChannel: channel }), + transceiver: null, setTransceiver: (transceiver: RTCRtpTransceiver) => set({ transceiver }), diff --git a/ui/src/hooks/useHidRpc.ts b/ui/src/hooks/useHidRpc.ts index d5690849..ff22885c 100644 --- a/ui/src/hooks/useHidRpc.ts +++ b/ui/src/hooks/useHidRpc.ts @@ -16,20 +16,35 @@ import { const KEEPALIVE_MESSAGE = new KeypressKeepAliveMessage(); +interface sendMessageParams { + ignoreHandshakeState?: boolean; + useUnreliableChannel?: boolean; +} + export function useHidRpc(onHidRpcMessage?: (payload: RpcMessage) => void) { - const { rpcHidChannel, setRpcHidProtocolVersion, rpcHidProtocolVersion } = useRTCStore(); + const { + rpcHidChannel, + rpcHidUnreliableChannel, + setRpcHidProtocolVersion, + rpcHidProtocolVersion, + } = useRTCStore(); + const rpcHidReady = useMemo(() => { return rpcHidChannel?.readyState === "open" && rpcHidProtocolVersion !== null; }, [rpcHidChannel, rpcHidProtocolVersion]); + const rpcHidUnreliableReady = useMemo(() => { + return rpcHidUnreliableChannel?.readyState === "open" && rpcHidProtocolVersion !== null; + }, [rpcHidUnreliableChannel, rpcHidProtocolVersion]); + const rpcHidStatus = useMemo(() => { if (!rpcHidChannel) return "N/A"; if (rpcHidChannel.readyState !== "open") return rpcHidChannel.readyState; if (!rpcHidProtocolVersion) return "handshaking"; - return `ready (v${rpcHidProtocolVersion})`; - }, [rpcHidChannel, rpcHidProtocolVersion]); + return `ready (v${rpcHidProtocolVersion}${rpcHidUnreliableReady ? "+u" : ""})`; + }, [rpcHidChannel, rpcHidUnreliableReady, rpcHidProtocolVersion]); - const sendMessage = useCallback((message: RpcMessage, ignoreHandshakeState = false) => { + const sendMessage = useCallback((message: RpcMessage, { ignoreHandshakeState, useUnreliableChannel }: sendMessageParams = {}) => { if (rpcHidChannel?.readyState !== "open") return; if (!rpcHidReady && !ignoreHandshakeState) return; @@ -41,8 +56,12 @@ export function useHidRpc(onHidRpcMessage?: (payload: RpcMessage) => void) { } if (!data) return; - rpcHidChannel?.send(data as unknown as ArrayBuffer); - }, [rpcHidChannel, rpcHidReady]); + if (useUnreliableChannel && rpcHidUnreliableReady) { + rpcHidUnreliableChannel?.send(data as unknown as ArrayBuffer); + } else { + rpcHidChannel?.send(data as unknown as ArrayBuffer); + } + }, [rpcHidChannel, rpcHidReady, rpcHidUnreliableChannel, rpcHidUnreliableReady]); const reportKeyboardEvent = useCallback( (keys: number[], modifier: number) => { @@ -72,14 +91,14 @@ export function useHidRpc(onHidRpcMessage?: (payload: RpcMessage) => void) { ); const reportKeypressKeepAlive = useCallback(() => { - sendMessage(KEEPALIVE_MESSAGE); + sendMessage(KEEPALIVE_MESSAGE, { useUnreliableChannel: true }); }, [sendMessage]); const sendHandshake = useCallback(() => { if (rpcHidProtocolVersion) return; if (!rpcHidChannel) return; - sendMessage(new HandshakeMessage(HID_RPC_VERSION), true); + sendMessage(new HandshakeMessage(HID_RPC_VERSION), { ignoreHandshakeState: true }); }, [rpcHidChannel, rpcHidProtocolVersion, sendMessage]); const handleHandshake = useCallback((message: HandshakeMessage) => { diff --git a/ui/src/routes/devices.$id.tsx b/ui/src/routes/devices.$id.tsx index 4318447e..444e347d 100644 --- a/ui/src/routes/devices.$id.tsx +++ b/ui/src/routes/devices.$id.tsx @@ -136,6 +136,7 @@ export default function KvmIdRoute() { rpcDataChannel, setTransceiver, setRpcHidChannel, + setRpcHidUnreliableChannel, } = useRTCStore(); const location = useLocation(); @@ -488,6 +489,16 @@ export default function KvmIdRoute() { setRpcHidChannel(rpcHidChannel); }; + const rpcHidUnreliableChannel = pc.createDataChannel("hidrpc-unreliable", { + // We don't need to be ordered, as we're using the unreliable channel for keepalive messages + ordered: false, + maxRetransmits: 0, + }); + rpcHidUnreliableChannel.binaryType = "arraybuffer"; + rpcHidUnreliableChannel.onopen = () => { + setRpcHidUnreliableChannel(rpcHidUnreliableChannel); + }; + setPeerConnection(pc); }, [ cleanupAndStopReconnecting, @@ -499,6 +510,7 @@ export default function KvmIdRoute() { setPeerConnectionState, setRpcDataChannel, setRpcHidChannel, + setRpcHidUnreliableChannel, setTransceiver, ]); diff --git a/webrtc.go b/webrtc.go index c3d0dc1b..e43c2512 100644 --- a/webrtc.go +++ b/webrtc.go @@ -145,6 +145,41 @@ func newSession(config SessionConfig) (*Session, error) { go session.handleQueues(i) } + onHidMessage := func(msg webrtc.DataChannelMessage) { + l := scopedLogger.With().Int("length", len(msg.Data)).Logger() + // only log data if the log level is debug or lower + if scopedLogger.GetLevel() > zerolog.DebugLevel { + l = l.With().Str("data", string(msg.Data)).Logger() + } + + if msg.IsString { + l.Warn().Msg("received string data in HID RPC message handler") + return + } + + if len(msg.Data) < 1 { + l.Warn().Msg("received empty data in HID RPC message handler") + return + } + + l.Trace().Msg("received data in HID RPC message handler") + + // Enqueue to ensure ordered processing + queueIndex := hidrpc.GetQueueIndex(hidrpc.MessageType(msg.Data[0])) + if queueIndex >= len(session.hidQueue) || queueIndex < 0 { + l.Warn().Int("queueIndex", queueIndex).Msg("received data in HID RPC message handler, but queue index not found") + queueIndex = 3 + } + + queue := session.hidQueue[queueIndex] + if queue != nil { + queue <- msg + } else { + l.Warn().Int("queueIndex", queueIndex).Msg("received data in HID RPC message handler, but queue is nil") + return + } + } + peerConnection.OnDataChannel(func(d *webrtc.DataChannel) { defer func() { if r := recover(); r != nil { @@ -157,40 +192,10 @@ func newSession(config SessionConfig) (*Session, error) { switch d.Label() { case "hidrpc": session.HidChannel = d - d.OnMessage(func(msg webrtc.DataChannelMessage) { - l := scopedLogger.With().Int("length", len(msg.Data)).Logger() - // only log data if the log level is debug or lower - if scopedLogger.GetLevel() > zerolog.DebugLevel { - l = l.With().Str("data", string(msg.Data)).Logger() - } - - if msg.IsString { - l.Warn().Msg("received string data in HID RPC message handler") - return - } - - if len(msg.Data) < 1 { - l.Warn().Msg("received empty data in HID RPC message handler") - return - } - - l.Trace().Msg("received data in HID RPC message handler") - - // Enqueue to ensure ordered processing - queueIndex := hidrpc.GetQueueIndex(hidrpc.MessageType(msg.Data[0])) - if queueIndex >= len(session.hidQueue) || queueIndex < 0 { - l.Warn().Int("queueIndex", queueIndex).Msg("received data in HID RPC message handler, but queue index not found") - queueIndex = 3 - } - - queue := session.hidQueue[queueIndex] - if queue != nil { - queue <- msg - } else { - l.Warn().Int("queueIndex", queueIndex).Msg("received data in HID RPC message handler, but queue is nil") - return - } - }) + d.OnMessage(onHidMessage) + // we won't send anything over the unreliable channel + case "hidrpc-unreliable": + d.OnMessage(onHidMessage) case "rpc": session.RPCChannel = d d.OnMessage(func(msg webrtc.DataChannelMessage) { From 3901ac91e0a4274a07c6fc11ecc21157a636588f Mon Sep 17 00:00:00 2001 From: Siyuan Miao Date: Thu, 11 Sep 2025 17:24:10 +0200 Subject: [PATCH 6/7] chore: use ordered unreliable channel for pointer events --- hidrpc.go | 9 ++- ui/src/hooks/stores.ts | 6 ++ ui/src/hooks/useHidRpc.ts | 34 +++++++++--- ui/src/routes/devices.$id.tsx | 16 +++++- webrtc.go | 101 +++++++++++++++++++--------------- 5 files changed, 110 insertions(+), 56 deletions(-) diff --git a/hidrpc.go b/hidrpc.go index 52022ba6..2c3ca27b 100644 --- a/hidrpc.go +++ b/hidrpc.go @@ -54,8 +54,13 @@ func handleHidRPCMessage(message hidrpc.Message, session *Session) { } } -func onHidMessage(data []byte, session *Session) { - scopedLogger := hidRPCLogger.With().Bytes("data", data).Logger() +func onHidMessage(msg hidQueueMessage, session *Session) { + data := msg.Data + + scopedLogger := hidRPCLogger.With(). + Str("channel", msg.channel). + Bytes("data", data). + Logger() scopedLogger.Debug().Msg("HID RPC message received") if len(data) < 1 { diff --git a/ui/src/hooks/stores.ts b/ui/src/hooks/stores.ts index 7007f4e6..b849fa55 100644 --- a/ui/src/hooks/stores.ts +++ b/ui/src/hooks/stores.ts @@ -114,6 +114,9 @@ export interface RTCState { rpcHidUnreliableChannel: RTCDataChannel | null; setRpcHidUnreliableChannel: (channel: RTCDataChannel) => void; + rpcHidUnreliableNonOrderedChannel: RTCDataChannel | null; + setRpcHidUnreliableNonOrderedChannel: (channel: RTCDataChannel) => void; + peerConnectionState: RTCPeerConnectionState | null; setPeerConnectionState: (state: RTCPeerConnectionState) => void; @@ -169,6 +172,9 @@ export const useRTCStore = create(set => ({ rpcHidUnreliableChannel: null, setRpcHidUnreliableChannel: (channel: RTCDataChannel) => set({ rpcHidUnreliableChannel: channel }), + rpcHidUnreliableNonOrderedChannel: null, + setRpcHidUnreliableNonOrderedChannel: (channel: RTCDataChannel) => set({ rpcHidUnreliableNonOrderedChannel: channel }), + transceiver: null, setTransceiver: (transceiver: RTCRtpTransceiver) => set({ transceiver }), diff --git a/ui/src/hooks/useHidRpc.ts b/ui/src/hooks/useHidRpc.ts index ff22885c..58c04f42 100644 --- a/ui/src/hooks/useHidRpc.ts +++ b/ui/src/hooks/useHidRpc.ts @@ -19,12 +19,14 @@ const KEEPALIVE_MESSAGE = new KeypressKeepAliveMessage(); interface sendMessageParams { ignoreHandshakeState?: boolean; useUnreliableChannel?: boolean; + requireOrdered?: boolean; } export function useHidRpc(onHidRpcMessage?: (payload: RpcMessage) => void) { const { rpcHidChannel, rpcHidUnreliableChannel, + rpcHidUnreliableNonOrderedChannel, setRpcHidProtocolVersion, rpcHidProtocolVersion, } = useRTCStore(); @@ -37,6 +39,10 @@ export function useHidRpc(onHidRpcMessage?: (payload: RpcMessage) => void) { return rpcHidUnreliableChannel?.readyState === "open" && rpcHidProtocolVersion !== null; }, [rpcHidUnreliableChannel, rpcHidProtocolVersion]); + const rpcHidUnreliableNonOrderedReady = useMemo(() => { + return rpcHidUnreliableNonOrderedChannel?.readyState === "open" && rpcHidProtocolVersion !== null; + }, [rpcHidUnreliableNonOrderedChannel, rpcHidProtocolVersion]); + const rpcHidStatus = useMemo(() => { if (!rpcHidChannel) return "N/A"; if (rpcHidChannel.readyState !== "open") return rpcHidChannel.readyState; @@ -44,7 +50,7 @@ export function useHidRpc(onHidRpcMessage?: (payload: RpcMessage) => void) { return `ready (v${rpcHidProtocolVersion}${rpcHidUnreliableReady ? "+u" : ""})`; }, [rpcHidChannel, rpcHidUnreliableReady, rpcHidProtocolVersion]); - const sendMessage = useCallback((message: RpcMessage, { ignoreHandshakeState, useUnreliableChannel }: sendMessageParams = {}) => { + const sendMessage = useCallback((message: RpcMessage, { ignoreHandshakeState, useUnreliableChannel, requireOrdered = true }: sendMessageParams = {}) => { if (rpcHidChannel?.readyState !== "open") return; if (!rpcHidReady && !ignoreHandshakeState) return; @@ -56,12 +62,24 @@ export function useHidRpc(onHidRpcMessage?: (payload: RpcMessage) => void) { } if (!data) return; - if (useUnreliableChannel && rpcHidUnreliableReady) { - rpcHidUnreliableChannel?.send(data as unknown as ArrayBuffer); - } else { - rpcHidChannel?.send(data as unknown as ArrayBuffer); + if (useUnreliableChannel) { + if (requireOrdered && rpcHidUnreliableReady) { + rpcHidUnreliableChannel?.send(data as unknown as ArrayBuffer); + } else if (!requireOrdered && rpcHidUnreliableNonOrderedReady) { + rpcHidUnreliableNonOrderedChannel?.send(data as unknown as ArrayBuffer); + } + return; } - }, [rpcHidChannel, rpcHidReady, rpcHidUnreliableChannel, rpcHidUnreliableReady]); + + rpcHidChannel?.send(data as unknown as ArrayBuffer); + }, [ + rpcHidChannel, + rpcHidUnreliableChannel, + rpcHidUnreliableNonOrderedChannel, + rpcHidReady, + rpcHidUnreliableReady, + rpcHidUnreliableNonOrderedReady, + ]); const reportKeyboardEvent = useCallback( (keys: number[], modifier: number) => { @@ -78,7 +96,7 @@ export function useHidRpc(onHidRpcMessage?: (payload: RpcMessage) => void) { const reportAbsMouseEvent = useCallback( (x: number, y: number, buttons: number) => { - sendMessage(new PointerReportMessage(x, y, buttons)); + sendMessage(new PointerReportMessage(x, y, buttons), { useUnreliableChannel: true }); }, [sendMessage], ); @@ -91,7 +109,7 @@ export function useHidRpc(onHidRpcMessage?: (payload: RpcMessage) => void) { ); const reportKeypressKeepAlive = useCallback(() => { - sendMessage(KEEPALIVE_MESSAGE, { useUnreliableChannel: true }); + sendMessage(KEEPALIVE_MESSAGE, { useUnreliableChannel: true, requireOrdered: false }); }, [sendMessage]); const sendHandshake = useCallback(() => { diff --git a/ui/src/routes/devices.$id.tsx b/ui/src/routes/devices.$id.tsx index 444e347d..8a363199 100644 --- a/ui/src/routes/devices.$id.tsx +++ b/ui/src/routes/devices.$id.tsx @@ -136,6 +136,7 @@ export default function KvmIdRoute() { rpcDataChannel, setTransceiver, setRpcHidChannel, + setRpcHidUnreliableNonOrderedChannel, setRpcHidUnreliableChannel, } = useRTCStore(); @@ -489,9 +490,8 @@ export default function KvmIdRoute() { setRpcHidChannel(rpcHidChannel); }; - const rpcHidUnreliableChannel = pc.createDataChannel("hidrpc-unreliable", { - // We don't need to be ordered, as we're using the unreliable channel for keepalive messages - ordered: false, + const rpcHidUnreliableChannel = pc.createDataChannel("hidrpc-unreliable-ordered", { + ordered: true, maxRetransmits: 0, }); rpcHidUnreliableChannel.binaryType = "arraybuffer"; @@ -499,6 +499,15 @@ export default function KvmIdRoute() { setRpcHidUnreliableChannel(rpcHidUnreliableChannel); }; + const rpcHidUnreliableNonOrderedChannel = pc.createDataChannel("hidrpc-unreliable-nonordered", { + ordered: false, + maxRetransmits: 0, + }); + rpcHidUnreliableNonOrderedChannel.binaryType = "arraybuffer"; + rpcHidUnreliableNonOrderedChannel.onopen = () => { + setRpcHidUnreliableNonOrderedChannel(rpcHidUnreliableNonOrderedChannel); + }; + setPeerConnection(pc); }, [ cleanupAndStopReconnecting, @@ -510,6 +519,7 @@ export default function KvmIdRoute() { setPeerConnectionState, setRpcDataChannel, setRpcHidChannel, + setRpcHidUnreliableNonOrderedChannel, setRpcHidUnreliableChannel, setTransceiver, ]); diff --git a/webrtc.go b/webrtc.go index e43c2512..333a58b8 100644 --- a/webrtc.go +++ b/webrtc.go @@ -29,7 +29,12 @@ type Session struct { hidRPCAvailable bool hidQueueLock sync.Mutex - hidQueue []chan webrtc.DataChannelMessage + hidQueue []chan hidQueueMessage +} + +type hidQueueMessage struct { + webrtc.DataChannelMessage + channel string } type SessionConfig struct { @@ -78,16 +83,59 @@ func (s *Session) initQueues() { s.hidQueueLock.Lock() defer s.hidQueueLock.Unlock() - s.hidQueue = make([]chan webrtc.DataChannelMessage, 0) + s.hidQueue = make([]chan hidQueueMessage, 0) for i := 0; i < 4; i++ { - q := make(chan webrtc.DataChannelMessage, 256) + q := make(chan hidQueueMessage, 256) s.hidQueue = append(s.hidQueue, q) } } func (s *Session) handleQueues(index int) { for msg := range s.hidQueue[index] { - onHidMessage(msg.Data, s) + onHidMessage(msg, s) + } +} + +func getOnHidMessageHandler(session *Session, scopedLogger *zerolog.Logger, channel string) func(msg webrtc.DataChannelMessage) { + return func(msg webrtc.DataChannelMessage) { + l := scopedLogger.With(). + Str("channel", channel). + Int("length", len(msg.Data)). + Logger() + // only log data if the log level is debug or lower + if scopedLogger.GetLevel() > zerolog.DebugLevel { + l = l.With().Str("data", string(msg.Data)).Logger() + } + + if msg.IsString { + l.Warn().Msg("received string data in HID RPC message handler") + return + } + + if len(msg.Data) < 1 { + l.Warn().Msg("received empty data in HID RPC message handler") + return + } + + l.Trace().Msg("received data in HID RPC message handler") + + // Enqueue to ensure ordered processing + queueIndex := hidrpc.GetQueueIndex(hidrpc.MessageType(msg.Data[0])) + if queueIndex >= len(session.hidQueue) || queueIndex < 0 { + l.Warn().Int("queueIndex", queueIndex).Msg("received data in HID RPC message handler, but queue index not found") + queueIndex = 3 + } + + queue := session.hidQueue[queueIndex] + if queue != nil { + queue <- hidQueueMessage{ + DataChannelMessage: msg, + channel: channel, + } + } else { + l.Warn().Int("queueIndex", queueIndex).Msg("received data in HID RPC message handler, but queue is nil") + return + } } } @@ -145,41 +193,6 @@ func newSession(config SessionConfig) (*Session, error) { go session.handleQueues(i) } - onHidMessage := func(msg webrtc.DataChannelMessage) { - l := scopedLogger.With().Int("length", len(msg.Data)).Logger() - // only log data if the log level is debug or lower - if scopedLogger.GetLevel() > zerolog.DebugLevel { - l = l.With().Str("data", string(msg.Data)).Logger() - } - - if msg.IsString { - l.Warn().Msg("received string data in HID RPC message handler") - return - } - - if len(msg.Data) < 1 { - l.Warn().Msg("received empty data in HID RPC message handler") - return - } - - l.Trace().Msg("received data in HID RPC message handler") - - // Enqueue to ensure ordered processing - queueIndex := hidrpc.GetQueueIndex(hidrpc.MessageType(msg.Data[0])) - if queueIndex >= len(session.hidQueue) || queueIndex < 0 { - l.Warn().Int("queueIndex", queueIndex).Msg("received data in HID RPC message handler, but queue index not found") - queueIndex = 3 - } - - queue := session.hidQueue[queueIndex] - if queue != nil { - queue <- msg - } else { - l.Warn().Int("queueIndex", queueIndex).Msg("received data in HID RPC message handler, but queue is nil") - return - } - } - peerConnection.OnDataChannel(func(d *webrtc.DataChannel) { defer func() { if r := recover(); r != nil { @@ -192,10 +205,12 @@ func newSession(config SessionConfig) (*Session, error) { switch d.Label() { case "hidrpc": session.HidChannel = d - d.OnMessage(onHidMessage) - // we won't send anything over the unreliable channel - case "hidrpc-unreliable": - d.OnMessage(onHidMessage) + d.OnMessage(getOnHidMessageHandler(session, scopedLogger, "hidrpc")) + // we won't send anything over the unreliable channels + case "hidrpc-unreliable-ordered": + d.OnMessage(getOnHidMessageHandler(session, scopedLogger, "hidrpc-unreliable-ordered")) + case "hidrpc-unreliable-nonordered": + d.OnMessage(getOnHidMessageHandler(session, scopedLogger, "hidrpc-unreliable-nonordered")) case "rpc": session.RPCChannel = d d.OnMessage(func(msg webrtc.DataChannelMessage) { From cbf1c7fba68dc9c4a0a083537444bc85d3a28020 Mon Sep 17 00:00:00 2001 From: Siyuan Miao Date: Thu, 11 Sep 2025 17:46:50 +0200 Subject: [PATCH 7/7] chore: adjust auto release key interval --- internal/usbgadget/hid_keyboard.go | 27 +++++++-------------------- ui/src/hooks/useKeyboard.ts | 2 +- 2 files changed, 8 insertions(+), 21 deletions(-) diff --git a/internal/usbgadget/hid_keyboard.go b/internal/usbgadget/hid_keyboard.go index 40dff0be..e3997ea6 100644 --- a/internal/usbgadget/hid_keyboard.go +++ b/internal/usbgadget/hid_keyboard.go @@ -22,6 +22,11 @@ var keyboardConfig = gadgetConfigItem{ reportDesc: keyboardReportDesc, } +// macOS default: 15 * 15 = 225ms https://discussions.apple.com/thread/1316947?sortBy=rank +// Linux default: 250ms https://man.archlinux.org/man/kbdrate.8.en +// Windows default: 1ms `HKEY_CURRENT_USER\Control Panel\Accessibility\Keyboard Response\AutoRepeatDelay` +const autoReleaseKeyboardInterval = time.Millisecond * 225 + // Source: https://www.kernel.org/doc/Documentation/usb/gadget_hid.txt var keyboardReportDesc = []byte{ 0x05, 0x01, /* USAGE_PAGE (Generic Desktop) */ @@ -173,8 +178,6 @@ func (u *UsbGadget) SetOnKeysDownChange(f func(state KeysDownState)) { u.onKeysDownChange = &f } -const autoReleaseKeyboardInterval = time.Millisecond * 450 - func (u *UsbGadget) scheduleAutoRelease(key byte) { u.kbdAutoReleaseLock.Lock() defer u.kbdAutoReleaseLock.Unlock() @@ -222,7 +225,8 @@ func (u *UsbGadget) performAutoRelease(key byte) { default: } - _, err := u.keypressReport(key, false, false) + // we just reset the keyboard state to 0 no matter what + _, err := u.keypressReport(0, false, false) if err != nil { u.log.Warn().Uint8("key", key).Msg("failed to auto-release keyboard key") } @@ -453,23 +457,6 @@ func (u *UsbGadget) keypressReport(key byte, press bool, autoRelease bool) (Keys } } - // if autoRelease { - // u.kbdAutoReleaseLock.Lock() - // u.kbdAutoReleaseLock.Unlock() - // ll.Uint8("key", key).Msg("locking kbdAutoReleaseLock, autoReleasLastKey reset") - - // defer func() { - // ll.Uint8("key", key).Msg("unlocked kbdAutoReleaseLock, autoReleasLastKey reset") - // }() - - // if u.kbdAutoReleaseLastKey == key { - // ll.Uint8("key", key).Msg("key already released by auto-release, skipping") - // u.kbdAutoReleaseLastKey = 0 - - // return u.UpdateKeysDown(modifier, keys), nil - // } - // } - err := u.keyboardWriteHidFile(modifier, keys) if err != nil { u.log.Warn().Uint8("modifier", modifier).Uints8("keys", keys).Msg("Could not write keypress report to hidg0") diff --git a/ui/src/hooks/useKeyboard.ts b/ui/src/hooks/useKeyboard.ts index fb9fa73a..cc9796f3 100644 --- a/ui/src/hooks/useKeyboard.ts +++ b/ui/src/hooks/useKeyboard.ts @@ -104,7 +104,7 @@ export default function useKeyboard() { } }; - const KEEPALIVE_INTERVAL = 200; // 200ms interval + const KEEPALIVE_INTERVAL = 75; // 200ms interval const cancelKeepAlive = useCallback(() => { if (keepAliveTimerRef.current) {