From 56dfb4febdcd2ec5884db1d403a776221822e890 Mon Sep 17 00:00:00 2001 From: Siyuan Miao Date: Thu, 11 Sep 2025 14:10:18 +0200 Subject: [PATCH] send keepalive when pressing the key --- hidrpc.go | 2 + internal/hidrpc/hidrpc.go | 9 +++ internal/usbgadget/hid_keyboard.go | 98 ++++++++++++++++++++++++------ internal/usbgadget/usbgadget.go | 15 ++--- ui/src/hooks/hidRpc.ts | 12 ++++ ui/src/hooks/useHidRpc.ts | 8 +++ ui/src/hooks/useKeyboard.ts | 30 ++++++++- 7 files changed, 147 insertions(+), 27 deletions(-) diff --git a/hidrpc.go b/hidrpc.go index 7ed5f1c4..3d458f3d 100644 --- a/hidrpc.go +++ b/hidrpc.go @@ -41,6 +41,8 @@ func handleHidRPCMessage(message hidrpc.Message, session *Session) { case hidrpc.TypeCancelKeyboardMacroReport: rpcCancelKeyboardMacro() return + case hidrpc.TypeKeypressKeepAliveReport: + gadget.DelayAutoRelease() case hidrpc.TypePointerReport: pointerReport, err := message.PointerReport() if err != nil { diff --git a/internal/hidrpc/hidrpc.go b/internal/hidrpc/hidrpc.go index 5d02d9ec..2fe0c1d5 100644 --- a/internal/hidrpc/hidrpc.go +++ b/internal/hidrpc/hidrpc.go @@ -15,6 +15,7 @@ const ( TypePointerReport MessageType = 0x03 TypeWheelReport MessageType = 0x04 TypeKeypressReport MessageType = 0x05 + TypeKeypressKeepAliveReport MessageType = 0x09 TypeMouseReport MessageType = 0x06 TypeKeyboardMacroReport MessageType = 0x07 TypeCancelKeyboardMacroReport MessageType = 0x08 @@ -120,3 +121,11 @@ func NewKeyboardMacroStateMessage(state bool, isPaste bool) *Message { d: data, } } + +// NewKeypressKeepAliveMessage creates a new keypress keep alive message. +func NewKeypressKeepAliveMessage() *Message { + return &Message{ + t: TypeKeypressKeepAliveReport, + d: []byte{}, + } +} diff --git a/internal/usbgadget/hid_keyboard.go b/internal/usbgadget/hid_keyboard.go index 8b7080a7..f9bd20db 100644 --- a/internal/usbgadget/hid_keyboard.go +++ b/internal/usbgadget/hid_keyboard.go @@ -173,33 +173,47 @@ func (u *UsbGadget) SetOnKeysDownChange(f func(state KeysDownState)) { u.onKeysDownChange = &f } -const autoReleaseKeyboardInterval = time.Millisecond * 300 +const autoReleaseKeyboardInterval = time.Millisecond * 450 func (u *UsbGadget) scheduleAutoRelease(key byte) { - u.keysAutoReleaseLock.Lock() - defer u.keysAutoReleaseLock.Unlock() + u.kbdAutoReleaseLock.Lock() + defer u.kbdAutoReleaseLock.Unlock() - if u.keysAutoReleaseTimer != nil { - u.keysAutoReleaseTimer.Stop() + if u.kbdAutoReleaseTimer != nil { + u.kbdAutoReleaseTimer.Stop() } - u.keysAutoReleaseTimer = time.AfterFunc(autoReleaseKeyboardInterval, func() { + u.kbdAutoReleaseTimer = time.AfterFunc(autoReleaseKeyboardInterval, func() { u.performAutoRelease(key) }) } func (u *UsbGadget) cancelAutoRelease() { - u.keysAutoReleaseLock.Lock() - defer u.keysAutoReleaseLock.Unlock() + u.kbdAutoReleaseLock.Lock() + defer u.kbdAutoReleaseLock.Unlock() - if u.keysAutoReleaseTimer != nil { - u.keysAutoReleaseTimer.Stop() + if u.kbdAutoReleaseTimer != nil { + u.kbdAutoReleaseTimer.Stop() } } +func (u *UsbGadget) DelayAutoRelease() { + u.kbdAutoReleaseLock.Lock() + defer u.kbdAutoReleaseLock.Unlock() + + u.log.Info().Msg("delaying auto-release") + + if u.kbdAutoReleaseTimer == nil { + return + } + + u.log.Info().Msg("resetting auto-release timer") + u.kbdAutoReleaseTimer.Reset(autoReleaseKeyboardInterval) +} + func (u *UsbGadget) performAutoRelease(key byte) { - u.keysAutoReleaseLock.Lock() - defer u.keysAutoReleaseLock.Unlock() + u.kbdAutoReleaseLock.Lock() + defer u.kbdAutoReleaseLock.Unlock() select { case <-u.keyboardStateCtx.Done(): @@ -207,12 +221,12 @@ func (u *UsbGadget) performAutoRelease(key byte) { default: } - _, err := u.KeypressReport(key, false) + _, err := u.keypressReport(key, false, false) if err != nil { u.log.Warn().Uint8("key", key).Msg("failed to auto-release keyboard key") } - u.keysAutoReleaseTimer = nil + u.kbdAutoReleaseTimer = nil u.log.Trace().Uint8("key", key).Msg("auto release performed") } @@ -375,11 +389,21 @@ var KeyCodeToMaskMap = map[byte]byte{ RightSuper: ModifierMaskRightSuper, } -func (u *UsbGadget) KeypressReport(key byte, press bool) (KeysDownState, error) { +func (u *UsbGadget) keypressReport(key byte, press bool, autoRelease bool) (KeysDownState, error) { + ll := u.log.Info().Str("component", "kbd") + ll.Uint8("key", key).Msg("locking keyboardLock") + u.keyboardLock.Lock() - defer u.keyboardLock.Unlock() + defer func() { + u.keyboardLock.Unlock() + ll.Uint8("key", key).Msg("unlocked keyboardLock") + }() + + ll.Uint8("key", key).Msg("resetting user input time") defer u.resetUserInputTime() + ll.Uint8("key", key).Msg("locked keyboardLock") + // IMPORTANT: This code parallels the logic in the kernel's hid-gadget driver // for handling key presses and releases. It ensures that the USB gadget // behaves similarly to a real USB HID keyboard. This logic is paralleled @@ -437,16 +461,54 @@ func (u *UsbGadget) KeypressReport(key byte, press bool) (KeysDownState, error) } } + ll.Uint8("key", key).Msg("checking if auto-release is enabled") + + // if autoRelease { + // u.kbdAutoReleaseLock.Lock() + // u.kbdAutoReleaseLock.Unlock() + // ll.Uint8("key", key).Msg("locking kbdAutoReleaseLock, autoReleasLastKey reset") + + // defer func() { + // ll.Uint8("key", key).Msg("unlocked kbdAutoReleaseLock, autoReleasLastKey reset") + // }() + + // if u.kbdAutoReleaseLastKey == key { + // ll.Uint8("key", key).Msg("key already released by auto-release, skipping") + // u.kbdAutoReleaseLastKey = 0 + + // return u.UpdateKeysDown(modifier, keys), nil + // } + // } + + ll.Uint8("key", key).Msg("writing keypress report to hidg0") + err := u.keyboardWriteHidFile(modifier, keys) if err != nil { u.log.Warn().Uint8("modifier", modifier).Uints8("keys", keys).Msg("Could not write keypress report to hidg0") } if press { - u.scheduleAutoRelease(key) + { + u.kbdAutoReleaseLock.Lock() + u.kbdAutoReleaseLastKey = key + u.kbdAutoReleaseLock.Unlock() + } + + if autoRelease { + ll.Uint8("key", key).Msg("scheduling auto-release") + u.scheduleAutoRelease(key) + } } else { - u.cancelAutoRelease() + if autoRelease { + ll.Uint8("key", key).Msg("canceling auto-release") + u.cancelAutoRelease() + ll.Uint8("key", key).Msg("auto-release canceled") + } } return u.UpdateKeysDown(modifier, keys), err } + +func (u *UsbGadget) KeypressReport(key byte, press bool) (KeysDownState, error) { + return u.keypressReport(key, press, true) +} diff --git a/internal/usbgadget/usbgadget.go b/internal/usbgadget/usbgadget.go index 7485fc20..e491a78c 100644 --- a/internal/usbgadget/usbgadget.go +++ b/internal/usbgadget/usbgadget.go @@ -68,8 +68,9 @@ type UsbGadget struct { keyboardState byte // keyboard latched state (NumLock, CapsLock, ScrollLock, Compose, Kana) keysDownState KeysDownState // keyboard dynamic state (modifier keys and pressed keys) - keysAutoReleaseLock sync.Mutex - keysAutoReleaseTimer *time.Timer + kbdAutoReleaseLock sync.Mutex + kbdAutoReleaseTimer *time.Timer + kbdAutoReleaseLastKey byte keyboardStateLock sync.Mutex keyboardStateCtx context.Context @@ -161,12 +162,12 @@ func (u *UsbGadget) Close() error { } // Stop auto-release timer - u.keysAutoReleaseLock.Lock() - if u.keysAutoReleaseTimer != nil { - u.keysAutoReleaseTimer.Stop() - u.keysAutoReleaseTimer = nil + u.kbdAutoReleaseLock.Lock() + if u.kbdAutoReleaseTimer != nil { + u.kbdAutoReleaseTimer.Stop() + u.kbdAutoReleaseTimer = nil } - u.keysAutoReleaseLock.Unlock() + u.kbdAutoReleaseLock.Unlock() // Close HID files if u.keyboardHidFile != nil { diff --git a/ui/src/hooks/hidRpc.ts b/ui/src/hooks/hidRpc.ts index 2606770d..823384ff 100644 --- a/ui/src/hooks/hidRpc.ts +++ b/ui/src/hooks/hidRpc.ts @@ -6,6 +6,7 @@ export const HID_RPC_MESSAGE_TYPES = { PointerReport: 0x03, WheelReport: 0x04, KeypressReport: 0x05, + KeypressKeepAliveReport: 0x09, MouseReport: 0x06, KeyboardMacroReport: 0x07, CancelKeyboardMacroReport: 0x08, @@ -409,6 +410,16 @@ export class MouseReportMessage extends RpcMessage { } } +export class KeypressKeepAliveMessage extends RpcMessage { + constructor() { + super(HID_RPC_MESSAGE_TYPES.KeypressKeepAliveReport); + } + + marshal(): Uint8Array { + return new Uint8Array([this.messageType]); + } +} + export const messageRegistry = { [HID_RPC_MESSAGE_TYPES.Handshake]: HandshakeMessage, [HID_RPC_MESSAGE_TYPES.KeysDownState]: KeysDownStateMessage, @@ -418,6 +429,7 @@ export const messageRegistry = { [HID_RPC_MESSAGE_TYPES.KeyboardMacroReport]: KeyboardMacroReportMessage, [HID_RPC_MESSAGE_TYPES.CancelKeyboardMacroReport]: CancelKeyboardMacroReportMessage, [HID_RPC_MESSAGE_TYPES.KeyboardMacroState]: KeyboardMacroStateMessage, + [HID_RPC_MESSAGE_TYPES.KeypressKeepAliveReport]: KeypressKeepAliveMessage, } export const unmarshalHidRpcMessage = (data: Uint8Array): RpcMessage | undefined => { diff --git a/ui/src/hooks/useHidRpc.ts b/ui/src/hooks/useHidRpc.ts index 87b0b816..6288b541 100644 --- a/ui/src/hooks/useHidRpc.ts +++ b/ui/src/hooks/useHidRpc.ts @@ -9,6 +9,7 @@ import { KeyboardMacroStep, KeyboardMacroReportMessage, KeyboardReportMessage, + KeypressKeepAliveMessage, KeypressReportMessage, MouseReportMessage, PointerReportMessage, @@ -16,6 +17,8 @@ import { unmarshalHidRpcMessage, } from "./hidRpc"; +const KEEPALIVE_MESSAGE = new KeypressKeepAliveMessage(); + export function useHidRpc(onHidRpcMessage?: (payload: RpcMessage) => void) { const { rpcHidChannel, setRpcHidProtocolVersion, rpcHidProtocolVersion, hidRpcDisabled } = useRTCStore(); const rpcHidReady = useMemo(() => { @@ -89,6 +92,10 @@ export function useHidRpc(onHidRpcMessage?: (payload: RpcMessage) => void) { [sendMessage], ); + const reportKeypressKeepAlive = useCallback(() => { + sendMessage(KEEPALIVE_MESSAGE); + }, [sendMessage]); + const sendHandshake = useCallback(() => { if (hidRpcDisabled) return; if (rpcHidProtocolVersion) return; @@ -171,6 +178,7 @@ export function useHidRpc(onHidRpcMessage?: (payload: RpcMessage) => void) { reportRelMouseEvent, reportKeyboardMacroEvent, cancelOngoingKeyboardMacro, + reportKeypressKeepAlive, rpcHidProtocolVersion, rpcHidReady, rpcHidStatus, diff --git a/ui/src/hooks/useKeyboard.ts b/ui/src/hooks/useKeyboard.ts index a276f431..ee3313b0 100644 --- a/ui/src/hooks/useKeyboard.ts +++ b/ui/src/hooks/useKeyboard.ts @@ -44,6 +44,9 @@ export default function useKeyboard() { abortController.current = ac; }, []); + // Keepalive timer management + const keepAliveTimerRef = useRef(null); + // INTRODUCTION: The earlier version of the JetKVM device shipped with all keyboard state // being tracked on the browser/client-side. When adding the keyPressReport API to the // device-side code, we have to still support the situation where the browser/client-side code @@ -61,6 +64,7 @@ export default function useKeyboard() { reportKeypressEvent: sendKeypressEventHidRpc, reportKeyboardMacroEvent: sendKeyboardMacroEventHidRpc, cancelOngoingKeyboardMacro: cancelOngoingKeyboardMacroHidRpc, + reportKeypressKeepAlive: sendKeypressKeepAliveHidRpc, rpcHidReady, } = useHidRpc(message => { switch (message.constructor) { @@ -223,6 +227,20 @@ export default function useKeyboard() { // If the keyPressReport API is not available, it simulates the device-side key // handling for legacy devices and updates the keysDownState accordingly. // It then sends the full keyboard state to the device. + + const sendKeypress = useCallback( + (key: number, press: boolean) => { + cancelKeepAlive(); + + sendKeypressEventHidRpc(key, press); + + if (press) { + scheduleKeepAlive(); + } + }, + [sendKeypressEventHidRpc, scheduleKeepAlive, cancelKeepAlive], + ); + const handleKeyPress = useCallback( async (key: number, press: boolean) => { if (rpcDataChannel?.readyState !== "open" && !rpcHidReady) return; @@ -235,7 +253,7 @@ export default function useKeyboard() { // Older device version doesn't support this API, so we will switch to local key handling // In that case we will switch to local key handling and update the keysDownState // in client/browser-side code using simulateDeviceSideKeyHandlingForLegacyDevices. - sendKeypressEventHidRpc(key, press); + sendKeypress(key, press); } else { // Older backends don't support the hidRpc API, so we need: // 1. Calculate the state @@ -261,6 +279,9 @@ export default function useKeyboard() { keysDownState, handleLegacyKeyboardReport, resetKeyboardState, + rpcDataChannel?.readyState, + sendKeyboardEvent, + sendKeypress, ], ); @@ -329,5 +350,10 @@ export default function useKeyboard() { return { modifier: modifiers, keys }; } - return { handleKeyPress, resetKeyboardState, executeMacro, cancelExecuteMacro }; + // Cleanup function to cancel keepalive timer + const cleanup = useCallback(() => { + cancelKeepAlive(); + }, [cancelKeepAlive]); + + return { handleKeyPress, resetKeyboardState, executeMacro, cleanup, cancelExecuteMacro }; }