send keepalive when pressing the key

hidrpc.go

@@ -41,6 +41,8 @@ func handleHidRPCMessage(message hidrpc.Message, session *Session) {
 	case hidrpc.TypeCancelKeyboardMacroReport:
 		rpcCancelKeyboardMacro()
 		return
+	case hidrpc.TypeKeypressKeepAliveReport:
+		gadget.DelayAutoRelease()
 	case hidrpc.TypePointerReport:
 		pointerReport, err := message.PointerReport()
 		if err != nil {

internal/hidrpc/hidrpc.go

@@ -15,6 +15,7 @@ const (
 	TypePointerReport             MessageType = 0x03
 	TypeWheelReport               MessageType = 0x04
 	TypeKeypressReport            MessageType = 0x05
+	TypeKeypressKeepAliveReport   MessageType = 0x09
 	TypeMouseReport               MessageType = 0x06
 	TypeKeyboardMacroReport       MessageType = 0x07
 	TypeCancelKeyboardMacroReport MessageType = 0x08
@@ -120,3 +121,11 @@ func NewKeyboardMacroStateMessage(state bool, isPaste bool) *Message {
 		d: data,
 	}
 }
+
+// NewKeypressKeepAliveMessage creates a new keypress keep alive message.
+func NewKeypressKeepAliveMessage() *Message {
+	return &Message{
+		t: TypeKeypressKeepAliveReport,
+		d: []byte{},
+	}
+}

internal/usbgadget/hid_keyboard.go

@@ -173,33 +173,47 @@ func (u *UsbGadget) SetOnKeysDownChange(f func(state KeysDownState)) {
 	u.onKeysDownChange = &f
 }
 
-const autoReleaseKeyboardInterval = time.Millisecond * 300
+const autoReleaseKeyboardInterval = time.Millisecond * 450
 
 func (u *UsbGadget) scheduleAutoRelease(key byte) {
-	u.keysAutoReleaseLock.Lock()
+	u.kbdAutoReleaseLock.Lock()
-	defer u.keysAutoReleaseLock.Unlock()
+	defer u.kbdAutoReleaseLock.Unlock()
 
-	if u.keysAutoReleaseTimer != nil {
+	if u.kbdAutoReleaseTimer != nil {
-		u.keysAutoReleaseTimer.Stop()
+		u.kbdAutoReleaseTimer.Stop()
 	}
 
-	u.keysAutoReleaseTimer = time.AfterFunc(autoReleaseKeyboardInterval, func() {
+	u.kbdAutoReleaseTimer = time.AfterFunc(autoReleaseKeyboardInterval, func() {
 		u.performAutoRelease(key)
 	})
 }
 
 func (u *UsbGadget) cancelAutoRelease() {
-	u.keysAutoReleaseLock.Lock()
+	u.kbdAutoReleaseLock.Lock()
-	defer u.keysAutoReleaseLock.Unlock()
+	defer u.kbdAutoReleaseLock.Unlock()
 
-	if u.keysAutoReleaseTimer != nil {
+	if u.kbdAutoReleaseTimer != nil {
-		u.keysAutoReleaseTimer.Stop()
+		u.kbdAutoReleaseTimer.Stop()
 	}
 }
 
+func (u *UsbGadget) DelayAutoRelease() {
+	u.kbdAutoReleaseLock.Lock()
+	defer u.kbdAutoReleaseLock.Unlock()
+
+	u.log.Info().Msg("delaying auto-release")
+
+	if u.kbdAutoReleaseTimer == nil {
+		return
+	}
+
+	u.log.Info().Msg("resetting auto-release timer")
+	u.kbdAutoReleaseTimer.Reset(autoReleaseKeyboardInterval)
+}
+
 func (u *UsbGadget) performAutoRelease(key byte) {
-	u.keysAutoReleaseLock.Lock()
+	u.kbdAutoReleaseLock.Lock()
-	defer u.keysAutoReleaseLock.Unlock()
+	defer u.kbdAutoReleaseLock.Unlock()
 
 	select {
 	case <-u.keyboardStateCtx.Done():
@@ -207,12 +221,12 @@ func (u *UsbGadget) performAutoRelease(key byte) {
 	default:
 	}
 
-	_, err := u.KeypressReport(key, false)
+	_, err := u.keypressReport(key, false, false)
 	if err != nil {
 		u.log.Warn().Uint8("key", key).Msg("failed to auto-release keyboard key")
 	}
 
-	u.keysAutoReleaseTimer = nil
+	u.kbdAutoReleaseTimer = nil
 
 	u.log.Trace().Uint8("key", key).Msg("auto release performed")
 }
@@ -375,11 +389,21 @@ var KeyCodeToMaskMap = map[byte]byte{
 	RightSuper:   ModifierMaskRightSuper,
 }
 
-func (u *UsbGadget) KeypressReport(key byte, press bool) (KeysDownState, error) {
+func (u *UsbGadget) keypressReport(key byte, press bool, autoRelease bool) (KeysDownState, error) {
+	ll := u.log.Info().Str("component", "kbd")
+	ll.Uint8("key", key).Msg("locking keyboardLock")
+
 	u.keyboardLock.Lock()
-	defer u.keyboardLock.Unlock()
+	defer func() {
+		u.keyboardLock.Unlock()
+		ll.Uint8("key", key).Msg("unlocked keyboardLock")
+	}()
+
+	ll.Uint8("key", key).Msg("resetting user input time")
 	defer u.resetUserInputTime()
 
+	ll.Uint8("key", key).Msg("locked keyboardLock")
+
 	// IMPORTANT: This code parallels the logic in the kernel's hid-gadget driver
 	// for handling key presses and releases. It ensures that the USB gadget
 	// behaves similarly to a real USB HID keyboard. This logic is paralleled
@@ -437,16 +461,54 @@ func (u *UsbGadget) KeypressReport(key byte, press bool) (KeysDownState, error)
 		}
 	}
 
+	ll.Uint8("key", key).Msg("checking if auto-release is enabled")
+
+	// if autoRelease {
+	// 	u.kbdAutoReleaseLock.Lock()
+	// 	u.kbdAutoReleaseLock.Unlock()
+	// 	ll.Uint8("key", key).Msg("locking kbdAutoReleaseLock, autoReleasLastKey reset")
+
+	// 	defer func() {
+	// 		ll.Uint8("key", key).Msg("unlocked kbdAutoReleaseLock, autoReleasLastKey reset")
+	// 	}()
+
+	// 	if u.kbdAutoReleaseLastKey == key {
+	// 		ll.Uint8("key", key).Msg("key already released by auto-release, skipping")
+	// 		u.kbdAutoReleaseLastKey = 0
+
+	// 		return u.UpdateKeysDown(modifier, keys), nil
+	// 	}
+	// }
+
+	ll.Uint8("key", key).Msg("writing keypress report to hidg0")
+
 	err := u.keyboardWriteHidFile(modifier, keys)
 	if err != nil {
 		u.log.Warn().Uint8("modifier", modifier).Uints8("keys", keys).Msg("Could not write keypress report to hidg0")
 	}
 
 	if press {
-		u.scheduleAutoRelease(key)
+		{
+			u.kbdAutoReleaseLock.Lock()
+			u.kbdAutoReleaseLastKey = key
+			u.kbdAutoReleaseLock.Unlock()
+		}
+
+		if autoRelease {
+			ll.Uint8("key", key).Msg("scheduling auto-release")
+			u.scheduleAutoRelease(key)
+		}
 	} else {
-		u.cancelAutoRelease()
+		if autoRelease {
+			ll.Uint8("key", key).Msg("canceling auto-release")
+			u.cancelAutoRelease()
+			ll.Uint8("key", key).Msg("auto-release canceled")
+		}
 	}
 
 	return u.UpdateKeysDown(modifier, keys), err
 }
+
+func (u *UsbGadget) KeypressReport(key byte, press bool) (KeysDownState, error) {
+	return u.keypressReport(key, press, true)
+}

internal/usbgadget/usbgadget.go

@@ -68,8 +68,9 @@ type UsbGadget struct {
 	keyboardState byte          // keyboard latched state (NumLock, CapsLock, ScrollLock, Compose, Kana)
 	keysDownState KeysDownState // keyboard dynamic state (modifier keys and pressed keys)
 
-	keysAutoReleaseLock  sync.Mutex
+	kbdAutoReleaseLock    sync.Mutex
-	keysAutoReleaseTimer *time.Timer
+	kbdAutoReleaseTimer   *time.Timer
+	kbdAutoReleaseLastKey byte
 
 	keyboardStateLock   sync.Mutex
 	keyboardStateCtx    context.Context
@@ -161,12 +162,12 @@ func (u *UsbGadget) Close() error {
 	}
 
 	// Stop auto-release timer
-	u.keysAutoReleaseLock.Lock()
+	u.kbdAutoReleaseLock.Lock()
-	if u.keysAutoReleaseTimer != nil {
+	if u.kbdAutoReleaseTimer != nil {
-		u.keysAutoReleaseTimer.Stop()
+		u.kbdAutoReleaseTimer.Stop()
-		u.keysAutoReleaseTimer = nil
+		u.kbdAutoReleaseTimer = nil
 	}
-	u.keysAutoReleaseLock.Unlock()
+	u.kbdAutoReleaseLock.Unlock()
 
 	// Close HID files
 	if u.keyboardHidFile != nil {

ui/src/hooks/hidRpc.ts

@@ -6,6 +6,7 @@ export const HID_RPC_MESSAGE_TYPES = {
     PointerReport: 0x03,
     WheelReport: 0x04,
     KeypressReport: 0x05,
+    KeypressKeepAliveReport: 0x09,
     MouseReport: 0x06,
     KeyboardMacroReport: 0x07,
     CancelKeyboardMacroReport: 0x08,
@@ -409,6 +410,16 @@ export class MouseReportMessage extends RpcMessage {
     }
 }
 
+export class KeypressKeepAliveMessage extends RpcMessage {
+    constructor() {
+        super(HID_RPC_MESSAGE_TYPES.KeypressKeepAliveReport);
+    }
+
+    marshal(): Uint8Array {
+        return new Uint8Array([this.messageType]);
+    }
+}
+
 export const messageRegistry = {
     [HID_RPC_MESSAGE_TYPES.Handshake]: HandshakeMessage,
     [HID_RPC_MESSAGE_TYPES.KeysDownState]: KeysDownStateMessage,
@@ -418,6 +429,7 @@ export const messageRegistry = {
     [HID_RPC_MESSAGE_TYPES.KeyboardMacroReport]: KeyboardMacroReportMessage,
     [HID_RPC_MESSAGE_TYPES.CancelKeyboardMacroReport]: CancelKeyboardMacroReportMessage,
     [HID_RPC_MESSAGE_TYPES.KeyboardMacroState]: KeyboardMacroStateMessage,
+    [HID_RPC_MESSAGE_TYPES.KeypressKeepAliveReport]: KeypressKeepAliveMessage,
 }
 
 export const unmarshalHidRpcMessage = (data: Uint8Array): RpcMessage | undefined => {

ui/src/hooks/useHidRpc.ts

@@ -9,6 +9,7 @@ import {
   KeyboardMacroStep,
   KeyboardMacroReportMessage,
   KeyboardReportMessage,
+  KeypressKeepAliveMessage,
   KeypressReportMessage,
   MouseReportMessage,
   PointerReportMessage,
@@ -16,6 +17,8 @@ import {
   unmarshalHidRpcMessage,
 } from "./hidRpc";
 
+const KEEPALIVE_MESSAGE = new KeypressKeepAliveMessage();
+
 export function useHidRpc(onHidRpcMessage?: (payload: RpcMessage) => void) {
   const { rpcHidChannel, setRpcHidProtocolVersion, rpcHidProtocolVersion, hidRpcDisabled } = useRTCStore();
   const rpcHidReady = useMemo(() => {
@@ -89,6 +92,10 @@ export function useHidRpc(onHidRpcMessage?: (payload: RpcMessage) => void) {
     [sendMessage],
   );
 
+  const reportKeypressKeepAlive = useCallback(() => {
+    sendMessage(KEEPALIVE_MESSAGE);
+  }, [sendMessage]);
+
   const sendHandshake = useCallback(() => {
     if (hidRpcDisabled) return;
     if (rpcHidProtocolVersion) return;
@@ -171,6 +178,7 @@ export function useHidRpc(onHidRpcMessage?: (payload: RpcMessage) => void) {
     reportRelMouseEvent,
     reportKeyboardMacroEvent,
     cancelOngoingKeyboardMacro,
+    reportKeypressKeepAlive,
     rpcHidProtocolVersion,
     rpcHidReady,
     rpcHidStatus,

ui/src/hooks/useKeyboard.ts

@@ -44,6 +44,9 @@ export default function useKeyboard() {
     abortController.current = ac;
   }, []);
 
+  // Keepalive timer management
+  const keepAliveTimerRef = useRef<number | null>(null);
+
   // INTRODUCTION: The earlier version of the JetKVM device shipped with all keyboard state
   // being tracked on the browser/client-side. When adding the keyPressReport API to the
   // device-side code, we have to still support the situation where the browser/client-side code
@@ -61,6 +64,7 @@ export default function useKeyboard() {
     reportKeypressEvent: sendKeypressEventHidRpc,
     reportKeyboardMacroEvent: sendKeyboardMacroEventHidRpc,
     cancelOngoingKeyboardMacro: cancelOngoingKeyboardMacroHidRpc,
+    reportKeypressKeepAlive: sendKeypressKeepAliveHidRpc,
     rpcHidReady,
   } = useHidRpc(message => {
     switch (message.constructor) {
@@ -223,6 +227,20 @@ export default function useKeyboard() {
   // If the keyPressReport API is not available, it simulates the device-side key
   // handling for legacy devices and updates the keysDownState accordingly.
   // It then sends the full keyboard state to the device.
+
+  const sendKeypress = useCallback(
+    (key: number, press: boolean) => {
+      cancelKeepAlive();
+
+      sendKeypressEventHidRpc(key, press);
+
+      if (press) {
+        scheduleKeepAlive();
+      }
+    },
+    [sendKeypressEventHidRpc, scheduleKeepAlive, cancelKeepAlive],
+  );
+
   const handleKeyPress = useCallback(
     async (key: number, press: boolean) => {
       if (rpcDataChannel?.readyState !== "open" && !rpcHidReady) return;
@@ -235,7 +253,7 @@ export default function useKeyboard() {
         // Older device version doesn't support this API, so we will switch to local key handling
         // In that case we will switch to local key handling and update the keysDownState
         // in client/browser-side code using simulateDeviceSideKeyHandlingForLegacyDevices.
-        sendKeypressEventHidRpc(key, press);
+        sendKeypress(key, press);
       } else {
         // Older backends don't support the hidRpc API, so we need:
         // 1. Calculate the state
@@ -261,6 +279,9 @@ export default function useKeyboard() {
       keysDownState,
       handleLegacyKeyboardReport,
       resetKeyboardState,
+      rpcDataChannel?.readyState,
+      sendKeyboardEvent,
+      sendKeypress,
     ],
   );
 
@@ -329,5 +350,10 @@ export default function useKeyboard() {
     return { modifier: modifiers, keys };
   }
 
-  return { handleKeyPress, resetKeyboardState, executeMacro, cancelExecuteMacro };
+  // Cleanup function to cancel keepalive timer
+  const cleanup = useCallback(() => {
+    cancelKeepAlive();
+  }, [cancelKeepAlive]);
+
+  return { handleKeyPress, resetKeyboardState, executeMacro, cleanup, cancelExecuteMacro };
 }