From 262df47f7d292030aa2df2747859e7a9326b09f4 Mon Sep 17 00:00:00 2001 From: Marc Brooks Date: Wed, 24 Sep 2025 19:20:51 -0500 Subject: [PATCH 1/4] Reduce traffic during pastes Suspend KeyDownMessages while processing a macro. Make sure we don't emit huge debugging traces. Allow 30 seconds for RPC to finish (not ideal) Reduced default delay between keys (and allow as low as 0) --- hidrpc.go | 10 +++++---- internal/usbgadget/hid_keyboard.go | 25 ++++++++++++++++++----- jsonrpc.go | 9 ++++++-- ui/src/components/popovers/PasteModal.tsx | 6 +++--- ui/src/hooks/useKeyboard.ts | 4 +--- 5 files changed, 37 insertions(+), 17 deletions(-) diff --git a/hidrpc.go b/hidrpc.go index ebe03daa..abf3bd73 100644 --- a/hidrpc.go +++ b/hidrpc.go @@ -65,15 +65,17 @@ func handleHidRPCMessage(message hidrpc.Message, session *Session) { func onHidMessage(msg hidQueueMessage, session *Session) { data := msg.Data + dataLen := len(data) scopedLogger := hidRPCLogger.With(). Str("channel", msg.channel). - Bytes("data", data). + Int("data_len", dataLen). + Bytes("data", data[:min(dataLen, 32)]). Logger() scopedLogger.Debug().Msg("HID RPC message received") - if len(data) < 1 { - scopedLogger.Warn().Int("length", len(data)).Msg("received empty data in HID RPC message handler") + if dataLen < 1 { + scopedLogger.Warn().Msg("received empty data in HID RPC message handler") return } @@ -96,7 +98,7 @@ func onHidMessage(msg hidQueueMessage, session *Session) { r <- nil }() select { - case <-time.After(1 * time.Second): + case <-time.After(30 * time.Second): scopedLogger.Warn().Msg("HID RPC message timed out") case <-r: scopedLogger.Debug().Dur("duration", time.Since(t)).Msg("HID RPC message handled") diff --git a/internal/usbgadget/hid_keyboard.go b/internal/usbgadget/hid_keyboard.go index 74cf76f9..1deb7c95 100644 --- a/internal/usbgadget/hid_keyboard.go +++ b/internal/usbgadget/hid_keyboard.go @@ -55,10 +55,10 @@ var keyboardReportDesc = []byte{ 0x95, 0x06, /* REPORT_COUNT (6) */ 0x75, 0x08, /* REPORT_SIZE (8) */ 0x15, 0x00, /* LOGICAL_MINIMUM (0) */ - 0x25, 0x65, /* LOGICAL_MAXIMUM (101) */ + 0x25, 104, /* LOGICAL_MAXIMUM (104-key) */ 0x05, 0x07, /* USAGE_PAGE (Keyboard) */ 0x19, 0x00, /* USAGE_MINIMUM (Reserved) */ - 0x29, 0x65, /* USAGE_MAXIMUM (Keyboard Application) */ + 0x29, 0xE7, /* USAGE_MAXIMUM (Keyboard Right GUI) */ 0x81, 0x00, /* INPUT (Data,Ary,Abs) */ 0xc0, /* END_COLLECTION */ } @@ -153,6 +153,16 @@ func (u *UsbGadget) SetOnKeysDownChange(f func(state KeysDownState)) { u.onKeysDownChange = &f } +var suspendedKeyDownMessages bool = false + +func (u *UsbGadget) SuspendKeyDownMessages() { + suspendedKeyDownMessages = true +} + +func (u *UsbGadget) ResumeSuspendKeyDownMessages() { + suspendedKeyDownMessages = false +} + func (u *UsbGadget) SetOnKeepAliveReset(f func()) { u.onKeepAliveReset = &f } @@ -169,9 +179,9 @@ func (u *UsbGadget) scheduleAutoRelease(key byte) { } // TODO: make this configurable - // We currently hardcode the duration to 100ms + // We currently hardcode the duration to the default of 100ms // However, it should be the same as the duration of the keep-alive reset called baseExtension. - u.kbdAutoReleaseTimers[key] = time.AfterFunc(100*time.Millisecond, func() { + u.kbdAutoReleaseTimers[key] = time.AfterFunc(DefaultAutoReleaseDuration, func() { u.performAutoRelease(key) }) } @@ -314,6 +324,7 @@ var keyboardWriteHidFileLock sync.Mutex func (u *UsbGadget) keyboardWriteHidFile(modifier byte, keys []byte) error { keyboardWriteHidFileLock.Lock() defer keyboardWriteHidFileLock.Unlock() + if err := u.openKeyboardHidFile(); err != nil { return err } @@ -353,7 +364,7 @@ func (u *UsbGadget) UpdateKeysDown(modifier byte, keys []byte) KeysDownState { u.keysDownState = state u.keyboardStateLock.Unlock() - if u.onKeysDownChange != nil { + if u.onKeysDownChange != nil && !suspendedKeyDownMessages { (*u.onKeysDownChange)(state) // this enques to the outgoing hidrpc queue via usb.go → currentSession.enqueueKeysDownState(...) } return state @@ -484,6 +495,10 @@ func (u *UsbGadget) keypressReport(key byte, press bool) (KeysDownState, error) } err := u.keyboardWriteHidFile(modifier, keys) + if err != nil { + u.log.Warn().Uint8("modifier", modifier).Uints8("keys", keys).Msg("Could not write keyboard report to hidg0") + } + return u.UpdateKeysDown(modifier, keys), err } diff --git a/jsonrpc.go b/jsonrpc.go index 3e3d9c94..e3db0fac 100644 --- a/jsonrpc.go +++ b/jsonrpc.go @@ -1132,7 +1132,11 @@ func isClearKeyStep(step hidrpc.KeyboardMacroStep) bool { } func rpcDoExecuteKeyboardMacro(ctx context.Context, macro []hidrpc.KeyboardMacroStep) error { - logger.Debug().Interface("macro", macro).Msg("Executing keyboard macro") + logger.Debug().Int("macro_steps", len(macro)).Msg("Executing keyboard macro") + + // don't report keyboard state changes while executing the macro + gadget.SuspendKeyDownMessages() + defer gadget.ResumeSuspendKeyDownMessages() for i, step := range macro { delay := time.Duration(step.Delay) * time.Millisecond @@ -1153,7 +1157,8 @@ func rpcDoExecuteKeyboardMacro(ctx context.Context, macro []hidrpc.KeyboardMacro case <-time.After(delay): // Sleep completed normally case <-ctx.Done(): - // make sure keyboard state is reset + // make sure keyboard state is reset and the client gets notified + gadget.ResumeSuspendKeyDownMessages() err := rpcKeyboardReport(0, keyboardClearStateKeys) if err != nil { logger.Warn().Err(err).Msg("failed to reset keyboard state") diff --git a/ui/src/components/popovers/PasteModal.tsx b/ui/src/components/popovers/PasteModal.tsx index ac97e29b..4b2ed49a 100644 --- a/ui/src/components/popovers/PasteModal.tsx +++ b/ui/src/components/popovers/PasteModal.tsx @@ -188,18 +188,18 @@ export default function PasteModal() { type="number" label="Delay between keys" placeholder="Delay between keys" - min={50} + min={0} max={65534} value={delayValue} onChange={e => { setDelayValue(parseInt(e.target.value, 10)); }} /> - {delayValue < 50 || delayValue > 65534 && ( + {delayValue < defaultDelay || delayValue > 65534 && (
- Delay must be between 50 and 65534 + Delay should be between 20 and 65534
)} diff --git a/ui/src/hooks/useKeyboard.ts b/ui/src/hooks/useKeyboard.ts index 8d101b3b..48a2cdfe 100644 --- a/ui/src/hooks/useKeyboard.ts +++ b/ui/src/hooks/useKeyboard.ts @@ -277,7 +277,6 @@ export default function useKeyboard() { cancelKeepAlive(); }, [cancelKeepAlive]); - // executeMacro is used to execute a macro consisting of multiple steps. // Each step can have multiple keys, multiple modifiers and a delay. // The keys and modifiers are pressed together and held for the delay duration. @@ -292,9 +291,7 @@ export default function useKeyboard() { for (const [_, step] of steps.entries()) { const keyValues = (step.keys || []).map(key => keys[key]).filter(Boolean); const modifierMask: number = (step.modifiers || []) - .map(mod => modifiers[mod]) - .reduce((acc, val) => acc + val, 0); // If the step has keys and/or modifiers, press them and hold for the delay @@ -306,6 +303,7 @@ export default function useKeyboard() { sendKeyboardMacroEventHidRpc(macro); }, [sendKeyboardMacroEventHidRpc]); + const executeMacroClientSide = useCallback(async (steps: MacroSteps) => { const promises: (() => Promise)[] = []; From f54b6283d66f10e31c3621d720e06909460b29bf Mon Sep 17 00:00:00 2001 From: Marc Brooks Date: Thu, 25 Sep 2025 16:36:28 -0500 Subject: [PATCH 2/4] Move the HID keyboard descriptor LED state Seems to interfere with boot mode --- internal/usbgadget/hid_keyboard.go | 42 +++++++++++++++++++++++------- 1 file changed, 32 insertions(+), 10 deletions(-) diff --git a/internal/usbgadget/hid_keyboard.go b/internal/usbgadget/hid_keyboard.go index 1deb7c95..1ffbe9fe 100644 --- a/internal/usbgadget/hid_keyboard.go +++ b/internal/usbgadget/hid_keyboard.go @@ -31,6 +31,8 @@ var keyboardReportDesc = []byte{ 0x05, 0x01, /* USAGE_PAGE (Generic Desktop) */ 0x09, 0x06, /* USAGE (Keyboard) */ 0xa1, 0x01, /* COLLECTION (Application) */ + + /* 8 modifier bits */ 0x05, 0x07, /* USAGE_PAGE (Keyboard) */ 0x19, 0xe0, /* USAGE_MINIMUM (Keyboard LeftControl) */ 0x29, 0xe7, /* USAGE_MAXIMUM (Keyboard Right GUI) */ @@ -39,27 +41,47 @@ var keyboardReportDesc = []byte{ 0x75, 0x01, /* REPORT_SIZE (1) */ 0x95, 0x08, /* REPORT_COUNT (8) */ 0x81, 0x02, /* INPUT (Data,Var,Abs) */ + + /* 8 bits of padding */ 0x95, 0x01, /* REPORT_COUNT (1) */ 0x75, 0x08, /* REPORT_SIZE (8) */ 0x81, 0x03, /* INPUT (Cnst,Var,Abs) */ - 0x95, 0x05, /* REPORT_COUNT (5) */ - 0x75, 0x01, /* REPORT_SIZE (1) */ - 0x05, 0x08, /* USAGE_PAGE (LEDs) */ - 0x19, 0x01, /* USAGE_MINIMUM (Num Lock) */ - 0x29, 0x05, /* USAGE_MAXIMUM (Kana) */ - 0x91, 0x02, /* OUTPUT (Data,Var,Abs) */ - 0x95, 0x01, /* REPORT_COUNT (1) */ - 0x75, 0x03, /* REPORT_SIZE (3) */ - 0x91, 0x03, /* OUTPUT (Cnst,Var,Abs) */ + /* 6 key codes for the 104 key keyboard */ 0x95, 0x06, /* REPORT_COUNT (6) */ 0x75, 0x08, /* REPORT_SIZE (8) */ 0x15, 0x00, /* LOGICAL_MINIMUM (0) */ - 0x25, 104, /* LOGICAL_MAXIMUM (104-key) */ + 0x25, 0xE7, /* LOGICAL_MAXIMUM (104-key HID) */ 0x05, 0x07, /* USAGE_PAGE (Keyboard) */ 0x19, 0x00, /* USAGE_MINIMUM (Reserved) */ 0x29, 0xE7, /* USAGE_MAXIMUM (Keyboard Right GUI) */ 0x81, 0x00, /* INPUT (Data,Ary,Abs) */ + + /* LED report 5 bits for Num Lock through Kana */ + 0x95, 0x05, /* REPORT_COUNT (5) */ + 0x75, 0x01, /* REPORT_SIZE (1) */ + 0x05, 0x08, /* USAGE_PAGE (LEDs) */ + 0x19, 0x01, /* USAGE_MINIMUM (Num Lock) */ + 0x29, 0x05, /* USAGE_MAXIMUM (Kana) */ + 0x91, 0x02, /* OUTPUT (Data,Var,Abs) */ + + /* 1 bit of padding for the Power LED (ignored) */ + 0x95, 0x01, /* REPORT_COUNT (1) */ + 0x75, 0x03, /* REPORT_SIZE (1) */ + 0x91, 0x03, /* OUTPUT (Cnst,Var,Abs) */ + + /* LED report 1 bit for Shift */ + 0x95, 0x01, /* REPORT_COUNT (1) */ + 0x75, 0x01, /* REPORT_SIZE (1) */ + 0x05, 0x08, /* USAGE_PAGE (LEDs) */ + 0x19, 0x07, /* USAGE_MINIMUM (Shift) */ + 0x29, 0x07, /* USAGE_MAXIMUM (Shift) */ + 0x91, 0x02, /* OUTPUT (Data,Var,Abs) */ + + /* 1 bit of padding for the rest of the byte */ + 0x95, 0x01, /* REPORT_COUNT (1) */ + 0x75, 0x03, /* REPORT_SIZE (1) */ + 0x91, 0x03, /* OUTPUT (Cnst,Var,Abs) */ 0xc0, /* END_COLLECTION */ } From faa35a4dca9a5d19fec1b35466f205f0932ae363 Mon Sep 17 00:00:00 2001 From: Marc Brooks Date: Thu, 25 Sep 2025 20:36:52 -0500 Subject: [PATCH 3/4] Run paste/macros in background on their own queue. Also returns a token for cancellation. Fixed error in length check for macro key state. Removed redundant clear operation. --- cloud.go | 2 +- hidrpc.go | 31 ++++++++- internal/hidrpc/hidrpc.go | 61 +++++++++++------ internal/hidrpc/message.go | 63 ++++++++++++++++- jsonrpc.go | 134 +++++++++++++++++++++++++------------ ui/package-lock.json | 14 ++++ ui/package.json | 1 + ui/src/hooks/hidRpc.ts | 27 +++++++- ui/src/hooks/useHidRpc.ts | 2 +- web.go | 2 +- webrtc.go | 6 +- 11 files changed, 265 insertions(+), 78 deletions(-) diff --git a/cloud.go b/cloud.go index fb138508..cf301f44 100644 --- a/cloud.go +++ b/cloud.go @@ -477,7 +477,7 @@ func handleSessionRequest( cloudLogger.Trace().Interface("session", session).Msg("new session accepted") // Cancel any ongoing keyboard macro when session changes - cancelKeyboardMacro() + cancelAllRunningKeyboardMacros() currentSession = session _ = wsjson.Write(context.Background(), c, gin.H{"type": "answer", "data": sd}) diff --git a/hidrpc.go b/hidrpc.go index abf3bd73..b2233673 100644 --- a/hidrpc.go +++ b/hidrpc.go @@ -26,20 +26,45 @@ func handleHidRPCMessage(message hidrpc.Message, session *Session) { return } session.hidRPCAvailable = true + case hidrpc.TypeKeypressReport, hidrpc.TypeKeyboardReport: rpcErr = handleHidRPCKeyboardInput(message) + case hidrpc.TypeKeyboardMacroReport: keyboardMacroReport, err := message.KeyboardMacroReport() if err != nil { logger.Warn().Err(err).Msg("failed to get keyboard macro report") return } - rpcErr = rpcExecuteKeyboardMacro(keyboardMacroReport.Steps) + token := rpcExecuteKeyboardMacro(keyboardMacroReport.IsPaste, keyboardMacroReport.Steps) + logger.Debug().Str("token", token.String()).Msg("started keyboard macro") + message, err := hidrpc.NewKeyboardMacroTokenMessage(token).Marshal() + + if err != nil { + logger.Warn().Err(err).Msg("failed to marshal running macro token message") + return + } + if err := session.HidChannel.Send(message); err != nil { + logger.Warn().Err(err).Msg("failed to send running macro token message") + return + } + case hidrpc.TypeCancelKeyboardMacroReport: rpcCancelKeyboardMacro() return + + case hidrpc.TypeCancelKeyboardMacroByTokenReport: + token, err := message.KeyboardMacroToken() + if err != nil { + logger.Warn().Err(err).Msg("failed to get keyboard macro token") + return + } + rpcCancelKeyboardMacroByToken(token) + return + case hidrpc.TypeKeypressKeepAliveReport: rpcErr = handleHidRPCKeypressKeepAlive(session) + case hidrpc.TypePointerReport: pointerReport, err := message.PointerReport() if err != nil { @@ -47,6 +72,7 @@ func handleHidRPCMessage(message hidrpc.Message, session *Session) { return } rpcErr = rpcAbsMouseReport(pointerReport.X, pointerReport.Y, pointerReport.Button) + case hidrpc.TypeMouseReport: mouseReport, err := message.MouseReport() if err != nil { @@ -54,6 +80,7 @@ func handleHidRPCMessage(message hidrpc.Message, session *Session) { return } rpcErr = rpcRelMouseReport(mouseReport.DX, mouseReport.DY, mouseReport.Button) + default: logger.Warn().Uint8("type", uint8(message.Type())).Msg("unknown HID RPC message type") } @@ -98,7 +125,7 @@ func onHidMessage(msg hidQueueMessage, session *Session) { r <- nil }() select { - case <-time.After(30 * time.Second): + case <-time.After(1 * time.Second): scopedLogger.Warn().Msg("HID RPC message timed out") case <-r: scopedLogger.Debug().Dur("duration", time.Since(t)).Msg("HID RPC message handled") diff --git a/internal/hidrpc/hidrpc.go b/internal/hidrpc/hidrpc.go index 7313e3b5..f8844478 100644 --- a/internal/hidrpc/hidrpc.go +++ b/internal/hidrpc/hidrpc.go @@ -3,6 +3,7 @@ package hidrpc import ( "fmt" + "github.com/google/uuid" "github.com/jetkvm/kvm/internal/usbgadget" ) @@ -10,38 +11,46 @@ import ( type MessageType byte const ( - TypeHandshake MessageType = 0x01 - TypeKeyboardReport MessageType = 0x02 - TypePointerReport MessageType = 0x03 - TypeWheelReport MessageType = 0x04 - TypeKeypressReport MessageType = 0x05 - TypeKeypressKeepAliveReport MessageType = 0x09 - TypeMouseReport MessageType = 0x06 - TypeKeyboardMacroReport MessageType = 0x07 - TypeCancelKeyboardMacroReport MessageType = 0x08 - TypeKeyboardLedState MessageType = 0x32 - TypeKeydownState MessageType = 0x33 - TypeKeyboardMacroState MessageType = 0x34 + TypeHandshake MessageType = 0x01 + TypeKeyboardReport MessageType = 0x02 + TypePointerReport MessageType = 0x03 + TypeWheelReport MessageType = 0x04 + TypeKeypressReport MessageType = 0x05 + TypeKeypressKeepAliveReport MessageType = 0x09 + TypeMouseReport MessageType = 0x06 + TypeKeyboardMacroReport MessageType = 0x07 + TypeCancelKeyboardMacroReport MessageType = 0x08 + TypeKeyboardLedState MessageType = 0x32 + TypeKeydownState MessageType = 0x33 + TypeKeyboardMacroState MessageType = 0x34 + TypeCancelKeyboardMacroByTokenReport MessageType = 0x35 ) +type QueueIndex int + const ( - Version byte = 0x01 // Version of the HID RPC protocol + Version byte = 0x01 // Version of the HID RPC protocol + HandshakeQueue int = 0 // Queue index for handshake messages + KeyboardQueue int = 1 // Queue index for keyboard and macro messages + MouseQueue int = 2 // Queue index for mouse messages + MacroQueue int = 3 // Queue index for macro cancel messages + OtherQueue int = 4 // Queue index for other messages ) // GetQueueIndex returns the index of the queue to which the message should be enqueued. func GetQueueIndex(messageType MessageType) int { switch messageType { case TypeHandshake: - return 0 - case TypeKeyboardReport, TypeKeypressReport, TypeKeyboardMacroReport, TypeKeyboardLedState, TypeKeydownState, TypeKeyboardMacroState: - return 1 + return HandshakeQueue + case TypeKeyboardReport, TypeKeypressReport, TypeKeyboardLedState, TypeKeydownState, TypeKeyboardMacroState: + return KeyboardQueue case TypePointerReport, TypeMouseReport, TypeWheelReport: - return 2 - // we don't want to block the queue for this message - case TypeCancelKeyboardMacroReport: - return 3 + return MouseQueue + // we don't want to block the queue for these messages + case TypeKeyboardMacroReport, TypeCancelKeyboardMacroReport, TypeCancelKeyboardMacroByTokenReport: + return MacroQueue default: - return 3 + return OtherQueue } } @@ -121,3 +130,13 @@ func NewKeyboardMacroStateMessage(state bool, isPaste bool) *Message { d: data, } } + +// NewKeyboardMacroTokenMessage creates a new keyboard macro token message. +func NewKeyboardMacroTokenMessage(token uuid.UUID) *Message { + data, _ := token.MarshalBinary() + + return &Message{ + t: TypeKeyboardMacroState, + d: data, + } +} diff --git a/internal/hidrpc/message.go b/internal/hidrpc/message.go index 3f3506f7..88ff6602 100644 --- a/internal/hidrpc/message.go +++ b/internal/hidrpc/message.go @@ -3,6 +3,8 @@ package hidrpc import ( "encoding/binary" "fmt" + + "github.com/google/uuid" ) // Message .. @@ -23,6 +25,9 @@ func (m *Message) Type() MessageType { func (m *Message) String() string { switch m.t { case TypeHandshake: + if len(m.d) != 0 { + return fmt.Sprintf("Handshake{Malformed: %v}", m.d) + } return "Handshake" case TypeKeypressReport: if len(m.d) < 2 { @@ -45,12 +50,45 @@ func (m *Message) String() string { } return fmt.Sprintf("MouseReport{DX: %d, DY: %d, Button: %d}", m.d[0], m.d[1], m.d[2]) case TypeKeypressKeepAliveReport: + if len(m.d) != 0 { + return fmt.Sprintf("KeypressKeepAliveReport{Malformed: %v}", m.d) + } return "KeypressKeepAliveReport" + case TypeWheelReport: + if len(m.d) < 3 { + return fmt.Sprintf("WheelReport{Malformed: %v}", m.d) + } + return fmt.Sprintf("WheelReport{Vertical: %d, Horizontal: %d}", int8(m.d[0]), int8(m.d[1])) case TypeKeyboardMacroReport: if len(m.d) < 5 { return fmt.Sprintf("KeyboardMacroReport{Malformed: %v}", m.d) } return fmt.Sprintf("KeyboardMacroReport{IsPaste: %v, Length: %d}", m.d[0] == uint8(1), binary.BigEndian.Uint32(m.d[1:5])) + case TypeCancelKeyboardMacroReport: + if len(m.d) != 0 { + return fmt.Sprintf("CancelKeyboardMacroReport{Malformed: %v}", m.d) + } + return "CancelKeyboardMacroReport" + case TypeCancelKeyboardMacroByTokenReport: + if len(m.d) != 16 { + return fmt.Sprintf("CancelKeyboardMacroByTokenReport{Malformed: %v}", m.d) + } + return fmt.Sprintf("CancelKeyboardMacroByTokenReport{Token: %s}", uuid.Must(uuid.FromBytes(m.d)).String()) + case TypeKeyboardLedState: + if len(m.d) < 1 { + return fmt.Sprintf("KeyboardLedState{Malformed: %v}", m.d) + } + return fmt.Sprintf("KeyboardLedState{State: %d}", m.d[0]) + case TypeKeydownState: + if len(m.d) < 1 { + return fmt.Sprintf("KeydownState{Malformed: %v}", m.d) + } + return fmt.Sprintf("KeydownState{State: %d}", m.d[0]) + case TypeKeyboardMacroState: + if len(m.d) < 2 { + return fmt.Sprintf("KeyboardMacroState{Malformed: %v}", m.d) + } + return fmt.Sprintf("KeyboardMacroState{State: %v, IsPaste: %v}", m.d[0] == uint8(1), m.d[1] == uint8(1)) default: return fmt.Sprintf("Unknown{Type: %d, Data: %v}", m.t, m.d) } @@ -67,7 +105,9 @@ func (m *Message) KeypressReport() (KeypressReport, error) { if m.t != TypeKeypressReport { return KeypressReport{}, fmt.Errorf("invalid message type: %d", m.t) } - + if len(m.d) < 2 { + return KeypressReport{}, fmt.Errorf("invalid message data length: %d", len(m.d)) + } return KeypressReport{ Key: m.d[0], Press: m.d[1] == uint8(1), @@ -95,7 +135,7 @@ func (m *Message) KeyboardReport() (KeyboardReport, error) { // Macro .. type KeyboardMacroStep struct { Modifier byte // 1 byte - Keys []byte // 6 bytes: hidKeyBufferSize + Keys []byte // 6 bytes: HidKeyBufferSize Delay uint16 // 2 bytes } type KeyboardMacroReport struct { @@ -105,7 +145,7 @@ type KeyboardMacroReport struct { } // HidKeyBufferSize is the size of the keys buffer in the keyboard report. -const HidKeyBufferSize = 6 +const HidKeyBufferSize int = 6 // KeyboardMacroReport returns the keyboard macro report from the message. func (m *Message) KeyboardMacroReport() (KeyboardMacroReport, error) { @@ -205,3 +245,20 @@ func (m *Message) KeyboardMacroState() (KeyboardMacroState, error) { IsPaste: m.d[1] == uint8(1), }, nil } + +// KeyboardMacroToken returns the keyboard macro token UUID from the message. +func (m *Message) KeyboardMacroToken() (uuid.UUID, error) { + if m.t != TypeCancelKeyboardMacroByTokenReport { + return uuid.Nil, fmt.Errorf("invalid message type: %d", m.t) + } + + if len(m.d) == 0 { + return uuid.Nil, nil + } + + if len(m.d) != 16 { + return uuid.Nil, fmt.Errorf("invalid UUID length: %d", len(m.d)) + } + + return uuid.FromBytes(m.d) +} diff --git a/jsonrpc.go b/jsonrpc.go index e3db0fac..bf21b9bf 100644 --- a/jsonrpc.go +++ b/jsonrpc.go @@ -1,7 +1,6 @@ package kvm import ( - "bytes" "context" "encoding/json" "errors" @@ -14,6 +13,7 @@ import ( "sync" "time" + "github.com/google/uuid" "github.com/pion/webrtc/v4" "github.com/rs/zerolog" "go.bug.st/serial" @@ -1070,68 +1070,121 @@ func rpcSetLocalLoopbackOnly(enabled bool) error { return nil } +type RunningMacro struct { + cancel context.CancelFunc + isPaste bool +} + var ( - keyboardMacroCancel context.CancelFunc - keyboardMacroLock sync.Mutex + keyboardMacroCancelMap map[uuid.UUID]RunningMacro + keyboardMacroLock sync.Mutex ) -// cancelKeyboardMacro cancels any ongoing keyboard macro execution -func cancelKeyboardMacro() { +func init() { + keyboardMacroCancelMap = make(map[uuid.UUID]RunningMacro) +} + +func addKeyboardMacro(isPaste bool, cancel context.CancelFunc) uuid.UUID { keyboardMacroLock.Lock() defer keyboardMacroLock.Unlock() - if keyboardMacroCancel != nil { - keyboardMacroCancel() - logger.Info().Msg("canceled keyboard macro") - keyboardMacroCancel = nil + token := uuid.New() // Generate a unique token + keyboardMacroCancelMap[token] = RunningMacro{ + isPaste: isPaste, + cancel: cancel, + } + return token +} + +func removeRunningKeyboardMacro(token uuid.UUID) { + keyboardMacroLock.Lock() + defer keyboardMacroLock.Unlock() + + delete(keyboardMacroCancelMap, token) +} + +func cancelRunningKeyboardMacro(token uuid.UUID) { + keyboardMacroLock.Lock() + defer keyboardMacroLock.Unlock() + + if runningMacro, exists := keyboardMacroCancelMap[token]; exists { + runningMacro.cancel() + delete(keyboardMacroCancelMap, token) + logger.Info().Interface("token", token).Msg("canceled keyboard macro by token") + } else { + logger.Debug().Interface("token", token).Msg("no running keyboard macro found for token") } } -func setKeyboardMacroCancel(cancel context.CancelFunc) { +func cancelAllRunningKeyboardMacros() { keyboardMacroLock.Lock() defer keyboardMacroLock.Unlock() - keyboardMacroCancel = cancel + for token, runningMacro := range keyboardMacroCancelMap { + runningMacro.cancel() + delete(keyboardMacroCancelMap, token) + logger.Info().Interface("token", token).Msg("cancelled keyboard macro") + } } -func rpcExecuteKeyboardMacro(macro []hidrpc.KeyboardMacroStep) error { - cancelKeyboardMacro() +func reportRunningMacrosState() { + if currentSession != nil { + keyboardMacroLock.Lock() + defer keyboardMacroLock.Unlock() + isPaste := false + anyRunning := false + for _, runningMacro := range keyboardMacroCancelMap { + anyRunning = true + if runningMacro.isPaste { + isPaste = true + break + } + } + + state := hidrpc.KeyboardMacroState{ + State: anyRunning, + IsPaste: isPaste, + } + + currentSession.reportHidRPCKeyboardMacroState(state) + } +} + +func rpcExecuteKeyboardMacro(isPaste bool, macro []hidrpc.KeyboardMacroStep) uuid.UUID { ctx, cancel := context.WithCancel(context.Background()) - setKeyboardMacroCancel(cancel) + token := addKeyboardMacro(isPaste, cancel) + reportRunningMacrosState() - s := hidrpc.KeyboardMacroState{ - State: true, - IsPaste: true, - } + go func() { + defer reportRunningMacrosState() // this executes last, so the map is already updated + defer removeRunningKeyboardMacro(token) // this executes first, to update the map - if currentSession != nil { - currentSession.reportHidRPCKeyboardMacroState(s) - } + err := executeKeyboardMacro(ctx, isPaste, macro) + if err != nil { + logger.Error().Err(err).Interface("token", token).Bool("isPaste", isPaste).Msg("keyboard macro execution failed") + } + }() - err := rpcDoExecuteKeyboardMacro(ctx, macro) - - setKeyboardMacroCancel(nil) - - s.State = false - if currentSession != nil { - currentSession.reportHidRPCKeyboardMacroState(s) - } - - return err + return token } func rpcCancelKeyboardMacro() { - cancelKeyboardMacro() + defer reportRunningMacrosState() + cancelAllRunningKeyboardMacros() } -var keyboardClearStateKeys = make([]byte, hidrpc.HidKeyBufferSize) +func rpcCancelKeyboardMacroByToken(token uuid.UUID) { + defer reportRunningMacrosState() -func isClearKeyStep(step hidrpc.KeyboardMacroStep) bool { - return step.Modifier == 0 && bytes.Equal(step.Keys, keyboardClearStateKeys) + if token == uuid.Nil { + cancelAllRunningKeyboardMacros() + } else { + cancelRunningKeyboardMacro(token) + } } -func rpcDoExecuteKeyboardMacro(ctx context.Context, macro []hidrpc.KeyboardMacroStep) error { +func executeKeyboardMacro(ctx context.Context, isPaste bool, macro []hidrpc.KeyboardMacroStep) error { logger.Debug().Int("macro_steps", len(macro)).Msg("Executing keyboard macro") // don't report keyboard state changes while executing the macro @@ -1143,15 +1196,10 @@ func rpcDoExecuteKeyboardMacro(ctx context.Context, macro []hidrpc.KeyboardMacro err := rpcKeyboardReport(step.Modifier, step.Keys) if err != nil { - logger.Warn().Err(err).Msg("failed to execute keyboard macro") + logger.Warn().Err(err).Int("step", i).Msg("failed to execute keyboard macro") return err } - // notify the device that the keyboard state is being cleared - if isClearKeyStep(step) { - gadget.UpdateKeysDown(0, keyboardClearStateKeys) - } - // Use context-aware sleep that can be cancelled select { case <-time.After(delay): @@ -1159,7 +1207,7 @@ func rpcDoExecuteKeyboardMacro(ctx context.Context, macro []hidrpc.KeyboardMacro case <-ctx.Done(): // make sure keyboard state is reset and the client gets notified gadget.ResumeSuspendKeyDownMessages() - err := rpcKeyboardReport(0, keyboardClearStateKeys) + err := rpcKeyboardReport(0, make([]byte, hidrpc.HidKeyBufferSize)) if err != nil { logger.Warn().Err(err).Msg("failed to reset keyboard state") } diff --git a/ui/package-lock.json b/ui/package-lock.json index e2b4b03f..108b353f 100644 --- a/ui/package-lock.json +++ b/ui/package-lock.json @@ -37,6 +37,7 @@ "recharts": "^3.2.1", "tailwind-merge": "^3.3.1", "usehooks-ts": "^3.1.1", + "uuid": "^13.0.0", "validator": "^13.15.15", "zustand": "^4.5.2" }, @@ -6880,6 +6881,19 @@ "dev": true, "license": "MIT" }, + "node_modules/uuid": { + "version": "13.0.0", + "resolved": "https://registry.npmjs.org/uuid/-/uuid-13.0.0.tgz", + "integrity": "sha512-XQegIaBTVUjSHliKqcnFqYypAd4S+WCYt5NIeRs6w/UAry7z8Y9j5ZwRRL4kzq9U3sD6v+85er9FvkEaBpji2w==", + "funding": [ + "https://github.com/sponsors/broofa", + "https://github.com/sponsors/ctavan" + ], + "license": "MIT", + "bin": { + "uuid": "dist-node/bin/uuid" + } + }, "node_modules/validator": { "version": "13.15.15", "resolved": "https://registry.npmjs.org/validator/-/validator-13.15.15.tgz", diff --git a/ui/package.json b/ui/package.json index 46b16c0c..43cbbe6b 100644 --- a/ui/package.json +++ b/ui/package.json @@ -48,6 +48,7 @@ "recharts": "^3.2.1", "tailwind-merge": "^3.3.1", "usehooks-ts": "^3.1.1", + "uuid": "^13.0.0", "validator": "^13.15.15", "zustand": "^4.5.2" }, diff --git a/ui/src/hooks/hidRpc.ts b/ui/src/hooks/hidRpc.ts index 823384ff..737ee58d 100644 --- a/ui/src/hooks/hidRpc.ts +++ b/ui/src/hooks/hidRpc.ts @@ -1,3 +1,5 @@ +import { parse as uuidParse , stringify as uuidStringify } from "uuid"; + import { hidKeyBufferSize, KeyboardLedState, KeysDownState } from "./stores"; export const HID_RPC_MESSAGE_TYPES = { @@ -13,6 +15,7 @@ export const HID_RPC_MESSAGE_TYPES = { KeyboardLedState: 0x32, KeysDownState: 0x33, KeyboardMacroState: 0x34, + CancelKeyboardMacroByTokenReport: 0x35, } export type HidRpcMessageType = typeof HID_RPC_MESSAGE_TYPES[keyof typeof HID_RPC_MESSAGE_TYPES]; @@ -299,7 +302,7 @@ export class KeyboardMacroStateMessage extends RpcMessage { } public static unmarshal(data: Uint8Array): KeyboardMacroStateMessage | undefined { - if (data.length < 1) { + if (data.length < 2) { throw new Error(`Invalid keyboard macro state report message length: ${data.length}`); } @@ -378,13 +381,30 @@ export class PointerReportMessage extends RpcMessage { } export class CancelKeyboardMacroReportMessage extends RpcMessage { + token: string; - constructor() { + constructor(token: string) { super(HID_RPC_MESSAGE_TYPES.CancelKeyboardMacroReport); + this.token = (token == null || token === undefined || token === "") + ? "00000000-0000-0000-0000-000000000000" + : token; } marshal(): Uint8Array { - return new Uint8Array([this.messageType]); + const tokenBytes = uuidParse(this.token); + return new Uint8Array([this.messageType, ...tokenBytes]); + } + + public static unmarshal(data: Uint8Array): CancelKeyboardMacroReportMessage | undefined { + if (data.length == 0) { + return new CancelKeyboardMacroReportMessage("00000000-0000-0000-0000-000000000000"); + } + + if (data.length != 16) { + throw new Error(`Invalid cancel message length: ${data.length}`); + } + + return new CancelKeyboardMacroReportMessage(uuidStringify(data.slice(0, 16))); } } @@ -430,6 +450,7 @@ export const messageRegistry = { [HID_RPC_MESSAGE_TYPES.CancelKeyboardMacroReport]: CancelKeyboardMacroReportMessage, [HID_RPC_MESSAGE_TYPES.KeyboardMacroState]: KeyboardMacroStateMessage, [HID_RPC_MESSAGE_TYPES.KeypressKeepAliveReport]: KeypressKeepAliveMessage, + [HID_RPC_MESSAGE_TYPES.CancelKeyboardMacroByTokenReport]: CancelKeyboardMacroReportMessage, } export const unmarshalHidRpcMessage = (data: Uint8Array): RpcMessage | undefined => { diff --git a/ui/src/hooks/useHidRpc.ts b/ui/src/hooks/useHidRpc.ts index aeb1c4fa..ad7fe477 100644 --- a/ui/src/hooks/useHidRpc.ts +++ b/ui/src/hooks/useHidRpc.ts @@ -142,7 +142,7 @@ export function useHidRpc(onHidRpcMessage?: (payload: RpcMessage) => void) { const cancelOngoingKeyboardMacro = useCallback( () => { - sendMessage(new CancelKeyboardMacroReportMessage()); + sendMessage(new CancelKeyboardMacroReportMessage("")); }, [sendMessage], ); diff --git a/web.go b/web.go index 45253579..2bf60d9b 100644 --- a/web.go +++ b/web.go @@ -230,7 +230,7 @@ func handleWebRTCSession(c *gin.Context) { } // Cancel any ongoing keyboard macro when session changes - cancelKeyboardMacro() + cancelAllRunningKeyboardMacros() currentSession = session c.JSON(http.StatusOK, gin.H{"sd": sd}) diff --git a/webrtc.go b/webrtc.go index 7fd13929..73879bdf 100644 --- a/webrtc.go +++ b/webrtc.go @@ -98,7 +98,7 @@ func (s *Session) initQueues() { defer s.hidQueueLock.Unlock() s.hidQueue = make([]chan hidQueueMessage, 0) - for i := 0; i < 4; i++ { + for i := 0; i <= hidrpc.OtherQueue; i++ { q := make(chan hidQueueMessage, 256) s.hidQueue = append(s.hidQueue, q) } @@ -163,7 +163,7 @@ func getOnHidMessageHandler(session *Session, scopedLogger *zerolog.Logger, chan queueIndex := hidrpc.GetQueueIndex(hidrpc.MessageType(msg.Data[0])) if queueIndex >= len(session.hidQueue) || queueIndex < 0 { l.Warn().Int("queueIndex", queueIndex).Msg("received data in HID RPC message handler, but queue index not found") - queueIndex = 3 + queueIndex = hidrpc.OtherQueue } queue := session.hidQueue[queueIndex] @@ -328,7 +328,7 @@ func newSession(config SessionConfig) (*Session, error) { scopedLogger.Debug().Msg("ICE Connection State is closed, unmounting virtual media") if session == currentSession { // Cancel any ongoing keyboard report multi when session closes - cancelKeyboardMacro() + cancelAllRunningKeyboardMacros() currentSession = nil } // Stop RPC processor From 9fafcfb94c7b614bfb19722090a648c2d1836647 Mon Sep 17 00:00:00 2001 From: Marc Brooks Date: Fri, 26 Sep 2025 12:46:53 -0500 Subject: [PATCH 4/4] Use Once instead of init() --- jsonrpc.go | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/jsonrpc.go b/jsonrpc.go index bf21b9bf..4a343951 100644 --- a/jsonrpc.go +++ b/jsonrpc.go @@ -1078,15 +1078,20 @@ type RunningMacro struct { var ( keyboardMacroCancelMap map[uuid.UUID]RunningMacro keyboardMacroLock sync.Mutex + keyboardMacroOnce sync.Once ) -func init() { - keyboardMacroCancelMap = make(map[uuid.UUID]RunningMacro) +func getKeyboardMacroCancelMap() map[uuid.UUID]RunningMacro { + keyboardMacroOnce.Do(func() { + keyboardMacroCancelMap = make(map[uuid.UUID]RunningMacro) + }) + return keyboardMacroCancelMap } func addKeyboardMacro(isPaste bool, cancel context.CancelFunc) uuid.UUID { keyboardMacroLock.Lock() defer keyboardMacroLock.Unlock() + keyboardMacroCancelMap := getKeyboardMacroCancelMap() token := uuid.New() // Generate a unique token keyboardMacroCancelMap[token] = RunningMacro{ @@ -1099,6 +1104,7 @@ func addKeyboardMacro(isPaste bool, cancel context.CancelFunc) uuid.UUID { func removeRunningKeyboardMacro(token uuid.UUID) { keyboardMacroLock.Lock() defer keyboardMacroLock.Unlock() + keyboardMacroCancelMap := getKeyboardMacroCancelMap() delete(keyboardMacroCancelMap, token) } @@ -1106,6 +1112,7 @@ func removeRunningKeyboardMacro(token uuid.UUID) { func cancelRunningKeyboardMacro(token uuid.UUID) { keyboardMacroLock.Lock() defer keyboardMacroLock.Unlock() + keyboardMacroCancelMap := getKeyboardMacroCancelMap() if runningMacro, exists := keyboardMacroCancelMap[token]; exists { runningMacro.cancel() @@ -1119,6 +1126,7 @@ func cancelRunningKeyboardMacro(token uuid.UUID) { func cancelAllRunningKeyboardMacros() { keyboardMacroLock.Lock() defer keyboardMacroLock.Unlock() + keyboardMacroCancelMap := getKeyboardMacroCancelMap() for token, runningMacro := range keyboardMacroCancelMap { runningMacro.cancel() @@ -1131,6 +1139,7 @@ func reportRunningMacrosState() { if currentSession != nil { keyboardMacroLock.Lock() defer keyboardMacroLock.Unlock() + keyboardMacroCancelMap := getKeyboardMacroCancelMap() isPaste := false anyRunning := false