From 4b0818502c71db213a467df101ecb421db3b098b Mon Sep 17 00:00:00 2001 From: Siyuan Miao Date: Tue, 9 Sep 2025 23:56:23 +0200 Subject: [PATCH 01/10] feat: send all paste keystrokes to backend --- cloud.go | 4 + hidrpc.go | 6 ++ jsonrpc.go | 97 +++++++++++++++++++ ui/src/components/popovers/PasteModal.tsx | 89 ++++++++--------- ui/src/hooks/useKeyboard.ts | 111 +++++++++++++--------- web.go | 4 + webrtc.go | 2 + 7 files changed, 224 insertions(+), 89 deletions(-) diff --git a/cloud.go b/cloud.go index cec749e4..f86a4815 100644 --- a/cloud.go +++ b/cloud.go @@ -475,6 +475,10 @@ func handleSessionRequest( cloudLogger.Info().Interface("session", session).Msg("new session accepted") cloudLogger.Trace().Interface("session", session).Msg("new session accepted") + + // Cancel any ongoing keyboard report multi when session changes + cancelKeyboardReportMulti() + currentSession = session _ = wsjson.Write(context.Background(), c, gin.H{"type": "answer", "data": sd}) return nil diff --git a/hidrpc.go b/hidrpc.go index 74fe687f..9537e5b9 100644 --- a/hidrpc.go +++ b/hidrpc.go @@ -1,7 +1,9 @@ package kvm import ( + "errors" "fmt" + "io" "time" "github.com/jetkvm/kvm/internal/hidrpc" @@ -143,6 +145,10 @@ func reportHidRPC(params any, session *Session) { } if err := session.HidChannel.Send(message); err != nil { + if errors.Is(err, io.ErrClosedPipe) { + logger.Debug().Err(err).Msg("HID RPC channel closed, skipping reportHidRPC") + return + } logger.Warn().Err(err).Msg("failed to send HID RPC message") } } diff --git a/jsonrpc.go b/jsonrpc.go index ff3a4b12..711edeb0 100644 --- a/jsonrpc.go +++ b/jsonrpc.go @@ -10,6 +10,7 @@ import ( "path/filepath" "reflect" "strconv" + "sync" "time" "github.com/pion/webrtc/v4" @@ -1049,6 +1050,101 @@ func rpcSetLocalLoopbackOnly(enabled bool) error { return nil } +// cancelKeyboardReportMulti cancels any ongoing keyboard report multi execution +func cancelKeyboardReportMulti() { + keyboardReportMultiLock.Lock() + defer keyboardReportMultiLock.Unlock() + + if keyboardReportMultiCancel != nil { + keyboardReportMultiCancel() + logger.Info().Msg("canceled keyboard report multi") + keyboardReportMultiCancel = nil + } +} + +func rpcKeyboardReportMultiWrapper(macro []map[string]any) (usbgadget.KeysDownState, error) { + keyboardReportMultiLock.Lock() + defer keyboardReportMultiLock.Unlock() + + if keyboardReportMultiCancel != nil { + keyboardReportMultiCancel() + logger.Info().Msg("canceled previous keyboard report multi") + } + + ctx, cancel := context.WithCancel(context.Background()) + keyboardReportMultiCancel = cancel + + result, err := rpcKeyboardReportMulti(ctx, macro) + + keyboardReportMultiCancel = nil + + return result, err +} + +var ( + keyboardReportMultiCancel context.CancelFunc + keyboardReportMultiLock sync.Mutex +) + +func rpcKeyboardReportMulti(ctx context.Context, macro []map[string]any) (usbgadget.KeysDownState, error) { + var last usbgadget.KeysDownState + var err error + + logger.Debug().Interface("macro", macro).Msg("Executing keyboard report multi") + + for i, step := range macro { + // Check for cancellation before each step + select { + case <-ctx.Done(): + logger.Debug().Msg("Keyboard report multi context cancelled") + return last, ctx.Err() + default: + } + + var modifier byte + if m, ok := step["modifier"].(float64); ok { + modifier = byte(int(m)) + } else if mi, ok := step["modifier"].(int); ok { + modifier = byte(mi) + } else if mb, ok := step["modifier"].(uint8); ok { + modifier = mb + } + + var keys []byte + if arr, ok := step["keys"].([]any); ok { + keys = make([]byte, 0, len(arr)) + for _, v := range arr { + if f, ok := v.(float64); ok { + keys = append(keys, byte(int(f))) + } else if i, ok := v.(int); ok { + keys = append(keys, byte(i)) + } else if b, ok := v.(uint8); ok { + keys = append(keys, b) + } + } + } else if bs, ok := step["keys"].([]byte); ok { + keys = bs + } + + // Use context-aware sleep that can be cancelled + select { + case <-time.After(100 * time.Millisecond): + // Sleep completed normally + case <-ctx.Done(): + logger.Debug().Int("step", i).Msg("Keyboard report multi cancelled during sleep") + return last, ctx.Err() + } + + last, err = rpcKeyboardReport(modifier, keys) + if err != nil { + logger.Warn().Err(err).Msg("failed to execute keyboard report multi") + return last, err + } + } + + return last, nil +} + var rpcHandlers = map[string]RPCHandler{ "ping": {Func: rpcPing}, "reboot": {Func: rpcReboot, Params: []string{"force"}}, @@ -1060,6 +1156,7 @@ var rpcHandlers = map[string]RPCHandler{ "setNetworkSettings": {Func: rpcSetNetworkSettings, Params: []string{"settings"}}, "renewDHCPLease": {Func: rpcRenewDHCPLease}, "keyboardReport": {Func: rpcKeyboardReport, Params: []string{"modifier", "keys"}}, + "keyboardReportMulti": {Func: rpcKeyboardReportMultiWrapper, Params: []string{"macro"}}, "getKeyboardLedState": {Func: rpcGetKeyboardLedState}, "keypressReport": {Func: rpcKeypressReport, Params: []string{"key", "press"}}, "getKeyDownState": {Func: rpcGetKeysDownState}, diff --git a/ui/src/components/popovers/PasteModal.tsx b/ui/src/components/popovers/PasteModal.tsx index 077759b7..ee17842c 100644 --- a/ui/src/components/popovers/PasteModal.tsx +++ b/ui/src/components/popovers/PasteModal.tsx @@ -8,35 +8,24 @@ import { GridCard } from "@components/Card"; import { TextAreaWithLabel } from "@components/TextArea"; import { SettingsPageHeader } from "@components/SettingsPageheader"; import { JsonRpcResponse, useJsonRpc } from "@/hooks/useJsonRpc"; -import { useHidStore, useRTCStore, useUiStore, useSettingsStore } from "@/hooks/stores"; -import { keys, modifiers } from "@/keyboardMappings"; -import { KeyStroke } from "@/keyboardLayouts"; +import { useHidStore, useSettingsStore, useUiStore } from "@/hooks/stores"; +import useKeyboard from "@/hooks/useKeyboard"; import useKeyboardLayout from "@/hooks/useKeyboardLayout"; import notifications from "@/notifications"; -const hidKeyboardPayload = (modifier: number, keys: number[]) => { - return { modifier, keys }; -}; - -const modifierCode = (shift?: boolean, altRight?: boolean) => { - return (shift ? modifiers.ShiftLeft : 0) - | (altRight ? modifiers.AltRight : 0) -} -const noModifier = 0 - export default function PasteModal() { const TextAreaRef = useRef(null); const { setPasteModeEnabled } = useHidStore(); const { setDisableVideoFocusTrap } = useUiStore(); const { send } = useJsonRpc(); - const { rpcDataChannel } = useRTCStore(); + const { executeMacro } = useKeyboard(); const [invalidChars, setInvalidChars] = useState([]); const close = useClose(); const { setKeyboardLayout } = useSettingsStore(); - const { selectedKeyboard } = useKeyboardLayout(); + const { selectedKeyboard } = useKeyboardLayout(); useEffect(() => { send("getKeyboardLayout", {}, (resp: JsonRpcResponse) => { @@ -52,15 +41,20 @@ export default function PasteModal() { }, [setDisableVideoFocusTrap, setPasteModeEnabled]); const onConfirmPaste = useCallback(async () => { - setPasteModeEnabled(false); - setDisableVideoFocusTrap(false); + // setPasteModeEnabled(false); + // setDisableVideoFocusTrap(false); - if (rpcDataChannel?.readyState !== "open" || !TextAreaRef.current) return; - if (!selectedKeyboard) return; + if (!TextAreaRef.current || !selectedKeyboard) return; const text = TextAreaRef.current.value; try { + const macroSteps: { + keys: string[] | null; + modifiers: string[] | null; + delay: number; + }[] = []; + for (const char of text) { const keyprops = selectedKeyboard.chars[char]; if (!keyprops) continue; @@ -70,39 +64,41 @@ export default function PasteModal() { // if this is an accented character, we need to send that accent FIRST if (accentKey) { - await sendKeystroke({modifier: modifierCode(accentKey.shift, accentKey.altRight), keys: [ keys[accentKey.key] ] }) + const accentModifiers: string[] = []; + if (accentKey.shift) accentModifiers.push("ShiftLeft"); + if (accentKey.altRight) accentModifiers.push("AltRight"); + + macroSteps.push({ + keys: [String(accentKey.key)], + modifiers: accentModifiers.length > 0 ? accentModifiers : null, + delay: 100, + }); } // now send the actual key - await sendKeystroke({ modifier: modifierCode(shift, altRight), keys: [ keys[key] ]}); + const modifiers: string[] = []; + if (shift) modifiers.push("ShiftLeft"); + if (altRight) modifiers.push("AltRight"); + + macroSteps.push({ + keys: [String(key)], + modifiers: modifiers.length > 0 ? modifiers : null, + delay: 100, + }); // if what was requested was a dead key, we need to send an unmodified space to emit // just the accent character - if (deadKey) { - await sendKeystroke({ modifier: noModifier, keys: [ keys["Space"] ] }); - } + if (deadKey) macroSteps.push({ keys: ["Space"], modifiers: null, delay: 100 }); + } - // now send a message with no keys down to "release" the keys - await sendKeystroke({ modifier: 0, keys: [] }); + if (macroSteps.length > 0) { + await executeMacro(macroSteps); } } catch (error) { console.error("Failed to paste text:", error); notifications.error("Failed to paste text"); } - - async function sendKeystroke(stroke: KeyStroke) { - await new Promise((resolve, reject) => { - send( - "keyboardReport", - hidKeyboardPayload(stroke.modifier, stroke.keys), - params => { - if ("error" in params) return reject(params.error); - resolve(); - } - ); - }); - } - }, [selectedKeyboard, rpcDataChannel?.readyState, send, setDisableVideoFocusTrap, setPasteModeEnabled]); + }, [selectedKeyboard, executeMacro]); useEffect(() => { if (TextAreaRef.current) { @@ -122,14 +118,18 @@ export default function PasteModal() { />
-
e.stopPropagation()} onKeyDown={e => e.stopPropagation()}> +
e.stopPropagation()} + onKeyDown={e => e.stopPropagation()} + >

- Sending text using keyboard layout: {selectedKeyboard.isoCode}-{selectedKeyboard.name} + Sending text using keyboard layout: {selectedKeyboard.isoCode}- + {selectedKeyboard.name}

@@ -181,7 +182,7 @@ export default function PasteModal() {
{ + } = useHidRpc(message => { switch (message.constructor) { case KeysDownStateMessage: setKeysDownState((message as KeysDownStateMessage).keysDownState); @@ -48,7 +54,9 @@ export default function useKeyboard() { async (state: KeysDownState) => { if (rpcDataChannel?.readyState !== "open" && !rpcHidReady) return; - console.debug(`Send keyboardReport keys: ${state.keys}, modifier: ${state.modifier}`); + console.debug( + `Send keyboardReport keys: ${state.keys}, modifier: ${state.modifier}`, + ); if (rpcHidReady) { console.debug("Sending keyboard report via HidRPC"); @@ -56,31 +64,29 @@ export default function useKeyboard() { return; } - send("keyboardReport", { keys: state.keys, modifier: state.modifier }, (resp: JsonRpcResponse) => { - if ("error" in resp) { - console.error(`Failed to send keyboard report ${state}`, resp.error); - } - }); + send( + "keyboardReport", + { keys: state.keys, modifier: state.modifier }, + (resp: JsonRpcResponse) => { + if ("error" in resp) { + console.error(`Failed to send keyboard report ${state}`, resp.error); + } + }, + ); }, - [ - rpcDataChannel?.readyState, - rpcHidReady, - send, - sendKeyboardEventHidRpc, - ], + [rpcDataChannel?.readyState, rpcHidReady, send, sendKeyboardEventHidRpc], ); // resetKeyboardState is used to reset the keyboard state to no keys pressed and no modifiers. // This is useful for macros and when the browser loses focus to ensure that the keyboard state // is clean. - const resetKeyboardState = useCallback( - async () => { - // Reset the keys buffer to zeros and the modifier state to zero - keysDownState.keys.length = hidKeyBufferSize; - keysDownState.keys.fill(0); - keysDownState.modifier = 0; - sendKeyboardEvent(keysDownState); - }, [keysDownState, sendKeyboardEvent]); + const resetKeyboardState = useCallback(async () => { + // Reset the keys buffer to zeros and the modifier state to zero + keysDownState.keys.length = hidKeyBufferSize; + keysDownState.keys.fill(0); + keysDownState.modifier = 0; + sendKeyboardEvent(keysDownState); + }, [keysDownState, sendKeyboardEvent]); // executeMacro is used to execute a macro consisting of multiple steps. // Each step can have multiple keys, multiple modifiers and a delay. @@ -88,27 +94,32 @@ export default function useKeyboard() { // After the delay, the keys and modifiers are released and the next step is executed. // If a step has no keys or modifiers, it is treated as a delay-only step. // A small pause is added between steps to ensure that the device can process the events. - const executeMacro = async (steps: { keys: string[] | null; modifiers: string[] | null; delay: number }[]) => { - for (const [index, step] of steps.entries()) { + const executeMacro = async ( + steps: { keys: string[] | null; modifiers: string[] | null; delay: number }[], + ) => { + const macro: KeysDownState[] = []; + + for (const [_, step] of steps.entries()) { const keyValues = (step.keys || []).map(key => keys[key]).filter(Boolean); - const modifierMask: number = (step.modifiers || []).map(mod => modifiers[mod]).reduce((acc, val) => acc + val, 0); + const modifierMask: number = (step.modifiers || []) + .map(mod => modifiers[mod]) + .reduce((acc, val) => acc + val, 0); // If the step has keys and/or modifiers, press them and hold for the delay if (keyValues.length > 0 || modifierMask > 0) { - sendKeyboardEvent({ keys: keyValues, modifier: modifierMask }); - await new Promise(resolve => setTimeout(resolve, step.delay || 50)); - - resetKeyboardState(); - } else { - // This is a delay-only step, just wait for the delay amount - await new Promise(resolve => setTimeout(resolve, step.delay || 50)); - } - - // Add a small pause between steps if not the last step - if (index < steps.length - 1) { - await new Promise(resolve => setTimeout(resolve, 10)); + macro.push({ keys: keyValues, modifier: modifierMask }); + keysDownState.keys.length = hidKeyBufferSize; + keysDownState.keys.fill(0); + keysDownState.modifier = 0; + macro.push(keysDownState); } } + // KeyboardReportMessage + send("keyboardReportMulti", { macro }, (resp: JsonRpcResponse) => { + if ("error" in resp) { + console.error(`Failed to send keyboard report ${macro}`, resp.error); + } + }); }; // handleKeyPress is used to handle a key press or release event. @@ -132,7 +143,11 @@ export default function useKeyboard() { sendKeypressEventHidRpc(key, press); } else { // if the keyPress api is not available, we need to handle the key locally - const downState = simulateDeviceSideKeyHandlingForLegacyDevices(keysDownState, key, press); + const downState = simulateDeviceSideKeyHandlingForLegacyDevices( + keysDownState, + key, + press, + ); sendKeyboardEvent(downState); // then we send the full state // if we just sent ErrorRollOver, reset to empty state @@ -152,7 +167,11 @@ export default function useKeyboard() { ); // IMPORTANT: See the keyPressReportApiAvailable comment above for the reason this exists - function simulateDeviceSideKeyHandlingForLegacyDevices(state: KeysDownState, key: number, press: boolean): KeysDownState { + function simulateDeviceSideKeyHandlingForLegacyDevices( + state: KeysDownState, + key: number, + press: boolean, + ): KeysDownState { // IMPORTANT: This code parallels the logic in the kernel's hid-gadget driver // for handling key presses and releases. It ensures that the USB gadget // behaves similarly to a real USB HID keyboard. This logic is paralleled @@ -164,7 +183,7 @@ export default function useKeyboard() { if (modifierMask !== 0) { // If the key is a modifier key, we update the keyboardModifier state // by setting or clearing the corresponding bit in the modifier byte. - // This allows us to track the state of dynamic modifier keys like + // This allows us to track the state of dynamic modifier keys like // Shift, Control, Alt, and Super. if (press) { modifiers |= modifierMask; @@ -181,7 +200,7 @@ export default function useKeyboard() { // and if we find a zero byte, we can place the key there (if press is true) if (keys[i] === key || keys[i] === 0) { if (press) { - keys[i] = key // overwrites the zero byte or the same key if already pressed + keys[i] = key; // overwrites the zero byte or the same key if already pressed } else { // we are releasing the key, remove it from the buffer if (keys[i] !== 0) { @@ -197,13 +216,15 @@ export default function useKeyboard() { // If we reach here it means we didn't find an empty slot or the key in the buffer if (overrun) { if (press) { - console.warn(`keyboard buffer overflow current keys ${keys}, key: ${key} not added`); + console.warn( + `keyboard buffer overflow current keys ${keys}, key: ${key} not added`, + ); // Fill all key slots with ErrorRollOver (0x01) to indicate overflow keys.length = hidKeyBufferSize; keys.fill(hidErrorRollOver); } else { // If we are releasing a key, and we didn't find it in a slot, who cares? - console.debug(`key ${key} not found in buffer, nothing to release`) + console.debug(`key ${key} not found in buffer, nothing to release`); } } } diff --git a/web.go b/web.go index 21e17e74..7987ebec 100644 --- a/web.go +++ b/web.go @@ -198,6 +198,10 @@ func handleWebRTCSession(c *gin.Context) { _ = peerConn.Close() }() } + + // Cancel any ongoing keyboard report multi when session changes + cancelKeyboardReportMulti() + currentSession = session c.JSON(http.StatusOK, gin.H{"sd": sd}) } diff --git a/webrtc.go b/webrtc.go index c3d0dc1b..43a72f83 100644 --- a/webrtc.go +++ b/webrtc.go @@ -266,6 +266,8 @@ func newSession(config SessionConfig) (*Session, error) { if connectionState == webrtc.ICEConnectionStateClosed { scopedLogger.Debug().Msg("ICE Connection State is closed, unmounting virtual media") if session == currentSession { + // Cancel any ongoing keyboard report multi when session closes + cancelKeyboardReportMulti() currentSession = nil } // Stop RPC processor From f58e5476bf63df53fc1508b0a5d4947abe5f5349 Mon Sep 17 00:00:00 2001 From: Siyuan Miao Date: Wed, 10 Sep 2025 00:13:10 +0200 Subject: [PATCH 02/10] feat: cancel paste mode --- jsonrpc.go | 192 ++++++++++++---------- ui/src/components/InfoBar.tsx | 8 +- ui/src/components/popovers/PasteModal.tsx | 9 +- ui/src/hooks/useKeyboard.ts | 10 +- ui/src/routes/devices.$id.tsx | 8 +- 5 files changed, 129 insertions(+), 98 deletions(-) diff --git a/jsonrpc.go b/jsonrpc.go index 711edeb0..47cc7960 100644 --- a/jsonrpc.go +++ b/jsonrpc.go @@ -1062,21 +1062,26 @@ func cancelKeyboardReportMulti() { } } -func rpcKeyboardReportMultiWrapper(macro []map[string]any) (usbgadget.KeysDownState, error) { +func setKeyboardReportMultiCancel(cancel context.CancelFunc) { keyboardReportMultiLock.Lock() defer keyboardReportMultiLock.Unlock() - if keyboardReportMultiCancel != nil { - keyboardReportMultiCancel() - logger.Info().Msg("canceled previous keyboard report multi") - } + keyboardReportMultiCancel = cancel +} + +func rpcKeyboardReportMultiWrapper(macro []map[string]any) (usbgadget.KeysDownState, error) { + cancelKeyboardReportMulti() ctx, cancel := context.WithCancel(context.Background()) - keyboardReportMultiCancel = cancel + setKeyboardReportMultiCancel(cancel) + + writeJSONRPCEvent("keyboardReportMultiState", true, currentSession) result, err := rpcKeyboardReportMulti(ctx, macro) - keyboardReportMultiCancel = nil + setKeyboardReportMultiCancel(nil) + + writeJSONRPCEvent("keyboardReportMultiState", false, currentSession) return result, err } @@ -1086,6 +1091,10 @@ var ( keyboardReportMultiLock sync.Mutex ) +func rpcCancelKeyboardReportMulti() { + cancelKeyboardReportMulti() +} + func rpcKeyboardReportMulti(ctx context.Context, macro []map[string]any) (usbgadget.KeysDownState, error) { var last usbgadget.KeysDownState var err error @@ -1146,88 +1155,89 @@ func rpcKeyboardReportMulti(ctx context.Context, macro []map[string]any) (usbgad } var rpcHandlers = map[string]RPCHandler{ - "ping": {Func: rpcPing}, - "reboot": {Func: rpcReboot, Params: []string{"force"}}, - "getDeviceID": {Func: rpcGetDeviceID}, - "deregisterDevice": {Func: rpcDeregisterDevice}, - "getCloudState": {Func: rpcGetCloudState}, - "getNetworkState": {Func: rpcGetNetworkState}, - "getNetworkSettings": {Func: rpcGetNetworkSettings}, - "setNetworkSettings": {Func: rpcSetNetworkSettings, Params: []string{"settings"}}, - "renewDHCPLease": {Func: rpcRenewDHCPLease}, - "keyboardReport": {Func: rpcKeyboardReport, Params: []string{"modifier", "keys"}}, - "keyboardReportMulti": {Func: rpcKeyboardReportMultiWrapper, Params: []string{"macro"}}, - "getKeyboardLedState": {Func: rpcGetKeyboardLedState}, - "keypressReport": {Func: rpcKeypressReport, Params: []string{"key", "press"}}, - "getKeyDownState": {Func: rpcGetKeysDownState}, - "absMouseReport": {Func: rpcAbsMouseReport, Params: []string{"x", "y", "buttons"}}, - "relMouseReport": {Func: rpcRelMouseReport, Params: []string{"dx", "dy", "buttons"}}, - "wheelReport": {Func: rpcWheelReport, Params: []string{"wheelY"}}, - "getVideoState": {Func: rpcGetVideoState}, - "getUSBState": {Func: rpcGetUSBState}, - "unmountImage": {Func: rpcUnmountImage}, - "rpcMountBuiltInImage": {Func: rpcMountBuiltInImage, Params: []string{"filename"}}, - "setJigglerState": {Func: rpcSetJigglerState, Params: []string{"enabled"}}, - "getJigglerState": {Func: rpcGetJigglerState}, - "setJigglerConfig": {Func: rpcSetJigglerConfig, Params: []string{"jigglerConfig"}}, - "getJigglerConfig": {Func: rpcGetJigglerConfig}, - "getTimezones": {Func: rpcGetTimezones}, - "sendWOLMagicPacket": {Func: rpcSendWOLMagicPacket, Params: []string{"macAddress"}}, - "getStreamQualityFactor": {Func: rpcGetStreamQualityFactor}, - "setStreamQualityFactor": {Func: rpcSetStreamQualityFactor, Params: []string{"factor"}}, - "getAutoUpdateState": {Func: rpcGetAutoUpdateState}, - "setAutoUpdateState": {Func: rpcSetAutoUpdateState, Params: []string{"enabled"}}, - "getEDID": {Func: rpcGetEDID}, - "setEDID": {Func: rpcSetEDID, Params: []string{"edid"}}, - "getDevChannelState": {Func: rpcGetDevChannelState}, - "setDevChannelState": {Func: rpcSetDevChannelState, Params: []string{"enabled"}}, - "getUpdateStatus": {Func: rpcGetUpdateStatus}, - "tryUpdate": {Func: rpcTryUpdate}, - "getDevModeState": {Func: rpcGetDevModeState}, - "setDevModeState": {Func: rpcSetDevModeState, Params: []string{"enabled"}}, - "getSSHKeyState": {Func: rpcGetSSHKeyState}, - "setSSHKeyState": {Func: rpcSetSSHKeyState, Params: []string{"sshKey"}}, - "getTLSState": {Func: rpcGetTLSState}, - "setTLSState": {Func: rpcSetTLSState, Params: []string{"state"}}, - "setMassStorageMode": {Func: rpcSetMassStorageMode, Params: []string{"mode"}}, - "getMassStorageMode": {Func: rpcGetMassStorageMode}, - "isUpdatePending": {Func: rpcIsUpdatePending}, - "getUsbEmulationState": {Func: rpcGetUsbEmulationState}, - "setUsbEmulationState": {Func: rpcSetUsbEmulationState, Params: []string{"enabled"}}, - "getUsbConfig": {Func: rpcGetUsbConfig}, - "setUsbConfig": {Func: rpcSetUsbConfig, Params: []string{"usbConfig"}}, - "checkMountUrl": {Func: rpcCheckMountUrl, Params: []string{"url"}}, - "getVirtualMediaState": {Func: rpcGetVirtualMediaState}, - "getStorageSpace": {Func: rpcGetStorageSpace}, - "mountWithHTTP": {Func: rpcMountWithHTTP, Params: []string{"url", "mode"}}, - "mountWithStorage": {Func: rpcMountWithStorage, Params: []string{"filename", "mode"}}, - "listStorageFiles": {Func: rpcListStorageFiles}, - "deleteStorageFile": {Func: rpcDeleteStorageFile, Params: []string{"filename"}}, - "startStorageFileUpload": {Func: rpcStartStorageFileUpload, Params: []string{"filename", "size"}}, - "getWakeOnLanDevices": {Func: rpcGetWakeOnLanDevices}, - "setWakeOnLanDevices": {Func: rpcSetWakeOnLanDevices, Params: []string{"params"}}, - "resetConfig": {Func: rpcResetConfig}, - "setDisplayRotation": {Func: rpcSetDisplayRotation, Params: []string{"params"}}, - "getDisplayRotation": {Func: rpcGetDisplayRotation}, - "setBacklightSettings": {Func: rpcSetBacklightSettings, Params: []string{"params"}}, - "getBacklightSettings": {Func: rpcGetBacklightSettings}, - "getDCPowerState": {Func: rpcGetDCPowerState}, - "setDCPowerState": {Func: rpcSetDCPowerState, Params: []string{"enabled"}}, - "setDCRestoreState": {Func: rpcSetDCRestoreState, Params: []string{"state"}}, - "getActiveExtension": {Func: rpcGetActiveExtension}, - "setActiveExtension": {Func: rpcSetActiveExtension, Params: []string{"extensionId"}}, - "getATXState": {Func: rpcGetATXState}, - "setATXPowerAction": {Func: rpcSetATXPowerAction, Params: []string{"action"}}, - "getSerialSettings": {Func: rpcGetSerialSettings}, - "setSerialSettings": {Func: rpcSetSerialSettings, Params: []string{"settings"}}, - "getUsbDevices": {Func: rpcGetUsbDevices}, - "setUsbDevices": {Func: rpcSetUsbDevices, Params: []string{"devices"}}, - "setUsbDeviceState": {Func: rpcSetUsbDeviceState, Params: []string{"device", "enabled"}}, - "setCloudUrl": {Func: rpcSetCloudUrl, Params: []string{"apiUrl", "appUrl"}}, - "getKeyboardLayout": {Func: rpcGetKeyboardLayout}, - "setKeyboardLayout": {Func: rpcSetKeyboardLayout, Params: []string{"layout"}}, - "getKeyboardMacros": {Func: getKeyboardMacros}, - "setKeyboardMacros": {Func: setKeyboardMacros, Params: []string{"params"}}, - "getLocalLoopbackOnly": {Func: rpcGetLocalLoopbackOnly}, - "setLocalLoopbackOnly": {Func: rpcSetLocalLoopbackOnly, Params: []string{"enabled"}}, + "ping": {Func: rpcPing}, + "reboot": {Func: rpcReboot, Params: []string{"force"}}, + "getDeviceID": {Func: rpcGetDeviceID}, + "deregisterDevice": {Func: rpcDeregisterDevice}, + "getCloudState": {Func: rpcGetCloudState}, + "getNetworkState": {Func: rpcGetNetworkState}, + "getNetworkSettings": {Func: rpcGetNetworkSettings}, + "setNetworkSettings": {Func: rpcSetNetworkSettings, Params: []string{"settings"}}, + "renewDHCPLease": {Func: rpcRenewDHCPLease}, + "keyboardReport": {Func: rpcKeyboardReport, Params: []string{"modifier", "keys"}}, + "keyboardReportMulti": {Func: rpcKeyboardReportMultiWrapper, Params: []string{"macro"}}, + "cancelKeyboardReportMulti": {Func: rpcCancelKeyboardReportMulti}, + "getKeyboardLedState": {Func: rpcGetKeyboardLedState}, + "keypressReport": {Func: rpcKeypressReport, Params: []string{"key", "press"}}, + "getKeyDownState": {Func: rpcGetKeysDownState}, + "absMouseReport": {Func: rpcAbsMouseReport, Params: []string{"x", "y", "buttons"}}, + "relMouseReport": {Func: rpcRelMouseReport, Params: []string{"dx", "dy", "buttons"}}, + "wheelReport": {Func: rpcWheelReport, Params: []string{"wheelY"}}, + "getVideoState": {Func: rpcGetVideoState}, + "getUSBState": {Func: rpcGetUSBState}, + "unmountImage": {Func: rpcUnmountImage}, + "rpcMountBuiltInImage": {Func: rpcMountBuiltInImage, Params: []string{"filename"}}, + "setJigglerState": {Func: rpcSetJigglerState, Params: []string{"enabled"}}, + "getJigglerState": {Func: rpcGetJigglerState}, + "setJigglerConfig": {Func: rpcSetJigglerConfig, Params: []string{"jigglerConfig"}}, + "getJigglerConfig": {Func: rpcGetJigglerConfig}, + "getTimezones": {Func: rpcGetTimezones}, + "sendWOLMagicPacket": {Func: rpcSendWOLMagicPacket, Params: []string{"macAddress"}}, + "getStreamQualityFactor": {Func: rpcGetStreamQualityFactor}, + "setStreamQualityFactor": {Func: rpcSetStreamQualityFactor, Params: []string{"factor"}}, + "getAutoUpdateState": {Func: rpcGetAutoUpdateState}, + "setAutoUpdateState": {Func: rpcSetAutoUpdateState, Params: []string{"enabled"}}, + "getEDID": {Func: rpcGetEDID}, + "setEDID": {Func: rpcSetEDID, Params: []string{"edid"}}, + "getDevChannelState": {Func: rpcGetDevChannelState}, + "setDevChannelState": {Func: rpcSetDevChannelState, Params: []string{"enabled"}}, + "getUpdateStatus": {Func: rpcGetUpdateStatus}, + "tryUpdate": {Func: rpcTryUpdate}, + "getDevModeState": {Func: rpcGetDevModeState}, + "setDevModeState": {Func: rpcSetDevModeState, Params: []string{"enabled"}}, + "getSSHKeyState": {Func: rpcGetSSHKeyState}, + "setSSHKeyState": {Func: rpcSetSSHKeyState, Params: []string{"sshKey"}}, + "getTLSState": {Func: rpcGetTLSState}, + "setTLSState": {Func: rpcSetTLSState, Params: []string{"state"}}, + "setMassStorageMode": {Func: rpcSetMassStorageMode, Params: []string{"mode"}}, + "getMassStorageMode": {Func: rpcGetMassStorageMode}, + "isUpdatePending": {Func: rpcIsUpdatePending}, + "getUsbEmulationState": {Func: rpcGetUsbEmulationState}, + "setUsbEmulationState": {Func: rpcSetUsbEmulationState, Params: []string{"enabled"}}, + "getUsbConfig": {Func: rpcGetUsbConfig}, + "setUsbConfig": {Func: rpcSetUsbConfig, Params: []string{"usbConfig"}}, + "checkMountUrl": {Func: rpcCheckMountUrl, Params: []string{"url"}}, + "getVirtualMediaState": {Func: rpcGetVirtualMediaState}, + "getStorageSpace": {Func: rpcGetStorageSpace}, + "mountWithHTTP": {Func: rpcMountWithHTTP, Params: []string{"url", "mode"}}, + "mountWithStorage": {Func: rpcMountWithStorage, Params: []string{"filename", "mode"}}, + "listStorageFiles": {Func: rpcListStorageFiles}, + "deleteStorageFile": {Func: rpcDeleteStorageFile, Params: []string{"filename"}}, + "startStorageFileUpload": {Func: rpcStartStorageFileUpload, Params: []string{"filename", "size"}}, + "getWakeOnLanDevices": {Func: rpcGetWakeOnLanDevices}, + "setWakeOnLanDevices": {Func: rpcSetWakeOnLanDevices, Params: []string{"params"}}, + "resetConfig": {Func: rpcResetConfig}, + "setDisplayRotation": {Func: rpcSetDisplayRotation, Params: []string{"params"}}, + "getDisplayRotation": {Func: rpcGetDisplayRotation}, + "setBacklightSettings": {Func: rpcSetBacklightSettings, Params: []string{"params"}}, + "getBacklightSettings": {Func: rpcGetBacklightSettings}, + "getDCPowerState": {Func: rpcGetDCPowerState}, + "setDCPowerState": {Func: rpcSetDCPowerState, Params: []string{"enabled"}}, + "setDCRestoreState": {Func: rpcSetDCRestoreState, Params: []string{"state"}}, + "getActiveExtension": {Func: rpcGetActiveExtension}, + "setActiveExtension": {Func: rpcSetActiveExtension, Params: []string{"extensionId"}}, + "getATXState": {Func: rpcGetATXState}, + "setATXPowerAction": {Func: rpcSetATXPowerAction, Params: []string{"action"}}, + "getSerialSettings": {Func: rpcGetSerialSettings}, + "setSerialSettings": {Func: rpcSetSerialSettings, Params: []string{"settings"}}, + "getUsbDevices": {Func: rpcGetUsbDevices}, + "setUsbDevices": {Func: rpcSetUsbDevices, Params: []string{"devices"}}, + "setUsbDeviceState": {Func: rpcSetUsbDeviceState, Params: []string{"device", "enabled"}}, + "setCloudUrl": {Func: rpcSetCloudUrl, Params: []string{"apiUrl", "appUrl"}}, + "getKeyboardLayout": {Func: rpcGetKeyboardLayout}, + "setKeyboardLayout": {Func: rpcSetKeyboardLayout, Params: []string{"layout"}}, + "getKeyboardMacros": {Func: getKeyboardMacros}, + "setKeyboardMacros": {Func: setKeyboardMacros, Params: []string{"params"}}, + "getLocalLoopbackOnly": {Func: rpcGetLocalLoopbackOnly}, + "setLocalLoopbackOnly": {Func: rpcSetLocalLoopbackOnly, Params: []string{"enabled"}}, } diff --git a/ui/src/components/InfoBar.tsx b/ui/src/components/InfoBar.tsx index 8d0b2822..62759d17 100644 --- a/ui/src/components/InfoBar.tsx +++ b/ui/src/components/InfoBar.tsx @@ -27,6 +27,7 @@ export default function InfoBar() { const { rpcDataChannel } = useRTCStore(); const { debugMode, mouseMode, showPressedKeys } = useSettingsStore(); + const { isPasteModeEnabled } = useHidStore(); useEffect(() => { if (!rpcDataChannel) return; @@ -108,7 +109,12 @@ export default function InfoBar() { {rpcHidStatus}
)} - + {isPasteModeEnabled && ( +
+ Paste Mode: + Enabled +
+ )} {showPressedKeys && (
Keys: diff --git a/ui/src/components/popovers/PasteModal.tsx b/ui/src/components/popovers/PasteModal.tsx index ee17842c..30744fd6 100644 --- a/ui/src/components/popovers/PasteModal.tsx +++ b/ui/src/components/popovers/PasteModal.tsx @@ -15,11 +15,11 @@ import notifications from "@/notifications"; export default function PasteModal() { const TextAreaRef = useRef(null); - const { setPasteModeEnabled } = useHidStore(); + const { isPasteModeEnabled } = useHidStore(); const { setDisableVideoFocusTrap } = useUiStore(); const { send } = useJsonRpc(); - const { executeMacro } = useKeyboard(); + const { executeMacro, cancelExecuteMacro } = useKeyboard(); const [invalidChars, setInvalidChars] = useState([]); const close = useClose(); @@ -35,10 +35,10 @@ export default function PasteModal() { }, [send, setKeyboardLayout]); const onCancelPasteMode = useCallback(() => { - setPasteModeEnabled(false); + cancelExecuteMacro(); setDisableVideoFocusTrap(false); setInvalidChars([]); - }, [setDisableVideoFocusTrap, setPasteModeEnabled]); + }, [setDisableVideoFocusTrap, cancelExecuteMacro]); const onConfirmPaste = useCallback(async () => { // setPasteModeEnabled(false); @@ -201,6 +201,7 @@ export default function PasteModal() { size="SM" theme="primary" text="Confirm Paste" + disabled={isPasteModeEnabled} onClick={onConfirmPaste} LeadingIcon={LuCornerDownLeft} /> diff --git a/ui/src/hooks/useKeyboard.ts b/ui/src/hooks/useKeyboard.ts index 46a1a4bb..bdc9fa69 100644 --- a/ui/src/hooks/useKeyboard.ts +++ b/ui/src/hooks/useKeyboard.ts @@ -122,6 +122,14 @@ export default function useKeyboard() { }); }; + const cancelExecuteMacro = useCallback(async () => { + send("cancelKeyboardReportMulti", {}, (resp: JsonRpcResponse) => { + if ("error" in resp) { + console.error(`Failed to cancel keyboard report multi`, resp.error); + } + }); + }, [send]); + // handleKeyPress is used to handle a key press or release event. // This function handle both key press and key release events. // It checks if the keyPressReport API is available and sends the key press event. @@ -231,5 +239,5 @@ export default function useKeyboard() { return { modifier: modifiers, keys }; } - return { handleKeyPress, resetKeyboardState, executeMacro }; + return { handleKeyPress, resetKeyboardState, executeMacro, cancelExecuteMacro }; } diff --git a/ui/src/routes/devices.$id.tsx b/ui/src/routes/devices.$id.tsx index 4318447e..9445e9fd 100644 --- a/ui/src/routes/devices.$id.tsx +++ b/ui/src/routes/devices.$id.tsx @@ -580,7 +580,7 @@ export default function KvmIdRoute() { const { setNetworkState} = useNetworkStateStore(); const { setHdmiState } = useVideoStore(); const { - keyboardLedState, setKeyboardLedState, + keyboardLedState, setKeyboardLedState, setPasteModeEnabled, keysDownState, setKeysDownState, setUsbState, } = useHidStore(); @@ -598,6 +598,12 @@ export default function KvmIdRoute() { setUsbState(usbState); } + if (resp.method === "keyboardReportMultiState") { + const reportMultiState = resp.params as unknown as boolean; + console.debug("Setting keyboard report multi state", reportMultiState); + setPasteModeEnabled(reportMultiState); + } + if (resp.method === "videoInputState") { const hdmiState = resp.params as Parameters[0]; console.debug("Setting HDMI state", hdmiState); From 024cbb8fb19a46d8539f05dc463372402fa3fae5 Mon Sep 17 00:00:00 2001 From: Siyuan Miao Date: Wed, 10 Sep 2025 02:42:36 +0200 Subject: [PATCH 03/10] wip: send macro using hidRPC channel --- hidrpc.go | 8 ++ internal/hidrpc/hidrpc.go | 21 +-- internal/hidrpc/message.go | 51 +++++++ jsonrpc.go | 274 +++++++++++++++++------------------- ui/src/hooks/hidRpc.ts | 83 +++++++++++ ui/src/hooks/useHidRpc.ts | 12 ++ ui/src/hooks/useKeyboard.ts | 35 +++-- 7 files changed, 310 insertions(+), 174 deletions(-) diff --git a/hidrpc.go b/hidrpc.go index 9537e5b9..604be89f 100644 --- a/hidrpc.go +++ b/hidrpc.go @@ -1,6 +1,7 @@ package kvm import ( + "context" "errors" "fmt" "io" @@ -31,6 +32,13 @@ func handleHidRPCMessage(message hidrpc.Message, session *Session) { session.reportHidRPCKeysDownState(*keysDownState) } rpcErr = err + case hidrpc.TypeKeyboardMacroReport: + keyboardMacroReport, err := message.KeyboardMacroReport() + if err != nil { + logger.Warn().Err(err).Msg("failed to get keyboard macro report") + return + } + _, rpcErr = rpcKeyboardReportMulti(context.Background(), keyboardMacroReport.Macro) case hidrpc.TypePointerReport: pointerReport, err := message.PointerReport() if err != nil { diff --git a/internal/hidrpc/hidrpc.go b/internal/hidrpc/hidrpc.go index e9c8c24d..c4d99615 100644 --- a/internal/hidrpc/hidrpc.go +++ b/internal/hidrpc/hidrpc.go @@ -10,14 +10,17 @@ import ( type MessageType byte const ( - TypeHandshake MessageType = 0x01 - TypeKeyboardReport MessageType = 0x02 - TypePointerReport MessageType = 0x03 - TypeWheelReport MessageType = 0x04 - TypeKeypressReport MessageType = 0x05 - TypeMouseReport MessageType = 0x06 - TypeKeyboardLedState MessageType = 0x32 - TypeKeydownState MessageType = 0x33 + TypeHandshake MessageType = 0x01 + TypeKeyboardReport MessageType = 0x02 + TypePointerReport MessageType = 0x03 + TypeWheelReport MessageType = 0x04 + TypeKeypressReport MessageType = 0x05 + TypeMouseReport MessageType = 0x06 + TypeKeyboardMacroReport MessageType = 0x07 + TypeCancelKeyboardMacroReport MessageType = 0x08 + TypeKeyboardLedState MessageType = 0x32 + TypeKeydownState MessageType = 0x33 + TypeKeyboardMacroStateReport MessageType = 0x34 ) const ( @@ -29,7 +32,7 @@ func GetQueueIndex(messageType MessageType) int { switch messageType { case TypeHandshake: return 0 - case TypeKeyboardReport, TypeKeypressReport, TypeKeyboardLedState, TypeKeydownState: + case TypeKeyboardReport, TypeKeypressReport, TypeKeyboardLedState, TypeKeydownState, TypeKeyboardMacroStateReport: return 1 case TypePointerReport, TypeMouseReport, TypeWheelReport: return 2 diff --git a/internal/hidrpc/message.go b/internal/hidrpc/message.go index 84bbda7c..fa403265 100644 --- a/internal/hidrpc/message.go +++ b/internal/hidrpc/message.go @@ -1,6 +1,7 @@ package hidrpc import ( + "encoding/binary" "fmt" ) @@ -43,6 +44,11 @@ func (m *Message) String() string { return fmt.Sprintf("MouseReport{Malformed: %v}", m.d) } return fmt.Sprintf("MouseReport{DX: %d, DY: %d, Button: %d}", m.d[0], m.d[1], m.d[2]) + case TypeKeyboardMacroReport: + if len(m.d) < 5 { + return fmt.Sprintf("KeyboardMacroReport{Malformed: %v}", m.d) + } + return fmt.Sprintf("KeyboardMacroReport{IsPaste: %v, Length: %d}", m.d[0] == uint8(1), binary.BigEndian.Uint32(m.d[1:5])) default: return fmt.Sprintf("Unknown{Type: %d, Data: %v}", m.t, m.d) } @@ -84,6 +90,51 @@ func (m *Message) KeyboardReport() (KeyboardReport, error) { }, nil } +// Macro .. +type KeyboardMacro struct { + Modifier byte // 1 byte + Keys []byte // 6 bytes, to make things easier, the keys length is fixed to 6 + Delay uint16 // 2 bytes +} +type KeyboardMacroReport struct { + IsPaste bool + Length uint32 + Macro []KeyboardMacro +} + +// KeyboardMacroReport returns the keyboard macro report from the message. +func (m *Message) KeyboardMacroReport() (KeyboardMacroReport, error) { + if m.t != TypeKeyboardMacroReport { + return KeyboardMacroReport{}, fmt.Errorf("invalid message type: %d", m.t) + } + + isPaste := m.d[0] == uint8(1) + length := binary.BigEndian.Uint32(m.d[1:5]) + + // check total length + expectedLength := int(length)*9 + 5 + if len(m.d) != expectedLength { + return KeyboardMacroReport{}, fmt.Errorf("invalid length: %d, expected: %d", len(m.d), expectedLength) + } + + macro := make([]KeyboardMacro, 0, int(length)) + for i := 0; i < int(length); i++ { + offset := 5 + i*9 + + macro = append(macro, KeyboardMacro{ + Modifier: m.d[offset], + Keys: m.d[offset+1 : offset+7], + Delay: binary.BigEndian.Uint16(m.d[offset+7 : offset+9]), + }) + } + + return KeyboardMacroReport{ + IsPaste: isPaste, + Macro: macro, + Length: length, + }, nil +} + // PointerReport .. type PointerReport struct { X int diff --git a/jsonrpc.go b/jsonrpc.go index 47cc7960..911793b3 100644 --- a/jsonrpc.go +++ b/jsonrpc.go @@ -10,13 +10,13 @@ import ( "path/filepath" "reflect" "strconv" - "sync" "time" "github.com/pion/webrtc/v4" "github.com/rs/zerolog" "go.bug.st/serial" + "github.com/jetkvm/kvm/internal/hidrpc" "github.com/jetkvm/kvm/internal/usbgadget" ) @@ -1050,52 +1050,56 @@ func rpcSetLocalLoopbackOnly(enabled bool) error { return nil } -// cancelKeyboardReportMulti cancels any ongoing keyboard report multi execution func cancelKeyboardReportMulti() { - keyboardReportMultiLock.Lock() - defer keyboardReportMultiLock.Unlock() - if keyboardReportMultiCancel != nil { - keyboardReportMultiCancel() - logger.Info().Msg("canceled keyboard report multi") - keyboardReportMultiCancel = nil - } } -func setKeyboardReportMultiCancel(cancel context.CancelFunc) { - keyboardReportMultiLock.Lock() - defer keyboardReportMultiLock.Unlock() +// // cancelKeyboardReportMulti cancels any ongoing keyboard report multi execution +// func cancelKeyboardReportMulti() { +// keyboardReportMultiLock.Lock() +// defer keyboardReportMultiLock.Unlock() - keyboardReportMultiCancel = cancel -} +// if keyboardReportMultiCancel != nil { +// keyboardReportMultiCancel() +// logger.Info().Msg("canceled keyboard report multi") +// keyboardReportMultiCancel = nil +// } +// } -func rpcKeyboardReportMultiWrapper(macro []map[string]any) (usbgadget.KeysDownState, error) { - cancelKeyboardReportMulti() +// func setKeyboardReportMultiCancel(cancel context.CancelFunc) { +// keyboardReportMultiLock.Lock() +// defer keyboardReportMultiLock.Unlock() - ctx, cancel := context.WithCancel(context.Background()) - setKeyboardReportMultiCancel(cancel) +// keyboardReportMultiCancel = cancel +// } - writeJSONRPCEvent("keyboardReportMultiState", true, currentSession) +// func rpcKeyboardReportMultiWrapper(macro []map[string]any) (usbgadget.KeysDownState, error) { +// // cancelKeyboardReportMulti() - result, err := rpcKeyboardReportMulti(ctx, macro) +// // ctx, cancel := context.WithCancel(context.Background()) +// // setKeyboardReportMultiCancel(cancel) - setKeyboardReportMultiCancel(nil) +// // writeJSONRPCEvent("keyboardReportMultiState", true, currentSession) - writeJSONRPCEvent("keyboardReportMultiState", false, currentSession) +// // result, err := rpcKeyboardReportMulti(ctx, macro) - return result, err -} +// // setKeyboardReportMultiCancel(nil) -var ( - keyboardReportMultiCancel context.CancelFunc - keyboardReportMultiLock sync.Mutex -) +// // writeJSONRPCEvent("keyboardReportMultiState", false, currentSession) -func rpcCancelKeyboardReportMulti() { - cancelKeyboardReportMulti() -} +// // return result, err +// } -func rpcKeyboardReportMulti(ctx context.Context, macro []map[string]any) (usbgadget.KeysDownState, error) { +// var ( +// keyboardReportMultiCancel context.CancelFunc +// keyboardReportMultiLock sync.Mutex +// ) + +// func rpcCancelKeyboardReportMulti() { +// cancelKeyboardReportMulti() +// } + +func rpcKeyboardReportMulti(ctx context.Context, macro []hidrpc.KeyboardMacro) (usbgadget.KeysDownState, error) { var last usbgadget.KeysDownState var err error @@ -1110,134 +1114,112 @@ func rpcKeyboardReportMulti(ctx context.Context, macro []map[string]any) (usbgad default: } - var modifier byte - if m, ok := step["modifier"].(float64); ok { - modifier = byte(int(m)) - } else if mi, ok := step["modifier"].(int); ok { - modifier = byte(mi) - } else if mb, ok := step["modifier"].(uint8); ok { - modifier = mb - } + delay := time.Duration(step.Delay) * time.Millisecond + logger.Info().Int("step", i).Uint16("delay", step.Delay).Msg("Keyboard report multi delay") - var keys []byte - if arr, ok := step["keys"].([]any); ok { - keys = make([]byte, 0, len(arr)) - for _, v := range arr { - if f, ok := v.(float64); ok { - keys = append(keys, byte(int(f))) - } else if i, ok := v.(int); ok { - keys = append(keys, byte(i)) - } else if b, ok := v.(uint8); ok { - keys = append(keys, b) - } - } - } else if bs, ok := step["keys"].([]byte); ok { - keys = bs + last, err = rpcKeyboardReport(step.Modifier, step.Keys) + if err != nil { + logger.Warn().Err(err).Msg("failed to execute keyboard report multi") + return last, err } // Use context-aware sleep that can be cancelled select { - case <-time.After(100 * time.Millisecond): + case <-time.After(delay): // Sleep completed normally case <-ctx.Done(): logger.Debug().Int("step", i).Msg("Keyboard report multi cancelled during sleep") return last, ctx.Err() } - - last, err = rpcKeyboardReport(modifier, keys) - if err != nil { - logger.Warn().Err(err).Msg("failed to execute keyboard report multi") - return last, err - } } return last, nil } var rpcHandlers = map[string]RPCHandler{ - "ping": {Func: rpcPing}, - "reboot": {Func: rpcReboot, Params: []string{"force"}}, - "getDeviceID": {Func: rpcGetDeviceID}, - "deregisterDevice": {Func: rpcDeregisterDevice}, - "getCloudState": {Func: rpcGetCloudState}, - "getNetworkState": {Func: rpcGetNetworkState}, - "getNetworkSettings": {Func: rpcGetNetworkSettings}, - "setNetworkSettings": {Func: rpcSetNetworkSettings, Params: []string{"settings"}}, - "renewDHCPLease": {Func: rpcRenewDHCPLease}, - "keyboardReport": {Func: rpcKeyboardReport, Params: []string{"modifier", "keys"}}, - "keyboardReportMulti": {Func: rpcKeyboardReportMultiWrapper, Params: []string{"macro"}}, - "cancelKeyboardReportMulti": {Func: rpcCancelKeyboardReportMulti}, - "getKeyboardLedState": {Func: rpcGetKeyboardLedState}, - "keypressReport": {Func: rpcKeypressReport, Params: []string{"key", "press"}}, - "getKeyDownState": {Func: rpcGetKeysDownState}, - "absMouseReport": {Func: rpcAbsMouseReport, Params: []string{"x", "y", "buttons"}}, - "relMouseReport": {Func: rpcRelMouseReport, Params: []string{"dx", "dy", "buttons"}}, - "wheelReport": {Func: rpcWheelReport, Params: []string{"wheelY"}}, - "getVideoState": {Func: rpcGetVideoState}, - "getUSBState": {Func: rpcGetUSBState}, - "unmountImage": {Func: rpcUnmountImage}, - "rpcMountBuiltInImage": {Func: rpcMountBuiltInImage, Params: []string{"filename"}}, - "setJigglerState": {Func: rpcSetJigglerState, Params: []string{"enabled"}}, - "getJigglerState": {Func: rpcGetJigglerState}, - "setJigglerConfig": {Func: rpcSetJigglerConfig, Params: []string{"jigglerConfig"}}, - "getJigglerConfig": {Func: rpcGetJigglerConfig}, - "getTimezones": {Func: rpcGetTimezones}, - "sendWOLMagicPacket": {Func: rpcSendWOLMagicPacket, Params: []string{"macAddress"}}, - "getStreamQualityFactor": {Func: rpcGetStreamQualityFactor}, - "setStreamQualityFactor": {Func: rpcSetStreamQualityFactor, Params: []string{"factor"}}, - "getAutoUpdateState": {Func: rpcGetAutoUpdateState}, - "setAutoUpdateState": {Func: rpcSetAutoUpdateState, Params: []string{"enabled"}}, - "getEDID": {Func: rpcGetEDID}, - "setEDID": {Func: rpcSetEDID, Params: []string{"edid"}}, - "getDevChannelState": {Func: rpcGetDevChannelState}, - "setDevChannelState": {Func: rpcSetDevChannelState, Params: []string{"enabled"}}, - "getUpdateStatus": {Func: rpcGetUpdateStatus}, - "tryUpdate": {Func: rpcTryUpdate}, - "getDevModeState": {Func: rpcGetDevModeState}, - "setDevModeState": {Func: rpcSetDevModeState, Params: []string{"enabled"}}, - "getSSHKeyState": {Func: rpcGetSSHKeyState}, - "setSSHKeyState": {Func: rpcSetSSHKeyState, Params: []string{"sshKey"}}, - "getTLSState": {Func: rpcGetTLSState}, - "setTLSState": {Func: rpcSetTLSState, Params: []string{"state"}}, - "setMassStorageMode": {Func: rpcSetMassStorageMode, Params: []string{"mode"}}, - "getMassStorageMode": {Func: rpcGetMassStorageMode}, - "isUpdatePending": {Func: rpcIsUpdatePending}, - "getUsbEmulationState": {Func: rpcGetUsbEmulationState}, - "setUsbEmulationState": {Func: rpcSetUsbEmulationState, Params: []string{"enabled"}}, - "getUsbConfig": {Func: rpcGetUsbConfig}, - "setUsbConfig": {Func: rpcSetUsbConfig, Params: []string{"usbConfig"}}, - "checkMountUrl": {Func: rpcCheckMountUrl, Params: []string{"url"}}, - "getVirtualMediaState": {Func: rpcGetVirtualMediaState}, - "getStorageSpace": {Func: rpcGetStorageSpace}, - "mountWithHTTP": {Func: rpcMountWithHTTP, Params: []string{"url", "mode"}}, - "mountWithStorage": {Func: rpcMountWithStorage, Params: []string{"filename", "mode"}}, - "listStorageFiles": {Func: rpcListStorageFiles}, - "deleteStorageFile": {Func: rpcDeleteStorageFile, Params: []string{"filename"}}, - "startStorageFileUpload": {Func: rpcStartStorageFileUpload, Params: []string{"filename", "size"}}, - "getWakeOnLanDevices": {Func: rpcGetWakeOnLanDevices}, - "setWakeOnLanDevices": {Func: rpcSetWakeOnLanDevices, Params: []string{"params"}}, - "resetConfig": {Func: rpcResetConfig}, - "setDisplayRotation": {Func: rpcSetDisplayRotation, Params: []string{"params"}}, - "getDisplayRotation": {Func: rpcGetDisplayRotation}, - "setBacklightSettings": {Func: rpcSetBacklightSettings, Params: []string{"params"}}, - "getBacklightSettings": {Func: rpcGetBacklightSettings}, - "getDCPowerState": {Func: rpcGetDCPowerState}, - "setDCPowerState": {Func: rpcSetDCPowerState, Params: []string{"enabled"}}, - "setDCRestoreState": {Func: rpcSetDCRestoreState, Params: []string{"state"}}, - "getActiveExtension": {Func: rpcGetActiveExtension}, - "setActiveExtension": {Func: rpcSetActiveExtension, Params: []string{"extensionId"}}, - "getATXState": {Func: rpcGetATXState}, - "setATXPowerAction": {Func: rpcSetATXPowerAction, Params: []string{"action"}}, - "getSerialSettings": {Func: rpcGetSerialSettings}, - "setSerialSettings": {Func: rpcSetSerialSettings, Params: []string{"settings"}}, - "getUsbDevices": {Func: rpcGetUsbDevices}, - "setUsbDevices": {Func: rpcSetUsbDevices, Params: []string{"devices"}}, - "setUsbDeviceState": {Func: rpcSetUsbDeviceState, Params: []string{"device", "enabled"}}, - "setCloudUrl": {Func: rpcSetCloudUrl, Params: []string{"apiUrl", "appUrl"}}, - "getKeyboardLayout": {Func: rpcGetKeyboardLayout}, - "setKeyboardLayout": {Func: rpcSetKeyboardLayout, Params: []string{"layout"}}, - "getKeyboardMacros": {Func: getKeyboardMacros}, - "setKeyboardMacros": {Func: setKeyboardMacros, Params: []string{"params"}}, - "getLocalLoopbackOnly": {Func: rpcGetLocalLoopbackOnly}, - "setLocalLoopbackOnly": {Func: rpcSetLocalLoopbackOnly, Params: []string{"enabled"}}, + "ping": {Func: rpcPing}, + "reboot": {Func: rpcReboot, Params: []string{"force"}}, + "getDeviceID": {Func: rpcGetDeviceID}, + "deregisterDevice": {Func: rpcDeregisterDevice}, + "getCloudState": {Func: rpcGetCloudState}, + "getNetworkState": {Func: rpcGetNetworkState}, + "getNetworkSettings": {Func: rpcGetNetworkSettings}, + "setNetworkSettings": {Func: rpcSetNetworkSettings, Params: []string{"settings"}}, + "renewDHCPLease": {Func: rpcRenewDHCPLease}, + "keyboardReport": {Func: rpcKeyboardReport, Params: []string{"modifier", "keys"}}, + // "keyboardReportMulti": {Func: rpcKeyboardReportMultiWrapper, Params: []string{"macro"}}, + // "cancelKeyboardReportMulti": {Func: rpcCancelKeyboardReportMulti}, + "getKeyboardLedState": {Func: rpcGetKeyboardLedState}, + "keypressReport": {Func: rpcKeypressReport, Params: []string{"key", "press"}}, + "getKeyDownState": {Func: rpcGetKeysDownState}, + "absMouseReport": {Func: rpcAbsMouseReport, Params: []string{"x", "y", "buttons"}}, + "relMouseReport": {Func: rpcRelMouseReport, Params: []string{"dx", "dy", "buttons"}}, + "wheelReport": {Func: rpcWheelReport, Params: []string{"wheelY"}}, + "getVideoState": {Func: rpcGetVideoState}, + "getUSBState": {Func: rpcGetUSBState}, + "unmountImage": {Func: rpcUnmountImage}, + "rpcMountBuiltInImage": {Func: rpcMountBuiltInImage, Params: []string{"filename"}}, + "setJigglerState": {Func: rpcSetJigglerState, Params: []string{"enabled"}}, + "getJigglerState": {Func: rpcGetJigglerState}, + "setJigglerConfig": {Func: rpcSetJigglerConfig, Params: []string{"jigglerConfig"}}, + "getJigglerConfig": {Func: rpcGetJigglerConfig}, + "getTimezones": {Func: rpcGetTimezones}, + "sendWOLMagicPacket": {Func: rpcSendWOLMagicPacket, Params: []string{"macAddress"}}, + "getStreamQualityFactor": {Func: rpcGetStreamQualityFactor}, + "setStreamQualityFactor": {Func: rpcSetStreamQualityFactor, Params: []string{"factor"}}, + "getAutoUpdateState": {Func: rpcGetAutoUpdateState}, + "setAutoUpdateState": {Func: rpcSetAutoUpdateState, Params: []string{"enabled"}}, + "getEDID": {Func: rpcGetEDID}, + "setEDID": {Func: rpcSetEDID, Params: []string{"edid"}}, + "getDevChannelState": {Func: rpcGetDevChannelState}, + "setDevChannelState": {Func: rpcSetDevChannelState, Params: []string{"enabled"}}, + "getUpdateStatus": {Func: rpcGetUpdateStatus}, + "tryUpdate": {Func: rpcTryUpdate}, + "getDevModeState": {Func: rpcGetDevModeState}, + "setDevModeState": {Func: rpcSetDevModeState, Params: []string{"enabled"}}, + "getSSHKeyState": {Func: rpcGetSSHKeyState}, + "setSSHKeyState": {Func: rpcSetSSHKeyState, Params: []string{"sshKey"}}, + "getTLSState": {Func: rpcGetTLSState}, + "setTLSState": {Func: rpcSetTLSState, Params: []string{"state"}}, + "setMassStorageMode": {Func: rpcSetMassStorageMode, Params: []string{"mode"}}, + "getMassStorageMode": {Func: rpcGetMassStorageMode}, + "isUpdatePending": {Func: rpcIsUpdatePending}, + "getUsbEmulationState": {Func: rpcGetUsbEmulationState}, + "setUsbEmulationState": {Func: rpcSetUsbEmulationState, Params: []string{"enabled"}}, + "getUsbConfig": {Func: rpcGetUsbConfig}, + "setUsbConfig": {Func: rpcSetUsbConfig, Params: []string{"usbConfig"}}, + "checkMountUrl": {Func: rpcCheckMountUrl, Params: []string{"url"}}, + "getVirtualMediaState": {Func: rpcGetVirtualMediaState}, + "getStorageSpace": {Func: rpcGetStorageSpace}, + "mountWithHTTP": {Func: rpcMountWithHTTP, Params: []string{"url", "mode"}}, + "mountWithStorage": {Func: rpcMountWithStorage, Params: []string{"filename", "mode"}}, + "listStorageFiles": {Func: rpcListStorageFiles}, + "deleteStorageFile": {Func: rpcDeleteStorageFile, Params: []string{"filename"}}, + "startStorageFileUpload": {Func: rpcStartStorageFileUpload, Params: []string{"filename", "size"}}, + "getWakeOnLanDevices": {Func: rpcGetWakeOnLanDevices}, + "setWakeOnLanDevices": {Func: rpcSetWakeOnLanDevices, Params: []string{"params"}}, + "resetConfig": {Func: rpcResetConfig}, + "setDisplayRotation": {Func: rpcSetDisplayRotation, Params: []string{"params"}}, + "getDisplayRotation": {Func: rpcGetDisplayRotation}, + "setBacklightSettings": {Func: rpcSetBacklightSettings, Params: []string{"params"}}, + "getBacklightSettings": {Func: rpcGetBacklightSettings}, + "getDCPowerState": {Func: rpcGetDCPowerState}, + "setDCPowerState": {Func: rpcSetDCPowerState, Params: []string{"enabled"}}, + "setDCRestoreState": {Func: rpcSetDCRestoreState, Params: []string{"state"}}, + "getActiveExtension": {Func: rpcGetActiveExtension}, + "setActiveExtension": {Func: rpcSetActiveExtension, Params: []string{"extensionId"}}, + "getATXState": {Func: rpcGetATXState}, + "setATXPowerAction": {Func: rpcSetATXPowerAction, Params: []string{"action"}}, + "getSerialSettings": {Func: rpcGetSerialSettings}, + "setSerialSettings": {Func: rpcSetSerialSettings, Params: []string{"settings"}}, + "getUsbDevices": {Func: rpcGetUsbDevices}, + "setUsbDevices": {Func: rpcSetUsbDevices, Params: []string{"devices"}}, + "setUsbDeviceState": {Func: rpcSetUsbDeviceState, Params: []string{"device", "enabled"}}, + "setCloudUrl": {Func: rpcSetCloudUrl, Params: []string{"apiUrl", "appUrl"}}, + "getKeyboardLayout": {Func: rpcGetKeyboardLayout}, + "setKeyboardLayout": {Func: rpcSetKeyboardLayout, Params: []string{"layout"}}, + "getKeyboardMacros": {Func: getKeyboardMacros}, + "setKeyboardMacros": {Func: setKeyboardMacros, Params: []string{"params"}}, + "getLocalLoopbackOnly": {Func: rpcGetLocalLoopbackOnly}, + "setLocalLoopbackOnly": {Func: rpcSetLocalLoopbackOnly, Params: []string{"enabled"}}, } diff --git a/ui/src/hooks/hidRpc.ts b/ui/src/hooks/hidRpc.ts index 20b8a108..05923c94 100644 --- a/ui/src/hooks/hidRpc.ts +++ b/ui/src/hooks/hidRpc.ts @@ -7,6 +7,7 @@ export const HID_RPC_MESSAGE_TYPES = { WheelReport: 0x04, KeypressReport: 0x05, MouseReport: 0x06, + KeyboardMacroReport: 0x07, KeyboardLedState: 0x32, KeysDownState: 0x33, } @@ -32,6 +33,30 @@ const fromInt32toUint8 = (n: number) => { ]); }; +const fromUint16toUint8 = (n: number) => { + if (n > 65535 || n < 0) { + throw new Error(`Number ${n} is not within the uint16 range`); + } + + return new Uint8Array([ + (n >> 8) & 0xFF, + (n >> 0) & 0xFF, + ]); +}; + +const fromUint32toUint8 = (n: number) => { + if (n > 4294967295 || n < 0) { + throw new Error(`Number ${n} is not within the uint32 range`); + } + + return new Uint8Array([ + (n >> 24) & 0xFF, + (n >> 16) & 0xFF, + (n >> 8) & 0xFF, + (n >> 0) & 0xFF, + ]); +}; + const fromInt8ToUint8 = (n: number) => { if (n < -128 || n > 127) { throw new Error(`Number ${n} is not within the int8 range`); @@ -186,6 +211,64 @@ export class KeyboardReportMessage extends RpcMessage { } } +export interface KeyboardMacro extends KeysDownState { + delay: number; +} + +export class KeyboardMacroReportMessage extends RpcMessage { + isPaste: boolean; + length: number; + macro: KeyboardMacro[]; + + KEYS_LENGTH = 6; + + constructor(isPaste: boolean, length: number, macro: KeyboardMacro[]) { + super(HID_RPC_MESSAGE_TYPES.KeyboardMacroReport); + this.isPaste = isPaste; + this.length = length; + this.macro = macro; + } + + marshal(): Uint8Array { + const dataHeader = new Uint8Array([ + this.messageType, + this.isPaste ? 1 : 0, + ...fromUint32toUint8(this.length), + ]); + + let dataBody = new Uint8Array(); + + for (const step of this.macro) { + if (!withinUint8Range(step.modifier)) { + throw new Error(`Modifier ${step.modifier} is not within the uint8 range`); + } + + // Ensure the keys are within the KEYS_LENGTH range + const keys = step.keys; + if (keys.length > this.KEYS_LENGTH) { + throw new Error(`Keys ${keys} is not within the hidKeyBufferSize range`); + } else if (keys.length < this.KEYS_LENGTH) { + keys.push(...Array(this.KEYS_LENGTH - keys.length).fill(0)); + } + + for (const key of keys) { + if (!withinUint8Range(key)) { + throw new Error(`Key ${key} is not within the uint8 range`); + } + } + + const macroBinary = new Uint8Array([ + step.modifier, + ...keys, + ...fromUint16toUint8(step.delay), + ]); + + dataBody = new Uint8Array([...dataBody, ...macroBinary]); + } + return new Uint8Array([...dataHeader, ...dataBody]); + } +} + export class KeyboardLedStateMessage extends RpcMessage { keyboardLedState: KeyboardLedState; diff --git a/ui/src/hooks/useHidRpc.ts b/ui/src/hooks/useHidRpc.ts index ea0c7112..3beb9c07 100644 --- a/ui/src/hooks/useHidRpc.ts +++ b/ui/src/hooks/useHidRpc.ts @@ -5,6 +5,8 @@ import { useRTCStore } from "@/hooks/stores"; import { HID_RPC_VERSION, HandshakeMessage, + KeyboardMacro, + KeyboardMacroReportMessage, KeyboardReportMessage, KeypressReportMessage, MouseReportMessage, @@ -68,6 +70,15 @@ export function useHidRpc(onHidRpcMessage?: (payload: RpcMessage) => void) { [sendMessage], ); + const reportKeyboardMacroEvent = useCallback( + (macro: KeyboardMacro[]) => { + const d = new KeyboardMacroReportMessage(false, macro.length, macro); + sendMessage(d); + console.log("Sent keyboard macro report", d, d.marshal()); + }, + [sendMessage], + ); + const sendHandshake = useCallback(() => { if (rpcHidProtocolVersion) return; if (!rpcHidChannel) return; @@ -143,6 +154,7 @@ export function useHidRpc(onHidRpcMessage?: (payload: RpcMessage) => void) { reportKeypressEvent, reportAbsMouseEvent, reportRelMouseEvent, + reportKeyboardMacroEvent, rpcHidProtocolVersion, rpcHidReady, rpcHidStatus, diff --git a/ui/src/hooks/useKeyboard.ts b/ui/src/hooks/useKeyboard.ts index bdc9fa69..315724fc 100644 --- a/ui/src/hooks/useKeyboard.ts +++ b/ui/src/hooks/useKeyboard.ts @@ -1,4 +1,4 @@ -import { useCallback } from "react"; +import { useCallback, useMemo } from "react"; import { hidErrorRollOver, @@ -9,7 +9,7 @@ import { } from "@/hooks/stores"; import { JsonRpcResponse, useJsonRpc } from "@/hooks/useJsonRpc"; import { useHidRpc } from "@/hooks/useHidRpc"; -import { KeyboardLedStateMessage, KeysDownStateMessage } from "@/hooks/hidRpc"; +import { KeyboardLedStateMessage, KeyboardMacro, KeysDownStateMessage } from "@/hooks/hidRpc"; import { hidKeyToModifierMask, keys, modifiers } from "@/keyboardMappings"; export default function useKeyboard() { @@ -32,6 +32,7 @@ export default function useKeyboard() { const { reportKeyboardEvent: sendKeyboardEventHidRpc, reportKeypressEvent: sendKeypressEventHidRpc, + reportKeyboardMacroEvent: sendKeyboardMacroEventHidRpc, rpcHidReady, } = useHidRpc(message => { switch (message.constructor) { @@ -77,16 +78,19 @@ export default function useKeyboard() { [rpcDataChannel?.readyState, rpcHidReady, send, sendKeyboardEventHidRpc], ); + const MACRO_RESET_KEYBOARD_STATE = useMemo(() => ({ + keys: new Array(hidKeyBufferSize).fill(0), + modifier: 0, + delay: 0, + }), []); + // resetKeyboardState is used to reset the keyboard state to no keys pressed and no modifiers. // This is useful for macros and when the browser loses focus to ensure that the keyboard state // is clean. const resetKeyboardState = useCallback(async () => { // Reset the keys buffer to zeros and the modifier state to zero - keysDownState.keys.length = hidKeyBufferSize; - keysDownState.keys.fill(0); - keysDownState.modifier = 0; - sendKeyboardEvent(keysDownState); - }, [keysDownState, sendKeyboardEvent]); + sendKeyboardEvent(MACRO_RESET_KEYBOARD_STATE); + }, [sendKeyboardEvent, MACRO_RESET_KEYBOARD_STATE]); // executeMacro is used to execute a macro consisting of multiple steps. // Each step can have multiple keys, multiple modifiers and a delay. @@ -97,7 +101,7 @@ export default function useKeyboard() { const executeMacro = async ( steps: { keys: string[] | null; modifiers: string[] | null; delay: number }[], ) => { - const macro: KeysDownState[] = []; + const macro: KeyboardMacro[] = []; for (const [_, step] of steps.entries()) { const keyValues = (step.keys || []).map(key => keys[key]).filter(Boolean); @@ -107,19 +111,12 @@ export default function useKeyboard() { // If the step has keys and/or modifiers, press them and hold for the delay if (keyValues.length > 0 || modifierMask > 0) { - macro.push({ keys: keyValues, modifier: modifierMask }); - keysDownState.keys.length = hidKeyBufferSize; - keysDownState.keys.fill(0); - keysDownState.modifier = 0; - macro.push(keysDownState); + macro.push({ keys: keyValues, modifier: modifierMask, delay: 50 }); + macro.push({ ...MACRO_RESET_KEYBOARD_STATE, delay: 200 }); } } - // KeyboardReportMessage - send("keyboardReportMulti", { macro }, (resp: JsonRpcResponse) => { - if ("error" in resp) { - console.error(`Failed to send keyboard report ${macro}`, resp.error); - } - }); + + sendKeyboardMacroEventHidRpc(macro); }; const cancelExecuteMacro = useCallback(async () => { From d7c8abbb11e3ada72abd3cf5d9a0269d90a737f5 Mon Sep 17 00:00:00 2001 From: Siyuan Miao Date: Wed, 10 Sep 2025 17:21:44 +0200 Subject: [PATCH 04/10] add delay --- jsonrpc.go | 1 - ui/src/hooks/useKeyboard.ts | 4 ++-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/jsonrpc.go b/jsonrpc.go index 911793b3..a0acad25 100644 --- a/jsonrpc.go +++ b/jsonrpc.go @@ -1115,7 +1115,6 @@ func rpcKeyboardReportMulti(ctx context.Context, macro []hidrpc.KeyboardMacro) ( } delay := time.Duration(step.Delay) * time.Millisecond - logger.Info().Int("step", i).Uint16("delay", step.Delay).Msg("Keyboard report multi delay") last, err = rpcKeyboardReport(step.Modifier, step.Keys) if err != nil { diff --git a/ui/src/hooks/useKeyboard.ts b/ui/src/hooks/useKeyboard.ts index 315724fc..4254b480 100644 --- a/ui/src/hooks/useKeyboard.ts +++ b/ui/src/hooks/useKeyboard.ts @@ -111,8 +111,8 @@ export default function useKeyboard() { // If the step has keys and/or modifiers, press them and hold for the delay if (keyValues.length > 0 || modifierMask > 0) { - macro.push({ keys: keyValues, modifier: modifierMask, delay: 50 }); - macro.push({ ...MACRO_RESET_KEYBOARD_STATE, delay: 200 }); + macro.push({ keys: keyValues, modifier: modifierMask, delay: 20 }); + macro.push({ ...MACRO_RESET_KEYBOARD_STATE, delay: 100 }); } } From 7014560b41f96eaad0e72c019749896effba1a6c Mon Sep 17 00:00:00 2001 From: Siyuan Miao Date: Wed, 10 Sep 2025 23:35:24 +0200 Subject: [PATCH 05/10] feat: allow paste progress to be cancelled --- cloud.go | 2 +- hidrpc.go | 15 ++- internal/hidrpc/hidrpc.go | 21 +++- internal/hidrpc/message.go | 17 ++++ jsonrpc.go | 113 +++++++++++----------- ui/src/components/popovers/PasteModal.tsx | 3 - ui/src/hooks/hidRpc.ts | 53 +++++++++- ui/src/hooks/useHidRpc.ts | 13 ++- ui/src/hooks/useKeyboard.ts | 20 ++-- ui/src/routes/devices.$id.tsx | 8 +- web.go | 2 +- webrtc.go | 2 +- 12 files changed, 178 insertions(+), 91 deletions(-) diff --git a/cloud.go b/cloud.go index f86a4815..39b7683c 100644 --- a/cloud.go +++ b/cloud.go @@ -477,7 +477,7 @@ func handleSessionRequest( cloudLogger.Trace().Interface("session", session).Msg("new session accepted") // Cancel any ongoing keyboard report multi when session changes - cancelKeyboardReportMulti() + cancelKeyboardMacro() currentSession = session _ = wsjson.Write(context.Background(), c, gin.H{"type": "answer", "data": sd}) diff --git a/hidrpc.go b/hidrpc.go index 604be89f..d9717e3a 100644 --- a/hidrpc.go +++ b/hidrpc.go @@ -1,7 +1,6 @@ package kvm import ( - "context" "errors" "fmt" "io" @@ -38,7 +37,10 @@ func handleHidRPCMessage(message hidrpc.Message, session *Session) { logger.Warn().Err(err).Msg("failed to get keyboard macro report") return } - _, rpcErr = rpcKeyboardReportMulti(context.Background(), keyboardMacroReport.Macro) + _, rpcErr = rpcExecuteKeyboardMacro(keyboardMacroReport.Macro) + case hidrpc.TypeCancelKeyboardMacroReport: + rpcCancelKeyboardMacro() + return case hidrpc.TypePointerReport: pointerReport, err := message.PointerReport() if err != nil { @@ -138,6 +140,8 @@ func reportHidRPC(params any, session *Session) { message, err = hidrpc.NewKeyboardLedMessage(params).Marshal() case usbgadget.KeysDownState: message, err = hidrpc.NewKeydownStateMessage(params).Marshal() + case hidrpc.KeyboardMacroStateReport: + message, err = hidrpc.NewKeyboardMacroStateMessage(params.State, params.IsPaste).Marshal() default: err = fmt.Errorf("unknown HID RPC message type: %T", params) } @@ -174,3 +178,10 @@ func (s *Session) reportHidRPCKeysDownState(state usbgadget.KeysDownState) { } reportHidRPC(state, s) } + +func (s *Session) reportHidRPCKeyboardMacroState(state hidrpc.KeyboardMacroStateReport) { + if !s.hidRPCAvailable { + writeJSONRPCEvent("keyboardMacroState", state, s) + } + reportHidRPC(state, s) +} diff --git a/internal/hidrpc/hidrpc.go b/internal/hidrpc/hidrpc.go index c4d99615..dc49965a 100644 --- a/internal/hidrpc/hidrpc.go +++ b/internal/hidrpc/hidrpc.go @@ -32,10 +32,13 @@ func GetQueueIndex(messageType MessageType) int { switch messageType { case TypeHandshake: return 0 - case TypeKeyboardReport, TypeKeypressReport, TypeKeyboardLedState, TypeKeydownState, TypeKeyboardMacroStateReport: + case TypeKeyboardReport, TypeKeypressReport, TypeKeyboardMacroReport, TypeKeyboardLedState, TypeKeydownState, TypeKeyboardMacroStateReport: return 1 case TypePointerReport, TypeMouseReport, TypeWheelReport: return 2 + // we don't want to block the queue for this message + case TypeCancelKeyboardMacroReport: + return 3 default: return 3 } @@ -101,3 +104,19 @@ func NewKeydownStateMessage(state usbgadget.KeysDownState) *Message { d: data, } } + +// NewKeyboardMacroStateMessage creates a new keyboard macro state message. +func NewKeyboardMacroStateMessage(state bool, isPaste bool) *Message { + data := make([]byte, 2) + if state { + data[0] = 1 + } + if isPaste { + data[1] = 1 + } + + return &Message{ + t: TypeKeyboardMacroStateReport, + d: data, + } +} diff --git a/internal/hidrpc/message.go b/internal/hidrpc/message.go index fa403265..6eae1779 100644 --- a/internal/hidrpc/message.go +++ b/internal/hidrpc/message.go @@ -182,3 +182,20 @@ func (m *Message) MouseReport() (MouseReport, error) { Button: uint8(m.d[2]), }, nil } + +type KeyboardMacroStateReport struct { + State bool + IsPaste bool +} + +// KeyboardMacroStateReport returns the keyboard macro state report from the message. +func (m *Message) KeyboardMacroStateReport() (KeyboardMacroStateReport, error) { + if m.t != TypeKeyboardMacroStateReport { + return KeyboardMacroStateReport{}, fmt.Errorf("invalid message type: %d", m.t) + } + + return KeyboardMacroStateReport{ + State: m.d[0] == uint8(1), + IsPaste: m.d[1] == uint8(1), + }, nil +} diff --git a/jsonrpc.go b/jsonrpc.go index a0acad25..d564b4a3 100644 --- a/jsonrpc.go +++ b/jsonrpc.go @@ -10,6 +10,7 @@ import ( "path/filepath" "reflect" "strconv" + "sync" "time" "github.com/pion/webrtc/v4" @@ -1050,75 +1051,69 @@ func rpcSetLocalLoopbackOnly(enabled bool) error { return nil } -func cancelKeyboardReportMulti() { +var ( + keyboardMacroCancel context.CancelFunc + keyboardMacroLock sync.Mutex +) +// cancelKeyboardMacro cancels any ongoing keyboard macro execution +func cancelKeyboardMacro() { + keyboardMacroLock.Lock() + defer keyboardMacroLock.Unlock() + + if keyboardMacroCancel != nil { + keyboardMacroCancel() + logger.Info().Msg("canceled keyboard macro") + keyboardMacroCancel = nil + } } -// // cancelKeyboardReportMulti cancels any ongoing keyboard report multi execution -// func cancelKeyboardReportMulti() { -// keyboardReportMultiLock.Lock() -// defer keyboardReportMultiLock.Unlock() +func setKeyboardMacroCancel(cancel context.CancelFunc) { + keyboardMacroLock.Lock() + defer keyboardMacroLock.Unlock() -// if keyboardReportMultiCancel != nil { -// keyboardReportMultiCancel() -// logger.Info().Msg("canceled keyboard report multi") -// keyboardReportMultiCancel = nil -// } -// } + keyboardMacroCancel = cancel +} -// func setKeyboardReportMultiCancel(cancel context.CancelFunc) { -// keyboardReportMultiLock.Lock() -// defer keyboardReportMultiLock.Unlock() +func rpcExecuteKeyboardMacro(macro []hidrpc.KeyboardMacro) (usbgadget.KeysDownState, error) { + cancelKeyboardMacro() -// keyboardReportMultiCancel = cancel -// } + ctx, cancel := context.WithCancel(context.Background()) + setKeyboardMacroCancel(cancel) -// func rpcKeyboardReportMultiWrapper(macro []map[string]any) (usbgadget.KeysDownState, error) { -// // cancelKeyboardReportMulti() + s := hidrpc.KeyboardMacroStateReport{ + State: true, + IsPaste: true, + } -// // ctx, cancel := context.WithCancel(context.Background()) -// // setKeyboardReportMultiCancel(cancel) + reportHidRPC(s, currentSession) -// // writeJSONRPCEvent("keyboardReportMultiState", true, currentSession) + result, err := rpcDoExecuteKeyboardMacro(ctx, macro) -// // result, err := rpcKeyboardReportMulti(ctx, macro) + setKeyboardMacroCancel(nil) -// // setKeyboardReportMultiCancel(nil) + s.State = false + reportHidRPC(s, currentSession) -// // writeJSONRPCEvent("keyboardReportMultiState", false, currentSession) + return result, err +} -// // return result, err -// } +func rpcCancelKeyboardMacro() { + cancelKeyboardMacro() +} -// var ( -// keyboardReportMultiCancel context.CancelFunc -// keyboardReportMultiLock sync.Mutex -// ) - -// func rpcCancelKeyboardReportMulti() { -// cancelKeyboardReportMulti() -// } - -func rpcKeyboardReportMulti(ctx context.Context, macro []hidrpc.KeyboardMacro) (usbgadget.KeysDownState, error) { +func rpcDoExecuteKeyboardMacro(ctx context.Context, macro []hidrpc.KeyboardMacro) (usbgadget.KeysDownState, error) { var last usbgadget.KeysDownState var err error - logger.Debug().Interface("macro", macro).Msg("Executing keyboard report multi") + logger.Debug().Interface("macro", macro).Msg("Executing keyboard macro") for i, step := range macro { - // Check for cancellation before each step - select { - case <-ctx.Done(): - logger.Debug().Msg("Keyboard report multi context cancelled") - return last, ctx.Err() - default: - } - delay := time.Duration(step.Delay) * time.Millisecond last, err = rpcKeyboardReport(step.Modifier, step.Keys) if err != nil { - logger.Warn().Err(err).Msg("failed to execute keyboard report multi") + logger.Warn().Err(err).Msg("failed to execute keyboard macro") return last, err } @@ -1127,7 +1122,9 @@ func rpcKeyboardReportMulti(ctx context.Context, macro []hidrpc.KeyboardMacro) ( case <-time.After(delay): // Sleep completed normally case <-ctx.Done(): - logger.Debug().Int("step", i).Msg("Keyboard report multi cancelled during sleep") + // make sure keyboard state is reset + rpcKeyboardReport(0, make([]byte, 6)) + logger.Debug().Int("step", i).Msg("Keyboard macro cancelled during sleep") return last, ctx.Err() } } @@ -1136,18 +1133,16 @@ func rpcKeyboardReportMulti(ctx context.Context, macro []hidrpc.KeyboardMacro) ( } var rpcHandlers = map[string]RPCHandler{ - "ping": {Func: rpcPing}, - "reboot": {Func: rpcReboot, Params: []string{"force"}}, - "getDeviceID": {Func: rpcGetDeviceID}, - "deregisterDevice": {Func: rpcDeregisterDevice}, - "getCloudState": {Func: rpcGetCloudState}, - "getNetworkState": {Func: rpcGetNetworkState}, - "getNetworkSettings": {Func: rpcGetNetworkSettings}, - "setNetworkSettings": {Func: rpcSetNetworkSettings, Params: []string{"settings"}}, - "renewDHCPLease": {Func: rpcRenewDHCPLease}, - "keyboardReport": {Func: rpcKeyboardReport, Params: []string{"modifier", "keys"}}, - // "keyboardReportMulti": {Func: rpcKeyboardReportMultiWrapper, Params: []string{"macro"}}, - // "cancelKeyboardReportMulti": {Func: rpcCancelKeyboardReportMulti}, + "ping": {Func: rpcPing}, + "reboot": {Func: rpcReboot, Params: []string{"force"}}, + "getDeviceID": {Func: rpcGetDeviceID}, + "deregisterDevice": {Func: rpcDeregisterDevice}, + "getCloudState": {Func: rpcGetCloudState}, + "getNetworkState": {Func: rpcGetNetworkState}, + "getNetworkSettings": {Func: rpcGetNetworkSettings}, + "setNetworkSettings": {Func: rpcSetNetworkSettings, Params: []string{"settings"}}, + "renewDHCPLease": {Func: rpcRenewDHCPLease}, + "keyboardReport": {Func: rpcKeyboardReport, Params: []string{"modifier", "keys"}}, "getKeyboardLedState": {Func: rpcGetKeyboardLedState}, "keypressReport": {Func: rpcKeypressReport, Params: []string{"key", "press"}}, "getKeyDownState": {Func: rpcGetKeysDownState}, diff --git a/ui/src/components/popovers/PasteModal.tsx b/ui/src/components/popovers/PasteModal.tsx index 30744fd6..1f5f6403 100644 --- a/ui/src/components/popovers/PasteModal.tsx +++ b/ui/src/components/popovers/PasteModal.tsx @@ -41,9 +41,6 @@ export default function PasteModal() { }, [setDisableVideoFocusTrap, cancelExecuteMacro]); const onConfirmPaste = useCallback(async () => { - // setPasteModeEnabled(false); - // setDisableVideoFocusTrap(false); - if (!TextAreaRef.current || !selectedKeyboard) return; const text = TextAreaRef.current.value; diff --git a/ui/src/hooks/hidRpc.ts b/ui/src/hooks/hidRpc.ts index 05923c94..edf75e75 100644 --- a/ui/src/hooks/hidRpc.ts +++ b/ui/src/hooks/hidRpc.ts @@ -8,8 +8,10 @@ export const HID_RPC_MESSAGE_TYPES = { KeypressReport: 0x05, MouseReport: 0x06, KeyboardMacroReport: 0x07, + CancelKeyboardMacroReport: 0x08, KeyboardLedState: 0x32, KeysDownState: 0x33, + KeyboardMacroStateReport: 0x34, } export type HidRpcMessageType = typeof HID_RPC_MESSAGE_TYPES[keyof typeof HID_RPC_MESSAGE_TYPES]; @@ -211,22 +213,22 @@ export class KeyboardReportMessage extends RpcMessage { } } -export interface KeyboardMacro extends KeysDownState { +export interface KeyboardMacroStep extends KeysDownState { delay: number; } export class KeyboardMacroReportMessage extends RpcMessage { isPaste: boolean; length: number; - macro: KeyboardMacro[]; + steps: KeyboardMacroStep[]; KEYS_LENGTH = 6; - constructor(isPaste: boolean, length: number, macro: KeyboardMacro[]) { + constructor(isPaste: boolean, length: number, steps: KeyboardMacroStep[]) { super(HID_RPC_MESSAGE_TYPES.KeyboardMacroReport); this.isPaste = isPaste; this.length = length; - this.macro = macro; + this.steps = steps; } marshal(): Uint8Array { @@ -238,7 +240,7 @@ export class KeyboardMacroReportMessage extends RpcMessage { let dataBody = new Uint8Array(); - for (const step of this.macro) { + for (const step of this.steps) { if (!withinUint8Range(step.modifier)) { throw new Error(`Modifier ${step.modifier} is not within the uint8 range`); } @@ -269,6 +271,33 @@ export class KeyboardMacroReportMessage extends RpcMessage { } } +export class KeyboardMacroStateReportMessage extends RpcMessage { + state: boolean; + isPaste: boolean; + + constructor(state: boolean, isPaste: boolean) { + super(HID_RPC_MESSAGE_TYPES.KeyboardMacroStateReport); + this.state = state; + this.isPaste = isPaste; + } + + marshal(): Uint8Array { + return new Uint8Array([ + this.messageType, + this.state ? 1 : 0, + this.isPaste ? 1 : 0, + ]); + } + + public static unmarshal(data: Uint8Array): KeyboardMacroStateReportMessage | undefined { + if (data.length < 1) { + throw new Error(`Invalid keyboard macro state report message length: ${data.length}`); + } + + return new KeyboardMacroStateReportMessage(data[0] === 1, data[1] === 1); + } +} + export class KeyboardLedStateMessage extends RpcMessage { keyboardLedState: KeyboardLedState; @@ -339,6 +368,17 @@ export class PointerReportMessage extends RpcMessage { } } +export class CancelKeyboardMacroReportMessage extends RpcMessage { + + constructor() { + super(HID_RPC_MESSAGE_TYPES.CancelKeyboardMacroReport); + } + + marshal(): Uint8Array { + return new Uint8Array([this.messageType]); + } +} + export class MouseReportMessage extends RpcMessage { dx: number; dy: number; @@ -367,6 +407,9 @@ export const messageRegistry = { [HID_RPC_MESSAGE_TYPES.KeyboardLedState]: KeyboardLedStateMessage, [HID_RPC_MESSAGE_TYPES.KeyboardReport]: KeyboardReportMessage, [HID_RPC_MESSAGE_TYPES.KeypressReport]: KeypressReportMessage, + [HID_RPC_MESSAGE_TYPES.KeyboardMacroReport]: KeyboardMacroReportMessage, + [HID_RPC_MESSAGE_TYPES.CancelKeyboardMacroReport]: CancelKeyboardMacroReportMessage, + [HID_RPC_MESSAGE_TYPES.KeyboardMacroStateReport]: KeyboardMacroStateReportMessage, } export const unmarshalHidRpcMessage = (data: Uint8Array): RpcMessage | undefined => { diff --git a/ui/src/hooks/useHidRpc.ts b/ui/src/hooks/useHidRpc.ts index 3beb9c07..fecb1661 100644 --- a/ui/src/hooks/useHidRpc.ts +++ b/ui/src/hooks/useHidRpc.ts @@ -3,9 +3,10 @@ import { useCallback, useEffect, useMemo } from "react"; import { useRTCStore } from "@/hooks/stores"; import { + CancelKeyboardMacroReportMessage, HID_RPC_VERSION, HandshakeMessage, - KeyboardMacro, + KeyboardMacroStep, KeyboardMacroReportMessage, KeyboardReportMessage, KeypressReportMessage, @@ -71,7 +72,7 @@ export function useHidRpc(onHidRpcMessage?: (payload: RpcMessage) => void) { ); const reportKeyboardMacroEvent = useCallback( - (macro: KeyboardMacro[]) => { + (macro: KeyboardMacroStep[]) => { const d = new KeyboardMacroReportMessage(false, macro.length, macro); sendMessage(d); console.log("Sent keyboard macro report", d, d.marshal()); @@ -79,6 +80,13 @@ export function useHidRpc(onHidRpcMessage?: (payload: RpcMessage) => void) { [sendMessage], ); + const cancelOngoingKeyboardMacro = useCallback( + () => { + sendMessage(new CancelKeyboardMacroReportMessage()); + }, + [sendMessage], + ); + const sendHandshake = useCallback(() => { if (rpcHidProtocolVersion) return; if (!rpcHidChannel) return; @@ -155,6 +163,7 @@ export function useHidRpc(onHidRpcMessage?: (payload: RpcMessage) => void) { reportAbsMouseEvent, reportRelMouseEvent, reportKeyboardMacroEvent, + cancelOngoingKeyboardMacro, rpcHidProtocolVersion, rpcHidReady, rpcHidStatus, diff --git a/ui/src/hooks/useKeyboard.ts b/ui/src/hooks/useKeyboard.ts index 4254b480..56c108e6 100644 --- a/ui/src/hooks/useKeyboard.ts +++ b/ui/src/hooks/useKeyboard.ts @@ -9,13 +9,13 @@ import { } from "@/hooks/stores"; import { JsonRpcResponse, useJsonRpc } from "@/hooks/useJsonRpc"; import { useHidRpc } from "@/hooks/useHidRpc"; -import { KeyboardLedStateMessage, KeyboardMacro, KeysDownStateMessage } from "@/hooks/hidRpc"; +import { KeyboardLedStateMessage, KeyboardMacroStateReportMessage, KeyboardMacroStep, KeysDownStateMessage } from "@/hooks/hidRpc"; import { hidKeyToModifierMask, keys, modifiers } from "@/keyboardMappings"; export default function useKeyboard() { const { send } = useJsonRpc(); const { rpcDataChannel } = useRTCStore(); - const { keysDownState, setKeysDownState, setKeyboardLedState } = useHidStore(); + const { keysDownState, setKeysDownState, setKeyboardLedState, setPasteModeEnabled } = useHidStore(); // INTRODUCTION: The earlier version of the JetKVM device shipped with all keyboard state // being tracked on the browser/client-side. When adding the keyPressReport API to the @@ -33,6 +33,7 @@ export default function useKeyboard() { reportKeyboardEvent: sendKeyboardEventHidRpc, reportKeypressEvent: sendKeypressEventHidRpc, reportKeyboardMacroEvent: sendKeyboardMacroEventHidRpc, + cancelOngoingKeyboardMacro: cancelOngoingKeyboardMacroHidRpc, rpcHidReady, } = useHidRpc(message => { switch (message.constructor) { @@ -42,6 +43,10 @@ export default function useKeyboard() { case KeyboardLedStateMessage: setKeyboardLedState((message as KeyboardLedStateMessage).keyboardLedState); break; + case KeyboardMacroStateReportMessage: + if (!(message as KeyboardMacroStateReportMessage).isPaste) break; + setPasteModeEnabled((message as KeyboardMacroStateReportMessage).state); + break; default: break; } @@ -101,7 +106,7 @@ export default function useKeyboard() { const executeMacro = async ( steps: { keys: string[] | null; modifiers: string[] | null; delay: number }[], ) => { - const macro: KeyboardMacro[] = []; + const macro: KeyboardMacroStep[] = []; for (const [_, step] of steps.entries()) { const keyValues = (step.keys || []).map(key => keys[key]).filter(Boolean); @@ -120,12 +125,9 @@ export default function useKeyboard() { }; const cancelExecuteMacro = useCallback(async () => { - send("cancelKeyboardReportMulti", {}, (resp: JsonRpcResponse) => { - if ("error" in resp) { - console.error(`Failed to cancel keyboard report multi`, resp.error); - } - }); - }, [send]); + if (!rpcHidReady) return; + cancelOngoingKeyboardMacroHidRpc(); + }, [rpcHidReady, cancelOngoingKeyboardMacroHidRpc]); // handleKeyPress is used to handle a key press or release event. // This function handle both key press and key release events. diff --git a/ui/src/routes/devices.$id.tsx b/ui/src/routes/devices.$id.tsx index 9445e9fd..4318447e 100644 --- a/ui/src/routes/devices.$id.tsx +++ b/ui/src/routes/devices.$id.tsx @@ -580,7 +580,7 @@ export default function KvmIdRoute() { const { setNetworkState} = useNetworkStateStore(); const { setHdmiState } = useVideoStore(); const { - keyboardLedState, setKeyboardLedState, setPasteModeEnabled, + keyboardLedState, setKeyboardLedState, keysDownState, setKeysDownState, setUsbState, } = useHidStore(); @@ -598,12 +598,6 @@ export default function KvmIdRoute() { setUsbState(usbState); } - if (resp.method === "keyboardReportMultiState") { - const reportMultiState = resp.params as unknown as boolean; - console.debug("Setting keyboard report multi state", reportMultiState); - setPasteModeEnabled(reportMultiState); - } - if (resp.method === "videoInputState") { const hdmiState = resp.params as Parameters[0]; console.debug("Setting HDMI state", hdmiState); diff --git a/web.go b/web.go index 7987ebec..b6ed667e 100644 --- a/web.go +++ b/web.go @@ -200,7 +200,7 @@ func handleWebRTCSession(c *gin.Context) { } // Cancel any ongoing keyboard report multi when session changes - cancelKeyboardReportMulti() + cancelKeyboardMacro() currentSession = session c.JSON(http.StatusOK, gin.H{"sd": sd}) diff --git a/webrtc.go b/webrtc.go index 43a72f83..db9a7c2c 100644 --- a/webrtc.go +++ b/webrtc.go @@ -267,7 +267,7 @@ func newSession(config SessionConfig) (*Session, error) { scopedLogger.Debug().Msg("ICE Connection State is closed, unmounting virtual media") if session == currentSession { // Cancel any ongoing keyboard report multi when session closes - cancelKeyboardReportMulti() + cancelKeyboardMacro() currentSession = nil } // Stop RPC processor From a667aefc961824be4a019d8cf7f49ec8c9334d34 Mon Sep 17 00:00:00 2001 From: Siyuan Miao Date: Wed, 10 Sep 2025 23:48:26 +0200 Subject: [PATCH 06/10] allow user to override delay --- ui/src/components/popovers/PasteModal.tsx | 39 ++++++++++++++++++++--- ui/src/hooks/useKeyboard.ts | 2 +- 2 files changed, 35 insertions(+), 6 deletions(-) diff --git a/ui/src/components/popovers/PasteModal.tsx b/ui/src/components/popovers/PasteModal.tsx index 1f5f6403..3466b199 100644 --- a/ui/src/components/popovers/PasteModal.tsx +++ b/ui/src/components/popovers/PasteModal.tsx @@ -1,4 +1,4 @@ -import { useCallback, useEffect, useRef, useState } from "react"; +import { useCallback, useEffect, useMemo, useRef, useState } from "react"; import { LuCornerDownLeft } from "react-icons/lu"; import { ExclamationCircleIcon } from "@heroicons/react/16/solid"; import { useClose } from "@headlessui/react"; @@ -12,6 +12,7 @@ import { useHidStore, useSettingsStore, useUiStore } from "@/hooks/stores"; import useKeyboard from "@/hooks/useKeyboard"; import useKeyboardLayout from "@/hooks/useKeyboardLayout"; import notifications from "@/notifications"; +import { InputFieldWithLabel } from "@components/InputField"; export default function PasteModal() { const TextAreaRef = useRef(null); @@ -22,6 +23,13 @@ export default function PasteModal() { const { executeMacro, cancelExecuteMacro } = useKeyboard(); const [invalidChars, setInvalidChars] = useState([]); + const [delayValue, setDelayValue] = useState(100); + const delay = useMemo(() => { + if (delayValue < 50 || delayValue > 65534) { + return 100; + } + return delayValue; + }, [delayValue]); const close = useClose(); const { setKeyboardLayout } = useSettingsStore(); @@ -68,7 +76,7 @@ export default function PasteModal() { macroSteps.push({ keys: [String(accentKey.key)], modifiers: accentModifiers.length > 0 ? accentModifiers : null, - delay: 100, + delay, }); } @@ -80,12 +88,12 @@ export default function PasteModal() { macroSteps.push({ keys: [String(key)], modifiers: modifiers.length > 0 ? modifiers : null, - delay: 100, + delay }); // if what was requested was a dead key, we need to send an unmodified space to emit // just the accent character - if (deadKey) macroSteps.push({ keys: ["Space"], modifiers: null, delay: 100 }); + if (deadKey) macroSteps.push({ keys: ["Space"], modifiers: null, delay }); } if (macroSteps.length > 0) { @@ -95,7 +103,7 @@ export default function PasteModal() { console.error("Failed to paste text:", error); notifications.error("Failed to paste text"); } - }, [selectedKeyboard, executeMacro]); + }, [selectedKeyboard, executeMacro, delay]); useEffect(() => { if (TextAreaRef.current) { @@ -168,6 +176,27 @@ export default function PasteModal() { )}
+
+ { + setDelayValue(parseInt(e.target.value, 10)); + }} + /> + {delayValue < 50 || delayValue > 65534 && ( +
+ + + Delay must be between 50 and 65534 + +
+ )} +

Sending text using keyboard layout: {selectedKeyboard.isoCode}- diff --git a/ui/src/hooks/useKeyboard.ts b/ui/src/hooks/useKeyboard.ts index 56c108e6..5c660805 100644 --- a/ui/src/hooks/useKeyboard.ts +++ b/ui/src/hooks/useKeyboard.ts @@ -117,7 +117,7 @@ export default function useKeyboard() { // If the step has keys and/or modifiers, press them and hold for the delay if (keyValues.length > 0 || modifierMask > 0) { macro.push({ keys: keyValues, modifier: modifierMask, delay: 20 }); - macro.push({ ...MACRO_RESET_KEYBOARD_STATE, delay: 100 }); + macro.push({ ...MACRO_RESET_KEYBOARD_STATE, delay: step.delay || 100 }); } } From d6de9668bd1e2da6e2910da09ddb3eadf64948eb Mon Sep 17 00:00:00 2001 From: Siyuan Miao Date: Thu, 11 Sep 2025 02:12:30 +0200 Subject: [PATCH 07/10] chore: clear keysDownState --- jsonrpc.go | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/jsonrpc.go b/jsonrpc.go index d564b4a3..32524bc5 100644 --- a/jsonrpc.go +++ b/jsonrpc.go @@ -1,6 +1,7 @@ package kvm import ( + "bytes" "context" "encoding/json" "errors" @@ -1102,6 +1103,12 @@ func rpcCancelKeyboardMacro() { cancelKeyboardMacro() } +var keyboardClearStateKeys = make([]byte, 6) + +func isClearKeyStep(step hidrpc.KeyboardMacro) bool { + return step.Modifier == 0 && bytes.Equal(step.Keys, keyboardClearStateKeys) +} + func rpcDoExecuteKeyboardMacro(ctx context.Context, macro []hidrpc.KeyboardMacro) (usbgadget.KeysDownState, error) { var last usbgadget.KeysDownState var err error @@ -1117,13 +1124,20 @@ func rpcDoExecuteKeyboardMacro(ctx context.Context, macro []hidrpc.KeyboardMacro return last, err } + // notify the device that the keyboard state is being cleared + if isClearKeyStep(step) { + gadget.UpdateKeysDown(0, keyboardClearStateKeys) + } + // Use context-aware sleep that can be cancelled select { case <-time.After(delay): // Sleep completed normally case <-ctx.Done(): // make sure keyboard state is reset - rpcKeyboardReport(0, make([]byte, 6)) + rpcKeyboardReport(0, keyboardClearStateKeys) + gadget.UpdateKeysDown(0, keyboardClearStateKeys) + logger.Debug().Int("step", i).Msg("Keyboard macro cancelled during sleep") return last, ctx.Err() } From 137d22b0b3dcb8204e5b55cbe9c49549064688fa Mon Sep 17 00:00:00 2001 From: Siyuan Miao Date: Thu, 11 Sep 2025 02:15:01 +0200 Subject: [PATCH 08/10] fix: use currentSession.reportHidRPCKeyboardMacroState --- jsonrpc.go | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/jsonrpc.go b/jsonrpc.go index 32524bc5..7b2d0a25 100644 --- a/jsonrpc.go +++ b/jsonrpc.go @@ -1087,14 +1087,18 @@ func rpcExecuteKeyboardMacro(macro []hidrpc.KeyboardMacro) (usbgadget.KeysDownSt IsPaste: true, } - reportHidRPC(s, currentSession) + if currentSession != nil { + currentSession.reportHidRPCKeyboardMacroState(s) + } result, err := rpcDoExecuteKeyboardMacro(ctx, macro) setKeyboardMacroCancel(nil) s.State = false - reportHidRPC(s, currentSession) + if currentSession != nil { + currentSession.reportHidRPCKeyboardMacroState(s) + } return result, err } From a86b516f9a941de911f5ffad97a1952a93dd1a71 Mon Sep 17 00:00:00 2001 From: Siyuan Miao Date: Thu, 11 Sep 2025 02:16:45 +0200 Subject: [PATCH 09/10] fix: jsonrpc.go:1142:21: Error return value is not checked (errcheck) --- jsonrpc.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/jsonrpc.go b/jsonrpc.go index 7b2d0a25..4dd6a4a8 100644 --- a/jsonrpc.go +++ b/jsonrpc.go @@ -1139,7 +1139,10 @@ func rpcDoExecuteKeyboardMacro(ctx context.Context, macro []hidrpc.KeyboardMacro // Sleep completed normally case <-ctx.Done(): // make sure keyboard state is reset - rpcKeyboardReport(0, keyboardClearStateKeys) + _, err := rpcKeyboardReport(0, keyboardClearStateKeys) + if err != nil { + logger.Warn().Err(err).Msg("failed to reset keyboard state") + } gadget.UpdateKeysDown(0, keyboardClearStateKeys) logger.Debug().Int("step", i).Msg("Keyboard macro cancelled during sleep") From 1e2cee7060dcca6a72335490233bc17185304f63 Mon Sep 17 00:00:00 2001 From: Siyuan Miao Date: Thu, 11 Sep 2025 02:56:47 +0200 Subject: [PATCH 10/10] fix: performance issue of Uint8Array concat --- ui/src/hooks/hidRpc.ts | 22 +++++++++++++++------- ui/src/hooks/useHidRpc.ts | 1 - 2 files changed, 15 insertions(+), 8 deletions(-) diff --git a/ui/src/hooks/hidRpc.ts b/ui/src/hooks/hidRpc.ts index edf75e75..cf2ef9cc 100644 --- a/ui/src/hooks/hidRpc.ts +++ b/ui/src/hooks/hidRpc.ts @@ -232,15 +232,20 @@ export class KeyboardMacroReportMessage extends RpcMessage { } marshal(): Uint8Array { - const dataHeader = new Uint8Array([ + // validate if length is correct + if (this.length !== this.steps.length) { + throw new Error(`Length ${this.length} is not equal to the number of steps ${this.steps.length}`); + } + + const data = new Uint8Array(this.length * 9 + 6); + data.set(new Uint8Array([ this.messageType, this.isPaste ? 1 : 0, ...fromUint32toUint8(this.length), - ]); + ]), 0); - let dataBody = new Uint8Array(); - - for (const step of this.steps) { + for (let i = 0; i < this.length; i++) { + const step = this.steps[i]; if (!withinUint8Range(step.modifier)) { throw new Error(`Modifier ${step.modifier} is not within the uint8 range`); } @@ -264,10 +269,13 @@ export class KeyboardMacroReportMessage extends RpcMessage { ...keys, ...fromUint16toUint8(step.delay), ]); + const offset = 6 + i * 9; - dataBody = new Uint8Array([...dataBody, ...macroBinary]); + + data.set(macroBinary, offset); } - return new Uint8Array([...dataHeader, ...dataBody]); + + return data; } } diff --git a/ui/src/hooks/useHidRpc.ts b/ui/src/hooks/useHidRpc.ts index fecb1661..ddfea95b 100644 --- a/ui/src/hooks/useHidRpc.ts +++ b/ui/src/hooks/useHidRpc.ts @@ -75,7 +75,6 @@ export function useHidRpc(onHidRpcMessage?: (payload: RpcMessage) => void) { (macro: KeyboardMacroStep[]) => { const d = new KeyboardMacroReportMessage(false, macro.length, macro); sendMessage(d); - console.log("Sent keyboard macro report", d, d.marshal()); }, [sendMessage], );