From 4b0818502c71db213a467df101ecb421db3b098b Mon Sep 17 00:00:00 2001 From: Siyuan Miao Date: Tue, 9 Sep 2025 23:56:23 +0200 Subject: [PATCH] feat: send all paste keystrokes to backend --- cloud.go | 4 + hidrpc.go | 6 ++ jsonrpc.go | 97 +++++++++++++++++++ ui/src/components/popovers/PasteModal.tsx | 89 ++++++++--------- ui/src/hooks/useKeyboard.ts | 111 +++++++++++++--------- web.go | 4 + webrtc.go | 2 + 7 files changed, 224 insertions(+), 89 deletions(-) diff --git a/cloud.go b/cloud.go index cec749e4..f86a4815 100644 --- a/cloud.go +++ b/cloud.go @@ -475,6 +475,10 @@ func handleSessionRequest( cloudLogger.Info().Interface("session", session).Msg("new session accepted") cloudLogger.Trace().Interface("session", session).Msg("new session accepted") + + // Cancel any ongoing keyboard report multi when session changes + cancelKeyboardReportMulti() + currentSession = session _ = wsjson.Write(context.Background(), c, gin.H{"type": "answer", "data": sd}) return nil diff --git a/hidrpc.go b/hidrpc.go index 74fe687f..9537e5b9 100644 --- a/hidrpc.go +++ b/hidrpc.go @@ -1,7 +1,9 @@ package kvm import ( + "errors" "fmt" + "io" "time" "github.com/jetkvm/kvm/internal/hidrpc" @@ -143,6 +145,10 @@ func reportHidRPC(params any, session *Session) { } if err := session.HidChannel.Send(message); err != nil { + if errors.Is(err, io.ErrClosedPipe) { + logger.Debug().Err(err).Msg("HID RPC channel closed, skipping reportHidRPC") + return + } logger.Warn().Err(err).Msg("failed to send HID RPC message") } } diff --git a/jsonrpc.go b/jsonrpc.go index ff3a4b12..711edeb0 100644 --- a/jsonrpc.go +++ b/jsonrpc.go @@ -10,6 +10,7 @@ import ( "path/filepath" "reflect" "strconv" + "sync" "time" "github.com/pion/webrtc/v4" @@ -1049,6 +1050,101 @@ func rpcSetLocalLoopbackOnly(enabled bool) error { return nil } +// cancelKeyboardReportMulti cancels any ongoing keyboard report multi execution +func cancelKeyboardReportMulti() { + keyboardReportMultiLock.Lock() + defer keyboardReportMultiLock.Unlock() + + if keyboardReportMultiCancel != nil { + keyboardReportMultiCancel() + logger.Info().Msg("canceled keyboard report multi") + keyboardReportMultiCancel = nil + } +} + +func rpcKeyboardReportMultiWrapper(macro []map[string]any) (usbgadget.KeysDownState, error) { + keyboardReportMultiLock.Lock() + defer keyboardReportMultiLock.Unlock() + + if keyboardReportMultiCancel != nil { + keyboardReportMultiCancel() + logger.Info().Msg("canceled previous keyboard report multi") + } + + ctx, cancel := context.WithCancel(context.Background()) + keyboardReportMultiCancel = cancel + + result, err := rpcKeyboardReportMulti(ctx, macro) + + keyboardReportMultiCancel = nil + + return result, err +} + +var ( + keyboardReportMultiCancel context.CancelFunc + keyboardReportMultiLock sync.Mutex +) + +func rpcKeyboardReportMulti(ctx context.Context, macro []map[string]any) (usbgadget.KeysDownState, error) { + var last usbgadget.KeysDownState + var err error + + logger.Debug().Interface("macro", macro).Msg("Executing keyboard report multi") + + for i, step := range macro { + // Check for cancellation before each step + select { + case <-ctx.Done(): + logger.Debug().Msg("Keyboard report multi context cancelled") + return last, ctx.Err() + default: + } + + var modifier byte + if m, ok := step["modifier"].(float64); ok { + modifier = byte(int(m)) + } else if mi, ok := step["modifier"].(int); ok { + modifier = byte(mi) + } else if mb, ok := step["modifier"].(uint8); ok { + modifier = mb + } + + var keys []byte + if arr, ok := step["keys"].([]any); ok { + keys = make([]byte, 0, len(arr)) + for _, v := range arr { + if f, ok := v.(float64); ok { + keys = append(keys, byte(int(f))) + } else if i, ok := v.(int); ok { + keys = append(keys, byte(i)) + } else if b, ok := v.(uint8); ok { + keys = append(keys, b) + } + } + } else if bs, ok := step["keys"].([]byte); ok { + keys = bs + } + + // Use context-aware sleep that can be cancelled + select { + case <-time.After(100 * time.Millisecond): + // Sleep completed normally + case <-ctx.Done(): + logger.Debug().Int("step", i).Msg("Keyboard report multi cancelled during sleep") + return last, ctx.Err() + } + + last, err = rpcKeyboardReport(modifier, keys) + if err != nil { + logger.Warn().Err(err).Msg("failed to execute keyboard report multi") + return last, err + } + } + + return last, nil +} + var rpcHandlers = map[string]RPCHandler{ "ping": {Func: rpcPing}, "reboot": {Func: rpcReboot, Params: []string{"force"}}, @@ -1060,6 +1156,7 @@ var rpcHandlers = map[string]RPCHandler{ "setNetworkSettings": {Func: rpcSetNetworkSettings, Params: []string{"settings"}}, "renewDHCPLease": {Func: rpcRenewDHCPLease}, "keyboardReport": {Func: rpcKeyboardReport, Params: []string{"modifier", "keys"}}, + "keyboardReportMulti": {Func: rpcKeyboardReportMultiWrapper, Params: []string{"macro"}}, "getKeyboardLedState": {Func: rpcGetKeyboardLedState}, "keypressReport": {Func: rpcKeypressReport, Params: []string{"key", "press"}}, "getKeyDownState": {Func: rpcGetKeysDownState}, diff --git a/ui/src/components/popovers/PasteModal.tsx b/ui/src/components/popovers/PasteModal.tsx index 077759b7..ee17842c 100644 --- a/ui/src/components/popovers/PasteModal.tsx +++ b/ui/src/components/popovers/PasteModal.tsx @@ -8,35 +8,24 @@ import { GridCard } from "@components/Card"; import { TextAreaWithLabel } from "@components/TextArea"; import { SettingsPageHeader } from "@components/SettingsPageheader"; import { JsonRpcResponse, useJsonRpc } from "@/hooks/useJsonRpc"; -import { useHidStore, useRTCStore, useUiStore, useSettingsStore } from "@/hooks/stores"; -import { keys, modifiers } from "@/keyboardMappings"; -import { KeyStroke } from "@/keyboardLayouts"; +import { useHidStore, useSettingsStore, useUiStore } from "@/hooks/stores"; +import useKeyboard from "@/hooks/useKeyboard"; import useKeyboardLayout from "@/hooks/useKeyboardLayout"; import notifications from "@/notifications"; -const hidKeyboardPayload = (modifier: number, keys: number[]) => { - return { modifier, keys }; -}; - -const modifierCode = (shift?: boolean, altRight?: boolean) => { - return (shift ? modifiers.ShiftLeft : 0) - | (altRight ? modifiers.AltRight : 0) -} -const noModifier = 0 - export default function PasteModal() { const TextAreaRef = useRef(null); const { setPasteModeEnabled } = useHidStore(); const { setDisableVideoFocusTrap } = useUiStore(); const { send } = useJsonRpc(); - const { rpcDataChannel } = useRTCStore(); + const { executeMacro } = useKeyboard(); const [invalidChars, setInvalidChars] = useState([]); const close = useClose(); const { setKeyboardLayout } = useSettingsStore(); - const { selectedKeyboard } = useKeyboardLayout(); + const { selectedKeyboard } = useKeyboardLayout(); useEffect(() => { send("getKeyboardLayout", {}, (resp: JsonRpcResponse) => { @@ -52,15 +41,20 @@ export default function PasteModal() { }, [setDisableVideoFocusTrap, setPasteModeEnabled]); const onConfirmPaste = useCallback(async () => { - setPasteModeEnabled(false); - setDisableVideoFocusTrap(false); + // setPasteModeEnabled(false); + // setDisableVideoFocusTrap(false); - if (rpcDataChannel?.readyState !== "open" || !TextAreaRef.current) return; - if (!selectedKeyboard) return; + if (!TextAreaRef.current || !selectedKeyboard) return; const text = TextAreaRef.current.value; try { + const macroSteps: { + keys: string[] | null; + modifiers: string[] | null; + delay: number; + }[] = []; + for (const char of text) { const keyprops = selectedKeyboard.chars[char]; if (!keyprops) continue; @@ -70,39 +64,41 @@ export default function PasteModal() { // if this is an accented character, we need to send that accent FIRST if (accentKey) { - await sendKeystroke({modifier: modifierCode(accentKey.shift, accentKey.altRight), keys: [ keys[accentKey.key] ] }) + const accentModifiers: string[] = []; + if (accentKey.shift) accentModifiers.push("ShiftLeft"); + if (accentKey.altRight) accentModifiers.push("AltRight"); + + macroSteps.push({ + keys: [String(accentKey.key)], + modifiers: accentModifiers.length > 0 ? accentModifiers : null, + delay: 100, + }); } // now send the actual key - await sendKeystroke({ modifier: modifierCode(shift, altRight), keys: [ keys[key] ]}); + const modifiers: string[] = []; + if (shift) modifiers.push("ShiftLeft"); + if (altRight) modifiers.push("AltRight"); + + macroSteps.push({ + keys: [String(key)], + modifiers: modifiers.length > 0 ? modifiers : null, + delay: 100, + }); // if what was requested was a dead key, we need to send an unmodified space to emit // just the accent character - if (deadKey) { - await sendKeystroke({ modifier: noModifier, keys: [ keys["Space"] ] }); - } + if (deadKey) macroSteps.push({ keys: ["Space"], modifiers: null, delay: 100 }); + } - // now send a message with no keys down to "release" the keys - await sendKeystroke({ modifier: 0, keys: [] }); + if (macroSteps.length > 0) { + await executeMacro(macroSteps); } } catch (error) { console.error("Failed to paste text:", error); notifications.error("Failed to paste text"); } - - async function sendKeystroke(stroke: KeyStroke) { - await new Promise((resolve, reject) => { - send( - "keyboardReport", - hidKeyboardPayload(stroke.modifier, stroke.keys), - params => { - if ("error" in params) return reject(params.error); - resolve(); - } - ); - }); - } - }, [selectedKeyboard, rpcDataChannel?.readyState, send, setDisableVideoFocusTrap, setPasteModeEnabled]); + }, [selectedKeyboard, executeMacro]); useEffect(() => { if (TextAreaRef.current) { @@ -122,14 +118,18 @@ export default function PasteModal() { />
-
e.stopPropagation()} onKeyDown={e => e.stopPropagation()}> +
e.stopPropagation()} + onKeyDown={e => e.stopPropagation()} + >

- Sending text using keyboard layout: {selectedKeyboard.isoCode}-{selectedKeyboard.name} + Sending text using keyboard layout: {selectedKeyboard.isoCode}- + {selectedKeyboard.name}

@@ -181,7 +182,7 @@ export default function PasteModal() {
{ + } = useHidRpc(message => { switch (message.constructor) { case KeysDownStateMessage: setKeysDownState((message as KeysDownStateMessage).keysDownState); @@ -48,7 +54,9 @@ export default function useKeyboard() { async (state: KeysDownState) => { if (rpcDataChannel?.readyState !== "open" && !rpcHidReady) return; - console.debug(`Send keyboardReport keys: ${state.keys}, modifier: ${state.modifier}`); + console.debug( + `Send keyboardReport keys: ${state.keys}, modifier: ${state.modifier}`, + ); if (rpcHidReady) { console.debug("Sending keyboard report via HidRPC"); @@ -56,31 +64,29 @@ export default function useKeyboard() { return; } - send("keyboardReport", { keys: state.keys, modifier: state.modifier }, (resp: JsonRpcResponse) => { - if ("error" in resp) { - console.error(`Failed to send keyboard report ${state}`, resp.error); - } - }); + send( + "keyboardReport", + { keys: state.keys, modifier: state.modifier }, + (resp: JsonRpcResponse) => { + if ("error" in resp) { + console.error(`Failed to send keyboard report ${state}`, resp.error); + } + }, + ); }, - [ - rpcDataChannel?.readyState, - rpcHidReady, - send, - sendKeyboardEventHidRpc, - ], + [rpcDataChannel?.readyState, rpcHidReady, send, sendKeyboardEventHidRpc], ); // resetKeyboardState is used to reset the keyboard state to no keys pressed and no modifiers. // This is useful for macros and when the browser loses focus to ensure that the keyboard state // is clean. - const resetKeyboardState = useCallback( - async () => { - // Reset the keys buffer to zeros and the modifier state to zero - keysDownState.keys.length = hidKeyBufferSize; - keysDownState.keys.fill(0); - keysDownState.modifier = 0; - sendKeyboardEvent(keysDownState); - }, [keysDownState, sendKeyboardEvent]); + const resetKeyboardState = useCallback(async () => { + // Reset the keys buffer to zeros and the modifier state to zero + keysDownState.keys.length = hidKeyBufferSize; + keysDownState.keys.fill(0); + keysDownState.modifier = 0; + sendKeyboardEvent(keysDownState); + }, [keysDownState, sendKeyboardEvent]); // executeMacro is used to execute a macro consisting of multiple steps. // Each step can have multiple keys, multiple modifiers and a delay. @@ -88,27 +94,32 @@ export default function useKeyboard() { // After the delay, the keys and modifiers are released and the next step is executed. // If a step has no keys or modifiers, it is treated as a delay-only step. // A small pause is added between steps to ensure that the device can process the events. - const executeMacro = async (steps: { keys: string[] | null; modifiers: string[] | null; delay: number }[]) => { - for (const [index, step] of steps.entries()) { + const executeMacro = async ( + steps: { keys: string[] | null; modifiers: string[] | null; delay: number }[], + ) => { + const macro: KeysDownState[] = []; + + for (const [_, step] of steps.entries()) { const keyValues = (step.keys || []).map(key => keys[key]).filter(Boolean); - const modifierMask: number = (step.modifiers || []).map(mod => modifiers[mod]).reduce((acc, val) => acc + val, 0); + const modifierMask: number = (step.modifiers || []) + .map(mod => modifiers[mod]) + .reduce((acc, val) => acc + val, 0); // If the step has keys and/or modifiers, press them and hold for the delay if (keyValues.length > 0 || modifierMask > 0) { - sendKeyboardEvent({ keys: keyValues, modifier: modifierMask }); - await new Promise(resolve => setTimeout(resolve, step.delay || 50)); - - resetKeyboardState(); - } else { - // This is a delay-only step, just wait for the delay amount - await new Promise(resolve => setTimeout(resolve, step.delay || 50)); - } - - // Add a small pause between steps if not the last step - if (index < steps.length - 1) { - await new Promise(resolve => setTimeout(resolve, 10)); + macro.push({ keys: keyValues, modifier: modifierMask }); + keysDownState.keys.length = hidKeyBufferSize; + keysDownState.keys.fill(0); + keysDownState.modifier = 0; + macro.push(keysDownState); } } + // KeyboardReportMessage + send("keyboardReportMulti", { macro }, (resp: JsonRpcResponse) => { + if ("error" in resp) { + console.error(`Failed to send keyboard report ${macro}`, resp.error); + } + }); }; // handleKeyPress is used to handle a key press or release event. @@ -132,7 +143,11 @@ export default function useKeyboard() { sendKeypressEventHidRpc(key, press); } else { // if the keyPress api is not available, we need to handle the key locally - const downState = simulateDeviceSideKeyHandlingForLegacyDevices(keysDownState, key, press); + const downState = simulateDeviceSideKeyHandlingForLegacyDevices( + keysDownState, + key, + press, + ); sendKeyboardEvent(downState); // then we send the full state // if we just sent ErrorRollOver, reset to empty state @@ -152,7 +167,11 @@ export default function useKeyboard() { ); // IMPORTANT: See the keyPressReportApiAvailable comment above for the reason this exists - function simulateDeviceSideKeyHandlingForLegacyDevices(state: KeysDownState, key: number, press: boolean): KeysDownState { + function simulateDeviceSideKeyHandlingForLegacyDevices( + state: KeysDownState, + key: number, + press: boolean, + ): KeysDownState { // IMPORTANT: This code parallels the logic in the kernel's hid-gadget driver // for handling key presses and releases. It ensures that the USB gadget // behaves similarly to a real USB HID keyboard. This logic is paralleled @@ -164,7 +183,7 @@ export default function useKeyboard() { if (modifierMask !== 0) { // If the key is a modifier key, we update the keyboardModifier state // by setting or clearing the corresponding bit in the modifier byte. - // This allows us to track the state of dynamic modifier keys like + // This allows us to track the state of dynamic modifier keys like // Shift, Control, Alt, and Super. if (press) { modifiers |= modifierMask; @@ -181,7 +200,7 @@ export default function useKeyboard() { // and if we find a zero byte, we can place the key there (if press is true) if (keys[i] === key || keys[i] === 0) { if (press) { - keys[i] = key // overwrites the zero byte or the same key if already pressed + keys[i] = key; // overwrites the zero byte or the same key if already pressed } else { // we are releasing the key, remove it from the buffer if (keys[i] !== 0) { @@ -197,13 +216,15 @@ export default function useKeyboard() { // If we reach here it means we didn't find an empty slot or the key in the buffer if (overrun) { if (press) { - console.warn(`keyboard buffer overflow current keys ${keys}, key: ${key} not added`); + console.warn( + `keyboard buffer overflow current keys ${keys}, key: ${key} not added`, + ); // Fill all key slots with ErrorRollOver (0x01) to indicate overflow keys.length = hidKeyBufferSize; keys.fill(hidErrorRollOver); } else { // If we are releasing a key, and we didn't find it in a slot, who cares? - console.debug(`key ${key} not found in buffer, nothing to release`) + console.debug(`key ${key} not found in buffer, nothing to release`); } } } diff --git a/web.go b/web.go index 21e17e74..7987ebec 100644 --- a/web.go +++ b/web.go @@ -198,6 +198,10 @@ func handleWebRTCSession(c *gin.Context) { _ = peerConn.Close() }() } + + // Cancel any ongoing keyboard report multi when session changes + cancelKeyboardReportMulti() + currentSession = session c.JSON(http.StatusOK, gin.H{"sd": sd}) } diff --git a/webrtc.go b/webrtc.go index c3d0dc1b..43a72f83 100644 --- a/webrtc.go +++ b/webrtc.go @@ -266,6 +266,8 @@ func newSession(config SessionConfig) (*Session, error) { if connectionState == webrtc.ICEConnectionStateClosed { scopedLogger.Debug().Msg("ICE Connection State is closed, unmounting virtual media") if session == currentSession { + // Cancel any ongoing keyboard report multi when session closes + cancelKeyboardReportMulti() currentSession = nil } // Stop RPC processor