From 7014560b41f96eaad0e72c019749896effba1a6c Mon Sep 17 00:00:00 2001 From: Siyuan Miao Date: Wed, 10 Sep 2025 23:35:24 +0200 Subject: [PATCH] feat: allow paste progress to be cancelled --- cloud.go | 2 +- hidrpc.go | 15 ++- internal/hidrpc/hidrpc.go | 21 +++- internal/hidrpc/message.go | 17 ++++ jsonrpc.go | 113 +++++++++++----------- ui/src/components/popovers/PasteModal.tsx | 3 - ui/src/hooks/hidRpc.ts | 53 +++++++++- ui/src/hooks/useHidRpc.ts | 13 ++- ui/src/hooks/useKeyboard.ts | 20 ++-- ui/src/routes/devices.$id.tsx | 8 +- web.go | 2 +- webrtc.go | 2 +- 12 files changed, 178 insertions(+), 91 deletions(-) diff --git a/cloud.go b/cloud.go index f86a4815..39b7683c 100644 --- a/cloud.go +++ b/cloud.go @@ -477,7 +477,7 @@ func handleSessionRequest( cloudLogger.Trace().Interface("session", session).Msg("new session accepted") // Cancel any ongoing keyboard report multi when session changes - cancelKeyboardReportMulti() + cancelKeyboardMacro() currentSession = session _ = wsjson.Write(context.Background(), c, gin.H{"type": "answer", "data": sd}) diff --git a/hidrpc.go b/hidrpc.go index 604be89f..d9717e3a 100644 --- a/hidrpc.go +++ b/hidrpc.go @@ -1,7 +1,6 @@ package kvm import ( - "context" "errors" "fmt" "io" @@ -38,7 +37,10 @@ func handleHidRPCMessage(message hidrpc.Message, session *Session) { logger.Warn().Err(err).Msg("failed to get keyboard macro report") return } - _, rpcErr = rpcKeyboardReportMulti(context.Background(), keyboardMacroReport.Macro) + _, rpcErr = rpcExecuteKeyboardMacro(keyboardMacroReport.Macro) + case hidrpc.TypeCancelKeyboardMacroReport: + rpcCancelKeyboardMacro() + return case hidrpc.TypePointerReport: pointerReport, err := message.PointerReport() if err != nil { @@ -138,6 +140,8 @@ func reportHidRPC(params any, session *Session) { message, err = hidrpc.NewKeyboardLedMessage(params).Marshal() case usbgadget.KeysDownState: message, err = hidrpc.NewKeydownStateMessage(params).Marshal() + case hidrpc.KeyboardMacroStateReport: + message, err = hidrpc.NewKeyboardMacroStateMessage(params.State, params.IsPaste).Marshal() default: err = fmt.Errorf("unknown HID RPC message type: %T", params) } @@ -174,3 +178,10 @@ func (s *Session) reportHidRPCKeysDownState(state usbgadget.KeysDownState) { } reportHidRPC(state, s) } + +func (s *Session) reportHidRPCKeyboardMacroState(state hidrpc.KeyboardMacroStateReport) { + if !s.hidRPCAvailable { + writeJSONRPCEvent("keyboardMacroState", state, s) + } + reportHidRPC(state, s) +} diff --git a/internal/hidrpc/hidrpc.go b/internal/hidrpc/hidrpc.go index c4d99615..dc49965a 100644 --- a/internal/hidrpc/hidrpc.go +++ b/internal/hidrpc/hidrpc.go @@ -32,10 +32,13 @@ func GetQueueIndex(messageType MessageType) int { switch messageType { case TypeHandshake: return 0 - case TypeKeyboardReport, TypeKeypressReport, TypeKeyboardLedState, TypeKeydownState, TypeKeyboardMacroStateReport: + case TypeKeyboardReport, TypeKeypressReport, TypeKeyboardMacroReport, TypeKeyboardLedState, TypeKeydownState, TypeKeyboardMacroStateReport: return 1 case TypePointerReport, TypeMouseReport, TypeWheelReport: return 2 + // we don't want to block the queue for this message + case TypeCancelKeyboardMacroReport: + return 3 default: return 3 } @@ -101,3 +104,19 @@ func NewKeydownStateMessage(state usbgadget.KeysDownState) *Message { d: data, } } + +// NewKeyboardMacroStateMessage creates a new keyboard macro state message. +func NewKeyboardMacroStateMessage(state bool, isPaste bool) *Message { + data := make([]byte, 2) + if state { + data[0] = 1 + } + if isPaste { + data[1] = 1 + } + + return &Message{ + t: TypeKeyboardMacroStateReport, + d: data, + } +} diff --git a/internal/hidrpc/message.go b/internal/hidrpc/message.go index fa403265..6eae1779 100644 --- a/internal/hidrpc/message.go +++ b/internal/hidrpc/message.go @@ -182,3 +182,20 @@ func (m *Message) MouseReport() (MouseReport, error) { Button: uint8(m.d[2]), }, nil } + +type KeyboardMacroStateReport struct { + State bool + IsPaste bool +} + +// KeyboardMacroStateReport returns the keyboard macro state report from the message. +func (m *Message) KeyboardMacroStateReport() (KeyboardMacroStateReport, error) { + if m.t != TypeKeyboardMacroStateReport { + return KeyboardMacroStateReport{}, fmt.Errorf("invalid message type: %d", m.t) + } + + return KeyboardMacroStateReport{ + State: m.d[0] == uint8(1), + IsPaste: m.d[1] == uint8(1), + }, nil +} diff --git a/jsonrpc.go b/jsonrpc.go index a0acad25..d564b4a3 100644 --- a/jsonrpc.go +++ b/jsonrpc.go @@ -10,6 +10,7 @@ import ( "path/filepath" "reflect" "strconv" + "sync" "time" "github.com/pion/webrtc/v4" @@ -1050,75 +1051,69 @@ func rpcSetLocalLoopbackOnly(enabled bool) error { return nil } -func cancelKeyboardReportMulti() { +var ( + keyboardMacroCancel context.CancelFunc + keyboardMacroLock sync.Mutex +) +// cancelKeyboardMacro cancels any ongoing keyboard macro execution +func cancelKeyboardMacro() { + keyboardMacroLock.Lock() + defer keyboardMacroLock.Unlock() + + if keyboardMacroCancel != nil { + keyboardMacroCancel() + logger.Info().Msg("canceled keyboard macro") + keyboardMacroCancel = nil + } } -// // cancelKeyboardReportMulti cancels any ongoing keyboard report multi execution -// func cancelKeyboardReportMulti() { -// keyboardReportMultiLock.Lock() -// defer keyboardReportMultiLock.Unlock() +func setKeyboardMacroCancel(cancel context.CancelFunc) { + keyboardMacroLock.Lock() + defer keyboardMacroLock.Unlock() -// if keyboardReportMultiCancel != nil { -// keyboardReportMultiCancel() -// logger.Info().Msg("canceled keyboard report multi") -// keyboardReportMultiCancel = nil -// } -// } + keyboardMacroCancel = cancel +} -// func setKeyboardReportMultiCancel(cancel context.CancelFunc) { -// keyboardReportMultiLock.Lock() -// defer keyboardReportMultiLock.Unlock() +func rpcExecuteKeyboardMacro(macro []hidrpc.KeyboardMacro) (usbgadget.KeysDownState, error) { + cancelKeyboardMacro() -// keyboardReportMultiCancel = cancel -// } + ctx, cancel := context.WithCancel(context.Background()) + setKeyboardMacroCancel(cancel) -// func rpcKeyboardReportMultiWrapper(macro []map[string]any) (usbgadget.KeysDownState, error) { -// // cancelKeyboardReportMulti() + s := hidrpc.KeyboardMacroStateReport{ + State: true, + IsPaste: true, + } -// // ctx, cancel := context.WithCancel(context.Background()) -// // setKeyboardReportMultiCancel(cancel) + reportHidRPC(s, currentSession) -// // writeJSONRPCEvent("keyboardReportMultiState", true, currentSession) + result, err := rpcDoExecuteKeyboardMacro(ctx, macro) -// // result, err := rpcKeyboardReportMulti(ctx, macro) + setKeyboardMacroCancel(nil) -// // setKeyboardReportMultiCancel(nil) + s.State = false + reportHidRPC(s, currentSession) -// // writeJSONRPCEvent("keyboardReportMultiState", false, currentSession) + return result, err +} -// // return result, err -// } +func rpcCancelKeyboardMacro() { + cancelKeyboardMacro() +} -// var ( -// keyboardReportMultiCancel context.CancelFunc -// keyboardReportMultiLock sync.Mutex -// ) - -// func rpcCancelKeyboardReportMulti() { -// cancelKeyboardReportMulti() -// } - -func rpcKeyboardReportMulti(ctx context.Context, macro []hidrpc.KeyboardMacro) (usbgadget.KeysDownState, error) { +func rpcDoExecuteKeyboardMacro(ctx context.Context, macro []hidrpc.KeyboardMacro) (usbgadget.KeysDownState, error) { var last usbgadget.KeysDownState var err error - logger.Debug().Interface("macro", macro).Msg("Executing keyboard report multi") + logger.Debug().Interface("macro", macro).Msg("Executing keyboard macro") for i, step := range macro { - // Check for cancellation before each step - select { - case <-ctx.Done(): - logger.Debug().Msg("Keyboard report multi context cancelled") - return last, ctx.Err() - default: - } - delay := time.Duration(step.Delay) * time.Millisecond last, err = rpcKeyboardReport(step.Modifier, step.Keys) if err != nil { - logger.Warn().Err(err).Msg("failed to execute keyboard report multi") + logger.Warn().Err(err).Msg("failed to execute keyboard macro") return last, err } @@ -1127,7 +1122,9 @@ func rpcKeyboardReportMulti(ctx context.Context, macro []hidrpc.KeyboardMacro) ( case <-time.After(delay): // Sleep completed normally case <-ctx.Done(): - logger.Debug().Int("step", i).Msg("Keyboard report multi cancelled during sleep") + // make sure keyboard state is reset + rpcKeyboardReport(0, make([]byte, 6)) + logger.Debug().Int("step", i).Msg("Keyboard macro cancelled during sleep") return last, ctx.Err() } } @@ -1136,18 +1133,16 @@ func rpcKeyboardReportMulti(ctx context.Context, macro []hidrpc.KeyboardMacro) ( } var rpcHandlers = map[string]RPCHandler{ - "ping": {Func: rpcPing}, - "reboot": {Func: rpcReboot, Params: []string{"force"}}, - "getDeviceID": {Func: rpcGetDeviceID}, - "deregisterDevice": {Func: rpcDeregisterDevice}, - "getCloudState": {Func: rpcGetCloudState}, - "getNetworkState": {Func: rpcGetNetworkState}, - "getNetworkSettings": {Func: rpcGetNetworkSettings}, - "setNetworkSettings": {Func: rpcSetNetworkSettings, Params: []string{"settings"}}, - "renewDHCPLease": {Func: rpcRenewDHCPLease}, - "keyboardReport": {Func: rpcKeyboardReport, Params: []string{"modifier", "keys"}}, - // "keyboardReportMulti": {Func: rpcKeyboardReportMultiWrapper, Params: []string{"macro"}}, - // "cancelKeyboardReportMulti": {Func: rpcCancelKeyboardReportMulti}, + "ping": {Func: rpcPing}, + "reboot": {Func: rpcReboot, Params: []string{"force"}}, + "getDeviceID": {Func: rpcGetDeviceID}, + "deregisterDevice": {Func: rpcDeregisterDevice}, + "getCloudState": {Func: rpcGetCloudState}, + "getNetworkState": {Func: rpcGetNetworkState}, + "getNetworkSettings": {Func: rpcGetNetworkSettings}, + "setNetworkSettings": {Func: rpcSetNetworkSettings, Params: []string{"settings"}}, + "renewDHCPLease": {Func: rpcRenewDHCPLease}, + "keyboardReport": {Func: rpcKeyboardReport, Params: []string{"modifier", "keys"}}, "getKeyboardLedState": {Func: rpcGetKeyboardLedState}, "keypressReport": {Func: rpcKeypressReport, Params: []string{"key", "press"}}, "getKeyDownState": {Func: rpcGetKeysDownState}, diff --git a/ui/src/components/popovers/PasteModal.tsx b/ui/src/components/popovers/PasteModal.tsx index 30744fd6..1f5f6403 100644 --- a/ui/src/components/popovers/PasteModal.tsx +++ b/ui/src/components/popovers/PasteModal.tsx @@ -41,9 +41,6 @@ export default function PasteModal() { }, [setDisableVideoFocusTrap, cancelExecuteMacro]); const onConfirmPaste = useCallback(async () => { - // setPasteModeEnabled(false); - // setDisableVideoFocusTrap(false); - if (!TextAreaRef.current || !selectedKeyboard) return; const text = TextAreaRef.current.value; diff --git a/ui/src/hooks/hidRpc.ts b/ui/src/hooks/hidRpc.ts index 05923c94..edf75e75 100644 --- a/ui/src/hooks/hidRpc.ts +++ b/ui/src/hooks/hidRpc.ts @@ -8,8 +8,10 @@ export const HID_RPC_MESSAGE_TYPES = { KeypressReport: 0x05, MouseReport: 0x06, KeyboardMacroReport: 0x07, + CancelKeyboardMacroReport: 0x08, KeyboardLedState: 0x32, KeysDownState: 0x33, + KeyboardMacroStateReport: 0x34, } export type HidRpcMessageType = typeof HID_RPC_MESSAGE_TYPES[keyof typeof HID_RPC_MESSAGE_TYPES]; @@ -211,22 +213,22 @@ export class KeyboardReportMessage extends RpcMessage { } } -export interface KeyboardMacro extends KeysDownState { +export interface KeyboardMacroStep extends KeysDownState { delay: number; } export class KeyboardMacroReportMessage extends RpcMessage { isPaste: boolean; length: number; - macro: KeyboardMacro[]; + steps: KeyboardMacroStep[]; KEYS_LENGTH = 6; - constructor(isPaste: boolean, length: number, macro: KeyboardMacro[]) { + constructor(isPaste: boolean, length: number, steps: KeyboardMacroStep[]) { super(HID_RPC_MESSAGE_TYPES.KeyboardMacroReport); this.isPaste = isPaste; this.length = length; - this.macro = macro; + this.steps = steps; } marshal(): Uint8Array { @@ -238,7 +240,7 @@ export class KeyboardMacroReportMessage extends RpcMessage { let dataBody = new Uint8Array(); - for (const step of this.macro) { + for (const step of this.steps) { if (!withinUint8Range(step.modifier)) { throw new Error(`Modifier ${step.modifier} is not within the uint8 range`); } @@ -269,6 +271,33 @@ export class KeyboardMacroReportMessage extends RpcMessage { } } +export class KeyboardMacroStateReportMessage extends RpcMessage { + state: boolean; + isPaste: boolean; + + constructor(state: boolean, isPaste: boolean) { + super(HID_RPC_MESSAGE_TYPES.KeyboardMacroStateReport); + this.state = state; + this.isPaste = isPaste; + } + + marshal(): Uint8Array { + return new Uint8Array([ + this.messageType, + this.state ? 1 : 0, + this.isPaste ? 1 : 0, + ]); + } + + public static unmarshal(data: Uint8Array): KeyboardMacroStateReportMessage | undefined { + if (data.length < 1) { + throw new Error(`Invalid keyboard macro state report message length: ${data.length}`); + } + + return new KeyboardMacroStateReportMessage(data[0] === 1, data[1] === 1); + } +} + export class KeyboardLedStateMessage extends RpcMessage { keyboardLedState: KeyboardLedState; @@ -339,6 +368,17 @@ export class PointerReportMessage extends RpcMessage { } } +export class CancelKeyboardMacroReportMessage extends RpcMessage { + + constructor() { + super(HID_RPC_MESSAGE_TYPES.CancelKeyboardMacroReport); + } + + marshal(): Uint8Array { + return new Uint8Array([this.messageType]); + } +} + export class MouseReportMessage extends RpcMessage { dx: number; dy: number; @@ -367,6 +407,9 @@ export const messageRegistry = { [HID_RPC_MESSAGE_TYPES.KeyboardLedState]: KeyboardLedStateMessage, [HID_RPC_MESSAGE_TYPES.KeyboardReport]: KeyboardReportMessage, [HID_RPC_MESSAGE_TYPES.KeypressReport]: KeypressReportMessage, + [HID_RPC_MESSAGE_TYPES.KeyboardMacroReport]: KeyboardMacroReportMessage, + [HID_RPC_MESSAGE_TYPES.CancelKeyboardMacroReport]: CancelKeyboardMacroReportMessage, + [HID_RPC_MESSAGE_TYPES.KeyboardMacroStateReport]: KeyboardMacroStateReportMessage, } export const unmarshalHidRpcMessage = (data: Uint8Array): RpcMessage | undefined => { diff --git a/ui/src/hooks/useHidRpc.ts b/ui/src/hooks/useHidRpc.ts index 3beb9c07..fecb1661 100644 --- a/ui/src/hooks/useHidRpc.ts +++ b/ui/src/hooks/useHidRpc.ts @@ -3,9 +3,10 @@ import { useCallback, useEffect, useMemo } from "react"; import { useRTCStore } from "@/hooks/stores"; import { + CancelKeyboardMacroReportMessage, HID_RPC_VERSION, HandshakeMessage, - KeyboardMacro, + KeyboardMacroStep, KeyboardMacroReportMessage, KeyboardReportMessage, KeypressReportMessage, @@ -71,7 +72,7 @@ export function useHidRpc(onHidRpcMessage?: (payload: RpcMessage) => void) { ); const reportKeyboardMacroEvent = useCallback( - (macro: KeyboardMacro[]) => { + (macro: KeyboardMacroStep[]) => { const d = new KeyboardMacroReportMessage(false, macro.length, macro); sendMessage(d); console.log("Sent keyboard macro report", d, d.marshal()); @@ -79,6 +80,13 @@ export function useHidRpc(onHidRpcMessage?: (payload: RpcMessage) => void) { [sendMessage], ); + const cancelOngoingKeyboardMacro = useCallback( + () => { + sendMessage(new CancelKeyboardMacroReportMessage()); + }, + [sendMessage], + ); + const sendHandshake = useCallback(() => { if (rpcHidProtocolVersion) return; if (!rpcHidChannel) return; @@ -155,6 +163,7 @@ export function useHidRpc(onHidRpcMessage?: (payload: RpcMessage) => void) { reportAbsMouseEvent, reportRelMouseEvent, reportKeyboardMacroEvent, + cancelOngoingKeyboardMacro, rpcHidProtocolVersion, rpcHidReady, rpcHidStatus, diff --git a/ui/src/hooks/useKeyboard.ts b/ui/src/hooks/useKeyboard.ts index 4254b480..56c108e6 100644 --- a/ui/src/hooks/useKeyboard.ts +++ b/ui/src/hooks/useKeyboard.ts @@ -9,13 +9,13 @@ import { } from "@/hooks/stores"; import { JsonRpcResponse, useJsonRpc } from "@/hooks/useJsonRpc"; import { useHidRpc } from "@/hooks/useHidRpc"; -import { KeyboardLedStateMessage, KeyboardMacro, KeysDownStateMessage } from "@/hooks/hidRpc"; +import { KeyboardLedStateMessage, KeyboardMacroStateReportMessage, KeyboardMacroStep, KeysDownStateMessage } from "@/hooks/hidRpc"; import { hidKeyToModifierMask, keys, modifiers } from "@/keyboardMappings"; export default function useKeyboard() { const { send } = useJsonRpc(); const { rpcDataChannel } = useRTCStore(); - const { keysDownState, setKeysDownState, setKeyboardLedState } = useHidStore(); + const { keysDownState, setKeysDownState, setKeyboardLedState, setPasteModeEnabled } = useHidStore(); // INTRODUCTION: The earlier version of the JetKVM device shipped with all keyboard state // being tracked on the browser/client-side. When adding the keyPressReport API to the @@ -33,6 +33,7 @@ export default function useKeyboard() { reportKeyboardEvent: sendKeyboardEventHidRpc, reportKeypressEvent: sendKeypressEventHidRpc, reportKeyboardMacroEvent: sendKeyboardMacroEventHidRpc, + cancelOngoingKeyboardMacro: cancelOngoingKeyboardMacroHidRpc, rpcHidReady, } = useHidRpc(message => { switch (message.constructor) { @@ -42,6 +43,10 @@ export default function useKeyboard() { case KeyboardLedStateMessage: setKeyboardLedState((message as KeyboardLedStateMessage).keyboardLedState); break; + case KeyboardMacroStateReportMessage: + if (!(message as KeyboardMacroStateReportMessage).isPaste) break; + setPasteModeEnabled((message as KeyboardMacroStateReportMessage).state); + break; default: break; } @@ -101,7 +106,7 @@ export default function useKeyboard() { const executeMacro = async ( steps: { keys: string[] | null; modifiers: string[] | null; delay: number }[], ) => { - const macro: KeyboardMacro[] = []; + const macro: KeyboardMacroStep[] = []; for (const [_, step] of steps.entries()) { const keyValues = (step.keys || []).map(key => keys[key]).filter(Boolean); @@ -120,12 +125,9 @@ export default function useKeyboard() { }; const cancelExecuteMacro = useCallback(async () => { - send("cancelKeyboardReportMulti", {}, (resp: JsonRpcResponse) => { - if ("error" in resp) { - console.error(`Failed to cancel keyboard report multi`, resp.error); - } - }); - }, [send]); + if (!rpcHidReady) return; + cancelOngoingKeyboardMacroHidRpc(); + }, [rpcHidReady, cancelOngoingKeyboardMacroHidRpc]); // handleKeyPress is used to handle a key press or release event. // This function handle both key press and key release events. diff --git a/ui/src/routes/devices.$id.tsx b/ui/src/routes/devices.$id.tsx index 9445e9fd..4318447e 100644 --- a/ui/src/routes/devices.$id.tsx +++ b/ui/src/routes/devices.$id.tsx @@ -580,7 +580,7 @@ export default function KvmIdRoute() { const { setNetworkState} = useNetworkStateStore(); const { setHdmiState } = useVideoStore(); const { - keyboardLedState, setKeyboardLedState, setPasteModeEnabled, + keyboardLedState, setKeyboardLedState, keysDownState, setKeysDownState, setUsbState, } = useHidStore(); @@ -598,12 +598,6 @@ export default function KvmIdRoute() { setUsbState(usbState); } - if (resp.method === "keyboardReportMultiState") { - const reportMultiState = resp.params as unknown as boolean; - console.debug("Setting keyboard report multi state", reportMultiState); - setPasteModeEnabled(reportMultiState); - } - if (resp.method === "videoInputState") { const hdmiState = resp.params as Parameters[0]; console.debug("Setting HDMI state", hdmiState); diff --git a/web.go b/web.go index 7987ebec..b6ed667e 100644 --- a/web.go +++ b/web.go @@ -200,7 +200,7 @@ func handleWebRTCSession(c *gin.Context) { } // Cancel any ongoing keyboard report multi when session changes - cancelKeyboardReportMulti() + cancelKeyboardMacro() currentSession = session c.JSON(http.StatusOK, gin.H{"sd": sd}) diff --git a/webrtc.go b/webrtc.go index 43a72f83..db9a7c2c 100644 --- a/webrtc.go +++ b/webrtc.go @@ -267,7 +267,7 @@ func newSession(config SessionConfig) (*Session, error) { scopedLogger.Debug().Msg("ICE Connection State is closed, unmounting virtual media") if session == currentSession { // Cancel any ongoing keyboard report multi when session closes - cancelKeyboardReportMulti() + cancelKeyboardMacro() currentSession = nil } // Stop RPC processor