diff --git a/cloud.go b/cloud.go index cec749e..fb13850 100644 --- a/cloud.go +++ b/cloud.go @@ -475,6 +475,10 @@ func handleSessionRequest( cloudLogger.Info().Interface("session", session).Msg("new session accepted") cloudLogger.Trace().Interface("session", session).Msg("new session accepted") + + // Cancel any ongoing keyboard macro when session changes + cancelKeyboardMacro() + currentSession = session _ = wsjson.Write(context.Background(), c, gin.H{"type": "answer", "data": sd}) return nil diff --git a/hidrpc.go b/hidrpc.go index 74fe687..7ed5f1c 100644 --- a/hidrpc.go +++ b/hidrpc.go @@ -1,7 +1,9 @@ package kvm import ( + "errors" "fmt" + "io" "time" "github.com/jetkvm/kvm/internal/hidrpc" @@ -29,6 +31,16 @@ func handleHidRPCMessage(message hidrpc.Message, session *Session) { session.reportHidRPCKeysDownState(*keysDownState) } rpcErr = err + case hidrpc.TypeKeyboardMacroReport: + keyboardMacroReport, err := message.KeyboardMacroReport() + if err != nil { + logger.Warn().Err(err).Msg("failed to get keyboard macro report") + return + } + _, rpcErr = rpcExecuteKeyboardMacro(keyboardMacroReport.Steps) + case hidrpc.TypeCancelKeyboardMacroReport: + rpcCancelKeyboardMacro() + return case hidrpc.TypePointerReport: pointerReport, err := message.PointerReport() if err != nil { @@ -128,6 +140,8 @@ func reportHidRPC(params any, session *Session) { message, err = hidrpc.NewKeyboardLedMessage(params).Marshal() case usbgadget.KeysDownState: message, err = hidrpc.NewKeydownStateMessage(params).Marshal() + case hidrpc.KeyboardMacroState: + message, err = hidrpc.NewKeyboardMacroStateMessage(params.State, params.IsPaste).Marshal() default: err = fmt.Errorf("unknown HID RPC message type: %T", params) } @@ -143,6 +157,10 @@ func reportHidRPC(params any, session *Session) { } if err := session.HidChannel.Send(message); err != nil { + if errors.Is(err, io.ErrClosedPipe) { + logger.Debug().Err(err).Msg("HID RPC channel closed, skipping reportHidRPC") + return + } logger.Warn().Err(err).Msg("failed to send HID RPC message") } } @@ -160,3 +178,10 @@ func (s *Session) reportHidRPCKeysDownState(state usbgadget.KeysDownState) { } reportHidRPC(state, s) } + +func (s *Session) reportHidRPCKeyboardMacroState(state hidrpc.KeyboardMacroState) { + if !s.hidRPCAvailable { + writeJSONRPCEvent("keyboardMacroState", state, s) + } + reportHidRPC(state, s) +} diff --git a/internal/hidrpc/hidrpc.go b/internal/hidrpc/hidrpc.go index e9c8c24..5d02d9e 100644 --- a/internal/hidrpc/hidrpc.go +++ b/internal/hidrpc/hidrpc.go @@ -10,14 +10,17 @@ import ( type MessageType byte const ( - TypeHandshake MessageType = 0x01 - TypeKeyboardReport MessageType = 0x02 - TypePointerReport MessageType = 0x03 - TypeWheelReport MessageType = 0x04 - TypeKeypressReport MessageType = 0x05 - TypeMouseReport MessageType = 0x06 - TypeKeyboardLedState MessageType = 0x32 - TypeKeydownState MessageType = 0x33 + TypeHandshake MessageType = 0x01 + TypeKeyboardReport MessageType = 0x02 + TypePointerReport MessageType = 0x03 + TypeWheelReport MessageType = 0x04 + TypeKeypressReport MessageType = 0x05 + TypeMouseReport MessageType = 0x06 + TypeKeyboardMacroReport MessageType = 0x07 + TypeCancelKeyboardMacroReport MessageType = 0x08 + TypeKeyboardLedState MessageType = 0x32 + TypeKeydownState MessageType = 0x33 + TypeKeyboardMacroState MessageType = 0x34 ) const ( @@ -29,10 +32,13 @@ func GetQueueIndex(messageType MessageType) int { switch messageType { case TypeHandshake: return 0 - case TypeKeyboardReport, TypeKeypressReport, TypeKeyboardLedState, TypeKeydownState: + case TypeKeyboardReport, TypeKeypressReport, TypeKeyboardMacroReport, TypeKeyboardLedState, TypeKeydownState, TypeKeyboardMacroState: return 1 case TypePointerReport, TypeMouseReport, TypeWheelReport: return 2 + // we don't want to block the queue for this message + case TypeCancelKeyboardMacroReport: + return 3 default: return 3 } @@ -98,3 +104,19 @@ func NewKeydownStateMessage(state usbgadget.KeysDownState) *Message { d: data, } } + +// NewKeyboardMacroStateMessage creates a new keyboard macro state message. +func NewKeyboardMacroStateMessage(state bool, isPaste bool) *Message { + data := make([]byte, 2) + if state { + data[0] = 1 + } + if isPaste { + data[1] = 1 + } + + return &Message{ + t: TypeKeyboardMacroState, + d: data, + } +} diff --git a/internal/hidrpc/message.go b/internal/hidrpc/message.go index 84bbda7..d1cc953 100644 --- a/internal/hidrpc/message.go +++ b/internal/hidrpc/message.go @@ -1,6 +1,7 @@ package hidrpc import ( + "encoding/binary" "fmt" ) @@ -43,6 +44,11 @@ func (m *Message) String() string { return fmt.Sprintf("MouseReport{Malformed: %v}", m.d) } return fmt.Sprintf("MouseReport{DX: %d, DY: %d, Button: %d}", m.d[0], m.d[1], m.d[2]) + case TypeKeyboardMacroReport: + if len(m.d) < 5 { + return fmt.Sprintf("KeyboardMacroReport{Malformed: %v}", m.d) + } + return fmt.Sprintf("KeyboardMacroReport{IsPaste: %v, Length: %d}", m.d[0] == uint8(1), binary.BigEndian.Uint32(m.d[1:5])) default: return fmt.Sprintf("Unknown{Type: %d, Data: %v}", m.t, m.d) } @@ -84,6 +90,55 @@ func (m *Message) KeyboardReport() (KeyboardReport, error) { }, nil } +// Macro .. +type KeyboardMacroStep struct { + Modifier byte // 1 byte + Keys []byte // 6 bytes: hidKeyBufferSize + Delay uint16 // 2 bytes +} +type KeyboardMacroReport struct { + IsPaste bool + StepCount uint32 + Steps []KeyboardMacroStep +} + +// HidKeyBufferSize is the size of the keys buffer in the keyboard report. +const HidKeyBufferSize = 6 + +// KeyboardMacroReport returns the keyboard macro report from the message. +func (m *Message) KeyboardMacroReport() (KeyboardMacroReport, error) { + if m.t != TypeKeyboardMacroReport { + return KeyboardMacroReport{}, fmt.Errorf("invalid message type: %d", m.t) + } + + isPaste := m.d[0] == uint8(1) + stepCount := binary.BigEndian.Uint32(m.d[1:5]) + + // check total length + expectedLength := int(stepCount)*9 + 5 + if len(m.d) != expectedLength { + return KeyboardMacroReport{}, fmt.Errorf("invalid length: %d, expected: %d", len(m.d), expectedLength) + } + + steps := make([]KeyboardMacroStep, 0, int(stepCount)) + offset := 5 + for i := 0; i < int(stepCount); i++ { + steps = append(steps, KeyboardMacroStep{ + Modifier: m.d[offset], + Keys: m.d[offset+1 : offset+7], + Delay: binary.BigEndian.Uint16(m.d[offset+7 : offset+9]), + }) + + offset += 1 + HidKeyBufferSize + 2 + } + + return KeyboardMacroReport{ + IsPaste: isPaste, + Steps: steps, + StepCount: stepCount, + }, nil +} + // PointerReport .. type PointerReport struct { X int @@ -131,3 +186,20 @@ func (m *Message) MouseReport() (MouseReport, error) { Button: uint8(m.d[2]), }, nil } + +type KeyboardMacroState struct { + State bool + IsPaste bool +} + +// KeyboardMacroState returns the keyboard macro state report from the message. +func (m *Message) KeyboardMacroState() (KeyboardMacroState, error) { + if m.t != TypeKeyboardMacroState { + return KeyboardMacroState{}, fmt.Errorf("invalid message type: %d", m.t) + } + + return KeyboardMacroState{ + State: m.d[0] == uint8(1), + IsPaste: m.d[1] == uint8(1), + }, nil +} diff --git a/jsonrpc.go b/jsonrpc.go index 61f28df..6be8633 100644 --- a/jsonrpc.go +++ b/jsonrpc.go @@ -1,6 +1,7 @@ package kvm import ( + "bytes" "context" "encoding/json" "errors" @@ -10,12 +11,14 @@ import ( "path/filepath" "reflect" "strconv" + "sync" "time" "github.com/pion/webrtc/v4" "github.com/rs/zerolog" "go.bug.st/serial" + "github.com/jetkvm/kvm/internal/hidrpc" "github.com/jetkvm/kvm/internal/usbgadget" "github.com/jetkvm/kvm/internal/utils" ) @@ -1056,6 +1059,106 @@ func rpcSetLocalLoopbackOnly(enabled bool) error { return nil } +var ( + keyboardMacroCancel context.CancelFunc + keyboardMacroLock sync.Mutex +) + +// cancelKeyboardMacro cancels any ongoing keyboard macro execution +func cancelKeyboardMacro() { + keyboardMacroLock.Lock() + defer keyboardMacroLock.Unlock() + + if keyboardMacroCancel != nil { + keyboardMacroCancel() + logger.Info().Msg("canceled keyboard macro") + keyboardMacroCancel = nil + } +} + +func setKeyboardMacroCancel(cancel context.CancelFunc) { + keyboardMacroLock.Lock() + defer keyboardMacroLock.Unlock() + + keyboardMacroCancel = cancel +} + +func rpcExecuteKeyboardMacro(macro []hidrpc.KeyboardMacroStep) (usbgadget.KeysDownState, error) { + cancelKeyboardMacro() + + ctx, cancel := context.WithCancel(context.Background()) + setKeyboardMacroCancel(cancel) + + s := hidrpc.KeyboardMacroState{ + State: true, + IsPaste: true, + } + + if currentSession != nil { + currentSession.reportHidRPCKeyboardMacroState(s) + } + + result, err := rpcDoExecuteKeyboardMacro(ctx, macro) + + setKeyboardMacroCancel(nil) + + s.State = false + if currentSession != nil { + currentSession.reportHidRPCKeyboardMacroState(s) + } + + return result, err +} + +func rpcCancelKeyboardMacro() { + cancelKeyboardMacro() +} + +var keyboardClearStateKeys = make([]byte, hidrpc.HidKeyBufferSize) + +func isClearKeyStep(step hidrpc.KeyboardMacroStep) bool { + return step.Modifier == 0 && bytes.Equal(step.Keys, keyboardClearStateKeys) +} + +func rpcDoExecuteKeyboardMacro(ctx context.Context, macro []hidrpc.KeyboardMacroStep) (usbgadget.KeysDownState, error) { + var last usbgadget.KeysDownState + var err error + + logger.Debug().Interface("macro", macro).Msg("Executing keyboard macro") + + for i, step := range macro { + delay := time.Duration(step.Delay) * time.Millisecond + + last, err = rpcKeyboardReport(step.Modifier, step.Keys) + if err != nil { + logger.Warn().Err(err).Msg("failed to execute keyboard macro") + return last, err + } + + // notify the device that the keyboard state is being cleared + if isClearKeyStep(step) { + gadget.UpdateKeysDown(0, keyboardClearStateKeys) + } + + // Use context-aware sleep that can be cancelled + select { + case <-time.After(delay): + // Sleep completed normally + case <-ctx.Done(): + // make sure keyboard state is reset + _, err := rpcKeyboardReport(0, keyboardClearStateKeys) + if err != nil { + logger.Warn().Err(err).Msg("failed to reset keyboard state") + } + + logger.Debug().Int("step", i).Msg("Keyboard macro cancelled during sleep") + return last, ctx.Err() + } + } + + return last, nil +} + var rpcHandlers = map[string]RPCHandler{ "ping": {Func: rpcPing}, "reboot": {Func: rpcReboot, Params: []string{"force"}}, diff --git a/ui/src/components/InfoBar.tsx b/ui/src/components/InfoBar.tsx index 8d0b282..ce444d8 100644 --- a/ui/src/components/InfoBar.tsx +++ b/ui/src/components/InfoBar.tsx @@ -27,6 +27,7 @@ export default function InfoBar() { const { rpcDataChannel } = useRTCStore(); const { debugMode, mouseMode, showPressedKeys } = useSettingsStore(); + const { isPasteInProgress } = useHidStore(); useEffect(() => { if (!rpcDataChannel) return; @@ -108,7 +109,12 @@ export default function InfoBar() { {rpcHidStatus} )} - + {isPasteInProgress && ( +
+ Paste Mode: + Enabled +
+ )} {showPressedKeys && (
Keys: diff --git a/ui/src/components/popovers/PasteModal.tsx b/ui/src/components/popovers/PasteModal.tsx index 077759b..5227222 100644 --- a/ui/src/components/popovers/PasteModal.tsx +++ b/ui/src/components/popovers/PasteModal.tsx @@ -1,42 +1,46 @@ -import { useCallback, useEffect, useRef, useState } from "react"; -import { LuCornerDownLeft } from "react-icons/lu"; -import { ExclamationCircleIcon } from "@heroicons/react/16/solid"; import { useClose } from "@headlessui/react"; +import { ExclamationCircleIcon } from "@heroicons/react/16/solid"; +import { useCallback, useEffect, useMemo, useRef, useState } from "react"; +import { LuCornerDownLeft } from "react-icons/lu"; -import { Button } from "@components/Button"; -import { GridCard } from "@components/Card"; -import { TextAreaWithLabel } from "@components/TextArea"; -import { SettingsPageHeader } from "@components/SettingsPageheader"; +import { cx } from "@/cva.config"; +import { useHidStore, useSettingsStore, useUiStore } from "@/hooks/stores"; import { JsonRpcResponse, useJsonRpc } from "@/hooks/useJsonRpc"; -import { useHidStore, useRTCStore, useUiStore, useSettingsStore } from "@/hooks/stores"; -import { keys, modifiers } from "@/keyboardMappings"; -import { KeyStroke } from "@/keyboardLayouts"; +import useKeyboard, { type MacroStep } from "@/hooks/useKeyboard"; import useKeyboardLayout from "@/hooks/useKeyboardLayout"; import notifications from "@/notifications"; +import { Button } from "@components/Button"; +import { GridCard } from "@components/Card"; +import { InputFieldWithLabel } from "@components/InputField"; +import { SettingsPageHeader } from "@components/SettingsPageheader"; +import { TextAreaWithLabel } from "@components/TextArea"; -const hidKeyboardPayload = (modifier: number, keys: number[]) => { - return { modifier, keys }; -}; - -const modifierCode = (shift?: boolean, altRight?: boolean) => { - return (shift ? modifiers.ShiftLeft : 0) - | (altRight ? modifiers.AltRight : 0) -} -const noModifier = 0 +// uint32 max value / 4 +const pasteMaxLength = 1073741824; export default function PasteModal() { const TextAreaRef = useRef(null); - const { setPasteModeEnabled } = useHidStore(); + const { isPasteInProgress } = useHidStore(); const { setDisableVideoFocusTrap } = useUiStore(); const { send } = useJsonRpc(); - const { rpcDataChannel } = useRTCStore(); + const { executeMacro, cancelExecuteMacro } = useKeyboard(); const [invalidChars, setInvalidChars] = useState([]); + const [delayValue, setDelayValue] = useState(100); + const delay = useMemo(() => { + if (delayValue < 50 || delayValue > 65534) { + return 100; + } + return delayValue; + }, [delayValue]); const close = useClose(); + const debugMode = useSettingsStore(state => state.debugMode); + const delayClassName = useMemo(() => debugMode ? "" : "hidden", [debugMode]); + const { setKeyboardLayout } = useSettingsStore(); - const { selectedKeyboard } = useKeyboardLayout(); + const { selectedKeyboard } = useKeyboardLayout(); useEffect(() => { send("getKeyboardLayout", {}, (resp: JsonRpcResponse) => { @@ -46,21 +50,19 @@ export default function PasteModal() { }, [send, setKeyboardLayout]); const onCancelPasteMode = useCallback(() => { - setPasteModeEnabled(false); + cancelExecuteMacro(); setDisableVideoFocusTrap(false); setInvalidChars([]); - }, [setDisableVideoFocusTrap, setPasteModeEnabled]); + }, [setDisableVideoFocusTrap, cancelExecuteMacro]); const onConfirmPaste = useCallback(async () => { - setPasteModeEnabled(false); - setDisableVideoFocusTrap(false); - - if (rpcDataChannel?.readyState !== "open" || !TextAreaRef.current) return; - if (!selectedKeyboard) return; + if (!TextAreaRef.current || !selectedKeyboard) return; const text = TextAreaRef.current.value; try { + const macroSteps: MacroStep[] = []; + for (const char of text) { const keyprops = selectedKeyboard.chars[char]; if (!keyprops) continue; @@ -70,39 +72,41 @@ export default function PasteModal() { // if this is an accented character, we need to send that accent FIRST if (accentKey) { - await sendKeystroke({modifier: modifierCode(accentKey.shift, accentKey.altRight), keys: [ keys[accentKey.key] ] }) + const accentModifiers: string[] = []; + if (accentKey.shift) accentModifiers.push("ShiftLeft"); + if (accentKey.altRight) accentModifiers.push("AltRight"); + + macroSteps.push({ + keys: [String(accentKey.key)], + modifiers: accentModifiers.length > 0 ? accentModifiers : null, + delay, + }); } // now send the actual key - await sendKeystroke({ modifier: modifierCode(shift, altRight), keys: [ keys[key] ]}); + const modifiers: string[] = []; + if (shift) modifiers.push("ShiftLeft"); + if (altRight) modifiers.push("AltRight"); + + macroSteps.push({ + keys: [String(key)], + modifiers: modifiers.length > 0 ? modifiers : null, + delay + }); // if what was requested was a dead key, we need to send an unmodified space to emit // just the accent character - if (deadKey) { - await sendKeystroke({ modifier: noModifier, keys: [ keys["Space"] ] }); - } + if (deadKey) macroSteps.push({ keys: ["Space"], modifiers: null, delay }); + } - // now send a message with no keys down to "release" the keys - await sendKeystroke({ modifier: 0, keys: [] }); + if (macroSteps.length > 0) { + await executeMacro(macroSteps); } } catch (error) { console.error("Failed to paste text:", error); notifications.error("Failed to paste text"); } - - async function sendKeystroke(stroke: KeyStroke) { - await new Promise((resolve, reject) => { - send( - "keyboardReport", - hidKeyboardPayload(stroke.modifier, stroke.keys), - params => { - if ("error" in params) return reject(params.error); - resolve(); - } - ); - }); - } - }, [selectedKeyboard, rpcDataChannel?.readyState, send, setDisableVideoFocusTrap, setPasteModeEnabled]); + }, [selectedKeyboard, executeMacro, delay]); useEffect(() => { if (TextAreaRef.current) { @@ -122,19 +126,25 @@ export default function PasteModal() { />
-
e.stopPropagation()} onKeyDown={e => e.stopPropagation()}> +
e.stopPropagation()} + onKeyDown={e => e.stopPropagation()} onKeyDownCapture={e => e.stopPropagation()} + onKeyUpCapture={e => e.stopPropagation()} + > e.stopPropagation()} + maxLength={pasteMaxLength} onKeyDown={e => { e.stopPropagation(); if (e.key === "Enter" && (e.metaKey || e.ctrlKey)) { @@ -171,9 +181,31 @@ export default function PasteModal() { )}
+
+ { + setDelayValue(parseInt(e.target.value, 10)); + }} + /> + {delayValue < 50 || delayValue > 65534 && ( +
+ + + Delay must be between 50 and 65534 + +
+ )} +

- Sending text using keyboard layout: {selectedKeyboard.isoCode}-{selectedKeyboard.name} + Sending text using keyboard layout: {selectedKeyboard.isoCode}- + {selectedKeyboard.name}

@@ -181,7 +213,7 @@ export default function PasteModal() {
diff --git a/ui/src/hooks/hidRpc.ts b/ui/src/hooks/hidRpc.ts index 20b8a10..2606770 100644 --- a/ui/src/hooks/hidRpc.ts +++ b/ui/src/hooks/hidRpc.ts @@ -1,4 +1,4 @@ -import { KeyboardLedState, KeysDownState } from "./stores"; +import { hidKeyBufferSize, KeyboardLedState, KeysDownState } from "./stores"; export const HID_RPC_MESSAGE_TYPES = { Handshake: 0x01, @@ -7,8 +7,11 @@ export const HID_RPC_MESSAGE_TYPES = { WheelReport: 0x04, KeypressReport: 0x05, MouseReport: 0x06, + KeyboardMacroReport: 0x07, + CancelKeyboardMacroReport: 0x08, KeyboardLedState: 0x32, KeysDownState: 0x33, + KeyboardMacroState: 0x34, } export type HidRpcMessageType = typeof HID_RPC_MESSAGE_TYPES[keyof typeof HID_RPC_MESSAGE_TYPES]; @@ -28,7 +31,31 @@ const fromInt32toUint8 = (n: number) => { (n >> 24) & 0xFF, (n >> 16) & 0xFF, (n >> 8) & 0xFF, - (n >> 0) & 0xFF, + n & 0xFF, + ]); +}; + +const fromUint16toUint8 = (n: number) => { + if (n > 65535 || n < 0) { + throw new Error(`Number ${n} is not within the uint16 range`); + } + + return new Uint8Array([ + (n >> 8) & 0xFF, + n & 0xFF, + ]); +}; + +const fromUint32toUint8 = (n: number) => { + if (n > 4294967295 || n < 0) { + throw new Error(`Number ${n} is not within the uint32 range`); + } + + return new Uint8Array([ + (n >> 24) & 0xFF, + (n >> 16) & 0xFF, + (n >> 8) & 0xFF, + n & 0xFF, ]); }; @@ -37,7 +64,7 @@ const fromInt8ToUint8 = (n: number) => { throw new Error(`Number ${n} is not within the int8 range`); } - return (n >> 0) & 0xFF; + return n & 0xFF; }; const keyboardLedStateMasks = { @@ -186,6 +213,99 @@ export class KeyboardReportMessage extends RpcMessage { } } +export interface KeyboardMacroStep extends KeysDownState { + delay: number; +} + +export class KeyboardMacroReportMessage extends RpcMessage { + isPaste: boolean; + stepCount: number; + steps: KeyboardMacroStep[]; + + KEYS_LENGTH = hidKeyBufferSize; + + constructor(isPaste: boolean, stepCount: number, steps: KeyboardMacroStep[]) { + super(HID_RPC_MESSAGE_TYPES.KeyboardMacroReport); + this.isPaste = isPaste; + this.stepCount = stepCount; + this.steps = steps; + } + + marshal(): Uint8Array { + // validate if length is correct + if (this.stepCount !== this.steps.length) { + throw new Error(`Length ${this.stepCount} is not equal to the number of steps ${this.steps.length}`); + } + + const data = new Uint8Array(this.stepCount * 9 + 6); + data.set(new Uint8Array([ + this.messageType, + this.isPaste ? 1 : 0, + ...fromUint32toUint8(this.stepCount), + ]), 0); + + for (let i = 0; i < this.stepCount; i++) { + const step = this.steps[i]; + if (!withinUint8Range(step.modifier)) { + throw new Error(`Modifier ${step.modifier} is not within the uint8 range`); + } + + // Ensure the keys are within the KEYS_LENGTH range + const keys = step.keys; + if (keys.length > this.KEYS_LENGTH) { + throw new Error(`Keys ${keys} is not within the hidKeyBufferSize range`); + } else if (keys.length < this.KEYS_LENGTH) { + keys.push(...Array(this.KEYS_LENGTH - keys.length).fill(0)); + } + + for (const key of keys) { + if (!withinUint8Range(key)) { + throw new Error(`Key ${key} is not within the uint8 range`); + } + } + + const macroBinary = new Uint8Array([ + step.modifier, + ...keys, + ...fromUint16toUint8(step.delay), + ]); + const offset = 6 + i * 9; + + + data.set(macroBinary, offset); + } + + return data; + } +} + +export class KeyboardMacroStateMessage extends RpcMessage { + state: boolean; + isPaste: boolean; + + constructor(state: boolean, isPaste: boolean) { + super(HID_RPC_MESSAGE_TYPES.KeyboardMacroState); + this.state = state; + this.isPaste = isPaste; + } + + marshal(): Uint8Array { + return new Uint8Array([ + this.messageType, + this.state ? 1 : 0, + this.isPaste ? 1 : 0, + ]); + } + + public static unmarshal(data: Uint8Array): KeyboardMacroStateMessage | undefined { + if (data.length < 1) { + throw new Error(`Invalid keyboard macro state report message length: ${data.length}`); + } + + return new KeyboardMacroStateMessage(data[0] === 1, data[1] === 1); + } +} + export class KeyboardLedStateMessage extends RpcMessage { keyboardLedState: KeyboardLedState; @@ -256,6 +376,17 @@ export class PointerReportMessage extends RpcMessage { } } +export class CancelKeyboardMacroReportMessage extends RpcMessage { + + constructor() { + super(HID_RPC_MESSAGE_TYPES.CancelKeyboardMacroReport); + } + + marshal(): Uint8Array { + return new Uint8Array([this.messageType]); + } +} + export class MouseReportMessage extends RpcMessage { dx: number; dy: number; @@ -284,6 +415,9 @@ export const messageRegistry = { [HID_RPC_MESSAGE_TYPES.KeyboardLedState]: KeyboardLedStateMessage, [HID_RPC_MESSAGE_TYPES.KeyboardReport]: KeyboardReportMessage, [HID_RPC_MESSAGE_TYPES.KeypressReport]: KeypressReportMessage, + [HID_RPC_MESSAGE_TYPES.KeyboardMacroReport]: KeyboardMacroReportMessage, + [HID_RPC_MESSAGE_TYPES.CancelKeyboardMacroReport]: CancelKeyboardMacroReportMessage, + [HID_RPC_MESSAGE_TYPES.KeyboardMacroState]: KeyboardMacroStateMessage, } export const unmarshalHidRpcMessage = (data: Uint8Array): RpcMessage | undefined => { diff --git a/ui/src/hooks/stores.ts b/ui/src/hooks/stores.ts index f99fd07..bb4b8dd 100644 --- a/ui/src/hooks/stores.ts +++ b/ui/src/hooks/stores.ts @@ -105,6 +105,9 @@ export interface RTCState { setRpcDataChannel: (channel: RTCDataChannel) => void; rpcDataChannel: RTCDataChannel | null; + hidRpcDisabled: boolean; + setHidRpcDisabled: (disabled: boolean) => void; + rpcHidProtocolVersion: number | null; setRpcHidProtocolVersion: (version: number) => void; @@ -157,6 +160,9 @@ export const useRTCStore = create(set => ({ rpcDataChannel: null, setRpcDataChannel: (channel: RTCDataChannel) => set({ rpcDataChannel: channel }), + hidRpcDisabled: false, + setHidRpcDisabled: (disabled: boolean) => set({ hidRpcDisabled: disabled }), + rpcHidProtocolVersion: null, setRpcHidProtocolVersion: (version: number) => set({ rpcHidProtocolVersion: version }), @@ -464,7 +470,7 @@ export interface HidState { isVirtualKeyboardEnabled: boolean; setVirtualKeyboardEnabled: (enabled: boolean) => void; - isPasteModeEnabled: boolean; + isPasteInProgress: boolean; setPasteModeEnabled: (enabled: boolean) => void; usbState: USBStates; @@ -481,8 +487,8 @@ export const useHidStore = create(set => ({ isVirtualKeyboardEnabled: false, setVirtualKeyboardEnabled: (enabled: boolean): void => set({ isVirtualKeyboardEnabled: enabled }), - isPasteModeEnabled: false, - setPasteModeEnabled: (enabled: boolean): void => set({ isPasteModeEnabled: enabled }), + isPasteInProgress: false, + setPasteModeEnabled: (enabled: boolean): void => set({ isPasteInProgress: enabled }), // Add these new properties for USB state usbState: "not attached", diff --git a/ui/src/hooks/useHidRpc.ts b/ui/src/hooks/useHidRpc.ts index ea0c711..87b0b81 100644 --- a/ui/src/hooks/useHidRpc.ts +++ b/ui/src/hooks/useHidRpc.ts @@ -3,8 +3,11 @@ import { useCallback, useEffect, useMemo } from "react"; import { useRTCStore } from "@/hooks/stores"; import { + CancelKeyboardMacroReportMessage, HID_RPC_VERSION, HandshakeMessage, + KeyboardMacroStep, + KeyboardMacroReportMessage, KeyboardReportMessage, KeypressReportMessage, MouseReportMessage, @@ -14,19 +17,23 @@ import { } from "./hidRpc"; export function useHidRpc(onHidRpcMessage?: (payload: RpcMessage) => void) { - const { rpcHidChannel, setRpcHidProtocolVersion, rpcHidProtocolVersion } = useRTCStore(); + const { rpcHidChannel, setRpcHidProtocolVersion, rpcHidProtocolVersion, hidRpcDisabled } = useRTCStore(); const rpcHidReady = useMemo(() => { + if (hidRpcDisabled) return false; return rpcHidChannel?.readyState === "open" && rpcHidProtocolVersion !== null; - }, [rpcHidChannel, rpcHidProtocolVersion]); + }, [rpcHidChannel, rpcHidProtocolVersion, hidRpcDisabled]); const rpcHidStatus = useMemo(() => { + if (hidRpcDisabled) return "disabled"; + if (!rpcHidChannel) return "N/A"; if (rpcHidChannel.readyState !== "open") return rpcHidChannel.readyState; if (!rpcHidProtocolVersion) return "handshaking"; return `ready (v${rpcHidProtocolVersion})`; - }, [rpcHidChannel, rpcHidProtocolVersion]); + }, [rpcHidChannel, rpcHidProtocolVersion, hidRpcDisabled]); const sendMessage = useCallback((message: RpcMessage, ignoreHandshakeState = false) => { + if (hidRpcDisabled) return; if (rpcHidChannel?.readyState !== "open") return; if (!rpcHidReady && !ignoreHandshakeState) return; @@ -39,7 +46,7 @@ export function useHidRpc(onHidRpcMessage?: (payload: RpcMessage) => void) { if (!data) return; rpcHidChannel?.send(data as unknown as ArrayBuffer); - }, [rpcHidChannel, rpcHidReady]); + }, [rpcHidChannel, rpcHidReady, hidRpcDisabled]); const reportKeyboardEvent = useCallback( (keys: number[], modifier: number) => { @@ -68,14 +75,31 @@ export function useHidRpc(onHidRpcMessage?: (payload: RpcMessage) => void) { [sendMessage], ); + const reportKeyboardMacroEvent = useCallback( + (steps: KeyboardMacroStep[]) => { + sendMessage(new KeyboardMacroReportMessage(false, steps.length, steps)); + }, + [sendMessage], + ); + + const cancelOngoingKeyboardMacro = useCallback( + () => { + sendMessage(new CancelKeyboardMacroReportMessage()); + }, + [sendMessage], + ); + const sendHandshake = useCallback(() => { + if (hidRpcDisabled) return; if (rpcHidProtocolVersion) return; if (!rpcHidChannel) return; sendMessage(new HandshakeMessage(HID_RPC_VERSION), true); - }, [rpcHidChannel, rpcHidProtocolVersion, sendMessage]); + }, [rpcHidChannel, rpcHidProtocolVersion, sendMessage, hidRpcDisabled]); const handleHandshake = useCallback((message: HandshakeMessage) => { + if (hidRpcDisabled) return; + if (!message.version) { console.error("Received handshake message without version", message); return; @@ -90,10 +114,11 @@ export function useHidRpc(onHidRpcMessage?: (payload: RpcMessage) => void) { } setRpcHidProtocolVersion(message.version); - }, [setRpcHidProtocolVersion]); + }, [setRpcHidProtocolVersion, hidRpcDisabled]); useEffect(() => { if (!rpcHidChannel) return; + if (hidRpcDisabled) return; // send handshake message sendHandshake(); @@ -135,6 +160,7 @@ export function useHidRpc(onHidRpcMessage?: (payload: RpcMessage) => void) { setRpcHidProtocolVersion, sendHandshake, handleHandshake, + hidRpcDisabled, ], ); @@ -143,6 +169,8 @@ export function useHidRpc(onHidRpcMessage?: (payload: RpcMessage) => void) { reportKeypressEvent, reportAbsMouseEvent, reportRelMouseEvent, + reportKeyboardMacroEvent, + cancelOngoingKeyboardMacro, rpcHidProtocolVersion, rpcHidReady, rpcHidStatus, diff --git a/ui/src/hooks/useKeyboard.ts b/ui/src/hooks/useKeyboard.ts index 787df9a..a276f43 100644 --- a/ui/src/hooks/useKeyboard.ts +++ b/ui/src/hooks/useKeyboard.ts @@ -1,15 +1,48 @@ -import { useCallback } from "react"; +import { useCallback, useRef } from "react"; -import { hidErrorRollOver, hidKeyBufferSize, KeysDownState, useHidStore, useRTCStore } from "@/hooks/stores"; +import { + hidErrorRollOver, + hidKeyBufferSize, + KeysDownState, + useHidStore, + useRTCStore, +} from "@/hooks/stores"; import { JsonRpcResponse, useJsonRpc } from "@/hooks/useJsonRpc"; import { useHidRpc } from "@/hooks/useHidRpc"; -import { KeyboardLedStateMessage, KeysDownStateMessage } from "@/hooks/hidRpc"; +import { + KeyboardLedStateMessage, + KeyboardMacroStateMessage, + KeyboardMacroStep, + KeysDownStateMessage, +} from "@/hooks/hidRpc"; import { hidKeyToModifierMask, keys, modifiers } from "@/keyboardMappings"; +const MACRO_RESET_KEYBOARD_STATE = { + keys: new Array(hidKeyBufferSize).fill(0), + modifier: 0, + delay: 0, +}; + +export interface MacroStep { + keys: string[] | null; + modifiers: string[] | null; + delay: number; +} + +export type MacroSteps = MacroStep[]; + +const sleep = (ms: number): Promise => new Promise(resolve => setTimeout(resolve, ms)); + export default function useKeyboard() { const { send } = useJsonRpc(); const { rpcDataChannel } = useRTCStore(); - const { keysDownState, setKeysDownState, setKeyboardLedState } = useHidStore(); + const { keysDownState, setKeysDownState, setKeyboardLedState, setPasteModeEnabled } = + useHidStore(); + + const abortController = useRef(null); + const setAbortController = useCallback((ac: AbortController | null) => { + abortController.current = ac; + }, []); // INTRODUCTION: The earlier version of the JetKVM device shipped with all keyboard state // being tracked on the browser/client-side. When adding the keyPressReport API to the @@ -17,17 +50,19 @@ export default function useKeyboard() { // is running on the cloud against a device that has not been updated yet and thus does not // support the keyPressReport API. In that case, we need to handle the key presses locally // and send the full state to the device, so it can behave like a real USB HID keyboard. - // This flag indicates whether the keyPressReport API is available on the device which is + // This flag indicates whether the keyPressReport API is available on the device which is // dynamically set when the device responds to the first key press event or reports its - // keysDownState when queried since the keyPressReport was introduced together with the + // keysDownState when queried since the keyPressReport was introduced together with the // getKeysDownState API. // HidRPC is a binary format for exchanging keyboard and mouse events const { reportKeyboardEvent: sendKeyboardEventHidRpc, reportKeypressEvent: sendKeypressEventHidRpc, + reportKeyboardMacroEvent: sendKeyboardMacroEventHidRpc, + cancelOngoingKeyboardMacro: cancelOngoingKeyboardMacroHidRpc, rpcHidReady, - } = useHidRpc((message) => { + } = useHidRpc(message => { switch (message.constructor) { case KeysDownStateMessage: setKeysDownState((message as KeysDownStateMessage).keysDownState); @@ -35,52 +70,61 @@ export default function useKeyboard() { case KeyboardLedStateMessage: setKeyboardLedState((message as KeyboardLedStateMessage).keyboardLedState); break; + case KeyboardMacroStateMessage: + if (!(message as KeyboardMacroStateMessage).isPaste) break; + setPasteModeEnabled((message as KeyboardMacroStateMessage).state); + break; default: break; } }); - // sendKeyboardEvent is used to send the full keyboard state to the device for macro handling - // and resetting keyboard state. It sends the keys currently pressed and the modifier state. - // The device will respond with the keysDownState if it supports the keyPressReport API - // or just accept the state if it does not support (returning no result) - const sendKeyboardEvent = useCallback( - async (state: KeysDownState) => { - if (rpcDataChannel?.readyState !== "open" && !rpcHidReady) return; - - console.debug(`Send keyboardReport keys: ${state.keys}, modifier: ${state.modifier}`); - - if (rpcHidReady) { - console.debug("Sending keyboard report via HidRPC"); - sendKeyboardEventHidRpc(state.keys, state.modifier); - return; - } - - send("keyboardReport", { keys: state.keys, modifier: state.modifier }, (resp: JsonRpcResponse) => { + const handleLegacyKeyboardReport = useCallback( + async (keys: number[], modifier: number) => { + send("keyboardReport", { keys, modifier }, (resp: JsonRpcResponse) => { if ("error" in resp) { - console.error(`Failed to send keyboard report ${state}`, resp.error); + console.error(`Failed to send keyboard report ${keys} ${modifier}`, resp.error); } + + // On older backends, we need to set the keysDownState manually since without the hidRpc API, the state doesn't trickle down from the backend + setKeysDownState({ modifier, keys }); }); }, - [ - rpcDataChannel?.readyState, - rpcHidReady, - send, - sendKeyboardEventHidRpc, - ], + [send, setKeysDownState], ); + const sendKeystrokeLegacy = useCallback(async (keys: number[], modifier: number, ac?: AbortController) => { + return await new Promise((resolve, reject) => { + const abortListener = () => { + reject(new Error("Keyboard report aborted")); + }; + + ac?.signal?.addEventListener("abort", abortListener); + + send( + "keyboardReport", + { keys, modifier }, + params => { + if ("error" in params) return reject(params.error); + resolve(); + }, + ); + }); + }, [send]); // resetKeyboardState is used to reset the keyboard state to no keys pressed and no modifiers. // This is useful for macros and when the browser loses focus to ensure that the keyboard state // is clean. - const resetKeyboardState = useCallback( - async () => { - // Reset the keys buffer to zeros and the modifier state to zero - keysDownState.keys.length = hidKeyBufferSize; - keysDownState.keys.fill(0); - keysDownState.modifier = 0; - sendKeyboardEvent(keysDownState); - }, [keysDownState, sendKeyboardEvent]); + const resetKeyboardState = useCallback(async () => { + // Reset the keys buffer to zeros and the modifier state to zero + const { keys, modifier } = MACRO_RESET_KEYBOARD_STATE; + if (rpcHidReady) { + sendKeyboardEventHidRpc(keys, modifier); + } else { + // Older backends don't support the hidRpc API, so we send the full reset state + handleLegacyKeyboardReport(keys, modifier); + } + }, [rpcHidReady, sendKeyboardEventHidRpc, handleLegacyKeyboardReport]); + // executeMacro is used to execute a macro consisting of multiple steps. // Each step can have multiple keys, multiple modifiers and a delay. @@ -88,28 +132,90 @@ export default function useKeyboard() { // After the delay, the keys and modifiers are released and the next step is executed. // If a step has no keys or modifiers, it is treated as a delay-only step. // A small pause is added between steps to ensure that the device can process the events. - const executeMacro = async (steps: { keys: string[] | null; modifiers: string[] | null; delay: number }[]) => { - for (const [index, step] of steps.entries()) { + const executeMacroRemote = useCallback(async (steps: MacroSteps) => { + const macro: KeyboardMacroStep[] = []; + + for (const [_, step] of steps.entries()) { const keyValues = (step.keys || []).map(key => keys[key]).filter(Boolean); - const modifierMask: number = (step.modifiers || []).map(mod => modifiers[mod]).reduce((acc, val) => acc + val, 0); + const modifierMask: number = (step.modifiers || []) + .map(mod => modifiers[mod]) + .reduce((acc, val) => acc + val, 0); // If the step has keys and/or modifiers, press them and hold for the delay if (keyValues.length > 0 || modifierMask > 0) { - sendKeyboardEvent({ keys: keyValues, modifier: modifierMask }); - await new Promise(resolve => setTimeout(resolve, step.delay || 50)); - - resetKeyboardState(); - } else { - // This is a delay-only step, just wait for the delay amount - await new Promise(resolve => setTimeout(resolve, step.delay || 50)); - } - - // Add a small pause between steps if not the last step - if (index < steps.length - 1) { - await new Promise(resolve => setTimeout(resolve, 10)); + macro.push({ keys: keyValues, modifier: modifierMask, delay: 20 }); + macro.push({ ...MACRO_RESET_KEYBOARD_STATE, delay: step.delay || 100 }); } } - }; + + sendKeyboardMacroEventHidRpc(macro); + }, [sendKeyboardMacroEventHidRpc]); + const executeMacroClientSide = useCallback(async (steps: MacroSteps) => { + const promises: (() => Promise)[] = []; + + const ac = new AbortController(); + setAbortController(ac); + + for (const [_, step] of steps.entries()) { + const keyValues = (step.keys || []).map(key => keys[key]).filter(Boolean); + const modifierMask: number = (step.modifiers || []) + .map(mod => modifiers[mod]) + .reduce((acc, val) => acc + val, 0); + + // If the step has keys and/or modifiers, press them and hold for the delay + if (keyValues.length > 0 || modifierMask > 0) { + promises.push(() => sendKeystrokeLegacy(keyValues, modifierMask, ac)); + promises.push(() => resetKeyboardState()); + promises.push(() => sleep(step.delay || 100)); + } + } + + const runAll = async () => { + for (const promise of promises) { + // Check if we've been aborted before executing each promise + if (ac.signal.aborted) { + throw new Error("Macro execution aborted"); + } + await promise(); + } + } + + return await new Promise((resolve, reject) => { + // Set up abort listener + const abortListener = () => { + reject(new Error("Macro execution aborted")); + }; + + ac.signal.addEventListener("abort", abortListener); + + runAll() + .then(() => { + ac.signal.removeEventListener("abort", abortListener); + resolve(); + }) + .catch((error) => { + ac.signal.removeEventListener("abort", abortListener); + reject(error); + }); + }); + }, [sendKeystrokeLegacy, resetKeyboardState, setAbortController]); + const executeMacro = useCallback(async (steps: MacroSteps) => { + if (rpcHidReady) { + return executeMacroRemote(steps); + } + return executeMacroClientSide(steps); + }, [rpcHidReady, executeMacroRemote, executeMacroClientSide]); + + const cancelExecuteMacro = useCallback(async () => { + if (abortController.current) { + abortController.current.abort(); + } + if (!rpcHidReady) return; + // older versions don't support this API, + // and all paste actions are pure-frontend, + // we don't need to cancel it actually + cancelOngoingKeyboardMacroHidRpc(); + }, [rpcHidReady, cancelOngoingKeyboardMacroHidRpc, abortController]); // handleKeyPress is used to handle a key press or release event. // This function handle both key press and key release events. @@ -131,9 +237,16 @@ export default function useKeyboard() { // in client/browser-side code using simulateDeviceSideKeyHandlingForLegacyDevices. sendKeypressEventHidRpc(key, press); } else { - // if the keyPress api is not available, we need to handle the key locally - const downState = simulateDeviceSideKeyHandlingForLegacyDevices(keysDownState, key, press); - sendKeyboardEvent(downState); // then we send the full state + // Older backends don't support the hidRpc API, so we need: + // 1. Calculate the state + // 2. Send the newly calculated state to the device + const downState = simulateDeviceSideKeyHandlingForLegacyDevices( + keysDownState, + key, + press, + ); + + handleLegacyKeyboardReport(downState.keys, downState.modifier); // if we just sent ErrorRollOver, reset to empty state if (downState.keys[0] === hidErrorRollOver) { @@ -142,17 +255,21 @@ export default function useKeyboard() { } }, [ - rpcHidReady, - keysDownState, - resetKeyboardState, rpcDataChannel?.readyState, - sendKeyboardEvent, + rpcHidReady, sendKeypressEventHidRpc, + keysDownState, + handleLegacyKeyboardReport, + resetKeyboardState, ], ); // IMPORTANT: See the keyPressReportApiAvailable comment above for the reason this exists - function simulateDeviceSideKeyHandlingForLegacyDevices(state: KeysDownState, key: number, press: boolean): KeysDownState { + function simulateDeviceSideKeyHandlingForLegacyDevices( + state: KeysDownState, + key: number, + press: boolean, + ): KeysDownState { // IMPORTANT: This code parallels the logic in the kernel's hid-gadget driver // for handling key presses and releases. It ensures that the USB gadget // behaves similarly to a real USB HID keyboard. This logic is paralleled @@ -164,7 +281,7 @@ export default function useKeyboard() { if (modifierMask !== 0) { // If the key is a modifier key, we update the keyboardModifier state // by setting or clearing the corresponding bit in the modifier byte. - // This allows us to track the state of dynamic modifier keys like + // This allows us to track the state of dynamic modifier keys like // Shift, Control, Alt, and Super. if (press) { modifiers |= modifierMask; @@ -181,7 +298,7 @@ export default function useKeyboard() { // and if we find a zero byte, we can place the key there (if press is true) if (keys[i] === key || keys[i] === 0) { if (press) { - keys[i] = key // overwrites the zero byte or the same key if already pressed + keys[i] = key; // overwrites the zero byte or the same key if already pressed } else { // we are releasing the key, remove it from the buffer if (keys[i] !== 0) { @@ -197,18 +314,20 @@ export default function useKeyboard() { // If we reach here it means we didn't find an empty slot or the key in the buffer if (overrun) { if (press) { - console.warn(`keyboard buffer overflow current keys ${keys}, key: ${key} not added`); + console.warn( + `keyboard buffer overflow current keys ${keys}, key: ${key} not added`, + ); // Fill all key slots with ErrorRollOver (0x01) to indicate overflow keys.length = hidKeyBufferSize; keys.fill(hidErrorRollOver); } else { // If we are releasing a key, and we didn't find it in a slot, who cares? - console.debug(`key ${key} not found in buffer, nothing to release`) + console.debug(`key ${key} not found in buffer, nothing to release`); } } } return { modifier: modifiers, keys }; } - return { handleKeyPress, resetKeyboardState, executeMacro }; + return { handleKeyPress, resetKeyboardState, executeMacro, cancelExecuteMacro }; } diff --git a/ui/src/routes/devices.$id.tsx b/ui/src/routes/devices.$id.tsx index 4318447..bdf6de9 100644 --- a/ui/src/routes/devices.$id.tsx +++ b/ui/src/routes/devices.$id.tsx @@ -583,6 +583,7 @@ export default function KvmIdRoute() { keyboardLedState, setKeyboardLedState, keysDownState, setKeysDownState, setUsbState, } = useHidStore(); + const setHidRpcDisabled = useRTCStore(state => state.setHidRpcDisabled); const [hasUpdated, setHasUpdated] = useState(false); const { navigateTo } = useDeviceUiNavigation(); @@ -695,6 +696,7 @@ export default function KvmIdRoute() { if (resp.error.code === -32601) { // if we don't support key down state, we know key press is also not available console.warn("Failed to get key down state, switching to old-school", resp.error); + setHidRpcDisabled(true); } else { console.error("Failed to get key down state", resp.error); } @@ -705,7 +707,7 @@ export default function KvmIdRoute() { } setNeedKeyDownState(false); }); - }, [keysDownState, needKeyDownState, rpcDataChannel?.readyState, send, setKeysDownState]); + }, [keysDownState, needKeyDownState, rpcDataChannel?.readyState, send, setKeysDownState, setHidRpcDisabled]); // When the update is successful, we need to refresh the client javascript and show a success modal useEffect(() => { diff --git a/web.go b/web.go index 7dfb696..4525357 100644 --- a/web.go +++ b/web.go @@ -228,6 +228,10 @@ func handleWebRTCSession(c *gin.Context) { _ = peerConn.Close() }() } + + // Cancel any ongoing keyboard macro when session changes + cancelKeyboardMacro() + currentSession = session c.JSON(http.StatusOK, gin.H{"sd": sd}) } diff --git a/webrtc.go b/webrtc.go index c3d0dc1..db9a7c2 100644 --- a/webrtc.go +++ b/webrtc.go @@ -266,6 +266,8 @@ func newSession(config SessionConfig) (*Session, error) { if connectionState == webrtc.ICEConnectionStateClosed { scopedLogger.Debug().Msg("ICE Connection State is closed, unmounting virtual media") if session == currentSession { + // Cancel any ongoing keyboard report multi when session closes + cancelKeyboardMacro() currentSession = nil } // Stop RPC processor