diff --git a/cloud.go b/cloud.go index cec749e4..39b7683c 100644 --- a/cloud.go +++ b/cloud.go @@ -475,6 +475,10 @@ func handleSessionRequest( cloudLogger.Info().Interface("session", session).Msg("new session accepted") cloudLogger.Trace().Interface("session", session).Msg("new session accepted") + + // Cancel any ongoing keyboard report multi when session changes + cancelKeyboardMacro() + currentSession = session _ = wsjson.Write(context.Background(), c, gin.H{"type": "answer", "data": sd}) return nil diff --git a/hidrpc.go b/hidrpc.go index 74fe687f..d9717e3a 100644 --- a/hidrpc.go +++ b/hidrpc.go @@ -1,7 +1,9 @@ package kvm import ( + "errors" "fmt" + "io" "time" "github.com/jetkvm/kvm/internal/hidrpc" @@ -29,6 +31,16 @@ func handleHidRPCMessage(message hidrpc.Message, session *Session) { session.reportHidRPCKeysDownState(*keysDownState) } rpcErr = err + case hidrpc.TypeKeyboardMacroReport: + keyboardMacroReport, err := message.KeyboardMacroReport() + if err != nil { + logger.Warn().Err(err).Msg("failed to get keyboard macro report") + return + } + _, rpcErr = rpcExecuteKeyboardMacro(keyboardMacroReport.Macro) + case hidrpc.TypeCancelKeyboardMacroReport: + rpcCancelKeyboardMacro() + return case hidrpc.TypePointerReport: pointerReport, err := message.PointerReport() if err != nil { @@ -128,6 +140,8 @@ func reportHidRPC(params any, session *Session) { message, err = hidrpc.NewKeyboardLedMessage(params).Marshal() case usbgadget.KeysDownState: message, err = hidrpc.NewKeydownStateMessage(params).Marshal() + case hidrpc.KeyboardMacroStateReport: + message, err = hidrpc.NewKeyboardMacroStateMessage(params.State, params.IsPaste).Marshal() default: err = fmt.Errorf("unknown HID RPC message type: %T", params) } @@ -143,6 +157,10 @@ func reportHidRPC(params any, session *Session) { } if err := session.HidChannel.Send(message); err != nil { + if errors.Is(err, io.ErrClosedPipe) { + logger.Debug().Err(err).Msg("HID RPC channel closed, skipping reportHidRPC") + return + } logger.Warn().Err(err).Msg("failed to send HID RPC message") } } @@ -160,3 +178,10 @@ func (s *Session) reportHidRPCKeysDownState(state usbgadget.KeysDownState) { } reportHidRPC(state, s) } + +func (s *Session) reportHidRPCKeyboardMacroState(state hidrpc.KeyboardMacroStateReport) { + if !s.hidRPCAvailable { + writeJSONRPCEvent("keyboardMacroState", state, s) + } + reportHidRPC(state, s) +} diff --git a/internal/hidrpc/hidrpc.go b/internal/hidrpc/hidrpc.go index e9c8c24d..dc49965a 100644 --- a/internal/hidrpc/hidrpc.go +++ b/internal/hidrpc/hidrpc.go @@ -10,14 +10,17 @@ import ( type MessageType byte const ( - TypeHandshake MessageType = 0x01 - TypeKeyboardReport MessageType = 0x02 - TypePointerReport MessageType = 0x03 - TypeWheelReport MessageType = 0x04 - TypeKeypressReport MessageType = 0x05 - TypeMouseReport MessageType = 0x06 - TypeKeyboardLedState MessageType = 0x32 - TypeKeydownState MessageType = 0x33 + TypeHandshake MessageType = 0x01 + TypeKeyboardReport MessageType = 0x02 + TypePointerReport MessageType = 0x03 + TypeWheelReport MessageType = 0x04 + TypeKeypressReport MessageType = 0x05 + TypeMouseReport MessageType = 0x06 + TypeKeyboardMacroReport MessageType = 0x07 + TypeCancelKeyboardMacroReport MessageType = 0x08 + TypeKeyboardLedState MessageType = 0x32 + TypeKeydownState MessageType = 0x33 + TypeKeyboardMacroStateReport MessageType = 0x34 ) const ( @@ -29,10 +32,13 @@ func GetQueueIndex(messageType MessageType) int { switch messageType { case TypeHandshake: return 0 - case TypeKeyboardReport, TypeKeypressReport, TypeKeyboardLedState, TypeKeydownState: + case TypeKeyboardReport, TypeKeypressReport, TypeKeyboardMacroReport, TypeKeyboardLedState, TypeKeydownState, TypeKeyboardMacroStateReport: return 1 case TypePointerReport, TypeMouseReport, TypeWheelReport: return 2 + // we don't want to block the queue for this message + case TypeCancelKeyboardMacroReport: + return 3 default: return 3 } @@ -98,3 +104,19 @@ func NewKeydownStateMessage(state usbgadget.KeysDownState) *Message { d: data, } } + +// NewKeyboardMacroStateMessage creates a new keyboard macro state message. +func NewKeyboardMacroStateMessage(state bool, isPaste bool) *Message { + data := make([]byte, 2) + if state { + data[0] = 1 + } + if isPaste { + data[1] = 1 + } + + return &Message{ + t: TypeKeyboardMacroStateReport, + d: data, + } +} diff --git a/internal/hidrpc/message.go b/internal/hidrpc/message.go index 84bbda7c..6eae1779 100644 --- a/internal/hidrpc/message.go +++ b/internal/hidrpc/message.go @@ -1,6 +1,7 @@ package hidrpc import ( + "encoding/binary" "fmt" ) @@ -43,6 +44,11 @@ func (m *Message) String() string { return fmt.Sprintf("MouseReport{Malformed: %v}", m.d) } return fmt.Sprintf("MouseReport{DX: %d, DY: %d, Button: %d}", m.d[0], m.d[1], m.d[2]) + case TypeKeyboardMacroReport: + if len(m.d) < 5 { + return fmt.Sprintf("KeyboardMacroReport{Malformed: %v}", m.d) + } + return fmt.Sprintf("KeyboardMacroReport{IsPaste: %v, Length: %d}", m.d[0] == uint8(1), binary.BigEndian.Uint32(m.d[1:5])) default: return fmt.Sprintf("Unknown{Type: %d, Data: %v}", m.t, m.d) } @@ -84,6 +90,51 @@ func (m *Message) KeyboardReport() (KeyboardReport, error) { }, nil } +// Macro .. +type KeyboardMacro struct { + Modifier byte // 1 byte + Keys []byte // 6 bytes, to make things easier, the keys length is fixed to 6 + Delay uint16 // 2 bytes +} +type KeyboardMacroReport struct { + IsPaste bool + Length uint32 + Macro []KeyboardMacro +} + +// KeyboardMacroReport returns the keyboard macro report from the message. +func (m *Message) KeyboardMacroReport() (KeyboardMacroReport, error) { + if m.t != TypeKeyboardMacroReport { + return KeyboardMacroReport{}, fmt.Errorf("invalid message type: %d", m.t) + } + + isPaste := m.d[0] == uint8(1) + length := binary.BigEndian.Uint32(m.d[1:5]) + + // check total length + expectedLength := int(length)*9 + 5 + if len(m.d) != expectedLength { + return KeyboardMacroReport{}, fmt.Errorf("invalid length: %d, expected: %d", len(m.d), expectedLength) + } + + macro := make([]KeyboardMacro, 0, int(length)) + for i := 0; i < int(length); i++ { + offset := 5 + i*9 + + macro = append(macro, KeyboardMacro{ + Modifier: m.d[offset], + Keys: m.d[offset+1 : offset+7], + Delay: binary.BigEndian.Uint16(m.d[offset+7 : offset+9]), + }) + } + + return KeyboardMacroReport{ + IsPaste: isPaste, + Macro: macro, + Length: length, + }, nil +} + // PointerReport .. type PointerReport struct { X int @@ -131,3 +182,20 @@ func (m *Message) MouseReport() (MouseReport, error) { Button: uint8(m.d[2]), }, nil } + +type KeyboardMacroStateReport struct { + State bool + IsPaste bool +} + +// KeyboardMacroStateReport returns the keyboard macro state report from the message. +func (m *Message) KeyboardMacroStateReport() (KeyboardMacroStateReport, error) { + if m.t != TypeKeyboardMacroStateReport { + return KeyboardMacroStateReport{}, fmt.Errorf("invalid message type: %d", m.t) + } + + return KeyboardMacroStateReport{ + State: m.d[0] == uint8(1), + IsPaste: m.d[1] == uint8(1), + }, nil +} diff --git a/jsonrpc.go b/jsonrpc.go index 61f28df5..62dc2026 100644 --- a/jsonrpc.go +++ b/jsonrpc.go @@ -1,6 +1,7 @@ package kvm import ( + "bytes" "context" "encoding/json" "errors" @@ -10,12 +11,14 @@ import ( "path/filepath" "reflect" "strconv" + "sync" "time" "github.com/pion/webrtc/v4" "github.com/rs/zerolog" "go.bug.st/serial" + "github.com/jetkvm/kvm/internal/hidrpc" "github.com/jetkvm/kvm/internal/usbgadget" "github.com/jetkvm/kvm/internal/utils" ) @@ -1056,6 +1059,107 @@ func rpcSetLocalLoopbackOnly(enabled bool) error { return nil } +var ( + keyboardMacroCancel context.CancelFunc + keyboardMacroLock sync.Mutex +) + +// cancelKeyboardMacro cancels any ongoing keyboard macro execution +func cancelKeyboardMacro() { + keyboardMacroLock.Lock() + defer keyboardMacroLock.Unlock() + + if keyboardMacroCancel != nil { + keyboardMacroCancel() + logger.Info().Msg("canceled keyboard macro") + keyboardMacroCancel = nil + } +} + +func setKeyboardMacroCancel(cancel context.CancelFunc) { + keyboardMacroLock.Lock() + defer keyboardMacroLock.Unlock() + + keyboardMacroCancel = cancel +} + +func rpcExecuteKeyboardMacro(macro []hidrpc.KeyboardMacro) (usbgadget.KeysDownState, error) { + cancelKeyboardMacro() + + ctx, cancel := context.WithCancel(context.Background()) + setKeyboardMacroCancel(cancel) + + s := hidrpc.KeyboardMacroStateReport{ + State: true, + IsPaste: true, + } + + if currentSession != nil { + currentSession.reportHidRPCKeyboardMacroState(s) + } + + result, err := rpcDoExecuteKeyboardMacro(ctx, macro) + + setKeyboardMacroCancel(nil) + + s.State = false + if currentSession != nil { + currentSession.reportHidRPCKeyboardMacroState(s) + } + + return result, err +} + +func rpcCancelKeyboardMacro() { + cancelKeyboardMacro() +} + +var keyboardClearStateKeys = make([]byte, 6) + +func isClearKeyStep(step hidrpc.KeyboardMacro) bool { + return step.Modifier == 0 && bytes.Equal(step.Keys, keyboardClearStateKeys) +} + +func rpcDoExecuteKeyboardMacro(ctx context.Context, macro []hidrpc.KeyboardMacro) (usbgadget.KeysDownState, error) { + var last usbgadget.KeysDownState + var err error + + logger.Debug().Interface("macro", macro).Msg("Executing keyboard macro") + + for i, step := range macro { + delay := time.Duration(step.Delay) * time.Millisecond + + last, err = rpcKeyboardReport(step.Modifier, step.Keys) + if err != nil { + logger.Warn().Err(err).Msg("failed to execute keyboard macro") + return last, err + } + + // notify the device that the keyboard state is being cleared + if isClearKeyStep(step) { + gadget.UpdateKeysDown(0, keyboardClearStateKeys) + } + + // Use context-aware sleep that can be cancelled + select { + case <-time.After(delay): + // Sleep completed normally + case <-ctx.Done(): + // make sure keyboard state is reset + _, err := rpcKeyboardReport(0, keyboardClearStateKeys) + if err != nil { + logger.Warn().Err(err).Msg("failed to reset keyboard state") + } + gadget.UpdateKeysDown(0, keyboardClearStateKeys) + + logger.Debug().Int("step", i).Msg("Keyboard macro cancelled during sleep") + return last, ctx.Err() + } + } + + return last, nil +} + var rpcHandlers = map[string]RPCHandler{ "ping": {Func: rpcPing}, "reboot": {Func: rpcReboot, Params: []string{"force"}}, diff --git a/ui/src/components/InfoBar.tsx b/ui/src/components/InfoBar.tsx index 8d0b2822..62759d17 100644 --- a/ui/src/components/InfoBar.tsx +++ b/ui/src/components/InfoBar.tsx @@ -27,6 +27,7 @@ export default function InfoBar() { const { rpcDataChannel } = useRTCStore(); const { debugMode, mouseMode, showPressedKeys } = useSettingsStore(); + const { isPasteModeEnabled } = useHidStore(); useEffect(() => { if (!rpcDataChannel) return; @@ -108,7 +109,12 @@ export default function InfoBar() { {rpcHidStatus} )} - + {isPasteModeEnabled && ( +
+ Paste Mode: + Enabled +
+ )} {showPressedKeys && (
Keys: diff --git a/ui/src/components/popovers/PasteModal.tsx b/ui/src/components/popovers/PasteModal.tsx index 077759b7..86212dea 100644 --- a/ui/src/components/popovers/PasteModal.tsx +++ b/ui/src/components/popovers/PasteModal.tsx @@ -1,42 +1,43 @@ -import { useCallback, useEffect, useRef, useState } from "react"; -import { LuCornerDownLeft } from "react-icons/lu"; -import { ExclamationCircleIcon } from "@heroicons/react/16/solid"; import { useClose } from "@headlessui/react"; +import { ExclamationCircleIcon } from "@heroicons/react/16/solid"; +import { useCallback, useEffect, useMemo, useRef, useState } from "react"; +import { LuCornerDownLeft } from "react-icons/lu"; -import { Button } from "@components/Button"; -import { GridCard } from "@components/Card"; -import { TextAreaWithLabel } from "@components/TextArea"; -import { SettingsPageHeader } from "@components/SettingsPageheader"; +import { cx } from "@/cva.config"; +import { useHidStore, useSettingsStore, useUiStore } from "@/hooks/stores"; import { JsonRpcResponse, useJsonRpc } from "@/hooks/useJsonRpc"; -import { useHidStore, useRTCStore, useUiStore, useSettingsStore } from "@/hooks/stores"; -import { keys, modifiers } from "@/keyboardMappings"; -import { KeyStroke } from "@/keyboardLayouts"; +import useKeyboard from "@/hooks/useKeyboard"; import useKeyboardLayout from "@/hooks/useKeyboardLayout"; import notifications from "@/notifications"; - -const hidKeyboardPayload = (modifier: number, keys: number[]) => { - return { modifier, keys }; -}; - -const modifierCode = (shift?: boolean, altRight?: boolean) => { - return (shift ? modifiers.ShiftLeft : 0) - | (altRight ? modifiers.AltRight : 0) -} -const noModifier = 0 +import { Button } from "@components/Button"; +import { GridCard } from "@components/Card"; +import { InputFieldWithLabel } from "@components/InputField"; +import { SettingsPageHeader } from "@components/SettingsPageheader"; +import { TextAreaWithLabel } from "@components/TextArea"; export default function PasteModal() { const TextAreaRef = useRef(null); - const { setPasteModeEnabled } = useHidStore(); + const { isPasteModeEnabled } = useHidStore(); const { setDisableVideoFocusTrap } = useUiStore(); const { send } = useJsonRpc(); - const { rpcDataChannel } = useRTCStore(); + const { executeMacro, cancelExecuteMacro } = useKeyboard(); const [invalidChars, setInvalidChars] = useState([]); + const [delayValue, setDelayValue] = useState(100); + const delay = useMemo(() => { + if (delayValue < 50 || delayValue > 65534) { + return 100; + } + return delayValue; + }, [delayValue]); const close = useClose(); + const debugMode = useSettingsStore(state => state.debugMode); + const delayClassName = useMemo(() => debugMode ? "" : "hidden", [debugMode]); + const { setKeyboardLayout } = useSettingsStore(); - const { selectedKeyboard } = useKeyboardLayout(); + const { selectedKeyboard } = useKeyboardLayout(); useEffect(() => { send("getKeyboardLayout", {}, (resp: JsonRpcResponse) => { @@ -46,21 +47,23 @@ export default function PasteModal() { }, [send, setKeyboardLayout]); const onCancelPasteMode = useCallback(() => { - setPasteModeEnabled(false); + cancelExecuteMacro(); setDisableVideoFocusTrap(false); setInvalidChars([]); - }, [setDisableVideoFocusTrap, setPasteModeEnabled]); + }, [setDisableVideoFocusTrap, cancelExecuteMacro]); const onConfirmPaste = useCallback(async () => { - setPasteModeEnabled(false); - setDisableVideoFocusTrap(false); - - if (rpcDataChannel?.readyState !== "open" || !TextAreaRef.current) return; - if (!selectedKeyboard) return; + if (!TextAreaRef.current || !selectedKeyboard) return; const text = TextAreaRef.current.value; try { + const macroSteps: { + keys: string[] | null; + modifiers: string[] | null; + delay: number; + }[] = []; + for (const char of text) { const keyprops = selectedKeyboard.chars[char]; if (!keyprops) continue; @@ -70,39 +73,41 @@ export default function PasteModal() { // if this is an accented character, we need to send that accent FIRST if (accentKey) { - await sendKeystroke({modifier: modifierCode(accentKey.shift, accentKey.altRight), keys: [ keys[accentKey.key] ] }) + const accentModifiers: string[] = []; + if (accentKey.shift) accentModifiers.push("ShiftLeft"); + if (accentKey.altRight) accentModifiers.push("AltRight"); + + macroSteps.push({ + keys: [String(accentKey.key)], + modifiers: accentModifiers.length > 0 ? accentModifiers : null, + delay, + }); } // now send the actual key - await sendKeystroke({ modifier: modifierCode(shift, altRight), keys: [ keys[key] ]}); + const modifiers: string[] = []; + if (shift) modifiers.push("ShiftLeft"); + if (altRight) modifiers.push("AltRight"); + + macroSteps.push({ + keys: [String(key)], + modifiers: modifiers.length > 0 ? modifiers : null, + delay + }); // if what was requested was a dead key, we need to send an unmodified space to emit // just the accent character - if (deadKey) { - await sendKeystroke({ modifier: noModifier, keys: [ keys["Space"] ] }); - } + if (deadKey) macroSteps.push({ keys: ["Space"], modifiers: null, delay }); + } - // now send a message with no keys down to "release" the keys - await sendKeystroke({ modifier: 0, keys: [] }); + if (macroSteps.length > 0) { + await executeMacro(macroSteps); } } catch (error) { console.error("Failed to paste text:", error); notifications.error("Failed to paste text"); } - - async function sendKeystroke(stroke: KeyStroke) { - await new Promise((resolve, reject) => { - send( - "keyboardReport", - hidKeyboardPayload(stroke.modifier, stroke.keys), - params => { - if ("error" in params) return reject(params.error); - resolve(); - } - ); - }); - } - }, [selectedKeyboard, rpcDataChannel?.readyState, send, setDisableVideoFocusTrap, setPasteModeEnabled]); + }, [selectedKeyboard, executeMacro, delay]); useEffect(() => { if (TextAreaRef.current) { @@ -122,14 +127,18 @@ export default function PasteModal() { />
-
e.stopPropagation()} onKeyDown={e => e.stopPropagation()}> +
e.stopPropagation()} + onKeyDown={e => e.stopPropagation()} + >
+
+ { + setDelayValue(parseInt(e.target.value, 10)); + }} + /> + {delayValue < 50 || delayValue > 65534 && ( +
+ + + Delay must be between 50 and 65534 + +
+ )} +

- Sending text using keyboard layout: {selectedKeyboard.isoCode}-{selectedKeyboard.name} + Sending text using keyboard layout: {selectedKeyboard.isoCode}- + {selectedKeyboard.name}

@@ -181,7 +212,7 @@ export default function PasteModal() {
diff --git a/ui/src/hooks/hidRpc.ts b/ui/src/hooks/hidRpc.ts index 20b8a108..cf2ef9cc 100644 --- a/ui/src/hooks/hidRpc.ts +++ b/ui/src/hooks/hidRpc.ts @@ -7,8 +7,11 @@ export const HID_RPC_MESSAGE_TYPES = { WheelReport: 0x04, KeypressReport: 0x05, MouseReport: 0x06, + KeyboardMacroReport: 0x07, + CancelKeyboardMacroReport: 0x08, KeyboardLedState: 0x32, KeysDownState: 0x33, + KeyboardMacroStateReport: 0x34, } export type HidRpcMessageType = typeof HID_RPC_MESSAGE_TYPES[keyof typeof HID_RPC_MESSAGE_TYPES]; @@ -32,6 +35,30 @@ const fromInt32toUint8 = (n: number) => { ]); }; +const fromUint16toUint8 = (n: number) => { + if (n > 65535 || n < 0) { + throw new Error(`Number ${n} is not within the uint16 range`); + } + + return new Uint8Array([ + (n >> 8) & 0xFF, + (n >> 0) & 0xFF, + ]); +}; + +const fromUint32toUint8 = (n: number) => { + if (n > 4294967295 || n < 0) { + throw new Error(`Number ${n} is not within the uint32 range`); + } + + return new Uint8Array([ + (n >> 24) & 0xFF, + (n >> 16) & 0xFF, + (n >> 8) & 0xFF, + (n >> 0) & 0xFF, + ]); +}; + const fromInt8ToUint8 = (n: number) => { if (n < -128 || n > 127) { throw new Error(`Number ${n} is not within the int8 range`); @@ -186,6 +213,99 @@ export class KeyboardReportMessage extends RpcMessage { } } +export interface KeyboardMacroStep extends KeysDownState { + delay: number; +} + +export class KeyboardMacroReportMessage extends RpcMessage { + isPaste: boolean; + length: number; + steps: KeyboardMacroStep[]; + + KEYS_LENGTH = 6; + + constructor(isPaste: boolean, length: number, steps: KeyboardMacroStep[]) { + super(HID_RPC_MESSAGE_TYPES.KeyboardMacroReport); + this.isPaste = isPaste; + this.length = length; + this.steps = steps; + } + + marshal(): Uint8Array { + // validate if length is correct + if (this.length !== this.steps.length) { + throw new Error(`Length ${this.length} is not equal to the number of steps ${this.steps.length}`); + } + + const data = new Uint8Array(this.length * 9 + 6); + data.set(new Uint8Array([ + this.messageType, + this.isPaste ? 1 : 0, + ...fromUint32toUint8(this.length), + ]), 0); + + for (let i = 0; i < this.length; i++) { + const step = this.steps[i]; + if (!withinUint8Range(step.modifier)) { + throw new Error(`Modifier ${step.modifier} is not within the uint8 range`); + } + + // Ensure the keys are within the KEYS_LENGTH range + const keys = step.keys; + if (keys.length > this.KEYS_LENGTH) { + throw new Error(`Keys ${keys} is not within the hidKeyBufferSize range`); + } else if (keys.length < this.KEYS_LENGTH) { + keys.push(...Array(this.KEYS_LENGTH - keys.length).fill(0)); + } + + for (const key of keys) { + if (!withinUint8Range(key)) { + throw new Error(`Key ${key} is not within the uint8 range`); + } + } + + const macroBinary = new Uint8Array([ + step.modifier, + ...keys, + ...fromUint16toUint8(step.delay), + ]); + const offset = 6 + i * 9; + + + data.set(macroBinary, offset); + } + + return data; + } +} + +export class KeyboardMacroStateReportMessage extends RpcMessage { + state: boolean; + isPaste: boolean; + + constructor(state: boolean, isPaste: boolean) { + super(HID_RPC_MESSAGE_TYPES.KeyboardMacroStateReport); + this.state = state; + this.isPaste = isPaste; + } + + marshal(): Uint8Array { + return new Uint8Array([ + this.messageType, + this.state ? 1 : 0, + this.isPaste ? 1 : 0, + ]); + } + + public static unmarshal(data: Uint8Array): KeyboardMacroStateReportMessage | undefined { + if (data.length < 1) { + throw new Error(`Invalid keyboard macro state report message length: ${data.length}`); + } + + return new KeyboardMacroStateReportMessage(data[0] === 1, data[1] === 1); + } +} + export class KeyboardLedStateMessage extends RpcMessage { keyboardLedState: KeyboardLedState; @@ -256,6 +376,17 @@ export class PointerReportMessage extends RpcMessage { } } +export class CancelKeyboardMacroReportMessage extends RpcMessage { + + constructor() { + super(HID_RPC_MESSAGE_TYPES.CancelKeyboardMacroReport); + } + + marshal(): Uint8Array { + return new Uint8Array([this.messageType]); + } +} + export class MouseReportMessage extends RpcMessage { dx: number; dy: number; @@ -284,6 +415,9 @@ export const messageRegistry = { [HID_RPC_MESSAGE_TYPES.KeyboardLedState]: KeyboardLedStateMessage, [HID_RPC_MESSAGE_TYPES.KeyboardReport]: KeyboardReportMessage, [HID_RPC_MESSAGE_TYPES.KeypressReport]: KeypressReportMessage, + [HID_RPC_MESSAGE_TYPES.KeyboardMacroReport]: KeyboardMacroReportMessage, + [HID_RPC_MESSAGE_TYPES.CancelKeyboardMacroReport]: CancelKeyboardMacroReportMessage, + [HID_RPC_MESSAGE_TYPES.KeyboardMacroStateReport]: KeyboardMacroStateReportMessage, } export const unmarshalHidRpcMessage = (data: Uint8Array): RpcMessage | undefined => { diff --git a/ui/src/hooks/useHidRpc.ts b/ui/src/hooks/useHidRpc.ts index ea0c7112..ddfea95b 100644 --- a/ui/src/hooks/useHidRpc.ts +++ b/ui/src/hooks/useHidRpc.ts @@ -3,8 +3,11 @@ import { useCallback, useEffect, useMemo } from "react"; import { useRTCStore } from "@/hooks/stores"; import { + CancelKeyboardMacroReportMessage, HID_RPC_VERSION, HandshakeMessage, + KeyboardMacroStep, + KeyboardMacroReportMessage, KeyboardReportMessage, KeypressReportMessage, MouseReportMessage, @@ -68,6 +71,21 @@ export function useHidRpc(onHidRpcMessage?: (payload: RpcMessage) => void) { [sendMessage], ); + const reportKeyboardMacroEvent = useCallback( + (macro: KeyboardMacroStep[]) => { + const d = new KeyboardMacroReportMessage(false, macro.length, macro); + sendMessage(d); + }, + [sendMessage], + ); + + const cancelOngoingKeyboardMacro = useCallback( + () => { + sendMessage(new CancelKeyboardMacroReportMessage()); + }, + [sendMessage], + ); + const sendHandshake = useCallback(() => { if (rpcHidProtocolVersion) return; if (!rpcHidChannel) return; @@ -143,6 +161,8 @@ export function useHidRpc(onHidRpcMessage?: (payload: RpcMessage) => void) { reportKeypressEvent, reportAbsMouseEvent, reportRelMouseEvent, + reportKeyboardMacroEvent, + cancelOngoingKeyboardMacro, rpcHidProtocolVersion, rpcHidReady, rpcHidStatus, diff --git a/ui/src/hooks/useKeyboard.ts b/ui/src/hooks/useKeyboard.ts index 787df9a9..5c660805 100644 --- a/ui/src/hooks/useKeyboard.ts +++ b/ui/src/hooks/useKeyboard.ts @@ -1,15 +1,21 @@ -import { useCallback } from "react"; +import { useCallback, useMemo } from "react"; -import { hidErrorRollOver, hidKeyBufferSize, KeysDownState, useHidStore, useRTCStore } from "@/hooks/stores"; +import { + hidErrorRollOver, + hidKeyBufferSize, + KeysDownState, + useHidStore, + useRTCStore, +} from "@/hooks/stores"; import { JsonRpcResponse, useJsonRpc } from "@/hooks/useJsonRpc"; import { useHidRpc } from "@/hooks/useHidRpc"; -import { KeyboardLedStateMessage, KeysDownStateMessage } from "@/hooks/hidRpc"; +import { KeyboardLedStateMessage, KeyboardMacroStateReportMessage, KeyboardMacroStep, KeysDownStateMessage } from "@/hooks/hidRpc"; import { hidKeyToModifierMask, keys, modifiers } from "@/keyboardMappings"; export default function useKeyboard() { const { send } = useJsonRpc(); const { rpcDataChannel } = useRTCStore(); - const { keysDownState, setKeysDownState, setKeyboardLedState } = useHidStore(); + const { keysDownState, setKeysDownState, setKeyboardLedState, setPasteModeEnabled } = useHidStore(); // INTRODUCTION: The earlier version of the JetKVM device shipped with all keyboard state // being tracked on the browser/client-side. When adding the keyPressReport API to the @@ -17,17 +23,19 @@ export default function useKeyboard() { // is running on the cloud against a device that has not been updated yet and thus does not // support the keyPressReport API. In that case, we need to handle the key presses locally // and send the full state to the device, so it can behave like a real USB HID keyboard. - // This flag indicates whether the keyPressReport API is available on the device which is + // This flag indicates whether the keyPressReport API is available on the device which is // dynamically set when the device responds to the first key press event or reports its - // keysDownState when queried since the keyPressReport was introduced together with the + // keysDownState when queried since the keyPressReport was introduced together with the // getKeysDownState API. // HidRPC is a binary format for exchanging keyboard and mouse events const { reportKeyboardEvent: sendKeyboardEventHidRpc, reportKeypressEvent: sendKeypressEventHidRpc, + reportKeyboardMacroEvent: sendKeyboardMacroEventHidRpc, + cancelOngoingKeyboardMacro: cancelOngoingKeyboardMacroHidRpc, rpcHidReady, - } = useHidRpc((message) => { + } = useHidRpc(message => { switch (message.constructor) { case KeysDownStateMessage: setKeysDownState((message as KeysDownStateMessage).keysDownState); @@ -35,6 +43,10 @@ export default function useKeyboard() { case KeyboardLedStateMessage: setKeyboardLedState((message as KeyboardLedStateMessage).keyboardLedState); break; + case KeyboardMacroStateReportMessage: + if (!(message as KeyboardMacroStateReportMessage).isPaste) break; + setPasteModeEnabled((message as KeyboardMacroStateReportMessage).state); + break; default: break; } @@ -48,7 +60,9 @@ export default function useKeyboard() { async (state: KeysDownState) => { if (rpcDataChannel?.readyState !== "open" && !rpcHidReady) return; - console.debug(`Send keyboardReport keys: ${state.keys}, modifier: ${state.modifier}`); + console.debug( + `Send keyboardReport keys: ${state.keys}, modifier: ${state.modifier}`, + ); if (rpcHidReady) { console.debug("Sending keyboard report via HidRPC"); @@ -56,31 +70,32 @@ export default function useKeyboard() { return; } - send("keyboardReport", { keys: state.keys, modifier: state.modifier }, (resp: JsonRpcResponse) => { - if ("error" in resp) { - console.error(`Failed to send keyboard report ${state}`, resp.error); - } - }); + send( + "keyboardReport", + { keys: state.keys, modifier: state.modifier }, + (resp: JsonRpcResponse) => { + if ("error" in resp) { + console.error(`Failed to send keyboard report ${state}`, resp.error); + } + }, + ); }, - [ - rpcDataChannel?.readyState, - rpcHidReady, - send, - sendKeyboardEventHidRpc, - ], + [rpcDataChannel?.readyState, rpcHidReady, send, sendKeyboardEventHidRpc], ); + const MACRO_RESET_KEYBOARD_STATE = useMemo(() => ({ + keys: new Array(hidKeyBufferSize).fill(0), + modifier: 0, + delay: 0, + }), []); + // resetKeyboardState is used to reset the keyboard state to no keys pressed and no modifiers. // This is useful for macros and when the browser loses focus to ensure that the keyboard state // is clean. - const resetKeyboardState = useCallback( - async () => { - // Reset the keys buffer to zeros and the modifier state to zero - keysDownState.keys.length = hidKeyBufferSize; - keysDownState.keys.fill(0); - keysDownState.modifier = 0; - sendKeyboardEvent(keysDownState); - }, [keysDownState, sendKeyboardEvent]); + const resetKeyboardState = useCallback(async () => { + // Reset the keys buffer to zeros and the modifier state to zero + sendKeyboardEvent(MACRO_RESET_KEYBOARD_STATE); + }, [sendKeyboardEvent, MACRO_RESET_KEYBOARD_STATE]); // executeMacro is used to execute a macro consisting of multiple steps. // Each step can have multiple keys, multiple modifiers and a delay. @@ -88,29 +103,32 @@ export default function useKeyboard() { // After the delay, the keys and modifiers are released and the next step is executed. // If a step has no keys or modifiers, it is treated as a delay-only step. // A small pause is added between steps to ensure that the device can process the events. - const executeMacro = async (steps: { keys: string[] | null; modifiers: string[] | null; delay: number }[]) => { - for (const [index, step] of steps.entries()) { + const executeMacro = async ( + steps: { keys: string[] | null; modifiers: string[] | null; delay: number }[], + ) => { + const macro: KeyboardMacroStep[] = []; + + for (const [_, step] of steps.entries()) { const keyValues = (step.keys || []).map(key => keys[key]).filter(Boolean); - const modifierMask: number = (step.modifiers || []).map(mod => modifiers[mod]).reduce((acc, val) => acc + val, 0); + const modifierMask: number = (step.modifiers || []) + .map(mod => modifiers[mod]) + .reduce((acc, val) => acc + val, 0); // If the step has keys and/or modifiers, press them and hold for the delay if (keyValues.length > 0 || modifierMask > 0) { - sendKeyboardEvent({ keys: keyValues, modifier: modifierMask }); - await new Promise(resolve => setTimeout(resolve, step.delay || 50)); - - resetKeyboardState(); - } else { - // This is a delay-only step, just wait for the delay amount - await new Promise(resolve => setTimeout(resolve, step.delay || 50)); - } - - // Add a small pause between steps if not the last step - if (index < steps.length - 1) { - await new Promise(resolve => setTimeout(resolve, 10)); + macro.push({ keys: keyValues, modifier: modifierMask, delay: 20 }); + macro.push({ ...MACRO_RESET_KEYBOARD_STATE, delay: step.delay || 100 }); } } + + sendKeyboardMacroEventHidRpc(macro); }; + const cancelExecuteMacro = useCallback(async () => { + if (!rpcHidReady) return; + cancelOngoingKeyboardMacroHidRpc(); + }, [rpcHidReady, cancelOngoingKeyboardMacroHidRpc]); + // handleKeyPress is used to handle a key press or release event. // This function handle both key press and key release events. // It checks if the keyPressReport API is available and sends the key press event. @@ -132,7 +150,11 @@ export default function useKeyboard() { sendKeypressEventHidRpc(key, press); } else { // if the keyPress api is not available, we need to handle the key locally - const downState = simulateDeviceSideKeyHandlingForLegacyDevices(keysDownState, key, press); + const downState = simulateDeviceSideKeyHandlingForLegacyDevices( + keysDownState, + key, + press, + ); sendKeyboardEvent(downState); // then we send the full state // if we just sent ErrorRollOver, reset to empty state @@ -152,7 +174,11 @@ export default function useKeyboard() { ); // IMPORTANT: See the keyPressReportApiAvailable comment above for the reason this exists - function simulateDeviceSideKeyHandlingForLegacyDevices(state: KeysDownState, key: number, press: boolean): KeysDownState { + function simulateDeviceSideKeyHandlingForLegacyDevices( + state: KeysDownState, + key: number, + press: boolean, + ): KeysDownState { // IMPORTANT: This code parallels the logic in the kernel's hid-gadget driver // for handling key presses and releases. It ensures that the USB gadget // behaves similarly to a real USB HID keyboard. This logic is paralleled @@ -164,7 +190,7 @@ export default function useKeyboard() { if (modifierMask !== 0) { // If the key is a modifier key, we update the keyboardModifier state // by setting or clearing the corresponding bit in the modifier byte. - // This allows us to track the state of dynamic modifier keys like + // This allows us to track the state of dynamic modifier keys like // Shift, Control, Alt, and Super. if (press) { modifiers |= modifierMask; @@ -181,7 +207,7 @@ export default function useKeyboard() { // and if we find a zero byte, we can place the key there (if press is true) if (keys[i] === key || keys[i] === 0) { if (press) { - keys[i] = key // overwrites the zero byte or the same key if already pressed + keys[i] = key; // overwrites the zero byte or the same key if already pressed } else { // we are releasing the key, remove it from the buffer if (keys[i] !== 0) { @@ -197,18 +223,20 @@ export default function useKeyboard() { // If we reach here it means we didn't find an empty slot or the key in the buffer if (overrun) { if (press) { - console.warn(`keyboard buffer overflow current keys ${keys}, key: ${key} not added`); + console.warn( + `keyboard buffer overflow current keys ${keys}, key: ${key} not added`, + ); // Fill all key slots with ErrorRollOver (0x01) to indicate overflow keys.length = hidKeyBufferSize; keys.fill(hidErrorRollOver); } else { // If we are releasing a key, and we didn't find it in a slot, who cares? - console.debug(`key ${key} not found in buffer, nothing to release`) + console.debug(`key ${key} not found in buffer, nothing to release`); } } } return { modifier: modifiers, keys }; } - return { handleKeyPress, resetKeyboardState, executeMacro }; + return { handleKeyPress, resetKeyboardState, executeMacro, cancelExecuteMacro }; } diff --git a/web.go b/web.go index f1aa1ab4..83873eab 100644 --- a/web.go +++ b/web.go @@ -228,6 +228,10 @@ func handleWebRTCSession(c *gin.Context) { _ = peerConn.Close() }() } + + // Cancel any ongoing keyboard report multi when session changes + cancelKeyboardMacro() + currentSession = session c.JSON(http.StatusOK, gin.H{"sd": sd}) } diff --git a/webrtc.go b/webrtc.go index c3d0dc1b..db9a7c2c 100644 --- a/webrtc.go +++ b/webrtc.go @@ -266,6 +266,8 @@ func newSession(config SessionConfig) (*Session, error) { if connectionState == webrtc.ICEConnectionStateClosed { scopedLogger.Debug().Msg("ICE Connection State is closed, unmounting virtual media") if session == currentSession { + // Cancel any ongoing keyboard report multi when session closes + cancelKeyboardMacro() currentSession = nil } // Stop RPC processor