wip: send macro using hidRPC channel

hidrpc.go

@@ -1,6 +1,7 @@
 package kvm
 
 import (
+	"context"
 	"errors"
 	"fmt"
 	"io"
@@ -31,6 +32,13 @@ func handleHidRPCMessage(message hidrpc.Message, session *Session) {
 			session.reportHidRPCKeysDownState(*keysDownState)
 		}
 		rpcErr = err
+	case hidrpc.TypeKeyboardMacroReport:
+		keyboardMacroReport, err := message.KeyboardMacroReport()
+		if err != nil {
+			logger.Warn().Err(err).Msg("failed to get keyboard macro report")
+			return
+		}
+		_, rpcErr = rpcKeyboardReportMulti(context.Background(), keyboardMacroReport.Macro)
 	case hidrpc.TypePointerReport:
 		pointerReport, err := message.PointerReport()
 		if err != nil {

internal/hidrpc/hidrpc.go

@@ -16,8 +16,11 @@ const (
 	TypeWheelReport               MessageType = 0x04
 	TypeKeypressReport            MessageType = 0x05
 	TypeMouseReport               MessageType = 0x06
+	TypeKeyboardMacroReport       MessageType = 0x07
+	TypeCancelKeyboardMacroReport MessageType = 0x08
 	TypeKeyboardLedState          MessageType = 0x32
 	TypeKeydownState              MessageType = 0x33
+	TypeKeyboardMacroStateReport  MessageType = 0x34
 )
 
 const (
@@ -29,7 +32,7 @@ func GetQueueIndex(messageType MessageType) int {
 	switch messageType {
 	case TypeHandshake:
 		return 0
-	case TypeKeyboardReport, TypeKeypressReport, TypeKeyboardLedState, TypeKeydownState:
+	case TypeKeyboardReport, TypeKeypressReport, TypeKeyboardLedState, TypeKeydownState, TypeKeyboardMacroStateReport:
 		return 1
 	case TypePointerReport, TypeMouseReport, TypeWheelReport:
 		return 2

internal/hidrpc/message.go

@@ -1,6 +1,7 @@
 package hidrpc
 
 import (
+	"encoding/binary"
 	"fmt"
 )
 
@@ -43,6 +44,11 @@ func (m *Message) String() string {
 			return fmt.Sprintf("MouseReport{Malformed: %v}", m.d)
 		}
 		return fmt.Sprintf("MouseReport{DX: %d, DY: %d, Button: %d}", m.d[0], m.d[1], m.d[2])
+	case TypeKeyboardMacroReport:
+		if len(m.d) < 5 {
+			return fmt.Sprintf("KeyboardMacroReport{Malformed: %v}", m.d)
+		}
+		return fmt.Sprintf("KeyboardMacroReport{IsPaste: %v, Length: %d}", m.d[0] == uint8(1), binary.BigEndian.Uint32(m.d[1:5]))
 	default:
 		return fmt.Sprintf("Unknown{Type: %d, Data: %v}", m.t, m.d)
 	}
@@ -84,6 +90,51 @@ func (m *Message) KeyboardReport() (KeyboardReport, error) {
 	}, nil
 }
 
+// Macro ..
+type KeyboardMacro struct {
+	Modifier byte   // 1 byte
+	Keys     []byte // 6 bytes, to make things easier, the keys length is fixed to 6
+	Delay    uint16 // 2 bytes
+}
+type KeyboardMacroReport struct {
+	IsPaste bool
+	Length  uint32
+	Macro   []KeyboardMacro
+}
+
+// KeyboardMacroReport returns the keyboard macro report from the message.
+func (m *Message) KeyboardMacroReport() (KeyboardMacroReport, error) {
+	if m.t != TypeKeyboardMacroReport {
+		return KeyboardMacroReport{}, fmt.Errorf("invalid message type: %d", m.t)
+	}
+
+	isPaste := m.d[0] == uint8(1)
+	length := binary.BigEndian.Uint32(m.d[1:5])
+
+	// check total length
+	expectedLength := int(length)*9 + 5
+	if len(m.d) != expectedLength {
+		return KeyboardMacroReport{}, fmt.Errorf("invalid length: %d, expected: %d", len(m.d), expectedLength)
+	}
+
+	macro := make([]KeyboardMacro, 0, int(length))
+	for i := 0; i < int(length); i++ {
+		offset := 5 + i*9
+
+		macro = append(macro, KeyboardMacro{
+			Modifier: m.d[offset],
+			Keys:     m.d[offset+1 : offset+7],
+			Delay:    binary.BigEndian.Uint16(m.d[offset+7 : offset+9]),
+		})
+	}
+
+	return KeyboardMacroReport{
+		IsPaste: isPaste,
+		Macro:   macro,
+		Length:  length,
+	}, nil
+}
+
 // PointerReport ..
 type PointerReport struct {
 	X      int

jsonrpc.go

@@ -10,13 +10,13 @@ import (
 	"path/filepath"
 	"reflect"
 	"strconv"
-	"sync"
 	"time"
 
 	"github.com/pion/webrtc/v4"
 	"github.com/rs/zerolog"
 	"go.bug.st/serial"
 
+	"github.com/jetkvm/kvm/internal/hidrpc"
 	"github.com/jetkvm/kvm/internal/usbgadget"
 )
 
@@ -1050,52 +1050,56 @@ func rpcSetLocalLoopbackOnly(enabled bool) error {
 	return nil
 }
 
-// cancelKeyboardReportMulti cancels any ongoing keyboard report multi execution
 func cancelKeyboardReportMulti() {
-	keyboardReportMultiLock.Lock()
-	defer keyboardReportMultiLock.Unlock()
 
-	if keyboardReportMultiCancel != nil {
-		keyboardReportMultiCancel()
-		logger.Info().Msg("canceled keyboard report multi")
-		keyboardReportMultiCancel = nil
-	}
 }
 
-func setKeyboardReportMultiCancel(cancel context.CancelFunc) {
+// // cancelKeyboardReportMulti cancels any ongoing keyboard report multi execution
-	keyboardReportMultiLock.Lock()
+// func cancelKeyboardReportMulti() {
-	defer keyboardReportMultiLock.Unlock()
+// 	keyboardReportMultiLock.Lock()
+// 	defer keyboardReportMultiLock.Unlock()
 
-	keyboardReportMultiCancel = cancel
+// 	if keyboardReportMultiCancel != nil {
-}
+// 		keyboardReportMultiCancel()
+// 		logger.Info().Msg("canceled keyboard report multi")
+// 		keyboardReportMultiCancel = nil
+// 	}
+// }
 
-func rpcKeyboardReportMultiWrapper(macro []map[string]any) (usbgadget.KeysDownState, error) {
+// func setKeyboardReportMultiCancel(cancel context.CancelFunc) {
-	cancelKeyboardReportMulti()
+// 	keyboardReportMultiLock.Lock()
+// 	defer keyboardReportMultiLock.Unlock()
 
-	ctx, cancel := context.WithCancel(context.Background())
+// 	keyboardReportMultiCancel = cancel
-	setKeyboardReportMultiCancel(cancel)
+// }
 
-	writeJSONRPCEvent("keyboardReportMultiState", true, currentSession)
+// func rpcKeyboardReportMultiWrapper(macro []map[string]any) (usbgadget.KeysDownState, error) {
+// 	// cancelKeyboardReportMulti()
 
-	result, err := rpcKeyboardReportMulti(ctx, macro)
+// 	// ctx, cancel := context.WithCancel(context.Background())
+// 	// setKeyboardReportMultiCancel(cancel)
 
-	setKeyboardReportMultiCancel(nil)
+// 	// writeJSONRPCEvent("keyboardReportMultiState", true, currentSession)
 
-	writeJSONRPCEvent("keyboardReportMultiState", false, currentSession)
+// 	// result, err := rpcKeyboardReportMulti(ctx, macro)
 
-	return result, err
+// 	// setKeyboardReportMultiCancel(nil)
-}
 
-var (
+// 	// writeJSONRPCEvent("keyboardReportMultiState", false, currentSession)
-	keyboardReportMultiCancel context.CancelFunc
-	keyboardReportMultiLock   sync.Mutex
-)
 
-func rpcCancelKeyboardReportMulti() {
+// 	// return result, err
-	cancelKeyboardReportMulti()
+// }
-}
 
-func rpcKeyboardReportMulti(ctx context.Context, macro []map[string]any) (usbgadget.KeysDownState, error) {
+// var (
+// 	keyboardReportMultiCancel context.CancelFunc
+// 	keyboardReportMultiLock   sync.Mutex
+// )
+
+// func rpcCancelKeyboardReportMulti() {
+// 	cancelKeyboardReportMulti()
+// }
+
+func rpcKeyboardReportMulti(ctx context.Context, macro []hidrpc.KeyboardMacro) (usbgadget.KeysDownState, error) {
 	var last usbgadget.KeysDownState
 	var err error
 
@@ -1110,45 +1114,23 @@ func rpcKeyboardReportMulti(ctx context.Context, macro []map[string]any) (usbgad
 		default:
 		}
 
-		var modifier byte
+		delay := time.Duration(step.Delay) * time.Millisecond
-		if m, ok := step["modifier"].(float64); ok {
+		logger.Info().Int("step", i).Uint16("delay", step.Delay).Msg("Keyboard report multi delay")
-			modifier = byte(int(m))
-		} else if mi, ok := step["modifier"].(int); ok {
-			modifier = byte(mi)
-		} else if mb, ok := step["modifier"].(uint8); ok {
-			modifier = mb
-		}
 
-		var keys []byte
+		last, err = rpcKeyboardReport(step.Modifier, step.Keys)
-		if arr, ok := step["keys"].([]any); ok {
+		if err != nil {
-			keys = make([]byte, 0, len(arr))
+			logger.Warn().Err(err).Msg("failed to execute keyboard report multi")
-			for _, v := range arr {
+			return last, err
-				if f, ok := v.(float64); ok {
-					keys = append(keys, byte(int(f)))
-				} else if i, ok := v.(int); ok {
-					keys = append(keys, byte(i))
-				} else if b, ok := v.(uint8); ok {
-					keys = append(keys, b)
-				}
-			}
-		} else if bs, ok := step["keys"].([]byte); ok {
-			keys = bs
 		}
 
 		// Use context-aware sleep that can be cancelled
 		select {
-		case <-time.After(100 * time.Millisecond):
+		case <-time.After(delay):
 			// Sleep completed normally
 		case <-ctx.Done():
 			logger.Debug().Int("step", i).Msg("Keyboard report multi cancelled during sleep")
 			return last, ctx.Err()
 		}
-
-		last, err = rpcKeyboardReport(modifier, keys)
-		if err != nil {
-			logger.Warn().Err(err).Msg("failed to execute keyboard report multi")
-			return last, err
-		}
 	}
 
 	return last, nil
@@ -1165,8 +1147,8 @@ var rpcHandlers = map[string]RPCHandler{
 	"setNetworkSettings": {Func: rpcSetNetworkSettings, Params: []string{"settings"}},
 	"renewDHCPLease":     {Func: rpcRenewDHCPLease},
 	"keyboardReport":     {Func: rpcKeyboardReport, Params: []string{"modifier", "keys"}},
-	"keyboardReportMulti":       {Func: rpcKeyboardReportMultiWrapper, Params: []string{"macro"}},
+	// "keyboardReportMulti":       {Func: rpcKeyboardReportMultiWrapper, Params: []string{"macro"}},
-	"cancelKeyboardReportMulti": {Func: rpcCancelKeyboardReportMulti},
+	// "cancelKeyboardReportMulti": {Func: rpcCancelKeyboardReportMulti},
 	"getKeyboardLedState":    {Func: rpcGetKeyboardLedState},
 	"keypressReport":         {Func: rpcKeypressReport, Params: []string{"key", "press"}},
 	"getKeyDownState":        {Func: rpcGetKeysDownState},

ui/src/hooks/hidRpc.ts

@@ -7,6 +7,7 @@ export const HID_RPC_MESSAGE_TYPES = {
     WheelReport: 0x04,
     KeypressReport: 0x05,
     MouseReport: 0x06,
+    KeyboardMacroReport: 0x07,
     KeyboardLedState: 0x32,
     KeysDownState: 0x33,
 }
@@ -32,6 +33,30 @@ const fromInt32toUint8 = (n: number) => {
     ]);
 };
 
+const fromUint16toUint8 = (n: number) => {
+    if (n > 65535 || n < 0) {
+        throw new Error(`Number ${n} is not within the uint16 range`);
+    }
+
+    return new Uint8Array([
+        (n >> 8) & 0xFF,
+        (n >> 0) & 0xFF,
+    ]);
+};
+
+const fromUint32toUint8 = (n: number) => {
+    if (n > 4294967295 || n < 0) {
+        throw new Error(`Number ${n} is not within the uint32 range`);
+    }
+
+    return new Uint8Array([
+        (n >> 24) & 0xFF,
+        (n >> 16) & 0xFF,
+        (n >> 8) & 0xFF,
+        (n >> 0) & 0xFF,
+    ]);
+};
+
 const fromInt8ToUint8 = (n: number) => {
     if (n < -128 || n > 127) {
         throw new Error(`Number ${n} is not within the int8 range`);
@@ -186,6 +211,64 @@ export class KeyboardReportMessage extends RpcMessage {
     }
 }
 
+export interface KeyboardMacro extends KeysDownState {
+    delay: number;
+}
+
+export class KeyboardMacroReportMessage extends RpcMessage {
+    isPaste: boolean;
+    length: number;
+    macro: KeyboardMacro[];
+
+    KEYS_LENGTH = 6;
+
+    constructor(isPaste: boolean, length: number, macro: KeyboardMacro[]) {
+        super(HID_RPC_MESSAGE_TYPES.KeyboardMacroReport);
+        this.isPaste = isPaste;
+        this.length = length;
+        this.macro = macro;
+    }
+
+    marshal(): Uint8Array {
+        const dataHeader = new Uint8Array([
+            this.messageType,
+            this.isPaste ? 1 : 0,
+            ...fromUint32toUint8(this.length),
+        ]);
+
+        let dataBody = new Uint8Array();
+
+        for (const step of this.macro) {
+            if (!withinUint8Range(step.modifier)) {
+                throw new Error(`Modifier ${step.modifier} is not within the uint8 range`);
+            }
+
+            // Ensure the keys are within the KEYS_LENGTH range
+            const keys = step.keys;
+            if (keys.length > this.KEYS_LENGTH) {
+                throw new Error(`Keys ${keys} is not within the hidKeyBufferSize range`);
+            } else if (keys.length < this.KEYS_LENGTH) {
+                keys.push(...Array(this.KEYS_LENGTH - keys.length).fill(0));
+            }
+
+            for (const key of keys) {
+                if (!withinUint8Range(key)) {
+                    throw new Error(`Key ${key} is not within the uint8 range`);
+                }
+            }
+
+            const macroBinary = new Uint8Array([
+                step.modifier,
+                ...keys,
+                ...fromUint16toUint8(step.delay),
+            ]);
+
+            dataBody = new Uint8Array([...dataBody, ...macroBinary]);
+        }
+        return new Uint8Array([...dataHeader, ...dataBody]);
+    }
+}
+
 export class KeyboardLedStateMessage extends RpcMessage {
     keyboardLedState: KeyboardLedState;
 

ui/src/hooks/useHidRpc.ts

@@ -5,6 +5,8 @@ import { useRTCStore } from "@/hooks/stores";
 import {
   HID_RPC_VERSION,
   HandshakeMessage,
+  KeyboardMacro,
+  KeyboardMacroReportMessage,
   KeyboardReportMessage,
   KeypressReportMessage,
   MouseReportMessage,
@@ -68,6 +70,15 @@ export function useHidRpc(onHidRpcMessage?: (payload: RpcMessage) => void) {
     [sendMessage],
   );
 
+  const reportKeyboardMacroEvent = useCallback(
+    (macro: KeyboardMacro[]) => {
+      const d = new KeyboardMacroReportMessage(false, macro.length, macro);
+      sendMessage(d);
+      console.log("Sent keyboard macro report", d, d.marshal());
+    },
+    [sendMessage],
+  );
+
   const sendHandshake = useCallback(() => {
     if (rpcHidProtocolVersion) return;
     if (!rpcHidChannel) return;
@@ -143,6 +154,7 @@ export function useHidRpc(onHidRpcMessage?: (payload: RpcMessage) => void) {
     reportKeypressEvent,
     reportAbsMouseEvent,
     reportRelMouseEvent,
+    reportKeyboardMacroEvent,
     rpcHidProtocolVersion,
     rpcHidReady,
     rpcHidStatus,

ui/src/hooks/useKeyboard.ts

@@ -1,4 +1,4 @@
-import { useCallback } from "react";
+import { useCallback, useMemo } from "react";
 
 import {
   hidErrorRollOver,
@@ -9,7 +9,7 @@ import {
 } from "@/hooks/stores";
 import { JsonRpcResponse, useJsonRpc } from "@/hooks/useJsonRpc";
 import { useHidRpc } from "@/hooks/useHidRpc";
-import { KeyboardLedStateMessage, KeysDownStateMessage } from "@/hooks/hidRpc";
+import { KeyboardLedStateMessage, KeyboardMacro, KeysDownStateMessage } from "@/hooks/hidRpc";
 import { hidKeyToModifierMask, keys, modifiers } from "@/keyboardMappings";
 
 export default function useKeyboard() {
@@ -32,6 +32,7 @@ export default function useKeyboard() {
   const {
     reportKeyboardEvent: sendKeyboardEventHidRpc,
     reportKeypressEvent: sendKeypressEventHidRpc,
+    reportKeyboardMacroEvent: sendKeyboardMacroEventHidRpc,
     rpcHidReady,
   } = useHidRpc(message => {
     switch (message.constructor) {
@@ -77,16 +78,19 @@ export default function useKeyboard() {
     [rpcDataChannel?.readyState, rpcHidReady, send, sendKeyboardEventHidRpc],
   );
 
+  const MACRO_RESET_KEYBOARD_STATE = useMemo(() => ({
+    keys: new Array(hidKeyBufferSize).fill(0),
+    modifier: 0,
+    delay: 0,
+  }), []);
+
   // resetKeyboardState is used to reset the keyboard state to no keys pressed and no modifiers.
   // This is useful for macros and when the browser loses focus to ensure that the keyboard state
   // is clean.
   const resetKeyboardState = useCallback(async () => {
     // Reset the keys buffer to zeros and the modifier state to zero
-    keysDownState.keys.length = hidKeyBufferSize;
+    sendKeyboardEvent(MACRO_RESET_KEYBOARD_STATE);
-    keysDownState.keys.fill(0);
+  }, [sendKeyboardEvent, MACRO_RESET_KEYBOARD_STATE]);
-    keysDownState.modifier = 0;
-    sendKeyboardEvent(keysDownState);
-  }, [keysDownState, sendKeyboardEvent]);
 
   // executeMacro is used to execute a macro consisting of multiple steps.
   // Each step can have multiple keys, multiple modifiers and a delay.
@@ -97,7 +101,7 @@ export default function useKeyboard() {
   const executeMacro = async (
     steps: { keys: string[] | null; modifiers: string[] | null; delay: number }[],
   ) => {
-    const macro: KeysDownState[] = [];
+    const macro: KeyboardMacro[] = [];
 
     for (const [_, step] of steps.entries()) {
       const keyValues = (step.keys || []).map(key => keys[key]).filter(Boolean);
@@ -107,19 +111,12 @@ export default function useKeyboard() {
 
       // If the step has keys and/or modifiers, press them and hold for the delay
       if (keyValues.length > 0 || modifierMask > 0) {
-        macro.push({ keys: keyValues, modifier: modifierMask });
+        macro.push({ keys: keyValues, modifier: modifierMask, delay: 50 });
-        keysDownState.keys.length = hidKeyBufferSize;
+        macro.push({ ...MACRO_RESET_KEYBOARD_STATE, delay: 200 });
-        keysDownState.keys.fill(0);
-        keysDownState.modifier = 0;
-        macro.push(keysDownState);
       }
     }
-    // KeyboardReportMessage
+
-    send("keyboardReportMulti", { macro }, (resp: JsonRpcResponse) => {
+    sendKeyboardMacroEventHidRpc(macro);
-      if ("error" in resp) {
-        console.error(`Failed to send keyboard report ${macro}`, resp.error);
-      }
-    });
   };
 
   const cancelExecuteMacro = useCallback(async () => {