Merge 854cdc8e82 into c8dd84c6b7

cloud.go

@@ -475,6 +475,10 @@ func handleSessionRequest(
 
 	cloudLogger.Info().Interface("session", session).Msg("new session accepted")
 	cloudLogger.Trace().Interface("session", session).Msg("new session accepted")
+
+	// Cancel any ongoing keyboard report multi when session changes
+	cancelKeyboardReportMulti()
+
 	currentSession = session
 	_ = wsjson.Write(context.Background(), c, gin.H{"type": "answer", "data": sd})
 	return nil

hidrpc.go

@@ -1,7 +1,9 @@
 package kvm
 
 import (
+	"errors"
 	"fmt"
+	"io"
 	"time"
 
 	"github.com/jetkvm/kvm/internal/hidrpc"
@@ -143,6 +145,10 @@ func reportHidRPC(params any, session *Session) {
 	}
 
 	if err := session.HidChannel.Send(message); err != nil {
+		if errors.Is(err, io.ErrClosedPipe) {
+			logger.Debug().Err(err).Msg("HID RPC channel closed, skipping reportHidRPC")
+			return
+		}
 		logger.Warn().Err(err).Msg("failed to send HID RPC message")
 	}
 }

jsonrpc.go

@@ -10,6 +10,7 @@ import (
 	"path/filepath"
 	"reflect"
 	"strconv"
+	"sync"
 	"time"
 
 	"github.com/pion/webrtc/v4"
@@ -1049,6 +1050,120 @@ func rpcSetLocalLoopbackOnly(enabled bool) error {
 	return nil
 }
 
+// cancelKeyboardReportMulti cancels any ongoing keyboard report multi execution
+func cancelKeyboardReportMulti() {
+	keyboardReportMultiLock.Lock()
+	defer keyboardReportMultiLock.Unlock()
+
+	if keyboardReportMultiCancel != nil {
+		keyboardReportMultiCancel()
+		logger.Info().Msg("canceled keyboard report multi")
+		keyboardReportMultiCancel = nil
+	}
+}
+
+func setKeyboardReportMultiCancel(cancel context.CancelFunc) {
+	keyboardReportMultiLock.Lock()
+	defer keyboardReportMultiLock.Unlock()
+
+	keyboardReportMultiCancel = cancel
+}
+
+func rpcKeyboardReportMultiWrapper(macro []map[string]any) (usbgadget.KeysDownState, error) {
+	cancelKeyboardReportMulti()
+
+	ctx, cancel := context.WithCancel(context.Background())
+	setKeyboardReportMultiCancel(cancel)
+
+	writeJSONRPCEvent("keyboardReportMultiState", true, currentSession)
+
+	result, err := rpcKeyboardReportMulti(ctx, macro)
+
+	setKeyboardReportMultiCancel(nil)
+
+	writeJSONRPCEvent("keyboardReportMultiState", false, currentSession)
+
+	return result, err
+}
+
+var (
+	keyboardReportMultiCancel context.CancelFunc
+	keyboardReportMultiLock   sync.Mutex
+)
+
+func rpcCancelKeyboardReportMulti() {
+	cancelKeyboardReportMulti()
+}
+
+func rpcKeyboardReportMulti(ctx context.Context, macro []map[string]any) (usbgadget.KeysDownState, error) {
+	var last usbgadget.KeysDownState
+	var err error
+
+	logger.Debug().Interface("macro", macro).Msg("Executing keyboard report multi")
+
+	for i, step := range macro {
+		// Check for cancellation before each step
+		select {
+		case <-ctx.Done():
+			logger.Debug().Msg("Keyboard report multi context cancelled")
+			return last, ctx.Err()
+		default:
+		}
+
+		var modifier byte
+		switch m := step["modifier"].(type) {
+		case uint8:
+			modifier = m
+		case int:
+			modifier = byte(m)
+		case float64:
+			modifier = byte(int(m))
+		default:
+			return last, fmt.Errorf("invalid modifier type: %T", m)
+		}
+
+		var keys []byte
+		switch k := step["keys"].(type) {
+		case []byte:
+			keys = k
+		case []any:
+			arr := k
+			keys = make([]byte, 0, len(arr))
+			for _, v := range arr {
+				switch f := v.(type) {
+				case uint8:
+					keys = append(keys, f)
+				case int:
+					keys = append(keys, byte(f))
+				case float64:
+					keys = append(keys, byte(int(f)))
+				default:
+					return last, fmt.Errorf("invalid key type: %T", f)
+				}
+			}
+		default:
+			return last, fmt.Errorf("invalid keys type: %T", k)
+		}
+
+		// Use context-aware sleep that can be cancelled
+		select {
+		case <-time.After(100 * time.Millisecond):
+			// Sleep completed normally
+		case <-ctx.Done():
+			logger.Debug().Int("step", i).Msg("Keyboard report multi cancelled during sleep")
+			return last, ctx.Err()
+		}
+
+		last, err = rpcKeyboardReport(modifier, keys)
+		if err != nil {
+			logger.Warn().Err(err).Msg("failed to execute keyboard report multi")
+			return last, err
+		}
+	}
+
+	return last, nil
+}
+
 var rpcHandlers = map[string]RPCHandler{
 	"ping":                      {Func: rpcPing},
 	"reboot":                    {Func: rpcReboot, Params: []string{"force"}},
@@ -1060,6 +1175,8 @@ var rpcHandlers = map[string]RPCHandler{
 	"setNetworkSettings":        {Func: rpcSetNetworkSettings, Params: []string{"settings"}},
 	"renewDHCPLease":            {Func: rpcRenewDHCPLease},
 	"keyboardReport":            {Func: rpcKeyboardReport, Params: []string{"modifier", "keys"}},
+	"keyboardReportMulti":       {Func: rpcKeyboardReportMultiWrapper, Params: []string{"macro"}},
+	"cancelKeyboardReportMulti": {Func: rpcCancelKeyboardReportMulti},
 	"getKeyboardLedState":       {Func: rpcGetKeyboardLedState},
 	"keypressReport":            {Func: rpcKeypressReport, Params: []string{"key", "press"}},
 	"getKeyDownState":           {Func: rpcGetKeysDownState},

ui/src/components/InfoBar.tsx

@@ -27,6 +27,7 @@ export default function InfoBar() {
 
   const { rpcDataChannel } = useRTCStore();
   const { debugMode, mouseMode, showPressedKeys } = useSettingsStore();
+  const { isPasteModeEnabled } = useHidStore();
 
   useEffect(() => {
     if (!rpcDataChannel) return;
@@ -108,7 +109,12 @@ export default function InfoBar() {
                 <span className="text-xs">{rpcHidStatus}</span>
               </div>
             )}
-
+            {isPasteModeEnabled && (
+              <div className="flex w-[156px] items-center gap-x-1">
+                <span className="text-xs font-semibold">Paste Mode:</span>
+                <span className="text-xs">Enabled</span>
+              </div>
+            )}
             {showPressedKeys && (
               <div className="flex items-center gap-x-1">
                 <span className="text-xs font-semibold">Keys:</span>

ui/src/components/popovers/PasteModal.tsx

@@ -8,29 +8,18 @@ import { GridCard } from "@components/Card";
 import { TextAreaWithLabel } from "@components/TextArea";
 import { SettingsPageHeader } from "@components/SettingsPageheader";
 import { JsonRpcResponse, useJsonRpc } from "@/hooks/useJsonRpc";
-import { useHidStore, useRTCStore, useUiStore, useSettingsStore } from "@/hooks/stores";
+import { useHidStore, useSettingsStore, useUiStore } from "@/hooks/stores";
-import { keys, modifiers } from "@/keyboardMappings";
+import useKeyboard from "@/hooks/useKeyboard";
-import { KeyStroke } from "@/keyboardLayouts";
 import useKeyboardLayout from "@/hooks/useKeyboardLayout";
 import notifications from "@/notifications";
 
-const hidKeyboardPayload = (modifier: number, keys: number[]) => {
-  return { modifier, keys };
-};
-
-const modifierCode = (shift?: boolean, altRight?: boolean) => {
-  return (shift ? modifiers.ShiftLeft : 0)
-       | (altRight ? modifiers.AltRight : 0)
-}
-const noModifier = 0
-
 export default function PasteModal() {
   const TextAreaRef = useRef<HTMLTextAreaElement>(null);
-  const { setPasteModeEnabled } = useHidStore();
+  const { isPasteModeEnabled } = useHidStore();
   const { setDisableVideoFocusTrap } = useUiStore();
 
   const { send } = useJsonRpc();
-  const { rpcDataChannel } = useRTCStore();
+  const { executeMacro, cancelExecuteMacro } = useKeyboard();
 
   const [invalidChars, setInvalidChars] = useState<string[]>([]);
   const close = useClose();
@@ -46,21 +35,26 @@ export default function PasteModal() {
   }, [send, setKeyboardLayout]);
 
   const onCancelPasteMode = useCallback(() => {
-    setPasteModeEnabled(false);
+    cancelExecuteMacro();
     setDisableVideoFocusTrap(false);
     setInvalidChars([]);
-  }, [setDisableVideoFocusTrap, setPasteModeEnabled]);
+  }, [setDisableVideoFocusTrap, cancelExecuteMacro]);
 
   const onConfirmPaste = useCallback(async () => {
-    setPasteModeEnabled(false);
+    // setPasteModeEnabled(false);
-    setDisableVideoFocusTrap(false);
+    // setDisableVideoFocusTrap(false);
 
-    if (rpcDataChannel?.readyState !== "open" || !TextAreaRef.current) return;
+    if (!TextAreaRef.current || !selectedKeyboard) return;
-    if (!selectedKeyboard) return;
 
     const text = TextAreaRef.current.value;
 
     try {
+      const macroSteps: {
+        keys: string[] | null;
+        modifiers: string[] | null;
+        delay: number;
+      }[] = [];
+
       for (const char of text) {
         const keyprops = selectedKeyboard.chars[char];
         if (!keyprops) continue;
@@ -70,39 +64,41 @@ export default function PasteModal() {
 
         // if this is an accented character, we need to send that accent FIRST
         if (accentKey) {
-          await sendKeystroke({modifier: modifierCode(accentKey.shift, accentKey.altRight), keys: [ keys[accentKey.key] ] })
+          const accentModifiers: string[] = [];
+          if (accentKey.shift) accentModifiers.push("ShiftLeft");
+          if (accentKey.altRight) accentModifiers.push("AltRight");
+
+          macroSteps.push({
+            keys: [String(accentKey.key)],
+            modifiers: accentModifiers.length > 0 ? accentModifiers : null,
+            delay: 100,
+          });
         }
 
         // now send the actual key
-        await sendKeystroke({ modifier: modifierCode(shift, altRight), keys: [ keys[key] ]});
+        const modifiers: string[] = [];
+        if (shift) modifiers.push("ShiftLeft");
+        if (altRight) modifiers.push("AltRight");
+
+        macroSteps.push({
+          keys: [String(key)],
+          modifiers: modifiers.length > 0 ? modifiers : null,
+          delay: 100,
+        });
 
         // if what was requested was a dead key, we need to send an unmodified space to emit
         // just the accent character
-        if (deadKey) {
+        if (deadKey) macroSteps.push({ keys: ["Space"], modifiers: null, delay: 100 });
-           await sendKeystroke({ modifier: noModifier, keys: [ keys["Space"] ] });
       }
 
-        // now send a message with no keys down to "release" the keys
+      if (macroSteps.length > 0) {
-        await sendKeystroke({ modifier: 0, keys: [] });
+        await executeMacro(macroSteps);
       }
     } catch (error) {
       console.error("Failed to paste text:", error);
       notifications.error("Failed to paste text");
     }
-
+  }, [selectedKeyboard, executeMacro]);
-    async function sendKeystroke(stroke: KeyStroke) {
-      await new Promise<void>((resolve, reject) => {
-        send(
-          "keyboardReport",
-          hidKeyboardPayload(stroke.modifier, stroke.keys),
-          params => {
-            if ("error" in params) return reject(params.error);
-            resolve();
-          }
-        );
-      });
-    }
-  }, [selectedKeyboard, rpcDataChannel?.readyState, send, setDisableVideoFocusTrap, setPasteModeEnabled]);
 
   useEffect(() => {
     if (TextAreaRef.current) {
@@ -122,14 +118,18 @@ export default function PasteModal() {
               />
 
               <div
-                className="animate-fadeIn opacity-0 space-y-2"
+                className="animate-fadeIn space-y-2 opacity-0"
                 style={{
                   animationDuration: "0.7s",
                   animationDelay: "0.1s",
                 }}
               >
                 <div>
-                  <div className="w-full" onKeyUp={e => e.stopPropagation()} onKeyDown={e => e.stopPropagation()}>
+                  <div
+                    className="w-full"
+                    onKeyUp={e => e.stopPropagation()}
+                    onKeyDown={e => e.stopPropagation()}
+                  >
                     <TextAreaWithLabel
                       ref={TextAreaRef}
                       label="Paste from host"
@@ -173,7 +173,8 @@ export default function PasteModal() {
                 </div>
                 <div className="space-y-4">
                   <p className="text-xs text-slate-600 dark:text-slate-400">
-                    Sending text using keyboard layout: {selectedKeyboard.isoCode}-{selectedKeyboard.name}
+                    Sending text using keyboard layout: {selectedKeyboard.isoCode}-
+                    {selectedKeyboard.name}
                   </p>
                 </div>
               </div>
@@ -181,7 +182,7 @@ export default function PasteModal() {
           </div>
         </div>
         <div
-          className="flex animate-fadeIn opacity-0 items-center justify-end gap-x-2"
+          className="flex animate-fadeIn items-center justify-end gap-x-2 opacity-0"
           style={{
             animationDuration: "0.7s",
             animationDelay: "0.2s",
@@ -200,6 +201,7 @@ export default function PasteModal() {
             size="SM"
             theme="primary"
             text="Confirm Paste"
+            disabled={isPasteModeEnabled}
             onClick={onConfirmPaste}
             LeadingIcon={LuCornerDownLeft}
           />

ui/src/hooks/useKeyboard.ts

@@ -1,6 +1,12 @@
 import { useCallback } from "react";
 
-import { hidErrorRollOver, hidKeyBufferSize, KeysDownState, useHidStore, useRTCStore } from "@/hooks/stores";
+import {
+  hidErrorRollOver,
+  hidKeyBufferSize,
+  KeysDownState,
+  useHidStore,
+  useRTCStore,
+} from "@/hooks/stores";
 import { JsonRpcResponse, useJsonRpc } from "@/hooks/useJsonRpc";
 import { useHidRpc } from "@/hooks/useHidRpc";
 import { KeyboardLedStateMessage, KeysDownStateMessage } from "@/hooks/hidRpc";
@@ -27,7 +33,7 @@ export default function useKeyboard() {
     reportKeyboardEvent: sendKeyboardEventHidRpc,
     reportKeypressEvent: sendKeypressEventHidRpc,
     rpcHidReady,
-  } = useHidRpc((message) => {
+  } = useHidRpc(message => {
     switch (message.constructor) {
       case KeysDownStateMessage:
         setKeysDownState((message as KeysDownStateMessage).keysDownState);
@@ -48,7 +54,9 @@ export default function useKeyboard() {
     async (state: KeysDownState) => {
       if (rpcDataChannel?.readyState !== "open" && !rpcHidReady) return;
 
-      console.debug(`Send keyboardReport keys: ${state.keys}, modifier: ${state.modifier}`);
+      console.debug(
+        `Send keyboardReport keys: ${state.keys}, modifier: ${state.modifier}`,
+      );
 
       if (rpcHidReady) {
         console.debug("Sending keyboard report via HidRPC");
@@ -56,25 +64,23 @@ export default function useKeyboard() {
         return;
       }
 
-      send("keyboardReport", { keys: state.keys, modifier: state.modifier }, (resp: JsonRpcResponse) => {
+      send(
+        "keyboardReport",
+        { keys: state.keys, modifier: state.modifier },
+        (resp: JsonRpcResponse) => {
           if ("error" in resp) {
             console.error(`Failed to send keyboard report ${state}`, resp.error);
           }
-      });
         },
-    [
+      );
-      rpcDataChannel?.readyState,
+    },
-      rpcHidReady,
+    [rpcDataChannel?.readyState, rpcHidReady, send, sendKeyboardEventHidRpc],
-      send,
-      sendKeyboardEventHidRpc,
-    ],
   );
 
   // resetKeyboardState is used to reset the keyboard state to no keys pressed and no modifiers.
   // This is useful for macros and when the browser loses focus to ensure that the keyboard state
   // is clean.
-  const resetKeyboardState = useCallback(
+  const resetKeyboardState = useCallback(async () => {
-    async () => {
     // Reset the keys buffer to zeros and the modifier state to zero
     keysDownState.keys.length = hidKeyBufferSize;
     keysDownState.keys.fill(0);
@@ -88,29 +94,42 @@ export default function useKeyboard() {
   // After the delay, the keys and modifiers are released and the next step is executed.
   // If a step has no keys or modifiers, it is treated as a delay-only step.
   // A small pause is added between steps to ensure that the device can process the events.
-  const executeMacro = async (steps: { keys: string[] | null; modifiers: string[] | null; delay: number }[]) => {
+  const executeMacro = async (
-    for (const [index, step] of steps.entries()) {
+    steps: { keys: string[] | null; modifiers: string[] | null; delay: number }[],
+  ) => {
+    const macro: KeysDownState[] = [];
+
+    for (const [_, step] of steps.entries()) {
       const keyValues = (step.keys || []).map(key => keys[key]).filter(Boolean);
-      const modifierMask: number = (step.modifiers || []).map(mod => modifiers[mod]).reduce((acc, val) => acc + val, 0);
+      const modifierMask: number = (step.modifiers || [])
+        .map(mod => modifiers[mod])
+        .reduce((acc, val) => acc + val, 0);
 
       // If the step has keys and/or modifiers, press them and hold for the delay
       if (keyValues.length > 0 || modifierMask > 0) {
-        sendKeyboardEvent({ keys: keyValues, modifier: modifierMask });
+        macro.push({ keys: keyValues, modifier: modifierMask });
-        await new Promise(resolve => setTimeout(resolve, step.delay || 50));
+        keysDownState.keys.length = hidKeyBufferSize;
-
+        keysDownState.keys.fill(0);
-        resetKeyboardState();
+        keysDownState.modifier = 0;
-      } else {
+        macro.push(keysDownState);
-        // This is a delay-only step, just wait for the delay amount
-        await new Promise(resolve => setTimeout(resolve, step.delay || 50));
-      }
-
-      // Add a small pause between steps if not the last step
-      if (index < steps.length - 1) {
-        await new Promise(resolve => setTimeout(resolve, 10));
       }
     }
+    // KeyboardReportMessage
+    send("keyboardReportMulti", { macro }, (resp: JsonRpcResponse) => {
+      if ("error" in resp) {
+        console.error(`Failed to send keyboard report ${macro}`, resp.error);
+      }
+    });
   };
 
+  const cancelExecuteMacro = useCallback(async () => {
+    send("cancelKeyboardReportMulti", {}, (resp: JsonRpcResponse) => {
+      if ("error" in resp) {
+        console.error(`Failed to cancel keyboard report multi`, resp.error);
+      }
+    });
+  }, [send]);
+
   // handleKeyPress is used to handle a key press or release event.
   // This function handle both key press and key release events.
   // It checks if the keyPressReport API is available and sends the key press event.
@@ -132,7 +151,11 @@ export default function useKeyboard() {
         sendKeypressEventHidRpc(key, press);
       } else {
         // if the keyPress api is not available, we need to handle the key locally
-        const downState = simulateDeviceSideKeyHandlingForLegacyDevices(keysDownState, key, press);
+        const downState = simulateDeviceSideKeyHandlingForLegacyDevices(
+          keysDownState,
+          key,
+          press,
+        );
         sendKeyboardEvent(downState); // then we send the full state
 
         // if we just sent ErrorRollOver, reset to empty state
@@ -152,7 +175,11 @@ export default function useKeyboard() {
   );
 
   // IMPORTANT: See the keyPressReportApiAvailable comment above for the reason this exists
-  function simulateDeviceSideKeyHandlingForLegacyDevices(state: KeysDownState, key: number, press: boolean): KeysDownState {
+  function simulateDeviceSideKeyHandlingForLegacyDevices(
+    state: KeysDownState,
+    key: number,
+    press: boolean,
+  ): KeysDownState {
     // IMPORTANT: This code parallels the logic in the kernel's hid-gadget driver
     // for handling key presses and releases. It ensures that the USB gadget
     // behaves similarly to a real USB HID keyboard. This logic is paralleled
@@ -181,7 +208,7 @@ export default function useKeyboard() {
         // and if we find a zero byte, we can place the key there (if press is true)
         if (keys[i] === key || keys[i] === 0) {
           if (press) {
-            keys[i] = key // overwrites the zero byte or the same key if already pressed
+            keys[i] = key; // overwrites the zero byte or the same key if already pressed
           } else {
             // we are releasing the key, remove it from the buffer
             if (keys[i] !== 0) {
@@ -197,18 +224,20 @@ export default function useKeyboard() {
       // If we reach here it means we didn't find an empty slot or the key in the buffer
       if (overrun) {
         if (press) {
-          console.warn(`keyboard buffer overflow current keys ${keys}, key: ${key} not added`);
+          console.warn(
+            `keyboard buffer overflow current keys ${keys}, key: ${key} not added`,
+          );
           // Fill all key slots with ErrorRollOver (0x01) to indicate overflow
           keys.length = hidKeyBufferSize;
           keys.fill(hidErrorRollOver);
         } else {
           // If we are releasing a key, and we didn't find it in a slot, who cares?
-          console.debug(`key ${key} not found in buffer, nothing to release`)
+          console.debug(`key ${key} not found in buffer, nothing to release`);
         }
       }
     }
     return { modifier: modifiers, keys };
   }
 
-  return { handleKeyPress, resetKeyboardState, executeMacro };
+  return { handleKeyPress, resetKeyboardState, executeMacro, cancelExecuteMacro };
 }

ui/src/routes/devices.$id.tsx

@@ -580,7 +580,7 @@ export default function KvmIdRoute() {
   const { setNetworkState} = useNetworkStateStore();
   const { setHdmiState } = useVideoStore();
   const { 
-    keyboardLedState,  setKeyboardLedState,
+    keyboardLedState,  setKeyboardLedState, setPasteModeEnabled,
     keysDownState, setKeysDownState, setUsbState,
   } = useHidStore();
 
@@ -598,6 +598,12 @@ export default function KvmIdRoute() {
       setUsbState(usbState);
     }
 
+    if (resp.method === "keyboardReportMultiState") {
+      const reportMultiState = resp.params as unknown as boolean;
+      console.debug("Setting keyboard report multi state", reportMultiState);
+      setPasteModeEnabled(reportMultiState);
+    }
+
     if (resp.method === "videoInputState") {
       const hdmiState = resp.params as Parameters<VideoState["setHdmiState"]>[0];
       console.debug("Setting HDMI state", hdmiState);

web.go

@@ -198,6 +198,10 @@ func handleWebRTCSession(c *gin.Context) {
 			_ = peerConn.Close()
 		}()
 	}
+
+	// Cancel any ongoing keyboard report multi when session changes
+	cancelKeyboardReportMulti()
+
 	currentSession = session
 	c.JSON(http.StatusOK, gin.H{"sd": sd})
 }

webrtc.go

@@ -266,6 +266,8 @@ func newSession(config SessionConfig) (*Session, error) {
 		if connectionState == webrtc.ICEConnectionStateClosed {
 			scopedLogger.Debug().Msg("ICE Connection State is closed, unmounting virtual media")
 			if session == currentSession {
+				// Cancel any ongoing keyboard report multi when session closes
+				cancelKeyboardReportMulti()
 				currentSession = nil
 			}
 			// Stop RPC processor