refactor: use less data to transfer keyboard and mouse events [WIP DO NOT MERGE]

dev_deploy.sh

@@ -159,10 +159,10 @@ else
 	msg_info "▶ Building development binary"
 	make build_dev
 	
-	# Kill any existing instances of the application
+	msg_info "▶ Killing any existing instances of the application"
 	ssh "${REMOTE_USER}@${REMOTE_HOST}" "killall jetkvm_app_debug || true"
 	
-	# Copy the binary to the remote host
+	msg_info "▶ Copying binary to remote host"
 	ssh "${REMOTE_USER}@${REMOTE_HOST}" "cat > ${REMOTE_PATH}/jetkvm_app_debug" < bin/jetkvm_app
 	
 	if [ "$RESET_USB_HID_DEVICE" = true ]; then
@@ -173,7 +173,7 @@ else
 	ssh "${REMOTE_USER}@${REMOTE_HOST}" "ls /sys/class/udc > /sys/kernel/config/usb_gadget/jetkvm/UDC"
 	fi
 	
-	# Deploy and run the application on the remote host
+	msg_info "▶ Deploying and running the application on the remote host"
 	ssh "${REMOTE_USER}@${REMOTE_HOST}" ash << EOF
 set -e
 

hid.go

@@ -0,0 +1,61 @@
+package kvm
+
+import (
+	"encoding/binary"
+
+	"github.com/pion/webrtc/v4"
+)
+
+type HidPayloadType uint8
+
+const (
+	HidPayloadTypeKeyboardReport           HidPayloadType = 1
+	HidPayloadTypeKeyboardReportNoModifier HidPayloadType = 2
+	HidPayloadTypeKeyboardReportNothing    HidPayloadType = 3
+	HidPayloadTypeAbsMouseReport           HidPayloadType = 8
+	HidPayloadTypeRelMouseReport           HidPayloadType = 9
+)
+
+func handleHidMessage(msg webrtc.DataChannelMessage) {
+	if msg.IsString {
+		webrtcLogger.Info().Interface("msg", msg.Data).Msg("Hid message is a string, skipping")
+		return
+	}
+
+	payloadType := HidPayloadType(msg.Data[0])
+	switch payloadType {
+	case HidPayloadTypeKeyboardReport:
+		if len(msg.Data) < 3 {
+			return
+		}
+		modifier := msg.Data[1]
+		keys := msg.Data[2:]
+		_ = gadget.KeyboardReport(modifier, keys)
+	case HidPayloadTypeKeyboardReportNoModifier:
+		if len(msg.Data) < 2 {
+			return
+		}
+		keys := msg.Data[1:]
+		_ = gadget.KeyboardReport(0, keys)
+	case HidPayloadTypeKeyboardReportNothing:
+		_ = gadget.KeyboardReport(0, []byte{})
+	case HidPayloadTypeAbsMouseReport:
+		if len(msg.Data) < 6 {
+			return
+		}
+		x := binary.LittleEndian.Uint16(msg.Data[1:3])
+		y := binary.LittleEndian.Uint16(msg.Data[3:5])
+		buttons := msg.Data[5]
+		webrtcLogger.Info().Uint16("x", x).Uint16("y", y).Uint8("buttons", buttons).Msg("Absolute mouse report")
+		_ = gadget.AbsMouseReport(int(x), int(y), buttons)
+	case HidPayloadTypeRelMouseReport:
+		if len(msg.Data) < 4 {
+			return
+		}
+		dx := int8(msg.Data[1])
+		dy := int8(msg.Data[2])
+		buttons := msg.Data[3]
+		webrtcLogger.Info().Int8("dx", dx).Int8("dy", dy).Uint8("buttons", buttons).Msg("Relative mouse report")
+		_ = gadget.RelMouseReport(dx, dy, buttons)
+	}
+}

internal/usbgadget/hid_keyboard.go

@@ -265,6 +265,8 @@ func (u *UsbGadget) KeyboardReport(modifier byte, keys []byte) (KeysDownState, e
 	defer u.keyboardLock.Unlock()
 	defer u.resetUserInputTime()
 
+	u.log.Trace().Uint8("modifier", modifier).Bytes("keys", keys).Msg("KeyboardReport")
+
 	if len(keys) > hidKeyBufferSize {
 		keys = keys[:hidKeyBufferSize]
 	}

internal/usbgadget/hid_mouse_absolute.go

@@ -89,6 +89,8 @@ func (u *UsbGadget) AbsMouseReport(x int, y int, buttons uint8) error {
 	u.absMouseLock.Lock()
 	defer u.absMouseLock.Unlock()
 
+	u.log.Trace().Int("x", x).Int("y", y).Uint8("buttons", buttons).Msg("AbsMouseReport")
+
 	err := u.absMouseWriteHidFile([]byte{
 		1,            // Report ID 1
 		buttons,      // Buttons
@@ -109,6 +111,8 @@ func (u *UsbGadget) AbsMouseWheelReport(wheelY int8) error {
 	u.absMouseLock.Lock()
 	defer u.absMouseLock.Unlock()
 
+	u.log.Trace().Int8("wheelY", wheelY).Msg("AbsMouseWheelReport")
+
 	// Only send a report if the value is non-zero
 	if wheelY == 0 {
 		return nil

ui/src/components/WebRTCVideo.tsx

@@ -50,7 +50,7 @@ export default function WebRTCVideo() {
 
   // RTC related states
   const { peerConnection } = useRTCStore();
-
+  const hidDataChannel = useRTCStore(state => state.hidDataChannel);
   // HDMI and UI states
   const hdmiError = ["no_lock", "no_signal", "out_of_range"].includes(hdmiState);
   const isVideoLoading = !isPlaying;
@@ -243,11 +243,21 @@ export default function WebRTCVideo() {
   const sendAbsMouseMovement = useCallback(
     (x: number, y: number, buttons: number) => {
       if (settings.mouseMode !== "absolute") return;
-      send("absMouseReport", { x, y, buttons });
+      if (hidDataChannel?.readyState === "open") {
+        const buffer = new ArrayBuffer(6);
+        const view = new DataView(buffer);
+        view.setUint8(0, 8); // type
+        view.setUint16(1, x, true); // x, little-endian
+        view.setUint16(3, y, true); // y, little-endian
+        view.setUint8(5, buttons); // buttons
+        hidDataChannel.send(buffer);
+      } else {
+        send("absMouseReport", { x, y, buttons });
+      }
       // We set that for the debug info bar
       setMousePosition(x, y);
     },
-    [send, setMousePosition, settings.mouseMode],
+    [hidDataChannel?.readyState, send, setMousePosition, settings.mouseMode],
   );
 
   const absMouseMoveHandler = useCallback(

ui/src/hooks/stores.ts

@@ -142,6 +142,9 @@ export interface RTCState {
 
   terminalChannel: RTCDataChannel | null;
   setTerminalChannel: (channel: RTCDataChannel) => void;
+
+  hidDataChannel: RTCDataChannel | null;
+  setHidDataChannel: (channel: RTCDataChannel) => void;
 }
 
 export const useRTCStore = create<RTCState>(set => ({
@@ -151,6 +154,9 @@ export const useRTCStore = create<RTCState>(set => ({
   rpcDataChannel: null,
   setRpcDataChannel: (channel: RTCDataChannel) => set({ rpcDataChannel: channel }),
 
+  hidDataChannel: null,
+  setHidDataChannel: channel => set({ hidDataChannel: channel }),
+
   transceiver: null,
   setTransceiver: (transceiver: RTCRtpTransceiver) => set({ transceiver }),
 

ui/src/hooks/useKeyboard.ts

@@ -25,6 +25,15 @@ export default function useKeyboard() {
   // and resetting keyboard state. It sends the keys currently pressed and the modifier state.
   // The device will respond with the keysDownState if it supports the keyPressReport API
   // or just accept the state if it does not support (returning no result)
+  const [send] = useJsonRpc();
+
+  const rpcDataChannel = useRTCStore(state => state.rpcDataChannel);
+  const hidDataChannel = useRTCStore(state => state.hidDataChannel);
+
+  const updateActiveKeysAndModifiers = useHidStore(
+    state => state.updateActiveKeysAndModifiers,
+  );
+
   const sendKeyboardEvent = useCallback(
     async (state: KeysDownState) => {
       if (rpcDataChannel?.readyState !== "open") return;
@@ -82,6 +91,29 @@ export default function useKeyboard() {
           }
         }
       });
+    (keys: number[], modifiers: number[]) => {
+      const rpcChannelReady = rpcDataChannel?.readyState === "open";
+      const hidChannelReady = hidDataChannel?.readyState === "open";
+      if (!rpcChannelReady && !hidChannelReady) return;
+
+      const accModifier = modifiers.reduce((acc, val) => acc + val, 0);
+
+      if (hidChannelReady) {
+        if (accModifier > 0) {
+          hidDataChannel?.send(new Uint8Array([1, accModifier, ...keys]));
+        } else {
+          if (keys.length > 0) {
+            hidDataChannel?.send(new Uint8Array([2, ...keys]));
+          } else {
+            hidDataChannel?.send(new Uint8Array([3]));
+          }
+        }
+      } else {
+        send("keyboardReport", { keys, modifier: accModifier });
+      }
+
+      // We do this for the info bar to display the currently pressed keys for the user
+      updateActiveKeysAndModifiers({ keys: keys, modifiers: modifiers });
     },
     [rpcDataChannel?.readyState, send, setkeyPressReportApiAvailable, setKeysDownState],
   );
@@ -97,6 +129,17 @@ export default function useKeyboard() {
       keysDownState.modifier = 0;
       sendKeyboardEvent(keysDownState);
     }, [keysDownState, sendKeyboardEvent]);
+    [
+      hidDataChannel?.readyState,
+      rpcDataChannel?.readyState,
+      send,
+      updateActiveKeysAndModifiers,
+    ],
+  );
+
+  const resetKeyboardState = useCallback(() => {
+    sendKeyboardEvent([], []);
+  }, [sendKeyboardEvent]);
 
   // executeMacro is used to execute a macro consisting of multiple steps.
   // Each step can have multiple keys, multiple modifiers and a delay.

ui/src/routes/devices.$id.tsx

@@ -482,6 +482,12 @@ export default function KvmIdRoute() {
       setRpcDataChannel(rpcDataChannel);
     };
 
+    const hidDataChannel = pc.createDataChannel("hid");
+    hidDataChannel.binaryType = "arraybuffer";
+    hidDataChannel.onopen = () => {
+      setHidDataChannel(hidDataChannel);
+    };
+
     setPeerConnection(pc);
   }, [
     cleanupAndStopReconnecting,
@@ -492,6 +498,7 @@ export default function KvmIdRoute() {
     setPeerConnection,
     setPeerConnectionState,
     setRpcDataChannel,
+    setHidDataChannel,
     setTransceiver,
   ]);
 

webrtc.go

@@ -125,6 +125,9 @@ func newSession(config SessionConfig) (*Session, error) {
 			triggerOTAStateUpdate()
 			triggerVideoStateUpdate()
 			triggerUSBStateUpdate()
+		case "hid":
+			session.HidChannel = d
+			d.OnMessage(handleHidMessage)
 		case "terminal":
 			handleTerminalChannel(d)
 		case "serial":