refactor: use less data to transfer keyboard and mouse events [WIP DO NOT MERGE]

dev_deploy.sh

@@ -142,9 +142,11 @@ fi
 msg_info "▶ Building go binary"
 make build_dev
 
+msg_info "▶ Killing any existing instances of the application"
 # Kill any existing instances of the application
 ssh "${REMOTE_USER}@${REMOTE_HOST}" "killall jetkvm_app_debug || true"
 
+msg_info "▶ Copying binary to remote host"
 # Copy the binary to the remote host
 ssh "${REMOTE_USER}@${REMOTE_HOST}" "cat > ${REMOTE_PATH}/jetkvm_app_debug" < bin/jetkvm_app
 
@@ -156,6 +158,7 @@ if [ "$RESET_USB_HID_DEVICE" = true ]; then
     ssh "${REMOTE_USER}@${REMOTE_HOST}" "ls /sys/class/udc > /sys/kernel/config/usb_gadget/jetkvm/UDC"
 fi
 
+msg_info "▶ Deploying and running the application on the remote host"
 # Deploy and run the application on the remote host
 ssh "${REMOTE_USER}@${REMOTE_HOST}" ash << EOF
 set -e

hid.go

@@ -0,0 +1,61 @@
+package kvm
+
+import (
+	"encoding/binary"
+
+	"github.com/pion/webrtc/v4"
+)
+
+type HidPayloadType uint8
+
+const (
+	HidPayloadTypeKeyboardReport           HidPayloadType = 1
+	HidPayloadTypeKeyboardReportNoModifier HidPayloadType = 2
+	HidPayloadTypeKeyboardReportNothing    HidPayloadType = 3
+	HidPayloadTypeAbsMouseReport           HidPayloadType = 8
+	HidPayloadTypeRelMouseReport           HidPayloadType = 9
+)
+
+func handleHidMessage(msg webrtc.DataChannelMessage) {
+	if msg.IsString {
+		webrtcLogger.Info().Interface("msg", msg.Data).Msg("Hid message is a string, skipping")
+		return
+	}
+
+	payloadType := HidPayloadType(msg.Data[0])
+	switch payloadType {
+	case HidPayloadTypeKeyboardReport:
+		if len(msg.Data) < 3 {
+			return
+		}
+		modifier := msg.Data[1]
+		keys := msg.Data[2:]
+		_ = gadget.KeyboardReport(modifier, keys)
+	case HidPayloadTypeKeyboardReportNoModifier:
+		if len(msg.Data) < 2 {
+			return
+		}
+		keys := msg.Data[1:]
+		_ = gadget.KeyboardReport(0, keys)
+	case HidPayloadTypeKeyboardReportNothing:
+		_ = gadget.KeyboardReport(0, []byte{})
+	case HidPayloadTypeAbsMouseReport:
+		if len(msg.Data) < 6 {
+			return
+		}
+		x := binary.LittleEndian.Uint16(msg.Data[1:3])
+		y := binary.LittleEndian.Uint16(msg.Data[3:5])
+		buttons := msg.Data[5]
+		webrtcLogger.Info().Uint16("x", x).Uint16("y", y).Uint8("buttons", buttons).Msg("Absolute mouse report")
+		_ = gadget.AbsMouseReport(int(x), int(y), buttons)
+	case HidPayloadTypeRelMouseReport:
+		if len(msg.Data) < 4 {
+			return
+		}
+		dx := int8(msg.Data[1])
+		dy := int8(msg.Data[2])
+		buttons := msg.Data[3]
+		webrtcLogger.Info().Int8("dx", dx).Int8("dy", dy).Uint8("buttons", buttons).Msg("Relative mouse report")
+		_ = gadget.RelMouseReport(dx, dy, buttons)
+	}
+}

internal/usbgadget/hid_keyboard.go

@@ -214,6 +214,8 @@ func (u *UsbGadget) KeyboardReport(modifier uint8, keys []uint8) error {
 	u.keyboardLock.Lock()
 	defer u.keyboardLock.Unlock()
 
+	u.log.Trace().Uint8("modifier", modifier).Bytes("keys", keys).Msg("KeyboardReport")
+
 	if len(keys) > 6 {
 		keys = keys[:6]
 	}

internal/usbgadget/hid_mouse_absolute.go

@@ -88,6 +88,8 @@ func (u *UsbGadget) AbsMouseReport(x, y int, buttons uint8) error {
 	u.absMouseLock.Lock()
 	defer u.absMouseLock.Unlock()
 
+	u.log.Trace().Int("x", x).Int("y", y).Uint8("buttons", buttons).Msg("AbsMouseReport")
+
 	err := u.absMouseWriteHidFile([]byte{
 		1,             // Report ID 1
 		buttons,       // Buttons
@@ -108,6 +110,8 @@ func (u *UsbGadget) AbsMouseWheelReport(wheelY int8) error {
 	u.absMouseLock.Lock()
 	defer u.absMouseLock.Unlock()
 
+	u.log.Trace().Int8("wheelY", wheelY).Msg("AbsMouseWheelReport")
+
 	// Only send a report if the value is non-zero
 	if wheelY == 0 {
 		return nil

ui/src/components/WebRTCVideo.tsx

@@ -65,7 +65,7 @@ export default function WebRTCVideo() {
 
   // RTC related states
   const peerConnection = useRTCStore(state => state.peerConnection);
-
+  const hidDataChannel = useRTCStore(state => state.hidDataChannel);
   // HDMI and UI states
   const hdmiState = useVideoStore(state => state.hdmiState);
   const hdmiError = ["no_lock", "no_signal", "out_of_range"].includes(hdmiState);
@@ -253,11 +253,21 @@ export default function WebRTCVideo() {
   const sendAbsMouseMovement = useCallback(
     (x: number, y: number, buttons: number) => {
       if (settings.mouseMode !== "absolute") return;
-      send("absMouseReport", { x, y, buttons });
+      if (hidDataChannel?.readyState === "open") {
+        const buffer = new ArrayBuffer(6);
+        const view = new DataView(buffer);
+        view.setUint8(0, 8); // type
+        view.setUint16(1, x, true); // x, little-endian
+        view.setUint16(3, y, true); // y, little-endian
+        view.setUint8(5, buttons); // buttons
+        hidDataChannel.send(buffer);
+      } else {
+        send("absMouseReport", { x, y, buttons });
+      }
       // We set that for the debug info bar
       setMousePosition(x, y);
     },
-    [send, setMousePosition, settings.mouseMode],
+    [hidDataChannel?.readyState, send, setMousePosition, settings.mouseMode],
   );
 
   const absMouseMoveHandler = useCallback(

ui/src/hooks/stores.ts

@@ -145,6 +145,9 @@ interface RTCState {
 
   terminalChannel: RTCDataChannel | null;
   setTerminalChannel: (channel: RTCDataChannel) => void;
+
+  hidDataChannel: RTCDataChannel | null;
+  setHidDataChannel: (channel: RTCDataChannel) => void;
 }
 
 export const useRTCStore = create<RTCState>(set => ({
@@ -154,6 +157,9 @@ export const useRTCStore = create<RTCState>(set => ({
   rpcDataChannel: null,
   setRpcDataChannel: channel => set({ rpcDataChannel: channel }),
 
+  hidDataChannel: null,
+  setHidDataChannel: channel => set({ hidDataChannel: channel }),
+
   transceiver: null,
   setTransceiver: transceiver => set({ transceiver }),
 

ui/src/hooks/useKeyboard.ts

@@ -8,21 +8,43 @@ export default function useKeyboard() {
   const [send] = useJsonRpc();
 
   const rpcDataChannel = useRTCStore(state => state.rpcDataChannel);
+  const hidDataChannel = useRTCStore(state => state.hidDataChannel);
+
   const updateActiveKeysAndModifiers = useHidStore(
     state => state.updateActiveKeysAndModifiers,
   );
 
   const sendKeyboardEvent = useCallback(
     (keys: number[], modifiers: number[]) => {
-      if (rpcDataChannel?.readyState !== "open") return;
+      const rpcChannelReady = rpcDataChannel?.readyState === "open";
+      const hidChannelReady = hidDataChannel?.readyState === "open";
+      if (!rpcChannelReady && !hidChannelReady) return;
+
       const accModifier = modifiers.reduce((acc, val) => acc + val, 0);
 
-      send("keyboardReport", { keys, modifier: accModifier });
+      if (hidChannelReady) {
+        if (accModifier > 0) {
+          hidDataChannel?.send(new Uint8Array([1, accModifier, ...keys]));
+        } else {
+          if (keys.length > 0) {
+            hidDataChannel?.send(new Uint8Array([2, ...keys]));
+          } else {
+            hidDataChannel?.send(new Uint8Array([3]));
+          }
+        }
+      } else {
+        send("keyboardReport", { keys, modifier: accModifier });
+      }
 
       // We do this for the info bar to display the currently pressed keys for the user
       updateActiveKeysAndModifiers({ keys: keys, modifiers: modifiers });
     },
-    [rpcDataChannel?.readyState, send, updateActiveKeysAndModifiers],
+    [
+      hidDataChannel?.readyState,
+      rpcDataChannel?.readyState,
+      send,
+      updateActiveKeysAndModifiers,
+    ],
   );
 
   const resetKeyboardState = useCallback(() => {

ui/src/routes/devices.$id.tsx

@@ -138,6 +138,7 @@ export default function KvmIdRoute() {
   const setPeerConnection = useRTCStore(state => state.setPeerConnection);
   const setDiskChannel = useRTCStore(state => state.setDiskChannel);
   const setRpcDataChannel = useRTCStore(state => state.setRpcDataChannel);
+  const setHidDataChannel = useRTCStore(state => state.setHidDataChannel);
   const setTransceiver = useRTCStore(state => state.setTransceiver);
   const location = useLocation();
 
@@ -485,6 +486,12 @@ export default function KvmIdRoute() {
       setRpcDataChannel(rpcDataChannel);
     };
 
+    const hidDataChannel = pc.createDataChannel("hid");
+    hidDataChannel.binaryType = "arraybuffer";
+    hidDataChannel.onopen = () => {
+      setHidDataChannel(hidDataChannel);
+    };
+
     const diskDataChannel = pc.createDataChannel("disk");
     diskDataChannel.onopen = () => {
       setDiskChannel(diskDataChannel);
@@ -501,6 +508,7 @@ export default function KvmIdRoute() {
     setPeerConnection,
     setPeerConnectionState,
     setRpcDataChannel,
+    setHidDataChannel,
     setTransceiver,
   ]);
 

webrtc.go

@@ -117,6 +117,9 @@ func newSession(config SessionConfig) (*Session, error) {
 			triggerOTAStateUpdate()
 			triggerVideoStateUpdate()
 			triggerUSBStateUpdate()
+		case "hid":
+			session.HidChannel = d
+			d.OnMessage(handleHidMessage)
 		case "disk":
 			session.DiskChannel = d
 			d.OnMessage(onDiskMessage)