From ae775aeeee8cb2bef6fe71bdbf78723c09e932aa Mon Sep 17 00:00:00 2001 From: Siyuan Miao Date: Tue, 12 Aug 2025 17:24:08 +0200 Subject: [PATCH] refactor: use less data to transfer keyboard and mouse events [WIP DO NOT MERGE] --- dev_deploy.sh | 3 ++ hid.go | 61 ++++++++++++++++++++++++ internal/usbgadget/hid_keyboard.go | 2 + internal/usbgadget/hid_mouse_absolute.go | 4 ++ ui/src/components/WebRTCVideo.tsx | 16 +++++-- ui/src/hooks/stores.ts | 6 +++ ui/src/hooks/useKeyboard.ts | 28 +++++++++-- ui/src/routes/devices.$id.tsx | 8 ++++ webrtc.go | 3 ++ 9 files changed, 125 insertions(+), 6 deletions(-) create mode 100644 hid.go diff --git a/dev_deploy.sh b/dev_deploy.sh index 059e416..114c8d3 100755 --- a/dev_deploy.sh +++ b/dev_deploy.sh @@ -142,9 +142,11 @@ fi msg_info "▶ Building go binary" make build_dev +msg_info "▶ Killing any existing instances of the application" # Kill any existing instances of the application ssh "${REMOTE_USER}@${REMOTE_HOST}" "killall jetkvm_app_debug || true" +msg_info "▶ Copying binary to remote host" # Copy the binary to the remote host ssh "${REMOTE_USER}@${REMOTE_HOST}" "cat > ${REMOTE_PATH}/jetkvm_app_debug" < bin/jetkvm_app @@ -156,6 +158,7 @@ if [ "$RESET_USB_HID_DEVICE" = true ]; then ssh "${REMOTE_USER}@${REMOTE_HOST}" "ls /sys/class/udc > /sys/kernel/config/usb_gadget/jetkvm/UDC" fi +msg_info "▶ Deploying and running the application on the remote host" # Deploy and run the application on the remote host ssh "${REMOTE_USER}@${REMOTE_HOST}" ash << EOF set -e diff --git a/hid.go b/hid.go new file mode 100644 index 0000000..64e4420 --- /dev/null +++ b/hid.go @@ -0,0 +1,61 @@ +package kvm + +import ( + "encoding/binary" + + "github.com/pion/webrtc/v4" +) + +type HidPayloadType uint8 + +const ( + HidPayloadTypeKeyboardReport HidPayloadType = 1 + HidPayloadTypeKeyboardReportNoModifier HidPayloadType = 2 + HidPayloadTypeKeyboardReportNothing HidPayloadType = 3 + HidPayloadTypeAbsMouseReport HidPayloadType = 8 + HidPayloadTypeRelMouseReport HidPayloadType = 9 +) + +func handleHidMessage(msg webrtc.DataChannelMessage) { + if msg.IsString { + webrtcLogger.Info().Interface("msg", msg.Data).Msg("Hid message is a string, skipping") + return + } + + payloadType := HidPayloadType(msg.Data[0]) + switch payloadType { + case HidPayloadTypeKeyboardReport: + if len(msg.Data) < 3 { + return + } + modifier := msg.Data[1] + keys := msg.Data[2:] + _ = gadget.KeyboardReport(modifier, keys) + case HidPayloadTypeKeyboardReportNoModifier: + if len(msg.Data) < 2 { + return + } + keys := msg.Data[1:] + _ = gadget.KeyboardReport(0, keys) + case HidPayloadTypeKeyboardReportNothing: + _ = gadget.KeyboardReport(0, []byte{}) + case HidPayloadTypeAbsMouseReport: + if len(msg.Data) < 6 { + return + } + x := binary.LittleEndian.Uint16(msg.Data[1:3]) + y := binary.LittleEndian.Uint16(msg.Data[3:5]) + buttons := msg.Data[5] + webrtcLogger.Info().Uint16("x", x).Uint16("y", y).Uint8("buttons", buttons).Msg("Absolute mouse report") + _ = gadget.AbsMouseReport(int(x), int(y), buttons) + case HidPayloadTypeRelMouseReport: + if len(msg.Data) < 4 { + return + } + dx := int8(msg.Data[1]) + dy := int8(msg.Data[2]) + buttons := msg.Data[3] + webrtcLogger.Info().Int8("dx", dx).Int8("dy", dy).Uint8("buttons", buttons).Msg("Relative mouse report") + _ = gadget.RelMouseReport(dx, dy, buttons) + } +} diff --git a/internal/usbgadget/hid_keyboard.go b/internal/usbgadget/hid_keyboard.go index dbd5d5a..24e2630 100644 --- a/internal/usbgadget/hid_keyboard.go +++ b/internal/usbgadget/hid_keyboard.go @@ -214,6 +214,8 @@ func (u *UsbGadget) KeyboardReport(modifier uint8, keys []uint8) error { u.keyboardLock.Lock() defer u.keyboardLock.Unlock() + u.log.Trace().Uint8("modifier", modifier).Bytes("keys", keys).Msg("KeyboardReport") + if len(keys) > 6 { keys = keys[:6] } diff --git a/internal/usbgadget/hid_mouse_absolute.go b/internal/usbgadget/hid_mouse_absolute.go index 7ba9958..4741c22 100644 --- a/internal/usbgadget/hid_mouse_absolute.go +++ b/internal/usbgadget/hid_mouse_absolute.go @@ -88,6 +88,8 @@ func (u *UsbGadget) AbsMouseReport(x, y int, buttons uint8) error { u.absMouseLock.Lock() defer u.absMouseLock.Unlock() + u.log.Trace().Int("x", x).Int("y", y).Uint8("buttons", buttons).Msg("AbsMouseReport") + err := u.absMouseWriteHidFile([]byte{ 1, // Report ID 1 buttons, // Buttons @@ -108,6 +110,8 @@ func (u *UsbGadget) AbsMouseWheelReport(wheelY int8) error { u.absMouseLock.Lock() defer u.absMouseLock.Unlock() + u.log.Trace().Int8("wheelY", wheelY).Msg("AbsMouseWheelReport") + // Only send a report if the value is non-zero if wheelY == 0 { return nil diff --git a/ui/src/components/WebRTCVideo.tsx b/ui/src/components/WebRTCVideo.tsx index 571fac8..bbc902c 100644 --- a/ui/src/components/WebRTCVideo.tsx +++ b/ui/src/components/WebRTCVideo.tsx @@ -65,7 +65,7 @@ export default function WebRTCVideo() { // RTC related states const peerConnection = useRTCStore(state => state.peerConnection); - + const hidDataChannel = useRTCStore(state => state.hidDataChannel); // HDMI and UI states const hdmiState = useVideoStore(state => state.hdmiState); const hdmiError = ["no_lock", "no_signal", "out_of_range"].includes(hdmiState); @@ -253,11 +253,21 @@ export default function WebRTCVideo() { const sendAbsMouseMovement = useCallback( (x: number, y: number, buttons: number) => { if (settings.mouseMode !== "absolute") return; - send("absMouseReport", { x, y, buttons }); + if (hidDataChannel?.readyState === "open") { + const buffer = new ArrayBuffer(6); + const view = new DataView(buffer); + view.setUint8(0, 8); // type + view.setUint16(1, x, true); // x, little-endian + view.setUint16(3, y, true); // y, little-endian + view.setUint8(5, buttons); // buttons + hidDataChannel.send(buffer); + } else { + send("absMouseReport", { x, y, buttons }); + } // We set that for the debug info bar setMousePosition(x, y); }, - [send, setMousePosition, settings.mouseMode], + [hidDataChannel?.readyState, send, setMousePosition, settings.mouseMode], ); const absMouseMoveHandler = useCallback( diff --git a/ui/src/hooks/stores.ts b/ui/src/hooks/stores.ts index 7281fa9..394259a 100644 --- a/ui/src/hooks/stores.ts +++ b/ui/src/hooks/stores.ts @@ -145,6 +145,9 @@ interface RTCState { terminalChannel: RTCDataChannel | null; setTerminalChannel: (channel: RTCDataChannel) => void; + + hidDataChannel: RTCDataChannel | null; + setHidDataChannel: (channel: RTCDataChannel) => void; } export const useRTCStore = create(set => ({ @@ -154,6 +157,9 @@ export const useRTCStore = create(set => ({ rpcDataChannel: null, setRpcDataChannel: channel => set({ rpcDataChannel: channel }), + hidDataChannel: null, + setHidDataChannel: channel => set({ hidDataChannel: channel }), + transceiver: null, setTransceiver: transceiver => set({ transceiver }), diff --git a/ui/src/hooks/useKeyboard.ts b/ui/src/hooks/useKeyboard.ts index 0ce1eef..1611aff 100644 --- a/ui/src/hooks/useKeyboard.ts +++ b/ui/src/hooks/useKeyboard.ts @@ -8,21 +8,43 @@ export default function useKeyboard() { const [send] = useJsonRpc(); const rpcDataChannel = useRTCStore(state => state.rpcDataChannel); + const hidDataChannel = useRTCStore(state => state.hidDataChannel); + const updateActiveKeysAndModifiers = useHidStore( state => state.updateActiveKeysAndModifiers, ); const sendKeyboardEvent = useCallback( (keys: number[], modifiers: number[]) => { - if (rpcDataChannel?.readyState !== "open") return; + const rpcChannelReady = rpcDataChannel?.readyState === "open"; + const hidChannelReady = hidDataChannel?.readyState === "open"; + if (!rpcChannelReady && !hidChannelReady) return; + const accModifier = modifiers.reduce((acc, val) => acc + val, 0); - send("keyboardReport", { keys, modifier: accModifier }); + if (hidChannelReady) { + if (accModifier > 0) { + hidDataChannel?.send(new Uint8Array([1, accModifier, ...keys])); + } else { + if (keys.length > 0) { + hidDataChannel?.send(new Uint8Array([2, ...keys])); + } else { + hidDataChannel?.send(new Uint8Array([3])); + } + } + } else { + send("keyboardReport", { keys, modifier: accModifier }); + } // We do this for the info bar to display the currently pressed keys for the user updateActiveKeysAndModifiers({ keys: keys, modifiers: modifiers }); }, - [rpcDataChannel?.readyState, send, updateActiveKeysAndModifiers], + [ + hidDataChannel?.readyState, + rpcDataChannel?.readyState, + send, + updateActiveKeysAndModifiers, + ], ); const resetKeyboardState = useCallback(() => { diff --git a/ui/src/routes/devices.$id.tsx b/ui/src/routes/devices.$id.tsx index 8cdb5b3..19b2b5f 100644 --- a/ui/src/routes/devices.$id.tsx +++ b/ui/src/routes/devices.$id.tsx @@ -138,6 +138,7 @@ export default function KvmIdRoute() { const setPeerConnection = useRTCStore(state => state.setPeerConnection); const setDiskChannel = useRTCStore(state => state.setDiskChannel); const setRpcDataChannel = useRTCStore(state => state.setRpcDataChannel); + const setHidDataChannel = useRTCStore(state => state.setHidDataChannel); const setTransceiver = useRTCStore(state => state.setTransceiver); const location = useLocation(); @@ -485,6 +486,12 @@ export default function KvmIdRoute() { setRpcDataChannel(rpcDataChannel); }; + const hidDataChannel = pc.createDataChannel("hid"); + hidDataChannel.binaryType = "arraybuffer"; + hidDataChannel.onopen = () => { + setHidDataChannel(hidDataChannel); + }; + const diskDataChannel = pc.createDataChannel("disk"); diskDataChannel.onopen = () => { setDiskChannel(diskDataChannel); @@ -501,6 +508,7 @@ export default function KvmIdRoute() { setPeerConnection, setPeerConnectionState, setRpcDataChannel, + setHidDataChannel, setTransceiver, ]); diff --git a/webrtc.go b/webrtc.go index f6c8529..95c34ab 100644 --- a/webrtc.go +++ b/webrtc.go @@ -117,6 +117,9 @@ func newSession(config SessionConfig) (*Session, error) { triggerOTAStateUpdate() triggerVideoStateUpdate() triggerUSBStateUpdate() + case "hid": + session.HidChannel = d + d.OnMessage(handleHidMessage) case "disk": session.DiskChannel = d d.OnMessage(onDiskMessage)