From e24f3c95cd584d2e3d5cd806582340ab94f3f2d2 Mon Sep 17 00:00:00 2001 From: Siyuan Miao Date: Tue, 12 Aug 2025 17:24:08 +0200 Subject: [PATCH] refactor: use less data to transfer keyboard and mouse events [WIP DO NOT MERGE] --- dev_deploy.sh | 6 +-- hid.go | 61 ++++++++++++++++++++++++ internal/usbgadget/hid_keyboard.go | 2 + internal/usbgadget/hid_mouse_absolute.go | 4 ++ ui/src/components/WebRTCVideo.tsx | 16 +++++-- ui/src/hooks/stores.ts | 6 +++ ui/src/hooks/useKeyboard.ts | 43 +++++++++++++++++ ui/src/routes/devices.$id.tsx | 7 +++ webrtc.go | 3 ++ 9 files changed, 142 insertions(+), 6 deletions(-) create mode 100644 hid.go diff --git a/dev_deploy.sh b/dev_deploy.sh index aac9acb..410104a 100755 --- a/dev_deploy.sh +++ b/dev_deploy.sh @@ -159,10 +159,10 @@ else msg_info "▶ Building development binary" make build_dev - # Kill any existing instances of the application + msg_info "▶ Killing any existing instances of the application" ssh "${REMOTE_USER}@${REMOTE_HOST}" "killall jetkvm_app_debug || true" - # Copy the binary to the remote host + msg_info "▶ Copying binary to remote host" ssh "${REMOTE_USER}@${REMOTE_HOST}" "cat > ${REMOTE_PATH}/jetkvm_app_debug" < bin/jetkvm_app if [ "$RESET_USB_HID_DEVICE" = true ]; then @@ -173,7 +173,7 @@ else ssh "${REMOTE_USER}@${REMOTE_HOST}" "ls /sys/class/udc > /sys/kernel/config/usb_gadget/jetkvm/UDC" fi - # Deploy and run the application on the remote host + msg_info "▶ Deploying and running the application on the remote host" ssh "${REMOTE_USER}@${REMOTE_HOST}" ash << EOF set -e diff --git a/hid.go b/hid.go new file mode 100644 index 0000000..64e4420 --- /dev/null +++ b/hid.go @@ -0,0 +1,61 @@ +package kvm + +import ( + "encoding/binary" + + "github.com/pion/webrtc/v4" +) + +type HidPayloadType uint8 + +const ( + HidPayloadTypeKeyboardReport HidPayloadType = 1 + HidPayloadTypeKeyboardReportNoModifier HidPayloadType = 2 + HidPayloadTypeKeyboardReportNothing HidPayloadType = 3 + HidPayloadTypeAbsMouseReport HidPayloadType = 8 + HidPayloadTypeRelMouseReport HidPayloadType = 9 +) + +func handleHidMessage(msg webrtc.DataChannelMessage) { + if msg.IsString { + webrtcLogger.Info().Interface("msg", msg.Data).Msg("Hid message is a string, skipping") + return + } + + payloadType := HidPayloadType(msg.Data[0]) + switch payloadType { + case HidPayloadTypeKeyboardReport: + if len(msg.Data) < 3 { + return + } + modifier := msg.Data[1] + keys := msg.Data[2:] + _ = gadget.KeyboardReport(modifier, keys) + case HidPayloadTypeKeyboardReportNoModifier: + if len(msg.Data) < 2 { + return + } + keys := msg.Data[1:] + _ = gadget.KeyboardReport(0, keys) + case HidPayloadTypeKeyboardReportNothing: + _ = gadget.KeyboardReport(0, []byte{}) + case HidPayloadTypeAbsMouseReport: + if len(msg.Data) < 6 { + return + } + x := binary.LittleEndian.Uint16(msg.Data[1:3]) + y := binary.LittleEndian.Uint16(msg.Data[3:5]) + buttons := msg.Data[5] + webrtcLogger.Info().Uint16("x", x).Uint16("y", y).Uint8("buttons", buttons).Msg("Absolute mouse report") + _ = gadget.AbsMouseReport(int(x), int(y), buttons) + case HidPayloadTypeRelMouseReport: + if len(msg.Data) < 4 { + return + } + dx := int8(msg.Data[1]) + dy := int8(msg.Data[2]) + buttons := msg.Data[3] + webrtcLogger.Info().Int8("dx", dx).Int8("dy", dy).Uint8("buttons", buttons).Msg("Relative mouse report") + _ = gadget.RelMouseReport(dx, dy, buttons) + } +} diff --git a/internal/usbgadget/hid_keyboard.go b/internal/usbgadget/hid_keyboard.go index f4fbaa6..3d0ff97 100644 --- a/internal/usbgadget/hid_keyboard.go +++ b/internal/usbgadget/hid_keyboard.go @@ -265,6 +265,8 @@ func (u *UsbGadget) KeyboardReport(modifier byte, keys []byte) (KeysDownState, e defer u.keyboardLock.Unlock() defer u.resetUserInputTime() + u.log.Trace().Uint8("modifier", modifier).Bytes("keys", keys).Msg("KeyboardReport") + if len(keys) > hidKeyBufferSize { keys = keys[:hidKeyBufferSize] } diff --git a/internal/usbgadget/hid_mouse_absolute.go b/internal/usbgadget/hid_mouse_absolute.go index c083b60..cd91555 100644 --- a/internal/usbgadget/hid_mouse_absolute.go +++ b/internal/usbgadget/hid_mouse_absolute.go @@ -89,6 +89,8 @@ func (u *UsbGadget) AbsMouseReport(x int, y int, buttons uint8) error { u.absMouseLock.Lock() defer u.absMouseLock.Unlock() + u.log.Trace().Int("x", x).Int("y", y).Uint8("buttons", buttons).Msg("AbsMouseReport") + err := u.absMouseWriteHidFile([]byte{ 1, // Report ID 1 buttons, // Buttons @@ -109,6 +111,8 @@ func (u *UsbGadget) AbsMouseWheelReport(wheelY int8) error { u.absMouseLock.Lock() defer u.absMouseLock.Unlock() + u.log.Trace().Int8("wheelY", wheelY).Msg("AbsMouseWheelReport") + // Only send a report if the value is non-zero if wheelY == 0 { return nil diff --git a/ui/src/components/WebRTCVideo.tsx b/ui/src/components/WebRTCVideo.tsx index 9e2f0f2..8d6f11f 100644 --- a/ui/src/components/WebRTCVideo.tsx +++ b/ui/src/components/WebRTCVideo.tsx @@ -50,7 +50,7 @@ export default function WebRTCVideo() { // RTC related states const { peerConnection } = useRTCStore(); - + const hidDataChannel = useRTCStore(state => state.hidDataChannel); // HDMI and UI states const hdmiError = ["no_lock", "no_signal", "out_of_range"].includes(hdmiState); const isVideoLoading = !isPlaying; @@ -243,11 +243,21 @@ export default function WebRTCVideo() { const sendAbsMouseMovement = useCallback( (x: number, y: number, buttons: number) => { if (settings.mouseMode !== "absolute") return; - send("absMouseReport", { x, y, buttons }); + if (hidDataChannel?.readyState === "open") { + const buffer = new ArrayBuffer(6); + const view = new DataView(buffer); + view.setUint8(0, 8); // type + view.setUint16(1, x, true); // x, little-endian + view.setUint16(3, y, true); // y, little-endian + view.setUint8(5, buttons); // buttons + hidDataChannel.send(buffer); + } else { + send("absMouseReport", { x, y, buttons }); + } // We set that for the debug info bar setMousePosition(x, y); }, - [send, setMousePosition, settings.mouseMode], + [hidDataChannel?.readyState, send, setMousePosition, settings.mouseMode], ); const absMouseMoveHandler = useCallback( diff --git a/ui/src/hooks/stores.ts b/ui/src/hooks/stores.ts index a6dc95b..f7a23a8 100644 --- a/ui/src/hooks/stores.ts +++ b/ui/src/hooks/stores.ts @@ -142,6 +142,9 @@ export interface RTCState { terminalChannel: RTCDataChannel | null; setTerminalChannel: (channel: RTCDataChannel) => void; + + hidDataChannel: RTCDataChannel | null; + setHidDataChannel: (channel: RTCDataChannel) => void; } export const useRTCStore = create(set => ({ @@ -151,6 +154,9 @@ export const useRTCStore = create(set => ({ rpcDataChannel: null, setRpcDataChannel: (channel: RTCDataChannel) => set({ rpcDataChannel: channel }), + hidDataChannel: null, + setHidDataChannel: channel => set({ hidDataChannel: channel }), + transceiver: null, setTransceiver: (transceiver: RTCRtpTransceiver) => set({ transceiver }), diff --git a/ui/src/hooks/useKeyboard.ts b/ui/src/hooks/useKeyboard.ts index 5f587b0..78d76d7 100644 --- a/ui/src/hooks/useKeyboard.ts +++ b/ui/src/hooks/useKeyboard.ts @@ -25,6 +25,15 @@ export default function useKeyboard() { // and resetting keyboard state. It sends the keys currently pressed and the modifier state. // The device will respond with the keysDownState if it supports the keyPressReport API // or just accept the state if it does not support (returning no result) + const [send] = useJsonRpc(); + + const rpcDataChannel = useRTCStore(state => state.rpcDataChannel); + const hidDataChannel = useRTCStore(state => state.hidDataChannel); + + const updateActiveKeysAndModifiers = useHidStore( + state => state.updateActiveKeysAndModifiers, + ); + const sendKeyboardEvent = useCallback( async (state: KeysDownState) => { if (rpcDataChannel?.readyState !== "open") return; @@ -82,6 +91,29 @@ export default function useKeyboard() { } } }); + (keys: number[], modifiers: number[]) => { + const rpcChannelReady = rpcDataChannel?.readyState === "open"; + const hidChannelReady = hidDataChannel?.readyState === "open"; + if (!rpcChannelReady && !hidChannelReady) return; + + const accModifier = modifiers.reduce((acc, val) => acc + val, 0); + + if (hidChannelReady) { + if (accModifier > 0) { + hidDataChannel?.send(new Uint8Array([1, accModifier, ...keys])); + } else { + if (keys.length > 0) { + hidDataChannel?.send(new Uint8Array([2, ...keys])); + } else { + hidDataChannel?.send(new Uint8Array([3])); + } + } + } else { + send("keyboardReport", { keys, modifier: accModifier }); + } + + // We do this for the info bar to display the currently pressed keys for the user + updateActiveKeysAndModifiers({ keys: keys, modifiers: modifiers }); }, [rpcDataChannel?.readyState, send, setkeyPressReportApiAvailable, setKeysDownState], ); @@ -97,6 +129,17 @@ export default function useKeyboard() { keysDownState.modifier = 0; sendKeyboardEvent(keysDownState); }, [keysDownState, sendKeyboardEvent]); + [ + hidDataChannel?.readyState, + rpcDataChannel?.readyState, + send, + updateActiveKeysAndModifiers, + ], + ); + + const resetKeyboardState = useCallback(() => { + sendKeyboardEvent([], []); + }, [sendKeyboardEvent]); // executeMacro is used to execute a macro consisting of multiple steps. // Each step can have multiple keys, multiple modifiers and a delay. diff --git a/ui/src/routes/devices.$id.tsx b/ui/src/routes/devices.$id.tsx index 9be05f6..e39810a 100644 --- a/ui/src/routes/devices.$id.tsx +++ b/ui/src/routes/devices.$id.tsx @@ -482,6 +482,12 @@ export default function KvmIdRoute() { setRpcDataChannel(rpcDataChannel); }; + const hidDataChannel = pc.createDataChannel("hid"); + hidDataChannel.binaryType = "arraybuffer"; + hidDataChannel.onopen = () => { + setHidDataChannel(hidDataChannel); + }; + setPeerConnection(pc); }, [ cleanupAndStopReconnecting, @@ -492,6 +498,7 @@ export default function KvmIdRoute() { setPeerConnection, setPeerConnectionState, setRpcDataChannel, + setHidDataChannel, setTransceiver, ]); diff --git a/webrtc.go b/webrtc.go index c0f159a..d925621 100644 --- a/webrtc.go +++ b/webrtc.go @@ -125,6 +125,9 @@ func newSession(config SessionConfig) (*Session, error) { triggerOTAStateUpdate() triggerVideoStateUpdate() triggerUSBStateUpdate() + case "hid": + session.HidChannel = d + d.OnMessage(handleHidMessage) case "terminal": handleTerminalChannel(d) case "serial":