refactor: use less data to transfer keyboard and mouse events [WIP DO NOT MERGE]

This commit is contained in:
Siyuan Miao 2025-08-12 17:24:08 +02:00
parent 584768bacf
commit ae775aeeee
9 changed files with 125 additions and 6 deletions

View File

@ -142,9 +142,11 @@ fi
msg_info "▶ Building go binary"
make build_dev
msg_info "▶ Killing any existing instances of the application"
# Kill any existing instances of the application
ssh "${REMOTE_USER}@${REMOTE_HOST}" "killall jetkvm_app_debug || true"
msg_info "▶ Copying binary to remote host"
# Copy the binary to the remote host
ssh "${REMOTE_USER}@${REMOTE_HOST}" "cat > ${REMOTE_PATH}/jetkvm_app_debug" < bin/jetkvm_app
@ -156,6 +158,7 @@ if [ "$RESET_USB_HID_DEVICE" = true ]; then
ssh "${REMOTE_USER}@${REMOTE_HOST}" "ls /sys/class/udc > /sys/kernel/config/usb_gadget/jetkvm/UDC"
fi
msg_info "▶ Deploying and running the application on the remote host"
# Deploy and run the application on the remote host
ssh "${REMOTE_USER}@${REMOTE_HOST}" ash << EOF
set -e

61
hid.go Normal file
View File

@ -0,0 +1,61 @@
package kvm
import (
"encoding/binary"
"github.com/pion/webrtc/v4"
)
type HidPayloadType uint8
const (
HidPayloadTypeKeyboardReport HidPayloadType = 1
HidPayloadTypeKeyboardReportNoModifier HidPayloadType = 2
HidPayloadTypeKeyboardReportNothing HidPayloadType = 3
HidPayloadTypeAbsMouseReport HidPayloadType = 8
HidPayloadTypeRelMouseReport HidPayloadType = 9
)
func handleHidMessage(msg webrtc.DataChannelMessage) {
if msg.IsString {
webrtcLogger.Info().Interface("msg", msg.Data).Msg("Hid message is a string, skipping")
return
}
payloadType := HidPayloadType(msg.Data[0])
switch payloadType {
case HidPayloadTypeKeyboardReport:
if len(msg.Data) < 3 {
return
}
modifier := msg.Data[1]
keys := msg.Data[2:]
_ = gadget.KeyboardReport(modifier, keys)
case HidPayloadTypeKeyboardReportNoModifier:
if len(msg.Data) < 2 {
return
}
keys := msg.Data[1:]
_ = gadget.KeyboardReport(0, keys)
case HidPayloadTypeKeyboardReportNothing:
_ = gadget.KeyboardReport(0, []byte{})
case HidPayloadTypeAbsMouseReport:
if len(msg.Data) < 6 {
return
}
x := binary.LittleEndian.Uint16(msg.Data[1:3])
y := binary.LittleEndian.Uint16(msg.Data[3:5])
buttons := msg.Data[5]
webrtcLogger.Info().Uint16("x", x).Uint16("y", y).Uint8("buttons", buttons).Msg("Absolute mouse report")
_ = gadget.AbsMouseReport(int(x), int(y), buttons)
case HidPayloadTypeRelMouseReport:
if len(msg.Data) < 4 {
return
}
dx := int8(msg.Data[1])
dy := int8(msg.Data[2])
buttons := msg.Data[3]
webrtcLogger.Info().Int8("dx", dx).Int8("dy", dy).Uint8("buttons", buttons).Msg("Relative mouse report")
_ = gadget.RelMouseReport(dx, dy, buttons)
}
}

View File

@ -214,6 +214,8 @@ func (u *UsbGadget) KeyboardReport(modifier uint8, keys []uint8) error {
u.keyboardLock.Lock()
defer u.keyboardLock.Unlock()
u.log.Trace().Uint8("modifier", modifier).Bytes("keys", keys).Msg("KeyboardReport")
if len(keys) > 6 {
keys = keys[:6]
}

View File

@ -88,6 +88,8 @@ func (u *UsbGadget) AbsMouseReport(x, y int, buttons uint8) error {
u.absMouseLock.Lock()
defer u.absMouseLock.Unlock()
u.log.Trace().Int("x", x).Int("y", y).Uint8("buttons", buttons).Msg("AbsMouseReport")
err := u.absMouseWriteHidFile([]byte{
1, // Report ID 1
buttons, // Buttons
@ -108,6 +110,8 @@ func (u *UsbGadget) AbsMouseWheelReport(wheelY int8) error {
u.absMouseLock.Lock()
defer u.absMouseLock.Unlock()
u.log.Trace().Int8("wheelY", wheelY).Msg("AbsMouseWheelReport")
// Only send a report if the value is non-zero
if wheelY == 0 {
return nil

View File

@ -65,7 +65,7 @@ export default function WebRTCVideo() {
// RTC related states
const peerConnection = useRTCStore(state => state.peerConnection);
const hidDataChannel = useRTCStore(state => state.hidDataChannel);
// HDMI and UI states
const hdmiState = useVideoStore(state => state.hdmiState);
const hdmiError = ["no_lock", "no_signal", "out_of_range"].includes(hdmiState);
@ -253,11 +253,21 @@ export default function WebRTCVideo() {
const sendAbsMouseMovement = useCallback(
(x: number, y: number, buttons: number) => {
if (settings.mouseMode !== "absolute") return;
send("absMouseReport", { x, y, buttons });
if (hidDataChannel?.readyState === "open") {
const buffer = new ArrayBuffer(6);
const view = new DataView(buffer);
view.setUint8(0, 8); // type
view.setUint16(1, x, true); // x, little-endian
view.setUint16(3, y, true); // y, little-endian
view.setUint8(5, buttons); // buttons
hidDataChannel.send(buffer);
} else {
send("absMouseReport", { x, y, buttons });
}
// We set that for the debug info bar
setMousePosition(x, y);
},
[send, setMousePosition, settings.mouseMode],
[hidDataChannel?.readyState, send, setMousePosition, settings.mouseMode],
);
const absMouseMoveHandler = useCallback(

View File

@ -145,6 +145,9 @@ interface RTCState {
terminalChannel: RTCDataChannel | null;
setTerminalChannel: (channel: RTCDataChannel) => void;
hidDataChannel: RTCDataChannel | null;
setHidDataChannel: (channel: RTCDataChannel) => void;
}
export const useRTCStore = create<RTCState>(set => ({
@ -154,6 +157,9 @@ export const useRTCStore = create<RTCState>(set => ({
rpcDataChannel: null,
setRpcDataChannel: channel => set({ rpcDataChannel: channel }),
hidDataChannel: null,
setHidDataChannel: channel => set({ hidDataChannel: channel }),
transceiver: null,
setTransceiver: transceiver => set({ transceiver }),

View File

@ -8,21 +8,43 @@ export default function useKeyboard() {
const [send] = useJsonRpc();
const rpcDataChannel = useRTCStore(state => state.rpcDataChannel);
const hidDataChannel = useRTCStore(state => state.hidDataChannel);
const updateActiveKeysAndModifiers = useHidStore(
state => state.updateActiveKeysAndModifiers,
);
const sendKeyboardEvent = useCallback(
(keys: number[], modifiers: number[]) => {
if (rpcDataChannel?.readyState !== "open") return;
const rpcChannelReady = rpcDataChannel?.readyState === "open";
const hidChannelReady = hidDataChannel?.readyState === "open";
if (!rpcChannelReady && !hidChannelReady) return;
const accModifier = modifiers.reduce((acc, val) => acc + val, 0);
send("keyboardReport", { keys, modifier: accModifier });
if (hidChannelReady) {
if (accModifier > 0) {
hidDataChannel?.send(new Uint8Array([1, accModifier, ...keys]));
} else {
if (keys.length > 0) {
hidDataChannel?.send(new Uint8Array([2, ...keys]));
} else {
hidDataChannel?.send(new Uint8Array([3]));
}
}
} else {
send("keyboardReport", { keys, modifier: accModifier });
}
// We do this for the info bar to display the currently pressed keys for the user
updateActiveKeysAndModifiers({ keys: keys, modifiers: modifiers });
},
[rpcDataChannel?.readyState, send, updateActiveKeysAndModifiers],
[
hidDataChannel?.readyState,
rpcDataChannel?.readyState,
send,
updateActiveKeysAndModifiers,
],
);
const resetKeyboardState = useCallback(() => {

View File

@ -138,6 +138,7 @@ export default function KvmIdRoute() {
const setPeerConnection = useRTCStore(state => state.setPeerConnection);
const setDiskChannel = useRTCStore(state => state.setDiskChannel);
const setRpcDataChannel = useRTCStore(state => state.setRpcDataChannel);
const setHidDataChannel = useRTCStore(state => state.setHidDataChannel);
const setTransceiver = useRTCStore(state => state.setTransceiver);
const location = useLocation();
@ -485,6 +486,12 @@ export default function KvmIdRoute() {
setRpcDataChannel(rpcDataChannel);
};
const hidDataChannel = pc.createDataChannel("hid");
hidDataChannel.binaryType = "arraybuffer";
hidDataChannel.onopen = () => {
setHidDataChannel(hidDataChannel);
};
const diskDataChannel = pc.createDataChannel("disk");
diskDataChannel.onopen = () => {
setDiskChannel(diskDataChannel);
@ -501,6 +508,7 @@ export default function KvmIdRoute() {
setPeerConnection,
setPeerConnectionState,
setRpcDataChannel,
setHidDataChannel,
setTransceiver,
]);

View File

@ -117,6 +117,9 @@ func newSession(config SessionConfig) (*Session, error) {
triggerOTAStateUpdate()
triggerVideoStateUpdate()
triggerUSBStateUpdate()
case "hid":
session.HidChannel = d
d.OnMessage(handleHidMessage)
case "disk":
session.DiskChannel = d
d.OnMessage(onDiskMessage)