Superminaren
/
kvm
mirror of https://github.com/jetkvm/kvm.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

feat: use hidRpcChannel to save bandwidth

This commit is contained in:
Siyuan Miao 2025-08-29 18:19:51 +02:00
parent 94521ef6db
commit 58b72add90
11 changed files with 697 additions and 21 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

129
hidrpc.go Normal file
View File

@ -0,0 +1,129 @@
package kvm
import (
"fmt"
"github.com/jetkvm/kvm/internal/hidrpc"
"github.com/jetkvm/kvm/internal/usbgadget"
)
func onHidMessage(data []byte, session *Session) {
if len(data) < 1 {
logger.Warn().Int("length", len(data)).Msg("received empty data in HID RPC message handler")
return
}
var (
message hidrpc.Message
rpcErr error
)
if err := hidrpc.Unmarshal(data, &message); err != nil {
logger.Warn().Err(err).Msg("failed to unmarshal HID RPC message")
return
}
switch message.Type() {
case hidrpc.TypeHandshake:
message, err := hidrpc.NewHandshakeMessage().Marshal()
if err != nil {
logger.Warn().Err(err).Msg("failed to marshal handshake message")
return
}
if err := session.HidChannel.Send(message); err != nil {
logger.Warn().Err(err).Msg("failed to send handshake message")
return
}
session.hidRpcAvailable = true
case hidrpc.TypeKeypressReport, hidrpc.TypeKeyboardReport:
keysDownState, err := handleHidRpcKeyboardInput(message)
if keysDownState != nil {
reportHidRpcKeysDownState(*keysDownState, session)
}
rpcErr = err
case hidrpc.TypePointerReport:
pointerReport, err := message.PointerReport()
if err != nil {
logger.Warn().Err(err).Msg("failed to get pointer report")
return
}
rpcErr = rpcAbsMouseReport(pointerReport.X, pointerReport.Y, pointerReport.Button)
case hidrpc.TypeMouseReport:
mouseReport, err := message.MouseReport()
if err != nil {
logger.Warn().Err(err).Msg("failed to get mouse report")
return
}
rpcErr = rpcRelMouseReport(mouseReport.DX, mouseReport.DY, mouseReport.Button)
default:
logger.Warn().Uint8("type", uint8(message.Type())).Msg("unknown HID RPC message type")
}
if rpcErr != nil {
logger.Warn().Err(rpcErr).Msg("failed to handle HID RPC message")
}
}
func handleHidRpcKeyboardInput(message hidrpc.Message) (*usbgadget.KeysDownState, error) {
switch message.Type() {
case hidrpc.TypeKeypressReport:
keypressReport, err := message.KeypressReport()
if err != nil {
logger.Warn().Err(err).Msg("failed to get keypress report")
return nil, err
}
keysDownState, rpcError := rpcKeypressReport(keypressReport.Key, keypressReport.Press)
return &keysDownState, rpcError
case hidrpc.TypeKeyboardReport:
keyboardReport, err := message.KeyboardReport()
if err != nil {
logger.Warn().Err(err).Msg("failed to get keyboard report")
return nil, err
}
keysDownState, rpcError := rpcKeyboardReport(keyboardReport.Modifier, keyboardReport.Keys)
return &keysDownState, rpcError
}
return nil, fmt.Errorf("unknown HID RPC message type: %d", message.Type())
}
func reportHidRpc(params any, session *Session) {
var (
message []byte
err error
)
switch params := params.(type) {
case usbgadget.KeyboardState:
message, err = hidrpc.NewKeyboardLedMessage(params).Marshal()
case usbgadget.KeysDownState:
message, err = hidrpc.NewKeydownStateMessage(params).Marshal()
}
if err != nil {
logger.Warn().Err(err).Msg("failed to marshal HID RPC message")
return
}
if message == nil {
logger.Warn().Msg("failed to marshal HID RPC message")
return
}
if err := session.HidChannel.Send(message); err != nil {
logger.Warn().Err(err).Msg("failed to send HID RPC message")
}
}
func reportHidRpcKeyboardLedState(state usbgadget.KeyboardState, session *Session) {
if !session.hidRpcAvailable {
writeJSONRPCEvent("keyboardLedState", state, currentSession)
}
reportHidRpc(state, session)
}
func reportHidRpcKeysDownState(state usbgadget.KeysDownState, session *Session) {
if !session.hidRpcAvailable {
writeJSONRPCEvent("keysDownState", state, currentSession)
}
reportHidRpc(state, session)
}

91
internal/hidrpc/hidrpc.go Normal file
View File

@ -0,0 +1,91 @@
package hidrpc
import (
"fmt"
"github.com/jetkvm/kvm/internal/usbgadget"
)
// HID RPC is a variable-length packet format that is used to exchange keyboard and mouse events between the client and the server.
// The packet format is as follows:
// 1 byte: Event Type
// MessageType is the type of the HID RPC message
type MessageType uint8
const (
TypeHandshake MessageType = 0x01
TypeKeyboardReport MessageType = 0x02
TypePointerReport MessageType = 0x03
TypeWheelReport MessageType = 0x04
TypeKeypressReport MessageType = 0x05
TypeMouseReport MessageType = 0x06
TypeKeyboardLedState MessageType = 0x32
TypeKeydownState MessageType = 0x33
)
// ShouldUseEnqueue returns true if the message type should be enqueued to the HID queue.
func ShouldUseEnqueue(messageType MessageType) bool {
return messageType == TypeMouseReport
}
// Unmarshal unmarshals the HID RPC message from the data.
func Unmarshal(data []byte, message *Message) error {
l := len(data)
if l < 1 {
return fmt.Errorf("invalid data length: %d", l)
}
message.t = MessageType(data[0])
message.d = data[1:]
return nil
}
// Marshal marshals the HID RPC message to the data.
func Marshal(message *Message) ([]byte, error) {
if message.t == 0 {
return nil, fmt.Errorf("invalid message type: %d", message.t)
}
data := make([]byte, len(message.d)+1)
data[0] = byte(message.t)
copy(data[1:], message.d)
return data, nil
}
// NewHandshakeMessage creates a new handshake message.
func NewHandshakeMessage() *Message {
return &Message{
t: TypeHandshake,
d: []byte{},
}
}
// NewKeyboardReportMessage creates a new keyboard report message.
func NewKeyboardReportMessage(keys []byte, modifier uint8) *Message {
return &Message{
t: TypeKeyboardReport,
d: append([]byte{modifier}, keys...),
}
}
// NewKeyboardLedMessage creates a new keyboard LED message.
func NewKeyboardLedMessage(state usbgadget.KeyboardState) *Message {
return &Message{
t: TypeKeyboardLedState,
d: []byte{state.Byte()},
}
}
// NewKeydownStateMessage creates a new keydown state message.
func NewKeydownStateMessage(state usbgadget.KeysDownState) *Message {
data := make([]byte, len(state.Keys)+1)
data[0] = state.Modifier
copy(data[1:], state.Keys)
return &Message{
t: TypeKeydownState,
d: data,
}
}

104
internal/hidrpc/message.go Normal file
View File

@ -0,0 +1,104 @@
package hidrpc
import (
"fmt"
)
// Message ..
type Message struct {
t MessageType
d []byte
}
// Marshal marshals the message to a byte array.
func (m *Message) Marshal() ([]byte, error) {
return Marshal(m)
}
func (m *Message) Type() MessageType {
return m.t
}
// KeypressReport ..
type KeypressReport struct {
Key byte
Press bool
}
// KeypressReport returns the keypress report from the message.
func (m *Message) KeypressReport() (KeypressReport, error) {
if m.t != TypeKeypressReport {
return KeypressReport{}, fmt.Errorf("invalid message type: %d", m.t)
}
return KeypressReport{
Key: m.d[0],
Press: m.d[1] == uint8(1),
}, nil
}
// KeyboardReport ..
type KeyboardReport struct {
Modifier byte
Keys []byte
}
// KeyboardReport returns the keyboard report from the message.
func (m *Message) KeyboardReport() (KeyboardReport, error) {
if m.t != TypeKeyboardReport {
return KeyboardReport{}, fmt.Errorf("invalid message type: %d", m.t)
}
return KeyboardReport{
Modifier: m.d[0],
Keys: m.d[1:],
}, nil
}
// PointerReport ..
type PointerReport struct {
X int
Y int
Button uint8
}
func toInt(b []byte) int {
return int(b[0])<<24 + int(b[1])<<16 + int(b[2])<<8 + int(b[3])<<0
}
// PointerReport returns the point report from the message.
func (m *Message) PointerReport() (PointerReport, error) {
if m.t != TypePointerReport {
return PointerReport{}, fmt.Errorf("invalid message type: %d", m.t)
}
if len(m.d) != 9 {
return PointerReport{}, fmt.Errorf("invalid message length: %d", len(m.d))
}
return PointerReport{
X: toInt(m.d[0:4]),
Y: toInt(m.d[4:8]),
Button: uint8(m.d[8]),
}, nil
}
// MouseReport ..
type MouseReport struct {
DX int8
DY int8
Button uint8
}
// MouseReport returns the mouse report from the message.
func (m *Message) MouseReport() (MouseReport, error) {
if m.t != TypeMouseReport {
return MouseReport{}, fmt.Errorf("invalid message type: %d", m.t)
}
return MouseReport{
DX: int8(m.d[0]),
DY: int8(m.d[1]),
Button: uint8(m.d[2]),
}, nil
}

6
internal/usbgadget/hid_keyboard.go
View File

@ -86,6 +86,12 @@ type KeyboardState struct {
Compose bool `json:"compose"`
Kana bool `json:"kana"`
Shift bool `json:"shift"` // This is not part of the main USB HID spec
raw byte
}
// Byte returns the raw byte representation of the keyboard state.
func (k *KeyboardState) Byte() byte {
return k.raw
}
func getKeyboardState(b byte) KeyboardState {

36
ui/src/components/WebRTCVideo.tsx
View File

@ -8,6 +8,7 @@ import InfoBar from "@components/InfoBar";
import notifications from "@/notifications";
import useKeyboard from "@/hooks/useKeyboard";
import { useJsonRpc } from "@/hooks/useJsonRpc";
import { useHidRpc } from "@/hooks/useHidRpc";
import { cx } from "@/cva.config";
import { keys } from "@/keyboardMappings";
import {
@ -60,10 +61,11 @@ export default function WebRTCVideo() {
// Misc states and hooks
const { send } = useJsonRpc();
const { reportAbsMouseEvent, reportRelMouseEvent, handshakeCompleted } = useHidRpc();
// Video-related
const handleResize = useCallback(
( { width, height }: { width: number | undefined; height: number | undefined }) => {
({ width, height }: { width: number | undefined; height: number | undefined }) => {
if (!videoElm.current) return;
// Do something with width and height, e.g.:
setVideoClientSize(width || 0, height || 0);
@ -222,10 +224,22 @@ export default function WebRTCVideo() {
if (settings.mouseMode !== "relative") return;
// if we ignore the event, double-click will not work
// if (x === 0 && y === 0 && buttons === 0) return;
send("relMouseReport", { dx: calcDelta(x), dy: calcDelta(y), buttons });
const dx = calcDelta(x);
const dy = calcDelta(y);
if (handshakeCompleted) {
reportRelMouseEvent(dx, dy, buttons);
} else {
send("relMouseReport", { dx, dy, buttons });
}
setMouseMove({ x, y, buttons });
},
[send, setMouseMove, settings.mouseMode],
[
send,
reportRelMouseEvent,
setMouseMove,
settings.mouseMode,
handshakeCompleted,
],
);
const relMouseMoveHandler = useCallback(
@ -243,11 +257,21 @@ export default function WebRTCVideo() {
const sendAbsMouseMovement = useCallback(
(x: number, y: number, buttons: number) => {
if (settings.mouseMode !== "absolute") return;
send("absMouseReport", { x, y, buttons });
if (handshakeCompleted) {
reportAbsMouseEvent(x, y, buttons);
} else {
send("absMouseReport", { x, y, buttons });
}
// We set that for the debug info bar
setMousePosition(x, y);
},
[send, setMousePosition, settings.mouseMode],
[
send,
reportAbsMouseEvent,
setMousePosition,
settings.mouseMode,
handshakeCompleted,
],
);
const absMouseMoveHandler = useCallback(
@ -357,7 +381,7 @@ export default function WebRTCVideo() {
}
console.debug(`Key down: ${hidKey}`);
handleKeyPress(hidKey, true);
if (!isKeyboardLockActive && hidKey === keys.MetaLeft) {
// If the left meta key was just pressed and we're not keyboard locked
// we'll never see the keyup event because the browser is going to lose

6
ui/src/hooks/stores.ts
View File

@ -105,6 +105,9 @@ export interface RTCState {
setRpcDataChannel: (channel: RTCDataChannel) => void;
rpcDataChannel: RTCDataChannel | null;
setRpcHidChannel: (channel: RTCDataChannel) => void;
rpcHidChannel: RTCDataChannel | null;
peerConnectionState: RTCPeerConnectionState | null;
setPeerConnectionState: (state: RTCPeerConnectionState) => void;
@ -151,6 +154,9 @@ export const useRTCStore = create<RTCState>(set => ({
rpcDataChannel: null,
setRpcDataChannel: (channel: RTCDataChannel) => set({ rpcDataChannel: channel }),
rpcHidChannel: null,
setRpcHidChannel: (channel: RTCDataChannel) => set({ rpcHidChannel: channel }),
transceiver: null,
setTransceiver: (transceiver: RTCRtpTransceiver) => set({ transceiver }),

217
ui/src/hooks/useHidRpc.ts Normal file
View File

@ -0,0 +1,217 @@
import { useCallback, useEffect, useState } from "react";
import { KeysDownState, useRTCStore } from "@/hooks/stores";
export const HID_RPC_MESSAGE_TYPES = {
Handshake: 0x01,
KeyboardReport: 0x02,
PointerReport: 0x03,
WheelReport: 0x04,
KeypressReport: 0x05,
MouseReport: 0x06,
KeyboardLedState: 0x32,
KeysDownState: 0x33,
}
export type HidRpcMessageType = typeof HID_RPC_MESSAGE_TYPES[keyof typeof HID_RPC_MESSAGE_TYPES];
const withinUint8Range = (value: number) => {
return value >= 0 && value <= 255;
};
const fromInt32toUint8 = (n: number) => {
if (n !== n >> 0) {
throw new Error(`Number ${n} is not within the int32 range`);
}
return new Uint8Array([
(n >> 24) & 0xFF,
(n >> 16) & 0xFF,
(n >> 8) & 0xFF,
(n >> 0) & 0xFF,
]);
};
const fromInt8ToUint8 = (n: number) => {
if (n < -128 || n > 127) {
throw new Error(`Number ${n} is not within the int8 range`);
}
return (n >> 0) & 0xFF;
};
const toPointerReportEvent = (x: number, y: number, buttons: number) => {
if (!withinUint8Range(buttons)) {
throw new Error(`Buttons ${buttons} is not within the uint8 range`);
}
return new Uint8Array([
HID_RPC_MESSAGE_TYPES.PointerReport,
...fromInt32toUint8(x),
...fromInt32toUint8(y),
buttons,
]);
};
const toMouseReportEvent = (dx: number, dy: number, buttons: number) => {
if (!withinUint8Range(buttons)) {
throw new Error(`Buttons ${buttons} is not within the uint8 range`);
}
return new Uint8Array([
HID_RPC_MESSAGE_TYPES.MouseReport,
fromInt8ToUint8(dx),
fromInt8ToUint8(dy),
buttons,
]);
};
const toKeyboardReportEvent = (keys: number[], modifier: number) => {
if (!withinUint8Range(modifier)) {
throw new Error(`Modifier ${modifier} is not within the uint8 range`);
}
keys.forEach((k) => {
if (!withinUint8Range(k)) {
throw new Error(`Key ${k} is not within the uint8 range`);
}
});
return new Uint8Array([
HID_RPC_MESSAGE_TYPES.KeyboardReport,
modifier,
...keys,
]);
};
const toKeypressReportEvent = (key: number, press: boolean) => {
if (!withinUint8Range(key)) {
throw new Error(`Key ${key} is not within the uint8 range`);
}
return new Uint8Array([
HID_RPC_MESSAGE_TYPES.KeypressReport,
key,
press ? 1 : 0,
]);
};
const toHandshakeMessage = () => {
return new Uint8Array([HID_RPC_MESSAGE_TYPES.Handshake]);
};
export interface HidRpcMessage {
type: HidRpcMessageType;
keysDownState?: KeysDownState;
}
const unmarshalHidRpcMessage = (data: Uint8Array): HidRpcMessage | undefined => {
if (data.length < 1) {
throw new Error(`Invalid HID RPC message length: ${data.length}`);
}
const payload = data.slice(1);
switch (data[0]) {
case HID_RPC_MESSAGE_TYPES.Handshake:
return {
type: HID_RPC_MESSAGE_TYPES.Handshake,
};
case HID_RPC_MESSAGE_TYPES.KeysDownState:
return {
type: HID_RPC_MESSAGE_TYPES.KeysDownState,
keysDownState: {
modifier: payload[0],
keys: Array.from(payload.slice(1))
},
};
default:
throw new Error(`Unknown HID RPC message type: ${data[0]}`);
}
};
export function useHidRpc(onHidRpcMessage?: (payload: HidRpcMessage) => void) {
const { rpcHidChannel } = useRTCStore();
const [handshakeCompleted, setHandshakeCompleted] = useState(false);
const reportKeyboardEvent = useCallback(
(keys: number[], modifier: number) => {
if (rpcHidChannel?.readyState !== "open") return;
rpcHidChannel.send(toKeyboardReportEvent(keys, modifier));
},
[rpcHidChannel],
);
const reportKeypressEvent = useCallback(
(key: number, press: boolean) => {
if (rpcHidChannel?.readyState !== "open") return;
rpcHidChannel.send(toKeypressReportEvent(key, press));
},
[rpcHidChannel],
);
const reportAbsMouseEvent = useCallback(
(x: number, y: number, buttons: number) => {
if (rpcHidChannel?.readyState !== "open") return;
rpcHidChannel.send(toPointerReportEvent(x, y, buttons));
},
[rpcHidChannel],
);
const reportRelMouseEvent = useCallback(
(dx: number, dy: number, buttons: number) => {
if (rpcHidChannel?.readyState !== "open") return;
rpcHidChannel.send(toMouseReportEvent(dx, dy, buttons));
},
[rpcHidChannel],
);
const doHandshake = useCallback(() => {
if (handshakeCompleted) return;
if (!rpcHidChannel) return;
rpcHidChannel.send(toHandshakeMessage());
}, [rpcHidChannel, handshakeCompleted]);
useEffect(() => {
if (!rpcHidChannel) return;
// send handshake message
doHandshake();
const messageHandler = (e: MessageEvent) => {
if (typeof e.data === "string") {
console.warn("Received string data in HID RPC message handler", e.data);
return;
}
console.log("Received HID RPC message", e.data);
const message = unmarshalHidRpcMessage(new Uint8Array(e.data));
if (!message) {
console.warn("Received invalid HID RPC message", e.data);
return;
}
if (message.type === HID_RPC_MESSAGE_TYPES.Handshake) {
setHandshakeCompleted(true);
}
onHidRpcMessage?.(message);
};
rpcHidChannel.addEventListener("message", messageHandler);
return () => {
rpcHidChannel.removeEventListener("message", messageHandler);
};
},
[rpcHidChannel, onHidRpcMessage, setHandshakeCompleted, doHandshake]);
return {
reportKeyboardEvent,
reportKeypressEvent,
reportAbsMouseEvent,
reportRelMouseEvent,
handshakeCompleted,
};
}

76
ui/src/hooks/useKeyboard.ts
View File

@ -1,12 +1,13 @@
import { useCallback } from "react";
import { KeysDownState, useHidStore, useRTCStore, hidKeyBufferSize, hidErrorRollOver } from "@/hooks/stores";
import { hidErrorRollOver, hidKeyBufferSize, KeysDownState, useHidStore, useRTCStore } from "@/hooks/stores";
import { JsonRpcResponse, useJsonRpc } from "@/hooks/useJsonRpc";
import { HID_RPC_MESSAGE_TYPES, useHidRpc } from "@/hooks/useHidRpc";
import { hidKeyToModifierMask, keys, modifiers } from "@/keyboardMappings";
export default function useKeyboard() {
const { send } = useJsonRpc();
const { rpcDataChannel } = useRTCStore();
const { rpcDataChannel, rpcHidChannel } = useRTCStore();
const { keysDownState, setKeysDownState } = useHidStore();
// INTRODUCTION: The earlier version of the JetKVM device shipped with all keyboard state
@ -19,7 +20,19 @@ export default function useKeyboard() {
// dynamically set when the device responds to the first key press event or reports its
// keysDownState when queried since the keyPressReport was introduced together with the
// getKeysDownState API.
const { keyPressReportApiAvailable, setkeyPressReportApiAvailable} = useHidStore();
const { keyPressReportApiAvailable, setkeyPressReportApiAvailable } = useHidStore();
// HidRPC is a binary format for exchanging keyboard and mouse events
const { reportKeyboardEvent, reportKeypressEvent } = useHidRpc((message) => {
if (message.type === HID_RPC_MESSAGE_TYPES.KeysDownState) {
if (!message.keysDownState) {
return;
}
setKeysDownState(message.keysDownState);
setkeyPressReportApiAvailable(true);
}
});
// sendKeyboardEvent is used to send the full keyboard state to the device for macro handling
// and resetting keyboard state. It sends the keys currently pressed and the modifier state.
@ -27,9 +40,16 @@ export default function useKeyboard() {
// or just accept the state if it does not support (returning no result)
const sendKeyboardEvent = useCallback(
async (state: KeysDownState) => {
if (rpcDataChannel?.readyState !== "open") return;
if (rpcDataChannel?.readyState !== "open" && rpcHidChannel?.readyState !== "open") return;
console.debug(`Send keyboardReport keys: ${state.keys}, modifier: ${state.modifier}`);
if (rpcHidChannel?.readyState === "open") {
console.debug("Sending keyboard report via HidRPC");
reportKeyboardEvent(state.keys, state.modifier);
return;
}
send("keyboardReport", { keys: state.keys, modifier: state.modifier }, (resp: JsonRpcResponse) => {
if ("error" in resp) {
console.error(`Failed to send keyboard report ${state}`, resp.error);
@ -44,13 +64,20 @@ export default function useKeyboard() {
} else {
// older devices versions do not return the keyDownState
// so we just pretend they accepted what we sent
setKeysDownState(state);
setKeysDownState(state);
setkeyPressReportApiAvailable(false); // we ALSO know they do not support keyPressReport
}
}
});
},
[rpcDataChannel?.readyState, send, setKeysDownState, setkeyPressReportApiAvailable],
[
rpcDataChannel?.readyState,
rpcHidChannel?.readyState,
send,
reportKeyboardEvent,
setKeysDownState,
setkeyPressReportApiAvailable,
],
);
// sendKeypressEvent is used to send a single key press/release event to the device.
@ -61,9 +88,16 @@ export default function useKeyboard() {
// in client/browser-side code using simulateDeviceSideKeyHandlingForLegacyDevices.
const sendKeypressEvent = useCallback(
async (key: number, press: boolean) => {
if (rpcDataChannel?.readyState !== "open") return;
if (rpcDataChannel?.readyState !== "open" && rpcHidChannel?.readyState !== "open") return;
console.debug(`Send keypressEvent key: ${key}, press: ${press}`);
if (rpcHidChannel?.readyState === "open") {
console.debug("Sending keypress event via HidRPC");
reportKeypressEvent(key, press);
return;
}
send("keypressReport", { key, press }, (resp: JsonRpcResponse) => {
if ("error" in resp) {
// -32601 means the method is not supported because the device is running an older version
@ -83,7 +117,14 @@ export default function useKeyboard() {
}
});
},
[rpcDataChannel?.readyState, send, setkeyPressReportApiAvailable, setKeysDownState],
[
rpcDataChannel?.readyState,
rpcHidChannel?.readyState,
send,
setkeyPressReportApiAvailable,
setKeysDownState,
reportKeypressEvent,
],
);
// resetKeyboardState is used to reset the keyboard state to no keys pressed and no modifiers.
@ -135,9 +176,15 @@ export default function useKeyboard() {
// It then sends the full keyboard state to the device.
const handleKeyPress = useCallback(
async (key: number, press: boolean) => {
if (rpcDataChannel?.readyState !== "open") return;
if (rpcDataChannel?.readyState !== "open" && rpcHidChannel?.readyState !== "open") return;
if ((key || 0) === 0) return; // ignore zero key presses (they are bad mappings)
if (rpcHidChannel?.readyState === "open") {
console.debug("Sending keypress event via HidRPC");
reportKeypressEvent(key, press);
return;
}
if (keyPressReportApiAvailable) {
// if the keyPress api is available, we can just send the key press event
sendKeypressEvent(key, press);
@ -152,7 +199,16 @@ export default function useKeyboard() {
}
}
},
[keyPressReportApiAvailable, keysDownState, resetKeyboardState, rpcDataChannel?.readyState, sendKeyboardEvent, sendKeypressEvent],
[
keyPressReportApiAvailable,
keysDownState,
resetKeyboardState,
rpcDataChannel?.readyState,
rpcHidChannel?.readyState,
sendKeyboardEvent,
sendKeypressEvent,
reportKeypressEvent,
],
);
// IMPORTANT: See the keyPressReportApiAvailable comment above for the reason this exists

10
ui/src/routes/devices.$id.tsx
View File

@ -135,7 +135,8 @@ export default function KvmIdRoute() {
setRpcDataChannel,
isTurnServerInUse, setTurnServerInUse,
rpcDataChannel,
setTransceiver
setTransceiver,
setRpcHidChannel,
} = useRTCStore();
const location = useLocation();
@ -482,6 +483,12 @@ export default function KvmIdRoute() {
setRpcDataChannel(rpcDataChannel);
};
const rpcHidChannel = pc.createDataChannel("hidrpc");
rpcHidChannel.binaryType = "arraybuffer";
rpcHidChannel.onopen = () => {
setRpcHidChannel(rpcHidChannel);
};
setPeerConnection(pc);
}, [
cleanupAndStopReconnecting,
@ -492,6 +499,7 @@ export default function KvmIdRoute() {
setPeerConnection,
setPeerConnectionState,
setRpcDataChannel,
setRpcHidChannel,
setTransceiver,
]);

4
usb.go
View File

@ -27,13 +27,13 @@ func initUsbGadget() {
gadget.SetOnKeyboardStateChange(func(state usbgadget.KeyboardState) {
if currentSession != nil {
writeJSONRPCEvent("keyboardLedState", state, currentSession)
reportHidRpcKeyboardLedState(state, currentSession)
}
})
gadget.SetOnKeysDownChange(func(state usbgadget.KeysDownState) {
if currentSession != nil {
writeJSONRPCEvent("keysDownState", state, currentSession)
reportHidRpcKeysDownState(state, currentSession)
}
})

39
webrtc.go
View File

@ -22,7 +22,10 @@ type Session struct {
RPCChannel *webrtc.DataChannel
HidChannel *webrtc.DataChannel
shouldUmountVirtualMedia bool
rpcQueue chan webrtc.DataChannelMessage
hidRpcAvailable bool
hidQueue chan webrtc.DataChannelMessage
rpcQueue chan webrtc.DataChannelMessage
}
type SessionConfig struct {
@ -105,17 +108,49 @@ func newSession(config SessionConfig) (*Session, error) {
scopedLogger.Warn().Err(err).Msg("Failed to create PeerConnection")
return nil, err
}
session := &Session{peerConnection: peerConnection}
session.rpcQueue = make(chan webrtc.DataChannelMessage, 256)
session.hidQueue = make(chan webrtc.DataChannelMessage, 1024)
go func() {
for msg := range session.rpcQueue {
onRPCMessage(msg, session)
// TODO: only use goroutine if the task is asynchronous
go onRPCMessage(msg, session)
}
}()
go func() {
for msg := range session.hidQueue {
onHidMessage(msg.Data, session)
}
}()
peerConnection.OnDataChannel(func(d *webrtc.DataChannel) {
defer func() {
if r := recover(); r != nil {
scopedLogger.Warn().Interface("error", r).Msg("Recovered from panic in DataChannel handler")
}
}()
scopedLogger.Info().Str("label", d.Label()).Uint16("id", *d.ID()).Msg("New DataChannel")
switch d.Label() {
case "hidrpc":
session.HidChannel = d
d.OnMessage(func(msg webrtc.DataChannelMessage) {
if msg.IsString {
scopedLogger.Warn().Str("data", string(msg.Data)).Msg("received string data in HID RPC message handler")
return
}
if len(msg.Data) < 1 {
scopedLogger.Warn().Int("length", len(msg.Data)).Msg("received empty data in HID RPC message handler")
return
}
// Enqueue to ensure ordered processing
session.hidQueue <- msg
})
case "rpc":
session.RPCChannel = d
d.OnMessage(func(msg webrtc.DataChannelMessage) {