Superminaren
/
kvm
mirror of https://github.com/jetkvm/kvm.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge af8fff7cee into 94521ef6db

This commit is contained in:
Aveline 2025-08-30 18:50:11 +02:00 committed by GitHub
parent 94521ef6db af8fff7cee
commit 2d836970e8
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
20 changed files with 1088 additions and 26 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

152
hidrpc.go Normal file
View File

@ -0,0 +1,152 @@
package kvm
import (
"fmt"
"time"
"github.com/jetkvm/kvm/internal/hidrpc"
"github.com/jetkvm/kvm/internal/usbgadget"
)
func handleHidRPCMessage(message hidrpc.Message, session *Session) {
var rpcErr error
switch message.Type() {
case hidrpc.TypeHandshake:
message, err := hidrpc.NewHandshakeMessage().Marshal()
if err != nil {
logger.Warn().Err(err).Msg("failed to marshal handshake message")
return
}
if err := session.HidChannel.Send(message); err != nil {
logger.Warn().Err(err).Msg("failed to send handshake message")
return
}
session.hidRPCAvailable = true
case hidrpc.TypeKeypressReport, hidrpc.TypeKeyboardReport:
keysDownState, err := handleHidRPCKeyboardInput(message)
if keysDownState != nil {
session.reportHidRPCKeysDownState(*keysDownState)
}
rpcErr = err
case hidrpc.TypePointerReport:
pointerReport, err := message.PointerReport()
if err != nil {
logger.Warn().Err(err).Msg("failed to get pointer report")
return
}
rpcErr = rpcAbsMouseReport(pointerReport.X, pointerReport.Y, pointerReport.Button)
case hidrpc.TypeMouseReport:
mouseReport, err := message.MouseReport()
if err != nil {
logger.Warn().Err(err).Msg("failed to get mouse report")
return
}
rpcErr = rpcRelMouseReport(mouseReport.DX, mouseReport.DY, mouseReport.Button)
default:
logger.Warn().Uint8("type", uint8(message.Type())).Msg("unknown HID RPC message type")
}
if rpcErr != nil {
logger.Warn().Err(rpcErr).Msg("failed to handle HID RPC message")
}
}
func onHidMessage(data []byte, session *Session) {
scopedLogger := hidRPCLogger.With().Bytes("data", data).Logger()
scopedLogger.Debug().Msg("HID RPC message received")
if len(data) < 1 {
scopedLogger.Warn().Int("length", len(data)).Msg("received empty data in HID RPC message handler")
return
}
var message hidrpc.Message
if err := hidrpc.Unmarshal(data, &message); err != nil {
scopedLogger.Warn().Err(err).Msg("failed to unmarshal HID RPC message")
return
}
scopedLogger = scopedLogger.With().Str("descr", message.String()).Logger()
t := time.Now()
r := make(chan interface{})
go func() {
handleHidRPCMessage(message, session)
r <- nil
}()
select {
case <-time.After(1 * time.Second):
scopedLogger.Warn().Msg("HID RPC message timed out")
case <-r:
scopedLogger.Debug().Dur("duration", time.Since(t)).Msg("HID RPC message handled")
}
}
func handleHidRPCKeyboardInput(message hidrpc.Message) (*usbgadget.KeysDownState, error) {
switch message.Type() {
case hidrpc.TypeKeypressReport:
keypressReport, err := message.KeypressReport()
if err != nil {
logger.Warn().Err(err).Msg("failed to get keypress report")
return nil, err
}
keysDownState, rpcError := rpcKeypressReport(keypressReport.Key, keypressReport.Press)
return &keysDownState, rpcError
case hidrpc.TypeKeyboardReport:
keyboardReport, err := message.KeyboardReport()
if err != nil {
logger.Warn().Err(err).Msg("failed to get keyboard report")
return nil, err
}
keysDownState, rpcError := rpcKeyboardReport(keyboardReport.Modifier, keyboardReport.Keys)
return &keysDownState, rpcError
}
return nil, fmt.Errorf("unknown HID RPC message type: %d", message.Type())
}
func reportHidRPC(params any, session *Session) {
var (
message []byte
err error
)
switch params := params.(type) {
case usbgadget.KeyboardState:
message, err = hidrpc.NewKeyboardLedMessage(params).Marshal()
case usbgadget.KeysDownState:
message, err = hidrpc.NewKeydownStateMessage(params).Marshal()
default:
err = fmt.Errorf("unknown HID RPC message type: %T", params)
}
if err != nil {
logger.Warn().Err(err).Msg("failed to marshal HID RPC message")
return
}
if message == nil {
logger.Warn().Msg("failed to marshal HID RPC message")
return
}
if err := session.HidChannel.Send(message); err != nil {
logger.Warn().Err(err).Msg("failed to send HID RPC message")
}
}
func (s *Session) reportHidRPCKeyboardLedState(state usbgadget.KeyboardState) {
if !s.hidRPCAvailable {
writeJSONRPCEvent("keyboardLedState", state, s)
}
reportHidRPC(state, s)
}
func (s *Session) reportHidRPCKeysDownState(state usbgadget.KeysDownState) {
if !s.hidRPCAvailable {
writeJSONRPCEvent("keysDownState", state, s)
}
reportHidRPC(state, s)
}

100
internal/hidrpc/hidrpc.go Normal file
View File

@ -0,0 +1,100 @@
package hidrpc
import (
"fmt"
"github.com/jetkvm/kvm/internal/usbgadget"
)
// MessageType is the type of the HID RPC message
type MessageType byte
const (
TypeHandshake MessageType = 0x01
TypeKeyboardReport MessageType = 0x02
TypePointerReport MessageType = 0x03
TypeWheelReport MessageType = 0x04
TypeKeypressReport MessageType = 0x05
TypeMouseReport MessageType = 0x06
TypeKeyboardLedState MessageType = 0x32
TypeKeydownState MessageType = 0x33
)
const (
Version byte = 0x01 // Version of the HID RPC protocol
)
// GetQueueIndex returns the index of the queue to which the message should be enqueued.
func GetQueueIndex(messageType MessageType) int {
switch messageType {
case TypeHandshake:
return 0
case TypeKeyboardReport, TypeKeypressReport, TypeKeyboardLedState, TypeKeydownState:
return 1
case TypePointerReport, TypeMouseReport, TypeWheelReport:
return 2
default:
return 3
}
}
// Unmarshal unmarshals the HID RPC message from the data.
func Unmarshal(data []byte, message *Message) error {
l := len(data)
if l < 1 {
return fmt.Errorf("invalid data length: %d", l)
}
message.t = MessageType(data[0])
message.d = data[1:]
return nil
}
// Marshal marshals the HID RPC message to the data.
func Marshal(message *Message) ([]byte, error) {
if message.t == 0 {
return nil, fmt.Errorf("invalid message type: %d", message.t)
}
data := make([]byte, len(message.d)+1)
data[0] = byte(message.t)
copy(data[1:], message.d)
return data, nil
}
// NewHandshakeMessage creates a new handshake message.
func NewHandshakeMessage() *Message {
return &Message{
t: TypeHandshake,
d: []byte{Version},
}
}
// NewKeyboardReportMessage creates a new keyboard report message.
func NewKeyboardReportMessage(keys []byte, modifier uint8) *Message {
return &Message{
t: TypeKeyboardReport,
d: append([]byte{modifier}, keys...),
}
}
// NewKeyboardLedMessage creates a new keyboard LED message.
func NewKeyboardLedMessage(state usbgadget.KeyboardState) *Message {
return &Message{
t: TypeKeyboardLedState,
d: []byte{state.Byte()},
}
}
// NewKeydownStateMessage creates a new keydown state message.
func NewKeydownStateMessage(state usbgadget.KeysDownState) *Message {
data := make([]byte, len(state.Keys)+1)
data[0] = state.Modifier
copy(data[1:], state.Keys)
return &Message{
t: TypeKeydownState,
d: data,
}
}

133
internal/hidrpc/message.go Normal file
View File

@ -0,0 +1,133 @@
package hidrpc
import (
"fmt"
)
// Message ..
type Message struct {
t MessageType
d []byte
}
// Marshal marshals the message to a byte array.
func (m *Message) Marshal() ([]byte, error) {
return Marshal(m)
}
func (m *Message) Type() MessageType {
return m.t
}
func (m *Message) String() string {
switch m.t {
case TypeHandshake:
return "Handshake"
case TypeKeypressReport:
if len(m.d) < 2 {
return fmt.Sprintf("KeypressReport{Malformed: %v}", m.d)
}
return fmt.Sprintf("KeypressReport{Key: %d, Press: %v}", m.d[0], m.d[1] == uint8(1))
case TypeKeyboardReport:
if len(m.d) < 2 {
return fmt.Sprintf("KeyboardReport{Malformed: %v}", m.d)
}
return fmt.Sprintf("KeyboardReport{Modifier: %d, Keys: %v}", m.d[0], m.d[1:])
case TypePointerReport:
if len(m.d) < 9 {
return fmt.Sprintf("PointerReport{Malformed: %v}", m.d)
}
return fmt.Sprintf("PointerReport{X: %d, Y: %d, Button: %d}", m.d[0:4], m.d[4:8], m.d[8])
case TypeMouseReport:
if len(m.d) < 3 {
return fmt.Sprintf("MouseReport{Malformed: %v}", m.d)
}
return fmt.Sprintf("MouseReport{DX: %d, DY: %d, Button: %d}", m.d[0], m.d[1], m.d[2])
default:
return fmt.Sprintf("Unknown{Type: %d, Data: %v}", m.t, m.d)
}
}
// KeypressReport ..
type KeypressReport struct {
Key byte
Press bool
}
// KeypressReport returns the keypress report from the message.
func (m *Message) KeypressReport() (KeypressReport, error) {
if m.t != TypeKeypressReport {
return KeypressReport{}, fmt.Errorf("invalid message type: %d", m.t)
}
return KeypressReport{
Key: m.d[0],
Press: m.d[1] == uint8(1),
}, nil
}
// KeyboardReport ..
type KeyboardReport struct {
Modifier byte
Keys []byte
}
// KeyboardReport returns the keyboard report from the message.
func (m *Message) KeyboardReport() (KeyboardReport, error) {
if m.t != TypeKeyboardReport {
return KeyboardReport{}, fmt.Errorf("invalid message type: %d", m.t)
}
return KeyboardReport{
Modifier: m.d[0],
Keys: m.d[1:],
}, nil
}
// PointerReport ..
type PointerReport struct {
X int
Y int
Button uint8
}
func toInt(b []byte) int {
return int(b[0])<<24 + int(b[1])<<16 + int(b[2])<<8 + int(b[3])<<0
}
// PointerReport returns the point report from the message.
func (m *Message) PointerReport() (PointerReport, error) {
if m.t != TypePointerReport {
return PointerReport{}, fmt.Errorf("invalid message type: %d", m.t)
}
if len(m.d) != 9 {
return PointerReport{}, fmt.Errorf("invalid message length: %d", len(m.d))
}
return PointerReport{
X: toInt(m.d[0:4]),
Y: toInt(m.d[4:8]),
Button: uint8(m.d[8]),
}, nil
}
// MouseReport ..
type MouseReport struct {
DX int8
DY int8
Button uint8
}
// MouseReport returns the mouse report from the message.
func (m *Message) MouseReport() (MouseReport, error) {
if m.t != TypeMouseReport {
return MouseReport{}, fmt.Errorf("invalid message type: %d", m.t)
}
return MouseReport{
DX: int8(m.d[0]),
DY: int8(m.d[1]),
Button: uint8(m.d[2]),
}, nil
}

4
internal/usbgadget/consts.go
View File

@ -1,3 +1,7 @@
package usbgadget
import "time"
const dwc3Path = "/sys/bus/platform/drivers/dwc3"
const hidWriteTimeout = 10 * time.Millisecond

11
internal/usbgadget/hid_keyboard.go
View File

@ -86,6 +86,12 @@ type KeyboardState struct {
Compose bool `json:"compose"`
Kana bool `json:"kana"`
Shift bool `json:"shift"` // This is not part of the main USB HID spec
raw byte
}
// Byte returns the raw byte representation of the keyboard state.
func (k *KeyboardState) Byte() byte {
return k.raw
}
func getKeyboardState(b byte) KeyboardState {
@ -97,6 +103,7 @@ func getKeyboardState(b byte) KeyboardState {
Compose: b&KeyboardLedMaskCompose != 0,
Kana: b&KeyboardLedMaskKana != 0,
Shift: b&KeyboardLedMaskShift != 0,
raw: b,
}
}
@ -151,7 +158,9 @@ func (u *UsbGadget) updateKeyDownState(state KeysDownState) {
u.keysDownState = state
if u.onKeysDownChange != nil {
u.log.Trace().Interface("state", state).Msg("calling onKeysDownChange")
(*u.onKeysDownChange)(state)
u.log.Trace().Interface("state", state).Msg("onKeysDownChange called")
}
}
@ -233,7 +242,7 @@ func (u *UsbGadget) keyboardWriteHidFile(modifier byte, keys []byte) error {
return err
}
_, err := u.keyboardHidFile.Write(append([]byte{modifier, 0x00}, keys[:hidKeyBufferSize]...))
_, err := writeWithTimeout(u.keyboardHidFile, append([]byte{modifier, 0x00}, keys[:hidKeyBufferSize]...))
if err != nil {
u.logWithSuppression("keyboardWriteHidFile", 100, u.log, err, "failed to write to hidg0")
u.keyboardHidFile.Close()

2
internal/usbgadget/hid_mouse_absolute.go
View File

@ -74,7 +74,7 @@ func (u *UsbGadget) absMouseWriteHidFile(data []byte) error {
}
}
_, err := u.absMouseHidFile.Write(data)
_, err := writeWithTimeout(u.absMouseHidFile, data)
if err != nil {
u.logWithSuppression("absMouseWriteHidFile", 100, u.log, err, "failed to write to hidg1")
u.absMouseHidFile.Close()

2
internal/usbgadget/hid_mouse_relative.go
View File

@ -64,7 +64,7 @@ func (u *UsbGadget) relMouseWriteHidFile(data []byte) error {
}
}
_, err := u.relMouseHidFile.Write(data)
_, err := writeWithTimeout(u.relMouseHidFile, data)
if err != nil {
u.logWithSuppression("relMouseWriteHidFile", 100, u.log, err, "failed to write to hidg2")
u.relMouseHidFile.Close()

20
internal/usbgadget/utils.go
View File

@ -3,10 +3,13 @@ package usbgadget
import (
"bytes"
"encoding/json"
"errors"
"fmt"
"os"
"path/filepath"
"strconv"
"strings"
"time"
"github.com/rs/zerolog"
)
@ -107,6 +110,23 @@ func compareFileContent(oldContent []byte, newContent []byte, looserMatch bool)
return false
}
func writeWithTimeout(file *os.File, data []byte) (n int, err error) {
if err := file.SetWriteDeadline(time.Now().Add(hidWriteTimeout)); err != nil {
return -1, err
}
n, err = file.Write(data)
if err == nil {
return
}
if errors.Is(err, os.ErrDeadlineExceeded) {
err = nil
}
return
}
func (u *UsbGadget) logWithSuppression(counterName string, every int, logger *zerolog.Logger, err error, msg string, args ...any) {
u.logSuppressionLock.Lock()
defer u.logSuppressionLock.Unlock()

2
jsonrpc.go
View File

@ -83,7 +83,7 @@ func writeJSONRPCEvent(event string, params any, session *Session) {
Str("data", requestString).
Logger()
scopedLogger.Info().Msg("sending JSONRPC event")
scopedLogger.Trace().Msg("sending JSONRPC event")
err = session.RPCChannel.SendText(requestString)
if err != nil {

1
log.go
View File

@ -19,6 +19,7 @@ var (
nbdLogger = logging.GetSubsystemLogger("nbd")
timesyncLogger = logging.GetSubsystemLogger("timesync")
jsonRpcLogger = logging.GetSubsystemLogger("jsonrpc")
hidRPCLogger = logging.GetSubsystemLogger("hidrpc")
watchdogLogger = logging.GetSubsystemLogger("watchdog")
websecureLogger = logging.GetSubsystemLogger("websecure")
otaLogger = logging.GetSubsystemLogger("ota")

10
ui/src/components/InfoBar.tsx
View File

@ -10,10 +10,12 @@ import {
VideoState
} from "@/hooks/stores";
import { keys, modifiers } from "@/keyboardMappings";
import { useHidRpc } from "@/hooks/useHidRpc";
export default function InfoBar() {
const { keysDownState } = useHidStore();
const { mouseX, mouseY, mouseMove } = useMouseStore();
const { rpcHidStatus } = useHidRpc();
const videoClientSize = useVideoStore(
(state: VideoState) => `${Math.round(state.clientWidth)}x${Math.round(state.clientHeight)}`,
@ -46,7 +48,7 @@ export default function InfoBar() {
const modifierNames = Object.entries(modifiers).filter(([_, mask]) => (activeModifierMask & mask) !== 0).map(([name, _]) => name);
const keyNames = Object.entries(keys).filter(([_, value]) => keysDown.includes(value)).map(([name, _]) => name);
return [...modifierNames,...keyNames].join(", ");
return [...modifierNames, ...keyNames].join(", ");
}, [keysDownState, showPressedKeys]);
return (
@ -100,6 +102,12 @@ export default function InfoBar() {
<span className="text-xs">{hdmiState}</span>
</div>
)}
{debugMode && (
<div className="flex w-[156px] items-center gap-x-1">
<span className="text-xs font-semibold">HidRPC State:</span>
<span className="text-xs">{rpcHidStatus}</span>
</div>
)}
{showPressedKeys && (
<div className="flex items-center gap-x-1">

36
ui/src/components/WebRTCVideo.tsx
View File

@ -8,6 +8,7 @@ import InfoBar from "@components/InfoBar";
import notifications from "@/notifications";
import useKeyboard from "@/hooks/useKeyboard";
import { useJsonRpc } from "@/hooks/useJsonRpc";
import { useHidRpc } from "@/hooks/useHidRpc";
import { cx } from "@/cva.config";
import { keys } from "@/keyboardMappings";
import {
@ -60,10 +61,11 @@ export default function WebRTCVideo() {
// Misc states and hooks
const { send } = useJsonRpc();
const { reportAbsMouseEvent, reportRelMouseEvent, rpcHidReady } = useHidRpc();
// Video-related
const handleResize = useCallback(
( { width, height }: { width: number | undefined; height: number | undefined }) => {
({ width, height }: { width: number | undefined; height: number | undefined }) => {
if (!videoElm.current) return;
// Do something with width and height, e.g.:
setVideoClientSize(width || 0, height || 0);
@ -222,10 +224,22 @@ export default function WebRTCVideo() {
if (settings.mouseMode !== "relative") return;
// if we ignore the event, double-click will not work
// if (x === 0 && y === 0 && buttons === 0) return;
send("relMouseReport", { dx: calcDelta(x), dy: calcDelta(y), buttons });
const dx = calcDelta(x);
const dy = calcDelta(y);
if (rpcHidReady) {
reportRelMouseEvent(dx, dy, buttons);
} else {
send("relMouseReport", { dx, dy, buttons });
}
setMouseMove({ x, y, buttons });
},
[send, setMouseMove, settings.mouseMode],
[
send,
reportRelMouseEvent,
setMouseMove,
settings.mouseMode,
rpcHidReady,
],
);
const relMouseMoveHandler = useCallback(
@ -243,11 +257,21 @@ export default function WebRTCVideo() {
const sendAbsMouseMovement = useCallback(
(x: number, y: number, buttons: number) => {
if (settings.mouseMode !== "absolute") return;
send("absMouseReport", { x, y, buttons });
if (rpcHidReady) {
reportAbsMouseEvent(x, y, buttons);
} else {
send("absMouseReport", { x, y, buttons });
}
// We set that for the debug info bar
setMousePosition(x, y);
},
[send, setMousePosition, settings.mouseMode],
[
send,
reportAbsMouseEvent,
setMousePosition,
settings.mouseMode,
rpcHidReady,
],
);
const absMouseMoveHandler = useCallback(
@ -357,7 +381,7 @@ export default function WebRTCVideo() {
}
console.debug(`Key down: ${hidKey}`);
handleKeyPress(hidKey, true);
if (!isKeyboardLockActive && hidKey === keys.MetaLeft) {
// If the left meta key was just pressed and we're not keyboard locked
// we'll never see the keyup event because the browser is going to lose

303
ui/src/hooks/hidRpc.ts Normal file
View File

@ -0,0 +1,303 @@
import { KeyboardLedState, KeysDownState } from "./stores";
export const HID_RPC_MESSAGE_TYPES = {
Handshake: 0x01,
KeyboardReport: 0x02,
PointerReport: 0x03,
WheelReport: 0x04,
KeypressReport: 0x05,
MouseReport: 0x06,
KeyboardLedState: 0x32,
KeysDownState: 0x33,
}
export type HidRpcMessageType = typeof HID_RPC_MESSAGE_TYPES[keyof typeof HID_RPC_MESSAGE_TYPES];
export const HID_RPC_VERSION = 0x01;
const withinUint8Range = (value: number) => {
return value >= 0 && value <= 255;
};
const fromInt32toUint8 = (n: number) => {
if (n !== n >> 0) {
throw new Error(`Number ${n} is not within the int32 range`);
}
return new Uint8Array([
(n >> 24) & 0xFF,
(n >> 16) & 0xFF,
(n >> 8) & 0xFF,
(n >> 0) & 0xFF,
]);
};
const fromInt8ToUint8 = (n: number) => {
if (n < -128 || n > 127) {
throw new Error(`Number ${n} is not within the int8 range`);
}
return (n >> 0) & 0xFF;
};
const keyboardLedStateMasks = {
num_lock: 1 << 0,
caps_lock: 1 << 1,
scroll_lock: 1 << 2,
compose: 1 << 3,
kana: 1 << 4,
shift: 1 << 6,
}
export class RpcMessage {
messageType: HidRpcMessageType;
constructor(messageType: HidRpcMessageType) {
this.messageType = messageType;
}
marshal(): Uint8Array {
throw new Error("Not implemented");
}
// @ts-expect-error: this is a base class, so we don't need to implement it
public static unmarshal(data: Uint8Array): RpcMessage | undefined {
throw new Error("Not implemented");
}
}
export class HandshakeMessage extends RpcMessage {
version: number;
constructor(version: number) {
super(HID_RPC_MESSAGE_TYPES.Handshake);
this.version = version;
}
marshal(): Uint8Array {
return new Uint8Array([this.messageType, this.version]);
}
public static unmarshal(data: Uint8Array): HandshakeMessage | undefined {
if (data.length < 1) {
throw new Error(`Invalid handshake message length: ${data.length}`);
}
return new HandshakeMessage(data[0]);
}
}
export class KeypressReportMessage extends RpcMessage {
private _key = 0;
private _press = false;
get key(): number {
return this._key;
}
set key(value: number) {
if (!withinUint8Range(value)) {
throw new Error(`Key ${value} is not within the uint8 range`);
}
this._key = value;
}
get press(): boolean {
return this._press;
}
set press(value: boolean) {
this._press = value;
}
constructor(key: number, press: boolean) {
super(HID_RPC_MESSAGE_TYPES.KeypressReport);
this.key = key;
this.press = press;
}
marshal(): Uint8Array {
return new Uint8Array([
this.messageType,
this.key,
this.press ? 1 : 0,
]);
}
public static unmarshal(data: Uint8Array): KeypressReportMessage | undefined {
if (data.length < 1) {
throw new Error(`Invalid keypress report message length: ${data.length}`);
}
return new KeypressReportMessage(data[0], data[1] === 1);
}
}
export class KeyboardReportMessage extends RpcMessage {
private _keys: number[] = [];
private _modifier = 0;
get keys(): number[] {
return this._keys;
}
set keys(value: number[]) {
value.forEach((k) => {
if (!withinUint8Range(k)) {
throw new Error(`Key ${k} is not within the uint8 range`);
}
});
this._keys = value;
}
get modifier(): number {
return this._modifier;
}
set modifier(value: number) {
if (!withinUint8Range(value)) {
throw new Error(`Modifier ${value} is not within the uint8 range`);
}
this._modifier = value;
}
constructor(keys: number[], modifier: number) {
super(HID_RPC_MESSAGE_TYPES.KeyboardReport);
this.keys = keys;
this.modifier = modifier;
}
marshal(): Uint8Array {
return new Uint8Array([
this.messageType,
this.modifier,
...this.keys,
]);
}
public static unmarshal(data: Uint8Array): KeyboardReportMessage | undefined {
if (data.length < 1) {
throw new Error(`Invalid keyboard report message length: ${data.length}`);
}
return new KeyboardReportMessage(Array.from(data.slice(1)), data[0]);
}
}
export class KeyboardLedStateMessage extends RpcMessage {
keyboardLedState: KeyboardLedState;
constructor(keyboardLedState: KeyboardLedState) {
super(HID_RPC_MESSAGE_TYPES.KeyboardLedState);
this.keyboardLedState = keyboardLedState;
}
public static unmarshal(data: Uint8Array): KeyboardLedStateMessage | undefined {
if (data.length < 1) {
throw new Error(`Invalid keyboard led state message length: ${data.length}`);
}
const s = data[0];
const state = {
num_lock: (s & keyboardLedStateMasks.num_lock) !== 0,
caps_lock: (s & keyboardLedStateMasks.caps_lock) !== 0,
scroll_lock: (s & keyboardLedStateMasks.scroll_lock) !== 0,
compose: (s & keyboardLedStateMasks.compose) !== 0,
kana: (s & keyboardLedStateMasks.kana) !== 0,
shift: (s & keyboardLedStateMasks.shift) !== 0,
} as KeyboardLedState;
return new KeyboardLedStateMessage(state);
}
}
export class KeysDownStateMessage extends RpcMessage {
keysDownState: KeysDownState;
constructor(keysDownState: KeysDownState) {
super(HID_RPC_MESSAGE_TYPES.KeysDownState);
this.keysDownState = keysDownState;
}
public static unmarshal(data: Uint8Array): KeysDownStateMessage | undefined {
if (data.length < 1) {
throw new Error(`Invalid keys down state message length: ${data.length}`);
}
return new KeysDownStateMessage({
modifier: data[0],
keys: Array.from(data.slice(1))
});
}
}
export class PointerReportMessage extends RpcMessage {
x: number;
y: number;
buttons: number;
constructor(x: number, y: number, buttons: number) {
super(HID_RPC_MESSAGE_TYPES.PointerReport);
this.x = x;
this.y = y;
this.buttons = buttons;
}
marshal(): Uint8Array {
return new Uint8Array([
this.messageType,
...fromInt32toUint8(this.x),
...fromInt32toUint8(this.y),
this.buttons,
]);
}
}
export class MouseReportMessage extends RpcMessage {
dx: number;
dy: number;
buttons: number;
constructor(dx: number, dy: number, buttons: number) {
super(HID_RPC_MESSAGE_TYPES.MouseReport);
this.dx = dx;
this.dy = dy;
this.buttons = buttons;
}
marshal(): Uint8Array {
return new Uint8Array([
this.messageType,
fromInt8ToUint8(this.dx),
fromInt8ToUint8(this.dy),
this.buttons,
]);
}
}
export const messageRegistry = {
[HID_RPC_MESSAGE_TYPES.Handshake]: HandshakeMessage,
[HID_RPC_MESSAGE_TYPES.KeysDownState]: KeysDownStateMessage,
[HID_RPC_MESSAGE_TYPES.KeyboardLedState]: KeyboardLedStateMessage,
[HID_RPC_MESSAGE_TYPES.KeyboardReport]: KeyboardReportMessage,
[HID_RPC_MESSAGE_TYPES.KeypressReport]: KeypressReportMessage,
}
export const unmarshalHidRpcMessage = (data: Uint8Array): RpcMessage | undefined => {
if (data.length < 1) {
throw new Error(`Invalid HID RPC message length: ${data.length}`);
}
const payload = data.slice(1);
const messageType = data[0];
if (!(messageType in messageRegistry)) {
throw new Error(`Unknown HID RPC message type: ${messageType}`);
}
return messageRegistry[messageType].unmarshal(payload);
};

12
ui/src/hooks/stores.ts
View File

@ -105,6 +105,12 @@ export interface RTCState {
setRpcDataChannel: (channel: RTCDataChannel) => void;
rpcDataChannel: RTCDataChannel | null;
rpcHidProtocolVersion: number | null;
setRpcHidProtocolVersion: (version: number) => void;
rpcHidChannel: RTCDataChannel | null;
setRpcHidChannel: (channel: RTCDataChannel) => void;
peerConnectionState: RTCPeerConnectionState | null;
setPeerConnectionState: (state: RTCPeerConnectionState) => void;
@ -151,6 +157,12 @@ export const useRTCStore = create<RTCState>(set => ({
rpcDataChannel: null,
setRpcDataChannel: (channel: RTCDataChannel) => set({ rpcDataChannel: channel }),
rpcHidProtocolVersion: null,
setRpcHidProtocolVersion: (version: number) => set({ rpcHidProtocolVersion: version }),
rpcHidChannel: null,
setRpcHidChannel: (channel: RTCDataChannel) => set({ rpcHidChannel: channel }),
transceiver: null,
setTransceiver: (transceiver: RTCRtpTransceiver) => set({ transceiver }),

147
ui/src/hooks/useHidRpc.ts Normal file
View File

@ -0,0 +1,147 @@
import { useCallback, useEffect, useMemo } from "react";
import { useRTCStore } from "@/hooks/stores";
import {
HID_RPC_VERSION,
HandshakeMessage,
KeyboardReportMessage,
KeypressReportMessage,
MouseReportMessage,
PointerReportMessage,
RpcMessage,
unmarshalHidRpcMessage,
} from "./hidRpc";
export function useHidRpc(onHidRpcMessage?: (payload: RpcMessage) => void) {
const { rpcHidChannel, setRpcHidProtocolVersion, rpcHidProtocolVersion } = useRTCStore();
const rpcHidReady = useMemo(() => {
return rpcHidChannel?.readyState === "open" && rpcHidProtocolVersion !== null;
}, [rpcHidChannel, rpcHidProtocolVersion]);
const rpcHidStatus = useMemo(() => {
if (!rpcHidChannel) return "N/A";
if (rpcHidChannel.readyState !== "open") return rpcHidChannel.readyState;
if (!rpcHidProtocolVersion) return "handshaking";
return `ready (v${rpcHidProtocolVersion})`;
}, [rpcHidChannel, rpcHidProtocolVersion]);
const sendMessage = useCallback((message: RpcMessage, ignoreHandshakeState = false) => {
if (rpcHidChannel?.readyState !== "open") return;
if (!rpcHidReady && !ignoreHandshakeState) return;
let data: Uint8Array | undefined;
try {
data = message.marshal();
} catch (e) {
console.error("Failed to send HID RPC message", e);
}
if (!data) return;
rpcHidChannel?.send(data as unknown as ArrayBuffer);
}, [rpcHidChannel, rpcHidReady]);
const reportKeyboardEvent = useCallback(
(keys: number[], modifier: number) => {
sendMessage(new KeyboardReportMessage(keys, modifier));
}, [sendMessage],
);
const reportKeypressEvent = useCallback(
(key: number, press: boolean) => {
sendMessage(new KeypressReportMessage(key, press));
},
[sendMessage],
);
const reportAbsMouseEvent = useCallback(
(x: number, y: number, buttons: number) => {
sendMessage(new PointerReportMessage(x, y, buttons));
},
[sendMessage],
);
const reportRelMouseEvent = useCallback(
(dx: number, dy: number, buttons: number) => {
sendMessage(new MouseReportMessage(dx, dy, buttons));
},
[sendMessage],
);
const sendHandshake = useCallback(() => {
if (rpcHidProtocolVersion) return;
if (!rpcHidChannel) return;
sendMessage(new HandshakeMessage(HID_RPC_VERSION), true);
}, [rpcHidChannel, rpcHidProtocolVersion, sendMessage]);
const handleHandshake = useCallback((message: HandshakeMessage) => {
if (!message.version) {
console.error("Received handshake message without version", message);
return;
}
if (message.version < HID_RPC_VERSION) {
console.error("Server is using an older HID RPC version than the client", message);
return;
}
setRpcHidProtocolVersion(message.version);
}, [setRpcHidProtocolVersion]);
useEffect(() => {
if (!rpcHidChannel) return;
// send handshake message
sendHandshake();
const messageHandler = (e: MessageEvent) => {
if (typeof e.data === "string") {
console.warn("Received string data in HID RPC message handler", e.data);
return;
}
const message = unmarshalHidRpcMessage(new Uint8Array(e.data));
if (!message) {
console.warn("Received invalid HID RPC message", e.data);
return;
}
console.debug("Received HID RPC message", message);
switch (message.constructor) {
case HandshakeMessage:
handleHandshake(message as HandshakeMessage);
break;
default:
// not all events are handled here, the rest are handled by the onHidRpcMessage callback
break;
}
onHidRpcMessage?.(message);
};
rpcHidChannel.addEventListener("message", messageHandler);
return () => {
rpcHidChannel.removeEventListener("message", messageHandler);
};
},
[
rpcHidChannel,
onHidRpcMessage,
setRpcHidProtocolVersion,
sendHandshake,
handleHandshake,
],
);
return {
reportKeyboardEvent,
reportKeypressEvent,
reportAbsMouseEvent,
reportRelMouseEvent,
rpcHidProtocolVersion,
rpcHidReady,
rpcHidStatus,
};
}

80
ui/src/hooks/useKeyboard.ts
View File

@ -1,13 +1,15 @@
import { useCallback } from "react";
import { KeysDownState, useHidStore, useRTCStore, hidKeyBufferSize, hidErrorRollOver } from "@/hooks/stores";
import { hidErrorRollOver, hidKeyBufferSize, KeysDownState, useHidStore, useRTCStore } from "@/hooks/stores";
import { JsonRpcResponse, useJsonRpc } from "@/hooks/useJsonRpc";
import { useHidRpc } from "@/hooks/useHidRpc";
import { KeyboardLedStateMessage, KeysDownStateMessage } from "@/hooks/hidRpc";
import { hidKeyToModifierMask, keys, modifiers } from "@/keyboardMappings";
export default function useKeyboard() {
const { send } = useJsonRpc();
const { rpcDataChannel } = useRTCStore();
const { keysDownState, setKeysDownState } = useHidStore();
const { keysDownState, setKeysDownState, setKeyboardLedState } = useHidStore();
// INTRODUCTION: The earlier version of the JetKVM device shipped with all keyboard state
// being tracked on the browser/client-side. When adding the keyPressReport API to the
@ -19,7 +21,22 @@ export default function useKeyboard() {
// dynamically set when the device responds to the first key press event or reports its
// keysDownState when queried since the keyPressReport was introduced together with the
// getKeysDownState API.
const { keyPressReportApiAvailable, setkeyPressReportApiAvailable} = useHidStore();
const { keyPressReportApiAvailable, setkeyPressReportApiAvailable } = useHidStore();
// HidRPC is a binary format for exchanging keyboard and mouse events
const { reportKeyboardEvent, reportKeypressEvent, rpcHidReady } = useHidRpc((message) => {
switch (message.constructor) {
case KeysDownStateMessage:
setKeysDownState((message as KeysDownStateMessage).keysDownState);
setkeyPressReportApiAvailable(true);
break;
case KeyboardLedStateMessage:
setKeyboardLedState((message as KeyboardLedStateMessage).keyboardLedState);
break;
default:
break;
}
});
// sendKeyboardEvent is used to send the full keyboard state to the device for macro handling
// and resetting keyboard state. It sends the keys currently pressed and the modifier state.
@ -27,9 +44,16 @@ export default function useKeyboard() {
// or just accept the state if it does not support (returning no result)
const sendKeyboardEvent = useCallback(
async (state: KeysDownState) => {
if (rpcDataChannel?.readyState !== "open") return;
if (rpcDataChannel?.readyState !== "open" && !rpcHidReady) return;
console.debug(`Send keyboardReport keys: ${state.keys}, modifier: ${state.modifier}`);
if (rpcHidReady) {
console.debug("Sending keyboard report via HidRPC");
reportKeyboardEvent(state.keys, state.modifier);
return;
}
send("keyboardReport", { keys: state.keys, modifier: state.modifier }, (resp: JsonRpcResponse) => {
if ("error" in resp) {
console.error(`Failed to send keyboard report ${state}`, resp.error);
@ -44,13 +68,20 @@ export default function useKeyboard() {
} else {
// older devices versions do not return the keyDownState
// so we just pretend they accepted what we sent
setKeysDownState(state);
setKeysDownState(state);
setkeyPressReportApiAvailable(false); // we ALSO know they do not support keyPressReport
}
}
});
},
[rpcDataChannel?.readyState, send, setKeysDownState, setkeyPressReportApiAvailable],
[
rpcDataChannel?.readyState,
rpcHidReady,
send,
reportKeyboardEvent,
setKeysDownState,
setkeyPressReportApiAvailable,
],
);
// sendKeypressEvent is used to send a single key press/release event to the device.
@ -61,9 +92,16 @@ export default function useKeyboard() {
// in client/browser-side code using simulateDeviceSideKeyHandlingForLegacyDevices.
const sendKeypressEvent = useCallback(
async (key: number, press: boolean) => {
if (rpcDataChannel?.readyState !== "open") return;
if (rpcDataChannel?.readyState !== "open" && !rpcHidReady) return;
console.debug(`Send keypressEvent key: ${key}, press: ${press}`);
if (rpcHidReady) {
console.debug("Sending keypress event via HidRPC");
reportKeypressEvent(key, press);
return;
}
send("keypressReport", { key, press }, (resp: JsonRpcResponse) => {
if ("error" in resp) {
// -32601 means the method is not supported because the device is running an older version
@ -83,7 +121,14 @@ export default function useKeyboard() {
}
});
},
[rpcDataChannel?.readyState, send, setkeyPressReportApiAvailable, setKeysDownState],
[
rpcDataChannel?.readyState,
rpcHidReady,
send,
setkeyPressReportApiAvailable,
setKeysDownState,
reportKeypressEvent,
],
);
// resetKeyboardState is used to reset the keyboard state to no keys pressed and no modifiers.
@ -135,9 +180,15 @@ export default function useKeyboard() {
// It then sends the full keyboard state to the device.
const handleKeyPress = useCallback(
async (key: number, press: boolean) => {
if (rpcDataChannel?.readyState !== "open") return;
if (rpcDataChannel?.readyState !== "open" && !rpcHidReady) return;
if ((key || 0) === 0) return; // ignore zero key presses (they are bad mappings)
if (rpcHidReady) {
console.debug("Sending keypress event via HidRPC");
reportKeypressEvent(key, press);
return;
}
if (keyPressReportApiAvailable) {
// if the keyPress api is available, we can just send the key press event
sendKeypressEvent(key, press);
@ -152,7 +203,16 @@ export default function useKeyboard() {
}
}
},
[keyPressReportApiAvailable, keysDownState, resetKeyboardState, rpcDataChannel?.readyState, sendKeyboardEvent, sendKeypressEvent],
[
keyPressReportApiAvailable,
keysDownState,
resetKeyboardState,
rpcDataChannel?.readyState,
rpcHidReady,
sendKeyboardEvent,
sendKeypressEvent,
reportKeypressEvent,
],
);
// IMPORTANT: See the keyPressReportApiAvailable comment above for the reason this exists

10
ui/src/routes/devices.$id.tsx
View File

@ -135,7 +135,8 @@ export default function KvmIdRoute() {
setRpcDataChannel,
isTurnServerInUse, setTurnServerInUse,
rpcDataChannel,
setTransceiver
setTransceiver,
setRpcHidChannel,
} = useRTCStore();
const location = useLocation();
@ -482,6 +483,12 @@ export default function KvmIdRoute() {
setRpcDataChannel(rpcDataChannel);
};
const rpcHidChannel = pc.createDataChannel("hidrpc");
rpcHidChannel.binaryType = "arraybuffer";
rpcHidChannel.onopen = () => {
setRpcHidChannel(rpcHidChannel);
};
setPeerConnection(pc);
}, [
cleanupAndStopReconnecting,
@ -492,6 +499,7 @@ export default function KvmIdRoute() {
setPeerConnection,
setPeerConnectionState,
setRpcDataChannel,
setRpcHidChannel,
setTransceiver,
]);

3
ui/vite.config.ts
View File

@ -28,6 +28,9 @@ export default defineConfig(({ mode, command }) => {
return {
plugins,
esbuild: {
pure: ["console.debug"],
},
build: { outDir: isCloud ? "dist" : "../static" },
server: {
host: "0.0.0.0",

4
usb.go
View File

@ -27,13 +27,13 @@ func initUsbGadget() {
gadget.SetOnKeyboardStateChange(func(state usbgadget.KeyboardState) {
if currentSession != nil {
writeJSONRPCEvent("keyboardLedState", state, currentSession)
currentSession.reportHidRPCKeyboardLedState(state)
}
})
gadget.SetOnKeysDownChange(func(state usbgadget.KeysDownState) {
if currentSession != nil {
writeJSONRPCEvent("keysDownState", state, currentSession)
currentSession.reportHidRPCKeysDownState(state)
}
})

82
webrtc.go
View File

@ -6,10 +6,12 @@ import (
"encoding/json"
"net"
"strings"
"sync"
"github.com/coder/websocket"
"github.com/coder/websocket/wsjson"
"github.com/gin-gonic/gin"
"github.com/jetkvm/kvm/internal/hidrpc"
"github.com/jetkvm/kvm/internal/logging"
"github.com/pion/webrtc/v4"
"github.com/rs/zerolog"
@ -22,7 +24,12 @@ type Session struct {
RPCChannel *webrtc.DataChannel
HidChannel *webrtc.DataChannel
shouldUmountVirtualMedia bool
rpcQueue chan webrtc.DataChannelMessage
rpcQueue chan webrtc.DataChannelMessage
hidRPCAvailable bool
hidQueueLock sync.Mutex
hidQueue []chan webrtc.DataChannelMessage
}
type SessionConfig struct {
@ -67,6 +74,23 @@ func (s *Session) ExchangeOffer(offerStr string) (string, error) {
return base64.StdEncoding.EncodeToString(localDescription), nil
}
func (s *Session) initQueues() {
s.hidQueueLock.Lock()
defer s.hidQueueLock.Unlock()
s.hidQueue = make([]chan webrtc.DataChannelMessage, 0)
for i := 0; i < 4; i++ {
q := make(chan webrtc.DataChannelMessage, 256)
s.hidQueue = append(s.hidQueue, q)
}
}
func (s *Session) handleQueues(index int) {
for msg := range s.hidQueue[index] {
onHidMessage(msg.Data, s)
}
}
func newSession(config SessionConfig) (*Session, error) {
webrtcSettingEngine := webrtc.SettingEngine{
LoggerFactory: logging.GetPionDefaultLoggerFactory(),
@ -105,17 +129,64 @@ func newSession(config SessionConfig) (*Session, error) {
scopedLogger.Warn().Err(err).Msg("Failed to create PeerConnection")
return nil, err
}
session := &Session{peerConnection: peerConnection}
session.rpcQueue = make(chan webrtc.DataChannelMessage, 256)
session.initQueues()
go func() {
for msg := range session.rpcQueue {
onRPCMessage(msg, session)
// TODO: only use goroutine if the task is asynchronous
go onRPCMessage(msg, session)
}
}()
for i := 0; i < len(session.hidQueue); i++ {
go session.handleQueues(i)
}
peerConnection.OnDataChannel(func(d *webrtc.DataChannel) {
defer func() {
if r := recover(); r != nil {
scopedLogger.Error().Interface("error", r).Msg("Recovered from panic in DataChannel handler")
}
}()
scopedLogger.Info().Str("label", d.Label()).Uint16("id", *d.ID()).Msg("New DataChannel")
switch d.Label() {
case "hidrpc":
session.HidChannel = d
d.OnMessage(func(msg webrtc.DataChannelMessage) {
l := scopedLogger.With().Str("data", string(msg.Data)).Int("length", len(msg.Data)).Logger()
if msg.IsString {
l.Warn().Msg("received string data in HID RPC message handler")
return
}
if len(msg.Data) < 1 {
l.Warn().Msg("received empty data in HID RPC message handler")
return
}
l.Trace().Msg("received data in HID RPC message handler")
// Enqueue to ensure ordered processing
queueIndex := hidrpc.GetQueueIndex(hidrpc.MessageType(msg.Data[0]))
if queueIndex >= len(session.hidQueue) || queueIndex < 0 {
l.Warn().Int("queueIndex", queueIndex).Msg("received data in HID RPC message handler, but queue index not found")
queueIndex = 3
}
queue := session.hidQueue[queueIndex]
if queue != nil {
queue <- msg
} else {
l.Warn().Int("queueIndex", queueIndex).Msg("received data in HID RPC message handler, but queue is nil")
return
}
})
case "rpc":
session.RPCChannel = d
d.OnMessage(func(msg webrtc.DataChannelMessage) {
@ -198,6 +269,13 @@ func newSession(config SessionConfig) (*Session, error) {
close(session.rpcQueue)
session.rpcQueue = nil
}
// Stop HID RPC processor
for i := 0; i < len(session.hidQueue); i++ {
close(session.hidQueue[i])
session.hidQueue[i] = nil
}
if session.shouldUmountVirtualMedia {
if err := rpcUnmountImage(); err != nil {
scopedLogger.Warn().Err(err).Msg("unmount image failed on connection close")