Superminaren
/
kvm
mirror of https://github.com/jetkvm/kvm.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

feat: hid rpc channel (#755) 

* feat: use hidRpcChannel to save bandwidth

* chore: simplify handshake of hid rpc

* add logs

* chore: add timeout when writing to hid endpoints

* fix issues

* chore: show hid rpc version

* refactor hidrpc marshal / unmarshal

* add queues for keyboard / mouse event

* chore: change logging level of JSONRPC send event to trace

* minor changes related to logging

* fix: nil check

* chore: add comments and remove unused code

* add useMouse

* chore: log msg data only when debug or trace mode

* chore: make tslint happy

* chore: unlock keyboardStateLock before calling onKeysDownChange

* chore: remove keyPressReportApiAvailable

* chore: change version handle

* chore: clean up unused functions

* remove comments
This commit is contained in:
Aveline 2025-09-04 22:27:56 +02:00 committed by GitHub
parent e8ef82e582
commit bcc307b147
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
21 changed files with 1292 additions and 216 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

162
hidrpc.go Normal file
View File

@ -0,0 +1,162 @@
package kvm
import (
"fmt"
"time"
"github.com/jetkvm/kvm/internal/hidrpc"
"github.com/jetkvm/kvm/internal/usbgadget"
)
func handleHidRPCMessage(message hidrpc.Message, session *Session) {
var rpcErr error
switch message.Type() {
case hidrpc.TypeHandshake:
message, err := hidrpc.NewHandshakeMessage().Marshal()
if err != nil {
logger.Warn().Err(err).Msg("failed to marshal handshake message")
return
}
if err := session.HidChannel.Send(message); err != nil {
logger.Warn().Err(err).Msg("failed to send handshake message")
return
}
session.hidRPCAvailable = true
case hidrpc.TypeKeypressReport, hidrpc.TypeKeyboardReport:
keysDownState, err := handleHidRPCKeyboardInput(message)
if keysDownState != nil {
session.reportHidRPCKeysDownState(*keysDownState)
}
rpcErr = err
case hidrpc.TypePointerReport:
pointerReport, err := message.PointerReport()
if err != nil {
logger.Warn().Err(err).Msg("failed to get pointer report")
return
}
rpcErr = rpcAbsMouseReport(pointerReport.X, pointerReport.Y, pointerReport.Button)
case hidrpc.TypeMouseReport:
mouseReport, err := message.MouseReport()
if err != nil {
logger.Warn().Err(err).Msg("failed to get mouse report")
return
}
rpcErr = rpcRelMouseReport(mouseReport.DX, mouseReport.DY, mouseReport.Button)
default:
logger.Warn().Uint8("type", uint8(message.Type())).Msg("unknown HID RPC message type")
}
if rpcErr != nil {
logger.Warn().Err(rpcErr).Msg("failed to handle HID RPC message")
}
}
func onHidMessage(data []byte, session *Session) {
scopedLogger := hidRPCLogger.With().Bytes("data", data).Logger()
scopedLogger.Debug().Msg("HID RPC message received")
if len(data) < 1 {
scopedLogger.Warn().Int("length", len(data)).Msg("received empty data in HID RPC message handler")
return
}
var message hidrpc.Message
if err := hidrpc.Unmarshal(data, &message); err != nil {
scopedLogger.Warn().Err(err).Msg("failed to unmarshal HID RPC message")
return
}
scopedLogger = scopedLogger.With().Str("descr", message.String()).Logger()
t := time.Now()
r := make(chan interface{})
go func() {
handleHidRPCMessage(message, session)
r <- nil
}()
select {
case <-time.After(1 * time.Second):
scopedLogger.Warn().Msg("HID RPC message timed out")
case <-r:
scopedLogger.Debug().Dur("duration", time.Since(t)).Msg("HID RPC message handled")
}
}
func handleHidRPCKeyboardInput(message hidrpc.Message) (*usbgadget.KeysDownState, error) {
switch message.Type() {
case hidrpc.TypeKeypressReport:
keypressReport, err := message.KeypressReport()
if err != nil {
logger.Warn().Err(err).Msg("failed to get keypress report")
return nil, err
}
keysDownState, rpcError := rpcKeypressReport(keypressReport.Key, keypressReport.Press)
return &keysDownState, rpcError
case hidrpc.TypeKeyboardReport:
keyboardReport, err := message.KeyboardReport()
if err != nil {
logger.Warn().Err(err).Msg("failed to get keyboard report")
return nil, err
}
keysDownState, rpcError := rpcKeyboardReport(keyboardReport.Modifier, keyboardReport.Keys)
return &keysDownState, rpcError
}
return nil, fmt.Errorf("unknown HID RPC message type: %d", message.Type())
}
func reportHidRPC(params any, session *Session) {
if session == nil {
logger.Warn().Msg("session is nil, skipping reportHidRPC")
return
}
if !session.hidRPCAvailable || session.HidChannel == nil {
logger.Warn().Msg("HID RPC is not available, skipping reportHidRPC")
return
}
var (
message []byte
err error
)
switch params := params.(type) {
case usbgadget.KeyboardState:
message, err = hidrpc.NewKeyboardLedMessage(params).Marshal()
case usbgadget.KeysDownState:
message, err = hidrpc.NewKeydownStateMessage(params).Marshal()
default:
err = fmt.Errorf("unknown HID RPC message type: %T", params)
}
if err != nil {
logger.Warn().Err(err).Msg("failed to marshal HID RPC message")
return
}
if message == nil {
logger.Warn().Msg("failed to marshal HID RPC message")
return
}
if err := session.HidChannel.Send(message); err != nil {
logger.Warn().Err(err).Msg("failed to send HID RPC message")
}
}
func (s *Session) reportHidRPCKeyboardLedState(state usbgadget.KeyboardState) {
if !s.hidRPCAvailable {
writeJSONRPCEvent("keyboardLedState", state, s)
}
reportHidRPC(state, s)
}
func (s *Session) reportHidRPCKeysDownState(state usbgadget.KeysDownState) {
if !s.hidRPCAvailable {
writeJSONRPCEvent("keysDownState", state, s)
}
reportHidRPC(state, s)
}

100
internal/hidrpc/hidrpc.go Normal file
View File

@ -0,0 +1,100 @@
package hidrpc
import (
"fmt"
"github.com/jetkvm/kvm/internal/usbgadget"
)
// MessageType is the type of the HID RPC message
type MessageType byte
const (
TypeHandshake MessageType = 0x01
TypeKeyboardReport MessageType = 0x02
TypePointerReport MessageType = 0x03
TypeWheelReport MessageType = 0x04
TypeKeypressReport MessageType = 0x05
TypeMouseReport MessageType = 0x06
TypeKeyboardLedState MessageType = 0x32
TypeKeydownState MessageType = 0x33
)
const (
Version byte = 0x01 // Version of the HID RPC protocol
)
// GetQueueIndex returns the index of the queue to which the message should be enqueued.
func GetQueueIndex(messageType MessageType) int {
switch messageType {
case TypeHandshake:
return 0
case TypeKeyboardReport, TypeKeypressReport, TypeKeyboardLedState, TypeKeydownState:
return 1
case TypePointerReport, TypeMouseReport, TypeWheelReport:
return 2
default:
return 3
}
}
// Unmarshal unmarshals the HID RPC message from the data.
func Unmarshal(data []byte, message *Message) error {
l := len(data)
if l < 1 {
return fmt.Errorf("invalid data length: %d", l)
}
message.t = MessageType(data[0])
message.d = data[1:]
return nil
}
// Marshal marshals the HID RPC message to the data.
func Marshal(message *Message) ([]byte, error) {
if message.t == 0 {
return nil, fmt.Errorf("invalid message type: %d", message.t)
}
data := make([]byte, len(message.d)+1)
data[0] = byte(message.t)
copy(data[1:], message.d)
return data, nil
}
// NewHandshakeMessage creates a new handshake message.
func NewHandshakeMessage() *Message {
return &Message{
t: TypeHandshake,
d: []byte{Version},
}
}
// NewKeyboardReportMessage creates a new keyboard report message.
func NewKeyboardReportMessage(keys []byte, modifier uint8) *Message {
return &Message{
t: TypeKeyboardReport,
d: append([]byte{modifier}, keys...),
}
}
// NewKeyboardLedMessage creates a new keyboard LED message.
func NewKeyboardLedMessage(state usbgadget.KeyboardState) *Message {
return &Message{
t: TypeKeyboardLedState,
d: []byte{state.Byte()},
}
}
// NewKeydownStateMessage creates a new keydown state message.
func NewKeydownStateMessage(state usbgadget.KeysDownState) *Message {
data := make([]byte, len(state.Keys)+1)
data[0] = state.Modifier
copy(data[1:], state.Keys)
return &Message{
t: TypeKeydownState,
d: data,
}
}

133
internal/hidrpc/message.go Normal file
View File

@ -0,0 +1,133 @@
package hidrpc
import (
"fmt"
)
// Message ..
type Message struct {
t MessageType
d []byte
}
// Marshal marshals the message to a byte array.
func (m *Message) Marshal() ([]byte, error) {
return Marshal(m)
}
func (m *Message) Type() MessageType {
return m.t
}
func (m *Message) String() string {
switch m.t {
case TypeHandshake:
return "Handshake"
case TypeKeypressReport:
if len(m.d) < 2 {
return fmt.Sprintf("KeypressReport{Malformed: %v}", m.d)
}
return fmt.Sprintf("KeypressReport{Key: %d, Press: %v}", m.d[0], m.d[1] == uint8(1))
case TypeKeyboardReport:
if len(m.d) < 2 {
return fmt.Sprintf("KeyboardReport{Malformed: %v}", m.d)
}
return fmt.Sprintf("KeyboardReport{Modifier: %d, Keys: %v}", m.d[0], m.d[1:])
case TypePointerReport:
if len(m.d) < 9 {
return fmt.Sprintf("PointerReport{Malformed: %v}", m.d)
}
return fmt.Sprintf("PointerReport{X: %d, Y: %d, Button: %d}", m.d[0:4], m.d[4:8], m.d[8])
case TypeMouseReport:
if len(m.d) < 3 {
return fmt.Sprintf("MouseReport{Malformed: %v}", m.d)
}
return fmt.Sprintf("MouseReport{DX: %d, DY: %d, Button: %d}", m.d[0], m.d[1], m.d[2])
default:
return fmt.Sprintf("Unknown{Type: %d, Data: %v}", m.t, m.d)
}
}
// KeypressReport ..
type KeypressReport struct {
Key byte
Press bool
}
// KeypressReport returns the keypress report from the message.
func (m *Message) KeypressReport() (KeypressReport, error) {
if m.t != TypeKeypressReport {
return KeypressReport{}, fmt.Errorf("invalid message type: %d", m.t)
}
return KeypressReport{
Key: m.d[0],
Press: m.d[1] == uint8(1),
}, nil
}
// KeyboardReport ..
type KeyboardReport struct {
Modifier byte
Keys []byte
}
// KeyboardReport returns the keyboard report from the message.
func (m *Message) KeyboardReport() (KeyboardReport, error) {
if m.t != TypeKeyboardReport {
return KeyboardReport{}, fmt.Errorf("invalid message type: %d", m.t)
}
return KeyboardReport{
Modifier: m.d[0],
Keys: m.d[1:],
}, nil
}
// PointerReport ..
type PointerReport struct {
X int
Y int
Button uint8
}
func toInt(b []byte) int {
return int(b[0])<<24 + int(b[1])<<16 + int(b[2])<<8 + int(b[3])<<0
}
// PointerReport returns the point report from the message.
func (m *Message) PointerReport() (PointerReport, error) {
if m.t != TypePointerReport {
return PointerReport{}, fmt.Errorf("invalid message type: %d", m.t)
}
if len(m.d) != 9 {
return PointerReport{}, fmt.Errorf("invalid message length: %d", len(m.d))
}
return PointerReport{
X: toInt(m.d[0:4]),
Y: toInt(m.d[4:8]),
Button: uint8(m.d[8]),
}, nil
}
// MouseReport ..
type MouseReport struct {
DX int8
DY int8
Button uint8
}
// MouseReport returns the mouse report from the message.
func (m *Message) MouseReport() (MouseReport, error) {
if m.t != TypeMouseReport {
return MouseReport{}, fmt.Errorf("invalid message type: %d", m.t)
}
return MouseReport{
DX: int8(m.d[0]),
DY: int8(m.d[1]),
Button: uint8(m.d[2]),
}, nil
}

4
internal/usbgadget/consts.go
View File

@ -1,3 +1,7 @@
package usbgadget
import "time"
const dwc3Path = "/sys/bus/platform/drivers/dwc3"
const hidWriteTimeout = 10 * time.Millisecond

32
internal/usbgadget/hid_keyboard.go
View File

@ -86,6 +86,12 @@ type KeyboardState struct {
Compose bool `json:"compose"`
Kana bool `json:"kana"`
Shift bool `json:"shift"` // This is not part of the main USB HID spec
raw byte
}
// Byte returns the raw byte representation of the keyboard state.
func (k *KeyboardState) Byte() byte {
return k.raw
}
func getKeyboardState(b byte) KeyboardState {
@ -97,6 +103,7 @@ func getKeyboardState(b byte) KeyboardState {
Compose: b&KeyboardLedMaskCompose != 0,
Kana: b&KeyboardLedMaskKana != 0,
Shift: b&KeyboardLedMaskShift != 0,
raw: b,
}
}
@ -139,19 +146,26 @@ func (u *UsbGadget) GetKeysDownState() KeysDownState {
}
func (u *UsbGadget) updateKeyDownState(state KeysDownState) {
u.keyboardStateLock.Lock()
defer u.keyboardStateLock.Unlock()
u.log.Trace().Interface("old", u.keysDownState).Interface("new", state).Msg("acquiring keyboardStateLock for updateKeyDownState")
if u.keysDownState.Modifier == state.Modifier &&
bytes.Equal(u.keysDownState.Keys, state.Keys) {
return // No change in key down state
// this is intentional to unlock keyboard state lock before onKeysDownChange callback
{
u.keyboardStateLock.Lock()
defer u.keyboardStateLock.Unlock()
if u.keysDownState.Modifier == state.Modifier &&
bytes.Equal(u.keysDownState.Keys, state.Keys) {
return // No change in key down state
}
u.log.Trace().Interface("old", u.keysDownState).Interface("new", state).Msg("keysDownState updated")
u.keysDownState = state
}
u.log.Trace().Interface("old", u.keysDownState).Interface("new", state).Msg("keysDownState updated")
u.keysDownState = state
if u.onKeysDownChange != nil {
u.log.Trace().Interface("state", state).Msg("calling onKeysDownChange")
(*u.onKeysDownChange)(state)
u.log.Trace().Interface("state", state).Msg("onKeysDownChange called")
}
}
@ -233,7 +247,7 @@ func (u *UsbGadget) keyboardWriteHidFile(modifier byte, keys []byte) error {
return err
}
_, err := u.keyboardHidFile.Write(append([]byte{modifier, 0x00}, keys[:hidKeyBufferSize]...))
_, err := u.writeWithTimeout(u.keyboardHidFile, append([]byte{modifier, 0x00}, keys[:hidKeyBufferSize]...))
if err != nil {
u.logWithSuppression("keyboardWriteHidFile", 100, u.log, err, "failed to write to hidg0")
u.keyboardHidFile.Close()

2
internal/usbgadget/hid_mouse_absolute.go
View File

@ -74,7 +74,7 @@ func (u *UsbGadget) absMouseWriteHidFile(data []byte) error {
}
}
_, err := u.absMouseHidFile.Write(data)
_, err := u.writeWithTimeout(u.absMouseHidFile, data)
if err != nil {
u.logWithSuppression("absMouseWriteHidFile", 100, u.log, err, "failed to write to hidg1")
u.absMouseHidFile.Close()

2
internal/usbgadget/hid_mouse_relative.go
View File

@ -64,7 +64,7 @@ func (u *UsbGadget) relMouseWriteHidFile(data []byte) error {
}
}
_, err := u.relMouseHidFile.Write(data)
_, err := u.writeWithTimeout(u.relMouseHidFile, data)
if err != nil {
u.logWithSuppression("relMouseWriteHidFile", 100, u.log, err, "failed to write to hidg2")
u.relMouseHidFile.Close()

28
internal/usbgadget/utils.go
View File

@ -3,10 +3,13 @@ package usbgadget
import (
"bytes"
"encoding/json"
"errors"
"fmt"
"os"
"path/filepath"
"strconv"
"strings"
"time"
"github.com/rs/zerolog"
)
@ -107,6 +110,31 @@ func compareFileContent(oldContent []byte, newContent []byte, looserMatch bool)
return false
}
func (u *UsbGadget) writeWithTimeout(file *os.File, data []byte) (n int, err error) {
if err := file.SetWriteDeadline(time.Now().Add(hidWriteTimeout)); err != nil {
return -1, err
}
n, err = file.Write(data)
if err == nil {
return
}
if errors.Is(err, os.ErrDeadlineExceeded) {
u.logWithSuppression(
fmt.Sprintf("writeWithTimeout_%s", file.Name()),
1000,
u.log,
err,
"write timed out: %s",
file.Name(),
)
err = nil
}
return
}
func (u *UsbGadget) logWithSuppression(counterName string, every int, logger *zerolog.Logger, err error, msg string, args ...any) {
u.logSuppressionLock.Lock()
defer u.logSuppressionLock.Unlock()

2
jsonrpc.go
View File

@ -83,7 +83,7 @@ func writeJSONRPCEvent(event string, params any, session *Session) {
Str("data", requestString).
Logger()
scopedLogger.Info().Msg("sending JSONRPC event")
scopedLogger.Trace().Msg("sending JSONRPC event")
err = session.RPCChannel.SendText(requestString)
if err != nil {

1
log.go
View File

@ -19,6 +19,7 @@ var (
nbdLogger = logging.GetSubsystemLogger("nbd")
timesyncLogger = logging.GetSubsystemLogger("timesync")
jsonRpcLogger = logging.GetSubsystemLogger("jsonrpc")
hidRPCLogger = logging.GetSubsystemLogger("hidrpc")
watchdogLogger = logging.GetSubsystemLogger("watchdog")
websecureLogger = logging.GetSubsystemLogger("websecure")
otaLogger = logging.GetSubsystemLogger("ota")

10
ui/src/components/InfoBar.tsx
View File

@ -10,10 +10,12 @@ import {
VideoState
} from "@/hooks/stores";
import { keys, modifiers } from "@/keyboardMappings";
import { useHidRpc } from "@/hooks/useHidRpc";
export default function InfoBar() {
const { keysDownState } = useHidStore();
const { mouseX, mouseY, mouseMove } = useMouseStore();
const { rpcHidStatus } = useHidRpc();
const videoClientSize = useVideoStore(
(state: VideoState) => `${Math.round(state.clientWidth)}x${Math.round(state.clientHeight)}`,
@ -46,7 +48,7 @@ export default function InfoBar() {
const modifierNames = Object.entries(modifiers).filter(([_, mask]) => (activeModifierMask & mask) !== 0).map(([name, _]) => name);
const keyNames = Object.entries(keys).filter(([_, value]) => keysDown.includes(value)).map(([name, _]) => name);
return [...modifierNames,...keyNames].join(", ");
return [...modifierNames, ...keyNames].join(", ");
}, [keysDownState, showPressedKeys]);
return (
@ -100,6 +102,12 @@ export default function InfoBar() {
<span className="text-xs">{hdmiState}</span>
</div>
)}
{debugMode && (
<div className="flex w-[156px] items-center gap-x-1">
<span className="text-xs font-semibold">HidRPC State:</span>
<span className="text-xs">{rpcHidStatus}</span>
</div>
)}
{showPressedKeys && (
<div className="flex items-center gap-x-1">

173
ui/src/components/WebRTCVideo.tsx
View File

@ -7,15 +7,14 @@ import MacroBar from "@/components/MacroBar";
import InfoBar from "@components/InfoBar";
import notifications from "@/notifications";
import useKeyboard from "@/hooks/useKeyboard";
import { useJsonRpc } from "@/hooks/useJsonRpc";
import { cx } from "@/cva.config";
import { keys } from "@/keyboardMappings";
import {
useMouseStore,
useRTCStore,
useSettingsStore,
useVideoStore,
} from "@/hooks/stores";
import useMouse from "@/hooks/useMouse";
import {
HDMIErrorOverlay,
@ -31,10 +30,18 @@ export default function WebRTCVideo() {
const [isPlaying, setIsPlaying] = useState(false);
const [isPointerLockActive, setIsPointerLockActive] = useState(false);
const [isKeyboardLockActive, setIsKeyboardLockActive] = useState(false);
const isPointerLockPossible = window.location.protocol === "https:" || window.location.hostname === "localhost";
// Store hooks
const settings = useSettingsStore();
const { handleKeyPress, resetKeyboardState } = useKeyboard();
const { setMousePosition, setMouseMove } = useMouseStore();
const {
getRelMouseMoveHandler,
getAbsMouseMoveHandler,
getMouseWheelHandler,
resetMousePosition,
} = useMouse();
const {
setClientSize: setVideoClientSize,
setSize: setVideoSize,
@ -55,15 +62,9 @@ export default function WebRTCVideo() {
const hdmiError = ["no_lock", "no_signal", "out_of_range"].includes(hdmiState);
const isVideoLoading = !isPlaying;
// Mouse wheel states
const [blockWheelEvent, setBlockWheelEvent] = useState(false);
// Misc states and hooks
const { send } = useJsonRpc();
// Video-related
const handleResize = useCallback(
( { width, height }: { width: number | undefined; height: number | undefined }) => {
({ width, height }: { width: number | undefined; height: number | undefined }) => {
if (!videoElm.current) return;
// Do something with width and height, e.g.:
setVideoClientSize(width || 0, height || 0);
@ -99,7 +100,6 @@ export default function WebRTCVideo() {
);
// Pointer lock and keyboard lock related
const isPointerLockPossible = window.location.protocol === "https:" || window.location.hostname === "localhost";
const isFullscreenEnabled = document.fullscreenEnabled;
const checkNavigatorPermissions = useCallback(async (permissionName: string) => {
@ -211,129 +211,29 @@ export default function WebRTCVideo() {
}
};
document.addEventListener("fullscreenchange ", handleFullscreenChange);
document.addEventListener("fullscreenchange", handleFullscreenChange);
}, [releaseKeyboardLock]);
// Mouse-related
const calcDelta = (pos: number) => (Math.abs(pos) < 10 ? pos * 2 : pos);
const sendRelMouseMovement = useCallback(
(x: number, y: number, buttons: number) => {
if (settings.mouseMode !== "relative") return;
// if we ignore the event, double-click will not work
// if (x === 0 && y === 0 && buttons === 0) return;
send("relMouseReport", { dx: calcDelta(x), dy: calcDelta(y), buttons });
setMouseMove({ x, y, buttons });
},
[send, setMouseMove, settings.mouseMode],
const absMouseMoveHandler = useMemo(
() => getAbsMouseMoveHandler({
videoClientWidth,
videoClientHeight,
videoWidth,
videoHeight,
}),
[getAbsMouseMoveHandler, videoClientWidth, videoClientHeight, videoWidth, videoHeight],
);
const relMouseMoveHandler = useCallback(
(e: MouseEvent) => {
if (settings.mouseMode !== "relative") return;
if (isPointerLockActive === false && isPointerLockPossible) return;
// Send mouse movement
const { buttons } = e;
sendRelMouseMovement(e.movementX, e.movementY, buttons);
},
[isPointerLockActive, isPointerLockPossible, sendRelMouseMovement, settings.mouseMode],
const relMouseMoveHandler = useMemo(
() => getRelMouseMoveHandler(),
[getRelMouseMoveHandler],
);
const sendAbsMouseMovement = useCallback(
(x: number, y: number, buttons: number) => {
if (settings.mouseMode !== "absolute") return;
send("absMouseReport", { x, y, buttons });
// We set that for the debug info bar
setMousePosition(x, y);
},
[send, setMousePosition, settings.mouseMode],
const mouseWheelHandler = useMemo(
() => getMouseWheelHandler(),
[getMouseWheelHandler],
);
const absMouseMoveHandler = useCallback(
(e: MouseEvent) => {
if (!videoClientWidth || !videoClientHeight) return;
if (settings.mouseMode !== "absolute") return;
// Get the aspect ratios of the video element and the video stream
const videoElementAspectRatio = videoClientWidth / videoClientHeight;
const videoStreamAspectRatio = videoWidth / videoHeight;
// Calculate the effective video display area
let effectiveWidth = videoClientWidth;
let effectiveHeight = videoClientHeight;
let offsetX = 0;
let offsetY = 0;
if (videoElementAspectRatio > videoStreamAspectRatio) {
// Pillarboxing: black bars on the left and right
effectiveWidth = videoClientHeight * videoStreamAspectRatio;
offsetX = (videoClientWidth - effectiveWidth) / 2;
} else if (videoElementAspectRatio < videoStreamAspectRatio) {
// Letterboxing: black bars on the top and bottom
effectiveHeight = videoClientWidth / videoStreamAspectRatio;
offsetY = (videoClientHeight - effectiveHeight) / 2;
}
// Clamp mouse position within the effective video boundaries
const clampedX = Math.min(Math.max(offsetX, e.offsetX), offsetX + effectiveWidth);
const clampedY = Math.min(Math.max(offsetY, e.offsetY), offsetY + effectiveHeight);
// Map clamped mouse position to the video stream's coordinate system
const relativeX = (clampedX - offsetX) / effectiveWidth;
const relativeY = (clampedY - offsetY) / effectiveHeight;
// Convert to HID absolute coordinate system (0-32767 range)
const x = Math.round(relativeX * 32767);
const y = Math.round(relativeY * 32767);
// Send mouse movement
const { buttons } = e;
sendAbsMouseMovement(x, y, buttons);
},
[settings.mouseMode, videoClientWidth, videoClientHeight, videoWidth, videoHeight, sendAbsMouseMovement],
);
const mouseWheelHandler = useCallback(
(e: WheelEvent) => {
if (settings.scrollThrottling && blockWheelEvent) {
return;
}
// Determine if the wheel event is an accel scroll value
const isAccel = Math.abs(e.deltaY) >= 100;
// Calculate the accel scroll value
const accelScrollValue = e.deltaY / 100;
// Calculate the no accel scroll value
const noAccelScrollValue = Math.sign(e.deltaY);
// Get scroll value
const scrollValue = isAccel ? accelScrollValue : noAccelScrollValue;
// Apply clamping (i.e. min and max mouse wheel hardware value)
const clampedScrollValue = Math.max(-127, Math.min(127, scrollValue));
// Invert the clamped scroll value to match expected behavior
const invertedScrollValue = -clampedScrollValue;
send("wheelReport", { wheelY: invertedScrollValue });
// Apply blocking delay based of throttling settings
if (settings.scrollThrottling && !blockWheelEvent) {
setBlockWheelEvent(true);
setTimeout(() => setBlockWheelEvent(false), settings.scrollThrottling);
}
},
[send, blockWheelEvent, settings],
);
const resetMousePosition = useCallback(() => {
sendAbsMouseMovement(0, 0, 0);
}, [sendAbsMouseMovement]);
const keyDownHandler = useCallback(
(e: KeyboardEvent) => {
e.preventDefault();
@ -357,7 +257,7 @@ export default function WebRTCVideo() {
}
console.debug(`Key down: ${hidKey}`);
handleKeyPress(hidKey, true);
if (!isKeyboardLockActive && hidKey === keys.MetaLeft) {
// If the left meta key was just pressed and we're not keyboard locked
// we'll never see the keyup event because the browser is going to lose
@ -488,14 +388,16 @@ export default function WebRTCVideo() {
function setMouseModeEventListeners() {
const videoElmRefValue = videoElm.current;
if (!videoElmRefValue) return;
const isRelativeMouseMode = (settings.mouseMode === "relative");
const mouseHandler = isRelativeMouseMode ? relMouseMoveHandler : absMouseMoveHandler;
const abortController = new AbortController();
const signal = abortController.signal;
videoElmRefValue.addEventListener("mousemove", isRelativeMouseMode ? relMouseMoveHandler : absMouseMoveHandler, { signal });
videoElmRefValue.addEventListener("pointerdown", isRelativeMouseMode ? relMouseMoveHandler : absMouseMoveHandler, { signal });
videoElmRefValue.addEventListener("pointerup", isRelativeMouseMode ? relMouseMoveHandler : absMouseMoveHandler, { signal });
videoElmRefValue.addEventListener("mousemove", mouseHandler, { signal });
videoElmRefValue.addEventListener("pointerdown", mouseHandler, { signal });
videoElmRefValue.addEventListener("pointerup", mouseHandler, { signal });
videoElmRefValue.addEventListener("wheel", mouseWheelHandler, {
signal,
passive: true,
@ -523,7 +425,16 @@ export default function WebRTCVideo() {
abortController.abort();
};
},
[absMouseMoveHandler, isPointerLockActive, isPointerLockPossible, mouseWheelHandler, relMouseMoveHandler, requestPointerLock, resetMousePosition, settings.mouseMode],
[
isPointerLockActive,
isPointerLockPossible,
requestPointerLock,
absMouseMoveHandler,
relMouseMoveHandler,
mouseWheelHandler,
resetMousePosition,
settings.mouseMode,
],
);
const containerRef = useRef<HTMLDivElement>(null);

302
ui/src/hooks/hidRpc.ts Normal file
View File

@ -0,0 +1,302 @@
import { KeyboardLedState, KeysDownState } from "./stores";
export const HID_RPC_MESSAGE_TYPES = {
Handshake: 0x01,
KeyboardReport: 0x02,
PointerReport: 0x03,
WheelReport: 0x04,
KeypressReport: 0x05,
MouseReport: 0x06,
KeyboardLedState: 0x32,
KeysDownState: 0x33,
}
export type HidRpcMessageType = typeof HID_RPC_MESSAGE_TYPES[keyof typeof HID_RPC_MESSAGE_TYPES];
export const HID_RPC_VERSION = 0x01;
const withinUint8Range = (value: number) => {
return value >= 0 && value <= 255;
};
const fromInt32toUint8 = (n: number) => {
if (n !== n >> 0) {
throw new Error(`Number ${n} is not within the int32 range`);
}
return new Uint8Array([
(n >> 24) & 0xFF,
(n >> 16) & 0xFF,
(n >> 8) & 0xFF,
(n >> 0) & 0xFF,
]);
};
const fromInt8ToUint8 = (n: number) => {
if (n < -128 || n > 127) {
throw new Error(`Number ${n} is not within the int8 range`);
}
return (n >> 0) & 0xFF;
};
const keyboardLedStateMasks = {
num_lock: 1 << 0,
caps_lock: 1 << 1,
scroll_lock: 1 << 2,
compose: 1 << 3,
kana: 1 << 4,
shift: 1 << 6,
}
export class RpcMessage {
messageType: HidRpcMessageType;
constructor(messageType: HidRpcMessageType) {
this.messageType = messageType;
}
marshal(): Uint8Array {
throw new Error("Not implemented");
}
public static unmarshal(_data: Uint8Array): RpcMessage | undefined {
throw new Error("Not implemented");
}
}
export class HandshakeMessage extends RpcMessage {
version: number;
constructor(version: number) {
super(HID_RPC_MESSAGE_TYPES.Handshake);
this.version = version;
}
marshal(): Uint8Array {
return new Uint8Array([this.messageType, this.version]);
}
public static unmarshal(data: Uint8Array): HandshakeMessage | undefined {
if (data.length < 1) {
throw new Error(`Invalid handshake message length: ${data.length}`);
}
return new HandshakeMessage(data[0]);
}
}
export class KeypressReportMessage extends RpcMessage {
private _key = 0;
private _press = false;
get key(): number {
return this._key;
}
set key(value: number) {
if (!withinUint8Range(value)) {
throw new Error(`Key ${value} is not within the uint8 range`);
}
this._key = value;
}
get press(): boolean {
return this._press;
}
set press(value: boolean) {
this._press = value;
}
constructor(key: number, press: boolean) {
super(HID_RPC_MESSAGE_TYPES.KeypressReport);
this.key = key;
this.press = press;
}
marshal(): Uint8Array {
return new Uint8Array([
this.messageType,
this.key,
this.press ? 1 : 0,
]);
}
public static unmarshal(data: Uint8Array): KeypressReportMessage | undefined {
if (data.length < 1) {
throw new Error(`Invalid keypress report message length: ${data.length}`);
}
return new KeypressReportMessage(data[0], data[1] === 1);
}
}
export class KeyboardReportMessage extends RpcMessage {
private _keys: number[] = [];
private _modifier = 0;
get keys(): number[] {
return this._keys;
}
set keys(value: number[]) {
value.forEach((k) => {
if (!withinUint8Range(k)) {
throw new Error(`Key ${k} is not within the uint8 range`);
}
});
this._keys = value;
}
get modifier(): number {
return this._modifier;
}
set modifier(value: number) {
if (!withinUint8Range(value)) {
throw new Error(`Modifier ${value} is not within the uint8 range`);
}
this._modifier = value;
}
constructor(keys: number[], modifier: number) {
super(HID_RPC_MESSAGE_TYPES.KeyboardReport);
this.keys = keys;
this.modifier = modifier;
}
marshal(): Uint8Array {
return new Uint8Array([
this.messageType,
this.modifier,
...this.keys,
]);
}
public static unmarshal(data: Uint8Array): KeyboardReportMessage | undefined {
if (data.length < 1) {
throw new Error(`Invalid keyboard report message length: ${data.length}`);
}
return new KeyboardReportMessage(Array.from(data.slice(1)), data[0]);
}
}
export class KeyboardLedStateMessage extends RpcMessage {
keyboardLedState: KeyboardLedState;
constructor(keyboardLedState: KeyboardLedState) {
super(HID_RPC_MESSAGE_TYPES.KeyboardLedState);
this.keyboardLedState = keyboardLedState;
}
public static unmarshal(data: Uint8Array): KeyboardLedStateMessage | undefined {
if (data.length < 1) {
throw new Error(`Invalid keyboard led state message length: ${data.length}`);
}
const s = data[0];
const state = {
num_lock: (s & keyboardLedStateMasks.num_lock) !== 0,
caps_lock: (s & keyboardLedStateMasks.caps_lock) !== 0,
scroll_lock: (s & keyboardLedStateMasks.scroll_lock) !== 0,
compose: (s & keyboardLedStateMasks.compose) !== 0,
kana: (s & keyboardLedStateMasks.kana) !== 0,
shift: (s & keyboardLedStateMasks.shift) !== 0,
} as KeyboardLedState;
return new KeyboardLedStateMessage(state);
}
}
export class KeysDownStateMessage extends RpcMessage {
keysDownState: KeysDownState;
constructor(keysDownState: KeysDownState) {
super(HID_RPC_MESSAGE_TYPES.KeysDownState);
this.keysDownState = keysDownState;
}
public static unmarshal(data: Uint8Array): KeysDownStateMessage | undefined {
if (data.length < 1) {
throw new Error(`Invalid keys down state message length: ${data.length}`);
}
return new KeysDownStateMessage({
modifier: data[0],
keys: Array.from(data.slice(1))
});
}
}
export class PointerReportMessage extends RpcMessage {
x: number;
y: number;
buttons: number;
constructor(x: number, y: number, buttons: number) {
super(HID_RPC_MESSAGE_TYPES.PointerReport);
this.x = x;
this.y = y;
this.buttons = buttons;
}
marshal(): Uint8Array {
return new Uint8Array([
this.messageType,
...fromInt32toUint8(this.x),
...fromInt32toUint8(this.y),
this.buttons,
]);
}
}
export class MouseReportMessage extends RpcMessage {
dx: number;
dy: number;
buttons: number;
constructor(dx: number, dy: number, buttons: number) {
super(HID_RPC_MESSAGE_TYPES.MouseReport);
this.dx = dx;
this.dy = dy;
this.buttons = buttons;
}
marshal(): Uint8Array {
return new Uint8Array([
this.messageType,
fromInt8ToUint8(this.dx),
fromInt8ToUint8(this.dy),
this.buttons,
]);
}
}
export const messageRegistry = {
[HID_RPC_MESSAGE_TYPES.Handshake]: HandshakeMessage,
[HID_RPC_MESSAGE_TYPES.KeysDownState]: KeysDownStateMessage,
[HID_RPC_MESSAGE_TYPES.KeyboardLedState]: KeyboardLedStateMessage,
[HID_RPC_MESSAGE_TYPES.KeyboardReport]: KeyboardReportMessage,
[HID_RPC_MESSAGE_TYPES.KeypressReport]: KeypressReportMessage,
}
export const unmarshalHidRpcMessage = (data: Uint8Array): RpcMessage | undefined => {
if (data.length < 1) {
throw new Error(`Invalid HID RPC message length: ${data.length}`);
}
const payload = data.slice(1);
const messageType = data[0];
if (!(messageType in messageRegistry)) {
throw new Error(`Unknown HID RPC message type: ${messageType}`);
}
return messageRegistry[messageType].unmarshal(payload);
};

18
ui/src/hooks/stores.ts
View File

@ -105,6 +105,12 @@ export interface RTCState {
setRpcDataChannel: (channel: RTCDataChannel) => void;
rpcDataChannel: RTCDataChannel | null;
rpcHidProtocolVersion: number | null;
setRpcHidProtocolVersion: (version: number) => void;
rpcHidChannel: RTCDataChannel | null;
setRpcHidChannel: (channel: RTCDataChannel) => void;
peerConnectionState: RTCPeerConnectionState | null;
setPeerConnectionState: (state: RTCPeerConnectionState) => void;
@ -151,6 +157,12 @@ export const useRTCStore = create<RTCState>(set => ({
rpcDataChannel: null,
setRpcDataChannel: (channel: RTCDataChannel) => set({ rpcDataChannel: channel }),
rpcHidProtocolVersion: null,
setRpcHidProtocolVersion: (version: number) => set({ rpcHidProtocolVersion: version }),
rpcHidChannel: null,
setRpcHidChannel: (channel: RTCDataChannel) => set({ rpcHidChannel: channel }),
transceiver: null,
setTransceiver: (transceiver: RTCRtpTransceiver) => set({ transceiver }),
@ -449,9 +461,6 @@ export interface HidState {
keysDownState: KeysDownState;
setKeysDownState: (state: KeysDownState) => void;
keyPressReportApiAvailable: boolean;
setkeyPressReportApiAvailable: (available: boolean) => void;
isVirtualKeyboardEnabled: boolean;
setVirtualKeyboardEnabled: (enabled: boolean) => void;
@ -469,9 +478,6 @@ export const useHidStore = create<HidState>(set => ({
keysDownState: { modifier: 0, keys: [0,0,0,0,0,0] } as KeysDownState,
setKeysDownState: (state: KeysDownState): void => set({ keysDownState: state }),
keyPressReportApiAvailable: true,
setkeyPressReportApiAvailable: (available: boolean) => set({ keyPressReportApiAvailable: available }),
isVirtualKeyboardEnabled: false,
setVirtualKeyboardEnabled: (enabled: boolean): void => set({ isVirtualKeyboardEnabled: enabled }),

150
ui/src/hooks/useHidRpc.ts Normal file
View File

@ -0,0 +1,150 @@
import { useCallback, useEffect, useMemo } from "react";
import { useRTCStore } from "@/hooks/stores";
import {
HID_RPC_VERSION,
HandshakeMessage,
KeyboardReportMessage,
KeypressReportMessage,
MouseReportMessage,
PointerReportMessage,
RpcMessage,
unmarshalHidRpcMessage,
} from "./hidRpc";
export function useHidRpc(onHidRpcMessage?: (payload: RpcMessage) => void) {
const { rpcHidChannel, setRpcHidProtocolVersion, rpcHidProtocolVersion } = useRTCStore();
const rpcHidReady = useMemo(() => {
return rpcHidChannel?.readyState === "open" && rpcHidProtocolVersion !== null;
}, [rpcHidChannel, rpcHidProtocolVersion]);
const rpcHidStatus = useMemo(() => {
if (!rpcHidChannel) return "N/A";
if (rpcHidChannel.readyState !== "open") return rpcHidChannel.readyState;
if (!rpcHidProtocolVersion) return "handshaking";
return `ready (v${rpcHidProtocolVersion})`;
}, [rpcHidChannel, rpcHidProtocolVersion]);
const sendMessage = useCallback((message: RpcMessage, ignoreHandshakeState = false) => {
if (rpcHidChannel?.readyState !== "open") return;
if (!rpcHidReady && !ignoreHandshakeState) return;
let data: Uint8Array | undefined;
try {
data = message.marshal();
} catch (e) {
console.error("Failed to send HID RPC message", e);
}
if (!data) return;
rpcHidChannel?.send(data as unknown as ArrayBuffer);
}, [rpcHidChannel, rpcHidReady]);
const reportKeyboardEvent = useCallback(
(keys: number[], modifier: number) => {
sendMessage(new KeyboardReportMessage(keys, modifier));
}, [sendMessage],
);
const reportKeypressEvent = useCallback(
(key: number, press: boolean) => {
sendMessage(new KeypressReportMessage(key, press));
},
[sendMessage],
);
const reportAbsMouseEvent = useCallback(
(x: number, y: number, buttons: number) => {
sendMessage(new PointerReportMessage(x, y, buttons));
},
[sendMessage],
);
const reportRelMouseEvent = useCallback(
(dx: number, dy: number, buttons: number) => {
sendMessage(new MouseReportMessage(dx, dy, buttons));
},
[sendMessage],
);
const sendHandshake = useCallback(() => {
if (rpcHidProtocolVersion) return;
if (!rpcHidChannel) return;
sendMessage(new HandshakeMessage(HID_RPC_VERSION), true);
}, [rpcHidChannel, rpcHidProtocolVersion, sendMessage]);
const handleHandshake = useCallback((message: HandshakeMessage) => {
if (!message.version) {
console.error("Received handshake message without version", message);
return;
}
if (message.version > HID_RPC_VERSION) {
// we assume that the UI is always using the latest version of the HID RPC protocol
// so we can't support this
// TODO: use capabilities to determine rather than version number
console.error("Server is using a newer HID RPC version than the client", message);
return;
}
setRpcHidProtocolVersion(message.version);
}, [setRpcHidProtocolVersion]);
useEffect(() => {
if (!rpcHidChannel) return;
// send handshake message
sendHandshake();
const messageHandler = (e: MessageEvent) => {
if (typeof e.data === "string") {
console.warn("Received string data in HID RPC message handler", e.data);
return;
}
const message = unmarshalHidRpcMessage(new Uint8Array(e.data));
if (!message) {
console.warn("Received invalid HID RPC message", e.data);
return;
}
console.debug("Received HID RPC message", message);
switch (message.constructor) {
case HandshakeMessage:
handleHandshake(message as HandshakeMessage);
break;
default:
// not all events are handled here, the rest are handled by the onHidRpcMessage callback
break;
}
onHidRpcMessage?.(message);
};
rpcHidChannel.addEventListener("message", messageHandler);
return () => {
rpcHidChannel.removeEventListener("message", messageHandler);
};
},
[
rpcHidChannel,
onHidRpcMessage,
setRpcHidProtocolVersion,
sendHandshake,
handleHandshake,
],
);
return {
reportKeyboardEvent,
reportKeypressEvent,
reportAbsMouseEvent,
reportRelMouseEvent,
rpcHidProtocolVersion,
rpcHidReady,
rpcHidStatus,
};
}

108
ui/src/hooks/useKeyboard.ts
View File

@ -1,13 +1,15 @@
import { useCallback } from "react";
import { KeysDownState, useHidStore, useRTCStore, hidKeyBufferSize, hidErrorRollOver } from "@/hooks/stores";
import { hidErrorRollOver, hidKeyBufferSize, KeysDownState, useHidStore, useRTCStore } from "@/hooks/stores";
import { JsonRpcResponse, useJsonRpc } from "@/hooks/useJsonRpc";
import { useHidRpc } from "@/hooks/useHidRpc";
import { KeyboardLedStateMessage, KeysDownStateMessage } from "@/hooks/hidRpc";
import { hidKeyToModifierMask, keys, modifiers } from "@/keyboardMappings";
export default function useKeyboard() {
const { send } = useJsonRpc();
const { rpcDataChannel } = useRTCStore();
const { keysDownState, setKeysDownState } = useHidStore();
const { keysDownState, setKeysDownState, setKeyboardLedState } = useHidStore();
// INTRODUCTION: The earlier version of the JetKVM device shipped with all keyboard state
// being tracked on the browser/client-side. When adding the keyPressReport API to the
@ -19,7 +21,24 @@ export default function useKeyboard() {
// dynamically set when the device responds to the first key press event or reports its
// keysDownState when queried since the keyPressReport was introduced together with the
// getKeysDownState API.
const { keyPressReportApiAvailable, setkeyPressReportApiAvailable} = useHidStore();
// HidRPC is a binary format for exchanging keyboard and mouse events
const {
reportKeyboardEvent: sendKeyboardEventHidRpc,
reportKeypressEvent: sendKeypressEventHidRpc,
rpcHidReady,
} = useHidRpc((message) => {
switch (message.constructor) {
case KeysDownStateMessage:
setKeysDownState((message as KeysDownStateMessage).keysDownState);
break;
case KeyboardLedStateMessage:
setKeyboardLedState((message as KeyboardLedStateMessage).keyboardLedState);
break;
default:
break;
}
});
// sendKeyboardEvent is used to send the full keyboard state to the device for macro handling
// and resetting keyboard state. It sends the keys currently pressed and the modifier state.
@ -27,63 +46,28 @@ export default function useKeyboard() {
// or just accept the state if it does not support (returning no result)
const sendKeyboardEvent = useCallback(
async (state: KeysDownState) => {
if (rpcDataChannel?.readyState !== "open") return;
if (rpcDataChannel?.readyState !== "open" && !rpcHidReady) return;
console.debug(`Send keyboardReport keys: ${state.keys}, modifier: ${state.modifier}`);
if (rpcHidReady) {
console.debug("Sending keyboard report via HidRPC");
sendKeyboardEventHidRpc(state.keys, state.modifier);
return;
}
send("keyboardReport", { keys: state.keys, modifier: state.modifier }, (resp: JsonRpcResponse) => {
if ("error" in resp) {
console.error(`Failed to send keyboard report ${state}`, resp.error);
} else {
// If the device supports keyPressReport API, it will (also) return the keysDownState when we send
// the keyboardReport
const keysDownState = resp.result as KeysDownState;
if (keysDownState) {
setKeysDownState(keysDownState); // treat the response as the canonical state
setkeyPressReportApiAvailable(true); // if they returned a keysDownState, we ALSO know they also support keyPressReport
} else {
// older devices versions do not return the keyDownState
// so we just pretend they accepted what we sent
setKeysDownState(state);
setkeyPressReportApiAvailable(false); // we ALSO know they do not support keyPressReport
}
}
});
},
[rpcDataChannel?.readyState, send, setKeysDownState, setkeyPressReportApiAvailable],
);
// sendKeypressEvent is used to send a single key press/release event to the device.
// It sends the key and whether it is pressed or released.
// Older device version will not understand this request and will respond with
// an error with code -32601, which means that the RPC method name was not recognized.
// In that case we will switch to local key handling and update the keysDownState
// in client/browser-side code using simulateDeviceSideKeyHandlingForLegacyDevices.
const sendKeypressEvent = useCallback(
async (key: number, press: boolean) => {
if (rpcDataChannel?.readyState !== "open") return;
console.debug(`Send keypressEvent key: ${key}, press: ${press}`);
send("keypressReport", { key, press }, (resp: JsonRpcResponse) => {
if ("error" in resp) {
// -32601 means the method is not supported because the device is running an older version
if (resp.error.code === -32601) {
console.error("Legacy device does not support keypressReport API, switching to local key down state handling", resp.error);
setkeyPressReportApiAvailable(false);
} else {
console.error(`Failed to send key ${key} press: ${press}`, resp.error);
}
} else {
const keysDownState = resp.result as KeysDownState;
if (keysDownState) {
setKeysDownState(keysDownState);
// we don't need to set keyPressReportApiAvailable here, because it's already true or we never landed here
}
}
});
},
[rpcDataChannel?.readyState, send, setkeyPressReportApiAvailable, setKeysDownState],
[
rpcDataChannel?.readyState,
rpcHidReady,
send,
sendKeyboardEventHidRpc,
],
);
// resetKeyboardState is used to reset the keyboard state to no keys pressed and no modifiers.
@ -135,12 +119,17 @@ export default function useKeyboard() {
// It then sends the full keyboard state to the device.
const handleKeyPress = useCallback(
async (key: number, press: boolean) => {
if (rpcDataChannel?.readyState !== "open") return;
if (rpcDataChannel?.readyState !== "open" && !rpcHidReady) return;
if ((key || 0) === 0) return; // ignore zero key presses (they are bad mappings)
if (keyPressReportApiAvailable) {
if (rpcHidReady) {
// if the keyPress api is available, we can just send the key press event
sendKeypressEvent(key, press);
// sendKeypressEvent is used to send a single key press/release event to the device.
// It sends the key and whether it is pressed or released.
// Older device version doesn't support this API, so we will switch to local key handling
// In that case we will switch to local key handling and update the keysDownState
// in client/browser-side code using simulateDeviceSideKeyHandlingForLegacyDevices.
sendKeypressEventHidRpc(key, press);
} else {
// if the keyPress api is not available, we need to handle the key locally
const downState = simulateDeviceSideKeyHandlingForLegacyDevices(keysDownState, key, press);
@ -152,7 +141,14 @@ export default function useKeyboard() {
}
}
},
[keyPressReportApiAvailable, keysDownState, resetKeyboardState, rpcDataChannel?.readyState, sendKeyboardEvent, sendKeypressEvent],
[
rpcHidReady,
keysDownState,
resetKeyboardState,
rpcDataChannel?.readyState,
sendKeyboardEvent,
sendKeypressEventHidRpc,
],
);
// IMPORTANT: See the keyPressReportApiAvailable comment above for the reason this exists

172
ui/src/hooks/useMouse.ts Normal file
View File

@ -0,0 +1,172 @@
import { useCallback, useState } from "react";
import { useJsonRpc } from "./useJsonRpc";
import { useHidRpc } from "./useHidRpc";
import { useMouseStore, useSettingsStore } from "./stores";
const calcDelta = (pos: number) => (Math.abs(pos) < 10 ? pos * 2 : pos);
export interface AbsMouseMoveHandlerProps {
videoClientWidth: number;
videoClientHeight: number;
videoWidth: number;
videoHeight: number;
}
export default function useMouse() {
// states
const { setMousePosition, setMouseMove } = useMouseStore();
const [blockWheelEvent, setBlockWheelEvent] = useState(false);
const { mouseMode, scrollThrottling } = useSettingsStore();
// RPC hooks
const { send } = useJsonRpc();
const { reportAbsMouseEvent, reportRelMouseEvent, rpcHidReady } = useHidRpc();
// Mouse-related
const sendRelMouseMovement = useCallback(
(x: number, y: number, buttons: number) => {
if (mouseMode !== "relative") return;
// if we ignore the event, double-click will not work
// if (x === 0 && y === 0 && buttons === 0) return;
const dx = calcDelta(x);
const dy = calcDelta(y);
if (rpcHidReady) {
reportRelMouseEvent(dx, dy, buttons);
} else {
// kept for backward compatibility
send("relMouseReport", { dx, dy, buttons });
}
setMouseMove({ x, y, buttons });
},
[
send,
reportRelMouseEvent,
setMouseMove,
mouseMode,
rpcHidReady,
],
);
const getRelMouseMoveHandler = useCallback(
() => (e: MouseEvent) => {
if (mouseMode !== "relative") return;
// Send mouse movement
const { buttons } = e;
sendRelMouseMovement(e.movementX, e.movementY, buttons);
},
[sendRelMouseMovement, mouseMode],
);
const sendAbsMouseMovement = useCallback(
(x: number, y: number, buttons: number) => {
if (mouseMode !== "absolute") return;
if (rpcHidReady) {
reportAbsMouseEvent(x, y, buttons);
} else {
// kept for backward compatibility
send("absMouseReport", { x, y, buttons });
}
// We set that for the debug info bar
setMousePosition(x, y);
},
[
send,
reportAbsMouseEvent,
setMousePosition,
mouseMode,
rpcHidReady,
],
);
const getAbsMouseMoveHandler = useCallback(
({ videoClientWidth, videoClientHeight, videoWidth, videoHeight }: AbsMouseMoveHandlerProps) => (e: MouseEvent) => {
if (!videoClientWidth || !videoClientHeight) return;
if (mouseMode !== "absolute") return;
// Get the aspect ratios of the video element and the video stream
const videoElementAspectRatio = videoClientWidth / videoClientHeight;
const videoStreamAspectRatio = videoWidth / videoHeight;
// Calculate the effective video display area
let effectiveWidth = videoClientWidth;
let effectiveHeight = videoClientHeight;
let offsetX = 0;
let offsetY = 0;
if (videoElementAspectRatio > videoStreamAspectRatio) {
// Pillarboxing: black bars on the left and right
effectiveWidth = videoClientHeight * videoStreamAspectRatio;
offsetX = (videoClientWidth - effectiveWidth) / 2;
} else if (videoElementAspectRatio < videoStreamAspectRatio) {
// Letterboxing: black bars on the top and bottom
effectiveHeight = videoClientWidth / videoStreamAspectRatio;
offsetY = (videoClientHeight - effectiveHeight) / 2;
}
// Clamp mouse position within the effective video boundaries
const clampedX = Math.min(Math.max(offsetX, e.offsetX), offsetX + effectiveWidth);
const clampedY = Math.min(Math.max(offsetY, e.offsetY), offsetY + effectiveHeight);
// Map clamped mouse position to the video stream's coordinate system
const relativeX = (clampedX - offsetX) / effectiveWidth;
const relativeY = (clampedY - offsetY) / effectiveHeight;
// Convert to HID absolute coordinate system (0-32767 range)
const x = Math.round(relativeX * 32767);
const y = Math.round(relativeY * 32767);
// Send mouse movement
const { buttons } = e;
sendAbsMouseMovement(x, y, buttons);
}, [mouseMode, sendAbsMouseMovement],
);
const getMouseWheelHandler = useCallback(
() => (e: WheelEvent) => {
if (scrollThrottling && blockWheelEvent) {
return;
}
// Determine if the wheel event is an accel scroll value
const isAccel = Math.abs(e.deltaY) >= 100;
// Calculate the accel scroll value
const accelScrollValue = e.deltaY / 100;
// Calculate the no accel scroll value
const noAccelScrollValue = Math.sign(e.deltaY);
// Get scroll value
const scrollValue = isAccel ? accelScrollValue : noAccelScrollValue;
// Apply clamping (i.e. min and max mouse wheel hardware value)
const clampedScrollValue = Math.max(-127, Math.min(127, scrollValue));
// Invert the clamped scroll value to match expected behavior
const invertedScrollValue = -clampedScrollValue;
send("wheelReport", { wheelY: invertedScrollValue });
// Apply blocking delay based of throttling settings
if (scrollThrottling && !blockWheelEvent) {
setBlockWheelEvent(true);
setTimeout(() => setBlockWheelEvent(false), scrollThrottling);
}
},
[send, blockWheelEvent, scrollThrottling],
);
const resetMousePosition = useCallback(() => {
sendAbsMouseMovement(0, 0, 0);
}, [sendAbsMouseMovement]);
return {
getRelMouseMoveHandler,
getAbsMouseMoveHandler,
getMouseWheelHandler,
resetMousePosition,
};
}

16
ui/src/routes/devices.$id.tsx
View File

@ -134,7 +134,8 @@ export default function KvmIdRoute() {
setRpcDataChannel,
isTurnServerInUse, setTurnServerInUse,
rpcDataChannel,
setTransceiver
setTransceiver,
setRpcHidChannel,
} = useRTCStore();
const location = useLocation();
@ -481,6 +482,12 @@ export default function KvmIdRoute() {
setRpcDataChannel(rpcDataChannel);
};
const rpcHidChannel = pc.createDataChannel("hidrpc");
rpcHidChannel.binaryType = "arraybuffer";
rpcHidChannel.onopen = () => {
setRpcHidChannel(rpcHidChannel);
};
setPeerConnection(pc);
}, [
cleanupAndStopReconnecting,
@ -491,6 +498,7 @@ export default function KvmIdRoute() {
setPeerConnection,
setPeerConnectionState,
setRpcDataChannel,
setRpcHidChannel,
setTransceiver,
]);
@ -574,7 +582,6 @@ export default function KvmIdRoute() {
const {
keyboardLedState, setKeyboardLedState,
keysDownState, setKeysDownState, setUsbState,
setkeyPressReportApiAvailable
} = useHidStore();
const [hasUpdated, setHasUpdated] = useState(false);
@ -612,7 +619,6 @@ export default function KvmIdRoute() {
const downState = resp.params as KeysDownState;
console.debug("Setting key down state:", downState);
setKeysDownState(downState);
setkeyPressReportApiAvailable(true); // if they returned a keyDownState, we know they also support keyPressReport
}
if (resp.method === "otaState") {
@ -689,7 +695,6 @@ export default function KvmIdRoute() {
if (resp.error.code === -32601) {
// if we don't support key down state, we know key press is also not available
console.warn("Failed to get key down state, switching to old-school", resp.error);
setkeyPressReportApiAvailable(false);
} else {
console.error("Failed to get key down state", resp.error);
}
@ -697,11 +702,10 @@ export default function KvmIdRoute() {
const downState = resp.result as KeysDownState;
console.debug("Keyboard key down state", downState);
setKeysDownState(downState);
setkeyPressReportApiAvailable(true); // if they returned a keyDownState, we know they also support keyPressReport
}
setNeedKeyDownState(false);
});
}, [keysDownState, needKeyDownState, rpcDataChannel?.readyState, send, setkeyPressReportApiAvailable, setKeysDownState]);
}, [keysDownState, needKeyDownState, rpcDataChannel?.readyState, send, setKeysDownState]);
// When the update is successful, we need to refresh the client javascript and show a success modal
useEffect(() => {

3
ui/vite.config.ts
View File

@ -28,6 +28,9 @@ export default defineConfig(({ mode, command }) => {
return {
plugins,
esbuild: {
pure: ["console.debug"],
},
build: { outDir: isCloud ? "dist" : "../static" },
server: {
host: "0.0.0.0",

4
usb.go
View File

@ -27,13 +27,13 @@ func initUsbGadget() {
gadget.SetOnKeyboardStateChange(func(state usbgadget.KeyboardState) {
if currentSession != nil {
writeJSONRPCEvent("keyboardLedState", state, currentSession)
currentSession.reportHidRPCKeyboardLedState(state)
}
})
gadget.SetOnKeysDownChange(func(state usbgadget.KeysDownState) {
if currentSession != nil {
writeJSONRPCEvent("keysDownState", state, currentSession)
currentSession.reportHidRPCKeysDownState(state)
}
})

86
webrtc.go
View File

@ -6,10 +6,12 @@ import (
"encoding/json"
"net"
"strings"
"sync"
"github.com/coder/websocket"
"github.com/coder/websocket/wsjson"
"github.com/gin-gonic/gin"
"github.com/jetkvm/kvm/internal/hidrpc"
"github.com/jetkvm/kvm/internal/logging"
"github.com/pion/webrtc/v4"
"github.com/rs/zerolog"
@ -22,7 +24,12 @@ type Session struct {
RPCChannel *webrtc.DataChannel
HidChannel *webrtc.DataChannel
shouldUmountVirtualMedia bool
rpcQueue chan webrtc.DataChannelMessage
rpcQueue chan webrtc.DataChannelMessage
hidRPCAvailable bool
hidQueueLock sync.Mutex
hidQueue []chan webrtc.DataChannelMessage
}
type SessionConfig struct {
@ -67,6 +74,23 @@ func (s *Session) ExchangeOffer(offerStr string) (string, error) {
return base64.StdEncoding.EncodeToString(localDescription), nil
}
func (s *Session) initQueues() {
s.hidQueueLock.Lock()
defer s.hidQueueLock.Unlock()
s.hidQueue = make([]chan webrtc.DataChannelMessage, 0)
for i := 0; i < 4; i++ {
q := make(chan webrtc.DataChannelMessage, 256)
s.hidQueue = append(s.hidQueue, q)
}
}
func (s *Session) handleQueues(index int) {
for msg := range s.hidQueue[index] {
onHidMessage(msg.Data, s)
}
}
func newSession(config SessionConfig) (*Session, error) {
webrtcSettingEngine := webrtc.SettingEngine{
LoggerFactory: logging.GetPionDefaultLoggerFactory(),
@ -105,17 +129,68 @@ func newSession(config SessionConfig) (*Session, error) {
scopedLogger.Warn().Err(err).Msg("Failed to create PeerConnection")
return nil, err
}
session := &Session{peerConnection: peerConnection}
session.rpcQueue = make(chan webrtc.DataChannelMessage, 256)
session.initQueues()
go func() {
for msg := range session.rpcQueue {
onRPCMessage(msg, session)
// TODO: only use goroutine if the task is asynchronous
go onRPCMessage(msg, session)
}
}()
for i := 0; i < len(session.hidQueue); i++ {
go session.handleQueues(i)
}
peerConnection.OnDataChannel(func(d *webrtc.DataChannel) {
defer func() {
if r := recover(); r != nil {
scopedLogger.Error().Interface("error", r).Msg("Recovered from panic in DataChannel handler")
}
}()
scopedLogger.Info().Str("label", d.Label()).Uint16("id", *d.ID()).Msg("New DataChannel")
switch d.Label() {
case "hidrpc":
session.HidChannel = d
d.OnMessage(func(msg webrtc.DataChannelMessage) {
l := scopedLogger.With().Int("length", len(msg.Data)).Logger()
// only log data if the log level is debug or lower
if scopedLogger.GetLevel() > zerolog.DebugLevel {
l = l.With().Str("data", string(msg.Data)).Logger()
}
if msg.IsString {
l.Warn().Msg("received string data in HID RPC message handler")
return
}
if len(msg.Data) < 1 {
l.Warn().Msg("received empty data in HID RPC message handler")
return
}
l.Trace().Msg("received data in HID RPC message handler")
// Enqueue to ensure ordered processing
queueIndex := hidrpc.GetQueueIndex(hidrpc.MessageType(msg.Data[0]))
if queueIndex >= len(session.hidQueue) || queueIndex < 0 {
l.Warn().Int("queueIndex", queueIndex).Msg("received data in HID RPC message handler, but queue index not found")
queueIndex = 3
}
queue := session.hidQueue[queueIndex]
if queue != nil {
queue <- msg
} else {
l.Warn().Int("queueIndex", queueIndex).Msg("received data in HID RPC message handler, but queue is nil")
return
}
})
case "rpc":
session.RPCChannel = d
d.OnMessage(func(msg webrtc.DataChannelMessage) {
@ -198,6 +273,13 @@ func newSession(config SessionConfig) (*Session, error) {
close(session.rpcQueue)
session.rpcQueue = nil
}
// Stop HID RPC processor
for i := 0; i < len(session.hidQueue); i++ {
close(session.hidQueue[i])
session.hidQueue[i] = nil
}
if session.shouldUmountVirtualMedia {
if err := rpcUnmountImage(); err != nil {
scopedLogger.Warn().Err(err).Msg("unmount image failed on connection close")