diff --git a/hidrpc.go b/hidrpc.go new file mode 100644 index 0000000..74fe687 --- /dev/null +++ b/hidrpc.go @@ -0,0 +1,162 @@ +package kvm + +import ( + "fmt" + "time" + + "github.com/jetkvm/kvm/internal/hidrpc" + "github.com/jetkvm/kvm/internal/usbgadget" +) + +func handleHidRPCMessage(message hidrpc.Message, session *Session) { + var rpcErr error + + switch message.Type() { + case hidrpc.TypeHandshake: + message, err := hidrpc.NewHandshakeMessage().Marshal() + if err != nil { + logger.Warn().Err(err).Msg("failed to marshal handshake message") + return + } + if err := session.HidChannel.Send(message); err != nil { + logger.Warn().Err(err).Msg("failed to send handshake message") + return + } + session.hidRPCAvailable = true + case hidrpc.TypeKeypressReport, hidrpc.TypeKeyboardReport: + keysDownState, err := handleHidRPCKeyboardInput(message) + if keysDownState != nil { + session.reportHidRPCKeysDownState(*keysDownState) + } + rpcErr = err + case hidrpc.TypePointerReport: + pointerReport, err := message.PointerReport() + if err != nil { + logger.Warn().Err(err).Msg("failed to get pointer report") + return + } + rpcErr = rpcAbsMouseReport(pointerReport.X, pointerReport.Y, pointerReport.Button) + case hidrpc.TypeMouseReport: + mouseReport, err := message.MouseReport() + if err != nil { + logger.Warn().Err(err).Msg("failed to get mouse report") + return + } + rpcErr = rpcRelMouseReport(mouseReport.DX, mouseReport.DY, mouseReport.Button) + default: + logger.Warn().Uint8("type", uint8(message.Type())).Msg("unknown HID RPC message type") + } + + if rpcErr != nil { + logger.Warn().Err(rpcErr).Msg("failed to handle HID RPC message") + } +} + +func onHidMessage(data []byte, session *Session) { + scopedLogger := hidRPCLogger.With().Bytes("data", data).Logger() + scopedLogger.Debug().Msg("HID RPC message received") + + if len(data) < 1 { + scopedLogger.Warn().Int("length", len(data)).Msg("received empty data in HID RPC message handler") + return + } + + var message hidrpc.Message + + if err := hidrpc.Unmarshal(data, &message); err != nil { + scopedLogger.Warn().Err(err).Msg("failed to unmarshal HID RPC message") + return + } + + scopedLogger = scopedLogger.With().Str("descr", message.String()).Logger() + + t := time.Now() + + r := make(chan interface{}) + go func() { + handleHidRPCMessage(message, session) + r <- nil + }() + select { + case <-time.After(1 * time.Second): + scopedLogger.Warn().Msg("HID RPC message timed out") + case <-r: + scopedLogger.Debug().Dur("duration", time.Since(t)).Msg("HID RPC message handled") + } +} + +func handleHidRPCKeyboardInput(message hidrpc.Message) (*usbgadget.KeysDownState, error) { + switch message.Type() { + case hidrpc.TypeKeypressReport: + keypressReport, err := message.KeypressReport() + if err != nil { + logger.Warn().Err(err).Msg("failed to get keypress report") + return nil, err + } + keysDownState, rpcError := rpcKeypressReport(keypressReport.Key, keypressReport.Press) + return &keysDownState, rpcError + case hidrpc.TypeKeyboardReport: + keyboardReport, err := message.KeyboardReport() + if err != nil { + logger.Warn().Err(err).Msg("failed to get keyboard report") + return nil, err + } + keysDownState, rpcError := rpcKeyboardReport(keyboardReport.Modifier, keyboardReport.Keys) + return &keysDownState, rpcError + } + + return nil, fmt.Errorf("unknown HID RPC message type: %d", message.Type()) +} + +func reportHidRPC(params any, session *Session) { + if session == nil { + logger.Warn().Msg("session is nil, skipping reportHidRPC") + return + } + + if !session.hidRPCAvailable || session.HidChannel == nil { + logger.Warn().Msg("HID RPC is not available, skipping reportHidRPC") + return + } + + var ( + message []byte + err error + ) + switch params := params.(type) { + case usbgadget.KeyboardState: + message, err = hidrpc.NewKeyboardLedMessage(params).Marshal() + case usbgadget.KeysDownState: + message, err = hidrpc.NewKeydownStateMessage(params).Marshal() + default: + err = fmt.Errorf("unknown HID RPC message type: %T", params) + } + + if err != nil { + logger.Warn().Err(err).Msg("failed to marshal HID RPC message") + return + } + + if message == nil { + logger.Warn().Msg("failed to marshal HID RPC message") + return + } + + if err := session.HidChannel.Send(message); err != nil { + logger.Warn().Err(err).Msg("failed to send HID RPC message") + } +} + +func (s *Session) reportHidRPCKeyboardLedState(state usbgadget.KeyboardState) { + if !s.hidRPCAvailable { + writeJSONRPCEvent("keyboardLedState", state, s) + } + reportHidRPC(state, s) +} + +func (s *Session) reportHidRPCKeysDownState(state usbgadget.KeysDownState) { + if !s.hidRPCAvailable { + writeJSONRPCEvent("keysDownState", state, s) + } + reportHidRPC(state, s) +} diff --git a/internal/hidrpc/hidrpc.go b/internal/hidrpc/hidrpc.go new file mode 100644 index 0000000..e9c8c24 --- /dev/null +++ b/internal/hidrpc/hidrpc.go @@ -0,0 +1,100 @@ +package hidrpc + +import ( + "fmt" + + "github.com/jetkvm/kvm/internal/usbgadget" +) + +// MessageType is the type of the HID RPC message +type MessageType byte + +const ( + TypeHandshake MessageType = 0x01 + TypeKeyboardReport MessageType = 0x02 + TypePointerReport MessageType = 0x03 + TypeWheelReport MessageType = 0x04 + TypeKeypressReport MessageType = 0x05 + TypeMouseReport MessageType = 0x06 + TypeKeyboardLedState MessageType = 0x32 + TypeKeydownState MessageType = 0x33 +) + +const ( + Version byte = 0x01 // Version of the HID RPC protocol +) + +// GetQueueIndex returns the index of the queue to which the message should be enqueued. +func GetQueueIndex(messageType MessageType) int { + switch messageType { + case TypeHandshake: + return 0 + case TypeKeyboardReport, TypeKeypressReport, TypeKeyboardLedState, TypeKeydownState: + return 1 + case TypePointerReport, TypeMouseReport, TypeWheelReport: + return 2 + default: + return 3 + } +} + +// Unmarshal unmarshals the HID RPC message from the data. +func Unmarshal(data []byte, message *Message) error { + l := len(data) + if l < 1 { + return fmt.Errorf("invalid data length: %d", l) + } + + message.t = MessageType(data[0]) + message.d = data[1:] + return nil +} + +// Marshal marshals the HID RPC message to the data. +func Marshal(message *Message) ([]byte, error) { + if message.t == 0 { + return nil, fmt.Errorf("invalid message type: %d", message.t) + } + + data := make([]byte, len(message.d)+1) + data[0] = byte(message.t) + copy(data[1:], message.d) + + return data, nil +} + +// NewHandshakeMessage creates a new handshake message. +func NewHandshakeMessage() *Message { + return &Message{ + t: TypeHandshake, + d: []byte{Version}, + } +} + +// NewKeyboardReportMessage creates a new keyboard report message. +func NewKeyboardReportMessage(keys []byte, modifier uint8) *Message { + return &Message{ + t: TypeKeyboardReport, + d: append([]byte{modifier}, keys...), + } +} + +// NewKeyboardLedMessage creates a new keyboard LED message. +func NewKeyboardLedMessage(state usbgadget.KeyboardState) *Message { + return &Message{ + t: TypeKeyboardLedState, + d: []byte{state.Byte()}, + } +} + +// NewKeydownStateMessage creates a new keydown state message. +func NewKeydownStateMessage(state usbgadget.KeysDownState) *Message { + data := make([]byte, len(state.Keys)+1) + data[0] = state.Modifier + copy(data[1:], state.Keys) + + return &Message{ + t: TypeKeydownState, + d: data, + } +} diff --git a/internal/hidrpc/message.go b/internal/hidrpc/message.go new file mode 100644 index 0000000..84bbda7 --- /dev/null +++ b/internal/hidrpc/message.go @@ -0,0 +1,133 @@ +package hidrpc + +import ( + "fmt" +) + +// Message .. +type Message struct { + t MessageType + d []byte +} + +// Marshal marshals the message to a byte array. +func (m *Message) Marshal() ([]byte, error) { + return Marshal(m) +} + +func (m *Message) Type() MessageType { + return m.t +} + +func (m *Message) String() string { + switch m.t { + case TypeHandshake: + return "Handshake" + case TypeKeypressReport: + if len(m.d) < 2 { + return fmt.Sprintf("KeypressReport{Malformed: %v}", m.d) + } + return fmt.Sprintf("KeypressReport{Key: %d, Press: %v}", m.d[0], m.d[1] == uint8(1)) + case TypeKeyboardReport: + if len(m.d) < 2 { + return fmt.Sprintf("KeyboardReport{Malformed: %v}", m.d) + } + return fmt.Sprintf("KeyboardReport{Modifier: %d, Keys: %v}", m.d[0], m.d[1:]) + case TypePointerReport: + if len(m.d) < 9 { + return fmt.Sprintf("PointerReport{Malformed: %v}", m.d) + } + return fmt.Sprintf("PointerReport{X: %d, Y: %d, Button: %d}", m.d[0:4], m.d[4:8], m.d[8]) + case TypeMouseReport: + if len(m.d) < 3 { + return fmt.Sprintf("MouseReport{Malformed: %v}", m.d) + } + return fmt.Sprintf("MouseReport{DX: %d, DY: %d, Button: %d}", m.d[0], m.d[1], m.d[2]) + default: + return fmt.Sprintf("Unknown{Type: %d, Data: %v}", m.t, m.d) + } +} + +// KeypressReport .. +type KeypressReport struct { + Key byte + Press bool +} + +// KeypressReport returns the keypress report from the message. +func (m *Message) KeypressReport() (KeypressReport, error) { + if m.t != TypeKeypressReport { + return KeypressReport{}, fmt.Errorf("invalid message type: %d", m.t) + } + + return KeypressReport{ + Key: m.d[0], + Press: m.d[1] == uint8(1), + }, nil +} + +// KeyboardReport .. +type KeyboardReport struct { + Modifier byte + Keys []byte +} + +// KeyboardReport returns the keyboard report from the message. +func (m *Message) KeyboardReport() (KeyboardReport, error) { + if m.t != TypeKeyboardReport { + return KeyboardReport{}, fmt.Errorf("invalid message type: %d", m.t) + } + + return KeyboardReport{ + Modifier: m.d[0], + Keys: m.d[1:], + }, nil +} + +// PointerReport .. +type PointerReport struct { + X int + Y int + Button uint8 +} + +func toInt(b []byte) int { + return int(b[0])<<24 + int(b[1])<<16 + int(b[2])<<8 + int(b[3])<<0 +} + +// PointerReport returns the point report from the message. +func (m *Message) PointerReport() (PointerReport, error) { + if m.t != TypePointerReport { + return PointerReport{}, fmt.Errorf("invalid message type: %d", m.t) + } + + if len(m.d) != 9 { + return PointerReport{}, fmt.Errorf("invalid message length: %d", len(m.d)) + } + + return PointerReport{ + X: toInt(m.d[0:4]), + Y: toInt(m.d[4:8]), + Button: uint8(m.d[8]), + }, nil +} + +// MouseReport .. +type MouseReport struct { + DX int8 + DY int8 + Button uint8 +} + +// MouseReport returns the mouse report from the message. +func (m *Message) MouseReport() (MouseReport, error) { + if m.t != TypeMouseReport { + return MouseReport{}, fmt.Errorf("invalid message type: %d", m.t) + } + + return MouseReport{ + DX: int8(m.d[0]), + DY: int8(m.d[1]), + Button: uint8(m.d[2]), + }, nil +} diff --git a/internal/usbgadget/consts.go b/internal/usbgadget/consts.go index 8204d0a..958aecc 100644 --- a/internal/usbgadget/consts.go +++ b/internal/usbgadget/consts.go @@ -1,3 +1,7 @@ package usbgadget +import "time" + const dwc3Path = "/sys/bus/platform/drivers/dwc3" + +const hidWriteTimeout = 10 * time.Millisecond diff --git a/internal/usbgadget/hid_keyboard.go b/internal/usbgadget/hid_keyboard.go index f4fbaa6..fb710c2 100644 --- a/internal/usbgadget/hid_keyboard.go +++ b/internal/usbgadget/hid_keyboard.go @@ -86,6 +86,12 @@ type KeyboardState struct { Compose bool `json:"compose"` Kana bool `json:"kana"` Shift bool `json:"shift"` // This is not part of the main USB HID spec + raw byte +} + +// Byte returns the raw byte representation of the keyboard state. +func (k *KeyboardState) Byte() byte { + return k.raw } func getKeyboardState(b byte) KeyboardState { @@ -97,6 +103,7 @@ func getKeyboardState(b byte) KeyboardState { Compose: b&KeyboardLedMaskCompose != 0, Kana: b&KeyboardLedMaskKana != 0, Shift: b&KeyboardLedMaskShift != 0, + raw: b, } } @@ -139,19 +146,26 @@ func (u *UsbGadget) GetKeysDownState() KeysDownState { } func (u *UsbGadget) updateKeyDownState(state KeysDownState) { - u.keyboardStateLock.Lock() - defer u.keyboardStateLock.Unlock() + u.log.Trace().Interface("old", u.keysDownState).Interface("new", state).Msg("acquiring keyboardStateLock for updateKeyDownState") - if u.keysDownState.Modifier == state.Modifier && - bytes.Equal(u.keysDownState.Keys, state.Keys) { - return // No change in key down state + // this is intentional to unlock keyboard state lock before onKeysDownChange callback + { + u.keyboardStateLock.Lock() + defer u.keyboardStateLock.Unlock() + + if u.keysDownState.Modifier == state.Modifier && + bytes.Equal(u.keysDownState.Keys, state.Keys) { + return // No change in key down state + } + + u.log.Trace().Interface("old", u.keysDownState).Interface("new", state).Msg("keysDownState updated") + u.keysDownState = state } - u.log.Trace().Interface("old", u.keysDownState).Interface("new", state).Msg("keysDownState updated") - u.keysDownState = state - if u.onKeysDownChange != nil { + u.log.Trace().Interface("state", state).Msg("calling onKeysDownChange") (*u.onKeysDownChange)(state) + u.log.Trace().Interface("state", state).Msg("onKeysDownChange called") } } @@ -233,7 +247,7 @@ func (u *UsbGadget) keyboardWriteHidFile(modifier byte, keys []byte) error { return err } - _, err := u.keyboardHidFile.Write(append([]byte{modifier, 0x00}, keys[:hidKeyBufferSize]...)) + _, err := u.writeWithTimeout(u.keyboardHidFile, append([]byte{modifier, 0x00}, keys[:hidKeyBufferSize]...)) if err != nil { u.logWithSuppression("keyboardWriteHidFile", 100, u.log, err, "failed to write to hidg0") u.keyboardHidFile.Close() diff --git a/internal/usbgadget/hid_mouse_absolute.go b/internal/usbgadget/hid_mouse_absolute.go index c083b60..374844f 100644 --- a/internal/usbgadget/hid_mouse_absolute.go +++ b/internal/usbgadget/hid_mouse_absolute.go @@ -74,7 +74,7 @@ func (u *UsbGadget) absMouseWriteHidFile(data []byte) error { } } - _, err := u.absMouseHidFile.Write(data) + _, err := u.writeWithTimeout(u.absMouseHidFile, data) if err != nil { u.logWithSuppression("absMouseWriteHidFile", 100, u.log, err, "failed to write to hidg1") u.absMouseHidFile.Close() diff --git a/internal/usbgadget/hid_mouse_relative.go b/internal/usbgadget/hid_mouse_relative.go index 70cb72c..070db6e 100644 --- a/internal/usbgadget/hid_mouse_relative.go +++ b/internal/usbgadget/hid_mouse_relative.go @@ -64,7 +64,7 @@ func (u *UsbGadget) relMouseWriteHidFile(data []byte) error { } } - _, err := u.relMouseHidFile.Write(data) + _, err := u.writeWithTimeout(u.relMouseHidFile, data) if err != nil { u.logWithSuppression("relMouseWriteHidFile", 100, u.log, err, "failed to write to hidg2") u.relMouseHidFile.Close() diff --git a/internal/usbgadget/utils.go b/internal/usbgadget/utils.go index 05fcd3a..d51f9e4 100644 --- a/internal/usbgadget/utils.go +++ b/internal/usbgadget/utils.go @@ -3,10 +3,13 @@ package usbgadget import ( "bytes" "encoding/json" + "errors" "fmt" + "os" "path/filepath" "strconv" "strings" + "time" "github.com/rs/zerolog" ) @@ -107,6 +110,31 @@ func compareFileContent(oldContent []byte, newContent []byte, looserMatch bool) return false } +func (u *UsbGadget) writeWithTimeout(file *os.File, data []byte) (n int, err error) { + if err := file.SetWriteDeadline(time.Now().Add(hidWriteTimeout)); err != nil { + return -1, err + } + + n, err = file.Write(data) + if err == nil { + return + } + + if errors.Is(err, os.ErrDeadlineExceeded) { + u.logWithSuppression( + fmt.Sprintf("writeWithTimeout_%s", file.Name()), + 1000, + u.log, + err, + "write timed out: %s", + file.Name(), + ) + err = nil + } + + return +} + func (u *UsbGadget) logWithSuppression(counterName string, every int, logger *zerolog.Logger, err error, msg string, args ...any) { u.logSuppressionLock.Lock() defer u.logSuppressionLock.Unlock() diff --git a/jsonrpc.go b/jsonrpc.go index 82b12d0..ff3a4b1 100644 --- a/jsonrpc.go +++ b/jsonrpc.go @@ -83,7 +83,7 @@ func writeJSONRPCEvent(event string, params any, session *Session) { Str("data", requestString). Logger() - scopedLogger.Info().Msg("sending JSONRPC event") + scopedLogger.Trace().Msg("sending JSONRPC event") err = session.RPCChannel.SendText(requestString) if err != nil { diff --git a/log.go b/log.go index 1a091b1..2047bbf 100644 --- a/log.go +++ b/log.go @@ -19,6 +19,7 @@ var ( nbdLogger = logging.GetSubsystemLogger("nbd") timesyncLogger = logging.GetSubsystemLogger("timesync") jsonRpcLogger = logging.GetSubsystemLogger("jsonrpc") + hidRPCLogger = logging.GetSubsystemLogger("hidrpc") watchdogLogger = logging.GetSubsystemLogger("watchdog") websecureLogger = logging.GetSubsystemLogger("websecure") otaLogger = logging.GetSubsystemLogger("ota") diff --git a/ui/src/components/InfoBar.tsx b/ui/src/components/InfoBar.tsx index 29f159d..8d0b282 100644 --- a/ui/src/components/InfoBar.tsx +++ b/ui/src/components/InfoBar.tsx @@ -10,10 +10,12 @@ import { VideoState } from "@/hooks/stores"; import { keys, modifiers } from "@/keyboardMappings"; +import { useHidRpc } from "@/hooks/useHidRpc"; export default function InfoBar() { const { keysDownState } = useHidStore(); const { mouseX, mouseY, mouseMove } = useMouseStore(); + const { rpcHidStatus } = useHidRpc(); const videoClientSize = useVideoStore( (state: VideoState) => `${Math.round(state.clientWidth)}x${Math.round(state.clientHeight)}`, @@ -46,7 +48,7 @@ export default function InfoBar() { const modifierNames = Object.entries(modifiers).filter(([_, mask]) => (activeModifierMask & mask) !== 0).map(([name, _]) => name); const keyNames = Object.entries(keys).filter(([_, value]) => keysDown.includes(value)).map(([name, _]) => name); - return [...modifierNames,...keyNames].join(", "); + return [...modifierNames, ...keyNames].join(", "); }, [keysDownState, showPressedKeys]); return ( @@ -100,6 +102,12 @@ export default function InfoBar() { {hdmiState} )} + {debugMode && ( +
+ HidRPC State: + {rpcHidStatus} +
+ )} {showPressedKeys && (
diff --git a/ui/src/components/WebRTCVideo.tsx b/ui/src/components/WebRTCVideo.tsx index 9e2f0f2..bc6897e 100644 --- a/ui/src/components/WebRTCVideo.tsx +++ b/ui/src/components/WebRTCVideo.tsx @@ -7,15 +7,14 @@ import MacroBar from "@/components/MacroBar"; import InfoBar from "@components/InfoBar"; import notifications from "@/notifications"; import useKeyboard from "@/hooks/useKeyboard"; -import { useJsonRpc } from "@/hooks/useJsonRpc"; import { cx } from "@/cva.config"; import { keys } from "@/keyboardMappings"; import { - useMouseStore, useRTCStore, useSettingsStore, useVideoStore, } from "@/hooks/stores"; +import useMouse from "@/hooks/useMouse"; import { HDMIErrorOverlay, @@ -31,10 +30,18 @@ export default function WebRTCVideo() { const [isPlaying, setIsPlaying] = useState(false); const [isPointerLockActive, setIsPointerLockActive] = useState(false); const [isKeyboardLockActive, setIsKeyboardLockActive] = useState(false); + + const isPointerLockPossible = window.location.protocol === "https:" || window.location.hostname === "localhost"; + // Store hooks const settings = useSettingsStore(); const { handleKeyPress, resetKeyboardState } = useKeyboard(); - const { setMousePosition, setMouseMove } = useMouseStore(); + const { + getRelMouseMoveHandler, + getAbsMouseMoveHandler, + getMouseWheelHandler, + resetMousePosition, + } = useMouse(); const { setClientSize: setVideoClientSize, setSize: setVideoSize, @@ -55,15 +62,9 @@ export default function WebRTCVideo() { const hdmiError = ["no_lock", "no_signal", "out_of_range"].includes(hdmiState); const isVideoLoading = !isPlaying; - // Mouse wheel states - const [blockWheelEvent, setBlockWheelEvent] = useState(false); - - // Misc states and hooks - const { send } = useJsonRpc(); - // Video-related const handleResize = useCallback( - ( { width, height }: { width: number | undefined; height: number | undefined }) => { + ({ width, height }: { width: number | undefined; height: number | undefined }) => { if (!videoElm.current) return; // Do something with width and height, e.g.: setVideoClientSize(width || 0, height || 0); @@ -99,7 +100,6 @@ export default function WebRTCVideo() { ); // Pointer lock and keyboard lock related - const isPointerLockPossible = window.location.protocol === "https:" || window.location.hostname === "localhost"; const isFullscreenEnabled = document.fullscreenEnabled; const checkNavigatorPermissions = useCallback(async (permissionName: string) => { @@ -211,129 +211,29 @@ export default function WebRTCVideo() { } }; - document.addEventListener("fullscreenchange ", handleFullscreenChange); + document.addEventListener("fullscreenchange", handleFullscreenChange); }, [releaseKeyboardLock]); - // Mouse-related - const calcDelta = (pos: number) => (Math.abs(pos) < 10 ? pos * 2 : pos); - - const sendRelMouseMovement = useCallback( - (x: number, y: number, buttons: number) => { - if (settings.mouseMode !== "relative") return; - // if we ignore the event, double-click will not work - // if (x === 0 && y === 0 && buttons === 0) return; - send("relMouseReport", { dx: calcDelta(x), dy: calcDelta(y), buttons }); - setMouseMove({ x, y, buttons }); - }, - [send, setMouseMove, settings.mouseMode], + const absMouseMoveHandler = useMemo( + () => getAbsMouseMoveHandler({ + videoClientWidth, + videoClientHeight, + videoWidth, + videoHeight, + }), + [getAbsMouseMoveHandler, videoClientWidth, videoClientHeight, videoWidth, videoHeight], ); - const relMouseMoveHandler = useCallback( - (e: MouseEvent) => { - if (settings.mouseMode !== "relative") return; - if (isPointerLockActive === false && isPointerLockPossible) return; - - // Send mouse movement - const { buttons } = e; - sendRelMouseMovement(e.movementX, e.movementY, buttons); - }, - [isPointerLockActive, isPointerLockPossible, sendRelMouseMovement, settings.mouseMode], + const relMouseMoveHandler = useMemo( + () => getRelMouseMoveHandler(), + [getRelMouseMoveHandler], ); - const sendAbsMouseMovement = useCallback( - (x: number, y: number, buttons: number) => { - if (settings.mouseMode !== "absolute") return; - send("absMouseReport", { x, y, buttons }); - // We set that for the debug info bar - setMousePosition(x, y); - }, - [send, setMousePosition, settings.mouseMode], + const mouseWheelHandler = useMemo( + () => getMouseWheelHandler(), + [getMouseWheelHandler], ); - const absMouseMoveHandler = useCallback( - (e: MouseEvent) => { - if (!videoClientWidth || !videoClientHeight) return; - if (settings.mouseMode !== "absolute") return; - - // Get the aspect ratios of the video element and the video stream - const videoElementAspectRatio = videoClientWidth / videoClientHeight; - const videoStreamAspectRatio = videoWidth / videoHeight; - - // Calculate the effective video display area - let effectiveWidth = videoClientWidth; - let effectiveHeight = videoClientHeight; - let offsetX = 0; - let offsetY = 0; - - if (videoElementAspectRatio > videoStreamAspectRatio) { - // Pillarboxing: black bars on the left and right - effectiveWidth = videoClientHeight * videoStreamAspectRatio; - offsetX = (videoClientWidth - effectiveWidth) / 2; - } else if (videoElementAspectRatio < videoStreamAspectRatio) { - // Letterboxing: black bars on the top and bottom - effectiveHeight = videoClientWidth / videoStreamAspectRatio; - offsetY = (videoClientHeight - effectiveHeight) / 2; - } - - // Clamp mouse position within the effective video boundaries - const clampedX = Math.min(Math.max(offsetX, e.offsetX), offsetX + effectiveWidth); - const clampedY = Math.min(Math.max(offsetY, e.offsetY), offsetY + effectiveHeight); - - // Map clamped mouse position to the video stream's coordinate system - const relativeX = (clampedX - offsetX) / effectiveWidth; - const relativeY = (clampedY - offsetY) / effectiveHeight; - - // Convert to HID absolute coordinate system (0-32767 range) - const x = Math.round(relativeX * 32767); - const y = Math.round(relativeY * 32767); - - // Send mouse movement - const { buttons } = e; - sendAbsMouseMovement(x, y, buttons); - }, - [settings.mouseMode, videoClientWidth, videoClientHeight, videoWidth, videoHeight, sendAbsMouseMovement], - ); - - const mouseWheelHandler = useCallback( - (e: WheelEvent) => { - - if (settings.scrollThrottling && blockWheelEvent) { - return; - } - - // Determine if the wheel event is an accel scroll value - const isAccel = Math.abs(e.deltaY) >= 100; - - // Calculate the accel scroll value - const accelScrollValue = e.deltaY / 100; - - // Calculate the no accel scroll value - const noAccelScrollValue = Math.sign(e.deltaY); - - // Get scroll value - const scrollValue = isAccel ? accelScrollValue : noAccelScrollValue; - - // Apply clamping (i.e. min and max mouse wheel hardware value) - const clampedScrollValue = Math.max(-127, Math.min(127, scrollValue)); - - // Invert the clamped scroll value to match expected behavior - const invertedScrollValue = -clampedScrollValue; - - send("wheelReport", { wheelY: invertedScrollValue }); - - // Apply blocking delay based of throttling settings - if (settings.scrollThrottling && !blockWheelEvent) { - setBlockWheelEvent(true); - setTimeout(() => setBlockWheelEvent(false), settings.scrollThrottling); - } - }, - [send, blockWheelEvent, settings], - ); - - const resetMousePosition = useCallback(() => { - sendAbsMouseMovement(0, 0, 0); - }, [sendAbsMouseMovement]); - const keyDownHandler = useCallback( (e: KeyboardEvent) => { e.preventDefault(); @@ -357,7 +257,7 @@ export default function WebRTCVideo() { } console.debug(`Key down: ${hidKey}`); handleKeyPress(hidKey, true); - + if (!isKeyboardLockActive && hidKey === keys.MetaLeft) { // If the left meta key was just pressed and we're not keyboard locked // we'll never see the keyup event because the browser is going to lose @@ -488,14 +388,16 @@ export default function WebRTCVideo() { function setMouseModeEventListeners() { const videoElmRefValue = videoElm.current; if (!videoElmRefValue) return; + const isRelativeMouseMode = (settings.mouseMode === "relative"); + const mouseHandler = isRelativeMouseMode ? relMouseMoveHandler : absMouseMoveHandler; const abortController = new AbortController(); const signal = abortController.signal; - videoElmRefValue.addEventListener("mousemove", isRelativeMouseMode ? relMouseMoveHandler : absMouseMoveHandler, { signal }); - videoElmRefValue.addEventListener("pointerdown", isRelativeMouseMode ? relMouseMoveHandler : absMouseMoveHandler, { signal }); - videoElmRefValue.addEventListener("pointerup", isRelativeMouseMode ? relMouseMoveHandler : absMouseMoveHandler, { signal }); + videoElmRefValue.addEventListener("mousemove", mouseHandler, { signal }); + videoElmRefValue.addEventListener("pointerdown", mouseHandler, { signal }); + videoElmRefValue.addEventListener("pointerup", mouseHandler, { signal }); videoElmRefValue.addEventListener("wheel", mouseWheelHandler, { signal, passive: true, @@ -523,7 +425,16 @@ export default function WebRTCVideo() { abortController.abort(); }; }, - [absMouseMoveHandler, isPointerLockActive, isPointerLockPossible, mouseWheelHandler, relMouseMoveHandler, requestPointerLock, resetMousePosition, settings.mouseMode], + [ + isPointerLockActive, + isPointerLockPossible, + requestPointerLock, + absMouseMoveHandler, + relMouseMoveHandler, + mouseWheelHandler, + resetMousePosition, + settings.mouseMode, + ], ); const containerRef = useRef(null); diff --git a/ui/src/hooks/hidRpc.ts b/ui/src/hooks/hidRpc.ts new file mode 100644 index 0000000..20b8a10 --- /dev/null +++ b/ui/src/hooks/hidRpc.ts @@ -0,0 +1,302 @@ +import { KeyboardLedState, KeysDownState } from "./stores"; + +export const HID_RPC_MESSAGE_TYPES = { + Handshake: 0x01, + KeyboardReport: 0x02, + PointerReport: 0x03, + WheelReport: 0x04, + KeypressReport: 0x05, + MouseReport: 0x06, + KeyboardLedState: 0x32, + KeysDownState: 0x33, +} + +export type HidRpcMessageType = typeof HID_RPC_MESSAGE_TYPES[keyof typeof HID_RPC_MESSAGE_TYPES]; + +export const HID_RPC_VERSION = 0x01; + +const withinUint8Range = (value: number) => { + return value >= 0 && value <= 255; +}; + +const fromInt32toUint8 = (n: number) => { + if (n !== n >> 0) { + throw new Error(`Number ${n} is not within the int32 range`); + } + + return new Uint8Array([ + (n >> 24) & 0xFF, + (n >> 16) & 0xFF, + (n >> 8) & 0xFF, + (n >> 0) & 0xFF, + ]); +}; + +const fromInt8ToUint8 = (n: number) => { + if (n < -128 || n > 127) { + throw new Error(`Number ${n} is not within the int8 range`); + } + + return (n >> 0) & 0xFF; +}; + +const keyboardLedStateMasks = { + num_lock: 1 << 0, + caps_lock: 1 << 1, + scroll_lock: 1 << 2, + compose: 1 << 3, + kana: 1 << 4, + shift: 1 << 6, +} + +export class RpcMessage { + messageType: HidRpcMessageType; + + constructor(messageType: HidRpcMessageType) { + this.messageType = messageType; + } + + marshal(): Uint8Array { + throw new Error("Not implemented"); + } + + public static unmarshal(_data: Uint8Array): RpcMessage | undefined { + throw new Error("Not implemented"); + } +} + +export class HandshakeMessage extends RpcMessage { + version: number; + + constructor(version: number) { + super(HID_RPC_MESSAGE_TYPES.Handshake); + this.version = version; + } + + marshal(): Uint8Array { + return new Uint8Array([this.messageType, this.version]); + } + + public static unmarshal(data: Uint8Array): HandshakeMessage | undefined { + if (data.length < 1) { + throw new Error(`Invalid handshake message length: ${data.length}`); + } + + return new HandshakeMessage(data[0]); + } +} + +export class KeypressReportMessage extends RpcMessage { + private _key = 0; + private _press = false; + + get key(): number { + return this._key; + } + + set key(value: number) { + if (!withinUint8Range(value)) { + throw new Error(`Key ${value} is not within the uint8 range`); + } + + this._key = value; + } + + get press(): boolean { + return this._press; + } + + set press(value: boolean) { + this._press = value; + } + + constructor(key: number, press: boolean) { + super(HID_RPC_MESSAGE_TYPES.KeypressReport); + this.key = key; + this.press = press; + } + + marshal(): Uint8Array { + return new Uint8Array([ + this.messageType, + this.key, + this.press ? 1 : 0, + ]); + } + + public static unmarshal(data: Uint8Array): KeypressReportMessage | undefined { + if (data.length < 1) { + throw new Error(`Invalid keypress report message length: ${data.length}`); + } + + return new KeypressReportMessage(data[0], data[1] === 1); + } +} + +export class KeyboardReportMessage extends RpcMessage { + private _keys: number[] = []; + private _modifier = 0; + + get keys(): number[] { + return this._keys; + } + + set keys(value: number[]) { + value.forEach((k) => { + if (!withinUint8Range(k)) { + throw new Error(`Key ${k} is not within the uint8 range`); + } + }); + + this._keys = value; + } + + get modifier(): number { + return this._modifier; + } + + set modifier(value: number) { + if (!withinUint8Range(value)) { + throw new Error(`Modifier ${value} is not within the uint8 range`); + } + + this._modifier = value; + } + + constructor(keys: number[], modifier: number) { + super(HID_RPC_MESSAGE_TYPES.KeyboardReport); + this.keys = keys; + this.modifier = modifier; + } + + marshal(): Uint8Array { + return new Uint8Array([ + this.messageType, + this.modifier, + ...this.keys, + ]); + } + + public static unmarshal(data: Uint8Array): KeyboardReportMessage | undefined { + if (data.length < 1) { + throw new Error(`Invalid keyboard report message length: ${data.length}`); + } + + return new KeyboardReportMessage(Array.from(data.slice(1)), data[0]); + } +} + +export class KeyboardLedStateMessage extends RpcMessage { + keyboardLedState: KeyboardLedState; + + constructor(keyboardLedState: KeyboardLedState) { + super(HID_RPC_MESSAGE_TYPES.KeyboardLedState); + this.keyboardLedState = keyboardLedState; + } + + public static unmarshal(data: Uint8Array): KeyboardLedStateMessage | undefined { + if (data.length < 1) { + throw new Error(`Invalid keyboard led state message length: ${data.length}`); + } + + const s = data[0]; + + const state = { + num_lock: (s & keyboardLedStateMasks.num_lock) !== 0, + caps_lock: (s & keyboardLedStateMasks.caps_lock) !== 0, + scroll_lock: (s & keyboardLedStateMasks.scroll_lock) !== 0, + compose: (s & keyboardLedStateMasks.compose) !== 0, + kana: (s & keyboardLedStateMasks.kana) !== 0, + shift: (s & keyboardLedStateMasks.shift) !== 0, + } as KeyboardLedState; + + return new KeyboardLedStateMessage(state); + } +} + +export class KeysDownStateMessage extends RpcMessage { + keysDownState: KeysDownState; + + constructor(keysDownState: KeysDownState) { + super(HID_RPC_MESSAGE_TYPES.KeysDownState); + this.keysDownState = keysDownState; + } + + public static unmarshal(data: Uint8Array): KeysDownStateMessage | undefined { + if (data.length < 1) { + throw new Error(`Invalid keys down state message length: ${data.length}`); + } + + return new KeysDownStateMessage({ + modifier: data[0], + keys: Array.from(data.slice(1)) + }); + } +} + +export class PointerReportMessage extends RpcMessage { + x: number; + y: number; + buttons: number; + + constructor(x: number, y: number, buttons: number) { + super(HID_RPC_MESSAGE_TYPES.PointerReport); + this.x = x; + this.y = y; + this.buttons = buttons; + } + + marshal(): Uint8Array { + return new Uint8Array([ + this.messageType, + ...fromInt32toUint8(this.x), + ...fromInt32toUint8(this.y), + this.buttons, + ]); + } +} + +export class MouseReportMessage extends RpcMessage { + dx: number; + dy: number; + buttons: number; + + constructor(dx: number, dy: number, buttons: number) { + super(HID_RPC_MESSAGE_TYPES.MouseReport); + this.dx = dx; + this.dy = dy; + this.buttons = buttons; + } + + marshal(): Uint8Array { + return new Uint8Array([ + this.messageType, + fromInt8ToUint8(this.dx), + fromInt8ToUint8(this.dy), + this.buttons, + ]); + } +} + +export const messageRegistry = { + [HID_RPC_MESSAGE_TYPES.Handshake]: HandshakeMessage, + [HID_RPC_MESSAGE_TYPES.KeysDownState]: KeysDownStateMessage, + [HID_RPC_MESSAGE_TYPES.KeyboardLedState]: KeyboardLedStateMessage, + [HID_RPC_MESSAGE_TYPES.KeyboardReport]: KeyboardReportMessage, + [HID_RPC_MESSAGE_TYPES.KeypressReport]: KeypressReportMessage, +} + +export const unmarshalHidRpcMessage = (data: Uint8Array): RpcMessage | undefined => { + if (data.length < 1) { + throw new Error(`Invalid HID RPC message length: ${data.length}`); + } + + const payload = data.slice(1); + + const messageType = data[0]; + if (!(messageType in messageRegistry)) { + throw new Error(`Unknown HID RPC message type: ${messageType}`); + } + + return messageRegistry[messageType].unmarshal(payload); +}; \ No newline at end of file diff --git a/ui/src/hooks/stores.ts b/ui/src/hooks/stores.ts index a3e05d8..f99fd07 100644 --- a/ui/src/hooks/stores.ts +++ b/ui/src/hooks/stores.ts @@ -105,6 +105,12 @@ export interface RTCState { setRpcDataChannel: (channel: RTCDataChannel) => void; rpcDataChannel: RTCDataChannel | null; + rpcHidProtocolVersion: number | null; + setRpcHidProtocolVersion: (version: number) => void; + + rpcHidChannel: RTCDataChannel | null; + setRpcHidChannel: (channel: RTCDataChannel) => void; + peerConnectionState: RTCPeerConnectionState | null; setPeerConnectionState: (state: RTCPeerConnectionState) => void; @@ -151,6 +157,12 @@ export const useRTCStore = create(set => ({ rpcDataChannel: null, setRpcDataChannel: (channel: RTCDataChannel) => set({ rpcDataChannel: channel }), + rpcHidProtocolVersion: null, + setRpcHidProtocolVersion: (version: number) => set({ rpcHidProtocolVersion: version }), + + rpcHidChannel: null, + setRpcHidChannel: (channel: RTCDataChannel) => set({ rpcHidChannel: channel }), + transceiver: null, setTransceiver: (transceiver: RTCRtpTransceiver) => set({ transceiver }), @@ -449,9 +461,6 @@ export interface HidState { keysDownState: KeysDownState; setKeysDownState: (state: KeysDownState) => void; - keyPressReportApiAvailable: boolean; - setkeyPressReportApiAvailable: (available: boolean) => void; - isVirtualKeyboardEnabled: boolean; setVirtualKeyboardEnabled: (enabled: boolean) => void; @@ -469,9 +478,6 @@ export const useHidStore = create(set => ({ keysDownState: { modifier: 0, keys: [0,0,0,0,0,0] } as KeysDownState, setKeysDownState: (state: KeysDownState): void => set({ keysDownState: state }), - keyPressReportApiAvailable: true, - setkeyPressReportApiAvailable: (available: boolean) => set({ keyPressReportApiAvailable: available }), - isVirtualKeyboardEnabled: false, setVirtualKeyboardEnabled: (enabled: boolean): void => set({ isVirtualKeyboardEnabled: enabled }), diff --git a/ui/src/hooks/useHidRpc.ts b/ui/src/hooks/useHidRpc.ts new file mode 100644 index 0000000..ea0c711 --- /dev/null +++ b/ui/src/hooks/useHidRpc.ts @@ -0,0 +1,150 @@ +import { useCallback, useEffect, useMemo } from "react"; + +import { useRTCStore } from "@/hooks/stores"; + +import { + HID_RPC_VERSION, + HandshakeMessage, + KeyboardReportMessage, + KeypressReportMessage, + MouseReportMessage, + PointerReportMessage, + RpcMessage, + unmarshalHidRpcMessage, +} from "./hidRpc"; + +export function useHidRpc(onHidRpcMessage?: (payload: RpcMessage) => void) { + const { rpcHidChannel, setRpcHidProtocolVersion, rpcHidProtocolVersion } = useRTCStore(); + const rpcHidReady = useMemo(() => { + return rpcHidChannel?.readyState === "open" && rpcHidProtocolVersion !== null; + }, [rpcHidChannel, rpcHidProtocolVersion]); + + const rpcHidStatus = useMemo(() => { + if (!rpcHidChannel) return "N/A"; + if (rpcHidChannel.readyState !== "open") return rpcHidChannel.readyState; + if (!rpcHidProtocolVersion) return "handshaking"; + return `ready (v${rpcHidProtocolVersion})`; + }, [rpcHidChannel, rpcHidProtocolVersion]); + + const sendMessage = useCallback((message: RpcMessage, ignoreHandshakeState = false) => { + if (rpcHidChannel?.readyState !== "open") return; + if (!rpcHidReady && !ignoreHandshakeState) return; + + let data: Uint8Array | undefined; + try { + data = message.marshal(); + } catch (e) { + console.error("Failed to send HID RPC message", e); + } + if (!data) return; + + rpcHidChannel?.send(data as unknown as ArrayBuffer); + }, [rpcHidChannel, rpcHidReady]); + + const reportKeyboardEvent = useCallback( + (keys: number[], modifier: number) => { + sendMessage(new KeyboardReportMessage(keys, modifier)); + }, [sendMessage], + ); + + const reportKeypressEvent = useCallback( + (key: number, press: boolean) => { + sendMessage(new KeypressReportMessage(key, press)); + }, + [sendMessage], + ); + + const reportAbsMouseEvent = useCallback( + (x: number, y: number, buttons: number) => { + sendMessage(new PointerReportMessage(x, y, buttons)); + }, + [sendMessage], + ); + + const reportRelMouseEvent = useCallback( + (dx: number, dy: number, buttons: number) => { + sendMessage(new MouseReportMessage(dx, dy, buttons)); + }, + [sendMessage], + ); + + const sendHandshake = useCallback(() => { + if (rpcHidProtocolVersion) return; + if (!rpcHidChannel) return; + + sendMessage(new HandshakeMessage(HID_RPC_VERSION), true); + }, [rpcHidChannel, rpcHidProtocolVersion, sendMessage]); + + const handleHandshake = useCallback((message: HandshakeMessage) => { + if (!message.version) { + console.error("Received handshake message without version", message); + return; + } + + if (message.version > HID_RPC_VERSION) { + // we assume that the UI is always using the latest version of the HID RPC protocol + // so we can't support this + // TODO: use capabilities to determine rather than version number + console.error("Server is using a newer HID RPC version than the client", message); + return; + } + + setRpcHidProtocolVersion(message.version); + }, [setRpcHidProtocolVersion]); + + useEffect(() => { + if (!rpcHidChannel) return; + + // send handshake message + sendHandshake(); + + const messageHandler = (e: MessageEvent) => { + if (typeof e.data === "string") { + console.warn("Received string data in HID RPC message handler", e.data); + return; + } + + const message = unmarshalHidRpcMessage(new Uint8Array(e.data)); + if (!message) { + console.warn("Received invalid HID RPC message", e.data); + return; + } + + console.debug("Received HID RPC message", message); + switch (message.constructor) { + case HandshakeMessage: + handleHandshake(message as HandshakeMessage); + break; + default: + // not all events are handled here, the rest are handled by the onHidRpcMessage callback + break; + } + + onHidRpcMessage?.(message); + }; + + rpcHidChannel.addEventListener("message", messageHandler); + + return () => { + rpcHidChannel.removeEventListener("message", messageHandler); + }; + }, + [ + rpcHidChannel, + onHidRpcMessage, + setRpcHidProtocolVersion, + sendHandshake, + handleHandshake, + ], + ); + + return { + reportKeyboardEvent, + reportKeypressEvent, + reportAbsMouseEvent, + reportRelMouseEvent, + rpcHidProtocolVersion, + rpcHidReady, + rpcHidStatus, + }; +} diff --git a/ui/src/hooks/useKeyboard.ts b/ui/src/hooks/useKeyboard.ts index 5f587b0..787df9a 100644 --- a/ui/src/hooks/useKeyboard.ts +++ b/ui/src/hooks/useKeyboard.ts @@ -1,13 +1,15 @@ import { useCallback } from "react"; -import { KeysDownState, useHidStore, useRTCStore, hidKeyBufferSize, hidErrorRollOver } from "@/hooks/stores"; +import { hidErrorRollOver, hidKeyBufferSize, KeysDownState, useHidStore, useRTCStore } from "@/hooks/stores"; import { JsonRpcResponse, useJsonRpc } from "@/hooks/useJsonRpc"; +import { useHidRpc } from "@/hooks/useHidRpc"; +import { KeyboardLedStateMessage, KeysDownStateMessage } from "@/hooks/hidRpc"; import { hidKeyToModifierMask, keys, modifiers } from "@/keyboardMappings"; export default function useKeyboard() { const { send } = useJsonRpc(); const { rpcDataChannel } = useRTCStore(); - const { keysDownState, setKeysDownState } = useHidStore(); + const { keysDownState, setKeysDownState, setKeyboardLedState } = useHidStore(); // INTRODUCTION: The earlier version of the JetKVM device shipped with all keyboard state // being tracked on the browser/client-side. When adding the keyPressReport API to the @@ -19,7 +21,24 @@ export default function useKeyboard() { // dynamically set when the device responds to the first key press event or reports its // keysDownState when queried since the keyPressReport was introduced together with the // getKeysDownState API. - const { keyPressReportApiAvailable, setkeyPressReportApiAvailable} = useHidStore(); + + // HidRPC is a binary format for exchanging keyboard and mouse events + const { + reportKeyboardEvent: sendKeyboardEventHidRpc, + reportKeypressEvent: sendKeypressEventHidRpc, + rpcHidReady, + } = useHidRpc((message) => { + switch (message.constructor) { + case KeysDownStateMessage: + setKeysDownState((message as KeysDownStateMessage).keysDownState); + break; + case KeyboardLedStateMessage: + setKeyboardLedState((message as KeyboardLedStateMessage).keyboardLedState); + break; + default: + break; + } + }); // sendKeyboardEvent is used to send the full keyboard state to the device for macro handling // and resetting keyboard state. It sends the keys currently pressed and the modifier state. @@ -27,63 +46,28 @@ export default function useKeyboard() { // or just accept the state if it does not support (returning no result) const sendKeyboardEvent = useCallback( async (state: KeysDownState) => { - if (rpcDataChannel?.readyState !== "open") return; + if (rpcDataChannel?.readyState !== "open" && !rpcHidReady) return; console.debug(`Send keyboardReport keys: ${state.keys}, modifier: ${state.modifier}`); + + if (rpcHidReady) { + console.debug("Sending keyboard report via HidRPC"); + sendKeyboardEventHidRpc(state.keys, state.modifier); + return; + } + send("keyboardReport", { keys: state.keys, modifier: state.modifier }, (resp: JsonRpcResponse) => { if ("error" in resp) { console.error(`Failed to send keyboard report ${state}`, resp.error); - } else { - // If the device supports keyPressReport API, it will (also) return the keysDownState when we send - // the keyboardReport - const keysDownState = resp.result as KeysDownState; - - if (keysDownState) { - setKeysDownState(keysDownState); // treat the response as the canonical state - setkeyPressReportApiAvailable(true); // if they returned a keysDownState, we ALSO know they also support keyPressReport - } else { - // older devices versions do not return the keyDownState - // so we just pretend they accepted what we sent - setKeysDownState(state); - setkeyPressReportApiAvailable(false); // we ALSO know they do not support keyPressReport - } } }); }, - [rpcDataChannel?.readyState, send, setKeysDownState, setkeyPressReportApiAvailable], - ); - - // sendKeypressEvent is used to send a single key press/release event to the device. - // It sends the key and whether it is pressed or released. - // Older device version will not understand this request and will respond with - // an error with code -32601, which means that the RPC method name was not recognized. - // In that case we will switch to local key handling and update the keysDownState - // in client/browser-side code using simulateDeviceSideKeyHandlingForLegacyDevices. - const sendKeypressEvent = useCallback( - async (key: number, press: boolean) => { - if (rpcDataChannel?.readyState !== "open") return; - - console.debug(`Send keypressEvent key: ${key}, press: ${press}`); - send("keypressReport", { key, press }, (resp: JsonRpcResponse) => { - if ("error" in resp) { - // -32601 means the method is not supported because the device is running an older version - if (resp.error.code === -32601) { - console.error("Legacy device does not support keypressReport API, switching to local key down state handling", resp.error); - setkeyPressReportApiAvailable(false); - } else { - console.error(`Failed to send key ${key} press: ${press}`, resp.error); - } - } else { - const keysDownState = resp.result as KeysDownState; - - if (keysDownState) { - setKeysDownState(keysDownState); - // we don't need to set keyPressReportApiAvailable here, because it's already true or we never landed here - } - } - }); - }, - [rpcDataChannel?.readyState, send, setkeyPressReportApiAvailable, setKeysDownState], + [ + rpcDataChannel?.readyState, + rpcHidReady, + send, + sendKeyboardEventHidRpc, + ], ); // resetKeyboardState is used to reset the keyboard state to no keys pressed and no modifiers. @@ -135,12 +119,17 @@ export default function useKeyboard() { // It then sends the full keyboard state to the device. const handleKeyPress = useCallback( async (key: number, press: boolean) => { - if (rpcDataChannel?.readyState !== "open") return; + if (rpcDataChannel?.readyState !== "open" && !rpcHidReady) return; if ((key || 0) === 0) return; // ignore zero key presses (they are bad mappings) - if (keyPressReportApiAvailable) { + if (rpcHidReady) { // if the keyPress api is available, we can just send the key press event - sendKeypressEvent(key, press); + // sendKeypressEvent is used to send a single key press/release event to the device. + // It sends the key and whether it is pressed or released. + // Older device version doesn't support this API, so we will switch to local key handling + // In that case we will switch to local key handling and update the keysDownState + // in client/browser-side code using simulateDeviceSideKeyHandlingForLegacyDevices. + sendKeypressEventHidRpc(key, press); } else { // if the keyPress api is not available, we need to handle the key locally const downState = simulateDeviceSideKeyHandlingForLegacyDevices(keysDownState, key, press); @@ -152,7 +141,14 @@ export default function useKeyboard() { } } }, - [keyPressReportApiAvailable, keysDownState, resetKeyboardState, rpcDataChannel?.readyState, sendKeyboardEvent, sendKeypressEvent], + [ + rpcHidReady, + keysDownState, + resetKeyboardState, + rpcDataChannel?.readyState, + sendKeyboardEvent, + sendKeypressEventHidRpc, + ], ); // IMPORTANT: See the keyPressReportApiAvailable comment above for the reason this exists diff --git a/ui/src/hooks/useMouse.ts b/ui/src/hooks/useMouse.ts new file mode 100644 index 0000000..a814bca --- /dev/null +++ b/ui/src/hooks/useMouse.ts @@ -0,0 +1,172 @@ +import { useCallback, useState } from "react"; + +import { useJsonRpc } from "./useJsonRpc"; +import { useHidRpc } from "./useHidRpc"; +import { useMouseStore, useSettingsStore } from "./stores"; + +const calcDelta = (pos: number) => (Math.abs(pos) < 10 ? pos * 2 : pos); + +export interface AbsMouseMoveHandlerProps { + videoClientWidth: number; + videoClientHeight: number; + videoWidth: number; + videoHeight: number; +} + +export default function useMouse() { + // states + const { setMousePosition, setMouseMove } = useMouseStore(); + const [blockWheelEvent, setBlockWheelEvent] = useState(false); + + const { mouseMode, scrollThrottling } = useSettingsStore(); + + // RPC hooks + const { send } = useJsonRpc(); + const { reportAbsMouseEvent, reportRelMouseEvent, rpcHidReady } = useHidRpc(); + // Mouse-related + + const sendRelMouseMovement = useCallback( + (x: number, y: number, buttons: number) => { + if (mouseMode !== "relative") return; + // if we ignore the event, double-click will not work + // if (x === 0 && y === 0 && buttons === 0) return; + const dx = calcDelta(x); + const dy = calcDelta(y); + if (rpcHidReady) { + reportRelMouseEvent(dx, dy, buttons); + } else { + // kept for backward compatibility + send("relMouseReport", { dx, dy, buttons }); + } + setMouseMove({ x, y, buttons }); + }, + [ + send, + reportRelMouseEvent, + setMouseMove, + mouseMode, + rpcHidReady, + ], + ); + + const getRelMouseMoveHandler = useCallback( + () => (e: MouseEvent) => { + if (mouseMode !== "relative") return; + + // Send mouse movement + const { buttons } = e; + sendRelMouseMovement(e.movementX, e.movementY, buttons); + }, + [sendRelMouseMovement, mouseMode], + ); + + const sendAbsMouseMovement = useCallback( + (x: number, y: number, buttons: number) => { + if (mouseMode !== "absolute") return; + if (rpcHidReady) { + reportAbsMouseEvent(x, y, buttons); + } else { + // kept for backward compatibility + send("absMouseReport", { x, y, buttons }); + } + // We set that for the debug info bar + setMousePosition(x, y); + }, + [ + send, + reportAbsMouseEvent, + setMousePosition, + mouseMode, + rpcHidReady, + ], + ); + + const getAbsMouseMoveHandler = useCallback( + ({ videoClientWidth, videoClientHeight, videoWidth, videoHeight }: AbsMouseMoveHandlerProps) => (e: MouseEvent) => { + if (!videoClientWidth || !videoClientHeight) return; + if (mouseMode !== "absolute") return; + + // Get the aspect ratios of the video element and the video stream + const videoElementAspectRatio = videoClientWidth / videoClientHeight; + const videoStreamAspectRatio = videoWidth / videoHeight; + + // Calculate the effective video display area + let effectiveWidth = videoClientWidth; + let effectiveHeight = videoClientHeight; + let offsetX = 0; + let offsetY = 0; + + if (videoElementAspectRatio > videoStreamAspectRatio) { + // Pillarboxing: black bars on the left and right + effectiveWidth = videoClientHeight * videoStreamAspectRatio; + offsetX = (videoClientWidth - effectiveWidth) / 2; + } else if (videoElementAspectRatio < videoStreamAspectRatio) { + // Letterboxing: black bars on the top and bottom + effectiveHeight = videoClientWidth / videoStreamAspectRatio; + offsetY = (videoClientHeight - effectiveHeight) / 2; + } + + // Clamp mouse position within the effective video boundaries + const clampedX = Math.min(Math.max(offsetX, e.offsetX), offsetX + effectiveWidth); + const clampedY = Math.min(Math.max(offsetY, e.offsetY), offsetY + effectiveHeight); + + // Map clamped mouse position to the video stream's coordinate system + const relativeX = (clampedX - offsetX) / effectiveWidth; + const relativeY = (clampedY - offsetY) / effectiveHeight; + + // Convert to HID absolute coordinate system (0-32767 range) + const x = Math.round(relativeX * 32767); + const y = Math.round(relativeY * 32767); + + // Send mouse movement + const { buttons } = e; + sendAbsMouseMovement(x, y, buttons); + }, [mouseMode, sendAbsMouseMovement], + ); + + const getMouseWheelHandler = useCallback( + () => (e: WheelEvent) => { + if (scrollThrottling && blockWheelEvent) { + return; + } + + // Determine if the wheel event is an accel scroll value + const isAccel = Math.abs(e.deltaY) >= 100; + + // Calculate the accel scroll value + const accelScrollValue = e.deltaY / 100; + + // Calculate the no accel scroll value + const noAccelScrollValue = Math.sign(e.deltaY); + + // Get scroll value + const scrollValue = isAccel ? accelScrollValue : noAccelScrollValue; + + // Apply clamping (i.e. min and max mouse wheel hardware value) + const clampedScrollValue = Math.max(-127, Math.min(127, scrollValue)); + + // Invert the clamped scroll value to match expected behavior + const invertedScrollValue = -clampedScrollValue; + + send("wheelReport", { wheelY: invertedScrollValue }); + + // Apply blocking delay based of throttling settings + if (scrollThrottling && !blockWheelEvent) { + setBlockWheelEvent(true); + setTimeout(() => setBlockWheelEvent(false), scrollThrottling); + } + }, + [send, blockWheelEvent, scrollThrottling], + ); + + const resetMousePosition = useCallback(() => { + sendAbsMouseMovement(0, 0, 0); + }, [sendAbsMouseMovement]); + + return { + getRelMouseMoveHandler, + getAbsMouseMoveHandler, + getMouseWheelHandler, + resetMousePosition, + }; +} \ No newline at end of file diff --git a/ui/src/routes/devices.$id.tsx b/ui/src/routes/devices.$id.tsx index a60c39a..4318447 100644 --- a/ui/src/routes/devices.$id.tsx +++ b/ui/src/routes/devices.$id.tsx @@ -134,7 +134,8 @@ export default function KvmIdRoute() { setRpcDataChannel, isTurnServerInUse, setTurnServerInUse, rpcDataChannel, - setTransceiver + setTransceiver, + setRpcHidChannel, } = useRTCStore(); const location = useLocation(); @@ -481,6 +482,12 @@ export default function KvmIdRoute() { setRpcDataChannel(rpcDataChannel); }; + const rpcHidChannel = pc.createDataChannel("hidrpc"); + rpcHidChannel.binaryType = "arraybuffer"; + rpcHidChannel.onopen = () => { + setRpcHidChannel(rpcHidChannel); + }; + setPeerConnection(pc); }, [ cleanupAndStopReconnecting, @@ -491,6 +498,7 @@ export default function KvmIdRoute() { setPeerConnection, setPeerConnectionState, setRpcDataChannel, + setRpcHidChannel, setTransceiver, ]); @@ -574,7 +582,6 @@ export default function KvmIdRoute() { const { keyboardLedState, setKeyboardLedState, keysDownState, setKeysDownState, setUsbState, - setkeyPressReportApiAvailable } = useHidStore(); const [hasUpdated, setHasUpdated] = useState(false); @@ -612,7 +619,6 @@ export default function KvmIdRoute() { const downState = resp.params as KeysDownState; console.debug("Setting key down state:", downState); setKeysDownState(downState); - setkeyPressReportApiAvailable(true); // if they returned a keyDownState, we know they also support keyPressReport } if (resp.method === "otaState") { @@ -689,7 +695,6 @@ export default function KvmIdRoute() { if (resp.error.code === -32601) { // if we don't support key down state, we know key press is also not available console.warn("Failed to get key down state, switching to old-school", resp.error); - setkeyPressReportApiAvailable(false); } else { console.error("Failed to get key down state", resp.error); } @@ -697,11 +702,10 @@ export default function KvmIdRoute() { const downState = resp.result as KeysDownState; console.debug("Keyboard key down state", downState); setKeysDownState(downState); - setkeyPressReportApiAvailable(true); // if they returned a keyDownState, we know they also support keyPressReport } setNeedKeyDownState(false); }); - }, [keysDownState, needKeyDownState, rpcDataChannel?.readyState, send, setkeyPressReportApiAvailable, setKeysDownState]); + }, [keysDownState, needKeyDownState, rpcDataChannel?.readyState, send, setKeysDownState]); // When the update is successful, we need to refresh the client javascript and show a success modal useEffect(() => { diff --git a/ui/vite.config.ts b/ui/vite.config.ts index 5871c4b..44eec3a 100644 --- a/ui/vite.config.ts +++ b/ui/vite.config.ts @@ -28,6 +28,9 @@ export default defineConfig(({ mode, command }) => { return { plugins, + esbuild: { + pure: ["console.debug"], + }, build: { outDir: isCloud ? "dist" : "../static" }, server: { host: "0.0.0.0", diff --git a/usb.go b/usb.go index d29e01a..131cd51 100644 --- a/usb.go +++ b/usb.go @@ -27,13 +27,13 @@ func initUsbGadget() { gadget.SetOnKeyboardStateChange(func(state usbgadget.KeyboardState) { if currentSession != nil { - writeJSONRPCEvent("keyboardLedState", state, currentSession) + currentSession.reportHidRPCKeyboardLedState(state) } }) gadget.SetOnKeysDownChange(func(state usbgadget.KeysDownState) { if currentSession != nil { - writeJSONRPCEvent("keysDownState", state, currentSession) + currentSession.reportHidRPCKeysDownState(state) } }) diff --git a/webrtc.go b/webrtc.go index c0f159a..c3d0dc1 100644 --- a/webrtc.go +++ b/webrtc.go @@ -6,10 +6,12 @@ import ( "encoding/json" "net" "strings" + "sync" "github.com/coder/websocket" "github.com/coder/websocket/wsjson" "github.com/gin-gonic/gin" + "github.com/jetkvm/kvm/internal/hidrpc" "github.com/jetkvm/kvm/internal/logging" "github.com/pion/webrtc/v4" "github.com/rs/zerolog" @@ -22,7 +24,12 @@ type Session struct { RPCChannel *webrtc.DataChannel HidChannel *webrtc.DataChannel shouldUmountVirtualMedia bool - rpcQueue chan webrtc.DataChannelMessage + + rpcQueue chan webrtc.DataChannelMessage + + hidRPCAvailable bool + hidQueueLock sync.Mutex + hidQueue []chan webrtc.DataChannelMessage } type SessionConfig struct { @@ -67,6 +74,23 @@ func (s *Session) ExchangeOffer(offerStr string) (string, error) { return base64.StdEncoding.EncodeToString(localDescription), nil } +func (s *Session) initQueues() { + s.hidQueueLock.Lock() + defer s.hidQueueLock.Unlock() + + s.hidQueue = make([]chan webrtc.DataChannelMessage, 0) + for i := 0; i < 4; i++ { + q := make(chan webrtc.DataChannelMessage, 256) + s.hidQueue = append(s.hidQueue, q) + } +} + +func (s *Session) handleQueues(index int) { + for msg := range s.hidQueue[index] { + onHidMessage(msg.Data, s) + } +} + func newSession(config SessionConfig) (*Session, error) { webrtcSettingEngine := webrtc.SettingEngine{ LoggerFactory: logging.GetPionDefaultLoggerFactory(), @@ -105,17 +129,68 @@ func newSession(config SessionConfig) (*Session, error) { scopedLogger.Warn().Err(err).Msg("Failed to create PeerConnection") return nil, err } + session := &Session{peerConnection: peerConnection} session.rpcQueue = make(chan webrtc.DataChannelMessage, 256) + session.initQueues() + go func() { for msg := range session.rpcQueue { - onRPCMessage(msg, session) + // TODO: only use goroutine if the task is asynchronous + go onRPCMessage(msg, session) } }() + for i := 0; i < len(session.hidQueue); i++ { + go session.handleQueues(i) + } + peerConnection.OnDataChannel(func(d *webrtc.DataChannel) { + defer func() { + if r := recover(); r != nil { + scopedLogger.Error().Interface("error", r).Msg("Recovered from panic in DataChannel handler") + } + }() + scopedLogger.Info().Str("label", d.Label()).Uint16("id", *d.ID()).Msg("New DataChannel") + switch d.Label() { + case "hidrpc": + session.HidChannel = d + d.OnMessage(func(msg webrtc.DataChannelMessage) { + l := scopedLogger.With().Int("length", len(msg.Data)).Logger() + // only log data if the log level is debug or lower + if scopedLogger.GetLevel() > zerolog.DebugLevel { + l = l.With().Str("data", string(msg.Data)).Logger() + } + + if msg.IsString { + l.Warn().Msg("received string data in HID RPC message handler") + return + } + + if len(msg.Data) < 1 { + l.Warn().Msg("received empty data in HID RPC message handler") + return + } + + l.Trace().Msg("received data in HID RPC message handler") + + // Enqueue to ensure ordered processing + queueIndex := hidrpc.GetQueueIndex(hidrpc.MessageType(msg.Data[0])) + if queueIndex >= len(session.hidQueue) || queueIndex < 0 { + l.Warn().Int("queueIndex", queueIndex).Msg("received data in HID RPC message handler, but queue index not found") + queueIndex = 3 + } + + queue := session.hidQueue[queueIndex] + if queue != nil { + queue <- msg + } else { + l.Warn().Int("queueIndex", queueIndex).Msg("received data in HID RPC message handler, but queue is nil") + return + } + }) case "rpc": session.RPCChannel = d d.OnMessage(func(msg webrtc.DataChannelMessage) { @@ -198,6 +273,13 @@ func newSession(config SessionConfig) (*Session, error) { close(session.rpcQueue) session.rpcQueue = nil } + + // Stop HID RPC processor + for i := 0; i < len(session.hidQueue); i++ { + close(session.hidQueue[i]) + session.hidQueue[i] = nil + } + if session.shouldUmountVirtualMedia { if err := rpcUnmountImage(); err != nil { scopedLogger.Warn().Err(err).Msg("unmount image failed on connection close")