This commit is contained in:
Aveline 2025-09-17 05:38:04 -05:00 committed by GitHub
commit 050555c001
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
12 changed files with 563 additions and 114 deletions

View File

@ -475,6 +475,10 @@ func handleSessionRequest(
cloudLogger.Info().Interface("session", session).Msg("new session accepted")
cloudLogger.Trace().Interface("session", session).Msg("new session accepted")
// Cancel any ongoing keyboard report multi when session changes
cancelKeyboardMacro()
currentSession = session
_ = wsjson.Write(context.Background(), c, gin.H{"type": "answer", "data": sd})
return nil

View File

@ -1,7 +1,9 @@
package kvm
import (
"errors"
"fmt"
"io"
"time"
"github.com/jetkvm/kvm/internal/hidrpc"
@ -29,6 +31,16 @@ func handleHidRPCMessage(message hidrpc.Message, session *Session) {
session.reportHidRPCKeysDownState(*keysDownState)
}
rpcErr = err
case hidrpc.TypeKeyboardMacroReport:
keyboardMacroReport, err := message.KeyboardMacroReport()
if err != nil {
logger.Warn().Err(err).Msg("failed to get keyboard macro report")
return
}
_, rpcErr = rpcExecuteKeyboardMacro(keyboardMacroReport.Macro)
case hidrpc.TypeCancelKeyboardMacroReport:
rpcCancelKeyboardMacro()
return
case hidrpc.TypePointerReport:
pointerReport, err := message.PointerReport()
if err != nil {
@ -128,6 +140,8 @@ func reportHidRPC(params any, session *Session) {
message, err = hidrpc.NewKeyboardLedMessage(params).Marshal()
case usbgadget.KeysDownState:
message, err = hidrpc.NewKeydownStateMessage(params).Marshal()
case hidrpc.KeyboardMacroStateReport:
message, err = hidrpc.NewKeyboardMacroStateMessage(params.State, params.IsPaste).Marshal()
default:
err = fmt.Errorf("unknown HID RPC message type: %T", params)
}
@ -143,6 +157,10 @@ func reportHidRPC(params any, session *Session) {
}
if err := session.HidChannel.Send(message); err != nil {
if errors.Is(err, io.ErrClosedPipe) {
logger.Debug().Err(err).Msg("HID RPC channel closed, skipping reportHidRPC")
return
}
logger.Warn().Err(err).Msg("failed to send HID RPC message")
}
}
@ -160,3 +178,10 @@ func (s *Session) reportHidRPCKeysDownState(state usbgadget.KeysDownState) {
}
reportHidRPC(state, s)
}
func (s *Session) reportHidRPCKeyboardMacroState(state hidrpc.KeyboardMacroStateReport) {
if !s.hidRPCAvailable {
writeJSONRPCEvent("keyboardMacroState", state, s)
}
reportHidRPC(state, s)
}

View File

@ -10,14 +10,17 @@ import (
type MessageType byte
const (
TypeHandshake MessageType = 0x01
TypeKeyboardReport MessageType = 0x02
TypePointerReport MessageType = 0x03
TypeWheelReport MessageType = 0x04
TypeKeypressReport MessageType = 0x05
TypeMouseReport MessageType = 0x06
TypeKeyboardLedState MessageType = 0x32
TypeKeydownState MessageType = 0x33
TypeHandshake MessageType = 0x01
TypeKeyboardReport MessageType = 0x02
TypePointerReport MessageType = 0x03
TypeWheelReport MessageType = 0x04
TypeKeypressReport MessageType = 0x05
TypeMouseReport MessageType = 0x06
TypeKeyboardMacroReport MessageType = 0x07
TypeCancelKeyboardMacroReport MessageType = 0x08
TypeKeyboardLedState MessageType = 0x32
TypeKeydownState MessageType = 0x33
TypeKeyboardMacroStateReport MessageType = 0x34
)
const (
@ -29,10 +32,13 @@ func GetQueueIndex(messageType MessageType) int {
switch messageType {
case TypeHandshake:
return 0
case TypeKeyboardReport, TypeKeypressReport, TypeKeyboardLedState, TypeKeydownState:
case TypeKeyboardReport, TypeKeypressReport, TypeKeyboardMacroReport, TypeKeyboardLedState, TypeKeydownState, TypeKeyboardMacroStateReport:
return 1
case TypePointerReport, TypeMouseReport, TypeWheelReport:
return 2
// we don't want to block the queue for this message
case TypeCancelKeyboardMacroReport:
return 3
default:
return 3
}
@ -98,3 +104,19 @@ func NewKeydownStateMessage(state usbgadget.KeysDownState) *Message {
d: data,
}
}
// NewKeyboardMacroStateMessage creates a new keyboard macro state message.
func NewKeyboardMacroStateMessage(state bool, isPaste bool) *Message {
data := make([]byte, 2)
if state {
data[0] = 1
}
if isPaste {
data[1] = 1
}
return &Message{
t: TypeKeyboardMacroStateReport,
d: data,
}
}

View File

@ -1,6 +1,7 @@
package hidrpc
import (
"encoding/binary"
"fmt"
)
@ -43,6 +44,11 @@ func (m *Message) String() string {
return fmt.Sprintf("MouseReport{Malformed: %v}", m.d)
}
return fmt.Sprintf("MouseReport{DX: %d, DY: %d, Button: %d}", m.d[0], m.d[1], m.d[2])
case TypeKeyboardMacroReport:
if len(m.d) < 5 {
return fmt.Sprintf("KeyboardMacroReport{Malformed: %v}", m.d)
}
return fmt.Sprintf("KeyboardMacroReport{IsPaste: %v, Length: %d}", m.d[0] == uint8(1), binary.BigEndian.Uint32(m.d[1:5]))
default:
return fmt.Sprintf("Unknown{Type: %d, Data: %v}", m.t, m.d)
}
@ -84,6 +90,51 @@ func (m *Message) KeyboardReport() (KeyboardReport, error) {
}, nil
}
// Macro ..
type KeyboardMacro struct {
Modifier byte // 1 byte
Keys []byte // 6 bytes, to make things easier, the keys length is fixed to 6
Delay uint16 // 2 bytes
}
type KeyboardMacroReport struct {
IsPaste bool
Length uint32
Macro []KeyboardMacro
}
// KeyboardMacroReport returns the keyboard macro report from the message.
func (m *Message) KeyboardMacroReport() (KeyboardMacroReport, error) {
if m.t != TypeKeyboardMacroReport {
return KeyboardMacroReport{}, fmt.Errorf("invalid message type: %d", m.t)
}
isPaste := m.d[0] == uint8(1)
length := binary.BigEndian.Uint32(m.d[1:5])
// check total length
expectedLength := int(length)*9 + 5
if len(m.d) != expectedLength {
return KeyboardMacroReport{}, fmt.Errorf("invalid length: %d, expected: %d", len(m.d), expectedLength)
}
macro := make([]KeyboardMacro, 0, int(length))
for i := 0; i < int(length); i++ {
offset := 5 + i*9
macro = append(macro, KeyboardMacro{
Modifier: m.d[offset],
Keys: m.d[offset+1 : offset+7],
Delay: binary.BigEndian.Uint16(m.d[offset+7 : offset+9]),
})
}
return KeyboardMacroReport{
IsPaste: isPaste,
Macro: macro,
Length: length,
}, nil
}
// PointerReport ..
type PointerReport struct {
X int
@ -131,3 +182,20 @@ func (m *Message) MouseReport() (MouseReport, error) {
Button: uint8(m.d[2]),
}, nil
}
type KeyboardMacroStateReport struct {
State bool
IsPaste bool
}
// KeyboardMacroStateReport returns the keyboard macro state report from the message.
func (m *Message) KeyboardMacroStateReport() (KeyboardMacroStateReport, error) {
if m.t != TypeKeyboardMacroStateReport {
return KeyboardMacroStateReport{}, fmt.Errorf("invalid message type: %d", m.t)
}
return KeyboardMacroStateReport{
State: m.d[0] == uint8(1),
IsPaste: m.d[1] == uint8(1),
}, nil
}

View File

@ -1,6 +1,7 @@
package kvm
import (
"bytes"
"context"
"encoding/json"
"errors"
@ -10,12 +11,14 @@ import (
"path/filepath"
"reflect"
"strconv"
"sync"
"time"
"github.com/pion/webrtc/v4"
"github.com/rs/zerolog"
"go.bug.st/serial"
"github.com/jetkvm/kvm/internal/hidrpc"
"github.com/jetkvm/kvm/internal/usbgadget"
"github.com/jetkvm/kvm/internal/utils"
)
@ -1056,6 +1059,107 @@ func rpcSetLocalLoopbackOnly(enabled bool) error {
return nil
}
var (
keyboardMacroCancel context.CancelFunc
keyboardMacroLock sync.Mutex
)
// cancelKeyboardMacro cancels any ongoing keyboard macro execution
func cancelKeyboardMacro() {
keyboardMacroLock.Lock()
defer keyboardMacroLock.Unlock()
if keyboardMacroCancel != nil {
keyboardMacroCancel()
logger.Info().Msg("canceled keyboard macro")
keyboardMacroCancel = nil
}
}
func setKeyboardMacroCancel(cancel context.CancelFunc) {
keyboardMacroLock.Lock()
defer keyboardMacroLock.Unlock()
keyboardMacroCancel = cancel
}
func rpcExecuteKeyboardMacro(macro []hidrpc.KeyboardMacro) (usbgadget.KeysDownState, error) {
cancelKeyboardMacro()
ctx, cancel := context.WithCancel(context.Background())
setKeyboardMacroCancel(cancel)
s := hidrpc.KeyboardMacroStateReport{
State: true,
IsPaste: true,
}
if currentSession != nil {
currentSession.reportHidRPCKeyboardMacroState(s)
}
result, err := rpcDoExecuteKeyboardMacro(ctx, macro)
setKeyboardMacroCancel(nil)
s.State = false
if currentSession != nil {
currentSession.reportHidRPCKeyboardMacroState(s)
}
return result, err
}
func rpcCancelKeyboardMacro() {
cancelKeyboardMacro()
}
var keyboardClearStateKeys = make([]byte, 6)
func isClearKeyStep(step hidrpc.KeyboardMacro) bool {
return step.Modifier == 0 && bytes.Equal(step.Keys, keyboardClearStateKeys)
}
func rpcDoExecuteKeyboardMacro(ctx context.Context, macro []hidrpc.KeyboardMacro) (usbgadget.KeysDownState, error) {
var last usbgadget.KeysDownState
var err error
logger.Debug().Interface("macro", macro).Msg("Executing keyboard macro")
for i, step := range macro {
delay := time.Duration(step.Delay) * time.Millisecond
last, err = rpcKeyboardReport(step.Modifier, step.Keys)
if err != nil {
logger.Warn().Err(err).Msg("failed to execute keyboard macro")
return last, err
}
// notify the device that the keyboard state is being cleared
if isClearKeyStep(step) {
gadget.UpdateKeysDown(0, keyboardClearStateKeys)
}
// Use context-aware sleep that can be cancelled
select {
case <-time.After(delay):
// Sleep completed normally
case <-ctx.Done():
// make sure keyboard state is reset
_, err := rpcKeyboardReport(0, keyboardClearStateKeys)
if err != nil {
logger.Warn().Err(err).Msg("failed to reset keyboard state")
}
gadget.UpdateKeysDown(0, keyboardClearStateKeys)
logger.Debug().Int("step", i).Msg("Keyboard macro cancelled during sleep")
return last, ctx.Err()
}
}
return last, nil
}
var rpcHandlers = map[string]RPCHandler{
"ping": {Func: rpcPing},
"reboot": {Func: rpcReboot, Params: []string{"force"}},

View File

@ -27,6 +27,7 @@ export default function InfoBar() {
const { rpcDataChannel } = useRTCStore();
const { debugMode, mouseMode, showPressedKeys } = useSettingsStore();
const { isPasteModeEnabled } = useHidStore();
useEffect(() => {
if (!rpcDataChannel) return;
@ -108,7 +109,12 @@ export default function InfoBar() {
<span className="text-xs">{rpcHidStatus}</span>
</div>
)}
{isPasteModeEnabled && (
<div className="flex w-[156px] items-center gap-x-1">
<span className="text-xs font-semibold">Paste Mode:</span>
<span className="text-xs">Enabled</span>
</div>
)}
{showPressedKeys && (
<div className="flex items-center gap-x-1">
<span className="text-xs font-semibold">Keys:</span>

View File

@ -1,42 +1,43 @@
import { useCallback, useEffect, useRef, useState } from "react";
import { LuCornerDownLeft } from "react-icons/lu";
import { ExclamationCircleIcon } from "@heroicons/react/16/solid";
import { useClose } from "@headlessui/react";
import { ExclamationCircleIcon } from "@heroicons/react/16/solid";
import { useCallback, useEffect, useMemo, useRef, useState } from "react";
import { LuCornerDownLeft } from "react-icons/lu";
import { Button } from "@components/Button";
import { GridCard } from "@components/Card";
import { TextAreaWithLabel } from "@components/TextArea";
import { SettingsPageHeader } from "@components/SettingsPageheader";
import { cx } from "@/cva.config";
import { useHidStore, useSettingsStore, useUiStore } from "@/hooks/stores";
import { JsonRpcResponse, useJsonRpc } from "@/hooks/useJsonRpc";
import { useHidStore, useRTCStore, useUiStore, useSettingsStore } from "@/hooks/stores";
import { keys, modifiers } from "@/keyboardMappings";
import { KeyStroke } from "@/keyboardLayouts";
import useKeyboard from "@/hooks/useKeyboard";
import useKeyboardLayout from "@/hooks/useKeyboardLayout";
import notifications from "@/notifications";
const hidKeyboardPayload = (modifier: number, keys: number[]) => {
return { modifier, keys };
};
const modifierCode = (shift?: boolean, altRight?: boolean) => {
return (shift ? modifiers.ShiftLeft : 0)
| (altRight ? modifiers.AltRight : 0)
}
const noModifier = 0
import { Button } from "@components/Button";
import { GridCard } from "@components/Card";
import { InputFieldWithLabel } from "@components/InputField";
import { SettingsPageHeader } from "@components/SettingsPageheader";
import { TextAreaWithLabel } from "@components/TextArea";
export default function PasteModal() {
const TextAreaRef = useRef<HTMLTextAreaElement>(null);
const { setPasteModeEnabled } = useHidStore();
const { isPasteModeEnabled } = useHidStore();
const { setDisableVideoFocusTrap } = useUiStore();
const { send } = useJsonRpc();
const { rpcDataChannel } = useRTCStore();
const { executeMacro, cancelExecuteMacro } = useKeyboard();
const [invalidChars, setInvalidChars] = useState<string[]>([]);
const [delayValue, setDelayValue] = useState(100);
const delay = useMemo(() => {
if (delayValue < 50 || delayValue > 65534) {
return 100;
}
return delayValue;
}, [delayValue]);
const close = useClose();
const debugMode = useSettingsStore(state => state.debugMode);
const delayClassName = useMemo(() => debugMode ? "" : "hidden", [debugMode]);
const { setKeyboardLayout } = useSettingsStore();
const { selectedKeyboard } = useKeyboardLayout();
const { selectedKeyboard } = useKeyboardLayout();
useEffect(() => {
send("getKeyboardLayout", {}, (resp: JsonRpcResponse) => {
@ -46,21 +47,23 @@ export default function PasteModal() {
}, [send, setKeyboardLayout]);
const onCancelPasteMode = useCallback(() => {
setPasteModeEnabled(false);
cancelExecuteMacro();
setDisableVideoFocusTrap(false);
setInvalidChars([]);
}, [setDisableVideoFocusTrap, setPasteModeEnabled]);
}, [setDisableVideoFocusTrap, cancelExecuteMacro]);
const onConfirmPaste = useCallback(async () => {
setPasteModeEnabled(false);
setDisableVideoFocusTrap(false);
if (rpcDataChannel?.readyState !== "open" || !TextAreaRef.current) return;
if (!selectedKeyboard) return;
if (!TextAreaRef.current || !selectedKeyboard) return;
const text = TextAreaRef.current.value;
try {
const macroSteps: {
keys: string[] | null;
modifiers: string[] | null;
delay: number;
}[] = [];
for (const char of text) {
const keyprops = selectedKeyboard.chars[char];
if (!keyprops) continue;
@ -70,39 +73,41 @@ export default function PasteModal() {
// if this is an accented character, we need to send that accent FIRST
if (accentKey) {
await sendKeystroke({modifier: modifierCode(accentKey.shift, accentKey.altRight), keys: [ keys[accentKey.key] ] })
const accentModifiers: string[] = [];
if (accentKey.shift) accentModifiers.push("ShiftLeft");
if (accentKey.altRight) accentModifiers.push("AltRight");
macroSteps.push({
keys: [String(accentKey.key)],
modifiers: accentModifiers.length > 0 ? accentModifiers : null,
delay,
});
}
// now send the actual key
await sendKeystroke({ modifier: modifierCode(shift, altRight), keys: [ keys[key] ]});
const modifiers: string[] = [];
if (shift) modifiers.push("ShiftLeft");
if (altRight) modifiers.push("AltRight");
macroSteps.push({
keys: [String(key)],
modifiers: modifiers.length > 0 ? modifiers : null,
delay
});
// if what was requested was a dead key, we need to send an unmodified space to emit
// just the accent character
if (deadKey) {
await sendKeystroke({ modifier: noModifier, keys: [ keys["Space"] ] });
}
if (deadKey) macroSteps.push({ keys: ["Space"], modifiers: null, delay });
}
// now send a message with no keys down to "release" the keys
await sendKeystroke({ modifier: 0, keys: [] });
if (macroSteps.length > 0) {
await executeMacro(macroSteps);
}
} catch (error) {
console.error("Failed to paste text:", error);
notifications.error("Failed to paste text");
}
async function sendKeystroke(stroke: KeyStroke) {
await new Promise<void>((resolve, reject) => {
send(
"keyboardReport",
hidKeyboardPayload(stroke.modifier, stroke.keys),
params => {
if ("error" in params) return reject(params.error);
resolve();
}
);
});
}
}, [selectedKeyboard, rpcDataChannel?.readyState, send, setDisableVideoFocusTrap, setPasteModeEnabled]);
}, [selectedKeyboard, executeMacro, delay]);
useEffect(() => {
if (TextAreaRef.current) {
@ -122,14 +127,18 @@ export default function PasteModal() {
/>
<div
className="animate-fadeIn opacity-0 space-y-2"
className="animate-fadeIn space-y-2 opacity-0"
style={{
animationDuration: "0.7s",
animationDelay: "0.1s",
}}
>
<div>
<div className="w-full" onKeyUp={e => e.stopPropagation()} onKeyDown={e => e.stopPropagation()}>
<div
className="w-full"
onKeyUp={e => e.stopPropagation()}
onKeyDown={e => e.stopPropagation()}
>
<TextAreaWithLabel
ref={TextAreaRef}
label="Paste from host"
@ -171,9 +180,31 @@ export default function PasteModal() {
)}
</div>
</div>
<div className={cx("text-xs text-slate-600 dark:text-slate-400", delayClassName)}>
<InputFieldWithLabel
type="number"
label="Delay between keys"
placeholder="Delay between keys"
min={50}
max={65534}
value={delayValue}
onChange={e => {
setDelayValue(parseInt(e.target.value, 10));
}}
/>
{delayValue < 50 || delayValue > 65534 && (
<div className="mt-2 flex items-center gap-x-2">
<ExclamationCircleIcon className="h-4 w-4 text-red-500 dark:text-red-400" />
<span className="text-xs text-red-500 dark:text-red-400">
Delay must be between 50 and 65534
</span>
</div>
)}
</div>
<div className="space-y-4">
<p className="text-xs text-slate-600 dark:text-slate-400">
Sending text using keyboard layout: {selectedKeyboard.isoCode}-{selectedKeyboard.name}
Sending text using keyboard layout: {selectedKeyboard.isoCode}-
{selectedKeyboard.name}
</p>
</div>
</div>
@ -181,7 +212,7 @@ export default function PasteModal() {
</div>
</div>
<div
className="flex animate-fadeIn opacity-0 items-center justify-end gap-x-2"
className="flex animate-fadeIn items-center justify-end gap-x-2 opacity-0"
style={{
animationDuration: "0.7s",
animationDelay: "0.2s",
@ -200,6 +231,7 @@ export default function PasteModal() {
size="SM"
theme="primary"
text="Confirm Paste"
disabled={isPasteModeEnabled}
onClick={onConfirmPaste}
LeadingIcon={LuCornerDownLeft}
/>

View File

@ -7,8 +7,11 @@ export const HID_RPC_MESSAGE_TYPES = {
WheelReport: 0x04,
KeypressReport: 0x05,
MouseReport: 0x06,
KeyboardMacroReport: 0x07,
CancelKeyboardMacroReport: 0x08,
KeyboardLedState: 0x32,
KeysDownState: 0x33,
KeyboardMacroStateReport: 0x34,
}
export type HidRpcMessageType = typeof HID_RPC_MESSAGE_TYPES[keyof typeof HID_RPC_MESSAGE_TYPES];
@ -32,6 +35,30 @@ const fromInt32toUint8 = (n: number) => {
]);
};
const fromUint16toUint8 = (n: number) => {
if (n > 65535 || n < 0) {
throw new Error(`Number ${n} is not within the uint16 range`);
}
return new Uint8Array([
(n >> 8) & 0xFF,
(n >> 0) & 0xFF,
]);
};
const fromUint32toUint8 = (n: number) => {
if (n > 4294967295 || n < 0) {
throw new Error(`Number ${n} is not within the uint32 range`);
}
return new Uint8Array([
(n >> 24) & 0xFF,
(n >> 16) & 0xFF,
(n >> 8) & 0xFF,
(n >> 0) & 0xFF,
]);
};
const fromInt8ToUint8 = (n: number) => {
if (n < -128 || n > 127) {
throw new Error(`Number ${n} is not within the int8 range`);
@ -186,6 +213,99 @@ export class KeyboardReportMessage extends RpcMessage {
}
}
export interface KeyboardMacroStep extends KeysDownState {
delay: number;
}
export class KeyboardMacroReportMessage extends RpcMessage {
isPaste: boolean;
length: number;
steps: KeyboardMacroStep[];
KEYS_LENGTH = 6;
constructor(isPaste: boolean, length: number, steps: KeyboardMacroStep[]) {
super(HID_RPC_MESSAGE_TYPES.KeyboardMacroReport);
this.isPaste = isPaste;
this.length = length;
this.steps = steps;
}
marshal(): Uint8Array {
// validate if length is correct
if (this.length !== this.steps.length) {
throw new Error(`Length ${this.length} is not equal to the number of steps ${this.steps.length}`);
}
const data = new Uint8Array(this.length * 9 + 6);
data.set(new Uint8Array([
this.messageType,
this.isPaste ? 1 : 0,
...fromUint32toUint8(this.length),
]), 0);
for (let i = 0; i < this.length; i++) {
const step = this.steps[i];
if (!withinUint8Range(step.modifier)) {
throw new Error(`Modifier ${step.modifier} is not within the uint8 range`);
}
// Ensure the keys are within the KEYS_LENGTH range
const keys = step.keys;
if (keys.length > this.KEYS_LENGTH) {
throw new Error(`Keys ${keys} is not within the hidKeyBufferSize range`);
} else if (keys.length < this.KEYS_LENGTH) {
keys.push(...Array(this.KEYS_LENGTH - keys.length).fill(0));
}
for (const key of keys) {
if (!withinUint8Range(key)) {
throw new Error(`Key ${key} is not within the uint8 range`);
}
}
const macroBinary = new Uint8Array([
step.modifier,
...keys,
...fromUint16toUint8(step.delay),
]);
const offset = 6 + i * 9;
data.set(macroBinary, offset);
}
return data;
}
}
export class KeyboardMacroStateReportMessage extends RpcMessage {
state: boolean;
isPaste: boolean;
constructor(state: boolean, isPaste: boolean) {
super(HID_RPC_MESSAGE_TYPES.KeyboardMacroStateReport);
this.state = state;
this.isPaste = isPaste;
}
marshal(): Uint8Array {
return new Uint8Array([
this.messageType,
this.state ? 1 : 0,
this.isPaste ? 1 : 0,
]);
}
public static unmarshal(data: Uint8Array): KeyboardMacroStateReportMessage | undefined {
if (data.length < 1) {
throw new Error(`Invalid keyboard macro state report message length: ${data.length}`);
}
return new KeyboardMacroStateReportMessage(data[0] === 1, data[1] === 1);
}
}
export class KeyboardLedStateMessage extends RpcMessage {
keyboardLedState: KeyboardLedState;
@ -256,6 +376,17 @@ export class PointerReportMessage extends RpcMessage {
}
}
export class CancelKeyboardMacroReportMessage extends RpcMessage {
constructor() {
super(HID_RPC_MESSAGE_TYPES.CancelKeyboardMacroReport);
}
marshal(): Uint8Array {
return new Uint8Array([this.messageType]);
}
}
export class MouseReportMessage extends RpcMessage {
dx: number;
dy: number;
@ -284,6 +415,9 @@ export const messageRegistry = {
[HID_RPC_MESSAGE_TYPES.KeyboardLedState]: KeyboardLedStateMessage,
[HID_RPC_MESSAGE_TYPES.KeyboardReport]: KeyboardReportMessage,
[HID_RPC_MESSAGE_TYPES.KeypressReport]: KeypressReportMessage,
[HID_RPC_MESSAGE_TYPES.KeyboardMacroReport]: KeyboardMacroReportMessage,
[HID_RPC_MESSAGE_TYPES.CancelKeyboardMacroReport]: CancelKeyboardMacroReportMessage,
[HID_RPC_MESSAGE_TYPES.KeyboardMacroStateReport]: KeyboardMacroStateReportMessage,
}
export const unmarshalHidRpcMessage = (data: Uint8Array): RpcMessage | undefined => {

View File

@ -3,8 +3,11 @@ import { useCallback, useEffect, useMemo } from "react";
import { useRTCStore } from "@/hooks/stores";
import {
CancelKeyboardMacroReportMessage,
HID_RPC_VERSION,
HandshakeMessage,
KeyboardMacroStep,
KeyboardMacroReportMessage,
KeyboardReportMessage,
KeypressReportMessage,
MouseReportMessage,
@ -68,6 +71,21 @@ export function useHidRpc(onHidRpcMessage?: (payload: RpcMessage) => void) {
[sendMessage],
);
const reportKeyboardMacroEvent = useCallback(
(macro: KeyboardMacroStep[]) => {
const d = new KeyboardMacroReportMessage(false, macro.length, macro);
sendMessage(d);
},
[sendMessage],
);
const cancelOngoingKeyboardMacro = useCallback(
() => {
sendMessage(new CancelKeyboardMacroReportMessage());
},
[sendMessage],
);
const sendHandshake = useCallback(() => {
if (rpcHidProtocolVersion) return;
if (!rpcHidChannel) return;
@ -143,6 +161,8 @@ export function useHidRpc(onHidRpcMessage?: (payload: RpcMessage) => void) {
reportKeypressEvent,
reportAbsMouseEvent,
reportRelMouseEvent,
reportKeyboardMacroEvent,
cancelOngoingKeyboardMacro,
rpcHidProtocolVersion,
rpcHidReady,
rpcHidStatus,

View File

@ -1,15 +1,21 @@
import { useCallback } from "react";
import { useCallback, useMemo } from "react";
import { hidErrorRollOver, hidKeyBufferSize, KeysDownState, useHidStore, useRTCStore } from "@/hooks/stores";
import {
hidErrorRollOver,
hidKeyBufferSize,
KeysDownState,
useHidStore,
useRTCStore,
} from "@/hooks/stores";
import { JsonRpcResponse, useJsonRpc } from "@/hooks/useJsonRpc";
import { useHidRpc } from "@/hooks/useHidRpc";
import { KeyboardLedStateMessage, KeysDownStateMessage } from "@/hooks/hidRpc";
import { KeyboardLedStateMessage, KeyboardMacroStateReportMessage, KeyboardMacroStep, KeysDownStateMessage } from "@/hooks/hidRpc";
import { hidKeyToModifierMask, keys, modifiers } from "@/keyboardMappings";
export default function useKeyboard() {
const { send } = useJsonRpc();
const { rpcDataChannel } = useRTCStore();
const { keysDownState, setKeysDownState, setKeyboardLedState } = useHidStore();
const { keysDownState, setKeysDownState, setKeyboardLedState, setPasteModeEnabled } = useHidStore();
// INTRODUCTION: The earlier version of the JetKVM device shipped with all keyboard state
// being tracked on the browser/client-side. When adding the keyPressReport API to the
@ -26,8 +32,10 @@ export default function useKeyboard() {
const {
reportKeyboardEvent: sendKeyboardEventHidRpc,
reportKeypressEvent: sendKeypressEventHidRpc,
reportKeyboardMacroEvent: sendKeyboardMacroEventHidRpc,
cancelOngoingKeyboardMacro: cancelOngoingKeyboardMacroHidRpc,
rpcHidReady,
} = useHidRpc((message) => {
} = useHidRpc(message => {
switch (message.constructor) {
case KeysDownStateMessage:
setKeysDownState((message as KeysDownStateMessage).keysDownState);
@ -35,6 +43,10 @@ export default function useKeyboard() {
case KeyboardLedStateMessage:
setKeyboardLedState((message as KeyboardLedStateMessage).keyboardLedState);
break;
case KeyboardMacroStateReportMessage:
if (!(message as KeyboardMacroStateReportMessage).isPaste) break;
setPasteModeEnabled((message as KeyboardMacroStateReportMessage).state);
break;
default:
break;
}
@ -48,7 +60,9 @@ export default function useKeyboard() {
async (state: KeysDownState) => {
if (rpcDataChannel?.readyState !== "open" && !rpcHidReady) return;
console.debug(`Send keyboardReport keys: ${state.keys}, modifier: ${state.modifier}`);
console.debug(
`Send keyboardReport keys: ${state.keys}, modifier: ${state.modifier}`,
);
if (rpcHidReady) {
console.debug("Sending keyboard report via HidRPC");
@ -56,31 +70,32 @@ export default function useKeyboard() {
return;
}
send("keyboardReport", { keys: state.keys, modifier: state.modifier }, (resp: JsonRpcResponse) => {
if ("error" in resp) {
console.error(`Failed to send keyboard report ${state}`, resp.error);
}
});
send(
"keyboardReport",
{ keys: state.keys, modifier: state.modifier },
(resp: JsonRpcResponse) => {
if ("error" in resp) {
console.error(`Failed to send keyboard report ${state}`, resp.error);
}
},
);
},
[
rpcDataChannel?.readyState,
rpcHidReady,
send,
sendKeyboardEventHidRpc,
],
[rpcDataChannel?.readyState, rpcHidReady, send, sendKeyboardEventHidRpc],
);
const MACRO_RESET_KEYBOARD_STATE = useMemo(() => ({
keys: new Array(hidKeyBufferSize).fill(0),
modifier: 0,
delay: 0,
}), []);
// resetKeyboardState is used to reset the keyboard state to no keys pressed and no modifiers.
// This is useful for macros and when the browser loses focus to ensure that the keyboard state
// is clean.
const resetKeyboardState = useCallback(
async () => {
// Reset the keys buffer to zeros and the modifier state to zero
keysDownState.keys.length = hidKeyBufferSize;
keysDownState.keys.fill(0);
keysDownState.modifier = 0;
sendKeyboardEvent(keysDownState);
}, [keysDownState, sendKeyboardEvent]);
const resetKeyboardState = useCallback(async () => {
// Reset the keys buffer to zeros and the modifier state to zero
sendKeyboardEvent(MACRO_RESET_KEYBOARD_STATE);
}, [sendKeyboardEvent, MACRO_RESET_KEYBOARD_STATE]);
// executeMacro is used to execute a macro consisting of multiple steps.
// Each step can have multiple keys, multiple modifiers and a delay.
@ -88,29 +103,32 @@ export default function useKeyboard() {
// After the delay, the keys and modifiers are released and the next step is executed.
// If a step has no keys or modifiers, it is treated as a delay-only step.
// A small pause is added between steps to ensure that the device can process the events.
const executeMacro = async (steps: { keys: string[] | null; modifiers: string[] | null; delay: number }[]) => {
for (const [index, step] of steps.entries()) {
const executeMacro = async (
steps: { keys: string[] | null; modifiers: string[] | null; delay: number }[],
) => {
const macro: KeyboardMacroStep[] = [];
for (const [_, step] of steps.entries()) {
const keyValues = (step.keys || []).map(key => keys[key]).filter(Boolean);
const modifierMask: number = (step.modifiers || []).map(mod => modifiers[mod]).reduce((acc, val) => acc + val, 0);
const modifierMask: number = (step.modifiers || [])
.map(mod => modifiers[mod])
.reduce((acc, val) => acc + val, 0);
// If the step has keys and/or modifiers, press them and hold for the delay
if (keyValues.length > 0 || modifierMask > 0) {
sendKeyboardEvent({ keys: keyValues, modifier: modifierMask });
await new Promise(resolve => setTimeout(resolve, step.delay || 50));
resetKeyboardState();
} else {
// This is a delay-only step, just wait for the delay amount
await new Promise(resolve => setTimeout(resolve, step.delay || 50));
}
// Add a small pause between steps if not the last step
if (index < steps.length - 1) {
await new Promise(resolve => setTimeout(resolve, 10));
macro.push({ keys: keyValues, modifier: modifierMask, delay: 20 });
macro.push({ ...MACRO_RESET_KEYBOARD_STATE, delay: step.delay || 100 });
}
}
sendKeyboardMacroEventHidRpc(macro);
};
const cancelExecuteMacro = useCallback(async () => {
if (!rpcHidReady) return;
cancelOngoingKeyboardMacroHidRpc();
}, [rpcHidReady, cancelOngoingKeyboardMacroHidRpc]);
// handleKeyPress is used to handle a key press or release event.
// This function handle both key press and key release events.
// It checks if the keyPressReport API is available and sends the key press event.
@ -132,7 +150,11 @@ export default function useKeyboard() {
sendKeypressEventHidRpc(key, press);
} else {
// if the keyPress api is not available, we need to handle the key locally
const downState = simulateDeviceSideKeyHandlingForLegacyDevices(keysDownState, key, press);
const downState = simulateDeviceSideKeyHandlingForLegacyDevices(
keysDownState,
key,
press,
);
sendKeyboardEvent(downState); // then we send the full state
// if we just sent ErrorRollOver, reset to empty state
@ -152,7 +174,11 @@ export default function useKeyboard() {
);
// IMPORTANT: See the keyPressReportApiAvailable comment above for the reason this exists
function simulateDeviceSideKeyHandlingForLegacyDevices(state: KeysDownState, key: number, press: boolean): KeysDownState {
function simulateDeviceSideKeyHandlingForLegacyDevices(
state: KeysDownState,
key: number,
press: boolean,
): KeysDownState {
// IMPORTANT: This code parallels the logic in the kernel's hid-gadget driver
// for handling key presses and releases. It ensures that the USB gadget
// behaves similarly to a real USB HID keyboard. This logic is paralleled
@ -181,7 +207,7 @@ export default function useKeyboard() {
// and if we find a zero byte, we can place the key there (if press is true)
if (keys[i] === key || keys[i] === 0) {
if (press) {
keys[i] = key // overwrites the zero byte or the same key if already pressed
keys[i] = key; // overwrites the zero byte or the same key if already pressed
} else {
// we are releasing the key, remove it from the buffer
if (keys[i] !== 0) {
@ -197,18 +223,20 @@ export default function useKeyboard() {
// If we reach here it means we didn't find an empty slot or the key in the buffer
if (overrun) {
if (press) {
console.warn(`keyboard buffer overflow current keys ${keys}, key: ${key} not added`);
console.warn(
`keyboard buffer overflow current keys ${keys}, key: ${key} not added`,
);
// Fill all key slots with ErrorRollOver (0x01) to indicate overflow
keys.length = hidKeyBufferSize;
keys.fill(hidErrorRollOver);
} else {
// If we are releasing a key, and we didn't find it in a slot, who cares?
console.debug(`key ${key} not found in buffer, nothing to release`)
console.debug(`key ${key} not found in buffer, nothing to release`);
}
}
}
return { modifier: modifiers, keys };
}
return { handleKeyPress, resetKeyboardState, executeMacro };
return { handleKeyPress, resetKeyboardState, executeMacro, cancelExecuteMacro };
}

4
web.go
View File

@ -228,6 +228,10 @@ func handleWebRTCSession(c *gin.Context) {
_ = peerConn.Close()
}()
}
// Cancel any ongoing keyboard report multi when session changes
cancelKeyboardMacro()
currentSession = session
c.JSON(http.StatusOK, gin.H{"sd": sd})
}

View File

@ -266,6 +266,8 @@ func newSession(config SessionConfig) (*Session, error) {
if connectionState == webrtc.ICEConnectionStateClosed {
scopedLogger.Debug().Msg("ICE Connection State is closed, unmounting virtual media")
if session == currentSession {
// Cancel any ongoing keyboard report multi when session closes
cancelKeyboardMacro()
currentSession = nil
}
// Stop RPC processor