mirror of https://github.com/jetkvm/kvm.git
feat: send all paste keystrokes to backend (#789)
* feat: send all paste keystrokes to backend * feat: cancel paste mode * wip: send macro using hidRPC channel * add delay * feat: allow paste progress to be cancelled * allow user to override delay * chore: clear keysDownState * fix: use currentSession.reportHidRPCKeyboardMacroState * fix: jsonrpc.go:1142:21: Error return value is not checked (errcheck) * fix: performance issue of Uint8Array concat * chore: hide delay option when debugMode isn't enabled * feat: use clientSide macro if backend doesn't support macros * fix: update keysDownState handling * minor issues * refactor * fix: send duplicated keyDownState * chore: add max length for paste text --------- Co-authored-by: Adam Shiervani <adam.shiervani@gmail.com>
This commit is contained in:
Aveline
GitHub
parent
25b102ac34
commit
72e3013337
4
cloud.go
4
cloud.go
|
|
@ -475,6 +475,10 @@ func handleSessionRequest(
|
|||
|
||||
cloudLogger.Info().Interface("session", session).Msg("new session accepted")
|
||||
cloudLogger.Trace().Interface("session", session).Msg("new session accepted")
|
||||
|
||||
// Cancel any ongoing keyboard macro when session changes
|
||||
cancelKeyboardMacro()
|
||||
|
||||
currentSession = session
|
||||
_ = wsjson.Write(context.Background(), c, gin.H{"type": "answer", "data": sd})
|
||||
return nil
|
||||
|
|
|
|||
25
hidrpc.go
25
hidrpc.go
|
|
@ -1,7 +1,9 @@
|
|||
package kvm
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"time"
|
||||
|
||||
"github.com/jetkvm/kvm/internal/hidrpc"
|
||||
|
|
@ -29,6 +31,16 @@ func handleHidRPCMessage(message hidrpc.Message, session *Session) {
|
|||
session.reportHidRPCKeysDownState(*keysDownState)
|
||||
}
|
||||
rpcErr = err
|
||||
case hidrpc.TypeKeyboardMacroReport:
|
||||
keyboardMacroReport, err := message.KeyboardMacroReport()
|
||||
if err != nil {
|
||||
logger.Warn().Err(err).Msg("failed to get keyboard macro report")
|
||||
return
|
||||
}
|
||||
_, rpcErr = rpcExecuteKeyboardMacro(keyboardMacroReport.Steps)
|
||||
case hidrpc.TypeCancelKeyboardMacroReport:
|
||||
rpcCancelKeyboardMacro()
|
||||
return
|
||||
case hidrpc.TypePointerReport:
|
||||
pointerReport, err := message.PointerReport()
|
||||
if err != nil {
|
||||
|
|
@ -128,6 +140,8 @@ func reportHidRPC(params any, session *Session) {
|
|||
message, err = hidrpc.NewKeyboardLedMessage(params).Marshal()
|
||||
case usbgadget.KeysDownState:
|
||||
message, err = hidrpc.NewKeydownStateMessage(params).Marshal()
|
||||
case hidrpc.KeyboardMacroState:
|
||||
message, err = hidrpc.NewKeyboardMacroStateMessage(params.State, params.IsPaste).Marshal()
|
||||
default:
|
||||
err = fmt.Errorf("unknown HID RPC message type: %T", params)
|
||||
}
|
||||
|
|
@ -143,6 +157,10 @@ func reportHidRPC(params any, session *Session) {
|
|||
}
|
||||
|
||||
if err := session.HidChannel.Send(message); err != nil {
|
||||
if errors.Is(err, io.ErrClosedPipe) {
|
||||
logger.Debug().Err(err).Msg("HID RPC channel closed, skipping reportHidRPC")
|
||||
return
|
||||
}
|
||||
logger.Warn().Err(err).Msg("failed to send HID RPC message")
|
||||
}
|
||||
}
|
||||
|
|
@ -160,3 +178,10 @@ func (s *Session) reportHidRPCKeysDownState(state usbgadget.KeysDownState) {
|
|||
}
|
||||
reportHidRPC(state, s)
|
||||
}
|
||||
|
||||
func (s *Session) reportHidRPCKeyboardMacroState(state hidrpc.KeyboardMacroState) {
|
||||
if !s.hidRPCAvailable {
|
||||
writeJSONRPCEvent("keyboardMacroState", state, s)
|
||||
}
|
||||
reportHidRPC(state, s)
|
||||
}
|
||||
|
|
|
|||
|
|
@ -10,14 +10,17 @@ import (
|
|||
type MessageType byte
|
||||
|
||||
const (
|
||||
TypeHandshake MessageType = 0x01
|
||||
TypeKeyboardReport MessageType = 0x02
|
||||
TypePointerReport MessageType = 0x03
|
||||
TypeWheelReport MessageType = 0x04
|
||||
TypeKeypressReport MessageType = 0x05
|
||||
TypeMouseReport MessageType = 0x06
|
||||
TypeKeyboardLedState MessageType = 0x32
|
||||
TypeKeydownState MessageType = 0x33
|
||||
TypeHandshake MessageType = 0x01
|
||||
TypeKeyboardReport MessageType = 0x02
|
||||
TypePointerReport MessageType = 0x03
|
||||
TypeWheelReport MessageType = 0x04
|
||||
TypeKeypressReport MessageType = 0x05
|
||||
TypeMouseReport MessageType = 0x06
|
||||
TypeKeyboardMacroReport MessageType = 0x07
|
||||
TypeCancelKeyboardMacroReport MessageType = 0x08
|
||||
TypeKeyboardLedState MessageType = 0x32
|
||||
TypeKeydownState MessageType = 0x33
|
||||
TypeKeyboardMacroState MessageType = 0x34
|
||||
)
|
||||
|
||||
const (
|
||||
|
|
@ -29,10 +32,13 @@ func GetQueueIndex(messageType MessageType) int {
|
|||
switch messageType {
|
||||
case TypeHandshake:
|
||||
return 0
|
||||
case TypeKeyboardReport, TypeKeypressReport, TypeKeyboardLedState, TypeKeydownState:
|
||||
case TypeKeyboardReport, TypeKeypressReport, TypeKeyboardMacroReport, TypeKeyboardLedState, TypeKeydownState, TypeKeyboardMacroState:
|
||||
return 1
|
||||
case TypePointerReport, TypeMouseReport, TypeWheelReport:
|
||||
return 2
|
||||
// we don't want to block the queue for this message
|
||||
case TypeCancelKeyboardMacroReport:
|
||||
return 3
|
||||
default:
|
||||
return 3
|
||||
}
|
||||
|
|
@ -98,3 +104,19 @@ func NewKeydownStateMessage(state usbgadget.KeysDownState) *Message {
|
|||
d: data,
|
||||
}
|
||||
}
|
||||
|
||||
// NewKeyboardMacroStateMessage creates a new keyboard macro state message.
|
||||
func NewKeyboardMacroStateMessage(state bool, isPaste bool) *Message {
|
||||
data := make([]byte, 2)
|
||||
if state {
|
||||
data[0] = 1
|
||||
}
|
||||
if isPaste {
|
||||
data[1] = 1
|
||||
}
|
||||
|
||||
return &Message{
|
||||
t: TypeKeyboardMacroState,
|
||||
d: data,
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,6 +1,7 @@
|
|||
package hidrpc
|
||||
|
||||
import (
|
||||
"encoding/binary"
|
||||
"fmt"
|
||||
)
|
||||
|
||||
|
|
@ -43,6 +44,11 @@ func (m *Message) String() string {
|
|||
return fmt.Sprintf("MouseReport{Malformed: %v}", m.d)
|
||||
}
|
||||
return fmt.Sprintf("MouseReport{DX: %d, DY: %d, Button: %d}", m.d[0], m.d[1], m.d[2])
|
||||
case TypeKeyboardMacroReport:
|
||||
if len(m.d) < 5 {
|
||||
return fmt.Sprintf("KeyboardMacroReport{Malformed: %v}", m.d)
|
||||
}
|
||||
return fmt.Sprintf("KeyboardMacroReport{IsPaste: %v, Length: %d}", m.d[0] == uint8(1), binary.BigEndian.Uint32(m.d[1:5]))
|
||||
default:
|
||||
return fmt.Sprintf("Unknown{Type: %d, Data: %v}", m.t, m.d)
|
||||
}
|
||||
|
|
@ -84,6 +90,55 @@ func (m *Message) KeyboardReport() (KeyboardReport, error) {
|
|||
}, nil
|
||||
}
|
||||
|
||||
// Macro ..
|
||||
type KeyboardMacroStep struct {
|
||||
Modifier byte // 1 byte
|
||||
Keys []byte // 6 bytes: hidKeyBufferSize
|
||||
Delay uint16 // 2 bytes
|
||||
}
|
||||
type KeyboardMacroReport struct {
|
||||
IsPaste bool
|
||||
StepCount uint32
|
||||
Steps []KeyboardMacroStep
|
||||
}
|
||||
|
||||
// HidKeyBufferSize is the size of the keys buffer in the keyboard report.
|
||||
const HidKeyBufferSize = 6
|
||||
|
||||
// KeyboardMacroReport returns the keyboard macro report from the message.
|
||||
func (m *Message) KeyboardMacroReport() (KeyboardMacroReport, error) {
|
||||
if m.t != TypeKeyboardMacroReport {
|
||||
return KeyboardMacroReport{}, fmt.Errorf("invalid message type: %d", m.t)
|
||||
}
|
||||
|
||||
isPaste := m.d[0] == uint8(1)
|
||||
stepCount := binary.BigEndian.Uint32(m.d[1:5])
|
||||
|
||||
// check total length
|
||||
expectedLength := int(stepCount)*9 + 5
|
||||
if len(m.d) != expectedLength {
|
||||
return KeyboardMacroReport{}, fmt.Errorf("invalid length: %d, expected: %d", len(m.d), expectedLength)
|
||||
}
|
||||
|
||||
steps := make([]KeyboardMacroStep, 0, int(stepCount))
|
||||
offset := 5
|
||||
for i := 0; i < int(stepCount); i++ {
|
||||
steps = append(steps, KeyboardMacroStep{
|
||||
Modifier: m.d[offset],
|
||||
Keys: m.d[offset+1 : offset+7],
|
||||
Delay: binary.BigEndian.Uint16(m.d[offset+7 : offset+9]),
|
||||
})
|
||||
|
||||
offset += 1 + HidKeyBufferSize + 2
|
||||
}
|
||||
|
||||
return KeyboardMacroReport{
|
||||
IsPaste: isPaste,
|
||||
Steps: steps,
|
||||
StepCount: stepCount,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// PointerReport ..
|
||||
type PointerReport struct {
|
||||
X int
|
||||
|
|
@ -131,3 +186,20 @@ func (m *Message) MouseReport() (MouseReport, error) {
|
|||
Button: uint8(m.d[2]),
|
||||
}, nil
|
||||
}
|
||||
|
||||
type KeyboardMacroState struct {
|
||||
State bool
|
||||
IsPaste bool
|
||||
}
|
||||
|
||||
// KeyboardMacroState returns the keyboard macro state report from the message.
|
||||
func (m *Message) KeyboardMacroState() (KeyboardMacroState, error) {
|
||||
if m.t != TypeKeyboardMacroState {
|
||||
return KeyboardMacroState{}, fmt.Errorf("invalid message type: %d", m.t)
|
||||
}
|
||||
|
||||
return KeyboardMacroState{
|
||||
State: m.d[0] == uint8(1),
|
||||
IsPaste: m.d[1] == uint8(1),
|
||||
}, nil
|
||||
}
|
||||
|
|
|
|||
103
jsonrpc.go
103
jsonrpc.go
|
|
@ -1,6 +1,7 @@
|
|||
package kvm
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
|
|
@ -10,12 +11,14 @@ import (
|
|||
"path/filepath"
|
||||
"reflect"
|
||||
"strconv"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/pion/webrtc/v4"
|
||||
"github.com/rs/zerolog"
|
||||
"go.bug.st/serial"
|
||||
|
||||
"github.com/jetkvm/kvm/internal/hidrpc"
|
||||
"github.com/jetkvm/kvm/internal/usbgadget"
|
||||
"github.com/jetkvm/kvm/internal/utils"
|
||||
)
|
||||
|
|
@ -1056,6 +1059,106 @@ func rpcSetLocalLoopbackOnly(enabled bool) error {
|
|||
return nil
|
||||
}
|
||||
|
||||
var (
|
||||
keyboardMacroCancel context.CancelFunc
|
||||
keyboardMacroLock sync.Mutex
|
||||
)
|
||||
|
||||
// cancelKeyboardMacro cancels any ongoing keyboard macro execution
|
||||
func cancelKeyboardMacro() {
|
||||
keyboardMacroLock.Lock()
|
||||
defer keyboardMacroLock.Unlock()
|
||||
|
||||
if keyboardMacroCancel != nil {
|
||||
keyboardMacroCancel()
|
||||
logger.Info().Msg("canceled keyboard macro")
|
||||
keyboardMacroCancel = nil
|
||||
}
|
||||
}
|
||||
|
||||
func setKeyboardMacroCancel(cancel context.CancelFunc) {
|
||||
keyboardMacroLock.Lock()
|
||||
defer keyboardMacroLock.Unlock()
|
||||
|
||||
keyboardMacroCancel = cancel
|
||||
}
|
||||
|
||||
func rpcExecuteKeyboardMacro(macro []hidrpc.KeyboardMacroStep) (usbgadget.KeysDownState, error) {
|
||||
cancelKeyboardMacro()
|
||||
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
setKeyboardMacroCancel(cancel)
|
||||
|
||||
s := hidrpc.KeyboardMacroState{
|
||||
State: true,
|
||||
IsPaste: true,
|
||||
}
|
||||
|
||||
if currentSession != nil {
|
||||
currentSession.reportHidRPCKeyboardMacroState(s)
|
||||
}
|
||||
|
||||
result, err := rpcDoExecuteKeyboardMacro(ctx, macro)
|
||||
|
||||
setKeyboardMacroCancel(nil)
|
||||
|
||||
s.State = false
|
||||
if currentSession != nil {
|
||||
currentSession.reportHidRPCKeyboardMacroState(s)
|
||||
}
|
||||
|
||||
return result, err
|
||||
}
|
||||
|
||||
func rpcCancelKeyboardMacro() {
|
||||
cancelKeyboardMacro()
|
||||
}
|
||||
|
||||
var keyboardClearStateKeys = make([]byte, hidrpc.HidKeyBufferSize)
|
||||
|
||||
func isClearKeyStep(step hidrpc.KeyboardMacroStep) bool {
|
||||
return step.Modifier == 0 && bytes.Equal(step.Keys, keyboardClearStateKeys)
|
||||
}
|
||||
|
||||
func rpcDoExecuteKeyboardMacro(ctx context.Context, macro []hidrpc.KeyboardMacroStep) (usbgadget.KeysDownState, error) {
|
||||
var last usbgadget.KeysDownState
|
||||
var err error
|
||||
|
||||
logger.Debug().Interface("macro", macro).Msg("Executing keyboard macro")
|
||||
|
||||
for i, step := range macro {
|
||||
delay := time.Duration(step.Delay) * time.Millisecond
|
||||
|
||||
last, err = rpcKeyboardReport(step.Modifier, step.Keys)
|
||||
if err != nil {
|
||||
logger.Warn().Err(err).Msg("failed to execute keyboard macro")
|
||||
return last, err
|
||||
}
|
||||
|
||||
// notify the device that the keyboard state is being cleared
|
||||
if isClearKeyStep(step) {
|
||||
gadget.UpdateKeysDown(0, keyboardClearStateKeys)
|
||||
}
|
||||
|
||||
// Use context-aware sleep that can be cancelled
|
||||
select {
|
||||
case <-time.After(delay):
|
||||
// Sleep completed normally
|
||||
case <-ctx.Done():
|
||||
// make sure keyboard state is reset
|
||||
_, err := rpcKeyboardReport(0, keyboardClearStateKeys)
|
||||
if err != nil {
|
||||
logger.Warn().Err(err).Msg("failed to reset keyboard state")
|
||||
}
|
||||
|
||||
logger.Debug().Int("step", i).Msg("Keyboard macro cancelled during sleep")
|
||||
return last, ctx.Err()
|
||||
}
|
||||
}
|
||||
|
||||
return last, nil
|
||||
}
|
||||
|
||||
var rpcHandlers = map[string]RPCHandler{
|
||||
"ping": {Func: rpcPing},
|
||||
"reboot": {Func: rpcReboot, Params: []string{"force"}},
|
||||
|
|
|
|||
|
|
@ -27,6 +27,7 @@ export default function InfoBar() {
|
|||
|
||||
const { rpcDataChannel } = useRTCStore();
|
||||
const { debugMode, mouseMode, showPressedKeys } = useSettingsStore();
|
||||
const { isPasteInProgress } = useHidStore();
|
||||
|
||||
useEffect(() => {
|
||||
if (!rpcDataChannel) return;
|
||||
|
|
@ -108,7 +109,12 @@ export default function InfoBar() {
|
|||
<span className="text-xs">{rpcHidStatus}</span>
|
||||
</div>
|
||||
)}
|
||||
|
||||
{isPasteInProgress && (
|
||||
<div className="flex w-[156px] items-center gap-x-1">
|
||||
<span className="text-xs font-semibold">Paste Mode:</span>
|
||||
<span className="text-xs">Enabled</span>
|
||||
</div>
|
||||
)}
|
||||
{showPressedKeys && (
|
||||
<div className="flex items-center gap-x-1">
|
||||
<span className="text-xs font-semibold">Keys:</span>
|
||||
|
|
|
|||
|
|
@ -1,42 +1,46 @@
|
|||
import { useCallback, useEffect, useRef, useState } from "react";
|
||||
import { LuCornerDownLeft } from "react-icons/lu";
|
||||
import { ExclamationCircleIcon } from "@heroicons/react/16/solid";
|
||||
import { useClose } from "@headlessui/react";
|
||||
import { ExclamationCircleIcon } from "@heroicons/react/16/solid";
|
||||
import { useCallback, useEffect, useMemo, useRef, useState } from "react";
|
||||
import { LuCornerDownLeft } from "react-icons/lu";
|
||||
|
||||
import { Button } from "@components/Button";
|
||||
import { GridCard } from "@components/Card";
|
||||
import { TextAreaWithLabel } from "@components/TextArea";
|
||||
import { SettingsPageHeader } from "@components/SettingsPageheader";
|
||||
import { cx } from "@/cva.config";
|
||||
import { useHidStore, useSettingsStore, useUiStore } from "@/hooks/stores";
|
||||
import { JsonRpcResponse, useJsonRpc } from "@/hooks/useJsonRpc";
|
||||
import { useHidStore, useRTCStore, useUiStore, useSettingsStore } from "@/hooks/stores";
|
||||
import { keys, modifiers } from "@/keyboardMappings";
|
||||
import { KeyStroke } from "@/keyboardLayouts";
|
||||
import useKeyboard, { type MacroStep } from "@/hooks/useKeyboard";
|
||||
import useKeyboardLayout from "@/hooks/useKeyboardLayout";
|
||||
import notifications from "@/notifications";
|
||||
import { Button } from "@components/Button";
|
||||
import { GridCard } from "@components/Card";
|
||||
import { InputFieldWithLabel } from "@components/InputField";
|
||||
import { SettingsPageHeader } from "@components/SettingsPageheader";
|
||||
import { TextAreaWithLabel } from "@components/TextArea";
|
||||
|
||||
const hidKeyboardPayload = (modifier: number, keys: number[]) => {
|
||||
return { modifier, keys };
|
||||
};
|
||||
|
||||
const modifierCode = (shift?: boolean, altRight?: boolean) => {
|
||||
return (shift ? modifiers.ShiftLeft : 0)
|
||||
| (altRight ? modifiers.AltRight : 0)
|
||||
}
|
||||
const noModifier = 0
|
||||
// uint32 max value / 4
|
||||
const pasteMaxLength = 1073741824;
|
||||
|
||||
export default function PasteModal() {
|
||||
const TextAreaRef = useRef<HTMLTextAreaElement>(null);
|
||||
const { setPasteModeEnabled } = useHidStore();
|
||||
const { isPasteInProgress } = useHidStore();
|
||||
const { setDisableVideoFocusTrap } = useUiStore();
|
||||
|
||||
const { send } = useJsonRpc();
|
||||
const { rpcDataChannel } = useRTCStore();
|
||||
const { executeMacro, cancelExecuteMacro } = useKeyboard();
|
||||
|
||||
const [invalidChars, setInvalidChars] = useState<string[]>([]);
|
||||
const [delayValue, setDelayValue] = useState(100);
|
||||
const delay = useMemo(() => {
|
||||
if (delayValue < 50 || delayValue > 65534) {
|
||||
return 100;
|
||||
}
|
||||
return delayValue;
|
||||
}, [delayValue]);
|
||||
const close = useClose();
|
||||
|
||||
const debugMode = useSettingsStore(state => state.debugMode);
|
||||
const delayClassName = useMemo(() => debugMode ? "" : "hidden", [debugMode]);
|
||||
|
||||
const { setKeyboardLayout } = useSettingsStore();
|
||||
const { selectedKeyboard } = useKeyboardLayout();
|
||||
const { selectedKeyboard } = useKeyboardLayout();
|
||||
|
||||
useEffect(() => {
|
||||
send("getKeyboardLayout", {}, (resp: JsonRpcResponse) => {
|
||||
|
|
@ -46,21 +50,19 @@ export default function PasteModal() {
|
|||
}, [send, setKeyboardLayout]);
|
||||
|
||||
const onCancelPasteMode = useCallback(() => {
|
||||
setPasteModeEnabled(false);
|
||||
cancelExecuteMacro();
|
||||
setDisableVideoFocusTrap(false);
|
||||
setInvalidChars([]);
|
||||
}, [setDisableVideoFocusTrap, setPasteModeEnabled]);
|
||||
}, [setDisableVideoFocusTrap, cancelExecuteMacro]);
|
||||
|
||||
const onConfirmPaste = useCallback(async () => {
|
||||
setPasteModeEnabled(false);
|
||||
setDisableVideoFocusTrap(false);
|
||||
|
||||
if (rpcDataChannel?.readyState !== "open" || !TextAreaRef.current) return;
|
||||
if (!selectedKeyboard) return;
|
||||
if (!TextAreaRef.current || !selectedKeyboard) return;
|
||||
|
||||
const text = TextAreaRef.current.value;
|
||||
|
||||
try {
|
||||
const macroSteps: MacroStep[] = [];
|
||||
|
||||
for (const char of text) {
|
||||
const keyprops = selectedKeyboard.chars[char];
|
||||
if (!keyprops) continue;
|
||||
|
|
@ -70,39 +72,41 @@ export default function PasteModal() {
|
|||
|
||||
// if this is an accented character, we need to send that accent FIRST
|
||||
if (accentKey) {
|
||||
await sendKeystroke({modifier: modifierCode(accentKey.shift, accentKey.altRight), keys: [ keys[accentKey.key] ] })
|
||||
const accentModifiers: string[] = [];
|
||||
if (accentKey.shift) accentModifiers.push("ShiftLeft");
|
||||
if (accentKey.altRight) accentModifiers.push("AltRight");
|
||||
|
||||
macroSteps.push({
|
||||
keys: [String(accentKey.key)],
|
||||
modifiers: accentModifiers.length > 0 ? accentModifiers : null,
|
||||
delay,
|
||||
});
|
||||
}
|
||||
|
||||
// now send the actual key
|
||||
await sendKeystroke({ modifier: modifierCode(shift, altRight), keys: [ keys[key] ]});
|
||||
const modifiers: string[] = [];
|
||||
if (shift) modifiers.push("ShiftLeft");
|
||||
if (altRight) modifiers.push("AltRight");
|
||||
|
||||
macroSteps.push({
|
||||
keys: [String(key)],
|
||||
modifiers: modifiers.length > 0 ? modifiers : null,
|
||||
delay
|
||||
});
|
||||
|
||||
// if what was requested was a dead key, we need to send an unmodified space to emit
|
||||
// just the accent character
|
||||
if (deadKey) {
|
||||
await sendKeystroke({ modifier: noModifier, keys: [ keys["Space"] ] });
|
||||
}
|
||||
if (deadKey) macroSteps.push({ keys: ["Space"], modifiers: null, delay });
|
||||
}
|
||||
|
||||
// now send a message with no keys down to "release" the keys
|
||||
await sendKeystroke({ modifier: 0, keys: [] });
|
||||
if (macroSteps.length > 0) {
|
||||
await executeMacro(macroSteps);
|
||||
}
|
||||
} catch (error) {
|
||||
console.error("Failed to paste text:", error);
|
||||
notifications.error("Failed to paste text");
|
||||
}
|
||||
|
||||
async function sendKeystroke(stroke: KeyStroke) {
|
||||
await new Promise<void>((resolve, reject) => {
|
||||
send(
|
||||
"keyboardReport",
|
||||
hidKeyboardPayload(stroke.modifier, stroke.keys),
|
||||
params => {
|
||||
if ("error" in params) return reject(params.error);
|
||||
resolve();
|
||||
}
|
||||
);
|
||||
});
|
||||
}
|
||||
}, [selectedKeyboard, rpcDataChannel?.readyState, send, setDisableVideoFocusTrap, setPasteModeEnabled]);
|
||||
}, [selectedKeyboard, executeMacro, delay]);
|
||||
|
||||
useEffect(() => {
|
||||
if (TextAreaRef.current) {
|
||||
|
|
@ -122,19 +126,25 @@ export default function PasteModal() {
|
|||
/>
|
||||
|
||||
<div
|
||||
className="animate-fadeIn opacity-0 space-y-2"
|
||||
className="animate-fadeIn space-y-2 opacity-0"
|
||||
style={{
|
||||
animationDuration: "0.7s",
|
||||
animationDelay: "0.1s",
|
||||
}}
|
||||
>
|
||||
<div>
|
||||
<div className="w-full" onKeyUp={e => e.stopPropagation()} onKeyDown={e => e.stopPropagation()}>
|
||||
<div
|
||||
className="w-full"
|
||||
onKeyUp={e => e.stopPropagation()}
|
||||
onKeyDown={e => e.stopPropagation()} onKeyDownCapture={e => e.stopPropagation()}
|
||||
onKeyUpCapture={e => e.stopPropagation()}
|
||||
>
|
||||
<TextAreaWithLabel
|
||||
ref={TextAreaRef}
|
||||
label="Paste from host"
|
||||
rows={4}
|
||||
onKeyUp={e => e.stopPropagation()}
|
||||
maxLength={pasteMaxLength}
|
||||
onKeyDown={e => {
|
||||
e.stopPropagation();
|
||||
if (e.key === "Enter" && (e.metaKey || e.ctrlKey)) {
|
||||
|
|
@ -171,9 +181,31 @@ export default function PasteModal() {
|
|||
)}
|
||||
</div>
|
||||
</div>
|
||||
<div className={cx("text-xs text-slate-600 dark:text-slate-400", delayClassName)}>
|
||||
<InputFieldWithLabel
|
||||
type="number"
|
||||
label="Delay between keys"
|
||||
placeholder="Delay between keys"
|
||||
min={50}
|
||||
max={65534}
|
||||
value={delayValue}
|
||||
onChange={e => {
|
||||
setDelayValue(parseInt(e.target.value, 10));
|
||||
}}
|
||||
/>
|
||||
{delayValue < 50 || delayValue > 65534 && (
|
||||
<div className="mt-2 flex items-center gap-x-2">
|
||||
<ExclamationCircleIcon className="h-4 w-4 text-red-500 dark:text-red-400" />
|
||||
<span className="text-xs text-red-500 dark:text-red-400">
|
||||
Delay must be between 50 and 65534
|
||||
</span>
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
<div className="space-y-4">
|
||||
<p className="text-xs text-slate-600 dark:text-slate-400">
|
||||
Sending text using keyboard layout: {selectedKeyboard.isoCode}-{selectedKeyboard.name}
|
||||
Sending text using keyboard layout: {selectedKeyboard.isoCode}-
|
||||
{selectedKeyboard.name}
|
||||
</p>
|
||||
</div>
|
||||
</div>
|
||||
|
|
@ -181,7 +213,7 @@ export default function PasteModal() {
|
|||
</div>
|
||||
</div>
|
||||
<div
|
||||
className="flex animate-fadeIn opacity-0 items-center justify-end gap-x-2"
|
||||
className="flex animate-fadeIn items-center justify-end gap-x-2 opacity-0"
|
||||
style={{
|
||||
animationDuration: "0.7s",
|
||||
animationDelay: "0.2s",
|
||||
|
|
@ -200,6 +232,7 @@ export default function PasteModal() {
|
|||
size="SM"
|
||||
theme="primary"
|
||||
text="Confirm Paste"
|
||||
disabled={isPasteInProgress}
|
||||
onClick={onConfirmPaste}
|
||||
LeadingIcon={LuCornerDownLeft}
|
||||
/>
|
||||
|
|
|
|||
|
|
@ -1,4 +1,4 @@
|
|||
import { KeyboardLedState, KeysDownState } from "./stores";
|
||||
import { hidKeyBufferSize, KeyboardLedState, KeysDownState } from "./stores";
|
||||
|
||||
export const HID_RPC_MESSAGE_TYPES = {
|
||||
Handshake: 0x01,
|
||||
|
|
@ -7,8 +7,11 @@ export const HID_RPC_MESSAGE_TYPES = {
|
|||
WheelReport: 0x04,
|
||||
KeypressReport: 0x05,
|
||||
MouseReport: 0x06,
|
||||
KeyboardMacroReport: 0x07,
|
||||
CancelKeyboardMacroReport: 0x08,
|
||||
KeyboardLedState: 0x32,
|
||||
KeysDownState: 0x33,
|
||||
KeyboardMacroState: 0x34,
|
||||
}
|
||||
|
||||
export type HidRpcMessageType = typeof HID_RPC_MESSAGE_TYPES[keyof typeof HID_RPC_MESSAGE_TYPES];
|
||||
|
|
@ -28,7 +31,31 @@ const fromInt32toUint8 = (n: number) => {
|
|||
(n >> 24) & 0xFF,
|
||||
(n >> 16) & 0xFF,
|
||||
(n >> 8) & 0xFF,
|
||||
(n >> 0) & 0xFF,
|
||||
n & 0xFF,
|
||||
]);
|
||||
};
|
||||
|
||||
const fromUint16toUint8 = (n: number) => {
|
||||
if (n > 65535 || n < 0) {
|
||||
throw new Error(`Number ${n} is not within the uint16 range`);
|
||||
}
|
||||
|
||||
return new Uint8Array([
|
||||
(n >> 8) & 0xFF,
|
||||
n & 0xFF,
|
||||
]);
|
||||
};
|
||||
|
||||
const fromUint32toUint8 = (n: number) => {
|
||||
if (n > 4294967295 || n < 0) {
|
||||
throw new Error(`Number ${n} is not within the uint32 range`);
|
||||
}
|
||||
|
||||
return new Uint8Array([
|
||||
(n >> 24) & 0xFF,
|
||||
(n >> 16) & 0xFF,
|
||||
(n >> 8) & 0xFF,
|
||||
n & 0xFF,
|
||||
]);
|
||||
};
|
||||
|
||||
|
|
@ -37,7 +64,7 @@ const fromInt8ToUint8 = (n: number) => {
|
|||
throw new Error(`Number ${n} is not within the int8 range`);
|
||||
}
|
||||
|
||||
return (n >> 0) & 0xFF;
|
||||
return n & 0xFF;
|
||||
};
|
||||
|
||||
const keyboardLedStateMasks = {
|
||||
|
|
@ -186,6 +213,99 @@ export class KeyboardReportMessage extends RpcMessage {
|
|||
}
|
||||
}
|
||||
|
||||
export interface KeyboardMacroStep extends KeysDownState {
|
||||
delay: number;
|
||||
}
|
||||
|
||||
export class KeyboardMacroReportMessage extends RpcMessage {
|
||||
isPaste: boolean;
|
||||
stepCount: number;
|
||||
steps: KeyboardMacroStep[];
|
||||
|
||||
KEYS_LENGTH = hidKeyBufferSize;
|
||||
|
||||
constructor(isPaste: boolean, stepCount: number, steps: KeyboardMacroStep[]) {
|
||||
super(HID_RPC_MESSAGE_TYPES.KeyboardMacroReport);
|
||||
this.isPaste = isPaste;
|
||||
this.stepCount = stepCount;
|
||||
this.steps = steps;
|
||||
}
|
||||
|
||||
marshal(): Uint8Array {
|
||||
// validate if length is correct
|
||||
if (this.stepCount !== this.steps.length) {
|
||||
throw new Error(`Length ${this.stepCount} is not equal to the number of steps ${this.steps.length}`);
|
||||
}
|
||||
|
||||
const data = new Uint8Array(this.stepCount * 9 + 6);
|
||||
data.set(new Uint8Array([
|
||||
this.messageType,
|
||||
this.isPaste ? 1 : 0,
|
||||
...fromUint32toUint8(this.stepCount),
|
||||
]), 0);
|
||||
|
||||
for (let i = 0; i < this.stepCount; i++) {
|
||||
const step = this.steps[i];
|
||||
if (!withinUint8Range(step.modifier)) {
|
||||
throw new Error(`Modifier ${step.modifier} is not within the uint8 range`);
|
||||
}
|
||||
|
||||
// Ensure the keys are within the KEYS_LENGTH range
|
||||
const keys = step.keys;
|
||||
if (keys.length > this.KEYS_LENGTH) {
|
||||
throw new Error(`Keys ${keys} is not within the hidKeyBufferSize range`);
|
||||
} else if (keys.length < this.KEYS_LENGTH) {
|
||||
keys.push(...Array(this.KEYS_LENGTH - keys.length).fill(0));
|
||||
}
|
||||
|
||||
for (const key of keys) {
|
||||
if (!withinUint8Range(key)) {
|
||||
throw new Error(`Key ${key} is not within the uint8 range`);
|
||||
}
|
||||
}
|
||||
|
||||
const macroBinary = new Uint8Array([
|
||||
step.modifier,
|
||||
...keys,
|
||||
...fromUint16toUint8(step.delay),
|
||||
]);
|
||||
const offset = 6 + i * 9;
|
||||
|
||||
|
||||
data.set(macroBinary, offset);
|
||||
}
|
||||
|
||||
return data;
|
||||
}
|
||||
}
|
||||
|
||||
export class KeyboardMacroStateMessage extends RpcMessage {
|
||||
state: boolean;
|
||||
isPaste: boolean;
|
||||
|
||||
constructor(state: boolean, isPaste: boolean) {
|
||||
super(HID_RPC_MESSAGE_TYPES.KeyboardMacroState);
|
||||
this.state = state;
|
||||
this.isPaste = isPaste;
|
||||
}
|
||||
|
||||
marshal(): Uint8Array {
|
||||
return new Uint8Array([
|
||||
this.messageType,
|
||||
this.state ? 1 : 0,
|
||||
this.isPaste ? 1 : 0,
|
||||
]);
|
||||
}
|
||||
|
||||
public static unmarshal(data: Uint8Array): KeyboardMacroStateMessage | undefined {
|
||||
if (data.length < 1) {
|
||||
throw new Error(`Invalid keyboard macro state report message length: ${data.length}`);
|
||||
}
|
||||
|
||||
return new KeyboardMacroStateMessage(data[0] === 1, data[1] === 1);
|
||||
}
|
||||
}
|
||||
|
||||
export class KeyboardLedStateMessage extends RpcMessage {
|
||||
keyboardLedState: KeyboardLedState;
|
||||
|
||||
|
|
@ -256,6 +376,17 @@ export class PointerReportMessage extends RpcMessage {
|
|||
}
|
||||
}
|
||||
|
||||
export class CancelKeyboardMacroReportMessage extends RpcMessage {
|
||||
|
||||
constructor() {
|
||||
super(HID_RPC_MESSAGE_TYPES.CancelKeyboardMacroReport);
|
||||
}
|
||||
|
||||
marshal(): Uint8Array {
|
||||
return new Uint8Array([this.messageType]);
|
||||
}
|
||||
}
|
||||
|
||||
export class MouseReportMessage extends RpcMessage {
|
||||
dx: number;
|
||||
dy: number;
|
||||
|
|
@ -284,6 +415,9 @@ export const messageRegistry = {
|
|||
[HID_RPC_MESSAGE_TYPES.KeyboardLedState]: KeyboardLedStateMessage,
|
||||
[HID_RPC_MESSAGE_TYPES.KeyboardReport]: KeyboardReportMessage,
|
||||
[HID_RPC_MESSAGE_TYPES.KeypressReport]: KeypressReportMessage,
|
||||
[HID_RPC_MESSAGE_TYPES.KeyboardMacroReport]: KeyboardMacroReportMessage,
|
||||
[HID_RPC_MESSAGE_TYPES.CancelKeyboardMacroReport]: CancelKeyboardMacroReportMessage,
|
||||
[HID_RPC_MESSAGE_TYPES.KeyboardMacroState]: KeyboardMacroStateMessage,
|
||||
}
|
||||
|
||||
export const unmarshalHidRpcMessage = (data: Uint8Array): RpcMessage | undefined => {
|
||||
|
|
|
|||
|
|
@ -105,6 +105,9 @@ export interface RTCState {
|
|||
setRpcDataChannel: (channel: RTCDataChannel) => void;
|
||||
rpcDataChannel: RTCDataChannel | null;
|
||||
|
||||
hidRpcDisabled: boolean;
|
||||
setHidRpcDisabled: (disabled: boolean) => void;
|
||||
|
||||
rpcHidProtocolVersion: number | null;
|
||||
setRpcHidProtocolVersion: (version: number) => void;
|
||||
|
||||
|
|
@ -157,6 +160,9 @@ export const useRTCStore = create<RTCState>(set => ({
|
|||
rpcDataChannel: null,
|
||||
setRpcDataChannel: (channel: RTCDataChannel) => set({ rpcDataChannel: channel }),
|
||||
|
||||
hidRpcDisabled: false,
|
||||
setHidRpcDisabled: (disabled: boolean) => set({ hidRpcDisabled: disabled }),
|
||||
|
||||
rpcHidProtocolVersion: null,
|
||||
setRpcHidProtocolVersion: (version: number) => set({ rpcHidProtocolVersion: version }),
|
||||
|
||||
|
|
@ -464,7 +470,7 @@ export interface HidState {
|
|||
isVirtualKeyboardEnabled: boolean;
|
||||
setVirtualKeyboardEnabled: (enabled: boolean) => void;
|
||||
|
||||
isPasteModeEnabled: boolean;
|
||||
isPasteInProgress: boolean;
|
||||
setPasteModeEnabled: (enabled: boolean) => void;
|
||||
|
||||
usbState: USBStates;
|
||||
|
|
@ -481,8 +487,8 @@ export const useHidStore = create<HidState>(set => ({
|
|||
isVirtualKeyboardEnabled: false,
|
||||
setVirtualKeyboardEnabled: (enabled: boolean): void => set({ isVirtualKeyboardEnabled: enabled }),
|
||||
|
||||
isPasteModeEnabled: false,
|
||||
setPasteModeEnabled: (enabled: boolean): void => set({ isPasteModeEnabled: enabled }),
|
||||
isPasteInProgress: false,
|
||||
setPasteModeEnabled: (enabled: boolean): void => set({ isPasteInProgress: enabled }),
|
||||
|
||||
// Add these new properties for USB state
|
||||
usbState: "not attached",
|
||||
|
|
|
|||
|
|
@ -3,8 +3,11 @@ import { useCallback, useEffect, useMemo } from "react";
|
|||
import { useRTCStore } from "@/hooks/stores";
|
||||
|
||||
import {
|
||||
CancelKeyboardMacroReportMessage,
|
||||
HID_RPC_VERSION,
|
||||
HandshakeMessage,
|
||||
KeyboardMacroStep,
|
||||
KeyboardMacroReportMessage,
|
||||
KeyboardReportMessage,
|
||||
KeypressReportMessage,
|
||||
MouseReportMessage,
|
||||
|
|
@ -14,19 +17,23 @@ import {
|
|||
} from "./hidRpc";
|
||||
|
||||
export function useHidRpc(onHidRpcMessage?: (payload: RpcMessage) => void) {
|
||||
const { rpcHidChannel, setRpcHidProtocolVersion, rpcHidProtocolVersion } = useRTCStore();
|
||||
const { rpcHidChannel, setRpcHidProtocolVersion, rpcHidProtocolVersion, hidRpcDisabled } = useRTCStore();
|
||||
const rpcHidReady = useMemo(() => {
|
||||
if (hidRpcDisabled) return false;
|
||||
return rpcHidChannel?.readyState === "open" && rpcHidProtocolVersion !== null;
|
||||
}, [rpcHidChannel, rpcHidProtocolVersion]);
|
||||
}, [rpcHidChannel, rpcHidProtocolVersion, hidRpcDisabled]);
|
||||
|
||||
const rpcHidStatus = useMemo(() => {
|
||||
if (hidRpcDisabled) return "disabled";
|
||||
|
||||
if (!rpcHidChannel) return "N/A";
|
||||
if (rpcHidChannel.readyState !== "open") return rpcHidChannel.readyState;
|
||||
if (!rpcHidProtocolVersion) return "handshaking";
|
||||
return `ready (v${rpcHidProtocolVersion})`;
|
||||
}, [rpcHidChannel, rpcHidProtocolVersion]);
|
||||
}, [rpcHidChannel, rpcHidProtocolVersion, hidRpcDisabled]);
|
||||
|
||||
const sendMessage = useCallback((message: RpcMessage, ignoreHandshakeState = false) => {
|
||||
if (hidRpcDisabled) return;
|
||||
if (rpcHidChannel?.readyState !== "open") return;
|
||||
if (!rpcHidReady && !ignoreHandshakeState) return;
|
||||
|
||||
|
|
@ -39,7 +46,7 @@ export function useHidRpc(onHidRpcMessage?: (payload: RpcMessage) => void) {
|
|||
if (!data) return;
|
||||
|
||||
rpcHidChannel?.send(data as unknown as ArrayBuffer);
|
||||
}, [rpcHidChannel, rpcHidReady]);
|
||||
}, [rpcHidChannel, rpcHidReady, hidRpcDisabled]);
|
||||
|
||||
const reportKeyboardEvent = useCallback(
|
||||
(keys: number[], modifier: number) => {
|
||||
|
|
@ -68,14 +75,31 @@ export function useHidRpc(onHidRpcMessage?: (payload: RpcMessage) => void) {
|
|||
[sendMessage],
|
||||
);
|
||||
|
||||
const reportKeyboardMacroEvent = useCallback(
|
||||
(steps: KeyboardMacroStep[]) => {
|
||||
sendMessage(new KeyboardMacroReportMessage(false, steps.length, steps));
|
||||
},
|
||||
[sendMessage],
|
||||
);
|
||||
|
||||
const cancelOngoingKeyboardMacro = useCallback(
|
||||
() => {
|
||||
sendMessage(new CancelKeyboardMacroReportMessage());
|
||||
},
|
||||
[sendMessage],
|
||||
);
|
||||
|
||||
const sendHandshake = useCallback(() => {
|
||||
if (hidRpcDisabled) return;
|
||||
if (rpcHidProtocolVersion) return;
|
||||
if (!rpcHidChannel) return;
|
||||
|
||||
sendMessage(new HandshakeMessage(HID_RPC_VERSION), true);
|
||||
}, [rpcHidChannel, rpcHidProtocolVersion, sendMessage]);
|
||||
}, [rpcHidChannel, rpcHidProtocolVersion, sendMessage, hidRpcDisabled]);
|
||||
|
||||
const handleHandshake = useCallback((message: HandshakeMessage) => {
|
||||
if (hidRpcDisabled) return;
|
||||
|
||||
if (!message.version) {
|
||||
console.error("Received handshake message without version", message);
|
||||
return;
|
||||
|
|
@ -90,10 +114,11 @@ export function useHidRpc(onHidRpcMessage?: (payload: RpcMessage) => void) {
|
|||
}
|
||||
|
||||
setRpcHidProtocolVersion(message.version);
|
||||
}, [setRpcHidProtocolVersion]);
|
||||
}, [setRpcHidProtocolVersion, hidRpcDisabled]);
|
||||
|
||||
useEffect(() => {
|
||||
if (!rpcHidChannel) return;
|
||||
if (hidRpcDisabled) return;
|
||||
|
||||
// send handshake message
|
||||
sendHandshake();
|
||||
|
|
@ -135,6 +160,7 @@ export function useHidRpc(onHidRpcMessage?: (payload: RpcMessage) => void) {
|
|||
setRpcHidProtocolVersion,
|
||||
sendHandshake,
|
||||
handleHandshake,
|
||||
hidRpcDisabled,
|
||||
],
|
||||
);
|
||||
|
||||
|
|
@ -143,6 +169,8 @@ export function useHidRpc(onHidRpcMessage?: (payload: RpcMessage) => void) {
|
|||
reportKeypressEvent,
|
||||
reportAbsMouseEvent,
|
||||
reportRelMouseEvent,
|
||||
reportKeyboardMacroEvent,
|
||||
cancelOngoingKeyboardMacro,
|
||||
rpcHidProtocolVersion,
|
||||
rpcHidReady,
|
||||
rpcHidStatus,
|
||||
|
|
|
|||
|
|
@ -1,15 +1,48 @@
|
|||
import { useCallback } from "react";
|
||||
import { useCallback, useRef } from "react";
|
||||
|
||||
import { hidErrorRollOver, hidKeyBufferSize, KeysDownState, useHidStore, useRTCStore } from "@/hooks/stores";
|
||||
import {
|
||||
hidErrorRollOver,
|
||||
hidKeyBufferSize,
|
||||
KeysDownState,
|
||||
useHidStore,
|
||||
useRTCStore,
|
||||
} from "@/hooks/stores";
|
||||
import { JsonRpcResponse, useJsonRpc } from "@/hooks/useJsonRpc";
|
||||
import { useHidRpc } from "@/hooks/useHidRpc";
|
||||
import { KeyboardLedStateMessage, KeysDownStateMessage } from "@/hooks/hidRpc";
|
||||
import {
|
||||
KeyboardLedStateMessage,
|
||||
KeyboardMacroStateMessage,
|
||||
KeyboardMacroStep,
|
||||
KeysDownStateMessage,
|
||||
} from "@/hooks/hidRpc";
|
||||
import { hidKeyToModifierMask, keys, modifiers } from "@/keyboardMappings";
|
||||
|
||||
const MACRO_RESET_KEYBOARD_STATE = {
|
||||
keys: new Array(hidKeyBufferSize).fill(0),
|
||||
modifier: 0,
|
||||
delay: 0,
|
||||
};
|
||||
|
||||
export interface MacroStep {
|
||||
keys: string[] | null;
|
||||
modifiers: string[] | null;
|
||||
delay: number;
|
||||
}
|
||||
|
||||
export type MacroSteps = MacroStep[];
|
||||
|
||||
const sleep = (ms: number): Promise<void> => new Promise(resolve => setTimeout(resolve, ms));
|
||||
|
||||
export default function useKeyboard() {
|
||||
const { send } = useJsonRpc();
|
||||
const { rpcDataChannel } = useRTCStore();
|
||||
const { keysDownState, setKeysDownState, setKeyboardLedState } = useHidStore();
|
||||
const { keysDownState, setKeysDownState, setKeyboardLedState, setPasteModeEnabled } =
|
||||
useHidStore();
|
||||
|
||||
const abortController = useRef<AbortController | null>(null);
|
||||
const setAbortController = useCallback((ac: AbortController | null) => {
|
||||
abortController.current = ac;
|
||||
}, []);
|
||||
|
||||
// INTRODUCTION: The earlier version of the JetKVM device shipped with all keyboard state
|
||||
// being tracked on the browser/client-side. When adding the keyPressReport API to the
|
||||
|
|
@ -17,17 +50,19 @@ export default function useKeyboard() {
|
|||
// is running on the cloud against a device that has not been updated yet and thus does not
|
||||
// support the keyPressReport API. In that case, we need to handle the key presses locally
|
||||
// and send the full state to the device, so it can behave like a real USB HID keyboard.
|
||||
// This flag indicates whether the keyPressReport API is available on the device which is
|
||||
// This flag indicates whether the keyPressReport API is available on the device which is
|
||||
// dynamically set when the device responds to the first key press event or reports its
|
||||
// keysDownState when queried since the keyPressReport was introduced together with the
|
||||
// keysDownState when queried since the keyPressReport was introduced together with the
|
||||
// getKeysDownState API.
|
||||
|
||||
// HidRPC is a binary format for exchanging keyboard and mouse events
|
||||
const {
|
||||
reportKeyboardEvent: sendKeyboardEventHidRpc,
|
||||
reportKeypressEvent: sendKeypressEventHidRpc,
|
||||
reportKeyboardMacroEvent: sendKeyboardMacroEventHidRpc,
|
||||
cancelOngoingKeyboardMacro: cancelOngoingKeyboardMacroHidRpc,
|
||||
rpcHidReady,
|
||||
} = useHidRpc((message) => {
|
||||
} = useHidRpc(message => {
|
||||
switch (message.constructor) {
|
||||
case KeysDownStateMessage:
|
||||
setKeysDownState((message as KeysDownStateMessage).keysDownState);
|
||||
|
|
@ -35,52 +70,61 @@ export default function useKeyboard() {
|
|||
case KeyboardLedStateMessage:
|
||||
setKeyboardLedState((message as KeyboardLedStateMessage).keyboardLedState);
|
||||
break;
|
||||
case KeyboardMacroStateMessage:
|
||||
if (!(message as KeyboardMacroStateMessage).isPaste) break;
|
||||
setPasteModeEnabled((message as KeyboardMacroStateMessage).state);
|
||||
break;
|
||||
default:
|
||||
break;
|
||||
}
|
||||
});
|
||||
|
||||
// sendKeyboardEvent is used to send the full keyboard state to the device for macro handling
|
||||
// and resetting keyboard state. It sends the keys currently pressed and the modifier state.
|
||||
// The device will respond with the keysDownState if it supports the keyPressReport API
|
||||
// or just accept the state if it does not support (returning no result)
|
||||
const sendKeyboardEvent = useCallback(
|
||||
async (state: KeysDownState) => {
|
||||
if (rpcDataChannel?.readyState !== "open" && !rpcHidReady) return;
|
||||
|
||||
console.debug(`Send keyboardReport keys: ${state.keys}, modifier: ${state.modifier}`);
|
||||
|
||||
if (rpcHidReady) {
|
||||
console.debug("Sending keyboard report via HidRPC");
|
||||
sendKeyboardEventHidRpc(state.keys, state.modifier);
|
||||
return;
|
||||
}
|
||||
|
||||
send("keyboardReport", { keys: state.keys, modifier: state.modifier }, (resp: JsonRpcResponse) => {
|
||||
const handleLegacyKeyboardReport = useCallback(
|
||||
async (keys: number[], modifier: number) => {
|
||||
send("keyboardReport", { keys, modifier }, (resp: JsonRpcResponse) => {
|
||||
if ("error" in resp) {
|
||||
console.error(`Failed to send keyboard report ${state}`, resp.error);
|
||||
console.error(`Failed to send keyboard report ${keys} ${modifier}`, resp.error);
|
||||
}
|
||||
|
||||
// On older backends, we need to set the keysDownState manually since without the hidRpc API, the state doesn't trickle down from the backend
|
||||
setKeysDownState({ modifier, keys });
|
||||
});
|
||||
},
|
||||
[
|
||||
rpcDataChannel?.readyState,
|
||||
rpcHidReady,
|
||||
send,
|
||||
sendKeyboardEventHidRpc,
|
||||
],
|
||||
[send, setKeysDownState],
|
||||
);
|
||||
const sendKeystrokeLegacy = useCallback(async (keys: number[], modifier: number, ac?: AbortController) => {
|
||||
return await new Promise<void>((resolve, reject) => {
|
||||
const abortListener = () => {
|
||||
reject(new Error("Keyboard report aborted"));
|
||||
};
|
||||
|
||||
ac?.signal?.addEventListener("abort", abortListener);
|
||||
|
||||
send(
|
||||
"keyboardReport",
|
||||
{ keys, modifier },
|
||||
params => {
|
||||
if ("error" in params) return reject(params.error);
|
||||
resolve();
|
||||
},
|
||||
);
|
||||
});
|
||||
}, [send]);
|
||||
|
||||
// resetKeyboardState is used to reset the keyboard state to no keys pressed and no modifiers.
|
||||
// This is useful for macros and when the browser loses focus to ensure that the keyboard state
|
||||
// is clean.
|
||||
const resetKeyboardState = useCallback(
|
||||
async () => {
|
||||
// Reset the keys buffer to zeros and the modifier state to zero
|
||||
keysDownState.keys.length = hidKeyBufferSize;
|
||||
keysDownState.keys.fill(0);
|
||||
keysDownState.modifier = 0;
|
||||
sendKeyboardEvent(keysDownState);
|
||||
}, [keysDownState, sendKeyboardEvent]);
|
||||
const resetKeyboardState = useCallback(async () => {
|
||||
// Reset the keys buffer to zeros and the modifier state to zero
|
||||
const { keys, modifier } = MACRO_RESET_KEYBOARD_STATE;
|
||||
if (rpcHidReady) {
|
||||
sendKeyboardEventHidRpc(keys, modifier);
|
||||
} else {
|
||||
// Older backends don't support the hidRpc API, so we send the full reset state
|
||||
handleLegacyKeyboardReport(keys, modifier);
|
||||
}
|
||||
}, [rpcHidReady, sendKeyboardEventHidRpc, handleLegacyKeyboardReport]);
|
||||
|
||||
|
||||
// executeMacro is used to execute a macro consisting of multiple steps.
|
||||
// Each step can have multiple keys, multiple modifiers and a delay.
|
||||
|
|
@ -88,28 +132,90 @@ export default function useKeyboard() {
|
|||
// After the delay, the keys and modifiers are released and the next step is executed.
|
||||
// If a step has no keys or modifiers, it is treated as a delay-only step.
|
||||
// A small pause is added between steps to ensure that the device can process the events.
|
||||
const executeMacro = async (steps: { keys: string[] | null; modifiers: string[] | null; delay: number }[]) => {
|
||||
for (const [index, step] of steps.entries()) {
|
||||
const executeMacroRemote = useCallback(async (steps: MacroSteps) => {
|
||||
const macro: KeyboardMacroStep[] = [];
|
||||
|
||||
for (const [_, step] of steps.entries()) {
|
||||
const keyValues = (step.keys || []).map(key => keys[key]).filter(Boolean);
|
||||
const modifierMask: number = (step.modifiers || []).map(mod => modifiers[mod]).reduce((acc, val) => acc + val, 0);
|
||||
const modifierMask: number = (step.modifiers || [])
|
||||
.map(mod => modifiers[mod])
|
||||
.reduce((acc, val) => acc + val, 0);
|
||||
|
||||
// If the step has keys and/or modifiers, press them and hold for the delay
|
||||
if (keyValues.length > 0 || modifierMask > 0) {
|
||||
sendKeyboardEvent({ keys: keyValues, modifier: modifierMask });
|
||||
await new Promise(resolve => setTimeout(resolve, step.delay || 50));
|
||||
|
||||
resetKeyboardState();
|
||||
} else {
|
||||
// This is a delay-only step, just wait for the delay amount
|
||||
await new Promise(resolve => setTimeout(resolve, step.delay || 50));
|
||||
}
|
||||
|
||||
// Add a small pause between steps if not the last step
|
||||
if (index < steps.length - 1) {
|
||||
await new Promise(resolve => setTimeout(resolve, 10));
|
||||
macro.push({ keys: keyValues, modifier: modifierMask, delay: 20 });
|
||||
macro.push({ ...MACRO_RESET_KEYBOARD_STATE, delay: step.delay || 100 });
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
sendKeyboardMacroEventHidRpc(macro);
|
||||
}, [sendKeyboardMacroEventHidRpc]);
|
||||
const executeMacroClientSide = useCallback(async (steps: MacroSteps) => {
|
||||
const promises: (() => Promise<void>)[] = [];
|
||||
|
||||
const ac = new AbortController();
|
||||
setAbortController(ac);
|
||||
|
||||
for (const [_, step] of steps.entries()) {
|
||||
const keyValues = (step.keys || []).map(key => keys[key]).filter(Boolean);
|
||||
const modifierMask: number = (step.modifiers || [])
|
||||
.map(mod => modifiers[mod])
|
||||
.reduce((acc, val) => acc + val, 0);
|
||||
|
||||
// If the step has keys and/or modifiers, press them and hold for the delay
|
||||
if (keyValues.length > 0 || modifierMask > 0) {
|
||||
promises.push(() => sendKeystrokeLegacy(keyValues, modifierMask, ac));
|
||||
promises.push(() => resetKeyboardState());
|
||||
promises.push(() => sleep(step.delay || 100));
|
||||
}
|
||||
}
|
||||
|
||||
const runAll = async () => {
|
||||
for (const promise of promises) {
|
||||
// Check if we've been aborted before executing each promise
|
||||
if (ac.signal.aborted) {
|
||||
throw new Error("Macro execution aborted");
|
||||
}
|
||||
await promise();
|
||||
}
|
||||
}
|
||||
|
||||
return await new Promise<void>((resolve, reject) => {
|
||||
// Set up abort listener
|
||||
const abortListener = () => {
|
||||
reject(new Error("Macro execution aborted"));
|
||||
};
|
||||
|
||||
ac.signal.addEventListener("abort", abortListener);
|
||||
|
||||
runAll()
|
||||
.then(() => {
|
||||
ac.signal.removeEventListener("abort", abortListener);
|
||||
resolve();
|
||||
})
|
||||
.catch((error) => {
|
||||
ac.signal.removeEventListener("abort", abortListener);
|
||||
reject(error);
|
||||
});
|
||||
});
|
||||
}, [sendKeystrokeLegacy, resetKeyboardState, setAbortController]);
|
||||
const executeMacro = useCallback(async (steps: MacroSteps) => {
|
||||
if (rpcHidReady) {
|
||||
return executeMacroRemote(steps);
|
||||
}
|
||||
return executeMacroClientSide(steps);
|
||||
}, [rpcHidReady, executeMacroRemote, executeMacroClientSide]);
|
||||
|
||||
const cancelExecuteMacro = useCallback(async () => {
|
||||
if (abortController.current) {
|
||||
abortController.current.abort();
|
||||
}
|
||||
if (!rpcHidReady) return;
|
||||
// older versions don't support this API,
|
||||
// and all paste actions are pure-frontend,
|
||||
// we don't need to cancel it actually
|
||||
cancelOngoingKeyboardMacroHidRpc();
|
||||
}, [rpcHidReady, cancelOngoingKeyboardMacroHidRpc, abortController]);
|
||||
|
||||
// handleKeyPress is used to handle a key press or release event.
|
||||
// This function handle both key press and key release events.
|
||||
|
|
@ -131,9 +237,16 @@ export default function useKeyboard() {
|
|||
// in client/browser-side code using simulateDeviceSideKeyHandlingForLegacyDevices.
|
||||
sendKeypressEventHidRpc(key, press);
|
||||
} else {
|
||||
// if the keyPress api is not available, we need to handle the key locally
|
||||
const downState = simulateDeviceSideKeyHandlingForLegacyDevices(keysDownState, key, press);
|
||||
sendKeyboardEvent(downState); // then we send the full state
|
||||
// Older backends don't support the hidRpc API, so we need:
|
||||
// 1. Calculate the state
|
||||
// 2. Send the newly calculated state to the device
|
||||
const downState = simulateDeviceSideKeyHandlingForLegacyDevices(
|
||||
keysDownState,
|
||||
key,
|
||||
press,
|
||||
);
|
||||
|
||||
handleLegacyKeyboardReport(downState.keys, downState.modifier);
|
||||
|
||||
// if we just sent ErrorRollOver, reset to empty state
|
||||
if (downState.keys[0] === hidErrorRollOver) {
|
||||
|
|
@ -142,17 +255,21 @@ export default function useKeyboard() {
|
|||
}
|
||||
},
|
||||
[
|
||||
rpcHidReady,
|
||||
keysDownState,
|
||||
resetKeyboardState,
|
||||
rpcDataChannel?.readyState,
|
||||
sendKeyboardEvent,
|
||||
rpcHidReady,
|
||||
sendKeypressEventHidRpc,
|
||||
keysDownState,
|
||||
handleLegacyKeyboardReport,
|
||||
resetKeyboardState,
|
||||
],
|
||||
);
|
||||
|
||||
// IMPORTANT: See the keyPressReportApiAvailable comment above for the reason this exists
|
||||
function simulateDeviceSideKeyHandlingForLegacyDevices(state: KeysDownState, key: number, press: boolean): KeysDownState {
|
||||
function simulateDeviceSideKeyHandlingForLegacyDevices(
|
||||
state: KeysDownState,
|
||||
key: number,
|
||||
press: boolean,
|
||||
): KeysDownState {
|
||||
// IMPORTANT: This code parallels the logic in the kernel's hid-gadget driver
|
||||
// for handling key presses and releases. It ensures that the USB gadget
|
||||
// behaves similarly to a real USB HID keyboard. This logic is paralleled
|
||||
|
|
@ -164,7 +281,7 @@ export default function useKeyboard() {
|
|||
if (modifierMask !== 0) {
|
||||
// If the key is a modifier key, we update the keyboardModifier state
|
||||
// by setting or clearing the corresponding bit in the modifier byte.
|
||||
// This allows us to track the state of dynamic modifier keys like
|
||||
// This allows us to track the state of dynamic modifier keys like
|
||||
// Shift, Control, Alt, and Super.
|
||||
if (press) {
|
||||
modifiers |= modifierMask;
|
||||
|
|
@ -181,7 +298,7 @@ export default function useKeyboard() {
|
|||
// and if we find a zero byte, we can place the key there (if press is true)
|
||||
if (keys[i] === key || keys[i] === 0) {
|
||||
if (press) {
|
||||
keys[i] = key // overwrites the zero byte or the same key if already pressed
|
||||
keys[i] = key; // overwrites the zero byte or the same key if already pressed
|
||||
} else {
|
||||
// we are releasing the key, remove it from the buffer
|
||||
if (keys[i] !== 0) {
|
||||
|
|
@ -197,18 +314,20 @@ export default function useKeyboard() {
|
|||
// If we reach here it means we didn't find an empty slot or the key in the buffer
|
||||
if (overrun) {
|
||||
if (press) {
|
||||
console.warn(`keyboard buffer overflow current keys ${keys}, key: ${key} not added`);
|
||||
console.warn(
|
||||
`keyboard buffer overflow current keys ${keys}, key: ${key} not added`,
|
||||
);
|
||||
// Fill all key slots with ErrorRollOver (0x01) to indicate overflow
|
||||
keys.length = hidKeyBufferSize;
|
||||
keys.fill(hidErrorRollOver);
|
||||
} else {
|
||||
// If we are releasing a key, and we didn't find it in a slot, who cares?
|
||||
console.debug(`key ${key} not found in buffer, nothing to release`)
|
||||
console.debug(`key ${key} not found in buffer, nothing to release`);
|
||||
}
|
||||
}
|
||||
}
|
||||
return { modifier: modifiers, keys };
|
||||
}
|
||||
|
||||
return { handleKeyPress, resetKeyboardState, executeMacro };
|
||||
return { handleKeyPress, resetKeyboardState, executeMacro, cancelExecuteMacro };
|
||||
}
|
||||
|
|
|
|||
|
|
@ -583,6 +583,7 @@ export default function KvmIdRoute() {
|
|||
keyboardLedState, setKeyboardLedState,
|
||||
keysDownState, setKeysDownState, setUsbState,
|
||||
} = useHidStore();
|
||||
const setHidRpcDisabled = useRTCStore(state => state.setHidRpcDisabled);
|
||||
|
||||
const [hasUpdated, setHasUpdated] = useState(false);
|
||||
const { navigateTo } = useDeviceUiNavigation();
|
||||
|
|
@ -695,6 +696,7 @@ export default function KvmIdRoute() {
|
|||
if (resp.error.code === -32601) {
|
||||
// if we don't support key down state, we know key press is also not available
|
||||
console.warn("Failed to get key down state, switching to old-school", resp.error);
|
||||
setHidRpcDisabled(true);
|
||||
} else {
|
||||
console.error("Failed to get key down state", resp.error);
|
||||
}
|
||||
|
|
@ -705,7 +707,7 @@ export default function KvmIdRoute() {
|
|||
}
|
||||
setNeedKeyDownState(false);
|
||||
});
|
||||
}, [keysDownState, needKeyDownState, rpcDataChannel?.readyState, send, setKeysDownState]);
|
||||
}, [keysDownState, needKeyDownState, rpcDataChannel?.readyState, send, setKeysDownState, setHidRpcDisabled]);
|
||||
|
||||
// When the update is successful, we need to refresh the client javascript and show a success modal
|
||||
useEffect(() => {
|
||||
|
|
|
|||
4
web.go
4
web.go
|
|
@ -228,6 +228,10 @@ func handleWebRTCSession(c *gin.Context) {
|
|||
_ = peerConn.Close()
|
||||
}()
|
||||
}
|
||||
|
||||
// Cancel any ongoing keyboard macro when session changes
|
||||
cancelKeyboardMacro()
|
||||
|
||||
currentSession = session
|
||||
c.JSON(http.StatusOK, gin.H{"sd": sd})
|
||||
}
|
||||
|
|
|
|||
|
|
@ -266,6 +266,8 @@ func newSession(config SessionConfig) (*Session, error) {
|
|||
if connectionState == webrtc.ICEConnectionStateClosed {
|
||||
scopedLogger.Debug().Msg("ICE Connection State is closed, unmounting virtual media")
|
||||
if session == currentSession {
|
||||
// Cancel any ongoing keyboard report multi when session closes
|
||||
cancelKeyboardMacro()
|
||||
currentSession = nil
|
||||
}
|
||||
// Stop RPC processor
|
||||
|
|
|
|||
Loading…
Reference in New Issue