-
e.stopPropagation()} onKeyDown={e => e.stopPropagation()}>
+
e.stopPropagation()}
+ onKeyDown={e => e.stopPropagation()}
+ >
- Sending text using keyboard layout: {selectedKeyboard.isoCode}-{selectedKeyboard.name}
+ Sending text using keyboard layout: {selectedKeyboard.isoCode}-
+ {selectedKeyboard.name}
@@ -181,7 +182,7 @@ export default function PasteModal() {
diff --git a/ui/src/hooks/hidRpc.ts b/ui/src/hooks/hidRpc.ts
index 20b8a108..05923c94 100644
--- a/ui/src/hooks/hidRpc.ts
+++ b/ui/src/hooks/hidRpc.ts
@@ -7,6 +7,7 @@ export const HID_RPC_MESSAGE_TYPES = {
WheelReport: 0x04,
KeypressReport: 0x05,
MouseReport: 0x06,
+ KeyboardMacroReport: 0x07,
KeyboardLedState: 0x32,
KeysDownState: 0x33,
}
@@ -32,6 +33,30 @@ const fromInt32toUint8 = (n: number) => {
]);
};
+const fromUint16toUint8 = (n: number) => {
+ if (n > 65535 || n < 0) {
+ throw new Error(`Number ${n} is not within the uint16 range`);
+ }
+
+ return new Uint8Array([
+ (n >> 8) & 0xFF,
+ (n >> 0) & 0xFF,
+ ]);
+};
+
+const fromUint32toUint8 = (n: number) => {
+ if (n > 4294967295 || n < 0) {
+ throw new Error(`Number ${n} is not within the uint32 range`);
+ }
+
+ return new Uint8Array([
+ (n >> 24) & 0xFF,
+ (n >> 16) & 0xFF,
+ (n >> 8) & 0xFF,
+ (n >> 0) & 0xFF,
+ ]);
+};
+
const fromInt8ToUint8 = (n: number) => {
if (n < -128 || n > 127) {
throw new Error(`Number ${n} is not within the int8 range`);
@@ -186,6 +211,64 @@ export class KeyboardReportMessage extends RpcMessage {
}
}
+export interface KeyboardMacro extends KeysDownState {
+ delay: number;
+}
+
+export class KeyboardMacroReportMessage extends RpcMessage {
+ isPaste: boolean;
+ length: number;
+ macro: KeyboardMacro[];
+
+ KEYS_LENGTH = 6;
+
+ constructor(isPaste: boolean, length: number, macro: KeyboardMacro[]) {
+ super(HID_RPC_MESSAGE_TYPES.KeyboardMacroReport);
+ this.isPaste = isPaste;
+ this.length = length;
+ this.macro = macro;
+ }
+
+ marshal(): Uint8Array {
+ const dataHeader = new Uint8Array([
+ this.messageType,
+ this.isPaste ? 1 : 0,
+ ...fromUint32toUint8(this.length),
+ ]);
+
+ let dataBody = new Uint8Array();
+
+ for (const step of this.macro) {
+ if (!withinUint8Range(step.modifier)) {
+ throw new Error(`Modifier ${step.modifier} is not within the uint8 range`);
+ }
+
+ // Ensure the keys are within the KEYS_LENGTH range
+ const keys = step.keys;
+ if (keys.length > this.KEYS_LENGTH) {
+ throw new Error(`Keys ${keys} is not within the hidKeyBufferSize range`);
+ } else if (keys.length < this.KEYS_LENGTH) {
+ keys.push(...Array(this.KEYS_LENGTH - keys.length).fill(0));
+ }
+
+ for (const key of keys) {
+ if (!withinUint8Range(key)) {
+ throw new Error(`Key ${key} is not within the uint8 range`);
+ }
+ }
+
+ const macroBinary = new Uint8Array([
+ step.modifier,
+ ...keys,
+ ...fromUint16toUint8(step.delay),
+ ]);
+
+ dataBody = new Uint8Array([...dataBody, ...macroBinary]);
+ }
+ return new Uint8Array([...dataHeader, ...dataBody]);
+ }
+}
+
export class KeyboardLedStateMessage extends RpcMessage {
keyboardLedState: KeyboardLedState;
diff --git a/ui/src/hooks/useHidRpc.ts b/ui/src/hooks/useHidRpc.ts
index ea0c7112..3beb9c07 100644
--- a/ui/src/hooks/useHidRpc.ts
+++ b/ui/src/hooks/useHidRpc.ts
@@ -5,6 +5,8 @@ import { useRTCStore } from "@/hooks/stores";
import {
HID_RPC_VERSION,
HandshakeMessage,
+ KeyboardMacro,
+ KeyboardMacroReportMessage,
KeyboardReportMessage,
KeypressReportMessage,
MouseReportMessage,
@@ -68,6 +70,15 @@ export function useHidRpc(onHidRpcMessage?: (payload: RpcMessage) => void) {
[sendMessage],
);
+ const reportKeyboardMacroEvent = useCallback(
+ (macro: KeyboardMacro[]) => {
+ const d = new KeyboardMacroReportMessage(false, macro.length, macro);
+ sendMessage(d);
+ console.log("Sent keyboard macro report", d, d.marshal());
+ },
+ [sendMessage],
+ );
+
const sendHandshake = useCallback(() => {
if (rpcHidProtocolVersion) return;
if (!rpcHidChannel) return;
@@ -143,6 +154,7 @@ export function useHidRpc(onHidRpcMessage?: (payload: RpcMessage) => void) {
reportKeypressEvent,
reportAbsMouseEvent,
reportRelMouseEvent,
+ reportKeyboardMacroEvent,
rpcHidProtocolVersion,
rpcHidReady,
rpcHidStatus,
diff --git a/ui/src/hooks/useKeyboard.ts b/ui/src/hooks/useKeyboard.ts
index 787df9a9..315724fc 100644
--- a/ui/src/hooks/useKeyboard.ts
+++ b/ui/src/hooks/useKeyboard.ts
@@ -1,9 +1,15 @@
-import { useCallback } from "react";
+import { useCallback, useMemo } from "react";
-import { hidErrorRollOver, hidKeyBufferSize, KeysDownState, useHidStore, useRTCStore } from "@/hooks/stores";
+import {
+ hidErrorRollOver,
+ hidKeyBufferSize,
+ KeysDownState,
+ useHidStore,
+ useRTCStore,
+} from "@/hooks/stores";
import { JsonRpcResponse, useJsonRpc } from "@/hooks/useJsonRpc";
import { useHidRpc } from "@/hooks/useHidRpc";
-import { KeyboardLedStateMessage, KeysDownStateMessage } from "@/hooks/hidRpc";
+import { KeyboardLedStateMessage, KeyboardMacro, KeysDownStateMessage } from "@/hooks/hidRpc";
import { hidKeyToModifierMask, keys, modifiers } from "@/keyboardMappings";
export default function useKeyboard() {
@@ -17,17 +23,18 @@ export default function useKeyboard() {
// is running on the cloud against a device that has not been updated yet and thus does not
// support the keyPressReport API. In that case, we need to handle the key presses locally
// and send the full state to the device, so it can behave like a real USB HID keyboard.
- // This flag indicates whether the keyPressReport API is available on the device which is
+ // This flag indicates whether the keyPressReport API is available on the device which is
// dynamically set when the device responds to the first key press event or reports its
- // keysDownState when queried since the keyPressReport was introduced together with the
+ // keysDownState when queried since the keyPressReport was introduced together with the
// getKeysDownState API.
// HidRPC is a binary format for exchanging keyboard and mouse events
const {
reportKeyboardEvent: sendKeyboardEventHidRpc,
reportKeypressEvent: sendKeypressEventHidRpc,
+ reportKeyboardMacroEvent: sendKeyboardMacroEventHidRpc,
rpcHidReady,
- } = useHidRpc((message) => {
+ } = useHidRpc(message => {
switch (message.constructor) {
case KeysDownStateMessage:
setKeysDownState((message as KeysDownStateMessage).keysDownState);
@@ -48,7 +55,9 @@ export default function useKeyboard() {
async (state: KeysDownState) => {
if (rpcDataChannel?.readyState !== "open" && !rpcHidReady) return;
- console.debug(`Send keyboardReport keys: ${state.keys}, modifier: ${state.modifier}`);
+ console.debug(
+ `Send keyboardReport keys: ${state.keys}, modifier: ${state.modifier}`,
+ );
if (rpcHidReady) {
console.debug("Sending keyboard report via HidRPC");
@@ -56,31 +65,32 @@ export default function useKeyboard() {
return;
}
- send("keyboardReport", { keys: state.keys, modifier: state.modifier }, (resp: JsonRpcResponse) => {
- if ("error" in resp) {
- console.error(`Failed to send keyboard report ${state}`, resp.error);
- }
- });
+ send(
+ "keyboardReport",
+ { keys: state.keys, modifier: state.modifier },
+ (resp: JsonRpcResponse) => {
+ if ("error" in resp) {
+ console.error(`Failed to send keyboard report ${state}`, resp.error);
+ }
+ },
+ );
},
- [
- rpcDataChannel?.readyState,
- rpcHidReady,
- send,
- sendKeyboardEventHidRpc,
- ],
+ [rpcDataChannel?.readyState, rpcHidReady, send, sendKeyboardEventHidRpc],
);
+ const MACRO_RESET_KEYBOARD_STATE = useMemo(() => ({
+ keys: new Array(hidKeyBufferSize).fill(0),
+ modifier: 0,
+ delay: 0,
+ }), []);
+
// resetKeyboardState is used to reset the keyboard state to no keys pressed and no modifiers.
// This is useful for macros and when the browser loses focus to ensure that the keyboard state
// is clean.
- const resetKeyboardState = useCallback(
- async () => {
- // Reset the keys buffer to zeros and the modifier state to zero
- keysDownState.keys.length = hidKeyBufferSize;
- keysDownState.keys.fill(0);
- keysDownState.modifier = 0;
- sendKeyboardEvent(keysDownState);
- }, [keysDownState, sendKeyboardEvent]);
+ const resetKeyboardState = useCallback(async () => {
+ // Reset the keys buffer to zeros and the modifier state to zero
+ sendKeyboardEvent(MACRO_RESET_KEYBOARD_STATE);
+ }, [sendKeyboardEvent, MACRO_RESET_KEYBOARD_STATE]);
// executeMacro is used to execute a macro consisting of multiple steps.
// Each step can have multiple keys, multiple modifiers and a delay.
@@ -88,29 +98,35 @@ export default function useKeyboard() {
// After the delay, the keys and modifiers are released and the next step is executed.
// If a step has no keys or modifiers, it is treated as a delay-only step.
// A small pause is added between steps to ensure that the device can process the events.
- const executeMacro = async (steps: { keys: string[] | null; modifiers: string[] | null; delay: number }[]) => {
- for (const [index, step] of steps.entries()) {
+ const executeMacro = async (
+ steps: { keys: string[] | null; modifiers: string[] | null; delay: number }[],
+ ) => {
+ const macro: KeyboardMacro[] = [];
+
+ for (const [_, step] of steps.entries()) {
const keyValues = (step.keys || []).map(key => keys[key]).filter(Boolean);
- const modifierMask: number = (step.modifiers || []).map(mod => modifiers[mod]).reduce((acc, val) => acc + val, 0);
+ const modifierMask: number = (step.modifiers || [])
+ .map(mod => modifiers[mod])
+ .reduce((acc, val) => acc + val, 0);
// If the step has keys and/or modifiers, press them and hold for the delay
if (keyValues.length > 0 || modifierMask > 0) {
- sendKeyboardEvent({ keys: keyValues, modifier: modifierMask });
- await new Promise(resolve => setTimeout(resolve, step.delay || 50));
-
- resetKeyboardState();
- } else {
- // This is a delay-only step, just wait for the delay amount
- await new Promise(resolve => setTimeout(resolve, step.delay || 50));
- }
-
- // Add a small pause between steps if not the last step
- if (index < steps.length - 1) {
- await new Promise(resolve => setTimeout(resolve, 10));
+ macro.push({ keys: keyValues, modifier: modifierMask, delay: 50 });
+ macro.push({ ...MACRO_RESET_KEYBOARD_STATE, delay: 200 });
}
}
+
+ sendKeyboardMacroEventHidRpc(macro);
};
+ const cancelExecuteMacro = useCallback(async () => {
+ send("cancelKeyboardReportMulti", {}, (resp: JsonRpcResponse) => {
+ if ("error" in resp) {
+ console.error(`Failed to cancel keyboard report multi`, resp.error);
+ }
+ });
+ }, [send]);
+
// handleKeyPress is used to handle a key press or release event.
// This function handle both key press and key release events.
// It checks if the keyPressReport API is available and sends the key press event.
@@ -132,7 +148,11 @@ export default function useKeyboard() {
sendKeypressEventHidRpc(key, press);
} else {
// if the keyPress api is not available, we need to handle the key locally
- const downState = simulateDeviceSideKeyHandlingForLegacyDevices(keysDownState, key, press);
+ const downState = simulateDeviceSideKeyHandlingForLegacyDevices(
+ keysDownState,
+ key,
+ press,
+ );
sendKeyboardEvent(downState); // then we send the full state
// if we just sent ErrorRollOver, reset to empty state
@@ -152,7 +172,11 @@ export default function useKeyboard() {
);
// IMPORTANT: See the keyPressReportApiAvailable comment above for the reason this exists
- function simulateDeviceSideKeyHandlingForLegacyDevices(state: KeysDownState, key: number, press: boolean): KeysDownState {
+ function simulateDeviceSideKeyHandlingForLegacyDevices(
+ state: KeysDownState,
+ key: number,
+ press: boolean,
+ ): KeysDownState {
// IMPORTANT: This code parallels the logic in the kernel's hid-gadget driver
// for handling key presses and releases. It ensures that the USB gadget
// behaves similarly to a real USB HID keyboard. This logic is paralleled
@@ -164,7 +188,7 @@ export default function useKeyboard() {
if (modifierMask !== 0) {
// If the key is a modifier key, we update the keyboardModifier state
// by setting or clearing the corresponding bit in the modifier byte.
- // This allows us to track the state of dynamic modifier keys like
+ // This allows us to track the state of dynamic modifier keys like
// Shift, Control, Alt, and Super.
if (press) {
modifiers |= modifierMask;
@@ -181,7 +205,7 @@ export default function useKeyboard() {
// and if we find a zero byte, we can place the key there (if press is true)
if (keys[i] === key || keys[i] === 0) {
if (press) {
- keys[i] = key // overwrites the zero byte or the same key if already pressed
+ keys[i] = key; // overwrites the zero byte or the same key if already pressed
} else {
// we are releasing the key, remove it from the buffer
if (keys[i] !== 0) {
@@ -197,18 +221,20 @@ export default function useKeyboard() {
// If we reach here it means we didn't find an empty slot or the key in the buffer
if (overrun) {
if (press) {
- console.warn(`keyboard buffer overflow current keys ${keys}, key: ${key} not added`);
+ console.warn(
+ `keyboard buffer overflow current keys ${keys}, key: ${key} not added`,
+ );
// Fill all key slots with ErrorRollOver (0x01) to indicate overflow
keys.length = hidKeyBufferSize;
keys.fill(hidErrorRollOver);
} else {
// If we are releasing a key, and we didn't find it in a slot, who cares?
- console.debug(`key ${key} not found in buffer, nothing to release`)
+ console.debug(`key ${key} not found in buffer, nothing to release`);
}
}
}
return { modifier: modifiers, keys };
}
- return { handleKeyPress, resetKeyboardState, executeMacro };
+ return { handleKeyPress, resetKeyboardState, executeMacro, cancelExecuteMacro };
}
diff --git a/ui/src/routes/devices.$id.tsx b/ui/src/routes/devices.$id.tsx
index 4318447e..9445e9fd 100644
--- a/ui/src/routes/devices.$id.tsx
+++ b/ui/src/routes/devices.$id.tsx
@@ -580,7 +580,7 @@ export default function KvmIdRoute() {
const { setNetworkState} = useNetworkStateStore();
const { setHdmiState } = useVideoStore();
const {
- keyboardLedState, setKeyboardLedState,
+ keyboardLedState, setKeyboardLedState, setPasteModeEnabled,
keysDownState, setKeysDownState, setUsbState,
} = useHidStore();
@@ -598,6 +598,12 @@ export default function KvmIdRoute() {
setUsbState(usbState);
}
+ if (resp.method === "keyboardReportMultiState") {
+ const reportMultiState = resp.params as unknown as boolean;
+ console.debug("Setting keyboard report multi state", reportMultiState);
+ setPasteModeEnabled(reportMultiState);
+ }
+
if (resp.method === "videoInputState") {
const hdmiState = resp.params as Parameters
[0];
console.debug("Setting HDMI state", hdmiState);
diff --git a/web.go b/web.go
index 21e17e74..7987ebec 100644
--- a/web.go
+++ b/web.go
@@ -198,6 +198,10 @@ func handleWebRTCSession(c *gin.Context) {
_ = peerConn.Close()
}()
}
+
+ // Cancel any ongoing keyboard report multi when session changes
+ cancelKeyboardReportMulti()
+
currentSession = session
c.JSON(http.StatusOK, gin.H{"sd": sd})
}
diff --git a/webrtc.go b/webrtc.go
index c3d0dc1b..43a72f83 100644
--- a/webrtc.go
+++ b/webrtc.go
@@ -266,6 +266,8 @@ func newSession(config SessionConfig) (*Session, error) {
if connectionState == webrtc.ICEConnectionStateClosed {
scopedLogger.Debug().Msg("ICE Connection State is closed, unmounting virtual media")
if session == currentSession {
+ // Cancel any ongoing keyboard report multi when session closes
+ cancelKeyboardReportMulti()
currentSession = nil
}
// Stop RPC processor