state.setMousePosition);
const setMouseMove = useMouseStore((state: MouseState) => state.setMouseMove);
const {
@@ -55,13 +55,14 @@ export default function WebRTCVideo() {
const videoContrast = useSettingsStore((state: SettingsState) => state.videoContrast);
// RTC related states
- const peerConnection = useRTCStore((state: RTCState ) => state.peerConnection);
+ const peerConnection = useRTCStore((state: RTCState) => state.peerConnection);
// HDMI and UI states
const hdmiState = useVideoStore((state: VideoState) => state.hdmiState);
const hdmiError = ["no_lock", "no_signal", "out_of_range"].includes(hdmiState);
const isVideoLoading = !isPlaying;
+ // Mouse wheel states
const [blockWheelEvent, setBlockWheelEvent] = useState(false);
// Misc states and hooks
@@ -104,7 +105,7 @@ export default function WebRTCVideo() {
// Pointer lock and keyboard lock related
const isPointerLockPossible = window.location.protocol === "https:" || window.location.hostname === "localhost";
const isFullscreenEnabled = document.fullscreenEnabled;
-
+
const checkNavigatorPermissions = useCallback(async (permissionName: string) => {
if (!navigator.permissions || !navigator.permissions.query) {
return false; // if can't query permissions, assume NOT granted
@@ -140,11 +141,11 @@ export default function WebRTCVideo() {
if (videoElm.current === null) return;
const isKeyboardLockGranted = await checkNavigatorPermissions("keyboard-lock");
-
+
if (isKeyboardLockGranted && "keyboard" in navigator) {
try {
// @ts-expect-error - keyboard lock is not supported in all browsers
- await navigator.keyboard.lock();
+ await navigator.keyboard.lock();
} catch {
// ignore errors
}
@@ -155,12 +156,12 @@ export default function WebRTCVideo() {
if (videoElm.current === null || document.fullscreenElement !== videoElm.current) return;
if ("keyboard" in navigator) {
- try {
- // @ts-expect-error - keyboard unlock is not supported in all browsers
- await navigator.keyboard.unlock();
- } catch {
- // ignore errors
- }
+ try {
+ // @ts-expect-error - keyboard unlock is not supported in all browsers
+ await navigator.keyboard.unlock();
+ } catch {
+ // ignore errors
+ }
}
}, []);
@@ -188,7 +189,7 @@ export default function WebRTCVideo() {
}, [isPointerLockPossible]);
const requestFullscreen = useCallback(async () => {
- if (!isFullscreenEnabled || !videoElm.current) return;
+ if (!isFullscreenEnabled || !videoElm.current) return;
// per https://wicg.github.io/keyboard-lock/#system-key-press-handler
// If keyboard lock is activated after fullscreen is already in effect, then the user my
@@ -352,12 +353,12 @@ export default function WebRTCVideo() {
// https://bugzilla.mozilla.org/show_bug.cgi?id=1299553
if (e.metaKey && hidKey < 0xE0) {
setTimeout(() => {
- sendKeypressEvent(hidKey, false);
+ handleKeyPress(hidKey, false);
}, 10);
}
- sendKeypressEvent(hidKey, true);
+ handleKeyPress(hidKey, true);
},
- [sendKeypressEvent],
+ [handleKeyPress],
);
const keyUpHandler = useCallback(
@@ -365,15 +366,15 @@ export default function WebRTCVideo() {
e.preventDefault();
const code = getAdjustedKeyCode(e);
const hidKey = keys[code];
-
+
if (hidKey === undefined) {
console.warn(`Key up not mapped: ${code}`);
return;
}
- sendKeypressEvent(hidKey, false);
+ handleKeyPress(hidKey, false);
},
- [sendKeypressEvent],
+ [handleKeyPress],
);
const videoKeyUpHandler = useCallback((e: KeyboardEvent) => {
@@ -489,7 +490,7 @@ export default function WebRTCVideo() {
videoElmRefValue.addEventListener("mousemove", isRelativeMouseMode ? relMouseMoveHandler : absMouseMoveHandler, { signal });
videoElmRefValue.addEventListener("pointerdown", isRelativeMouseMode ? relMouseMoveHandler : absMouseMoveHandler, { signal });
- videoElmRefValue.addEventListener("pointerup", isRelativeMouseMode ? relMouseMoveHandler :absMouseMoveHandler, { signal });
+ videoElmRefValue.addEventListener("pointerup", isRelativeMouseMode ? relMouseMoveHandler : absMouseMoveHandler, { signal });
videoElmRefValue.addEventListener("wheel", mouseWheelHandler, {
signal,
passive: true,
@@ -546,8 +547,8 @@ export default function WebRTCVideo() {
return isDefault
? {} // No filter if all settings are default (1.0)
: {
- filter: `saturate(${videoSaturation}) brightness(${videoBrightness}) contrast(${videoContrast})`,
- };
+ filter: `saturate(${videoSaturation}) brightness(${videoBrightness}) contrast(${videoContrast})`,
+ };
}, [videoSaturation, videoBrightness, videoContrast]);
function getAdjustedKeyCode(e: KeyboardEvent) {
@@ -594,48 +595,48 @@ export default function WebRTCVideo() {
-
- {peerConnection?.connectionState == "connected" && (
-
-
-
-
- {
- videoElm.current?.play();
- }}
- />
-
-
+
+ {peerConnection?.connectionState == "connected" && (
+
+
+
+
+ {
+ videoElm.current?.play();
+ }}
+ />
+
+
+ )}
diff --git a/ui/src/hooks/stores.ts b/ui/src/hooks/stores.ts
index 223d994..c5d714c 100644
--- a/ui/src/hooks/stores.ts
+++ b/ui/src/hooks/stores.ts
@@ -436,27 +436,24 @@ export interface KeyboardLedState {
shift: boolean; // Optional, as not all keyboards have a shift LED
};
+export const hidKeyBufferSize = 6;
+export const hidErrorRollOver = 0x01;
+
export interface KeysDownState {
modifier: number;
keys: number[];
}
export interface HidState {
- altGrArmed: boolean;
- setAltGrArmed: (armed: boolean) => void;
-
- altGrTimer: number | null; // _altGrCtrlTime
- setAltGrTimer: (timeout: number | null) => void;
-
- altGrCtrlTime: number; // _altGrCtrlTime
- setAltGrCtrlTime: (time: number) => void;
-
- keyboardLedState?: KeyboardLedState;
+ keyboardLedState: KeyboardLedState;
setKeyboardLedState: (state: KeyboardLedState) => void;
- keysDownState?: KeysDownState;
+ keysDownState: KeysDownState;
setKeysDownState: (state: KeysDownState) => void;
+ keyPressAvailable: boolean;
+ setKeyPressAvailable: (available: boolean) => void;
+
isVirtualKeyboardEnabled: boolean;
setVirtualKeyboardEnabled: (enabled: boolean) => void;
@@ -468,21 +465,15 @@ export interface HidState {
}
export const useHidStore = create
(set => ({
- altGrArmed: false,
- setAltGrArmed: (armed: boolean): void => set({ altGrArmed: armed }),
-
- altGrTimer: 0,
- setAltGrTimer: (timeout: number | null): void => set({ altGrTimer: timeout }),
-
- altGrCtrlTime: 0,
- setAltGrCtrlTime: (time: number): void => set({ altGrCtrlTime: time }),
-
- keyboardLedState: undefined,
+ keyboardLedState: {} as KeyboardLedState,
setKeyboardLedState: (ledState: KeyboardLedState): void => set({ keyboardLedState: ledState }),
- keysDownState: undefined,
+ keysDownState: { modifier: 0, keys: [0,0,0,0,0,0] } as KeysDownState,
setKeysDownState: (state: KeysDownState): void => set({ keysDownState: state }),
+ keyPressAvailable: true,
+ setKeyPressAvailable: (available: boolean) => set({ keyPressAvailable: available }),
+
isVirtualKeyboardEnabled: false,
setVirtualKeyboardEnabled: (enabled: boolean): void => set({ isVirtualKeyboardEnabled: enabled }),
diff --git a/ui/src/hooks/useKeyboard.ts b/ui/src/hooks/useKeyboard.ts
index a77946c..f98066d 100644
--- a/ui/src/hooks/useKeyboard.ts
+++ b/ui/src/hooks/useKeyboard.ts
@@ -1,24 +1,29 @@
import { useCallback } from "react";
-import { KeysDownState, HidState, useHidStore, RTCState, useRTCStore } from "@/hooks/stores";
+import { KeysDownState, HidState, useHidStore, RTCState, useRTCStore, hidKeyBufferSize, hidErrorRollOver } from "@/hooks/stores";
import { JsonRpcResponse, useJsonRpc } from "@/hooks/useJsonRpc";
-import { keys, modifiers } from "@/keyboardMappings";
+import { hidKeyToModifierMask, keys, modifiers } from "@/keyboardMappings";
export default function useKeyboard() {
const [send] = useJsonRpc();
const rpcDataChannel = useRTCStore((state: RTCState) => state.rpcDataChannel);
+
+ const keysDownState = useHidStore((state: HidState) => state.keysDownState);
const setKeysDownState = useHidStore((state: HidState) => state.setKeysDownState);
- const sendKeyboardEvent = useCallback(
- (keys: number[], modifiers: number[]) => {
- if (rpcDataChannel?.readyState !== "open") return;
- const accModifier = modifiers.reduce((acc, val) => acc + val, 0);
+ const keyPressAvailable = useHidStore((state: HidState) => state.keyPressAvailable);
+ const setKeyPressAvailable = useHidStore((state: HidState) => state.setKeyPressAvailable);
+
+ const sendKeyboardEvent = useCallback(
+ (state: KeysDownState) => {
+ if (rpcDataChannel?.readyState !== "open") return;
- send("keyboardReport", { keys, modifier: accModifier });
//TODO would be nice if the keyboardReport rpc call returned the current state like keypressReport does
+ send("keyboardReport", { keys: state.keys, modifier: state.modifier });
+
// We do this for the info bar to display the currently pressed keys for the user
- setKeysDownState({ keys: keys, modifier: accModifier });
+ setKeysDownState(state);
},
[rpcDataChannel?.readyState, send, setKeysDownState],
);
@@ -28,30 +33,37 @@ export default function useKeyboard() {
if (rpcDataChannel?.readyState !== "open") return;
send("keypressReport", { key, press }, (resp: JsonRpcResponse) => {
- if ("error" in resp) {
- console.error("Failed to send keypress:", resp.error);
- } else {
- const keyDownState = resp.result as KeysDownState;
- // We do this for the info bar to display the currently pressed keys for the user
- setKeysDownState(keyDownState);
- }
- });
+ if ("error" in resp) {
+ // -32601 means the method is not supported
+ if (resp.error.code === -32601) {
+ // if we don't support key press report, we need to disable all that handling
+ console.error("Failed calling keypressReport, switching to local handling", resp.error);
+ setKeyPressAvailable(false);
+ } else {
+ console.error(`Failed to send key ${key} press: ${press}`, resp.error);
+ }
+ } else {
+ const keyDownState = resp.result as KeysDownState;
+ // We do this for the info bar to display the currently pressed keys for the user
+ setKeysDownState(keyDownState);
+ }
+ });
},
- [rpcDataChannel?.readyState, send, setKeysDownState],
+ [rpcDataChannel?.readyState, send, setKeyPressAvailable, setKeysDownState],
);
const resetKeyboardState = useCallback(() => {
- sendKeyboardEvent([], []);
+ sendKeyboardEvent({ keys: [], modifier: 0 });
}, [sendKeyboardEvent]);
const executeMacro = async (steps: { keys: string[] | null; modifiers: string[] | null; delay: number }[]) => {
for (const [index, step] of steps.entries()) {
- const keyValues = step.keys?.map(key => keys[key]).filter(Boolean) || [];
- const modifierValues = step.modifiers?.map(mod => modifiers[mod]).filter(Boolean) || [];
+ const keyValues = (step.keys || []).map(key => keys[key]).filter(Boolean);
+ const modifierMask: number = (step.modifiers || []).map(mod => modifiers[mod]).reduce((acc, val) => acc + val, 0);
// If the step has keys and/or modifiers, press them and hold for the delay
- if (keyValues.length > 0 || modifierValues.length > 0) {
- sendKeyboardEvent(keyValues, modifierValues);
+ if (keyValues.length > 0 || modifierMask > 0) {
+ sendKeyboardEvent({ keys: keyValues, modifier: modifierMask });
await new Promise(resolve => setTimeout(resolve, step.delay || 50));
resetKeyboardState();
@@ -67,5 +79,75 @@ export default function useKeyboard() {
}
};
- return { sendKeyboardEvent, sendKeypressEvent, resetKeyboardState, executeMacro };
+ // this code exists because we have devices that don't support the keysPress api yet (not current)
+ // so we mirror the device-side code here to keep track of the keyboard state
+ function handleKeyLocally(state: KeysDownState, key: number, press: boolean): KeysDownState {
+ const keys = state.keys;
+ let modifiers = state.modifier;
+ const modifierMask = hidKeyToModifierMask[key] || 0;
+
+ if (modifierMask !== 0) {
+ if (press) {
+ modifiers |= modifierMask;
+ } else {
+ modifiers &= ~modifierMask;
+ }
+ } else {
+ // handle other keys that are not modifier keys by placing or removing them
+ // from the key buffer since the buffer tracks currently pressed keys
+ let overrun = true;
+ for (let i = 0; i < hidKeyBufferSize && overrun; i++) {
+ // If we find the key in the buffer the buffer, we either remove it (if press is false)
+ // or do nothing (if down is true) because the buffer tracks currently pressed keys
+ // and if we find a zero byte, we can place the key there (if press is true)
+ if (keys[i] == key || keys[i] == 0) {
+ if (press) {
+ keys[i] = key // overwrites the zero byte or the same key if already pressed
+ } else {
+ // we are releasing the key, remove it from the buffer
+ if (keys[i] != 0) {
+ keys.splice(i, 1);
+ keys.push(0); // add a zero at the end
+ }
+ }
+ overrun = false // We found a slot for the key
+ }
+
+ // If we reach here it means we didn't find an empty slot or the key in the buffer
+ if (overrun) {
+ if (press) {
+ console.warn(`keyboard buffer overflow, key: ${key} not added`);
+ // Fill all key slots with ErrorRollOver (0x01) to indicate overflow
+ keys.length = 6;
+ keys.fill(hidErrorRollOver);
+ } else {
+ // If we are releasing a key, and we didn't find it in a slot, who cares?
+ console.debug(`key ${key} not found in buffer, nothing to release`)
+ }
+ }
+ }
+ }
+ return { modifier: modifiers, keys };
+ }
+
+ const handleKeyPress = useCallback(
+ (key: number, press: boolean) => {
+ if (keyPressAvailable) {
+ // if the keyPress api is available, we can just send the key press event
+ sendKeypressEvent(key, press);
+
+ // TODO handle the case where the keyPress api is not available and we need to handle the key locally now...
+ } else {
+ // if the keyPress api is not available, we need to handle the key locally
+ const newKeysDownState = handleKeyLocally(keysDownState, key, press);
+ setKeysDownState(newKeysDownState);
+
+ // then we send the full state
+ sendKeyboardEvent(newKeysDownState);
+ }
+ },
+ [keyPressAvailable, keysDownState, sendKeyboardEvent, sendKeypressEvent, setKeysDownState],
+ );
+
+ return { handleKeyPress, sendKeyboardEvent, sendKeypressEvent, resetKeyboardState, executeMacro };
}
diff --git a/ui/src/keyboardMappings.ts b/ui/src/keyboardMappings.ts
index 7122254..c91729b 100644
--- a/ui/src/keyboardMappings.ts
+++ b/ui/src/keyboardMappings.ts
@@ -142,6 +142,17 @@ export const modifiers = {
MetaRight: 0x80,
} as Record;
+export const hidKeyToModifierMask = {
+ 0xe0: modifiers.ControlLeft,
+ 0xe1: modifiers.ShiftLeft,
+ 0xe2: modifiers.AltLeft,
+ 0xe3: modifiers.MetaLeft,
+ 0xe4: modifiers.ControlRight,
+ 0xe5: modifiers.ShiftRight,
+ 0xe6: modifiers.AltRight,
+ 0xe7: modifiers.MetaRight,
+} as Record;
+
export const modifierDisplayMap: Record = {
ControlLeft: "Left Ctrl",
ControlRight: "Right Ctrl",
diff --git a/ui/src/routes/devices.$id.settings.tsx b/ui/src/routes/devices.$id.settings.tsx
index 5075ab5..09703c3 100644
--- a/ui/src/routes/devices.$id.settings.tsx
+++ b/ui/src/routes/devices.$id.settings.tsx
@@ -29,7 +29,7 @@ import { cx } from "../cva.config";
export default function SettingsRoute() {
const location = useLocation();
const setDisableVideoFocusTrap = useUiStore(state => state.setDisableVideoFocusTrap);
- const { sendKeyboardEvent } = useKeyboard();
+ const { resetKeyboardState } = useKeyboard();
const scrollContainerRef = useRef(null);
const [showLeftGradient, setShowLeftGradient] = useState(false);
const [showRightGradient, setShowRightGradient] = useState(false);
@@ -67,8 +67,8 @@ export default function SettingsRoute() {
useEffect(() => {
// disable focus trap
setTimeout(() => {
- // Reset keyboard state. Incase the user is pressing a key while enabling the sidebar
- sendKeyboardEvent([], []);
+ // Reset keyboard state. In case the user is pressing a key while enabling the sidebar
+ resetKeyboardState();
setDisableVideoFocusTrap(true);
// For some reason, the focus trap is not disabled immediately
// so we need to blur the active element
@@ -79,7 +79,7 @@ export default function SettingsRoute() {
return () => {
setDisableVideoFocusTrap(false);
};
- }, [sendKeyboardEvent, setDisableVideoFocusTrap]);
+ }, [resetKeyboardState, setDisableVideoFocusTrap]);
return (
diff --git a/ui/src/routes/devices.$id.tsx b/ui/src/routes/devices.$id.tsx
index 53177c5..efc52df 100644
--- a/ui/src/routes/devices.$id.tsx
+++ b/ui/src/routes/devices.$id.tsx
@@ -519,7 +519,7 @@ export default function KvmIdRoute() {
// Cleanup effect
const clearInboundRtpStats = useRTCStore((state: RTCState) => state.clearInboundRtpStats);
const clearCandidatePairStats = useRTCStore((state: RTCState) => state.clearCandidatePairStats);
- const setSidebarView = useUiStore((state: UIState) => state.setSidebarView);
+ const setSidebarView = useUiStore((state: UIState) => state.setSidebarView);
useEffect(() => {
return () => {
@@ -597,6 +597,7 @@ export default function KvmIdRoute() {
const keysDownState = useHidStore((state: HidState) => state.keysDownState);
const setKeysDownState = useHidStore((state: HidState) => state.setKeysDownState);
+ const setKeyPressAvailable = useHidStore((state: HidState) => state.setKeyPressAvailable);
const [hasUpdated, setHasUpdated] = useState(false);
const { navigateTo } = useDeviceUiNavigation();
@@ -624,7 +625,7 @@ export default function KvmIdRoute() {
console.log("Setting keyboard led state", ledState);
setKeyboardLedState(ledState);
}
-
+
if (resp.method === "keysDownState") {
const downState = resp.params as KeysDownState;
console.log("Setting key down state", downState);
@@ -667,10 +668,12 @@ export default function KvmIdRoute() {
});
}, [rpcDataChannel?.readyState, send, setHdmiState]);
+ const [needLedState, setNeedLedState] = useState(true);
+
// request keyboard led state from the device
useEffect(() => {
if (rpcDataChannel?.readyState !== "open") return;
- if (keyboardLedState !== undefined) return;
+ if (!needLedState) return;
console.log("Requesting keyboard led state");
send("getKeyboardLedState", {}, (resp: JsonRpcResponse) => {
@@ -680,24 +683,35 @@ export default function KvmIdRoute() {
}
console.log("Keyboard led state", resp.result);
setKeyboardLedState(resp.result as KeyboardLedState);
+ setNeedLedState(false);
});
- }, [rpcDataChannel?.readyState, send, setKeyboardLedState, keyboardLedState]);
+ }, [rpcDataChannel?.readyState, send, setKeyboardLedState, keyboardLedState, needLedState]);
+
+ const [needKeyDownState, setNeedKeyDownState] = useState(true);
// request keyboard key down state from the device
useEffect(() => {
if (rpcDataChannel?.readyState !== "open") return;
- if (keysDownState !== undefined) return;
+ if (!needKeyDownState) return;
console.log("Requesting keys down state");
send("getKeyDownState", {}, (resp: JsonRpcResponse) => {
if ("error" in resp) {
- console.error("Failed to get key down state", resp.error);
+ // -32601 means the method is not supported
+ if (resp.error.code === -32601) {
+ // if we don't support key down state, we know key press is also not available
+ console.error("Failed to get key down state, switching to old-school", resp.error);
+ setKeyPressAvailable(false);
+ } else {
+ console.error("Failed to get key down state", resp.error);
+ }
return;
}
console.log("Keyboard key down state", resp.result);
setKeysDownState(resp.result as KeysDownState);
+ setNeedKeyDownState(false);
});
- }, [keysDownState, rpcDataChannel?.readyState, send, setKeysDownState]);
+ }, [keysDownState, needKeyDownState, rpcDataChannel?.readyState, send, setKeyPressAvailable, setKeysDownState]);
// When the update is successful, we need to refresh the client javascript and show a success modal
useEffect(() => {
@@ -758,7 +772,7 @@ export default function KvmIdRoute() {
send("getUpdateStatus", {}, async (resp: JsonRpcResponse) => {
if ("error" in resp) {
notifications.error(`Failed to get device version: ${resp.error}`);
- return
+ return
}
const result = resp.result as SystemVersionInfo;
@@ -895,7 +909,7 @@ interface SidebarContainerProps {
}
function SidebarContainer(props: SidebarContainerProps) {
- const { sidebarView }= props;
+ const { sidebarView } = props;
return (