Superminaren
/
kvm
mirror of https://github.com/jetkvm/kvm.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Enable still working with devices that haven't been upgraded

This commit is contained in:
Marc Brooks 2025-08-14 19:39:25 -05:00
parent 1e57f4bf4f
commit 894e66efaa
No known key found for this signature in database
GPG Key ID: 583A6AF2D6AE1DC6
8 changed files with 258 additions and 177 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
ui/src/components/InfoBar.tsx
View File

@ -133,7 +133,7 @@ export default function InfoBar() {
<div <div
className={cx( className={cx(
"shrink-0 p-1 px-1.5 text-xs", "shrink-0 p-1 px-1.5 text-xs",
keyboardLedState?.caps_lock keyboardLedState.caps_lock
? "text-black dark:text-white" ? "text-black dark:text-white"
: "text-slate-800/20 dark:text-slate-300/20", : "text-slate-800/20 dark:text-slate-300/20",
)} )}
@ -143,7 +143,7 @@ export default function InfoBar() {
<div <div
className={cx( className={cx(
"shrink-0 p-1 px-1.5 text-xs", "shrink-0 p-1 px-1.5 text-xs",
keyboardLedState?.num_lock keyboardLedState.num_lock
? "text-black dark:text-white" ? "text-black dark:text-white"
: "text-slate-800/20 dark:text-slate-300/20", : "text-slate-800/20 dark:text-slate-300/20",
)} )}
@ -153,24 +153,24 @@ export default function InfoBar() {
<div <div
className={cx( className={cx(
"shrink-0 p-1 px-1.5 text-xs", "shrink-0 p-1 px-1.5 text-xs",
keyboardLedState?.scroll_lock keyboardLedState.scroll_lock
? "text-black dark:text-white" ? "text-black dark:text-white"
: "text-slate-800/20 dark:text-slate-300/20", : "text-slate-800/20 dark:text-slate-300/20",
)} )}
> >
Scroll Lock Scroll Lock
</div> </div>
{keyboardLedState?.compose ? ( {keyboardLedState.compose ? (
<div className="shrink-0 p-1 px-1.5 text-xs"> <div className="shrink-0 p-1 px-1.5 text-xs">
Compose Compose
</div> </div>
) : null} ) : null}
{keyboardLedState?.kana ? ( {keyboardLedState.kana ? (
<div className="shrink-0 p-1 px-1.5 text-xs"> <div className="shrink-0 p-1 px-1.5 text-xs">
Kana Kana
</div> </div>
) : null} ) : null}
{keyboardLedState?.shift ? ( {keyboardLedState.shift ? (
<div className="shrink-0 p-1 px-1.5 text-xs"> <div className="shrink-0 p-1 px-1.5 text-xs">
Shift Shift
</div> </div>

84
ui/src/components/VirtualKeyboard.tsx
View File

@ -1,4 +1,3 @@
import { useShallow } from "zustand/react/shallow";
import { ChevronDownIcon } from "@heroicons/react/16/solid"; import { ChevronDownIcon } from "@heroicons/react/16/solid";
import { AnimatePresence, motion } from "framer-motion"; import { AnimatePresence, motion } from "framer-motion";
import { useCallback, useEffect, useRef, useState } from "react"; import { useCallback, useEffect, useRef, useState } from "react";
@ -13,7 +12,7 @@ import "react-simple-keyboard/build/css/index.css";
import AttachIconRaw from "@/assets/attach-icon.svg"; import AttachIconRaw from "@/assets/attach-icon.svg";
import DetachIconRaw from "@/assets/detach-icon.svg"; import DetachIconRaw from "@/assets/detach-icon.svg";
import { cx } from "@/cva.config"; import { cx } from "@/cva.config";
import { useHidStore, useUiStore } from "@/hooks/stores"; import { HidState, useHidStore, useUiStore } from "@/hooks/stores";
import useKeyboard from "@/hooks/useKeyboard"; import useKeyboard from "@/hooks/useKeyboard";
import { keyDisplayMap, keys, modifiers } from "@/keyboardMappings"; import { keyDisplayMap, keys, modifiers } from "@/keyboardMappings";
@ -26,7 +25,7 @@ const AttachIcon = ({ className }: { className?: string }) => {
}; };
function KeyboardWrapper() { function KeyboardWrapper() {
const [layoutName, setLayoutName] = useState("default"); const [layoutName] = useState("default");
const keyboardRef = useRef<HTMLDivElement>(null); const keyboardRef = useRef<HTMLDivElement>(null);
const showAttachedVirtualKeyboard = useUiStore( const showAttachedVirtualKeyboard = useUiStore(
@ -36,14 +35,16 @@ function KeyboardWrapper() {
state => state.setAttachedVirtualKeyboardVisibility, state => state.setAttachedVirtualKeyboardVisibility,
); );
const { sendKeyboardEvent, resetKeyboardState } = useKeyboard(); const virtualKeyboard = useHidStore(state => state.isVirtualKeyboardEnabled);
const setVirtualKeyboard = useHidStore(state => state.setVirtualKeyboardEnabled);
const keysDownState = useHidStore((state: HidState) => state.keysDownState);
const { handleKeyPress, sendKeyboardEvent, resetKeyboardState } = useKeyboard();
const [isDragging, setIsDragging] = useState(false); const [isDragging, setIsDragging] = useState(false);
const [position, setPosition] = useState({ x: 0, y: 0 }); const [position, setPosition] = useState({ x: 0, y: 0 });
const [newPosition, setNewPosition] = useState({ x: 0, y: 0 }); const [newPosition, setNewPosition] = useState({ x: 0, y: 0 });
const isCapsLockActive = useHidStore(useShallow(state => state.keyboardLedState?.caps_lock));
/* /*
// These will be used to display the currently pressed keys and modifiers on the virtual keyboard // These will be used to display the currently pressed keys and modifiers on the virtual keyboard
@ -129,74 +130,55 @@ function KeyboardWrapper() {
const onKeyDown = useCallback( const onKeyDown = useCallback(
(key: string) => { (key: string) => {
const isKeyShift = key === "{shift}" || key === "ShiftLeft" || key === "ShiftRight"; const latchingKeys = ["CapsLock", "ScrollLock", "NumLock", "Meta", "Compose", "Kana"];
const isKeyCaps = key === "CapsLock"; const dynamicKeys = ["ShiftLeft", "ShiftRight", "ControlLeft", "ControlRight", "AltLeft", "AltRight", "MetaLeft", "MetaRight"];
const cleanKey = key.replace(/[()]/g, "");
const keyHasShiftModifier = key.includes("(");
// Handle toggle of layout for shift or caps lock
const toggleLayout = () => {
setLayoutName(prevLayout => (prevLayout === "default" ? "shift" : "default"));
};
if (key === "CtrlAltDelete") { if (key === "CtrlAltDelete") {
sendKeyboardEvent( sendKeyboardEvent({ keys: [keys.Delete], modifier: modifiers.ControlLeft | modifiers.AltLeft });
[keys["Delete"]],
[modifiers["ControlLeft"], modifiers["AltLeft"]],
);
setTimeout(resetKeyboardState, 100); setTimeout(resetKeyboardState, 100);
return; return;
} }
if (key === "AltMetaEscape") { if (key === "AltMetaEscape") {
sendKeyboardEvent( sendKeyboardEvent({ keys: [keys.Escape], modifier: modifiers.AltLeft | modifiers.MetaLeft });
[keys["Escape"]],
[modifiers["MetaLeft"], modifiers["AltLeft"]],
);
setTimeout(resetKeyboardState, 100); setTimeout(resetKeyboardState, 100);
return; return;
} }
if (key === "CtrlAltBackspace") { if (key === "CtrlAltBackspace") {
sendKeyboardEvent( sendKeyboardEvent({ keys: [keys.Backspace], modifier: modifiers.ControlLeft | modifiers.AltLeft });
[keys["Backspace"]],
[modifiers["ControlLeft"], modifiers["AltLeft"]],
);
setTimeout(resetKeyboardState, 100); setTimeout(resetKeyboardState, 100);
return; return;
} }
if (isKeyShift || isKeyCaps) { // if they press any of the latching keys, we send a keypress down event and the release it automatically (on timer)
toggleLayout(); if (latchingKeys.includes(key)) {
handleKeyPress(keys[key], true)
if (isCapsLockActive) { setTimeout(() => handleKeyPress(keys[key], false), 100);
sendKeyboardEvent([keys["CapsLock"]], []); return;
return;
}
} }
// Collect new active keys and modifiers // if they press any of the dynamic keys, we send a keypress down event but we don't release it until they click it again
const newKeys = keys[cleanKey] ? [keys[cleanKey]] : []; if (dynamicKeys.includes(key)) {
const newModifiers = const currentlyDown = keysDownState.keys.includes(keys[key]);
keyHasShiftModifier && !isCapsLockActive ? [modifiers["ShiftLeft"]] : []; handleKeyPress(keys[key], !currentlyDown)
return;
// Update current keys and modifiers
sendKeyboardEvent(newKeys, newModifiers);
// If shift was used as a modifier and caps lock is not active, revert to default layout
if (keyHasShiftModifier && !isCapsLockActive) {
setLayoutName("default");
} }
setTimeout(resetKeyboardState, 100); // otherwise, just treat it as a down+up pair
const cleanKey = key.replace(/[()]/g, "");
handleKeyPress(keys[cleanKey], true);
setTimeout(() => handleKeyPress(keys[cleanKey], false), 50);
}, },
[isCapsLockActive, sendKeyboardEvent, resetKeyboardState], [handleKeyPress, sendKeyboardEvent, resetKeyboardState, keysDownState],
); );
const virtualKeyboard = useHidStore(state => state.isVirtualKeyboardEnabled); // TODO handle the display of down keys and the layout change for shift/caps lock
const setVirtualKeyboard = useHidStore(state => state.setVirtualKeyboardEnabled); // const isCapsLockActive = useHidStore(useShallow(state => state.keyboardLedState.caps_lock));
// // Handle toggle of layout for shift or caps lock
// const toggleLayout = () => {
// setLayoutName(prevLayout => (prevLayout === "default" ? "shift" : "default"));
// };
return ( return (
<div <div

125
ui/src/components/WebRTCVideo.tsx
View File

@ -37,7 +37,7 @@ export default function WebRTCVideo() {
const [isPointerLockActive, setIsPointerLockActive] = useState(false); const [isPointerLockActive, setIsPointerLockActive] = useState(false);
// Store hooks // Store hooks
const settings = useSettingsStore(); const settings = useSettingsStore();
const { sendKeypressEvent, resetKeyboardState } = useKeyboard(); const { handleKeyPress, resetKeyboardState } = useKeyboard();
const setMousePosition = useMouseStore((state: MouseState) => state.setMousePosition); const setMousePosition = useMouseStore((state: MouseState) => state.setMousePosition);
const setMouseMove = useMouseStore((state: MouseState) => state.setMouseMove); const setMouseMove = useMouseStore((state: MouseState) => state.setMouseMove);
const { const {
@ -55,13 +55,14 @@ export default function WebRTCVideo() {
const videoContrast = useSettingsStore((state: SettingsState) => state.videoContrast); const videoContrast = useSettingsStore((state: SettingsState) => state.videoContrast);
// RTC related states // RTC related states
const peerConnection = useRTCStore((state: RTCState ) => state.peerConnection); const peerConnection = useRTCStore((state: RTCState) => state.peerConnection);
// HDMI and UI states // HDMI and UI states
const hdmiState = useVideoStore((state: VideoState) => state.hdmiState); const hdmiState = useVideoStore((state: VideoState) => state.hdmiState);
const hdmiError = ["no_lock", "no_signal", "out_of_range"].includes(hdmiState); const hdmiError = ["no_lock", "no_signal", "out_of_range"].includes(hdmiState);
const isVideoLoading = !isPlaying; const isVideoLoading = !isPlaying;
// Mouse wheel states
const [blockWheelEvent, setBlockWheelEvent] = useState(false); const [blockWheelEvent, setBlockWheelEvent] = useState(false);
// Misc states and hooks // Misc states and hooks
@ -104,7 +105,7 @@ export default function WebRTCVideo() {
// Pointer lock and keyboard lock related // Pointer lock and keyboard lock related
const isPointerLockPossible = window.location.protocol === "https:" || window.location.hostname === "localhost"; const isPointerLockPossible = window.location.protocol === "https:" || window.location.hostname === "localhost";
const isFullscreenEnabled = document.fullscreenEnabled; const isFullscreenEnabled = document.fullscreenEnabled;
const checkNavigatorPermissions = useCallback(async (permissionName: string) => { const checkNavigatorPermissions = useCallback(async (permissionName: string) => {
if (!navigator.permissions || !navigator.permissions.query) { if (!navigator.permissions || !navigator.permissions.query) {
return false; // if can't query permissions, assume NOT granted return false; // if can't query permissions, assume NOT granted
@ -140,11 +141,11 @@ export default function WebRTCVideo() {
if (videoElm.current === null) return; if (videoElm.current === null) return;
const isKeyboardLockGranted = await checkNavigatorPermissions("keyboard-lock"); const isKeyboardLockGranted = await checkNavigatorPermissions("keyboard-lock");
if (isKeyboardLockGranted && "keyboard" in navigator) { if (isKeyboardLockGranted && "keyboard" in navigator) {
try { try {
// @ts-expect-error - keyboard lock is not supported in all browsers // @ts-expect-error - keyboard lock is not supported in all browsers
await navigator.keyboard.lock(); await navigator.keyboard.lock();
} catch { } catch {
// ignore errors // ignore errors
} }
@ -155,12 +156,12 @@ export default function WebRTCVideo() {
if (videoElm.current === null || document.fullscreenElement !== videoElm.current) return; if (videoElm.current === null || document.fullscreenElement !== videoElm.current) return;
if ("keyboard" in navigator) { if ("keyboard" in navigator) {
try { try {
// @ts-expect-error - keyboard unlock is not supported in all browsers // @ts-expect-error - keyboard unlock is not supported in all browsers
await navigator.keyboard.unlock(); await navigator.keyboard.unlock();
} catch { } catch {
// ignore errors // ignore errors
} }
} }
}, []); }, []);
@ -188,7 +189,7 @@ export default function WebRTCVideo() {
}, [isPointerLockPossible]); }, [isPointerLockPossible]);
const requestFullscreen = useCallback(async () => { const requestFullscreen = useCallback(async () => {
if (!isFullscreenEnabled || !videoElm.current) return; if (!isFullscreenEnabled || !videoElm.current) return;
// per https://wicg.github.io/keyboard-lock/#system-key-press-handler // per https://wicg.github.io/keyboard-lock/#system-key-press-handler
// If keyboard lock is activated after fullscreen is already in effect, then the user my // If keyboard lock is activated after fullscreen is already in effect, then the user my
@ -352,12 +353,12 @@ export default function WebRTCVideo() {
// https://bugzilla.mozilla.org/show_bug.cgi?id=1299553 // https://bugzilla.mozilla.org/show_bug.cgi?id=1299553
if (e.metaKey && hidKey < 0xE0) { if (e.metaKey && hidKey < 0xE0) {
setTimeout(() => { setTimeout(() => {
sendKeypressEvent(hidKey, false); handleKeyPress(hidKey, false);
}, 10); }, 10);
} }
sendKeypressEvent(hidKey, true); handleKeyPress(hidKey, true);
}, },
[sendKeypressEvent], [handleKeyPress],
); );
const keyUpHandler = useCallback( const keyUpHandler = useCallback(
@ -365,15 +366,15 @@ export default function WebRTCVideo() {
e.preventDefault(); e.preventDefault();
const code = getAdjustedKeyCode(e); const code = getAdjustedKeyCode(e);
const hidKey = keys[code]; const hidKey = keys[code];
if (hidKey === undefined) { if (hidKey === undefined) {
console.warn(`Key up not mapped: ${code}`); console.warn(`Key up not mapped: ${code}`);
return; return;
} }
sendKeypressEvent(hidKey, false); handleKeyPress(hidKey, false);
}, },
[sendKeypressEvent], [handleKeyPress],
); );
const videoKeyUpHandler = useCallback((e: KeyboardEvent) => { const videoKeyUpHandler = useCallback((e: KeyboardEvent) => {
@ -489,7 +490,7 @@ export default function WebRTCVideo() {
videoElmRefValue.addEventListener("mousemove", isRelativeMouseMode ? relMouseMoveHandler : absMouseMoveHandler, { signal }); videoElmRefValue.addEventListener("mousemove", isRelativeMouseMode ? relMouseMoveHandler : absMouseMoveHandler, { signal });
videoElmRefValue.addEventListener("pointerdown", isRelativeMouseMode ? relMouseMoveHandler : absMouseMoveHandler, { signal }); videoElmRefValue.addEventListener("pointerdown", isRelativeMouseMode ? relMouseMoveHandler : absMouseMoveHandler, { signal });
videoElmRefValue.addEventListener("pointerup", isRelativeMouseMode ? relMouseMoveHandler :absMouseMoveHandler, { signal }); videoElmRefValue.addEventListener("pointerup", isRelativeMouseMode ? relMouseMoveHandler : absMouseMoveHandler, { signal });
videoElmRefValue.addEventListener("wheel", mouseWheelHandler, { videoElmRefValue.addEventListener("wheel", mouseWheelHandler, {
signal, signal,
passive: true, passive: true,
@ -546,8 +547,8 @@ export default function WebRTCVideo() {
return isDefault return isDefault
? {} // No filter if all settings are default (1.0) ? {} // No filter if all settings are default (1.0)
: { : {
filter: `saturate(${videoSaturation}) brightness(${videoBrightness}) contrast(${videoContrast})`, filter: `saturate(${videoSaturation}) brightness(${videoBrightness}) contrast(${videoContrast})`,
}; };
}, [videoSaturation, videoBrightness, videoContrast]); }, [videoSaturation, videoBrightness, videoContrast]);
function getAdjustedKeyCode(e: KeyboardEvent) { function getAdjustedKeyCode(e: KeyboardEvent) {
@ -594,48 +595,48 @@ export default function WebRTCVideo() {
<PointerLockBar show={showPointerLockBar} /> <PointerLockBar show={showPointerLockBar} />
<div className="relative mx-4 my-2 flex items-center justify-center overflow-hidden"> <div className="relative mx-4 my-2 flex items-center justify-center overflow-hidden">
<div className="relative flex h-full w-full items-center justify-center"> <div className="relative flex h-full w-full items-center justify-center">
<video <video
ref={videoElm} ref={videoElm}
autoPlay autoPlay
controls={false} controls={false}
onPlaying={onVideoPlaying} onPlaying={onVideoPlaying}
onPlay={onVideoPlaying} onPlay={onVideoPlaying}
muted muted
playsInline playsInline
disablePictureInPicture disablePictureInPicture
controlsList="nofullscreen" controlsList="nofullscreen"
style={videoStyle} style={videoStyle}
className={cx( className={cx(
"max-h-full min-h-[384px] max-w-full min-w-[512px] bg-black/50 object-contain transition-all duration-1000", "max-h-full min-h-[384px] max-w-full min-w-[512px] bg-black/50 object-contain transition-all duration-1000",
{ {
"cursor-none": settings.isCursorHidden, "cursor-none": settings.isCursorHidden,
"opacity-0": "opacity-0":
isVideoLoading || isVideoLoading ||
hdmiError || hdmiError ||
peerConnectionState !== "connected", peerConnectionState !== "connected",
"opacity-60!": showPointerLockBar, "opacity-60!": showPointerLockBar,
"animate-slideUpFade border border-slate-800/30 shadow-xs dark:border-slate-300/20": "animate-slideUpFade border border-slate-800/30 shadow-xs dark:border-slate-300/20":
isPlaying, isPlaying,
}, },
)}
/>
{peerConnection?.connectionState == "connected" && (
<div
style={{ animationDuration: "500ms" }}
className="animate-slideUpFade pointer-events-none absolute inset-0 flex items-center justify-center"
>
<div className="relative h-full w-full rounded-md">
<LoadingVideoOverlay show={isVideoLoading} />
<HDMIErrorOverlay show={hdmiError} hdmiState={hdmiState} />
<NoAutoplayPermissionsOverlay
show={hasNoAutoPlayPermissions}
onPlayClick={() => {
videoElm.current?.play();
}}
/>
</div>
</div>
)} )}
/>
{peerConnection?.connectionState == "connected" && (
<div
style={{ animationDuration: "500ms" }}
className="animate-slideUpFade pointer-events-none absolute inset-0 flex items-center justify-center"
>
<div className="relative h-full w-full rounded-md">
<LoadingVideoOverlay show={isVideoLoading} />
<HDMIErrorOverlay show={hdmiError} hdmiState={hdmiState} />
<NoAutoplayPermissionsOverlay
show={hasNoAutoPlayPermissions}
onPlayClick={() => {
videoElm.current?.play();
}}
/>
</div>
</div>
)}
</div> </div>
</div> </div>
<VirtualKeyboard /> <VirtualKeyboard />

35
ui/src/hooks/stores.ts
View File

@ -436,27 +436,24 @@ export interface KeyboardLedState {
shift: boolean; // Optional, as not all keyboards have a shift LED shift: boolean; // Optional, as not all keyboards have a shift LED
}; };
export const hidKeyBufferSize = 6;
export const hidErrorRollOver = 0x01;
export interface KeysDownState { export interface KeysDownState {
modifier: number; modifier: number;
keys: number[]; keys: number[];
} }
export interface HidState { export interface HidState {
altGrArmed: boolean; keyboardLedState: KeyboardLedState;
setAltGrArmed: (armed: boolean) => void;
altGrTimer: number | null; // _altGrCtrlTime
setAltGrTimer: (timeout: number | null) => void;
altGrCtrlTime: number; // _altGrCtrlTime
setAltGrCtrlTime: (time: number) => void;
keyboardLedState?: KeyboardLedState;
setKeyboardLedState: (state: KeyboardLedState) => void; setKeyboardLedState: (state: KeyboardLedState) => void;
keysDownState?: KeysDownState; keysDownState: KeysDownState;
setKeysDownState: (state: KeysDownState) => void; setKeysDownState: (state: KeysDownState) => void;
keyPressAvailable: boolean;
setKeyPressAvailable: (available: boolean) => void;
isVirtualKeyboardEnabled: boolean; isVirtualKeyboardEnabled: boolean;
setVirtualKeyboardEnabled: (enabled: boolean) => void; setVirtualKeyboardEnabled: (enabled: boolean) => void;
@ -468,21 +465,15 @@ export interface HidState {
} }
export const useHidStore = create<HidState>(set => ({ export const useHidStore = create<HidState>(set => ({
altGrArmed: false, keyboardLedState: {} as KeyboardLedState,
setAltGrArmed: (armed: boolean): void => set({ altGrArmed: armed }),
altGrTimer: 0,
setAltGrTimer: (timeout: number | null): void => set({ altGrTimer: timeout }),
altGrCtrlTime: 0,
setAltGrCtrlTime: (time: number): void => set({ altGrCtrlTime: time }),
keyboardLedState: undefined,
setKeyboardLedState: (ledState: KeyboardLedState): void => set({ keyboardLedState: ledState }), setKeyboardLedState: (ledState: KeyboardLedState): void => set({ keyboardLedState: ledState }),
keysDownState: undefined, keysDownState: { modifier: 0, keys: [0,0,0,0,0,0] } as KeysDownState,
setKeysDownState: (state: KeysDownState): void => set({ keysDownState: state }), setKeysDownState: (state: KeysDownState): void => set({ keysDownState: state }),
keyPressAvailable: true,
setKeyPressAvailable: (available: boolean) => set({ keyPressAvailable: available }),
isVirtualKeyboardEnabled: false, isVirtualKeyboardEnabled: false,
setVirtualKeyboardEnabled: (enabled: boolean): void => set({ isVirtualKeyboardEnabled: enabled }), setVirtualKeyboardEnabled: (enabled: boolean): void => set({ isVirtualKeyboardEnabled: enabled }),

128
ui/src/hooks/useKeyboard.ts
View File

@ -1,24 +1,29 @@
import { useCallback } from "react"; import { useCallback } from "react";
import { KeysDownState, HidState, useHidStore, RTCState, useRTCStore } from "@/hooks/stores"; import { KeysDownState, HidState, useHidStore, RTCState, useRTCStore, hidKeyBufferSize, hidErrorRollOver } from "@/hooks/stores";
import { JsonRpcResponse, useJsonRpc } from "@/hooks/useJsonRpc"; import { JsonRpcResponse, useJsonRpc } from "@/hooks/useJsonRpc";
import { keys, modifiers } from "@/keyboardMappings"; import { hidKeyToModifierMask, keys, modifiers } from "@/keyboardMappings";
export default function useKeyboard() { export default function useKeyboard() {
const [send] = useJsonRpc(); const [send] = useJsonRpc();
const rpcDataChannel = useRTCStore((state: RTCState) => state.rpcDataChannel); const rpcDataChannel = useRTCStore((state: RTCState) => state.rpcDataChannel);
const keysDownState = useHidStore((state: HidState) => state.keysDownState);
const setKeysDownState = useHidStore((state: HidState) => state.setKeysDownState); const setKeysDownState = useHidStore((state: HidState) => state.setKeysDownState);
const sendKeyboardEvent = useCallback( const keyPressAvailable = useHidStore((state: HidState) => state.keyPressAvailable);
(keys: number[], modifiers: number[]) => { const setKeyPressAvailable = useHidStore((state: HidState) => state.setKeyPressAvailable);
if (rpcDataChannel?.readyState !== "open") return;
const accModifier = modifiers.reduce((acc, val) => acc + val, 0); const sendKeyboardEvent = useCallback(
(state: KeysDownState) => {
if (rpcDataChannel?.readyState !== "open") return;
send("keyboardReport", { keys, modifier: accModifier });
//TODO would be nice if the keyboardReport rpc call returned the current state like keypressReport does //TODO would be nice if the keyboardReport rpc call returned the current state like keypressReport does
send("keyboardReport", { keys: state.keys, modifier: state.modifier });
// We do this for the info bar to display the currently pressed keys for the user // We do this for the info bar to display the currently pressed keys for the user
setKeysDownState({ keys: keys, modifier: accModifier }); setKeysDownState(state);
}, },
[rpcDataChannel?.readyState, send, setKeysDownState], [rpcDataChannel?.readyState, send, setKeysDownState],
); );
@ -28,30 +33,37 @@ export default function useKeyboard() {
if (rpcDataChannel?.readyState !== "open") return; if (rpcDataChannel?.readyState !== "open") return;
send("keypressReport", { key, press }, (resp: JsonRpcResponse) => { send("keypressReport", { key, press }, (resp: JsonRpcResponse) => {
if ("error" in resp) { if ("error" in resp) {
console.error("Failed to send keypress:", resp.error); // -32601 means the method is not supported
} else { if (resp.error.code === -32601) {
const keyDownState = resp.result as KeysDownState; // if we don't support key press report, we need to disable all that handling
// We do this for the info bar to display the currently pressed keys for the user console.error("Failed calling keypressReport, switching to local handling", resp.error);
setKeysDownState(keyDownState); setKeyPressAvailable(false);
} } else {
}); console.error(`Failed to send key ${key} press: ${press}`, resp.error);
}
} else {
const keyDownState = resp.result as KeysDownState;
// We do this for the info bar to display the currently pressed keys for the user
setKeysDownState(keyDownState);
}
});
}, },
[rpcDataChannel?.readyState, send, setKeysDownState], [rpcDataChannel?.readyState, send, setKeyPressAvailable, setKeysDownState],
); );
const resetKeyboardState = useCallback(() => { const resetKeyboardState = useCallback(() => {
sendKeyboardEvent([], []); sendKeyboardEvent({ keys: [], modifier: 0 });
}, [sendKeyboardEvent]); }, [sendKeyboardEvent]);
const executeMacro = async (steps: { keys: string[] | null; modifiers: string[] | null; delay: number }[]) => { const executeMacro = async (steps: { keys: string[] | null; modifiers: string[] | null; delay: number }[]) => {
for (const [index, step] of steps.entries()) { for (const [index, step] of steps.entries()) {
const keyValues = step.keys?.map(key => keys[key]).filter(Boolean) || []; const keyValues = (step.keys || []).map(key => keys[key]).filter(Boolean);
const modifierValues = step.modifiers?.map(mod => modifiers[mod]).filter(Boolean) || []; const modifierMask: number = (step.modifiers || []).map(mod => modifiers[mod]).reduce((acc, val) => acc + val, 0);
// If the step has keys and/or modifiers, press them and hold for the delay // If the step has keys and/or modifiers, press them and hold for the delay
if (keyValues.length > 0 || modifierValues.length > 0) { if (keyValues.length > 0 || modifierMask > 0) {
sendKeyboardEvent(keyValues, modifierValues); sendKeyboardEvent({ keys: keyValues, modifier: modifierMask });
await new Promise(resolve => setTimeout(resolve, step.delay || 50)); await new Promise(resolve => setTimeout(resolve, step.delay || 50));
resetKeyboardState(); resetKeyboardState();
@ -67,5 +79,75 @@ export default function useKeyboard() {
} }
}; };
return { sendKeyboardEvent, sendKeypressEvent, resetKeyboardState, executeMacro }; // this code exists because we have devices that don't support the keysPress api yet (not current)
// so we mirror the device-side code here to keep track of the keyboard state
function handleKeyLocally(state: KeysDownState, key: number, press: boolean): KeysDownState {
const keys = state.keys;
let modifiers = state.modifier;
const modifierMask = hidKeyToModifierMask[key] || 0;
if (modifierMask !== 0) {
if (press) {
modifiers |= modifierMask;
} else {
modifiers &= ~modifierMask;
}
} else {
// handle other keys that are not modifier keys by placing or removing them
// from the key buffer since the buffer tracks currently pressed keys
let overrun = true;
for (let i = 0; i < hidKeyBufferSize && overrun; i++) {
// If we find the key in the buffer the buffer, we either remove it (if press is false)
// or do nothing (if down is true) because the buffer tracks currently pressed keys
// and if we find a zero byte, we can place the key there (if press is true)
if (keys[i] == key || keys[i] == 0) {
if (press) {
keys[i] = key // overwrites the zero byte or the same key if already pressed
} else {
// we are releasing the key, remove it from the buffer
if (keys[i] != 0) {
keys.splice(i, 1);
keys.push(0); // add a zero at the end
}
}
overrun = false // We found a slot for the key
}
// If we reach here it means we didn't find an empty slot or the key in the buffer
if (overrun) {
if (press) {
console.warn(`keyboard buffer overflow, key: ${key} not added`);
// Fill all key slots with ErrorRollOver (0x01) to indicate overflow
keys.length = 6;
keys.fill(hidErrorRollOver);
} else {
// If we are releasing a key, and we didn't find it in a slot, who cares?
console.debug(`key ${key} not found in buffer, nothing to release`)
}
}
}
}
return { modifier: modifiers, keys };
}
const handleKeyPress = useCallback(
(key: number, press: boolean) => {
if (keyPressAvailable) {
// if the keyPress api is available, we can just send the key press event
sendKeypressEvent(key, press);
// TODO handle the case where the keyPress api is not available and we need to handle the key locally now...
} else {
// if the keyPress api is not available, we need to handle the key locally
const newKeysDownState = handleKeyLocally(keysDownState, key, press);
setKeysDownState(newKeysDownState);
// then we send the full state
sendKeyboardEvent(newKeysDownState);
}
},
[keyPressAvailable, keysDownState, sendKeyboardEvent, sendKeypressEvent, setKeysDownState],
);
return { handleKeyPress, sendKeyboardEvent, sendKeypressEvent, resetKeyboardState, executeMacro };
} }

11
ui/src/keyboardMappings.ts
View File

@ -142,6 +142,17 @@ export const modifiers = {
MetaRight: 0x80, MetaRight: 0x80,
} as Record<string, number>; } as Record<string, number>;
export const hidKeyToModifierMask = {
0xe0: modifiers.ControlLeft,
0xe1: modifiers.ShiftLeft,
0xe2: modifiers.AltLeft,
0xe3: modifiers.MetaLeft,
0xe4: modifiers.ControlRight,
0xe5: modifiers.ShiftRight,
0xe6: modifiers.AltRight,
0xe7: modifiers.MetaRight,
} as Record<number, number>;
export const modifierDisplayMap: Record<string, string> = { export const modifierDisplayMap: Record<string, string> = {
ControlLeft: "Left Ctrl", ControlLeft: "Left Ctrl",
ControlRight: "Right Ctrl", ControlRight: "Right Ctrl",

8
ui/src/routes/devices.$id.settings.tsx
View File

@ -29,7 +29,7 @@ import { cx } from "../cva.config";
export default function SettingsRoute() { export default function SettingsRoute() {
const location = useLocation(); const location = useLocation();
const setDisableVideoFocusTrap = useUiStore(state => state.setDisableVideoFocusTrap); const setDisableVideoFocusTrap = useUiStore(state => state.setDisableVideoFocusTrap);
const { sendKeyboardEvent } = useKeyboard(); const { resetKeyboardState } = useKeyboard();
const scrollContainerRef = useRef<HTMLDivElement>(null); const scrollContainerRef = useRef<HTMLDivElement>(null);
const [showLeftGradient, setShowLeftGradient] = useState(false); const [showLeftGradient, setShowLeftGradient] = useState(false);
const [showRightGradient, setShowRightGradient] = useState(false); const [showRightGradient, setShowRightGradient] = useState(false);
@ -67,8 +67,8 @@ export default function SettingsRoute() {
useEffect(() => { useEffect(() => {
// disable focus trap // disable focus trap
setTimeout(() => { setTimeout(() => {
// Reset keyboard state. Incase the user is pressing a key while enabling the sidebar // Reset keyboard state. In case the user is pressing a key while enabling the sidebar
sendKeyboardEvent([], []); resetKeyboardState();
setDisableVideoFocusTrap(true); setDisableVideoFocusTrap(true);
// For some reason, the focus trap is not disabled immediately // For some reason, the focus trap is not disabled immediately
// so we need to blur the active element // so we need to blur the active element
@ -79,7 +79,7 @@ export default function SettingsRoute() {
return () => { return () => {
setDisableVideoFocusTrap(false); setDisableVideoFocusTrap(false);
}; };
}, [sendKeyboardEvent, setDisableVideoFocusTrap]); }, [resetKeyboardState, setDisableVideoFocusTrap]);
return ( return (
<div className="pointer-events-auto relative mx-auto max-w-4xl translate-x-0 transform text-left dark:text-white"> <div className="pointer-events-auto relative mx-auto max-w-4xl translate-x-0 transform text-left dark:text-white">

32
ui/src/routes/devices.$id.tsx
View File

@ -519,7 +519,7 @@ export default function KvmIdRoute() {
// Cleanup effect // Cleanup effect
const clearInboundRtpStats = useRTCStore((state: RTCState) => state.clearInboundRtpStats); const clearInboundRtpStats = useRTCStore((state: RTCState) => state.clearInboundRtpStats);
const clearCandidatePairStats = useRTCStore((state: RTCState) => state.clearCandidatePairStats); const clearCandidatePairStats = useRTCStore((state: RTCState) => state.clearCandidatePairStats);
const setSidebarView = useUiStore((state: UIState) => state.setSidebarView); const setSidebarView = useUiStore((state: UIState) => state.setSidebarView);
useEffect(() => { useEffect(() => {
return () => { return () => {
@ -597,6 +597,7 @@ export default function KvmIdRoute() {
const keysDownState = useHidStore((state: HidState) => state.keysDownState); const keysDownState = useHidStore((state: HidState) => state.keysDownState);
const setKeysDownState = useHidStore((state: HidState) => state.setKeysDownState); const setKeysDownState = useHidStore((state: HidState) => state.setKeysDownState);
const setKeyPressAvailable = useHidStore((state: HidState) => state.setKeyPressAvailable);
const [hasUpdated, setHasUpdated] = useState(false); const [hasUpdated, setHasUpdated] = useState(false);
const { navigateTo } = useDeviceUiNavigation(); const { navigateTo } = useDeviceUiNavigation();
@ -624,7 +625,7 @@ export default function KvmIdRoute() {
console.log("Setting keyboard led state", ledState); console.log("Setting keyboard led state", ledState);
setKeyboardLedState(ledState); setKeyboardLedState(ledState);
} }
if (resp.method === "keysDownState") { if (resp.method === "keysDownState") {
const downState = resp.params as KeysDownState; const downState = resp.params as KeysDownState;
console.log("Setting key down state", downState); console.log("Setting key down state", downState);
@ -667,10 +668,12 @@ export default function KvmIdRoute() {
}); });
}, [rpcDataChannel?.readyState, send, setHdmiState]); }, [rpcDataChannel?.readyState, send, setHdmiState]);
const [needLedState, setNeedLedState] = useState(true);
// request keyboard led state from the device // request keyboard led state from the device
useEffect(() => { useEffect(() => {
if (rpcDataChannel?.readyState !== "open") return; if (rpcDataChannel?.readyState !== "open") return;
if (keyboardLedState !== undefined) return; if (!needLedState) return;
console.log("Requesting keyboard led state"); console.log("Requesting keyboard led state");
send("getKeyboardLedState", {}, (resp: JsonRpcResponse) => { send("getKeyboardLedState", {}, (resp: JsonRpcResponse) => {
@ -680,24 +683,35 @@ export default function KvmIdRoute() {
} }
console.log("Keyboard led state", resp.result); console.log("Keyboard led state", resp.result);
setKeyboardLedState(resp.result as KeyboardLedState); setKeyboardLedState(resp.result as KeyboardLedState);
setNeedLedState(false);
}); });
}, [rpcDataChannel?.readyState, send, setKeyboardLedState, keyboardLedState]); }, [rpcDataChannel?.readyState, send, setKeyboardLedState, keyboardLedState, needLedState]);
const [needKeyDownState, setNeedKeyDownState] = useState(true);
// request keyboard key down state from the device // request keyboard key down state from the device
useEffect(() => { useEffect(() => {
if (rpcDataChannel?.readyState !== "open") return; if (rpcDataChannel?.readyState !== "open") return;
if (keysDownState !== undefined) return; if (!needKeyDownState) return;
console.log("Requesting keys down state"); console.log("Requesting keys down state");
send("getKeyDownState", {}, (resp: JsonRpcResponse) => { send("getKeyDownState", {}, (resp: JsonRpcResponse) => {
if ("error" in resp) { if ("error" in resp) {
console.error("Failed to get key down state", resp.error); // -32601 means the method is not supported
if (resp.error.code === -32601) {
// if we don't support key down state, we know key press is also not available
console.error("Failed to get key down state, switching to old-school", resp.error);
setKeyPressAvailable(false);
} else {
console.error("Failed to get key down state", resp.error);
}
return; return;
} }
console.log("Keyboard key down state", resp.result); console.log("Keyboard key down state", resp.result);
setKeysDownState(resp.result as KeysDownState); setKeysDownState(resp.result as KeysDownState);
setNeedKeyDownState(false);
}); });
}, [keysDownState, rpcDataChannel?.readyState, send, setKeysDownState]); }, [keysDownState, needKeyDownState, rpcDataChannel?.readyState, send, setKeyPressAvailable, setKeysDownState]);
// When the update is successful, we need to refresh the client javascript and show a success modal // When the update is successful, we need to refresh the client javascript and show a success modal
useEffect(() => { useEffect(() => {
@ -758,7 +772,7 @@ export default function KvmIdRoute() {
send("getUpdateStatus", {}, async (resp: JsonRpcResponse) => { send("getUpdateStatus", {}, async (resp: JsonRpcResponse) => {
if ("error" in resp) { if ("error" in resp) {
notifications.error(`Failed to get device version: ${resp.error}`); notifications.error(`Failed to get device version: ${resp.error}`);
return return
} }
const result = resp.result as SystemVersionInfo; const result = resp.result as SystemVersionInfo;
@ -895,7 +909,7 @@ interface SidebarContainerProps {
} }
function SidebarContainer(props: SidebarContainerProps) { function SidebarContainer(props: SidebarContainerProps) {
const { sidebarView }= props; const { sidebarView } = props;
return ( return (
<div <div
className={cx( className={cx(