From 84e4b44df0f56d5c1ade606aeac97703d806ef4e Mon Sep 17 00:00:00 2001 From: Marc Brooks Date: Wed, 21 May 2025 23:14:19 -0500 Subject: [PATCH] Treats all modifiers as "sticky" keys for use with just touch/mouse Made the operation keys look more like a keyboard Add persistent state management for the Shift/Ctrl/Alt/Meta Add infoBar indicators for Shift/Ctr/Alt/Meta/AltGr Remove redundant Break it's actually on keyboard as (Pause) Add a style for the virtual keyboard "depressed" buttons. Delete the no-longer-needed button variants Added missing keycodes Added modifier tracking for all the other keys Ensures the InfoBar tracks for physical and virtual keyboard Shows what buttons are depressed for sticky keys Now treats all the Shift/Control/Alt/Meta/AltGr keys as if they were sticky keys so users can click the button and hit the next key, --- ui/src/components/InfoBar.tsx | 56 +++++++ ui/src/components/VirtualKeyboard.tsx | 220 ++++++++++++++++---------- ui/src/components/WebRTCVideo.tsx | 39 ++++- ui/src/hooks/stores.ts | 30 ++++ ui/src/index.css | 4 + ui/src/keyboardMappings.ts | 110 ++++++++----- 6 files changed, 336 insertions(+), 123 deletions(-) diff --git a/ui/src/components/InfoBar.tsx b/ui/src/components/InfoBar.tsx index 7ce67a4..c769de0 100644 --- a/ui/src/components/InfoBar.tsx +++ b/ui/src/components/InfoBar.tsx @@ -40,6 +40,12 @@ export default function InfoBar() { const keyboardLedState = useHidStore(state => state.keyboardLedState); const keyboardLedStateSyncAvailable = useHidStore(state => state.keyboardLedStateSyncAvailable); const keyboardLedSync = useSettingsStore(state => state.keyboardLedSync); + + const isShiftActive = useHidStore(state => state.isShiftActive); + const isCtrlActive = useHidStore(state => state.isCtrlActive); + const isAltActive = useHidStore(state => state.isAltActive); + const isMetaActive = useHidStore(state => state.isMetaActive); + const isAltGrActive = useHidStore(state => state.isAltGrActive); const isTurnServerInUse = useRTCStore(state => state.isTurnServerInUse); @@ -135,6 +141,56 @@ export default function InfoBar() { {keyboardLedSync === "browser" ? "Browser" : "Host"} ) : null} +
+ Shift +
+
+ Ctrl +
+
+ Alt +
+
+ Meta +
+
+ AltGr +
{ function KeyboardWrapper() { const [layoutName, setLayoutName] = useState("default"); + const [depressedButtons, setDepressedButtons] = useState(""); const keyboardRef = useRef(null); const showAttachedVirtualKeyboard = useUiStore( @@ -54,6 +55,21 @@ function KeyboardWrapper() { const setIsCapsLockActive = useHidStore(state => state.setIsCapsLockActive); + const isShiftActive = useHidStore(state => state.isShiftActive); + const setIsShiftActive = useHidStore(state => state.setIsShiftActive); + + const isCtrlActive = useHidStore(state => state.isCtrlActive); + const setIsCtrlActive = useHidStore(state => state.setIsCtrlActive); + + const isAltActive = useHidStore(state => state.isAltActive); + const setIsAltActive = useHidStore(state => state.setIsAltActive); + + const isMetaActive = useHidStore(state => state.isMetaActive); + const setIsMetaActive = useHidStore(state => state.setIsMetaActive); + + const isAltGrActive = useHidStore(state => state.isAltGrActive); + const setIsAltGrActive = useHidStore(state => state.setIsAltGrActive); + const startDrag = useCallback((e: MouseEvent | TouchEvent) => { if (!keyboardRef.current) return; if (e instanceof TouchEvent && e.touches.length > 1) return; @@ -123,80 +139,123 @@ function KeyboardWrapper() { }; }, [endDrag, onDrag, startDrag]); - const onKeyDown = useCallback( - (key: string) => { - const isKeyShift = key === "{shift}" || key === "ShiftLeft" || key === "ShiftRight"; - const isKeyCaps = key === "CapsLock"; - const cleanKey = key.replace(/[()]/g, ""); - const keyHasShiftModifier = key.includes("("); + useEffect(() => { + // if you have the CapsLock "down", then the shift state is inverted + const effectiveShift = isCapsLockActive ? false === isShiftActive : isShiftActive; + setLayoutName(effectiveShift ? "shift" : "default"); + }, + [setLayoutName, isCapsLockActive, isShiftActive] + ); - // Handle toggle of layout for shift or caps lock - const toggleLayout = () => { - setLayoutName(prevLayout => (prevLayout === "default" ? "shift" : "default")); - }; + // this causes the buttons to look depressed/clicked depending on the sticky state + useEffect(() => { + let buttons = "None "; // make sure we name at least one (fake) button + if (isCapsLockActive) buttons += "CapsLock "; + if (isShiftActive) buttons += "ShiftLeft ShiftRight "; + if (isCtrlActive) buttons += "ControlLeft ControlRight "; + if (isAltActive) buttons += "AltLeft AltRight "; + if (isMetaActive) buttons += "MetaLeft MetaRight "; + setDepressedButtons(buttons.trimEnd()); + }, + [setDepressedButtons, isCapsLockActive, isShiftActive, isCtrlActive, isAltActive, isMetaActive, isAltGrActive] + ); - if (key === "CtrlAltDelete") { - sendKeyboardEvent( - [keys["Delete"]], - [modifiers["ControlLeft"], modifiers["AltLeft"]], - ); - setTimeout(resetKeyboardState, 100); - return; - } + const onKeyPress = useCallback((key: string) => { + // handle the fake combo keys first + if (key === "CtrlAltDelete") { + sendKeyboardEvent( + [keys["Delete"]], + [modifiers["ControlLeft"], modifiers["AltLeft"]], + ); + setTimeout(resetKeyboardState, 100); + return; + } - if (key === "AltMetaEscape") { - sendKeyboardEvent( - [keys["Escape"]], - [modifiers["MetaLeft"], modifiers["AltLeft"]], - ); - - setTimeout(resetKeyboardState, 100); - return; - } - - if (key === "CtrlAltBackspace") { - sendKeyboardEvent( - [keys["Backspace"]], - [modifiers["ControlLeft"], modifiers["AltLeft"]], - ); - - setTimeout(resetKeyboardState, 100); - return; - } - - if (isKeyShift || isKeyCaps) { - toggleLayout(); - - if (isCapsLockActive) { - if (!isKeyboardLedManagedByHost) { - setIsCapsLockActive(false); - } - sendKeyboardEvent([keys["CapsLock"]], []); - return; - } - } - - // Handle caps lock state change - if (isKeyCaps && !isKeyboardLedManagedByHost) { - setIsCapsLockActive(!isCapsLockActive); - } - - // Collect new active keys and modifiers - const newKeys = keys[cleanKey] ? [keys[cleanKey]] : []; - const newModifiers = - keyHasShiftModifier && !isCapsLockActive ? [modifiers["ShiftLeft"]] : []; - - // Update current keys and modifiers - sendKeyboardEvent(newKeys, newModifiers); - - // If shift was used as a modifier and caps lock is not active, revert to default layout - if (keyHasShiftModifier && !isCapsLockActive) { - setLayoutName("default"); - } + if (key === "AltMetaEscape") { + sendKeyboardEvent( + [keys["Escape"]], + [modifiers["MetaLeft"], modifiers["AltLeft"]], + ); setTimeout(resetKeyboardState, 100); - }, - [isCapsLockActive, isKeyboardLedManagedByHost, sendKeyboardEvent, resetKeyboardState, setIsCapsLockActive], + return; + } + + if (key === "CtrlAltBackspace") { + sendKeyboardEvent( + [keys["Backspace"]], + [modifiers["ControlLeft"], modifiers["AltLeft"]], + ); + + setTimeout(resetKeyboardState, 100); + return; + } + + // strip away the parens for shifted characters + const cleanKey = key.replace(/[()]/g, ""); + + const passthrough = ["PrintScreen", "SystemRequest", "Pause", "Break", "ScrollLock", "Enter", "Space"].find((value) => value === cleanKey); + + if (passthrough) { + emitkeycode(cleanKey); + return; + } + + // adjust the sticky state of the Shift/Ctrl/Alt/Meta/AltGr + if (key === "CapsLock" && !isKeyboardLedManagedByHost) + setIsCapsLockActive(!isCapsLockActive); + else if (key === "ShiftLeft" || key === "ShiftRight") + setIsShiftActive(!isShiftActive); + else if (key === "ControlLeft" || key === "ControlRight") + setIsCtrlActive(!isCtrlActive); + else if (key === "AltLeft" || key === "AltRight") + setIsAltActive(!isAltActive); + else if (key === "MetaLeft" || key === "MetaRight") + setIsMetaActive(!isMetaActive); + else if (key === "AltGr") + setIsAltGrActive(!isAltGrActive); + + emitkeycode(cleanKey); + + function emitkeycode(key: string) { + const effectiveMods: number[] = []; + + if (isShiftActive) + effectiveMods.push(modifiers["ShiftLeft"]); + + if (isCtrlActive) + effectiveMods.push(modifiers["ControlLeft"]); + + if (isAltActive) + effectiveMods.push(modifiers["AltLeft"]); + + if (isMetaActive) + effectiveMods.push(modifiers["MetaLeft"]); + + if (isAltGrActive) { + effectiveMods.push(modifiers["MetaRight"]); + effectiveMods.push(modifiers["CtrlLeft"]); + } + + const keycode = keys[key]; + if (keycode) { + // send the keycode with modifiers + sendKeyboardEvent([keycode], effectiveMods); + } + + // release the key (if one pressed), but retain the modifiers + setTimeout(() => sendKeyboardEvent([], effectiveMods), 50); + } + }, + [isKeyboardLedManagedByHost, + setIsCapsLockActive, isCapsLockActive, + setIsShiftActive, isShiftActive, + setIsCtrlActive, isCtrlActive, + setIsAltActive, isAltActive, + setIsMetaActive, isMetaActive, + setIsAltGrActive, isAltGrActive, + sendKeyboardEvent, resetKeyboardState + ], ); const virtualKeyboard = useHidStore(state => state.isVirtualKeyboardEnabled); @@ -276,12 +335,16 @@ function KeyboardWrapper() {
@@ -314,25 +375,24 @@ function KeyboardWrapper() { baseClass="simple-keyboard-control" theme="simple-keyboard hg-theme-default hg-layout-default" layoutName={layoutName} - onKeyPress={onKeyDown} + onKeyPress={onKeyPress} display={keyDisplayMap} layout={{ - default: ["PrintScreen ScrollLock Pause", "Insert Home Pageup", "Delete End Pagedown"], - shift: ["(PrintScreen) ScrollLock (Pause)", "Insert Home Pageup", "Delete End Pagedown"], + default: ["PrintScreen ScrollLock Pause", "Insert Home PageUp", "Delete End PageDown"], + shift: ["(PrintScreen) ScrollLock (Pause)", "Insert Home PageUp", "Delete End PageDown"], }} - syncInstanceInputs={true} - debug={false} + disableButtonHold={true} />
diff --git a/ui/src/components/WebRTCVideo.tsx b/ui/src/components/WebRTCVideo.tsx index e28f571..ed0744e 100644 --- a/ui/src/components/WebRTCVideo.tsx +++ b/ui/src/components/WebRTCVideo.tsx @@ -71,6 +71,15 @@ export default function WebRTCVideo() { const hdmiError = ["no_lock", "no_signal", "out_of_range"].includes(hdmiState); const isVideoLoading = !isPlaying; + // Keyboard related states + const { + setIsShiftActive, + setIsCtrlActive, + setIsAltActive, + setIsMetaActive, + setIsAltGrActive + } = useHidStore(); + const [blockWheelEvent, setBlockWheelEvent] = useState(false); // Misc states and hooks @@ -403,6 +412,12 @@ export default function WebRTCVideo() { setIsScrollLockActive(e.getModifierState("ScrollLock")); } + setIsShiftActive(e.getModifierState("Shift")) + setIsCtrlActive(e.getModifierState("Control")) + setIsAltActive(e.getModifierState("Alt")) + setIsMetaActive(e.getModifierState("Meta")) + setIsAltGrActive(e.getModifierState("AltGraph")) + if (code == "IntlBackslash" && ["`", "~"].includes(key)) { code = "Backquote"; } else if (code == "Backquote" && ["§", "±"].includes(key)) { @@ -432,12 +447,17 @@ export default function WebRTCVideo() { sendKeyboardEvent([...new Set(newKeys)], [...new Set(newModifiers)]); }, [ - handleModifierKeys, - sendKeyboardEvent, isKeyboardLedManagedByHost, setIsNumLockActive, setIsCapsLockActive, setIsScrollLockActive, + setIsShiftActive, + setIsCtrlActive, + setIsAltActive, + setIsMetaActive, + setIsAltGrActive, + handleModifierKeys, + sendKeyboardEvent ], ); @@ -452,6 +472,12 @@ export default function WebRTCVideo() { setIsScrollLockActive(e.getModifierState("ScrollLock")); } + setIsShiftActive(e.getModifierState("Shift")) + setIsCtrlActive(e.getModifierState("Control")) + setIsAltActive(e.getModifierState("Alt")) + setIsMetaActive(e.getModifierState("Meta")) + setIsAltGrActive(e.getModifierState("AltGraph")) + // Filtering out the key that was just released (keys[e.code]) const newKeys = prev.activeKeys.filter(k => k !== keys[e.code]).filter(Boolean); @@ -464,12 +490,17 @@ export default function WebRTCVideo() { sendKeyboardEvent([...new Set(newKeys)], [...new Set(newModifiers)]); }, [ - handleModifierKeys, - sendKeyboardEvent, isKeyboardLedManagedByHost, setIsNumLockActive, setIsCapsLockActive, setIsScrollLockActive, + setIsShiftActive, + setIsCtrlActive, + setIsAltActive, + setIsMetaActive, + setIsAltGrActive, + handleModifierKeys, + sendKeyboardEvent ], ); diff --git a/ui/src/hooks/stores.ts b/ui/src/hooks/stores.ts index 6bc7e17..ee9dc59 100644 --- a/ui/src/hooks/stores.ts +++ b/ui/src/hooks/stores.ts @@ -481,6 +481,21 @@ export interface HidState { isVirtualKeyboardEnabled: boolean; setVirtualKeyboardEnabled: (enabled: boolean) => void; + isShiftActive: boolean; + setIsShiftActive: (enabled: boolean) => void; + + isCtrlActive: boolean; + setIsCtrlActive: (enabled: boolean) => void; + + isAltActive: boolean; + setIsAltActive: (enabled: boolean) => void; + + isMetaActive: boolean; + setIsMetaActive: (enabled: boolean) => void; + + isAltGrActive: boolean; + setIsAltGrActive: (enabled: boolean) => void; + isPasteModeEnabled: boolean; setPasteModeEnabled: (enabled: boolean) => void; @@ -527,6 +542,21 @@ export const useHidStore = create((set, get) => ({ isVirtualKeyboardEnabled: false, setVirtualKeyboardEnabled: enabled => set({ isVirtualKeyboardEnabled: enabled }), + isShiftActive: false, + setIsShiftActive: enabled => set({ isShiftActive: enabled }), + + isCtrlActive: false, + setIsCtrlActive: enabled => set({ isCtrlActive: enabled }), + + isAltActive: false, + setIsAltActive: enabled => set({ isAltActive: enabled }), + + isMetaActive: false, + setIsMetaActive: enabled => set({ isMetaActive: enabled }), + + isAltGrActive: false, + setIsAltGrActive: enabled => set({ isAltGrActive: enabled }), + isPasteModeEnabled: false, setPasteModeEnabled: enabled => set({ isPasteModeEnabled: enabled }), diff --git a/ui/src/index.css b/ui/src/index.css index 44acd2a..1a79749 100644 --- a/ui/src/index.css +++ b/ui/src/index.css @@ -222,6 +222,10 @@ video::-webkit-media-controls { background: none; } +.simple-keyboard .hg-button.depressed-key { + @apply border-gray-800! border-b-gray-600! border-t-gray-900! bg-gray-700!; +} + .simple-keyboard .hg-button.selectedButton { background: rgba(5, 25, 70, 0.53); @apply text-white; diff --git a/ui/src/keyboardMappings.ts b/ui/src/keyboardMappings.ts index 891b96e..99ab23d 100644 --- a/ui/src/keyboardMappings.ts +++ b/ui/src/keyboardMappings.ts @@ -5,14 +5,14 @@ export const keys = { ArrowLeft: 0x50, ArrowRight: 0x4f, ArrowUp: 0x52, - Backquote: 0x35, + Backquote: 0x35, // aka Grave Backslash: 0x31, Backspace: 0x2a, - BracketLeft: 0x2f, - BracketRight: 0x30, + BracketLeft: 0x2f, // aka LeftBrace + BracketRight: 0x30, // aka RightBrace CapsLock: 0x39, Comma: 0x36, - ContextMenu: 0, + Compose: 0x65, Delete: 0x4c, Digit0: 0x27, Digit1: 0x1e, @@ -41,6 +41,18 @@ export const keys = { F11: 0x44, F12: 0x45, F13: 0x68, + F14: 0x69, + F15: 0x6a, + F16: 0x6b, + F17: 0x6c, + F18: 0x6d, + F19: 0x6e, + F20: 0x6f, + F21: 0x70, + F22: 0x71, + F23: 0x72, + F24: 0x73, + HashTilde: 0x32, Home: 0x4a, Insert: 0x49, IntlBackslash: 0x64, @@ -70,37 +82,58 @@ export const keys = { KeyX: 0x1b, KeyY: 0x1c, KeyZ: 0x1d, + KeypadComma: 0x85, + KeypadEqual: 0x86, + KeyRO: 0x87, + KatakanaHiragana: 0x88, + Yen: 0x89, + Henkan: 0x8a, + Muhenkan: 0x8b, + KPJPComma: 0x8c, + International7: 0x8d, + International8: 0x8e, + International9: 0x8f, + Hangeul: 0x90, + Hanja: 0x91, + Katakana: 0x92, + Hiragana: 0x93, + Zenkakuhankaku:0x94, + KeypadLeftParen: 0xb6, + KeypadRightParen: 0xb7, KeypadExclamation: 0xcf, Minus: 0x2d, - NumLock: 0x53, - Numpad0: 0x62, - Numpad1: 0x59, - Numpad2: 0x5a, - Numpad3: 0x5b, - Numpad4: 0x5c, + None: 0x00, + NumLock: 0x53, // and Clear + Numpad0: 0x62, // and Insert + Numpad1: 0x59, // and End + Numpad2: 0x5a, // and Down Arrow + Numpad3: 0x5b, // and Page Down + Numpad4: 0x5c, // and Left Arrow Numpad5: 0x5d, - Numpad6: 0x5e, - Numpad7: 0x5f, - Numpad8: 0x60, - Numpad9: 0x61, - NumpadAdd: 0x57, - NumpadDivide: 0x54, + Numpad6: 0x5e, // and Right Arrow + Numpad7: 0x5f, // and Home + Numpad8: 0x60, // and Up Arrow + Numpad9: 0x61, // and Page Up + NumpadPlus: 0x57, + NumpadSlash: 0x54, NumpadEnter: 0x58, NumpadEqual: 0x67, - NumpadMultiply: 0x55, - NumpadSubtract: 0x56, - NumpadDecimal: 0x63, + NumpadAsterisk: 0x55, + NumpadMinus: 0x56, + NumpadDecimal: 0x63, // aka NumpadDot and Delete + Overflow: 0x01, PageDown: 0x4e, PageUp: 0x4b, - Period: 0x37, + Period: 0x37, // aka Dot PrintScreen: 0x46, Pause: 0x48, - Quote: 0x34, + Power: 0x66, + Quote: 0x34, // aka Apostrophe ScrollLock: 0x47, Semicolon: 0x33, Slash: 0x38, Space: 0x2c, - SystemRequest: 0x9a, + SystemRequest: 0x9a, // aka Attention Tab: 0x2b, } as Record; @@ -131,23 +164,23 @@ export const keyDisplayMap: Record = { AltMetaEscape: "Alt + Meta + Escape", CtrlAltBackspace: "Ctrl + Alt + Backspace", Escape: "esc", - Tab: "tab", - Backspace: "backspace", - "(Backspace)": "backspace", + Tab: "tab ⇥", + Backspace: "backspace ⌫", Enter: "enter", - CapsLock: "caps lock", - ShiftLeft: "shift", - ShiftRight: "shift", - ControlLeft: "ctrl", - AltLeft: "alt", - AltRight: "alt", - MetaLeft: "meta", - MetaRight: "meta", + CapsLock: "caps lock ⇪", + ShiftLeft: "shift ⇧", + ShiftRight: "⇧ shift", + ControlLeft: "ctrl ⌃", + ControlRight: "⌃ ctrl", + AltLeft: "alt ⌥", + AltRight: "⌥ alt", + MetaLeft: "meta ⌘", // "meta ⊞" for windows + MetaRight: "⌘ meta",// "≣ meta" for windows Space: " ", - Insert: "insert", + Insert: "ins", Home: "home", PageUp: "page up", - Delete: "delete", + Delete: "del ⌦", End: "end", PageDown: "page down", ArrowLeft: "←", @@ -213,13 +246,12 @@ export const keyDisplayMap: Record = { Numpad0: "Num 0", Numpad1: "Num 1", Numpad2: "Num 2", Numpad3: "Num 3", Numpad4: "Num 4", Numpad5: "Num 5", Numpad6: "Num 6", Numpad7: "Num 7", Numpad8: "Num 8", - Numpad9: "Num 9", NumpadAdd: "Num +", NumpadSubtract: "Num -", - NumpadMultiply: "Num *", NumpadDivide: "Num /", NumpadDecimal: "Num .", + Numpad9: "Num 9", NumpadPlus: "Num +", NumpadMinus: "Num -", + NumpadAsterisk: "Num *", NumpadSlash: "Num /", NumpadDecimal: "Num .", NumpadEqual: "Num =", NumpadEnter: "Num Enter", NumLock: "Num Lock", // Modals PrintScreen: "prt sc", ScrollLock: "scr lk", Pause: "pause", - "(PrintScreen)": "sys rq", "(Pause)": "break", - SystemRequest: "sys rq", Break: "break" + "(PrintScreen)": "sys rq", "(Pause)": "break" };