diff --git a/ui/src/components/InfoBar.tsx b/ui/src/components/InfoBar.tsx index 7ce67a4..c769de0 100644 --- a/ui/src/components/InfoBar.tsx +++ b/ui/src/components/InfoBar.tsx @@ -40,6 +40,12 @@ export default function InfoBar() { const keyboardLedState = useHidStore(state => state.keyboardLedState); const keyboardLedStateSyncAvailable = useHidStore(state => state.keyboardLedStateSyncAvailable); const keyboardLedSync = useSettingsStore(state => state.keyboardLedSync); + + const isShiftActive = useHidStore(state => state.isShiftActive); + const isCtrlActive = useHidStore(state => state.isCtrlActive); + const isAltActive = useHidStore(state => state.isAltActive); + const isMetaActive = useHidStore(state => state.isMetaActive); + const isAltGrActive = useHidStore(state => state.isAltGrActive); const isTurnServerInUse = useRTCStore(state => state.isTurnServerInUse); @@ -135,6 +141,56 @@ export default function InfoBar() { {keyboardLedSync === "browser" ? "Browser" : "Host"} ) : null} +
+ Shift +
+
+ Ctrl +
+
+ Alt +
+
+ Meta +
+
+ AltGr +
{ function KeyboardWrapper() { const [layoutName, setLayoutName] = useState("default"); + const [depressedButtons, setDepressedButtons] = useState(""); const keyboardRef = useRef(null); const showAttachedVirtualKeyboard = useUiStore( @@ -54,6 +55,21 @@ function KeyboardWrapper() { const setIsCapsLockActive = useHidStore(state => state.setIsCapsLockActive); + const isShiftActive = useHidStore(state => state.isShiftActive); + const setIsShiftActive = useHidStore(state => state.setIsShiftActive); + + const isCtrlActive = useHidStore(state => state.isCtrlActive); + const setIsCtrlActive = useHidStore(state => state.setIsCtrlActive); + + const isAltActive = useHidStore(state => state.isAltActive); + const setIsAltActive = useHidStore(state => state.setIsAltActive); + + const isMetaActive = useHidStore(state => state.isMetaActive); + const setIsMetaActive = useHidStore(state => state.setIsMetaActive); + + const isAltGrActive = useHidStore(state => state.isAltGrActive); + const setIsAltGrActive = useHidStore(state => state.setIsAltGrActive); + const startDrag = useCallback((e: MouseEvent | TouchEvent) => { if (!keyboardRef.current) return; if (e instanceof TouchEvent && e.touches.length > 1) return; @@ -123,80 +139,123 @@ function KeyboardWrapper() { }; }, [endDrag, onDrag, startDrag]); - const onKeyDown = useCallback( - (key: string) => { - const isKeyShift = key === "{shift}" || key === "ShiftLeft" || key === "ShiftRight"; - const isKeyCaps = key === "CapsLock"; - const cleanKey = key.replace(/[()]/g, ""); - const keyHasShiftModifier = key.includes("("); + useEffect(() => { + // if you have the CapsLock "down", then the shift state is inverted + const effectiveShift = isCapsLockActive ? false === isShiftActive : isShiftActive; + setLayoutName(effectiveShift ? "shift" : "default"); + }, + [setLayoutName, isCapsLockActive, isShiftActive] + ); - // Handle toggle of layout for shift or caps lock - const toggleLayout = () => { - setLayoutName(prevLayout => (prevLayout === "default" ? "shift" : "default")); - }; + // this causes the buttons to look depressed/clicked depending on the sticky state + useEffect(() => { + let buttons = "None "; // make sure we name at least one (fake) button + if (isCapsLockActive) buttons += "CapsLock "; + if (isShiftActive) buttons += "ShiftLeft ShiftRight "; + if (isCtrlActive) buttons += "ControlLeft ControlRight "; + if (isAltActive) buttons += "AltLeft AltRight "; + if (isMetaActive) buttons += "MetaLeft MetaRight "; + setDepressedButtons(buttons.trimEnd()); + }, + [setDepressedButtons, isCapsLockActive, isShiftActive, isCtrlActive, isAltActive, isMetaActive, isAltGrActive] + ); - if (key === "CtrlAltDelete") { - sendKeyboardEvent( - [keys["Delete"]], - [modifiers["ControlLeft"], modifiers["AltLeft"]], - ); - setTimeout(resetKeyboardState, 100); - return; - } + const onKeyPress = useCallback((key: string) => { + // handle the fake combo keys first + if (key === "CtrlAltDelete") { + sendKeyboardEvent( + [keys["Delete"]], + [modifiers["ControlLeft"], modifiers["AltLeft"]], + ); + setTimeout(resetKeyboardState, 100); + return; + } - if (key === "AltMetaEscape") { - sendKeyboardEvent( - [keys["Escape"]], - [modifiers["MetaLeft"], modifiers["AltLeft"]], - ); - - setTimeout(resetKeyboardState, 100); - return; - } - - if (key === "CtrlAltBackspace") { - sendKeyboardEvent( - [keys["Backspace"]], - [modifiers["ControlLeft"], modifiers["AltLeft"]], - ); - - setTimeout(resetKeyboardState, 100); - return; - } - - if (isKeyShift || isKeyCaps) { - toggleLayout(); - - if (isCapsLockActive) { - if (!isKeyboardLedManagedByHost) { - setIsCapsLockActive(false); - } - sendKeyboardEvent([keys["CapsLock"]], []); - return; - } - } - - // Handle caps lock state change - if (isKeyCaps && !isKeyboardLedManagedByHost) { - setIsCapsLockActive(!isCapsLockActive); - } - - // Collect new active keys and modifiers - const newKeys = keys[cleanKey] ? [keys[cleanKey]] : []; - const newModifiers = - keyHasShiftModifier && !isCapsLockActive ? [modifiers["ShiftLeft"]] : []; - - // Update current keys and modifiers - sendKeyboardEvent(newKeys, newModifiers); - - // If shift was used as a modifier and caps lock is not active, revert to default layout - if (keyHasShiftModifier && !isCapsLockActive) { - setLayoutName("default"); - } + if (key === "AltMetaEscape") { + sendKeyboardEvent( + [keys["Escape"]], + [modifiers["MetaLeft"], modifiers["AltLeft"]], + ); setTimeout(resetKeyboardState, 100); - }, - [isCapsLockActive, isKeyboardLedManagedByHost, sendKeyboardEvent, resetKeyboardState, setIsCapsLockActive], + return; + } + + if (key === "CtrlAltBackspace") { + sendKeyboardEvent( + [keys["Backspace"]], + [modifiers["ControlLeft"], modifiers["AltLeft"]], + ); + + setTimeout(resetKeyboardState, 100); + return; + } + + // strip away the parens for shifted characters + const cleanKey = key.replace(/[()]/g, ""); + + const passthrough = ["PrintScreen", "SystemRequest", "Pause", "Break", "ScrollLock", "Enter", "Space"].find((value) => value === cleanKey); + + if (passthrough) { + emitkeycode(cleanKey); + return; + } + + // adjust the sticky state of the Shift/Ctrl/Alt/Meta/AltGr + if (key === "CapsLock" && !isKeyboardLedManagedByHost) + setIsCapsLockActive(!isCapsLockActive); + else if (key === "ShiftLeft" || key === "ShiftRight") + setIsShiftActive(!isShiftActive); + else if (key === "ControlLeft" || key === "ControlRight") + setIsCtrlActive(!isCtrlActive); + else if (key === "AltLeft" || key === "AltRight") + setIsAltActive(!isAltActive); + else if (key === "MetaLeft" || key === "MetaRight") + setIsMetaActive(!isMetaActive); + else if (key === "AltGr") + setIsAltGrActive(!isAltGrActive); + + emitkeycode(cleanKey); + + function emitkeycode(key: string) { + const effectiveMods: number[] = []; + + if (isShiftActive) + effectiveMods.push(modifiers["ShiftLeft"]); + + if (isCtrlActive) + effectiveMods.push(modifiers["ControlLeft"]); + + if (isAltActive) + effectiveMods.push(modifiers["AltLeft"]); + + if (isMetaActive) + effectiveMods.push(modifiers["MetaLeft"]); + + if (isAltGrActive) { + effectiveMods.push(modifiers["MetaRight"]); + effectiveMods.push(modifiers["CtrlLeft"]); + } + + const keycode = keys[key]; + if (keycode) { + // send the keycode with modifiers + sendKeyboardEvent([keycode], effectiveMods); + } + + // release the key (if one pressed), but retain the modifiers + setTimeout(() => sendKeyboardEvent([], effectiveMods), 50); + } + }, + [isKeyboardLedManagedByHost, + setIsCapsLockActive, isCapsLockActive, + setIsShiftActive, isShiftActive, + setIsCtrlActive, isCtrlActive, + setIsAltActive, isAltActive, + setIsMetaActive, isMetaActive, + setIsAltGrActive, isAltGrActive, + sendKeyboardEvent, resetKeyboardState + ], ); const virtualKeyboard = useHidStore(state => state.isVirtualKeyboardEnabled); @@ -276,12 +335,16 @@ function KeyboardWrapper() {
@@ -314,25 +375,24 @@ function KeyboardWrapper() { baseClass="simple-keyboard-control" theme="simple-keyboard hg-theme-default hg-layout-default" layoutName={layoutName} - onKeyPress={onKeyDown} + onKeyPress={onKeyPress} display={keyDisplayMap} layout={{ - default: ["PrintScreen ScrollLock Pause", "Insert Home Pageup", "Delete End Pagedown"], - shift: ["(PrintScreen) ScrollLock (Pause)", "Insert Home Pageup", "Delete End Pagedown"], + default: ["PrintScreen ScrollLock Pause", "Insert Home PageUp", "Delete End PageDown"], + shift: ["(PrintScreen) ScrollLock (Pause)", "Insert Home PageUp", "Delete End PageDown"], }} - syncInstanceInputs={true} - debug={false} + disableButtonHold={true} />
diff --git a/ui/src/components/WebRTCVideo.tsx b/ui/src/components/WebRTCVideo.tsx index 571fac8..256ccfc 100644 --- a/ui/src/components/WebRTCVideo.tsx +++ b/ui/src/components/WebRTCVideo.tsx @@ -71,6 +71,15 @@ export default function WebRTCVideo() { const hdmiError = ["no_lock", "no_signal", "out_of_range"].includes(hdmiState); const isVideoLoading = !isPlaying; + // Keyboard related states + const { + setIsShiftActive, + setIsCtrlActive, + setIsAltActive, + setIsMetaActive, + setIsAltGrActive + } = useHidStore(); + const [blockWheelEvent, setBlockWheelEvent] = useState(false); // Misc states and hooks @@ -423,6 +432,12 @@ export default function WebRTCVideo() { setIsScrollLockActive(e.getModifierState("ScrollLock")); } + setIsShiftActive(e.getModifierState("Shift")) + setIsCtrlActive(e.getModifierState("Control")) + setIsAltActive(e.getModifierState("Alt")) + setIsMetaActive(e.getModifierState("Meta")) + setIsAltGrActive(e.getModifierState("AltGraph")) + if (code == "IntlBackslash" && ["`", "~"].includes(key)) { code = "Backquote"; } else if (code == "Backquote" && ["§", "±"].includes(key)) { @@ -452,12 +467,17 @@ export default function WebRTCVideo() { sendKeyboardEvent([...new Set(newKeys)], [...new Set(newModifiers)]); }, [ - handleModifierKeys, - sendKeyboardEvent, isKeyboardLedManagedByHost, setIsNumLockActive, setIsCapsLockActive, setIsScrollLockActive, + setIsShiftActive, + setIsCtrlActive, + setIsAltActive, + setIsMetaActive, + setIsAltGrActive, + handleModifierKeys, + sendKeyboardEvent ], ); @@ -472,6 +492,12 @@ export default function WebRTCVideo() { setIsScrollLockActive(e.getModifierState("ScrollLock")); } + setIsShiftActive(e.getModifierState("Shift")) + setIsCtrlActive(e.getModifierState("Control")) + setIsAltActive(e.getModifierState("Alt")) + setIsMetaActive(e.getModifierState("Meta")) + setIsAltGrActive(e.getModifierState("AltGraph")) + // Filtering out the key that was just released (keys[e.code]) const newKeys = prev.activeKeys.filter(k => k !== keys[e.code]).filter(Boolean); @@ -484,12 +510,17 @@ export default function WebRTCVideo() { sendKeyboardEvent([...new Set(newKeys)], [...new Set(newModifiers)]); }, [ - handleModifierKeys, - sendKeyboardEvent, isKeyboardLedManagedByHost, setIsNumLockActive, setIsCapsLockActive, setIsScrollLockActive, + setIsShiftActive, + setIsCtrlActive, + setIsAltActive, + setIsMetaActive, + setIsAltGrActive, + handleModifierKeys, + sendKeyboardEvent ], ); diff --git a/ui/src/hooks/stores.ts b/ui/src/hooks/stores.ts index 6bc7e17..ee9dc59 100644 --- a/ui/src/hooks/stores.ts +++ b/ui/src/hooks/stores.ts @@ -481,6 +481,21 @@ export interface HidState { isVirtualKeyboardEnabled: boolean; setVirtualKeyboardEnabled: (enabled: boolean) => void; + isShiftActive: boolean; + setIsShiftActive: (enabled: boolean) => void; + + isCtrlActive: boolean; + setIsCtrlActive: (enabled: boolean) => void; + + isAltActive: boolean; + setIsAltActive: (enabled: boolean) => void; + + isMetaActive: boolean; + setIsMetaActive: (enabled: boolean) => void; + + isAltGrActive: boolean; + setIsAltGrActive: (enabled: boolean) => void; + isPasteModeEnabled: boolean; setPasteModeEnabled: (enabled: boolean) => void; @@ -527,6 +542,21 @@ export const useHidStore = create((set, get) => ({ isVirtualKeyboardEnabled: false, setVirtualKeyboardEnabled: enabled => set({ isVirtualKeyboardEnabled: enabled }), + isShiftActive: false, + setIsShiftActive: enabled => set({ isShiftActive: enabled }), + + isCtrlActive: false, + setIsCtrlActive: enabled => set({ isCtrlActive: enabled }), + + isAltActive: false, + setIsAltActive: enabled => set({ isAltActive: enabled }), + + isMetaActive: false, + setIsMetaActive: enabled => set({ isMetaActive: enabled }), + + isAltGrActive: false, + setIsAltGrActive: enabled => set({ isAltGrActive: enabled }), + isPasteModeEnabled: false, setPasteModeEnabled: enabled => set({ isPasteModeEnabled: enabled }), diff --git a/ui/src/index.css b/ui/src/index.css index 44acd2a..1a79749 100644 --- a/ui/src/index.css +++ b/ui/src/index.css @@ -222,6 +222,10 @@ video::-webkit-media-controls { background: none; } +.simple-keyboard .hg-button.depressed-key { + @apply border-gray-800! border-b-gray-600! border-t-gray-900! bg-gray-700!; +} + .simple-keyboard .hg-button.selectedButton { background: rgba(5, 25, 70, 0.53); @apply text-white; diff --git a/ui/src/keyboardMappings.ts b/ui/src/keyboardMappings.ts index 891b96e..99ab23d 100644 --- a/ui/src/keyboardMappings.ts +++ b/ui/src/keyboardMappings.ts @@ -5,14 +5,14 @@ export const keys = { ArrowLeft: 0x50, ArrowRight: 0x4f, ArrowUp: 0x52, - Backquote: 0x35, + Backquote: 0x35, // aka Grave Backslash: 0x31, Backspace: 0x2a, - BracketLeft: 0x2f, - BracketRight: 0x30, + BracketLeft: 0x2f, // aka LeftBrace + BracketRight: 0x30, // aka RightBrace CapsLock: 0x39, Comma: 0x36, - ContextMenu: 0, + Compose: 0x65, Delete: 0x4c, Digit0: 0x27, Digit1: 0x1e, @@ -41,6 +41,18 @@ export const keys = { F11: 0x44, F12: 0x45, F13: 0x68, + F14: 0x69, + F15: 0x6a, + F16: 0x6b, + F17: 0x6c, + F18: 0x6d, + F19: 0x6e, + F20: 0x6f, + F21: 0x70, + F22: 0x71, + F23: 0x72, + F24: 0x73, + HashTilde: 0x32, Home: 0x4a, Insert: 0x49, IntlBackslash: 0x64, @@ -70,37 +82,58 @@ export const keys = { KeyX: 0x1b, KeyY: 0x1c, KeyZ: 0x1d, + KeypadComma: 0x85, + KeypadEqual: 0x86, + KeyRO: 0x87, + KatakanaHiragana: 0x88, + Yen: 0x89, + Henkan: 0x8a, + Muhenkan: 0x8b, + KPJPComma: 0x8c, + International7: 0x8d, + International8: 0x8e, + International9: 0x8f, + Hangeul: 0x90, + Hanja: 0x91, + Katakana: 0x92, + Hiragana: 0x93, + Zenkakuhankaku:0x94, + KeypadLeftParen: 0xb6, + KeypadRightParen: 0xb7, KeypadExclamation: 0xcf, Minus: 0x2d, - NumLock: 0x53, - Numpad0: 0x62, - Numpad1: 0x59, - Numpad2: 0x5a, - Numpad3: 0x5b, - Numpad4: 0x5c, + None: 0x00, + NumLock: 0x53, // and Clear + Numpad0: 0x62, // and Insert + Numpad1: 0x59, // and End + Numpad2: 0x5a, // and Down Arrow + Numpad3: 0x5b, // and Page Down + Numpad4: 0x5c, // and Left Arrow Numpad5: 0x5d, - Numpad6: 0x5e, - Numpad7: 0x5f, - Numpad8: 0x60, - Numpad9: 0x61, - NumpadAdd: 0x57, - NumpadDivide: 0x54, + Numpad6: 0x5e, // and Right Arrow + Numpad7: 0x5f, // and Home + Numpad8: 0x60, // and Up Arrow + Numpad9: 0x61, // and Page Up + NumpadPlus: 0x57, + NumpadSlash: 0x54, NumpadEnter: 0x58, NumpadEqual: 0x67, - NumpadMultiply: 0x55, - NumpadSubtract: 0x56, - NumpadDecimal: 0x63, + NumpadAsterisk: 0x55, + NumpadMinus: 0x56, + NumpadDecimal: 0x63, // aka NumpadDot and Delete + Overflow: 0x01, PageDown: 0x4e, PageUp: 0x4b, - Period: 0x37, + Period: 0x37, // aka Dot PrintScreen: 0x46, Pause: 0x48, - Quote: 0x34, + Power: 0x66, + Quote: 0x34, // aka Apostrophe ScrollLock: 0x47, Semicolon: 0x33, Slash: 0x38, Space: 0x2c, - SystemRequest: 0x9a, + SystemRequest: 0x9a, // aka Attention Tab: 0x2b, } as Record; @@ -131,23 +164,23 @@ export const keyDisplayMap: Record = { AltMetaEscape: "Alt + Meta + Escape", CtrlAltBackspace: "Ctrl + Alt + Backspace", Escape: "esc", - Tab: "tab", - Backspace: "backspace", - "(Backspace)": "backspace", + Tab: "tab ⇥", + Backspace: "backspace ⌫", Enter: "enter", - CapsLock: "caps lock", - ShiftLeft: "shift", - ShiftRight: "shift", - ControlLeft: "ctrl", - AltLeft: "alt", - AltRight: "alt", - MetaLeft: "meta", - MetaRight: "meta", + CapsLock: "caps lock ⇪", + ShiftLeft: "shift ⇧", + ShiftRight: "⇧ shift", + ControlLeft: "ctrl ⌃", + ControlRight: "⌃ ctrl", + AltLeft: "alt ⌥", + AltRight: "⌥ alt", + MetaLeft: "meta ⌘", // "meta ⊞" for windows + MetaRight: "⌘ meta",// "≣ meta" for windows Space: " ", - Insert: "insert", + Insert: "ins", Home: "home", PageUp: "page up", - Delete: "delete", + Delete: "del ⌦", End: "end", PageDown: "page down", ArrowLeft: "←", @@ -213,13 +246,12 @@ export const keyDisplayMap: Record = { Numpad0: "Num 0", Numpad1: "Num 1", Numpad2: "Num 2", Numpad3: "Num 3", Numpad4: "Num 4", Numpad5: "Num 5", Numpad6: "Num 6", Numpad7: "Num 7", Numpad8: "Num 8", - Numpad9: "Num 9", NumpadAdd: "Num +", NumpadSubtract: "Num -", - NumpadMultiply: "Num *", NumpadDivide: "Num /", NumpadDecimal: "Num .", + Numpad9: "Num 9", NumpadPlus: "Num +", NumpadMinus: "Num -", + NumpadAsterisk: "Num *", NumpadSlash: "Num /", NumpadDecimal: "Num .", NumpadEqual: "Num =", NumpadEnter: "Num Enter", NumLock: "Num Lock", // Modals PrintScreen: "prt sc", ScrollLock: "scr lk", Pause: "pause", - "(PrintScreen)": "sys rq", "(Pause)": "break", - SystemRequest: "sys rq", Break: "break" + "(PrintScreen)": "sys rq", "(Pause)": "break" };