From 0e855adc351d9df0f7f488c46ff9e40d5334757d Mon Sep 17 00:00:00 2001 From: William Johnstone Date: Wed, 29 Jan 2025 01:52:13 +0000 Subject: [PATCH 1/5] Early implementation of different keyboard layouts. Store is functioning as expected, adding new layouts should be trivial and easily scalable. Implementation is different for each function that uses the keyboard (PasteModal vs Typing in the WebRTC window) these will all require their own testing. --- config.go | 2 + jsonrpc.go | 14 ++ ui/src/components/InfoBar.tsx | 15 +- ui/src/components/VirtualKeyboard.tsx | 14 +- ui/src/components/WebRTCVideo.tsx | 13 +- ui/src/components/popovers/PasteModal.tsx | 24 +- ui/src/components/sidebar/settings.tsx | 49 ++++ ui/src/keyboardMappings.ts | 214 ----------------- ui/src/keyboardMappings/KeyboardLayouts.ts | 25 ++ .../keyboardMappings/KeyboardMappingStore.ts | 39 ++++ ui/src/keyboardMappings/layouts/uk_apple.ts | 24 ++ ui/src/keyboardMappings/layouts/us.ts | 215 ++++++++++++++++++ 12 files changed, 427 insertions(+), 221 deletions(-) delete mode 100644 ui/src/keyboardMappings.ts create mode 100644 ui/src/keyboardMappings/KeyboardLayouts.ts create mode 100644 ui/src/keyboardMappings/KeyboardMappingStore.ts create mode 100644 ui/src/keyboardMappings/layouts/uk_apple.ts create mode 100644 ui/src/keyboardMappings/layouts/us.ts diff --git a/config.go b/config.go index 1636434..3ae4066 100644 --- a/config.go +++ b/config.go @@ -17,6 +17,7 @@ type Config struct { GoogleIdentity string `json:"google_identity"` JigglerEnabled bool `json:"jiggler_enabled"` AutoUpdateEnabled bool `json:"auto_update_enabled"` + KeyboardLayout string `json:"keyboard_layout"` IncludePreRelease bool `json:"include_pre_release"` HashedPassword string `json:"hashed_password"` LocalAuthToken string `json:"local_auth_token"` @@ -29,6 +30,7 @@ const configPath = "/userdata/kvm_config.json" var defaultConfig = &Config{ CloudURL: "https://api.jetkvm.com", AutoUpdateEnabled: true, // Set a default value + KeyboardLayout: "us", } var config *Config diff --git a/jsonrpc.go b/jsonrpc.go index 2ce5f18..f7543bb 100644 --- a/jsonrpc.go +++ b/jsonrpc.go @@ -131,6 +131,18 @@ func rpcGetDeviceID() (string, error) { return GetDeviceID(), nil } +func rpcGetKeyboardLayout() (string, error) { + return config.KeyboardLayout, nil +} + +func rpcSetKeyboardLayout(KeyboardLayout string) (string, error) { + config.KeyboardLayout = KeyboardLayout + if err := SaveConfig(); err != nil { + return config.KeyboardLayout, fmt.Errorf("failed to save config: %w", err) + } + return KeyboardLayout, nil +} + var streamFactor = 1.0 func rpcGetStreamQualityFactor() (float64, error) { @@ -523,6 +535,8 @@ var rpcHandlers = map[string]RPCHandler{ "setJigglerState": {Func: rpcSetJigglerState, Params: []string{"enabled"}}, "getJigglerState": {Func: rpcGetJigglerState}, "sendWOLMagicPacket": {Func: rpcSendWOLMagicPacket, Params: []string{"macAddress"}}, + "getKeyboardLayout": {Func: rpcGetKeyboardLayout}, + "setKeyboardLayout": {Func: rpcSetKeyboardLayout, Params: []string{"kbLayout"}}, "getStreamQualityFactor": {Func: rpcGetStreamQualityFactor}, "setStreamQualityFactor": {Func: rpcSetStreamQualityFactor, Params: []string{"factor"}}, "getAutoUpdateState": {Func: rpcGetAutoUpdateState}, diff --git a/ui/src/components/InfoBar.tsx b/ui/src/components/InfoBar.tsx index be94043..4490afe 100644 --- a/ui/src/components/InfoBar.tsx +++ b/ui/src/components/InfoBar.tsx @@ -6,10 +6,21 @@ import { useSettingsStore, useVideoStore, } from "@/hooks/stores"; -import { useEffect } from "react"; -import { keys, modifiers } from "@/keyboardMappings"; +import { useEffect, useState } from "react"; +import { keyboardMappingsStore } from "@/keyboardMappings/KeyboardMappingStore"; export default function InfoBar() { + const [keys, setKeys] = useState(keyboardMappingsStore.keys); + const [modifiers, setModifiers] = useState(keyboardMappingsStore.modifiers); + + useEffect(() => { + const unsubscribeKeyboardStore = keyboardMappingsStore.subscribe(() => { + setKeys(keyboardMappingsStore.keys); + setModifiers(keyboardMappingsStore.modifiers); + }); + return unsubscribeKeyboardStore; // Cleanup on unmount + }, []); + const activeKeys = useHidStore(state => state.activeKeys); const activeModifiers = useHidStore(state => state.activeModifiers); const mouseX = useMouseStore(state => state.mouseX); diff --git a/ui/src/components/VirtualKeyboard.tsx b/ui/src/components/VirtualKeyboard.tsx index e3858c0..f056c11 100644 --- a/ui/src/components/VirtualKeyboard.tsx +++ b/ui/src/components/VirtualKeyboard.tsx @@ -7,7 +7,8 @@ import "react-simple-keyboard/build/css/index.css"; import { useHidStore, useUiStore } from "@/hooks/stores"; import { Transition } from "@headlessui/react"; import { cx } from "@/cva.config"; -import { keys, modifiers } from "@/keyboardMappings"; +//import { keys, modifiers } from "@/keyboardMappings/KeyboardMappingStore"; +import { keyboardMappingsStore } from "@/keyboardMappings/KeyboardMappingStore"; import useKeyboard from "@/hooks/useKeyboard"; import DetachIconRaw from "@/assets/detach-icon.svg"; import AttachIconRaw from "@/assets/attach-icon.svg"; @@ -21,6 +22,17 @@ const AttachIcon = ({ className }: { className?: string }) => { }; function KeyboardWrapper() { + const [keys, setKeys] = useState(keyboardMappingsStore.keys); + const [modifiers, setModifiers] = useState(keyboardMappingsStore.modifiers); + + useEffect(() => { + const unsubscribeKeyboardStore = keyboardMappingsStore.subscribe(() => { + setKeys(keyboardMappingsStore.keys); + setModifiers(keyboardMappingsStore.modifiers); + }); + return unsubscribeKeyboardStore; // Cleanup on unmount + }, []); + const [layoutName, setLayoutName] = useState("default"); const keyboardRef = useRef(null); diff --git a/ui/src/components/WebRTCVideo.tsx b/ui/src/components/WebRTCVideo.tsx index f5f083b..8e6b867 100644 --- a/ui/src/components/WebRTCVideo.tsx +++ b/ui/src/components/WebRTCVideo.tsx @@ -7,7 +7,7 @@ import { useUiStore, useVideoStore, } from "@/hooks/stores"; -import { keys, modifiers } from "@/keyboardMappings"; +import { keyboardMappingsStore } from "@/keyboardMappings/KeyboardMappingStore"; import { useResizeObserver } from "@/hooks/useResizeObserver"; import { cx } from "@/cva.config"; import VirtualKeyboard from "@components/VirtualKeyboard"; @@ -18,6 +18,17 @@ import { useJsonRpc } from "@/hooks/useJsonRpc"; import { ConnectionErrorOverlay, HDMIErrorOverlay, LoadingOverlay } from "./VideoOverlay"; export default function WebRTCVideo() { + const [keys, setKeys] = useState(keyboardMappingsStore.keys); + const [modifiers, setModifiers] = useState(keyboardMappingsStore.modifiers); + + useEffect(() => { + const unsubscribeKeyboardStore = keyboardMappingsStore.subscribe(() => { + setKeys(keyboardMappingsStore.keys); + setModifiers(keyboardMappingsStore.modifiers); + }); + return unsubscribeKeyboardStore; // Cleanup on unmount + }, []); + // Video and stream related refs and states const videoElm = useRef(null); const mediaStream = useRTCStore(state => state.mediaStream); diff --git a/ui/src/components/popovers/PasteModal.tsx b/ui/src/components/popovers/PasteModal.tsx index 661c48d..6b3878f 100644 --- a/ui/src/components/popovers/PasteModal.tsx +++ b/ui/src/components/popovers/PasteModal.tsx @@ -9,13 +9,26 @@ import { useCallback, useEffect, useRef, useState } from "react"; import { LuCornerDownLeft } from "react-icons/lu"; import { ExclamationCircleIcon } from "@heroicons/react/16/solid"; import { useClose } from "@headlessui/react"; -import { chars, keys, modifiers } from "@/keyboardMappings"; +import { keyboardMappingsStore } from "@/keyboardMappings/KeyboardMappingStore"; const hidKeyboardPayload = (keys: number[], modifier: number) => { return { keys, modifier }; }; export default function PasteModal() { + const [keys, setKeys] = useState(keyboardMappingsStore.keys); + const [chars, setChars] = useState(keyboardMappingsStore.chars); + const [modifiers, setModifiers] = useState(keyboardMappingsStore.modifiers); + + useEffect(() => { + const unsubscribeKeyboardStore = keyboardMappingsStore.subscribe(() => { + setKeys(keyboardMappingsStore.keys); + setChars(keyboardMappingsStore.chars); + setModifiers(keyboardMappingsStore.modifiers); + }); + return unsubscribeKeyboardStore; // Cleanup on unmount + }, []); + const TextAreaRef = useRef(null); const setPasteMode = useHidStore(state => state.setPasteModeEnabled); const setDisableVideoFocusTrap = useUiStore(state => state.setDisableVideoFocusTrap); @@ -41,13 +54,18 @@ export default function PasteModal() { try { for (const char of text) { - const { key, shift } = chars[char] ?? {}; + const { key, shift, alt } = chars[char] ?? {}; if (!key) continue; + // Build the modifier bitmask + const modifier = + (shift ? modifiers["ShiftLeft"] : 0) | + (alt ? modifiers["AltLeft"] : 0); + await new Promise((resolve, reject) => { send( "keyboardReport", - hidKeyboardPayload([keys[key]], shift ? modifiers["ShiftLeft"] : 0), + hidKeyboardPayload([keys[key]], modifier), params => { if ("error" in params) return reject(params.error); send("keyboardReport", hidKeyboardPayload([], 0), params => { diff --git a/ui/src/components/sidebar/settings.tsx b/ui/src/components/sidebar/settings.tsx index ec606a6..f7bd99a 100644 --- a/ui/src/components/sidebar/settings.tsx +++ b/ui/src/components/sidebar/settings.tsx @@ -25,6 +25,8 @@ import LocalAuthPasswordDialog from "@/components/LocalAuthPasswordDialog"; import { LocalDevice } from "@routes/devices.$id"; import { useRevalidator } from "react-router-dom"; import { ShieldCheckIcon } from "@heroicons/react/20/solid"; +import { keyboardMappingsStore } from "@/keyboardMappings/KeyboardMappingStore"; +import { KeyboardLayout } from "@/keyboardMappings/KeyboardLayouts"; export function SettingsItem({ title, @@ -77,6 +79,7 @@ export default function SettingsSidebar() { const setSidebarView = useUiStore(state => state.setSidebarView); const settings = useSettingsStore(); const [send] = useJsonRpc(); + const [keyboardLayout, setKeyboardLayout] = useState("us"); const [streamQuality, setStreamQuality] = useState("1"); const [autoUpdate, setAutoUpdate] = useState(true); const [devChannel, setDevChannel] = useState(false); @@ -146,6 +149,20 @@ export default function SettingsSidebar() { }); }; + const handleKeyboardLayoutChange = (keyboardLayout: string) => { + send("setKeyboardLayout", { kbLayout: keyboardLayout }, resp => { + if ("error" in resp) { + notifications.error( + `Failed to set keyboard layout: ${resp.error.data || "Unknown error"}`, + ); + return; + } + // TODO set this to update to the actual layout chosen + keyboardMappingsStore.setLayout(KeyboardLayout.UKApple) + setKeyboardLayout(keyboardLayout); + }); + }; + const handleStreamQualityChange = (factor: string) => { send("setStreamQualityFactor", { factor: Number(factor) }, resp => { if ("error" in resp) { @@ -274,6 +291,11 @@ export default function SettingsSidebar() { setDevChannel(resp.result as boolean); }); + send("getKeyboardLayout", {}, resp => { + if ("error" in resp) return; + setKeyboardLayout(String(resp.result)); + }); + send("getStreamQualityFactor", {}, resp => { if ("error" in resp) return; setStreamQuality(String(resp.result)); @@ -509,6 +531,33 @@ export default function SettingsSidebar() {
+
+ +
+ + handleKeyboardLayoutChange(e.target.value)} + /> + +
+
+
; - -export const chars = { - A: { key: "KeyA", shift: true }, - B: { key: "KeyB", shift: true }, - C: { key: "KeyC", shift: true }, - D: { key: "KeyD", shift: true }, - E: { key: "KeyE", shift: true }, - F: { key: "KeyF", shift: true }, - G: { key: "KeyG", shift: true }, - H: { key: "KeyH", shift: true }, - I: { key: "KeyI", shift: true }, - J: { key: "KeyJ", shift: true }, - K: { key: "KeyK", shift: true }, - L: { key: "KeyL", shift: true }, - M: { key: "KeyM", shift: true }, - N: { key: "KeyN", shift: true }, - O: { key: "KeyO", shift: true }, - P: { key: "KeyP", shift: true }, - Q: { key: "KeyQ", shift: true }, - R: { key: "KeyR", shift: true }, - S: { key: "KeyS", shift: true }, - T: { key: "KeyT", shift: true }, - U: { key: "KeyU", shift: true }, - V: { key: "KeyV", shift: true }, - W: { key: "KeyW", shift: true }, - X: { key: "KeyX", shift: true }, - Y: { key: "KeyY", shift: true }, - Z: { key: "KeyZ", shift: true }, - a: { key: "KeyA", shift: false }, - b: { key: "KeyB", shift: false }, - c: { key: "KeyC", shift: false }, - d: { key: "KeyD", shift: false }, - e: { key: "KeyE", shift: false }, - f: { key: "KeyF", shift: false }, - g: { key: "KeyG", shift: false }, - h: { key: "KeyH", shift: false }, - i: { key: "KeyI", shift: false }, - j: { key: "KeyJ", shift: false }, - k: { key: "KeyK", shift: false }, - l: { key: "KeyL", shift: false }, - m: { key: "KeyM", shift: false }, - n: { key: "KeyN", shift: false }, - o: { key: "KeyO", shift: false }, - p: { key: "KeyP", shift: false }, - q: { key: "KeyQ", shift: false }, - r: { key: "KeyR", shift: false }, - s: { key: "KeyS", shift: false }, - t: { key: "KeyT", shift: false }, - u: { key: "KeyU", shift: false }, - v: { key: "KeyV", shift: false }, - w: { key: "KeyW", shift: false }, - x: { key: "KeyX", shift: false }, - y: { key: "KeyY", shift: false }, - z: { key: "KeyZ", shift: false }, - 1: { key: "Digit1", shift: false }, - "!": { key: "Digit1", shift: true }, - 2: { key: "Digit2", shift: false }, - "@": { key: "Digit2", shift: true }, - 3: { key: "Digit3", shift: false }, - "#": { key: "Digit3", shift: true }, - 4: { key: "Digit4", shift: false }, - $: { key: "Digit4", shift: true }, - "%": { key: "Digit5", shift: true }, - 5: { key: "Digit5", shift: false }, - "^": { key: "Digit6", shift: true }, - 6: { key: "Digit6", shift: false }, - "&": { key: "Digit7", shift: true }, - 7: { key: "Digit7", shift: false }, - "*": { key: "Digit8", shift: true }, - 8: { key: "Digit8", shift: false }, - "(": { key: "Digit9", shift: true }, - 9: { key: "Digit9", shift: false }, - ")": { key: "Digit0", shift: true }, - 0: { key: "Digit0", shift: false }, - "-": { key: "Minus", shift: false }, - _: { key: "Minus", shift: true }, - "=": { key: "Equal", shift: false }, - "+": { key: "Equal", shift: true }, - "'": { key: "Quote", shift: false }, - '"': { key: "Quote", shift: true }, - ",": { key: "Comma", shift: false }, - "<": { key: "Comma", shift: true }, - "/": { key: "Slash", shift: false }, - "?": { key: "Slash", shift: true }, - ".": { key: "Period", shift: false }, - ">": { key: "Period", shift: true }, - ";": { key: "Semicolon", shift: false }, - ":": { key: "Semicolon", shift: true }, - "[": { key: "BracketLeft", shift: false }, - "{": { key: "BracketLeft", shift: true }, - "]": { key: "BracketRight", shift: false }, - "}": { key: "BracketRight", shift: true }, - "\\": { key: "Backslash", shift: false }, - "|": { key: "Backslash", shift: true }, - "`": { key: "Backquote", shift: false }, - "~": { key: "Backquote", shift: true }, - "§": { key: "IntlBackslash", shift: false }, - "±": { key: "IntlBackslash", shift: true }, - " ": { key: "Space", shift: false }, - "\n": { key: "Enter", shift: false }, - Enter: { key: "Enter", shift: false }, - Tab: { key: "Tab", shift: false }, -} as Record; - -export const modifiers = { - ControlLeft: 0x01, - ControlRight: 0x10, - ShiftLeft: 0x02, - ShiftRight: 0x20, - AltLeft: 0x04, - AltRight: 0x40, - MetaLeft: 0x08, - MetaRight: 0x80, -} as Record; diff --git a/ui/src/keyboardMappings/KeyboardLayouts.ts b/ui/src/keyboardMappings/KeyboardLayouts.ts new file mode 100644 index 0000000..baadeab --- /dev/null +++ b/ui/src/keyboardMappings/KeyboardLayouts.ts @@ -0,0 +1,25 @@ +import {keysUKApple, charsUKApple, modifiersUKApple } from './layouts/uk_apple'; +import {keysUS, charsUS, modifiersUS } from './layouts/us'; + +export enum KeyboardLayout { + US = "us", + UKApple = "uk_apple", + } + +export function getKeyboardMappings(layout: KeyboardLayout) { + switch (layout) { + case KeyboardLayout.UKApple: + return { + keys: keysUKApple, + chars: charsUKApple, + modifiers: modifiersUKApple, + }; + case KeyboardLayout.US: + default: + return { + keys: keysUS, + chars: charsUS, + modifiers: modifiersUS, + }; + } + } \ No newline at end of file diff --git a/ui/src/keyboardMappings/KeyboardMappingStore.ts b/ui/src/keyboardMappings/KeyboardMappingStore.ts new file mode 100644 index 0000000..2d41bc1 --- /dev/null +++ b/ui/src/keyboardMappings/KeyboardMappingStore.ts @@ -0,0 +1,39 @@ +import { getKeyboardMappings, KeyboardLayout } from "@/keyboardMappings/KeyboardLayouts"; + +// TODO Move this in with all the other stores? + +class KeyboardMappingsStore { + private _layout: KeyboardLayout = KeyboardLayout.US; + private _subscribers: (() => void)[] = []; + + public keys = getKeyboardMappings(this._layout).keys; + public chars = getKeyboardMappings(this._layout).chars; + public modifiers = getKeyboardMappings(this._layout).modifiers; + + setLayout(newLayout: KeyboardLayout) { + if (this._layout === newLayout) return; + this._layout = newLayout; + const updatedMappings = getKeyboardMappings(newLayout); + this.keys = updatedMappings.keys; + this.chars = updatedMappings.chars; + this.modifiers = updatedMappings.modifiers; + this._notifySubscribers(); + } + + getLayout() { + return this._layout; + } + + subscribe(callback: () => void) { + this._subscribers.push(callback); + return () => { + this._subscribers = this._subscribers.filter(sub => sub !== callback); // Cleanup + }; + } + + private _notifySubscribers() { + this._subscribers.forEach(callback => callback()); + } +} + +export const keyboardMappingsStore = new KeyboardMappingsStore(); \ No newline at end of file diff --git a/ui/src/keyboardMappings/layouts/uk_apple.ts b/ui/src/keyboardMappings/layouts/uk_apple.ts new file mode 100644 index 0000000..b9107ea --- /dev/null +++ b/ui/src/keyboardMappings/layouts/uk_apple.ts @@ -0,0 +1,24 @@ +import { charsUS, keysUS, modifiersUS } from "./us"; + +// Extend US Keys with UK Apple-specific changes +export const keysUKApple = { + ...keysUS, +} as Record; + +// Extend US Chars with UK Apple-specific changes +export const charsUKApple = { + ...charsUS, + "`": { key: "Backquote", shift: false }, + "~": { key: "Backquote", shift: true }, + "\\" : { key: "Backslash", shift: false }, + "|": { key: "Backslash", shift: true }, + "#": { key: "Digit3", shift: false, alt: true }, + "£": { key: "Digit3", shift: true }, + "@": { key: "Digit2", shift: true }, + "\"": { key: "Quote", shift: true }, +} as Record; + +// Modifiers are typically the same between UK and US layouts +export const modifiersUKApple = { + ...modifiersUS, +}; \ No newline at end of file diff --git a/ui/src/keyboardMappings/layouts/us.ts b/ui/src/keyboardMappings/layouts/us.ts new file mode 100644 index 0000000..4b75b77 --- /dev/null +++ b/ui/src/keyboardMappings/layouts/us.ts @@ -0,0 +1,215 @@ +export const keysUS = { + AltLeft: 0xe2, + AltRight: 0xe6, + ArrowDown: 0x51, + ArrowLeft: 0x50, + ArrowRight: 0x4f, + ArrowUp: 0x52, + Backquote: 0x35, + Backslash: 0x31, + Backspace: 0x2a, + BracketLeft: 0x2f, + BracketRight: 0x30, + CapsLock: 0x39, + Comma: 0x36, + ContextMenu: 0, + Delete: 0x4c, + Digit0: 0x27, + Digit1: 0x1e, + Digit2: 0x1f, + Digit3: 0x20, + Digit4: 0x21, + Digit5: 0x22, + Digit6: 0x23, + Digit7: 0x24, + Digit8: 0x25, + Digit9: 0x26, + End: 0x4d, + Enter: 0x28, + Equal: 0x2e, + Escape: 0x29, + F1: 0x3a, + F2: 0x3b, + F3: 0x3c, + F4: 0x3d, + F5: 0x3e, + F6: 0x3f, + F7: 0x40, + F8: 0x41, + F9: 0x42, + F10: 0x43, + F11: 0x44, + F12: 0x45, + F13: 0x68, + Home: 0x4a, + Insert: 0x49, + IntlBackslash: 0x31, + KeyA: 0x04, + KeyB: 0x05, + KeyC: 0x06, + KeyD: 0x07, + KeyE: 0x08, + KeyF: 0x09, + KeyG: 0x0a, + KeyH: 0x0b, + KeyI: 0x0c, + KeyJ: 0x0d, + KeyK: 0x0e, + KeyL: 0x0f, + KeyM: 0x10, + KeyN: 0x11, + KeyO: 0x12, + KeyP: 0x13, + KeyQ: 0x14, + KeyR: 0x15, + KeyS: 0x16, + KeyT: 0x17, + KeyU: 0x18, + KeyV: 0x19, + KeyW: 0x1a, + KeyX: 0x1b, + KeyY: 0x1c, + KeyZ: 0x1d, + KeypadExclamation: 0xcf, + Minus: 0x2d, + NumLock: 0x53, + Numpad0: 0x62, + Numpad1: 0x59, + Numpad2: 0x5a, + Numpad3: 0x5b, + Numpad4: 0x5c, + Numpad5: 0x5d, + Numpad6: 0x5e, + Numpad7: 0x5f, + Numpad8: 0x60, + Numpad9: 0x61, + NumpadAdd: 0x57, + NumpadDivide: 0x54, + NumpadEnter: 0x58, + NumpadMultiply: 0x55, + NumpadSubtract: 0x56, + NumpadDecimal: 0x63, + PageDown: 0x4e, + PageUp: 0x4b, + Period: 0x37, + Quote: 0x34, + Semicolon: 0x33, + Slash: 0x38, + Space: 0x2c, + Tab: 0x2b, +} as Record; + +export const charsUS = { + A: { key: "KeyA", shift: true }, + B: { key: "KeyB", shift: true }, + C: { key: "KeyC", shift: true }, + D: { key: "KeyD", shift: true }, + E: { key: "KeyE", shift: true }, + F: { key: "KeyF", shift: true }, + G: { key: "KeyG", shift: true }, + H: { key: "KeyH", shift: true }, + I: { key: "KeyI", shift: true }, + J: { key: "KeyJ", shift: true }, + K: { key: "KeyK", shift: true }, + L: { key: "KeyL", shift: true }, + M: { key: "KeyM", shift: true }, + N: { key: "KeyN", shift: true }, + O: { key: "KeyO", shift: true }, + P: { key: "KeyP", shift: true }, + Q: { key: "KeyQ", shift: true }, + R: { key: "KeyR", shift: true }, + S: { key: "KeyS", shift: true }, + T: { key: "KeyT", shift: true }, + U: { key: "KeyU", shift: true }, + V: { key: "KeyV", shift: true }, + W: { key: "KeyW", shift: true }, + X: { key: "KeyX", shift: true }, + Y: { key: "KeyY", shift: true }, + Z: { key: "KeyZ", shift: true }, + a: { key: "KeyA", shift: false }, + b: { key: "KeyB", shift: false }, + c: { key: "KeyC", shift: false }, + d: { key: "KeyD", shift: false }, + e: { key: "KeyE", shift: false }, + f: { key: "KeyF", shift: false }, + g: { key: "KeyG", shift: false }, + h: { key: "KeyH", shift: false }, + i: { key: "KeyI", shift: false }, + j: { key: "KeyJ", shift: false }, + k: { key: "KeyK", shift: false }, + l: { key: "KeyL", shift: false }, + m: { key: "KeyM", shift: false }, + n: { key: "KeyN", shift: false }, + o: { key: "KeyO", shift: false }, + p: { key: "KeyP", shift: false }, + q: { key: "KeyQ", shift: false }, + r: { key: "KeyR", shift: false }, + s: { key: "KeyS", shift: false }, + t: { key: "KeyT", shift: false }, + u: { key: "KeyU", shift: false }, + v: { key: "KeyV", shift: false }, + w: { key: "KeyW", shift: false }, + x: { key: "KeyX", shift: false }, + y: { key: "KeyY", shift: false }, + z: { key: "KeyZ", shift: false }, + 1: { key: "Digit1", shift: false }, + "!": { key: "Digit1", shift: true }, + 2: { key: "Digit2", shift: false }, + "@": { key: "Digit2", shift: true }, + 3: { key: "Digit3", shift: false }, + "#": { key: "Digit3", shift: true }, + 4: { key: "Digit4", shift: false }, + $: { key: "Digit4", shift: true }, + "%": { key: "Digit5", shift: true }, + 5: { key: "Digit5", shift: false }, + "^": { key: "Digit6", shift: true }, + 6: { key: "Digit6", shift: false }, + "&": { key: "Digit7", shift: true }, + 7: { key: "Digit7", shift: false }, + "*": { key: "Digit8", shift: true }, + 8: { key: "Digit8", shift: false }, + "(": { key: "Digit9", shift: true }, + 9: { key: "Digit9", shift: false }, + ")": { key: "Digit0", shift: true }, + 0: { key: "Digit0", shift: false }, + "-": { key: "Minus", shift: false }, + _: { key: "Minus", shift: true }, + "=": { key: "Equal", shift: false }, + "+": { key: "Equal", shift: true }, + "'": { key: "Quote", shift: false }, + '"': { key: "Quote", shift: true }, + ",": { key: "Comma", shift: false }, + "<": { key: "Comma", shift: true }, + "/": { key: "Slash", shift: false }, + "?": { key: "Slash", shift: true }, + ".": { key: "Period", shift: false }, + ">": { key: "Period", shift: true }, + ";": { key: "Semicolon", shift: false }, + ":": { key: "Semicolon", shift: true }, + "[": { key: "BracketLeft", shift: false }, + "{": { key: "BracketLeft", shift: true }, + "]": { key: "BracketRight", shift: false }, + "}": { key: "BracketRight", shift: true }, + "\\": { key: "Backslash", shift: false }, + "|": { key: "Backslash", shift: true }, + "`": { key: "Backquote", shift: false }, + "~": { key: "Backquote", shift: true }, + "§": { key: "IntlBackslash", shift: false }, + "±": { key: "IntlBackslash", shift: true }, + " ": { key: "Space", shift: false }, + "\n": { key: "Enter", shift: false }, + Enter: { key: "Enter", shift: false }, + Tab: { key: "Tab", shift: false }, +} as Record; + +export const modifiersUS = { + ControlLeft: 0x01, + ControlRight: 0x10, + ShiftLeft: 0x02, + ShiftRight: 0x20, + AltLeft: 0x04, + AltRight: 0x40, + MetaLeft: 0x08, + MetaRight: 0x80, +} as Record; + \ No newline at end of file From 7c40e2e01109e11df912900450ee04d1832793ec Mon Sep 17 00:00:00 2001 From: William Johnstone Date: Sat, 1 Feb 2025 18:38:40 +0000 Subject: [PATCH 2/5] Move keyboardmapping store to stores.ts, simplified some things, updated settings.tsx to set the keyboard layout properly. --- ui/src/components/InfoBar.tsx | 12 +++--- ui/src/components/VirtualKeyboard.tsx | 14 +++---- ui/src/components/WebRTCVideo.tsx | 20 ++++++---- ui/src/components/popovers/PasteModal.tsx | 22 +++++------ ui/src/components/sidebar/settings.tsx | 9 ++--- ui/src/hooks/stores.ts | 37 ++++++++++++++++++ ui/src/keyboardMappings/KeyboardLayouts.ts | 25 +++++------- .../keyboardMappings/KeyboardMappingStore.ts | 39 ------------------- ui/src/keyboardMappings/layouts/uk.ts | 0 ui/src/keyboardMappings/layouts/uk_apple.ts | 4 +- ui/src/keyboardMappings/layouts/us.ts | 4 +- 11 files changed, 90 insertions(+), 96 deletions(-) delete mode 100644 ui/src/keyboardMappings/KeyboardMappingStore.ts create mode 100644 ui/src/keyboardMappings/layouts/uk.ts diff --git a/ui/src/components/InfoBar.tsx b/ui/src/components/InfoBar.tsx index 4490afe..dbbaa40 100644 --- a/ui/src/components/InfoBar.tsx +++ b/ui/src/components/InfoBar.tsx @@ -5,18 +5,18 @@ import { useRTCStore, useSettingsStore, useVideoStore, + useKeyboardMappingsStore, } from "@/hooks/stores"; import { useEffect, useState } from "react"; -import { keyboardMappingsStore } from "@/keyboardMappings/KeyboardMappingStore"; export default function InfoBar() { - const [keys, setKeys] = useState(keyboardMappingsStore.keys); - const [modifiers, setModifiers] = useState(keyboardMappingsStore.modifiers); + const [keys, setKeys] = useState(useKeyboardMappingsStore.keys); + const [modifiers, setModifiers] = useState(useKeyboardMappingsStore.modifiers); useEffect(() => { - const unsubscribeKeyboardStore = keyboardMappingsStore.subscribe(() => { - setKeys(keyboardMappingsStore.keys); - setModifiers(keyboardMappingsStore.modifiers); + const unsubscribeKeyboardStore = useKeyboardMappingsStore.subscribe(() => { + setKeys(useKeyboardMappingsStore.keys); + setModifiers(useKeyboardMappingsStore.modifiers); }); return unsubscribeKeyboardStore; // Cleanup on unmount }, []); diff --git a/ui/src/components/VirtualKeyboard.tsx b/ui/src/components/VirtualKeyboard.tsx index f056c11..fab78f9 100644 --- a/ui/src/components/VirtualKeyboard.tsx +++ b/ui/src/components/VirtualKeyboard.tsx @@ -4,11 +4,9 @@ import { Button } from "@components/Button"; import Card from "@components/Card"; import { ChevronDownIcon } from "@heroicons/react/16/solid"; import "react-simple-keyboard/build/css/index.css"; -import { useHidStore, useUiStore } from "@/hooks/stores"; +import { useHidStore, useUiStore, useKeyboardMappingsStore } from "@/hooks/stores"; import { Transition } from "@headlessui/react"; import { cx } from "@/cva.config"; -//import { keys, modifiers } from "@/keyboardMappings/KeyboardMappingStore"; -import { keyboardMappingsStore } from "@/keyboardMappings/KeyboardMappingStore"; import useKeyboard from "@/hooks/useKeyboard"; import DetachIconRaw from "@/assets/detach-icon.svg"; import AttachIconRaw from "@/assets/attach-icon.svg"; @@ -22,13 +20,13 @@ const AttachIcon = ({ className }: { className?: string }) => { }; function KeyboardWrapper() { - const [keys, setKeys] = useState(keyboardMappingsStore.keys); - const [modifiers, setModifiers] = useState(keyboardMappingsStore.modifiers); + const [keys, setKeys] = useState(useKeyboardMappingsStore.keys); + const [modifiers, setModifiers] = useState(useKeyboardMappingsStore.modifiers); useEffect(() => { - const unsubscribeKeyboardStore = keyboardMappingsStore.subscribe(() => { - setKeys(keyboardMappingsStore.keys); - setModifiers(keyboardMappingsStore.modifiers); + const unsubscribeKeyboardStore = useKeyboardMappingsStore.subscribe(() => { + setKeys(useKeyboardMappingsStore.keys); + setModifiers(useKeyboardMappingsStore.modifiers); }); return unsubscribeKeyboardStore; // Cleanup on unmount }, []); diff --git a/ui/src/components/WebRTCVideo.tsx b/ui/src/components/WebRTCVideo.tsx index 8e6b867..61b1f9a 100644 --- a/ui/src/components/WebRTCVideo.tsx +++ b/ui/src/components/WebRTCVideo.tsx @@ -6,8 +6,8 @@ import { useSettingsStore, useUiStore, useVideoStore, + useKeyboardMappingsStore, } from "@/hooks/stores"; -import { keyboardMappingsStore } from "@/keyboardMappings/KeyboardMappingStore"; import { useResizeObserver } from "@/hooks/useResizeObserver"; import { cx } from "@/cva.config"; import VirtualKeyboard from "@components/VirtualKeyboard"; @@ -18,13 +18,13 @@ import { useJsonRpc } from "@/hooks/useJsonRpc"; import { ConnectionErrorOverlay, HDMIErrorOverlay, LoadingOverlay } from "./VideoOverlay"; export default function WebRTCVideo() { - const [keys, setKeys] = useState(keyboardMappingsStore.keys); - const [modifiers, setModifiers] = useState(keyboardMappingsStore.modifiers); + const [keys, setKeys] = useState(useKeyboardMappingsStore.keys); + const [modifiers, setModifiers] = useState(useKeyboardMappingsStore.modifiers); useEffect(() => { - const unsubscribeKeyboardStore = keyboardMappingsStore.subscribe(() => { - setKeys(keyboardMappingsStore.keys); - setModifiers(keyboardMappingsStore.modifiers); + const unsubscribeKeyboardStore = useKeyboardMappingsStore.subscribe(() => { + setKeys(useKeyboardMappingsStore.keys); + setModifiers(useKeyboardMappingsStore.modifiers); }); return unsubscribeKeyboardStore; // Cleanup on unmount }, []); @@ -218,12 +218,15 @@ export default function WebRTCVideo() { const prev = useHidStore.getState(); let code = e.code; const key = e.key; + console.log(e); + console.log(key); - // if (document.activeElement?.id !== "videoFocusTrap") { + // if (document.activeElement?.id !== "videoFocusTrap") {hH // console.log("KEYUP: Not focusing on the video", document.activeElement); // return; // } - console.log(document.activeElement); + // + // console.log(document.activeElement); setIsNumLockActive(e.getModifierState("NumLock")); setIsCapsLockActive(e.getModifierState("CapsLock")); @@ -289,6 +292,7 @@ export default function WebRTCVideo() { prev.activeModifiers.filter(k => k !== modifiers[e.code]), ); + console.log(e.key); sendKeyboardEvent([...new Set(newKeys)], [...new Set(newModifiers)]); }, [ diff --git a/ui/src/components/popovers/PasteModal.tsx b/ui/src/components/popovers/PasteModal.tsx index 6b3878f..2387888 100644 --- a/ui/src/components/popovers/PasteModal.tsx +++ b/ui/src/components/popovers/PasteModal.tsx @@ -3,28 +3,27 @@ import { GridCard } from "@components/Card"; import { TextAreaWithLabel } from "@components/TextArea"; import { SectionHeader } from "@components/SectionHeader"; import { useJsonRpc } from "@/hooks/useJsonRpc"; -import { useHidStore, useRTCStore, useUiStore } from "@/hooks/stores"; +import { useHidStore, useRTCStore, useUiStore, useKeyboardMappingsStore } from "@/hooks/stores"; import notifications from "../../notifications"; import { useCallback, useEffect, useRef, useState } from "react"; import { LuCornerDownLeft } from "react-icons/lu"; import { ExclamationCircleIcon } from "@heroicons/react/16/solid"; import { useClose } from "@headlessui/react"; -import { keyboardMappingsStore } from "@/keyboardMappings/KeyboardMappingStore"; const hidKeyboardPayload = (keys: number[], modifier: number) => { return { keys, modifier }; }; export default function PasteModal() { - const [keys, setKeys] = useState(keyboardMappingsStore.keys); - const [chars, setChars] = useState(keyboardMappingsStore.chars); - const [modifiers, setModifiers] = useState(keyboardMappingsStore.modifiers); + const [keys, setKeys] = useState(useKeyboardMappingsStore.keys); + const [chars, setChars] = useState(useKeyboardMappingsStore.chars); + const [modifiers, setModifiers] = useState(useKeyboardMappingsStore.modifiers); useEffect(() => { - const unsubscribeKeyboardStore = keyboardMappingsStore.subscribe(() => { - setKeys(keyboardMappingsStore.keys); - setChars(keyboardMappingsStore.chars); - setModifiers(keyboardMappingsStore.modifiers); + const unsubscribeKeyboardStore = useKeyboardMappingsStore.subscribe(() => { + setKeys(useKeyboardMappingsStore.keys); + setChars(useKeyboardMappingsStore.chars); + setModifiers(useKeyboardMappingsStore.modifiers); }); return unsubscribeKeyboardStore; // Cleanup on unmount }, []); @@ -54,13 +53,14 @@ export default function PasteModal() { try { for (const char of text) { - const { key, shift, alt } = chars[char] ?? {}; + const { key, shift, altLeft, altRight } = chars[char] ?? {}; if (!key) continue; // Build the modifier bitmask const modifier = (shift ? modifiers["ShiftLeft"] : 0) | - (alt ? modifiers["AltLeft"] : 0); + (altLeft ? modifiers["AltLeft"] : 0) | + (altRight ? modifiers["AltRight"] : 0); // This is important for a lot of keyboard layouts, right and left alt have different functions await new Promise((resolve, reject) => { send( diff --git a/ui/src/components/sidebar/settings.tsx b/ui/src/components/sidebar/settings.tsx index f7bd99a..a6c3fe1 100644 --- a/ui/src/components/sidebar/settings.tsx +++ b/ui/src/components/sidebar/settings.tsx @@ -4,6 +4,7 @@ import { useSettingsStore, useUiStore, useUpdateStore, + useKeyboardMappingsStore, } from "@/hooks/stores"; import { Checkbox } from "@components/Checkbox"; import { Button, LinkButton } from "@components/Button"; @@ -25,8 +26,6 @@ import LocalAuthPasswordDialog from "@/components/LocalAuthPasswordDialog"; import { LocalDevice } from "@routes/devices.$id"; import { useRevalidator } from "react-router-dom"; import { ShieldCheckIcon } from "@heroicons/react/20/solid"; -import { keyboardMappingsStore } from "@/keyboardMappings/KeyboardMappingStore"; -import { KeyboardLayout } from "@/keyboardMappings/KeyboardLayouts"; export function SettingsItem({ title, @@ -157,8 +156,7 @@ export default function SettingsSidebar() { ); return; } - // TODO set this to update to the actual layout chosen - keyboardMappingsStore.setLayout(KeyboardLayout.UKApple) + useKeyboardMappingsStore.setLayout(keyboardLayout) setKeyboardLayout(keyboardLayout); }); }; @@ -294,6 +292,7 @@ export default function SettingsSidebar() { send("getKeyboardLayout", {}, resp => { if ("error" in resp) return; setKeyboardLayout(String(resp.result)); + useKeyboardMappingsStore.setLayout(String(resp.result)) }); send("getStreamQualityFactor", {}, resp => { @@ -545,7 +544,7 @@ export default function SettingsSidebar() { size="SM" label="" // TODO figure out how to make this selector wider like the EDID one? - //fullWidth + //fullWidthƒ value={keyboardLayout} options={[ { value: "uk", label: "GB" }, diff --git a/ui/src/hooks/stores.ts b/ui/src/hooks/stores.ts index b4cfbec..29daa82 100644 --- a/ui/src/hooks/stores.ts +++ b/ui/src/hooks/stores.ts @@ -1,5 +1,6 @@ import { create } from "zustand"; import { createJSONStorage, persist } from "zustand/middleware"; +import { getKeyboardMappings } from "@/keyboardMappings/KeyboardLayouts"; // Utility function to append stats to a Map const appendStatToMap = ( @@ -528,3 +529,39 @@ export const useLocalAuthModalStore = create(set => ({ setModalView: view => set({ modalView: view }), setErrorMessage: message => set({ errorMessage: message }), })); + +class KeyboardMappingsStore { + private _layout: string = 'us'; + private _subscribers: (() => void)[] = []; + + public keys = getKeyboardMappings(this._layout).keys; + public chars = getKeyboardMappings(this._layout).chars; + public modifiers = getKeyboardMappings(this._layout).modifiers; + + setLayout(newLayout: string) { + if (this._layout === newLayout) return; + this._layout = newLayout; + const updatedMappings = getKeyboardMappings(newLayout); + this.keys = updatedMappings.keys; + this.chars = updatedMappings.chars; + this.modifiers = updatedMappings.modifiers; + this._notifySubscribers(); + } + + getLayout() { + return this._layout; + } + + subscribe(callback: () => void) { + this._subscribers.push(callback); + return () => { + this._subscribers = this._subscribers.filter(sub => sub !== callback); // Cleanup + }; + } + + private _notifySubscribers() { + this._subscribers.forEach(callback => callback()); + } +} + +export const useKeyboardMappingsStore = new KeyboardMappingsStore(); \ No newline at end of file diff --git a/ui/src/keyboardMappings/KeyboardLayouts.ts b/ui/src/keyboardMappings/KeyboardLayouts.ts index baadeab..b631d76 100644 --- a/ui/src/keyboardMappings/KeyboardLayouts.ts +++ b/ui/src/keyboardMappings/KeyboardLayouts.ts @@ -1,20 +1,15 @@ import {keysUKApple, charsUKApple, modifiersUKApple } from './layouts/uk_apple'; import {keysUS, charsUS, modifiersUS } from './layouts/us'; -export enum KeyboardLayout { - US = "us", - UKApple = "uk_apple", - } - -export function getKeyboardMappings(layout: KeyboardLayout) { - switch (layout) { - case KeyboardLayout.UKApple: - return { - keys: keysUKApple, - chars: charsUKApple, - modifiers: modifiersUKApple, - }; - case KeyboardLayout.US: +export function getKeyboardMappings(layout: string) { + switch (layout) { + case "uk_apple": + return { + keys: keysUKApple, + chars: charsUKApple, + modifiers: modifiersUKApple, + }; + case "us": default: return { keys: keysUS, @@ -22,4 +17,4 @@ export function getKeyboardMappings(layout: KeyboardLayout) { modifiers: modifiersUS, }; } - } \ No newline at end of file +} \ No newline at end of file diff --git a/ui/src/keyboardMappings/KeyboardMappingStore.ts b/ui/src/keyboardMappings/KeyboardMappingStore.ts deleted file mode 100644 index 2d41bc1..0000000 --- a/ui/src/keyboardMappings/KeyboardMappingStore.ts +++ /dev/null @@ -1,39 +0,0 @@ -import { getKeyboardMappings, KeyboardLayout } from "@/keyboardMappings/KeyboardLayouts"; - -// TODO Move this in with all the other stores? - -class KeyboardMappingsStore { - private _layout: KeyboardLayout = KeyboardLayout.US; - private _subscribers: (() => void)[] = []; - - public keys = getKeyboardMappings(this._layout).keys; - public chars = getKeyboardMappings(this._layout).chars; - public modifiers = getKeyboardMappings(this._layout).modifiers; - - setLayout(newLayout: KeyboardLayout) { - if (this._layout === newLayout) return; - this._layout = newLayout; - const updatedMappings = getKeyboardMappings(newLayout); - this.keys = updatedMappings.keys; - this.chars = updatedMappings.chars; - this.modifiers = updatedMappings.modifiers; - this._notifySubscribers(); - } - - getLayout() { - return this._layout; - } - - subscribe(callback: () => void) { - this._subscribers.push(callback); - return () => { - this._subscribers = this._subscribers.filter(sub => sub !== callback); // Cleanup - }; - } - - private _notifySubscribers() { - this._subscribers.forEach(callback => callback()); - } -} - -export const keyboardMappingsStore = new KeyboardMappingsStore(); \ No newline at end of file diff --git a/ui/src/keyboardMappings/layouts/uk.ts b/ui/src/keyboardMappings/layouts/uk.ts new file mode 100644 index 0000000..e69de29 diff --git a/ui/src/keyboardMappings/layouts/uk_apple.ts b/ui/src/keyboardMappings/layouts/uk_apple.ts index b9107ea..d257f43 100644 --- a/ui/src/keyboardMappings/layouts/uk_apple.ts +++ b/ui/src/keyboardMappings/layouts/uk_apple.ts @@ -12,11 +12,11 @@ export const charsUKApple = { "~": { key: "Backquote", shift: true }, "\\" : { key: "Backslash", shift: false }, "|": { key: "Backslash", shift: true }, - "#": { key: "Digit3", shift: false, alt: true }, + "#": { key: "Digit3", shift: false, altLeft: true }, "£": { key: "Digit3", shift: true }, "@": { key: "Digit2", shift: true }, "\"": { key: "Quote", shift: true }, -} as Record; +} as Record; // Modifiers are typically the same between UK and US layouts export const modifiersUKApple = { diff --git a/ui/src/keyboardMappings/layouts/us.ts b/ui/src/keyboardMappings/layouts/us.ts index 4b75b77..15a0071 100644 --- a/ui/src/keyboardMappings/layouts/us.ts +++ b/ui/src/keyboardMappings/layouts/us.ts @@ -200,8 +200,8 @@ export const charsUS = { "\n": { key: "Enter", shift: false }, Enter: { key: "Enter", shift: false }, Tab: { key: "Tab", shift: false }, -} as Record; - +} as Record; + export const modifiersUS = { ControlLeft: 0x01, ControlRight: 0x10, From 5f1e53f24a8d00e5bfab79f074b0bd678b0b89c6 Mon Sep 17 00:00:00 2001 From: William Johnstone Date: Mon, 3 Feb 2025 00:39:53 +0000 Subject: [PATCH 3/5] Fully implemented layout setting and switching in the UI. Updated PasteModal to add more clarity to error message. Begin working on key remapping in WebRTC (working to a reasonable degree). --- dev_deploy.sh | 1 + ui/src/components/WebRTCVideo.tsx | 68 ++++++++++++++++++--- ui/src/components/popovers/PasteModal.tsx | 2 +- ui/src/keyboardMappings/layouts/uk_apple.ts | 2 +- ui/src/keyboardMappings/layouts/us.ts | 2 +- 5 files changed, 65 insertions(+), 10 deletions(-) diff --git a/dev_deploy.sh b/dev_deploy.sh index a106395..72bb6dd 100755 --- a/dev_deploy.sh +++ b/dev_deploy.sh @@ -71,6 +71,7 @@ export LD_LIBRARY_PATH=/oem/usr/lib:\$LD_LIBRARY_PATH # Kill any existing instances of the application killall jetkvm_app || true killall jetkvm_app_debug || true +killall jetkvm_native || true # Navigate to the directory where the binary will be stored cd "$REMOTE_PATH" diff --git a/ui/src/components/WebRTCVideo.tsx b/ui/src/components/WebRTCVideo.tsx index 61b1f9a..9b16045 100644 --- a/ui/src/components/WebRTCVideo.tsx +++ b/ui/src/components/WebRTCVideo.tsx @@ -19,11 +19,17 @@ import { ConnectionErrorOverlay, HDMIErrorOverlay, LoadingOverlay } from "./Vide export default function WebRTCVideo() { const [keys, setKeys] = useState(useKeyboardMappingsStore.keys); + const [chars, setChars] = useState(useKeyboardMappingsStore.chars); const [modifiers, setModifiers] = useState(useKeyboardMappingsStore.modifiers); + // TODO move this into stores as well as I think this will need to be used in InfoBar + // This map is used to maintain consistency between localised key mappings + const activeKeyState = useRef>(new Map()); + useEffect(() => { const unsubscribeKeyboardStore = useKeyboardMappingsStore.subscribe(() => { - setKeys(useKeyboardMappingsStore.keys); + setKeys(useKeyboardMappingsStore.keys); + setChars(useKeyboardMappingsStore.chars); setModifiers(useKeyboardMappingsStore.modifiers); }); return unsubscribeKeyboardStore; // Cleanup on unmount @@ -217,9 +223,9 @@ export default function WebRTCVideo() { e.preventDefault(); const prev = useHidStore.getState(); let code = e.code; - const key = e.key; + const localisedKey = e.key; console.log(e); - console.log(key); + console.log("Localised Key: " + localisedKey); // if (document.activeElement?.id !== "videoFocusTrap") {hH // console.log("KEYUP: Not focusing on the video", document.activeElement); @@ -232,25 +238,42 @@ export default function WebRTCVideo() { setIsCapsLockActive(e.getModifierState("CapsLock")); setIsScrollLockActive(e.getModifierState("ScrollLock")); - if (code == "IntlBackslash" && ["`", "~"].includes(key)) { + /*if (code == "IntlBackslash" && ["`", "~"].includes(key)) { code = "Backquote"; } else if (code == "Backquote" && ["§", "±"].includes(key)) { code = "IntlBackslash"; - } + }*/ + + const { key: mappedKey, shift, altLeft, altRight } = chars[localisedKey] ?? { key: e.code }; + //if (!key) continue; + console.log("Mapped Key: " + mappedKey) + console.log("Current KB Layout:" + useKeyboardMappingsStore.getLayout()); + + // Build the modifier bitmask + const modifier = + (shift ? modifiers["ShiftLeft"] : 0) | + (altLeft ? modifiers["AltLeft"] : 0) | + (altRight ? modifiers["AltRight"] : 0); // This is important for a lot of keyboard layouts, right and left alt have different functions + + // Add the mapped key to keyState + activeKeyState.current.set(e.code, { mappedKey, modifiers: modifier }); + console.log(activeKeyState) // Add the key to the active keys - const newKeys = [...prev.activeKeys, keys[code]].filter(Boolean); + const newKeys = [...prev.activeKeys, keys[mappedKey]].filter(Boolean); // Add the modifier to the active modifiers const newModifiers = handleModifierKeys(e, [ ...prev.activeModifiers, modifiers[code], + modifier, //Is this bad, will we have duplicate modifiers? ]); // When pressing the meta key + another key, the key will never trigger a keyup // event, so we need to clear the keys after a short delay // https://bugs.chromium.org/p/chromium/issues/detail?id=28089 // https://bugzilla.mozilla.org/show_bug.cgi?id=1299553 + // TODO add this to the activekey state if (e.metaKey) { setTimeout(() => { const prev = useHidStore.getState(); @@ -283,14 +306,44 @@ export default function WebRTCVideo() { setIsCapsLockActive(e.getModifierState("CapsLock")); setIsScrollLockActive(e.getModifierState("ScrollLock")); + // Retrieve the mapped key and modifiers from keyState + const keyInfo = activeKeyState.current.get(e.code); + if (!keyInfo) return; // Ignore if no record exists + + const { mappedKey, modifiers: modifier } = keyInfo; + + // Remove the key from keyState + activeKeyState.current.delete(e.code); + + // Filter out the key that was just released + const newKeys = prev.activeKeys.filter(k => k !== keys[mappedKey]).filter(Boolean); + console.log(activeKeyState) + + // Filter out the associated modifier + //const newModifiers = prev.activeModifiers.filter(k => k !== modifier).filter(Boolean); + const newModifiers = handleModifierKeys( + e, + prev.activeModifiers.filter(k => k !== modifier), + ); + /* + const { key: mappedKey/*, shift, altLeft, altRight*//* } = chars[e.key] ?? { key: e.code }; + //if (!key) continue; + console.log("Mapped Key: " + mappedKey) + // Build the modifier bitmask + /*const modifier = + (shift ? modifiers["ShiftLeft"] : 0) | + (altLeft ? modifiers["AltLeft"] : 0) | + (altRight ? modifiers["AltRight"] : 0); // This is important for a lot of keyboard layouts, right and left alt have different functions*//* + // Filtering out the key that was just released (keys[e.code]) - const newKeys = prev.activeKeys.filter(k => k !== keys[e.code]).filter(Boolean); + const newKeys = prev.activeKeys.filter(k => k !== keys[mappedKey]).filter(Boolean); // Filter out the modifier that was just released const newModifiers = handleModifierKeys( e, prev.activeModifiers.filter(k => k !== modifiers[e.code]), ); + */ console.log(e.key); sendKeyboardEvent([...new Set(newKeys)], [...new Set(newModifiers)]); @@ -321,6 +374,7 @@ export default function WebRTCVideo() { return () => { abortController.abort(); + activeKeyState.current.clear(); }; }, [keyDownHandler, keyUpHandler, resetKeyboardState, sendKeyboardEvent], diff --git a/ui/src/components/popovers/PasteModal.tsx b/ui/src/components/popovers/PasteModal.tsx index 2387888..9728cb8 100644 --- a/ui/src/components/popovers/PasteModal.tsx +++ b/ui/src/components/popovers/PasteModal.tsx @@ -141,7 +141,7 @@ export default function PasteModal() {
- The following characters won't be pasted:{" "} + The following characters won't be pasted as the current keyboard layout does not contain a valid mapping:{" "} {invalidChars.join(", ")}
diff --git a/ui/src/keyboardMappings/layouts/uk_apple.ts b/ui/src/keyboardMappings/layouts/uk_apple.ts index d257f43..24d974e 100644 --- a/ui/src/keyboardMappings/layouts/uk_apple.ts +++ b/ui/src/keyboardMappings/layouts/uk_apple.ts @@ -16,7 +16,7 @@ export const charsUKApple = { "£": { key: "Digit3", shift: true }, "@": { key: "Digit2", shift: true }, "\"": { key: "Quote", shift: true }, -} as Record; +} as Record; // Modifiers are typically the same between UK and US layouts export const modifiersUKApple = { diff --git a/ui/src/keyboardMappings/layouts/us.ts b/ui/src/keyboardMappings/layouts/us.ts index 15a0071..a5395c2 100644 --- a/ui/src/keyboardMappings/layouts/us.ts +++ b/ui/src/keyboardMappings/layouts/us.ts @@ -200,7 +200,7 @@ export const charsUS = { "\n": { key: "Enter", shift: false }, Enter: { key: "Enter", shift: false }, Tab: { key: "Tab", shift: false }, -} as Record; +} as Record; export const modifiersUS = { ControlLeft: 0x01, From f1de6639ef7526a6d0694a31949f5419878bdd9e Mon Sep 17 00:00:00 2001 From: William Johnstone Date: Tue, 4 Feb 2025 01:45:34 +0000 Subject: [PATCH 4/5] Continued work on keyboard overhaul, found many more issues caused by my changes (YAY!). Also spent 4 hours troubleshooting to find out I didn't realise how useCallback works... :/ Anway, not much longer before work on just the mappings can begin. --- ui/src/components/VirtualKeyboard.tsx | 5 +- ui/src/components/WebRTCVideo.tsx | 96 +++++++++++++++++++-- ui/src/keyboardMappings/layouts/uk_apple.ts | 2 +- 3 files changed, 93 insertions(+), 10 deletions(-) diff --git a/ui/src/components/VirtualKeyboard.tsx b/ui/src/components/VirtualKeyboard.tsx index fab78f9..9f09240 100644 --- a/ui/src/components/VirtualKeyboard.tsx +++ b/ui/src/components/VirtualKeyboard.tsx @@ -20,12 +20,15 @@ const AttachIcon = ({ className }: { className?: string }) => { }; function KeyboardWrapper() { + // TODO implement virtual keyboard mapping const [keys, setKeys] = useState(useKeyboardMappingsStore.keys); + //const [chars, setChars] = useState(useKeyboardMappingsStore.chars); const [modifiers, setModifiers] = useState(useKeyboardMappingsStore.modifiers); useEffect(() => { const unsubscribeKeyboardStore = useKeyboardMappingsStore.subscribe(() => { - setKeys(useKeyboardMappingsStore.keys); + setKeys(useKeyboardMappingsStore.keys); + //setChars(useKeyboardMappingsStore.chars); setModifiers(useKeyboardMappingsStore.modifiers); }); return unsubscribeKeyboardStore; // Cleanup on unmount diff --git a/ui/src/components/WebRTCVideo.tsx b/ui/src/components/WebRTCVideo.tsx index 9b16045..cf3990d 100644 --- a/ui/src/components/WebRTCVideo.tsx +++ b/ui/src/components/WebRTCVideo.tsx @@ -22,9 +22,8 @@ export default function WebRTCVideo() { const [chars, setChars] = useState(useKeyboardMappingsStore.chars); const [modifiers, setModifiers] = useState(useKeyboardMappingsStore.modifiers); - // TODO move this into stores as well as I think this will need to be used in InfoBar // This map is used to maintain consistency between localised key mappings - const activeKeyState = useRef>(new Map()); + const activeKeyState = useRef>(new Map()); useEffect(() => { const unsubscribeKeyboardStore = useKeyboardMappingsStore.subscribe(() => { @@ -140,6 +139,7 @@ export default function WebRTCVideo() { if (blockWheelEvent) return; e.preventDefault(); + // TODO this should be user controllable // Define a scaling factor to adjust scrolling sensitivity const scrollSensitivity = 0.8; // Adjust this value to change scroll speed @@ -155,9 +155,11 @@ export default function WebRTCVideo() { // Invert the scroll value to match expected behavior const invertedScroll = -roundedScroll; + // TODO remove debug logs console.log("wheelReport", { wheelY: invertedScroll }); send("wheelReport", { wheelY: invertedScroll }); + // TODO this is making scrolling feel slow and sluggish, also throwing a violation in chrome setBlockWheelEvent(true); setTimeout(() => setBlockWheelEvent(false), 50); }, @@ -168,6 +170,7 @@ export default function WebRTCVideo() { sendMouseMovement(0, 0, 0); }, [sendMouseMovement]); + // TODO this needs reworked ot work with mappings. // Keyboard-related const handleModifierKeys = useCallback( (e: KeyboardEvent, activeModifiers: number[]) => { @@ -244,19 +247,23 @@ export default function WebRTCVideo() { code = "IntlBackslash"; }*/ - const { key: mappedKey, shift, altLeft, altRight } = chars[localisedKey] ?? { key: e.code }; + const { key: mappedKey, shift, altLeft, altRight } = useKeyboardMappingsStore.chars[localisedKey] ?? { key: e.code }; //if (!key) continue; console.log("Mapped Key: " + mappedKey) console.log("Current KB Layout:" + useKeyboardMappingsStore.getLayout()); + console.log(chars[localisedKey]); // Build the modifier bitmask - const modifier = + const modifierBitmask = (shift ? modifiers["ShiftLeft"] : 0) | (altLeft ? modifiers["AltLeft"] : 0) | (altRight ? modifiers["AltRight"] : 0); // This is important for a lot of keyboard layouts, right and left alt have different functions - + // On second thought this may not be relevant here, may be best to just send altRight through, needs testing + console.log("Modifier Bitmask: " + modifierBitmask) + console.log("Shift: " + shift + ", altLeft: " + altLeft + ", altRight: " + altRight) + // Add the mapped key to keyState - activeKeyState.current.set(e.code, { mappedKey, modifiers: modifier }); + activeKeyState.current.set(e.code, { mappedKey, modifiers: {shift, altLeft, altRight, bitmask: modifierBitmask}}); console.log(activeKeyState) // Add the key to the active keys @@ -266,7 +273,7 @@ export default function WebRTCVideo() { const newModifiers = handleModifierKeys(e, [ ...prev.activeModifiers, modifiers[code], - modifier, //Is this bad, will we have duplicate modifiers? + modifierBitmask, //Is this bad, will we have duplicate modifiers? ]); // When pressing the meta key + another key, the key will never trigger a keyup @@ -289,6 +296,9 @@ export default function WebRTCVideo() { setIsScrollLockActive, handleModifierKeys, sendKeyboardEvent, + chars, + keys, + modifiers, ], ); @@ -306,6 +316,73 @@ export default function WebRTCVideo() { setIsCapsLockActive(e.getModifierState("CapsLock")); setIsScrollLockActive(e.getModifierState("ScrollLock")); + // Check if the released key is a modifier (e.g., Shift, Alt, Control) + const isModifierKey = + e.code === "ShiftLeft" || + e.code === "ShiftRight" || + e.code === "AltLeft" || + e.code === "AltRight"; + //e.code === "ControlLeft" || These shouldn't make a difference for mappings + //e.code === "ControlRight"; + + // Handle modifier release + if (isModifierKey) { + // Update all affected keys when this modifier is released + activeKeyState.current.forEach((value, code) => { + const { mappedKey, modifiers: mappedModifiers} = value; + + // Remove the released modifier from the modifier bitmask + //const updatedModifiers = modifiers & ~modifiers[e.code]; + + // Recalculate the remapped key based on the updated modifiers + //const updatedMappedKey = chars[originalKey]?.key || originalKey; + + var removeCurrentKey = false; + + // Shift Handling + if (mappedModifiers.shift && (e.code === "ShiftLeft" || e.code === "ShiftRight")) { + activeKeyState.current.delete(code); + removeCurrentKey = true; + }; + // Left Alt handling + if (mappedModifiers.altLeft && e.code === "AltLeft") { + activeKeyState.current.delete(code); + removeCurrentKey = true; + }; + // Right Alt handling + if (mappedModifiers.altRight && e.code === "AltRight") { + activeKeyState.current.delete(code); + removeCurrentKey = true; + }; + + var newKeys = prev.activeKeys; + + if (removeCurrentKey) { + newKeys = prev.activeKeys + .filter(k => k !== keys[mappedKey]) // Remove the previously mapped key + //.concat(keys[updatedMappedKey]) // Add the new remapped key, don't need to do this. + .filter(Boolean); + }; + + const newModifiers = prev.activeModifiers.filter(k => k !== modifiers[e.code]); + + // Update the keyState + /*activeKeyState.current.delete(code);/*.set(code, { + mappedKey: updatedMappedKey, + modifiers: updatedModifiers, + originalKey, + });*/ + + // Remove the modifer key from keyState + activeKeyState.current.delete(e.code); + + // Send the updated HID payload + sendKeyboardEvent([...new Set(newKeys)], [...new Set(newModifiers)]); + }); + + return; // Exit as we've already handled the modifier release + } + // Retrieve the mapped key and modifiers from keyState const keyInfo = activeKeyState.current.get(e.code); if (!keyInfo) return; // Ignore if no record exists @@ -323,7 +400,7 @@ export default function WebRTCVideo() { //const newModifiers = prev.activeModifiers.filter(k => k !== modifier).filter(Boolean); const newModifiers = handleModifierKeys( e, - prev.activeModifiers.filter(k => k !== modifier), + prev.activeModifiers.filter(k => k !== modifier.bitmask), ); /* const { key: mappedKey/*, shift, altLeft, altRight*//* } = chars[e.key] ?? { key: e.code }; @@ -354,6 +431,9 @@ export default function WebRTCVideo() { setIsScrollLockActive, handleModifierKeys, sendKeyboardEvent, + chars, + keys, + modifiers, ], ); diff --git a/ui/src/keyboardMappings/layouts/uk_apple.ts b/ui/src/keyboardMappings/layouts/uk_apple.ts index 24d974e..c5ec7f9 100644 --- a/ui/src/keyboardMappings/layouts/uk_apple.ts +++ b/ui/src/keyboardMappings/layouts/uk_apple.ts @@ -21,4 +21,4 @@ export const charsUKApple = { // Modifiers are typically the same between UK and US layouts export const modifiersUKApple = { ...modifiersUS, -}; \ No newline at end of file +} as Record; \ No newline at end of file From 8732a6aff878358a22569c34094e987868fc391e Mon Sep 17 00:00:00 2001 From: William Johnstone Date: Wed, 5 Feb 2025 01:26:24 +0000 Subject: [PATCH 5/5] Fix ghost keys issue, properly implemented modifer mapping --- ui/src/components/WebRTCVideo.tsx | 75 ++++++++++++++++++++----------- ui/src/hooks/stores.ts | 2 + 2 files changed, 51 insertions(+), 26 deletions(-) diff --git a/ui/src/components/WebRTCVideo.tsx b/ui/src/components/WebRTCVideo.tsx index cf3990d..c636bdc 100644 --- a/ui/src/components/WebRTCVideo.tsx +++ b/ui/src/components/WebRTCVideo.tsx @@ -23,7 +23,7 @@ export default function WebRTCVideo() { const [modifiers, setModifiers] = useState(useKeyboardMappingsStore.modifiers); // This map is used to maintain consistency between localised key mappings - const activeKeyState = useRef>(new Map()); + const activeKeyState = useRef>(new Map()); useEffect(() => { const unsubscribeKeyboardStore = useKeyboardMappingsStore.subscribe(() => { @@ -170,14 +170,16 @@ export default function WebRTCVideo() { sendMouseMovement(0, 0, 0); }, [sendMouseMovement]); - // TODO this needs reworked ot work with mappings. + // TODO this needs reworked ot work with mappings // Keyboard-related const handleModifierKeys = useCallback( - (e: KeyboardEvent, activeModifiers: number[]) => { + (e: KeyboardEvent, activeModifiers: number[], mappedKeyModifers: { shift: boolean; altLeft: boolean; altRight: boolean; }) => { const { shiftKey, ctrlKey, altKey, metaKey } = e; - const filteredModifiers = activeModifiers.filter(Boolean); + // TODO remove debug logging + console.log(shiftKey + " " +ctrlKey + " " +altKey + " " +metaKey + " " +mappedKeyModifers.shift + " "+mappedKeyModifers.altLeft + " "+mappedKeyModifers.altRight + " ") + const filteredModifiers = activeModifiers.filter(Boolean);3 // Example: activeModifiers = [0x01, 0x02, 0x04, 0x08] // Assuming 0x01 = ControlLeft, 0x02 = ShiftLeft, 0x04 = AltLeft, 0x08 = MetaLeft return ( @@ -188,6 +190,7 @@ export default function WebRTCVideo() { .filter( modifier => shiftKey || + mappedKeyModifers.shift || (modifier !== modifiers["ShiftLeft"] && modifier !== modifiers["ShiftRight"]), ) @@ -206,6 +209,8 @@ export default function WebRTCVideo() { .filter( modifier => altKey || + mappedKeyModifers.altLeft || + mappedKeyModifers.altRight || (modifier !== modifiers["AltLeft"] && modifier !== modifiers["AltRight"]), ) // Meta: Keep if Meta is pressed or if the key isn't a Meta key @@ -247,40 +252,38 @@ export default function WebRTCVideo() { code = "IntlBackslash"; }*/ - const { key: mappedKey, shift, altLeft, altRight } = useKeyboardMappingsStore.chars[localisedKey] ?? { key: e.code }; + const { key: mappedKey, shift, altLeft, altRight } = chars[localisedKey] ?? { key: code }; //if (!key) continue; console.log("Mapped Key: " + mappedKey) console.log("Current KB Layout:" + useKeyboardMappingsStore.getLayout()); console.log(chars[localisedKey]); - // Build the modifier bitmask - const modifierBitmask = - (shift ? modifiers["ShiftLeft"] : 0) | - (altLeft ? modifiers["AltLeft"] : 0) | - (altRight ? modifiers["AltRight"] : 0); // This is important for a lot of keyboard layouts, right and left alt have different functions - // On second thought this may not be relevant here, may be best to just send altRight through, needs testing - console.log("Modifier Bitmask: " + modifierBitmask) console.log("Shift: " + shift + ", altLeft: " + altLeft + ", altRight: " + altRight) // Add the mapped key to keyState - activeKeyState.current.set(e.code, { mappedKey, modifiers: {shift, altLeft, altRight, bitmask: modifierBitmask}}); + activeKeyState.current.set(e.code, { mappedKey, modifiers: {shift, altLeft, altRight}}); console.log(activeKeyState) // Add the key to the active keys const newKeys = [...prev.activeKeys, keys[mappedKey]].filter(Boolean); + // TODO I feel this may not be applying the modifiers correctly, specifically altRight // Add the modifier to the active modifiers const newModifiers = handleModifierKeys(e, [ ...prev.activeModifiers, modifiers[code], - modifierBitmask, //Is this bad, will we have duplicate modifiers? - ]); + (shift? modifiers['ShiftLeft'] : 0), + (altLeft? modifiers['AltLeft'] : 0), + (altRight? modifiers['AltRight'] : 0),], + {shift: shift, altLeft: altLeft? true : false, altRight: altRight ? true : false} + ); // When pressing the meta key + another key, the key will never trigger a keyup // event, so we need to clear the keys after a short delay // https://bugs.chromium.org/p/chromium/issues/detail?id=28089 // https://bugzilla.mozilla.org/show_bug.cgi?id=1299553 // TODO add this to the activekey state + // TODO set this to remove from activekeystate as well if (e.metaKey) { setTimeout(() => { const prev = useHidStore.getState(); @@ -305,6 +308,7 @@ export default function WebRTCVideo() { const keyUpHandler = useCallback( (e: KeyboardEvent) => { e.preventDefault(); + console.log(e) const prev = useHidStore.getState(); // if (document.activeElement?.id !== "videoFocusTrap") { @@ -321,12 +325,15 @@ export default function WebRTCVideo() { e.code === "ShiftLeft" || e.code === "ShiftRight" || e.code === "AltLeft" || - e.code === "AltRight"; - //e.code === "ControlLeft" || These shouldn't make a difference for mappings - //e.code === "ControlRight"; + e.code === "AltRight" || + e.code === "ControlLeft" || + e.code === "ControlRight"; + + var newKeys = prev.activeKeys; // Handle modifier release if (isModifierKey) { + console.log("ITS A MODIFER") // Update all affected keys when this modifier is released activeKeyState.current.forEach((value, code) => { const { mappedKey, modifiers: mappedModifiers} = value; @@ -355,16 +362,21 @@ export default function WebRTCVideo() { removeCurrentKey = true; }; - var newKeys = prev.activeKeys; - if (removeCurrentKey) { - newKeys = prev.activeKeys + newKeys = newKeys .filter(k => k !== keys[mappedKey]) // Remove the previously mapped key //.concat(keys[updatedMappedKey]) // Add the new remapped key, don't need to do this. .filter(Boolean); }; - - const newModifiers = prev.activeModifiers.filter(k => k !== modifiers[e.code]); + }); + console.log("prev.activemodifers: " + prev.activeModifiers) + console.log("prev.activemodifers.filtered: " + prev.activeModifiers.filter(k => k !== modifiers[e.code])) + const newModifiers = handleModifierKeys( + e, + prev.activeModifiers.filter(k => k !== modifiers[e.code]), + {shift: false, altLeft: false, altRight: false} + ); + console.log("New modifiers in keyup: " + newModifiers) // Update the keyState /*activeKeyState.current.delete(code);/*.set(code, { @@ -376,9 +388,14 @@ export default function WebRTCVideo() { // Remove the modifer key from keyState activeKeyState.current.delete(e.code); + // This is required to filter out the alt keys as well as the modifier. + newKeys = newKeys + .filter(k => k !== keys[e.code]) // Remove the previously mapped key + //.concat(keys[updatedMappedKey]) // Add the new remapped key, don't need to do this. + .filter(Boolean); + // Send the updated HID payload sendKeyboardEvent([...new Set(newKeys)], [...new Set(newModifiers)]); - }); return; // Exit as we've already handled the modifier release } @@ -393,14 +410,20 @@ export default function WebRTCVideo() { activeKeyState.current.delete(e.code); // Filter out the key that was just released - const newKeys = prev.activeKeys.filter(k => k !== keys[mappedKey]).filter(Boolean); + newKeys = newKeys.filter(k => k !== keys[mappedKey]).filter(Boolean); console.log(activeKeyState) // Filter out the associated modifier //const newModifiers = prev.activeModifiers.filter(k => k !== modifier).filter(Boolean); const newModifiers = handleModifierKeys( e, - prev.activeModifiers.filter(k => k !== modifier.bitmask), + prev.activeModifiers.filter(k => { + if (modifier.shift && k == modifiers["ShiftLeft"]) return false; + if (modifier.altLeft && k == modifiers["AltLeft"]) return false; + if (modifier.altRight && k == modifiers["AltRight"]) return false; + return true; + }), + {shift: modifier.shift, altLeft: modifier.altLeft? true : false, altRight: modifier.altRight ? true : false} ); /* const { key: mappedKey/*, shift, altLeft, altRight*//* } = chars[e.key] ?? { key: e.code }; diff --git a/ui/src/hooks/stores.ts b/ui/src/hooks/stores.ts index 29daa82..6118b0b 100644 --- a/ui/src/hooks/stores.ts +++ b/ui/src/hooks/stores.ts @@ -387,6 +387,8 @@ export const useHidStore = create(set => ({ activeKeys: [], activeModifiers: [], updateActiveKeysAndModifiers: ({ keys, modifiers }) => { + // TODO remove debug logs + console.log("keys: " + keys + "modifiers: " + modifiers) return set({ activeKeys: keys, activeModifiers: modifiers }); },