From f1de6639ef7526a6d0694a31949f5419878bdd9e Mon Sep 17 00:00:00 2001 From: William Johnstone Date: Tue, 4 Feb 2025 01:45:34 +0000 Subject: [PATCH] Continued work on keyboard overhaul, found many more issues caused by my changes (YAY!). Also spent 4 hours troubleshooting to find out I didn't realise how useCallback works... :/ Anway, not much longer before work on just the mappings can begin. --- ui/src/components/VirtualKeyboard.tsx | 5 +- ui/src/components/WebRTCVideo.tsx | 96 +++++++++++++++++++-- ui/src/keyboardMappings/layouts/uk_apple.ts | 2 +- 3 files changed, 93 insertions(+), 10 deletions(-) diff --git a/ui/src/components/VirtualKeyboard.tsx b/ui/src/components/VirtualKeyboard.tsx index fab78f9..9f09240 100644 --- a/ui/src/components/VirtualKeyboard.tsx +++ b/ui/src/components/VirtualKeyboard.tsx @@ -20,12 +20,15 @@ const AttachIcon = ({ className }: { className?: string }) => { }; function KeyboardWrapper() { + // TODO implement virtual keyboard mapping const [keys, setKeys] = useState(useKeyboardMappingsStore.keys); + //const [chars, setChars] = useState(useKeyboardMappingsStore.chars); const [modifiers, setModifiers] = useState(useKeyboardMappingsStore.modifiers); useEffect(() => { const unsubscribeKeyboardStore = useKeyboardMappingsStore.subscribe(() => { - setKeys(useKeyboardMappingsStore.keys); + setKeys(useKeyboardMappingsStore.keys); + //setChars(useKeyboardMappingsStore.chars); setModifiers(useKeyboardMappingsStore.modifiers); }); return unsubscribeKeyboardStore; // Cleanup on unmount diff --git a/ui/src/components/WebRTCVideo.tsx b/ui/src/components/WebRTCVideo.tsx index 9b16045..cf3990d 100644 --- a/ui/src/components/WebRTCVideo.tsx +++ b/ui/src/components/WebRTCVideo.tsx @@ -22,9 +22,8 @@ export default function WebRTCVideo() { const [chars, setChars] = useState(useKeyboardMappingsStore.chars); const [modifiers, setModifiers] = useState(useKeyboardMappingsStore.modifiers); - // TODO move this into stores as well as I think this will need to be used in InfoBar // This map is used to maintain consistency between localised key mappings - const activeKeyState = useRef>(new Map()); + const activeKeyState = useRef>(new Map()); useEffect(() => { const unsubscribeKeyboardStore = useKeyboardMappingsStore.subscribe(() => { @@ -140,6 +139,7 @@ export default function WebRTCVideo() { if (blockWheelEvent) return; e.preventDefault(); + // TODO this should be user controllable // Define a scaling factor to adjust scrolling sensitivity const scrollSensitivity = 0.8; // Adjust this value to change scroll speed @@ -155,9 +155,11 @@ export default function WebRTCVideo() { // Invert the scroll value to match expected behavior const invertedScroll = -roundedScroll; + // TODO remove debug logs console.log("wheelReport", { wheelY: invertedScroll }); send("wheelReport", { wheelY: invertedScroll }); + // TODO this is making scrolling feel slow and sluggish, also throwing a violation in chrome setBlockWheelEvent(true); setTimeout(() => setBlockWheelEvent(false), 50); }, @@ -168,6 +170,7 @@ export default function WebRTCVideo() { sendMouseMovement(0, 0, 0); }, [sendMouseMovement]); + // TODO this needs reworked ot work with mappings. // Keyboard-related const handleModifierKeys = useCallback( (e: KeyboardEvent, activeModifiers: number[]) => { @@ -244,19 +247,23 @@ export default function WebRTCVideo() { code = "IntlBackslash"; }*/ - const { key: mappedKey, shift, altLeft, altRight } = chars[localisedKey] ?? { key: e.code }; + const { key: mappedKey, shift, altLeft, altRight } = useKeyboardMappingsStore.chars[localisedKey] ?? { key: e.code }; //if (!key) continue; console.log("Mapped Key: " + mappedKey) console.log("Current KB Layout:" + useKeyboardMappingsStore.getLayout()); + console.log(chars[localisedKey]); // Build the modifier bitmask - const modifier = + const modifierBitmask = (shift ? modifiers["ShiftLeft"] : 0) | (altLeft ? modifiers["AltLeft"] : 0) | (altRight ? modifiers["AltRight"] : 0); // This is important for a lot of keyboard layouts, right and left alt have different functions - + // On second thought this may not be relevant here, may be best to just send altRight through, needs testing + console.log("Modifier Bitmask: " + modifierBitmask) + console.log("Shift: " + shift + ", altLeft: " + altLeft + ", altRight: " + altRight) + // Add the mapped key to keyState - activeKeyState.current.set(e.code, { mappedKey, modifiers: modifier }); + activeKeyState.current.set(e.code, { mappedKey, modifiers: {shift, altLeft, altRight, bitmask: modifierBitmask}}); console.log(activeKeyState) // Add the key to the active keys @@ -266,7 +273,7 @@ export default function WebRTCVideo() { const newModifiers = handleModifierKeys(e, [ ...prev.activeModifiers, modifiers[code], - modifier, //Is this bad, will we have duplicate modifiers? + modifierBitmask, //Is this bad, will we have duplicate modifiers? ]); // When pressing the meta key + another key, the key will never trigger a keyup @@ -289,6 +296,9 @@ export default function WebRTCVideo() { setIsScrollLockActive, handleModifierKeys, sendKeyboardEvent, + chars, + keys, + modifiers, ], ); @@ -306,6 +316,73 @@ export default function WebRTCVideo() { setIsCapsLockActive(e.getModifierState("CapsLock")); setIsScrollLockActive(e.getModifierState("ScrollLock")); + // Check if the released key is a modifier (e.g., Shift, Alt, Control) + const isModifierKey = + e.code === "ShiftLeft" || + e.code === "ShiftRight" || + e.code === "AltLeft" || + e.code === "AltRight"; + //e.code === "ControlLeft" || These shouldn't make a difference for mappings + //e.code === "ControlRight"; + + // Handle modifier release + if (isModifierKey) { + // Update all affected keys when this modifier is released + activeKeyState.current.forEach((value, code) => { + const { mappedKey, modifiers: mappedModifiers} = value; + + // Remove the released modifier from the modifier bitmask + //const updatedModifiers = modifiers & ~modifiers[e.code]; + + // Recalculate the remapped key based on the updated modifiers + //const updatedMappedKey = chars[originalKey]?.key || originalKey; + + var removeCurrentKey = false; + + // Shift Handling + if (mappedModifiers.shift && (e.code === "ShiftLeft" || e.code === "ShiftRight")) { + activeKeyState.current.delete(code); + removeCurrentKey = true; + }; + // Left Alt handling + if (mappedModifiers.altLeft && e.code === "AltLeft") { + activeKeyState.current.delete(code); + removeCurrentKey = true; + }; + // Right Alt handling + if (mappedModifiers.altRight && e.code === "AltRight") { + activeKeyState.current.delete(code); + removeCurrentKey = true; + }; + + var newKeys = prev.activeKeys; + + if (removeCurrentKey) { + newKeys = prev.activeKeys + .filter(k => k !== keys[mappedKey]) // Remove the previously mapped key + //.concat(keys[updatedMappedKey]) // Add the new remapped key, don't need to do this. + .filter(Boolean); + }; + + const newModifiers = prev.activeModifiers.filter(k => k !== modifiers[e.code]); + + // Update the keyState + /*activeKeyState.current.delete(code);/*.set(code, { + mappedKey: updatedMappedKey, + modifiers: updatedModifiers, + originalKey, + });*/ + + // Remove the modifer key from keyState + activeKeyState.current.delete(e.code); + + // Send the updated HID payload + sendKeyboardEvent([...new Set(newKeys)], [...new Set(newModifiers)]); + }); + + return; // Exit as we've already handled the modifier release + } + // Retrieve the mapped key and modifiers from keyState const keyInfo = activeKeyState.current.get(e.code); if (!keyInfo) return; // Ignore if no record exists @@ -323,7 +400,7 @@ export default function WebRTCVideo() { //const newModifiers = prev.activeModifiers.filter(k => k !== modifier).filter(Boolean); const newModifiers = handleModifierKeys( e, - prev.activeModifiers.filter(k => k !== modifier), + prev.activeModifiers.filter(k => k !== modifier.bitmask), ); /* const { key: mappedKey/*, shift, altLeft, altRight*//* } = chars[e.key] ?? { key: e.code }; @@ -354,6 +431,9 @@ export default function WebRTCVideo() { setIsScrollLockActive, handleModifierKeys, sendKeyboardEvent, + chars, + keys, + modifiers, ], ); diff --git a/ui/src/keyboardMappings/layouts/uk_apple.ts b/ui/src/keyboardMappings/layouts/uk_apple.ts index 24d974e..c5ec7f9 100644 --- a/ui/src/keyboardMappings/layouts/uk_apple.ts +++ b/ui/src/keyboardMappings/layouts/uk_apple.ts @@ -21,4 +21,4 @@ export const charsUKApple = { // Modifiers are typically the same between UK and US layouts export const modifiersUKApple = { ...modifiersUS, -}; \ No newline at end of file +} as Record; \ No newline at end of file