Almost complete implementation of mapped virtual keyboard. Still to implement proper modifer key holding.

ui/src/components/VirtualKeyboard.tsx

@@ -20,21 +20,40 @@ const AttachIcon = ({ className }: { className?: string }) => {
 };
 
 function KeyboardWrapper() {
-  // TODO implement virtual keyboard mapping
   const [keys, setKeys] = useState(useKeyboardMappingsStore.keys);
-  //const [chars, setChars] = useState(useKeyboardMappingsStore.chars);
+  const [chars, setChars] = useState(useKeyboardMappingsStore.chars);
   const [modifiers, setModifiers] = useState(useKeyboardMappingsStore.modifiers);
 
   useEffect(() => {
     const unsubscribeKeyboardStore = useKeyboardMappingsStore.subscribe(() => {
       setKeys(useKeyboardMappingsStore.keys);
-      //setChars(useKeyboardMappingsStore.chars);
+      setChars(useKeyboardMappingsStore.chars);
       setModifiers(useKeyboardMappingsStore.modifiers);
+      setMappingsEnabled(useKeyboardMappingsStore.getMappingState());
     });
     return unsubscribeKeyboardStore; // Cleanup on unmount
   }, []); 
 
   const [layoutName, setLayoutName] = useState("default");
+  const [mappingsEnabled, setMappingsEnabled] = useState(useKeyboardMappingsStore.getMappingState());
+
+  useEffect(() => {
+    if (mappingsEnabled) {
+      if (layoutName == "default" ) {
+        setLayoutName("mappedLower")
+      }
+      if (layoutName == "shift") {
+        setLayoutName("mappedUpper")
+      }
+    } else {
+      if (layoutName == "mappedLower") {
+        setLayoutName("default")
+      }
+      if (layoutName == "mappedUpper") {
+        setLayoutName("shift")
+      }
+    }
+  }, [mappingsEnabled, layoutName]);
 
   const keyboardRef = useRef<HTMLDivElement>(null);
   const showAttachedVirtualKeyboard = useUiStore(
@@ -121,16 +140,28 @@ function KeyboardWrapper() {
     };
   }, [endDrag, onDrag, startDrag]);
 
+  // TODO implement meta key and meta key modifer
+  // TODO implement hold functionality for key combos. (add a hold button, add all keys to an array, when released send as one)
   const onKeyDown = useCallback(
     (key: string) => {
+      const cleanKey = key.replace(/[()]/g, "");
+      // Mappings
+      const { key: mappedKey, shift, altLeft, altRight } = chars[cleanKey] ?? {};
+
       const isKeyShift = key === "{shift}" || key === "ShiftLeft" || key === "ShiftRight";
       const isKeyCaps = key === "CapsLock";
-      const cleanKey = key.replace(/[()]/g, "");
+      const keyHasShiftModifier = (key.includes("(") && key !== "(") || shift;
-      const keyHasShiftModifier = key.includes("(");
+
+      //TODO remove debug logs
+      console.log(layoutName)
 
       // Handle toggle of layout for shift or caps lock
       const toggleLayout = () => {
-        setLayoutName(prevLayout => (prevLayout === "default" ? "shift" : "default"));
+        if (mappingsEnabled) {
+          setLayoutName(prevLayout => (prevLayout === "mappedLower" ? "mappedUpper" : "mappedLower"));
+        } else {
+          setLayoutName(prevLayout => (prevLayout === "default" ? "shift" : "default"));
+        }
       };
 
       if (key === "CtrlAltDelete") {
@@ -152,10 +183,17 @@ function KeyboardWrapper() {
         return;
       }
 
-      if (isKeyShift || isKeyCaps) {
+      if (isKeyShift || (!(layoutName == "shift" || layoutName == "mappedUpper") && isCapsLockActive)) {
         toggleLayout();
+      }
 
-        if (isCapsLockActive) {
+      if (layoutName == "shift" || layoutName == "mappedUpper") {
+        if (!isCapsLockActive) {
+          toggleLayout();
+        }
+
+        if (isKeyCaps && isCapsLockActive) {
+          toggleLayout();
           setIsCapsLockActive(false);
           sendKeyboardEvent([keys["CapsLock"]], []);
           return;
@@ -164,25 +202,38 @@ function KeyboardWrapper() {
 
       // Handle caps lock state change
       if (isKeyCaps) {
+        toggleLayout();
         setIsCapsLockActive(!isCapsLockActive);
       }
 
+      //TODO remove debug logs
+      console.log(cleanKey)
+      console.log(chars[cleanKey])
+
+      console.log(mappedKey)
+
       // Collect new active keys and modifiers
-      const newKeys = keys[cleanKey] ? [keys[cleanKey]] : [];
+      const newKeys = keys[mappedKey ?? cleanKey] ? [keys[mappedKey ?? cleanKey]] : [];
       const newModifiers =
-        keyHasShiftModifier && !isCapsLockActive ? [modifiers["ShiftLeft"]] : [];
+      [
+        ((shift || isKeyShift)? modifiers['ShiftLeft'] : 0),
+        (altLeft? modifiers['AltLeft'] : 0),
+        (altRight? modifiers['AltRight'] : 0),
+      ].filter(Boolean);
+
+      console.log(newModifiers);
 
       // Update current keys and modifiers
-      sendKeyboardEvent(newKeys, newModifiers);
+      sendKeyboardEvent(newKeys, [...new Set(newModifiers)]);
 
       // If shift was used as a modifier and caps lock is not active, revert to default layout
       if (keyHasShiftModifier && !isCapsLockActive) {
-        setLayoutName("default");
+        setLayoutName(mappingsEnabled ? "mappedLower" : "default");
       }
 
       setTimeout(resetKeyboardState, 100);
     },
-    [isCapsLockActive, sendKeyboardEvent, resetKeyboardState, setIsCapsLockActive],
+    [isCapsLockActive, sendKeyboardEvent, resetKeyboardState, setIsCapsLockActive, mappingsEnabled, chars, keys, modifiers, layoutName],
   );
 
   const virtualKeyboard = useHidStore(state => state.isVirtualKeyboardEnabled);
@@ -398,6 +449,115 @@ function KeyboardWrapper() {
                       F10: "F10",
                       F11: "F11",
                       F12: "F12",
+
+                      "q": "q",
+                      "w": "w",
+                      "e": "e",
+                      "r": "r",
+                      "t": "t",
+                      "y": "y",
+                      "u": "u",
+                      "i": "i",
+                      "o": "o",
+                      "p": "p",
+                      "a": "a",
+                      "s": "s",
+                      "d": "d",
+                      "f": "f",
+                      "g": "g",
+                      "h": "h",
+                      "j": "j",
+                      "k": "k",
+                      "l": "l",
+                      "z": "z",
+                      "x": "x",
+                      "c": "c",
+                      "v": "v",
+                      "b": "b",
+                      "n": "n",
+                      "m": "m",
+                      
+                      "Q": "Q",
+                      "W": "W",
+                      "E": "E",
+                      "R": "R",
+                      "T": "T",
+                      "Y": "Y",
+                      "U": "U",
+                      "I": "I",
+                      "O": "O",
+                      "P": "P",
+                      "A": "A",
+                      "S": "S",
+                      "D": "D",
+                      "F": "F",
+                      "G": "G",
+                      "H": "H",
+                      "J": "J",
+                      "K": "K",
+                      "L": "L",
+                      "Z": "Z",
+                      "X": "X",
+                      "C": "C",
+                      "V": "V",
+                      "B": "B",
+                      "N": "N",
+                      "M": "M",
+                      
+                      "1": "1",
+                      "2": "2",
+                      "3": "3",
+                      "4": "4",
+                      "5": "5",
+                      "6": "6",
+                      "7": "7",
+                      "8": "8",
+                      "9": "9",
+                      "0": "0",
+                      
+                      "!": "!",
+                      "@": "@",
+                      "#": "#",
+                      "$": "$",
+                      "%": "%",
+                      "^": "^",
+                      "&": "&",
+                      "*": "*",
+                      "(": "(",
+                      ")": ")",
+                      
+                      "-": "-",
+                      "_": "_",
+                      
+                      "=": "=",
+                      "+": "+",
+                      
+                      "[": "[",
+                      "]": "]",
+                      "{": "{",
+                      "}": "}",
+                      
+                      "|": "|",
+                      
+                      ";": ";",
+                      ":": ":",
+                      
+                      "'": "'",
+                      "\"": "\"",
+                      
+                      ",": ",",
+                      "<": "<",
+                      
+                      ".": ".",
+                      ">": ">",
+                      
+                      "/": "/",
+                      "?": "?",
+                      
+                      "`": "`",
+                      "~": "~",
+                      
+                      "\\": "\\"
                     }}
                     layout={{
                       default: [
@@ -418,6 +578,25 @@ function KeyboardWrapper() {
                         "ShiftLeft (KeyZ) (KeyX) (KeyC) (KeyV) (KeyB) (KeyN) (KeyM) (Comma) (Period) (Slash) ShiftRight",
                         "ControlLeft AltLeft MetaLeft Space MetaRight AltRight",
                       ],
+                      mappedLower: [
+                        "CtrlAltDelete AltMetaEscape",
+                        "Escape F1 F2 F3 F4 F5 F6 F7 F8 F9 F10 F11 F12",
+                        "` 1 2 3 4 5 6 7 8 9 0 - = Backspace",
+                        "Tab q w e r t y u i o p [ ] \\",
+                        "CapsLock a s d f g h j k l ; ' Enter",
+                        "ShiftLeft z x c v b n m , . / ShiftRight",
+                        "ControlLeft AltLeft MetaLeft Space MetaRight AltRight"
+                      ],
+
+                      mappedUpper: [
+                        "CtrlAltDelete AltMetaEscape",
+                        "Escape F1 F2 F3 F4 F5 F6 F7 F8 F9 F10 F11 F12",
+                        "~ ! @ # $ % ^ & * ( ) _ + Backspace",
+                        "Tab Q W E R T Y U I O P { } |",
+                        "CapsLock A S D F G H J K L : \" Enter",
+                        "ShiftLeft Z X C V B N M < > ? ShiftRight",
+                        "ControlLeft AltLeft MetaLeft Space MetaRight AltRight"
+                      ],
                     }}
                     disableButtonHold={true}
                     mergeDisplay={true}

ui/src/components/WebRTCVideo.tsx

@@ -20,6 +20,7 @@ import { ConnectionErrorOverlay, HDMIErrorOverlay, LoadingOverlay } from "./Vide
 // TODO Implement keyboard lock API to resolve #127
 // https://developer.chrome.com/docs/capabilities/web-apis/keyboard-lock
 // An appropriate error message will need to be displayed in order to alert users to browser compatibility issues.
+// This requires TLS, waiting on TLS support.
 
 export default function WebRTCVideo() {
   const [keys, setKeys] = useState(useKeyboardMappingsStore.keys);

ui/src/hooks/stores.ts

@@ -580,6 +580,10 @@ class KeyboardMappingsStore {
     this._notifySubscribers(); 
   }
 
+  getMappingState() {
+    return this._mappingsEnabled;
+  }
+
   getLayout() {
     return this._layout;
   }