Merge 84e4b44df0 into 3e7d8fb0f5

ui/src/components/InfoBar.tsx

@@ -41,6 +41,12 @@ export default function InfoBar() {
   const keyboardLedStateSyncAvailable = useHidStore(state => state.keyboardLedStateSyncAvailable);
   const keyboardLedSync = useSettingsStore(state => state.keyboardLedSync);
   
+  const isShiftActive = useHidStore(state => state.isShiftActive);
+  const isCtrlActive = useHidStore(state => state.isCtrlActive);
+  const isAltActive = useHidStore(state => state.isAltActive);
+  const isMetaActive = useHidStore(state => state.isMetaActive);
+  const isAltGrActive = useHidStore(state => state.isAltGrActive);
+
   const isTurnServerInUse = useRTCStore(state => state.isTurnServerInUse);
 
   const usbState = useHidStore(state => state.usbState);
@@ -135,6 +141,56 @@ export default function InfoBar() {
               {keyboardLedSync === "browser" ? "Browser" : "Host"}
             </div>
           ) : null}
+          <div
+            className={cx(
+              "shrink-0 p-1 px-1.5 text-xs",
+              isShiftActive
+                ? "text-black dark:text-white"
+                : "text-slate-800/20 dark:text-slate-300/20",
+            )}
+          >
+            Shift
+          </div>
+          <div
+            className={cx(
+              "shrink-0 p-1 px-1.5 text-xs",
+              isCtrlActive
+                ? "text-black dark:text-white"
+                : "text-slate-800/20 dark:text-slate-300/20",
+            )}
+          >
+            Ctrl
+          </div>
+          <div
+            className={cx(
+              "shrink-0 p-1 px-1.5 text-xs",
+              isAltActive
+                ? "text-black dark:text-white"
+                : "text-slate-800/20 dark:text-slate-300/20",
+            )}
+          >
+            Alt
+          </div>
+          <div
+            className={cx(
+              "shrink-0 p-1 px-1.5 text-xs",
+              isMetaActive
+                ? "text-black dark:text-white"
+                : "text-slate-800/20 dark:text-slate-300/20",
+            )}
+          >
+            Meta
+          </div>
+          <div
+            className={cx(
+              "shrink-0 p-1 px-1.5 text-xs",
+              isAltGrActive
+                ? "text-black dark:text-white"
+                : "text-slate-800/20 dark:text-slate-300/20",
+            )}
+          >
+            AltGr
+          </div>
           <div
             className={cx(
               "shrink-0 p-1 px-1.5 text-xs",

ui/src/components/VirtualKeyboard.tsx

@@ -27,6 +27,7 @@ const AttachIcon = ({ className }: { className?: string }) => {
 
 function KeyboardWrapper() {
   const [layoutName, setLayoutName] = useState("default");
+  const [depressedButtons, setDepressedButtons] = useState("");
 
   const keyboardRef = useRef<HTMLDivElement>(null);
   const showAttachedVirtualKeyboard = useUiStore(
@@ -54,6 +55,21 @@ function KeyboardWrapper() {
 
   const setIsCapsLockActive = useHidStore(state => state.setIsCapsLockActive);
 
+  const isShiftActive = useHidStore(state => state.isShiftActive);
+  const setIsShiftActive = useHidStore(state => state.setIsShiftActive);
+
+  const isCtrlActive = useHidStore(state => state.isCtrlActive);
+  const setIsCtrlActive = useHidStore(state => state.setIsCtrlActive);
+
+  const isAltActive = useHidStore(state => state.isAltActive);
+  const setIsAltActive = useHidStore(state => state.setIsAltActive);
+
+  const isMetaActive = useHidStore(state => state.isMetaActive);
+  const setIsMetaActive = useHidStore(state => state.setIsMetaActive);
+
+  const isAltGrActive = useHidStore(state => state.isAltGrActive);
+  const setIsAltGrActive = useHidStore(state => state.setIsAltGrActive);
+
   const startDrag = useCallback((e: MouseEvent | TouchEvent) => {
     if (!keyboardRef.current) return;
     if (e instanceof TouchEvent && e.touches.length > 1) return;
@@ -123,18 +139,29 @@ function KeyboardWrapper() {
     };
   }, [endDrag, onDrag, startDrag]);
 
-  const onKeyDown = useCallback(
+  useEffect(() => {
-    (key: string) => {
+    // if you have the CapsLock "down", then the shift state is inverted
-      const isKeyShift = key === "{shift}" || key === "ShiftLeft" || key === "ShiftRight";
+    const effectiveShift = isCapsLockActive ? false === isShiftActive : isShiftActive;
-      const isKeyCaps = key === "CapsLock";
+    setLayoutName(effectiveShift ? "shift" : "default");
-      const cleanKey = key.replace(/[()]/g, "");
+  },
-      const keyHasShiftModifier = key.includes("(");
+    [setLayoutName, isCapsLockActive, isShiftActive]
+  );
 
-      // Handle toggle of layout for shift or caps lock
+  // this causes the buttons to look depressed/clicked depending on the sticky state
-      const toggleLayout = () => {
+  useEffect(() => {
-        setLayoutName(prevLayout => (prevLayout === "default" ? "shift" : "default"));
+    let buttons = "None ";  // make sure we name at least one (fake) button
-      };
+    if (isCapsLockActive) buttons += "CapsLock ";
+    if (isShiftActive) buttons += "ShiftLeft ShiftRight ";
+    if (isCtrlActive) buttons += "ControlLeft ControlRight ";
+    if (isAltActive) buttons += "AltLeft AltRight ";
+    if (isMetaActive) buttons += "MetaLeft MetaRight ";
+    setDepressedButtons(buttons.trimEnd());
+  },
+    [setDepressedButtons, isCapsLockActive, isShiftActive, isCtrlActive, isAltActive, isMetaActive, isAltGrActive]
+  );
 
+  const onKeyPress = useCallback((key: string) => {
+    // handle the fake combo keys first
     if (key === "CtrlAltDelete") {
       sendKeyboardEvent(
         [keys["Delete"]],
@@ -164,39 +191,71 @@ function KeyboardWrapper() {
       return;
     }
 
-      if (isKeyShift || isKeyCaps) {
+    // strip away the parens for shifted characters
-        toggleLayout();
+    const cleanKey = key.replace(/[()]/g, "");
 
-        if (isCapsLockActive) {
+    const passthrough = ["PrintScreen", "SystemRequest", "Pause", "Break", "ScrollLock", "Enter", "Space"].find((value) => value === cleanKey);
-          if (!isKeyboardLedManagedByHost) {
+
-            setIsCapsLockActive(false);
+    if (passthrough) {
-          }
+      emitkeycode(cleanKey);
-          sendKeyboardEvent([keys["CapsLock"]], []);
       return;
     }
-      }
 
-      // Handle caps lock state change
+    // adjust the sticky state of the Shift/Ctrl/Alt/Meta/AltGr
-      if (isKeyCaps && !isKeyboardLedManagedByHost) {
+    if (key === "CapsLock" && !isKeyboardLedManagedByHost)
       setIsCapsLockActive(!isCapsLockActive);
+    else if (key === "ShiftLeft" || key === "ShiftRight")
+      setIsShiftActive(!isShiftActive);
+    else if (key === "ControlLeft" || key === "ControlRight")
+      setIsCtrlActive(!isCtrlActive);
+    else if (key === "AltLeft" || key === "AltRight")
+      setIsAltActive(!isAltActive);
+    else if (key === "MetaLeft" || key === "MetaRight")
+      setIsMetaActive(!isMetaActive);
+    else if (key === "AltGr")
+      setIsAltGrActive(!isAltGrActive);
+
+    emitkeycode(cleanKey);
+
+    function emitkeycode(key: string) {
+      const effectiveMods: number[] = [];
+
+      if (isShiftActive)
+        effectiveMods.push(modifiers["ShiftLeft"]);
+
+      if (isCtrlActive)
+        effectiveMods.push(modifiers["ControlLeft"]);
+
+      if (isAltActive)
+        effectiveMods.push(modifiers["AltLeft"]);
+
+      if (isMetaActive)
+        effectiveMods.push(modifiers["MetaLeft"]);
+
+      if (isAltGrActive) {
+        effectiveMods.push(modifiers["MetaRight"]);
+        effectiveMods.push(modifiers["CtrlLeft"]);
       }
 
-      // Collect new active keys and modifiers
+      const keycode = keys[key];
-      const newKeys = keys[cleanKey] ? [keys[cleanKey]] : [];
+      if (keycode) {
-      const newModifiers =
+        // send the keycode with modifiers
-        keyHasShiftModifier && !isCapsLockActive ? [modifiers["ShiftLeft"]] : [];
+        sendKeyboardEvent([keycode], effectiveMods);
-
-      // Update current keys and modifiers
-      sendKeyboardEvent(newKeys, newModifiers);
-
-      // If shift was used as a modifier and caps lock is not active, revert to default layout
-      if (keyHasShiftModifier && !isCapsLockActive) {
-        setLayoutName("default");
       }
 
-      setTimeout(resetKeyboardState, 100);
+      // release the key (if one pressed), but retain the modifiers
+      setTimeout(() => sendKeyboardEvent([], effectiveMods), 50);
+    }
   },
-    [isCapsLockActive, isKeyboardLedManagedByHost, sendKeyboardEvent, resetKeyboardState, setIsCapsLockActive],
+    [isKeyboardLedManagedByHost,
+      setIsCapsLockActive, isCapsLockActive,
+      setIsShiftActive, isShiftActive,
+      setIsCtrlActive, isCtrlActive,
+      setIsAltActive, isAltActive,
+      setIsMetaActive, isMetaActive,
+      setIsAltGrActive, isAltGrActive,
+      sendKeyboardEvent, resetKeyboardState
+    ],
   );
 
   const virtualKeyboard = useHidStore(state => state.isVirtualKeyboardEnabled);
@@ -276,12 +335,16 @@ function KeyboardWrapper() {
                     <Keyboard
                       baseClass="simple-keyboard-main"
                       layoutName={layoutName}
-                      onKeyPress={onKeyDown}
+                      onKeyPress={onKeyPress}
                       buttonTheme={[
                         {
                           class: "combination-key",
                           buttons: "CtrlAltDelete AltMetaEscape CtrlAltBackspace",
                         },
+                        {
+                          class: "depressed-key",
+                          buttons: depressedButtons
+                        },
                       ]}
                       display={keyDisplayMap}
                       layout={{
@@ -305,8 +368,6 @@ function KeyboardWrapper() {
                         ],
                       }}
                       disableButtonHold={true}
-                      syncInstanceInputs={true}
-                      debug={false}
                     />
 
                     <div className="controlArrows">
@@ -314,25 +375,24 @@ function KeyboardWrapper() {
                         baseClass="simple-keyboard-control"
                         theme="simple-keyboard hg-theme-default hg-layout-default"
                         layoutName={layoutName}
-                        onKeyPress={onKeyDown}
+                        onKeyPress={onKeyPress}
                         display={keyDisplayMap}
                         layout={{
-                          default: ["PrintScreen ScrollLock Pause", "Insert Home Pageup", "Delete End Pagedown"],
+                          default: ["PrintScreen ScrollLock Pause", "Insert Home PageUp", "Delete End PageDown"],
-                          shift: ["(PrintScreen) ScrollLock (Pause)", "Insert Home Pageup", "Delete End Pagedown"],
+                          shift: ["(PrintScreen) ScrollLock (Pause)", "Insert Home PageUp", "Delete End PageDown"],
                         }}
-                        syncInstanceInputs={true}
+                        disableButtonHold={true}
-                        debug={false}
                       />
                       <Keyboard
                         baseClass="simple-keyboard-arrows"
                         theme="simple-keyboard hg-theme-default hg-layout-default"
-                        onKeyPress={onKeyDown}
+                        onKeyPress={onKeyPress}
                         display={keyDisplayMap}
                         layout={{
                           default: ["ArrowUp", "ArrowLeft ArrowDown ArrowRight"],
+                          shift: ["ArrowUp", "ArrowLeft ArrowDown ArrowRight"],
                         }}
-                        syncInstanceInputs={true}
+                        disableButtonHold={true}
-                        debug={false}
                       />
                     </div>
                   </div>

ui/src/components/WebRTCVideo.tsx

@@ -71,6 +71,15 @@ export default function WebRTCVideo() {
   const hdmiError = ["no_lock", "no_signal", "out_of_range"].includes(hdmiState);
   const isVideoLoading = !isPlaying;
 
+  // Keyboard related states
+  const { 
+    setIsShiftActive,
+    setIsCtrlActive,
+    setIsAltActive,
+    setIsMetaActive,
+    setIsAltGrActive
+  } = useHidStore();
+
   const [blockWheelEvent, setBlockWheelEvent] = useState(false);
 
   // Misc states and hooks
@@ -423,6 +432,12 @@ export default function WebRTCVideo() {
         setIsScrollLockActive(e.getModifierState("ScrollLock"));
       }
 
+      setIsShiftActive(e.getModifierState("Shift"))
+      setIsCtrlActive(e.getModifierState("Control"))
+      setIsAltActive(e.getModifierState("Alt"))
+      setIsMetaActive(e.getModifierState("Meta"))
+      setIsAltGrActive(e.getModifierState("AltGraph"))
+
       if (code == "IntlBackslash" && ["`", "~"].includes(key)) {
         code = "Backquote";
       } else if (code == "Backquote" && ["§", "±"].includes(key)) {
@@ -452,12 +467,17 @@ export default function WebRTCVideo() {
       sendKeyboardEvent([...new Set(newKeys)], [...new Set(newModifiers)]);
     },
     [
-      handleModifierKeys,
-      sendKeyboardEvent,
       isKeyboardLedManagedByHost,
       setIsNumLockActive,
       setIsCapsLockActive,
       setIsScrollLockActive,
+      setIsShiftActive,
+      setIsCtrlActive,
+      setIsAltActive,
+      setIsMetaActive,
+      setIsAltGrActive,
+      handleModifierKeys,
+      sendKeyboardEvent
     ],
   );
 
@@ -472,6 +492,12 @@ export default function WebRTCVideo() {
         setIsScrollLockActive(e.getModifierState("ScrollLock"));
       }
 
+      setIsShiftActive(e.getModifierState("Shift"))
+      setIsCtrlActive(e.getModifierState("Control"))
+      setIsAltActive(e.getModifierState("Alt"))
+      setIsMetaActive(e.getModifierState("Meta"))
+      setIsAltGrActive(e.getModifierState("AltGraph"))
+
       // Filtering out the key that was just released (keys[e.code])
       const newKeys = prev.activeKeys.filter(k => k !== keys[e.code]).filter(Boolean);
 
@@ -484,12 +510,17 @@ export default function WebRTCVideo() {
       sendKeyboardEvent([...new Set(newKeys)], [...new Set(newModifiers)]);
     },
     [
-      handleModifierKeys,
-      sendKeyboardEvent,
       isKeyboardLedManagedByHost,
       setIsNumLockActive,
       setIsCapsLockActive,
       setIsScrollLockActive,
+      setIsShiftActive,
+      setIsCtrlActive,
+      setIsAltActive,
+      setIsMetaActive,
+      setIsAltGrActive,
+      handleModifierKeys,
+      sendKeyboardEvent
     ],
   );
 

ui/src/hooks/stores.ts

@@ -481,6 +481,21 @@ export interface HidState {
   isVirtualKeyboardEnabled: boolean;
   setVirtualKeyboardEnabled: (enabled: boolean) => void;
 
+  isShiftActive: boolean;
+  setIsShiftActive: (enabled: boolean) => void;
+
+  isCtrlActive: boolean;
+  setIsCtrlActive: (enabled: boolean) => void;
+
+  isAltActive: boolean;
+  setIsAltActive: (enabled: boolean) => void;
+
+  isMetaActive: boolean;
+  setIsMetaActive: (enabled: boolean) => void;
+
+  isAltGrActive: boolean;
+  setIsAltGrActive: (enabled: boolean) => void;
+
   isPasteModeEnabled: boolean;
   setPasteModeEnabled: (enabled: boolean) => void;
 
@@ -527,6 +542,21 @@ export const useHidStore = create<HidState>((set, get) => ({
   isVirtualKeyboardEnabled: false,
   setVirtualKeyboardEnabled: enabled => set({ isVirtualKeyboardEnabled: enabled }),
 
+  isShiftActive: false,
+  setIsShiftActive: enabled => set({ isShiftActive: enabled }),
+
+  isCtrlActive: false,
+  setIsCtrlActive: enabled => set({ isCtrlActive: enabled }),
+
+  isAltActive: false,
+  setIsAltActive: enabled => set({ isAltActive: enabled }),
+
+  isMetaActive: false,
+  setIsMetaActive: enabled => set({ isMetaActive: enabled }),
+
+  isAltGrActive: false,
+  setIsAltGrActive: enabled => set({ isAltGrActive: enabled }),
+
   isPasteModeEnabled: false,
   setPasteModeEnabled: enabled => set({ isPasteModeEnabled: enabled }),
 

ui/src/index.css

@@ -222,6 +222,10 @@ video::-webkit-media-controls {
   background: none;
 }
 
+.simple-keyboard .hg-button.depressed-key {
+  @apply border-gray-800! border-b-gray-600! border-t-gray-900! bg-gray-700!;
+}
+
 .simple-keyboard .hg-button.selectedButton {
   background: rgba(5, 25, 70, 0.53);
   @apply text-white;

ui/src/keyboardMappings.ts

@@ -5,14 +5,14 @@ export const keys = {
   ArrowLeft: 0x50,
   ArrowRight: 0x4f,
   ArrowUp: 0x52,
-  Backquote: 0x35,
+  Backquote: 0x35,    // aka Grave
   Backslash: 0x31,
   Backspace: 0x2a,
-  BracketLeft: 0x2f,
+  BracketLeft: 0x2f,  // aka LeftBrace
-  BracketRight: 0x30,
+  BracketRight: 0x30, // aka RightBrace
   CapsLock: 0x39,
   Comma: 0x36,
-  ContextMenu: 0,
+  Compose: 0x65,
   Delete: 0x4c,
   Digit0: 0x27,
   Digit1: 0x1e,
@@ -41,6 +41,18 @@ export const keys = {
   F11: 0x44,
   F12: 0x45,
   F13: 0x68,
+  F14: 0x69,
+  F15: 0x6a,
+  F16: 0x6b,
+  F17: 0x6c,
+  F18: 0x6d,
+  F19: 0x6e,
+  F20: 0x6f,
+  F21: 0x70,
+  F22: 0x71,
+  F23: 0x72,
+  F24: 0x73,
+  HashTilde: 0x32,
   Home: 0x4a,
   Insert: 0x49,
   IntlBackslash: 0x64,
@@ -70,37 +82,58 @@ export const keys = {
   KeyX: 0x1b,
   KeyY: 0x1c,
   KeyZ: 0x1d,
+  KeypadComma: 0x85,
+  KeypadEqual: 0x86,
+  KeyRO: 0x87,
+  KatakanaHiragana: 0x88,
+  Yen: 0x89,
+  Henkan: 0x8a,
+  Muhenkan: 0x8b,
+  KPJPComma: 0x8c,
+  International7: 0x8d,
+  International8: 0x8e,
+  International9: 0x8f,
+  Hangeul: 0x90,
+  Hanja: 0x91,
+  Katakana: 0x92,
+  Hiragana: 0x93,
+  Zenkakuhankaku:0x94,
+  KeypadLeftParen: 0xb6,
+  KeypadRightParen: 0xb7,
   KeypadExclamation: 0xcf,
   Minus: 0x2d,
-  NumLock: 0x53,
+  None: 0x00,
-  Numpad0: 0x62,
+  NumLock: 0x53,  // and Clear
-  Numpad1: 0x59,
+  Numpad0: 0x62,  // and Insert
-  Numpad2: 0x5a,
+  Numpad1: 0x59,  // and End
-  Numpad3: 0x5b,
+  Numpad2: 0x5a,  // and Down Arrow
-  Numpad4: 0x5c,
+  Numpad3: 0x5b,  // and Page Down
+  Numpad4: 0x5c,  // and Left Arrow
   Numpad5: 0x5d,
-  Numpad6: 0x5e,
+  Numpad6: 0x5e,  // and Right Arrow
-  Numpad7: 0x5f,
+  Numpad7: 0x5f,  // and Home
-  Numpad8: 0x60,
+  Numpad8: 0x60,  // and Up Arrow
-  Numpad9: 0x61,
+  Numpad9: 0x61,  // and Page Up
-  NumpadAdd: 0x57,
+  NumpadPlus: 0x57,
-  NumpadDivide: 0x54,
+  NumpadSlash: 0x54,
   NumpadEnter: 0x58,
   NumpadEqual: 0x67,
-  NumpadMultiply: 0x55,
+  NumpadAsterisk: 0x55,
-  NumpadSubtract: 0x56,
+  NumpadMinus: 0x56,
-  NumpadDecimal: 0x63,
+  NumpadDecimal: 0x63,  // aka NumpadDot and Delete
+  Overflow: 0x01,
   PageDown: 0x4e,
   PageUp: 0x4b,
-  Period: 0x37,
+  Period: 0x37,     // aka Dot
   PrintScreen: 0x46,
   Pause: 0x48,
-  Quote: 0x34,
+  Power: 0x66,
+  Quote: 0x34,      // aka Apostrophe
   ScrollLock: 0x47,
   Semicolon: 0x33,
   Slash: 0x38,
   Space: 0x2c,
-  SystemRequest: 0x9a,
+  SystemRequest: 0x9a,  // aka Attention
   Tab: 0x2b,
 } as Record<string, number>;
 
@@ -131,23 +164,23 @@ export const keyDisplayMap: Record<string, string> = {
   AltMetaEscape: "Alt + Meta + Escape",
   CtrlAltBackspace: "Ctrl + Alt + Backspace",
   Escape: "esc",
-  Tab: "tab",
+  Tab: "tab ⇥",
-  Backspace: "backspace",
+  Backspace: "backspace ⌫",
-  "(Backspace)": "backspace",
   Enter: "enter",
-  CapsLock: "caps lock",
+  CapsLock: "caps lock ⇪",
-  ShiftLeft: "shift",
+  ShiftLeft: "shift ⇧",
-  ShiftRight: "shift",
+  ShiftRight: "⇧ shift",
-  ControlLeft: "ctrl",
+  ControlLeft: "ctrl ⌃",
-  AltLeft: "alt",
+  ControlRight: "⌃ ctrl",
-  AltRight: "alt",
+  AltLeft: "alt ⌥",
-  MetaLeft: "meta",
+  AltRight: "⌥ alt",
-  MetaRight: "meta",
+  MetaLeft: "meta ⌘", // "meta ⊞" for windows
+  MetaRight: "⌘ meta",// "≣ meta" for windows
   Space: " ",
-  Insert: "insert",
+  Insert: "ins",
   Home: "home",
   PageUp: "page up",
-  Delete: "delete",
+  Delete: "del ⌦",
   End: "end",
   PageDown: "page down",
   ArrowLeft: "←",
@@ -213,13 +246,12 @@ export const keyDisplayMap: Record<string, string> = {
   Numpad0: "Num 0", Numpad1: "Num 1", Numpad2: "Num 2",
   Numpad3: "Num 3", Numpad4: "Num 4", Numpad5: "Num 5",
   Numpad6: "Num 6", Numpad7: "Num 7", Numpad8: "Num 8",
-  Numpad9: "Num 9", NumpadAdd: "Num +", NumpadSubtract: "Num -",
+  Numpad9: "Num 9", NumpadPlus: "Num +", NumpadMinus: "Num -",
-  NumpadMultiply: "Num *", NumpadDivide: "Num /", NumpadDecimal: "Num .",
+  NumpadAsterisk: "Num *", NumpadSlash: "Num /", NumpadDecimal: "Num .",
   NumpadEqual: "Num =", NumpadEnter: "Num Enter",
   NumLock: "Num Lock",
 
   // Modals
   PrintScreen: "prt sc", ScrollLock: "scr lk", Pause: "pause",
-  "(PrintScreen)": "sys rq", "(Pause)": "break",
+  "(PrintScreen)": "sys rq", "(Pause)": "break"
-  SystemRequest: "sys rq", Break: "break"
 };