Merge 84e4b44df0 into 090e0b4b47

Bumps [actions/setup-go](https://github.com/actions/setup-go) from 4.2.1 to 5.5.0.
- [Release notes](https://github.com/actions/setup-go/releases)
- [Commits](https://github.com/actions/setup-go/compare/v4.2.1...v5.5.0)

---
updated-dependencies:
- dependency-name: actions/setup-go
  dependency-version: 5.5.0
  dependency-type: direct:production
  update-type: version-update:semver-major
...

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>

.github/workflows/build.yml

@@ -23,7 +23,7 @@ jobs:
           cache: "npm"
           cache-dependency-path: "**/package-lock.json"
       - name: Set up Golang
-        uses: actions/setup-go@v5
+        uses: actions/setup-go@v5.5.0
         with:
           go-version: "1.24.4"
       - name: Build frontend

.github/workflows/golangci-lint.yml

@@ -24,7 +24,7 @@ jobs:
       - name: Checkout repository
         uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
       - name: Install Go
-        uses: actions/setup-go@19bb51245e9c80abacb2e91cc42b33fa478b8639 # v4.2.1
+        uses: actions/setup-go@fa96338abe5531f6e34c5cc0bbe28c1a533d5505 # v4.2.1
         with:
           go-version: 1.24.4
       - name: Create empty resource directory

.github/workflows/smoketest.yml

@@ -104,7 +104,7 @@ jobs:
           EOF
           ssh jkci "cat /tmp/device-tests.json" > device-tests.json
       - name: Set up Golang
-        uses: actions/setup-go@v5
+        uses: actions/setup-go@v5.5.0
         with:
           go-version: "1.24.4"
       - name: Golang Test Report

display.go

@@ -9,9 +9,14 @@ import (
 	"time"
 )
 
-var currentScreen = "ui_Boot_Screen"
 var backlightState = 0 // 0 - NORMAL, 1 - DIMMED, 2 - OFF
 
+var (
+	currentScreen   = "ui_Boot_Screen"
+	displayedTexts  = make(map[string]string)
+	screenStateLock = sync.Mutex{}
+)
+
 var (
 	dimTicker *time.Ticker
 	offTicker *time.Ticker
@@ -22,6 +27,8 @@ const (
 	backlightControlClass string = "/sys/class/backlight/backlight/brightness"
 )
 
+// do not call this function directly, use switchToScreenIfDifferent instead
+// this function is not thread safe
 func switchToScreen(screen string) {
 	_, err := CallCtrlAction("lv_scr_load", map[string]interface{}{"obj": screen})
 	if err != nil {
@@ -31,8 +38,6 @@ func switchToScreen(screen string) {
 	currentScreen = screen
 }
 
-var displayedTexts = make(map[string]string)
-
 func lvObjSetState(objName string, state string) (*CtrlResponse, error) {
 	return CallCtrlAction("lv_obj_set_state", map[string]interface{}{"obj": objName, "state": state})
 }
@@ -78,6 +83,9 @@ func lvDispSetRotation(rotation string) (*CtrlResponse, error) {
 }
 
 func updateLabelIfChanged(objName string, newText string) {
+	screenStateLock.Lock()
+	defer screenStateLock.Unlock()
+
 	if newText != "" && newText != displayedTexts[objName] {
 		_, _ = lvLabelSetText(objName, newText)
 		displayedTexts[objName] = newText
@@ -85,12 +93,23 @@ func updateLabelIfChanged(objName string, newText string) {
 }
 
 func switchToScreenIfDifferent(screenName string) {
+	screenStateLock.Lock()
+	defer screenStateLock.Unlock()
+
 	if currentScreen != screenName {
 		displayLogger.Info().Str("from", currentScreen).Str("to", screenName).Msg("switching screen")
 		switchToScreen(screenName)
 	}
 }
 
+func clearDisplayState() {
+	screenStateLock.Lock()
+	defer screenStateLock.Unlock()
+
+	displayedTexts = make(map[string]string)
+	currentScreen = "ui_Boot_Screen"
+}
+
 var (
 	cloudBlinkLock    sync.Mutex = sync.Mutex{}
 	cloudBlinkStopped bool

go.mod

@@ -3,7 +3,7 @@ module github.com/jetkvm/kvm
 go 1.24.4
 
 require (
-	github.com/Masterminds/semver/v3 v3.3.1
+	github.com/Masterminds/semver/v3 v3.4.0
 	github.com/beevik/ntp v1.4.3
 	github.com/coder/websocket v1.8.13
 	github.com/coreos/go-oidc/v3 v3.14.1
@@ -17,7 +17,7 @@ require (
 	github.com/hanwen/go-fuse/v2 v2.8.0
 	github.com/pion/logging v0.2.4
 	github.com/pion/mdns/v2 v2.0.7
-	github.com/pion/webrtc/v4 v4.1.2
+	github.com/pion/webrtc/v4 v4.1.3
 	github.com/pojntfx/go-nbd v0.3.2
 	github.com/prometheus/client_golang v1.22.0
 	github.com/prometheus/common v0.65.0
@@ -66,7 +66,7 @@ require (
 	github.com/pion/interceptor v0.1.40 // indirect
 	github.com/pion/randutil v0.1.0 // indirect
 	github.com/pion/rtcp v1.2.15 // indirect
-	github.com/pion/rtp v1.8.19 // indirect
+	github.com/pion/rtp v1.8.20 // indirect
 	github.com/pion/sctp v1.8.39 // indirect
 	github.com/pion/sdp/v3 v3.0.14 // indirect
 	github.com/pion/srtp/v3 v3.0.6 // indirect

go.sum

@@ -1,5 +1,5 @@
-github.com/Masterminds/semver/v3 v3.3.1 h1:QtNSWtVZ3nBfk8mAOu/B6v7FMJ+NHTIgUPi7rj+4nv4=
-github.com/Masterminds/semver/v3 v3.3.1/go.mod h1:4V+yj/TJE1HU9XfppCwVMZq3I84lprf4nC11bSS5beM=
+github.com/Masterminds/semver/v3 v3.4.0 h1:Zog+i5UMtVoCU8oKka5P7i9q9HgrJeGzI9SA1Xbatp0=
+github.com/Masterminds/semver/v3 v3.4.0/go.mod h1:4V+yj/TJE1HU9XfppCwVMZq3I84lprf4nC11bSS5beM=
 github.com/beevik/ntp v1.4.3 h1:PlbTvE5NNy4QHmA4Mg57n7mcFTmr1W1j3gcK7L1lqho=
 github.com/beevik/ntp v1.4.3/go.mod h1:Unr8Zg+2dRn7d8bHFuehIMSvvUYssHMxW3Q5Nx4RW5Q=
 github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
@@ -117,8 +117,8 @@ github.com/pion/randutil v0.1.0 h1:CFG1UdESneORglEsnimhUjf33Rwjubwj6xfiOXBa3mA=
 github.com/pion/randutil v0.1.0/go.mod h1:XcJrSMMbbMRhASFVOlj/5hQial/Y8oH/HVo7TBZq+j8=
 github.com/pion/rtcp v1.2.15 h1:LZQi2JbdipLOj4eBjK4wlVoQWfrZbh3Q6eHtWtJBZBo=
 github.com/pion/rtcp v1.2.15/go.mod h1:jlGuAjHMEXwMUHK78RgX0UmEJFV4zUKOFHR7OP+D3D0=
-github.com/pion/rtp v1.8.19 h1:jhdO/3XhL/aKm/wARFVmvTfq0lC/CvN1xwYKmduly3c=
-github.com/pion/rtp v1.8.19/go.mod h1:bAu2UFKScgzyFqvUKmbvzSdPr+NGbZtv6UB2hesqXBk=
+github.com/pion/rtp v1.8.20 h1:8zcyqohadZE8FCBeGdyEvHiclPIezcwRQH9zfapFyYI=
+github.com/pion/rtp v1.8.20/go.mod h1:bAu2UFKScgzyFqvUKmbvzSdPr+NGbZtv6UB2hesqXBk=
 github.com/pion/sctp v1.8.39 h1:PJma40vRHa3UTO3C4MyeJDQ+KIobVYRZQZ0Nt7SjQnE=
 github.com/pion/sctp v1.8.39/go.mod h1:cNiLdchXra8fHQwmIoqw0MbLLMs+f7uQ+dGMG2gWebE=
 github.com/pion/sdp/v3 v3.0.14 h1:1h7gBr9FhOWH5GjWWY5lcw/U85MtdcibTyt/o6RxRUI=
@@ -131,8 +131,8 @@ github.com/pion/transport/v3 v3.0.7 h1:iRbMH05BzSNwhILHoBoAPxoB9xQgOaJk+591KC9P1
 github.com/pion/transport/v3 v3.0.7/go.mod h1:YleKiTZ4vqNxVwh77Z0zytYi7rXHl7j6uPLGhhz9rwo=
 github.com/pion/turn/v4 v4.0.2 h1:ZqgQ3+MjP32ug30xAbD6Mn+/K4Sxi3SdNOTFf+7mpps=
 github.com/pion/turn/v4 v4.0.2/go.mod h1:pMMKP/ieNAG/fN5cZiN4SDuyKsXtNTr0ccN7IToA1zs=
-github.com/pion/webrtc/v4 v4.1.2 h1:mpuUo/EJ1zMNKGE79fAdYNFZBX790KE7kQQpLMjjR54=
-github.com/pion/webrtc/v4 v4.1.2/go.mod h1:xsCXiNAmMEjIdFxAYU0MbB3RwRieJsegSB2JZsGN+8U=
+github.com/pion/webrtc/v4 v4.1.3 h1:YZ67Boj9X/hk190jJZ8+HFGQ6DqSZ/fYP3sLAZv7c3c=
+github.com/pion/webrtc/v4 v4.1.3/go.mod h1:rsq+zQ82ryfR9vbb0L1umPJ6Ogq7zm8mcn9fcGnxomM=
 github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=

native.go

@@ -272,6 +272,13 @@ func restartNativeBinary(binaryPath string) error {
 		nativeLogger.Warn().Err(err).Msg("failed to restart binary")
 	}
 	nativeCmd = cmd
+
+	// reset the display state
+	time.Sleep(1 * time.Second)
+	clearDisplayState()
+	updateStaticContents()
+	requestDisplayUpdate(true)
+
 	return err
 }
 

ui/src/components/InfoBar.tsx

@@ -41,6 +41,12 @@ export default function InfoBar() {
   const keyboardLedStateSyncAvailable = useHidStore(state => state.keyboardLedStateSyncAvailable);
   const keyboardLedSync = useSettingsStore(state => state.keyboardLedSync);
   
+  const isShiftActive = useHidStore(state => state.isShiftActive);
+  const isCtrlActive = useHidStore(state => state.isCtrlActive);
+  const isAltActive = useHidStore(state => state.isAltActive);
+  const isMetaActive = useHidStore(state => state.isMetaActive);
+  const isAltGrActive = useHidStore(state => state.isAltGrActive);
+
   const isTurnServerInUse = useRTCStore(state => state.isTurnServerInUse);
 
   const usbState = useHidStore(state => state.usbState);
@@ -135,6 +141,56 @@ export default function InfoBar() {
               {keyboardLedSync === "browser" ? "Browser" : "Host"}
             </div>
           ) : null}
+          <div
+            className={cx(
+              "shrink-0 p-1 px-1.5 text-xs",
+              isShiftActive
+                ? "text-black dark:text-white"
+                : "text-slate-800/20 dark:text-slate-300/20",
+            )}
+          >
+            Shift
+          </div>
+          <div
+            className={cx(
+              "shrink-0 p-1 px-1.5 text-xs",
+              isCtrlActive
+                ? "text-black dark:text-white"
+                : "text-slate-800/20 dark:text-slate-300/20",
+            )}
+          >
+            Ctrl
+          </div>
+          <div
+            className={cx(
+              "shrink-0 p-1 px-1.5 text-xs",
+              isAltActive
+                ? "text-black dark:text-white"
+                : "text-slate-800/20 dark:text-slate-300/20",
+            )}
+          >
+            Alt
+          </div>
+          <div
+            className={cx(
+              "shrink-0 p-1 px-1.5 text-xs",
+              isMetaActive
+                ? "text-black dark:text-white"
+                : "text-slate-800/20 dark:text-slate-300/20",
+            )}
+          >
+            Meta
+          </div>
+          <div
+            className={cx(
+              "shrink-0 p-1 px-1.5 text-xs",
+              isAltGrActive
+                ? "text-black dark:text-white"
+                : "text-slate-800/20 dark:text-slate-300/20",
+            )}
+          >
+            AltGr
+          </div>
           <div
             className={cx(
               "shrink-0 p-1 px-1.5 text-xs",

ui/src/components/VirtualKeyboard.tsx

@@ -27,6 +27,7 @@ const AttachIcon = ({ className }: { className?: string }) => {
 
 function KeyboardWrapper() {
   const [layoutName, setLayoutName] = useState("default");
+  const [depressedButtons, setDepressedButtons] = useState("");
 
   const keyboardRef = useRef<HTMLDivElement>(null);
   const showAttachedVirtualKeyboard = useUiStore(
@@ -54,6 +55,21 @@ function KeyboardWrapper() {
 
   const setIsCapsLockActive = useHidStore(state => state.setIsCapsLockActive);
 
+  const isShiftActive = useHidStore(state => state.isShiftActive);
+  const setIsShiftActive = useHidStore(state => state.setIsShiftActive);
+
+  const isCtrlActive = useHidStore(state => state.isCtrlActive);
+  const setIsCtrlActive = useHidStore(state => state.setIsCtrlActive);
+
+  const isAltActive = useHidStore(state => state.isAltActive);
+  const setIsAltActive = useHidStore(state => state.setIsAltActive);
+
+  const isMetaActive = useHidStore(state => state.isMetaActive);
+  const setIsMetaActive = useHidStore(state => state.setIsMetaActive);
+
+  const isAltGrActive = useHidStore(state => state.isAltGrActive);
+  const setIsAltGrActive = useHidStore(state => state.setIsAltGrActive);
+
   const startDrag = useCallback((e: MouseEvent | TouchEvent) => {
     if (!keyboardRef.current) return;
     if (e instanceof TouchEvent && e.touches.length > 1) return;
@@ -123,80 +139,123 @@ function KeyboardWrapper() {
     };
   }, [endDrag, onDrag, startDrag]);
 
-  const onKeyDown = useCallback(
-    (key: string) => {
-      const isKeyShift = key === "{shift}" || key === "ShiftLeft" || key === "ShiftRight";
-      const isKeyCaps = key === "CapsLock";
-      const cleanKey = key.replace(/[()]/g, "");
-      const keyHasShiftModifier = key.includes("(");
+  useEffect(() => {
+    // if you have the CapsLock "down", then the shift state is inverted
+    const effectiveShift = isCapsLockActive ? false === isShiftActive : isShiftActive;
+    setLayoutName(effectiveShift ? "shift" : "default");
+  },
+    [setLayoutName, isCapsLockActive, isShiftActive]
+  );
 
-      // Handle toggle of layout for shift or caps lock
-      const toggleLayout = () => {
-        setLayoutName(prevLayout => (prevLayout === "default" ? "shift" : "default"));
-      };
+  // this causes the buttons to look depressed/clicked depending on the sticky state
+  useEffect(() => {
+    let buttons = "None ";  // make sure we name at least one (fake) button
+    if (isCapsLockActive) buttons += "CapsLock ";
+    if (isShiftActive) buttons += "ShiftLeft ShiftRight ";
+    if (isCtrlActive) buttons += "ControlLeft ControlRight ";
+    if (isAltActive) buttons += "AltLeft AltRight ";
+    if (isMetaActive) buttons += "MetaLeft MetaRight ";
+    setDepressedButtons(buttons.trimEnd());
+  },
+    [setDepressedButtons, isCapsLockActive, isShiftActive, isCtrlActive, isAltActive, isMetaActive, isAltGrActive]
+  );
 
-      if (key === "CtrlAltDelete") {
-        sendKeyboardEvent(
-          [keys["Delete"]],
-          [modifiers["ControlLeft"], modifiers["AltLeft"]],
-        );
-        setTimeout(resetKeyboardState, 100);
-        return;
-      }
+  const onKeyPress = useCallback((key: string) => {
+    // handle the fake combo keys first
+    if (key === "CtrlAltDelete") {
+      sendKeyboardEvent(
+        [keys["Delete"]],
+        [modifiers["ControlLeft"], modifiers["AltLeft"]],
+      );
+      setTimeout(resetKeyboardState, 100);
+      return;
+    }
 
-      if (key === "AltMetaEscape") {
-        sendKeyboardEvent(
-          [keys["Escape"]],
-          [modifiers["MetaLeft"], modifiers["AltLeft"]],
-        );
-
-        setTimeout(resetKeyboardState, 100);
-        return;
-      }
-
-      if (key === "CtrlAltBackspace") {
-        sendKeyboardEvent(
-          [keys["Backspace"]],
-          [modifiers["ControlLeft"], modifiers["AltLeft"]],
-        );
-
-        setTimeout(resetKeyboardState, 100);
-        return;
-      }
-
-      if (isKeyShift || isKeyCaps) {
-        toggleLayout();
-
-        if (isCapsLockActive) {
-          if (!isKeyboardLedManagedByHost) {
-            setIsCapsLockActive(false);
-          }
-          sendKeyboardEvent([keys["CapsLock"]], []);
-          return;
-        }
-      }
-
-      // Handle caps lock state change
-      if (isKeyCaps && !isKeyboardLedManagedByHost) {
-        setIsCapsLockActive(!isCapsLockActive);
-      }
-
-      // Collect new active keys and modifiers
-      const newKeys = keys[cleanKey] ? [keys[cleanKey]] : [];
-      const newModifiers =
-        keyHasShiftModifier && !isCapsLockActive ? [modifiers["ShiftLeft"]] : [];
-
-      // Update current keys and modifiers
-      sendKeyboardEvent(newKeys, newModifiers);
-
-      // If shift was used as a modifier and caps lock is not active, revert to default layout
-      if (keyHasShiftModifier && !isCapsLockActive) {
-        setLayoutName("default");
-      }
+    if (key === "AltMetaEscape") {
+      sendKeyboardEvent(
+        [keys["Escape"]],
+        [modifiers["MetaLeft"], modifiers["AltLeft"]],
+      );
 
       setTimeout(resetKeyboardState, 100);
-    },
-    [isCapsLockActive, isKeyboardLedManagedByHost, sendKeyboardEvent, resetKeyboardState, setIsCapsLockActive],
+      return;
+    }
+
+    if (key === "CtrlAltBackspace") {
+      sendKeyboardEvent(
+        [keys["Backspace"]],
+        [modifiers["ControlLeft"], modifiers["AltLeft"]],
+      );
+
+      setTimeout(resetKeyboardState, 100);
+      return;
+    }
+
+    // strip away the parens for shifted characters
+    const cleanKey = key.replace(/[()]/g, "");
+
+    const passthrough = ["PrintScreen", "SystemRequest", "Pause", "Break", "ScrollLock", "Enter", "Space"].find((value) => value === cleanKey);
+
+    if (passthrough) {
+      emitkeycode(cleanKey);
+      return;
+    }
+
+    // adjust the sticky state of the Shift/Ctrl/Alt/Meta/AltGr
+    if (key === "CapsLock" && !isKeyboardLedManagedByHost)
+      setIsCapsLockActive(!isCapsLockActive);
+    else if (key === "ShiftLeft" || key === "ShiftRight")
+      setIsShiftActive(!isShiftActive);
+    else if (key === "ControlLeft" || key === "ControlRight")
+      setIsCtrlActive(!isCtrlActive);
+    else if (key === "AltLeft" || key === "AltRight")
+      setIsAltActive(!isAltActive);
+    else if (key === "MetaLeft" || key === "MetaRight")
+      setIsMetaActive(!isMetaActive);
+    else if (key === "AltGr")
+      setIsAltGrActive(!isAltGrActive);
+
+    emitkeycode(cleanKey);
+
+    function emitkeycode(key: string) {
+      const effectiveMods: number[] = [];
+
+      if (isShiftActive)
+        effectiveMods.push(modifiers["ShiftLeft"]);
+
+      if (isCtrlActive)
+        effectiveMods.push(modifiers["ControlLeft"]);
+
+      if (isAltActive)
+        effectiveMods.push(modifiers["AltLeft"]);
+
+      if (isMetaActive)
+        effectiveMods.push(modifiers["MetaLeft"]);
+
+      if (isAltGrActive) {
+        effectiveMods.push(modifiers["MetaRight"]);
+        effectiveMods.push(modifiers["CtrlLeft"]);
+      }
+
+      const keycode = keys[key];
+      if (keycode) {
+        // send the keycode with modifiers
+        sendKeyboardEvent([keycode], effectiveMods);
+      }
+
+      // release the key (if one pressed), but retain the modifiers
+      setTimeout(() => sendKeyboardEvent([], effectiveMods), 50);
+    }
+  },
+    [isKeyboardLedManagedByHost,
+      setIsCapsLockActive, isCapsLockActive,
+      setIsShiftActive, isShiftActive,
+      setIsCtrlActive, isCtrlActive,
+      setIsAltActive, isAltActive,
+      setIsMetaActive, isMetaActive,
+      setIsAltGrActive, isAltGrActive,
+      sendKeyboardEvent, resetKeyboardState
+    ],
   );
 
   const virtualKeyboard = useHidStore(state => state.isVirtualKeyboardEnabled);
@@ -276,12 +335,16 @@ function KeyboardWrapper() {
                     <Keyboard
                       baseClass="simple-keyboard-main"
                       layoutName={layoutName}
-                      onKeyPress={onKeyDown}
+                      onKeyPress={onKeyPress}
                       buttonTheme={[
                         {
                           class: "combination-key",
                           buttons: "CtrlAltDelete AltMetaEscape CtrlAltBackspace",
                         },
+                        {
+                          class: "depressed-key",
+                          buttons: depressedButtons
+                        },
                       ]}
                       display={keyDisplayMap}
                       layout={{
@@ -305,8 +368,6 @@ function KeyboardWrapper() {
                         ],
                       }}
                       disableButtonHold={true}
-                      syncInstanceInputs={true}
-                      debug={false}
                     />
 
                     <div className="controlArrows">
@@ -314,25 +375,24 @@ function KeyboardWrapper() {
                         baseClass="simple-keyboard-control"
                         theme="simple-keyboard hg-theme-default hg-layout-default"
                         layoutName={layoutName}
-                        onKeyPress={onKeyDown}
+                        onKeyPress={onKeyPress}
                         display={keyDisplayMap}
                         layout={{
-                          default: ["PrintScreen ScrollLock Pause", "Insert Home Pageup", "Delete End Pagedown"],
-                          shift: ["(PrintScreen) ScrollLock (Pause)", "Insert Home Pageup", "Delete End Pagedown"],
+                          default: ["PrintScreen ScrollLock Pause", "Insert Home PageUp", "Delete End PageDown"],
+                          shift: ["(PrintScreen) ScrollLock (Pause)", "Insert Home PageUp", "Delete End PageDown"],
                         }}
-                        syncInstanceInputs={true}
-                        debug={false}
+                        disableButtonHold={true}
                       />
                       <Keyboard
                         baseClass="simple-keyboard-arrows"
                         theme="simple-keyboard hg-theme-default hg-layout-default"
-                        onKeyPress={onKeyDown}
+                        onKeyPress={onKeyPress}
                         display={keyDisplayMap}
                         layout={{
                           default: ["ArrowUp", "ArrowLeft ArrowDown ArrowRight"],
+                          shift: ["ArrowUp", "ArrowLeft ArrowDown ArrowRight"],
                         }}
-                        syncInstanceInputs={true}
-                        debug={false}
+                        disableButtonHold={true}
                       />
                     </div>
                   </div>

ui/src/components/WebRTCVideo.tsx

@@ -71,6 +71,15 @@ export default function WebRTCVideo() {
   const hdmiError = ["no_lock", "no_signal", "out_of_range"].includes(hdmiState);
   const isVideoLoading = !isPlaying;
 
+  // Keyboard related states
+  const { 
+    setIsShiftActive,
+    setIsCtrlActive,
+    setIsAltActive,
+    setIsMetaActive,
+    setIsAltGrActive
+  } = useHidStore();
+
   const [blockWheelEvent, setBlockWheelEvent] = useState(false);
 
   // Misc states and hooks
@@ -423,6 +432,12 @@ export default function WebRTCVideo() {
         setIsScrollLockActive(e.getModifierState("ScrollLock"));
       }
 
+      setIsShiftActive(e.getModifierState("Shift"))
+      setIsCtrlActive(e.getModifierState("Control"))
+      setIsAltActive(e.getModifierState("Alt"))
+      setIsMetaActive(e.getModifierState("Meta"))
+      setIsAltGrActive(e.getModifierState("AltGraph"))
+
       if (code == "IntlBackslash" && ["`", "~"].includes(key)) {
         code = "Backquote";
       } else if (code == "Backquote" && ["§", "±"].includes(key)) {
@@ -452,12 +467,17 @@ export default function WebRTCVideo() {
       sendKeyboardEvent([...new Set(newKeys)], [...new Set(newModifiers)]);
     },
     [
-      handleModifierKeys,
-      sendKeyboardEvent,
       isKeyboardLedManagedByHost,
       setIsNumLockActive,
       setIsCapsLockActive,
       setIsScrollLockActive,
+      setIsShiftActive,
+      setIsCtrlActive,
+      setIsAltActive,
+      setIsMetaActive,
+      setIsAltGrActive,
+      handleModifierKeys,
+      sendKeyboardEvent
     ],
   );
 
@@ -472,6 +492,12 @@ export default function WebRTCVideo() {
         setIsScrollLockActive(e.getModifierState("ScrollLock"));
       }
 
+      setIsShiftActive(e.getModifierState("Shift"))
+      setIsCtrlActive(e.getModifierState("Control"))
+      setIsAltActive(e.getModifierState("Alt"))
+      setIsMetaActive(e.getModifierState("Meta"))
+      setIsAltGrActive(e.getModifierState("AltGraph"))
+
       // Filtering out the key that was just released (keys[e.code])
       const newKeys = prev.activeKeys.filter(k => k !== keys[e.code]).filter(Boolean);
 
@@ -484,12 +510,17 @@ export default function WebRTCVideo() {
       sendKeyboardEvent([...new Set(newKeys)], [...new Set(newModifiers)]);
     },
     [
-      handleModifierKeys,
-      sendKeyboardEvent,
       isKeyboardLedManagedByHost,
       setIsNumLockActive,
       setIsCapsLockActive,
       setIsScrollLockActive,
+      setIsShiftActive,
+      setIsCtrlActive,
+      setIsAltActive,
+      setIsMetaActive,
+      setIsAltGrActive,
+      handleModifierKeys,
+      sendKeyboardEvent
     ],
   );
 

ui/src/hooks/stores.ts

@@ -481,6 +481,21 @@ export interface HidState {
   isVirtualKeyboardEnabled: boolean;
   setVirtualKeyboardEnabled: (enabled: boolean) => void;
 
+  isShiftActive: boolean;
+  setIsShiftActive: (enabled: boolean) => void;
+
+  isCtrlActive: boolean;
+  setIsCtrlActive: (enabled: boolean) => void;
+
+  isAltActive: boolean;
+  setIsAltActive: (enabled: boolean) => void;
+
+  isMetaActive: boolean;
+  setIsMetaActive: (enabled: boolean) => void;
+
+  isAltGrActive: boolean;
+  setIsAltGrActive: (enabled: boolean) => void;
+
   isPasteModeEnabled: boolean;
   setPasteModeEnabled: (enabled: boolean) => void;
 
@@ -527,6 +542,21 @@ export const useHidStore = create<HidState>((set, get) => ({
   isVirtualKeyboardEnabled: false,
   setVirtualKeyboardEnabled: enabled => set({ isVirtualKeyboardEnabled: enabled }),
 
+  isShiftActive: false,
+  setIsShiftActive: enabled => set({ isShiftActive: enabled }),
+
+  isCtrlActive: false,
+  setIsCtrlActive: enabled => set({ isCtrlActive: enabled }),
+
+  isAltActive: false,
+  setIsAltActive: enabled => set({ isAltActive: enabled }),
+
+  isMetaActive: false,
+  setIsMetaActive: enabled => set({ isMetaActive: enabled }),
+
+  isAltGrActive: false,
+  setIsAltGrActive: enabled => set({ isAltGrActive: enabled }),
+
   isPasteModeEnabled: false,
   setPasteModeEnabled: enabled => set({ isPasteModeEnabled: enabled }),
 

ui/src/index.css

@@ -222,6 +222,10 @@ video::-webkit-media-controls {
   background: none;
 }
 
+.simple-keyboard .hg-button.depressed-key {
+  @apply border-gray-800! border-b-gray-600! border-t-gray-900! bg-gray-700!;
+}
+
 .simple-keyboard .hg-button.selectedButton {
   background: rgba(5, 25, 70, 0.53);
   @apply text-white;

ui/src/keyboardMappings.ts

@@ -5,14 +5,14 @@ export const keys = {
   ArrowLeft: 0x50,
   ArrowRight: 0x4f,
   ArrowUp: 0x52,
-  Backquote: 0x35,
+  Backquote: 0x35,    // aka Grave
   Backslash: 0x31,
   Backspace: 0x2a,
-  BracketLeft: 0x2f,
-  BracketRight: 0x30,
+  BracketLeft: 0x2f,  // aka LeftBrace
+  BracketRight: 0x30, // aka RightBrace
   CapsLock: 0x39,
   Comma: 0x36,
-  ContextMenu: 0,
+  Compose: 0x65,
   Delete: 0x4c,
   Digit0: 0x27,
   Digit1: 0x1e,
@@ -41,6 +41,18 @@ export const keys = {
   F11: 0x44,
   F12: 0x45,
   F13: 0x68,
+  F14: 0x69,
+  F15: 0x6a,
+  F16: 0x6b,
+  F17: 0x6c,
+  F18: 0x6d,
+  F19: 0x6e,
+  F20: 0x6f,
+  F21: 0x70,
+  F22: 0x71,
+  F23: 0x72,
+  F24: 0x73,
+  HashTilde: 0x32,
   Home: 0x4a,
   Insert: 0x49,
   IntlBackslash: 0x64,
@@ -70,37 +82,58 @@ export const keys = {
   KeyX: 0x1b,
   KeyY: 0x1c,
   KeyZ: 0x1d,
+  KeypadComma: 0x85,
+  KeypadEqual: 0x86,
+  KeyRO: 0x87,
+  KatakanaHiragana: 0x88,
+  Yen: 0x89,
+  Henkan: 0x8a,
+  Muhenkan: 0x8b,
+  KPJPComma: 0x8c,
+  International7: 0x8d,
+  International8: 0x8e,
+  International9: 0x8f,
+  Hangeul: 0x90,
+  Hanja: 0x91,
+  Katakana: 0x92,
+  Hiragana: 0x93,
+  Zenkakuhankaku:0x94,
+  KeypadLeftParen: 0xb6,
+  KeypadRightParen: 0xb7,
   KeypadExclamation: 0xcf,
   Minus: 0x2d,
-  NumLock: 0x53,
-  Numpad0: 0x62,
-  Numpad1: 0x59,
-  Numpad2: 0x5a,
-  Numpad3: 0x5b,
-  Numpad4: 0x5c,
+  None: 0x00,
+  NumLock: 0x53,  // and Clear
+  Numpad0: 0x62,  // and Insert
+  Numpad1: 0x59,  // and End
+  Numpad2: 0x5a,  // and Down Arrow
+  Numpad3: 0x5b,  // and Page Down
+  Numpad4: 0x5c,  // and Left Arrow
   Numpad5: 0x5d,
-  Numpad6: 0x5e,
-  Numpad7: 0x5f,
-  Numpad8: 0x60,
-  Numpad9: 0x61,
-  NumpadAdd: 0x57,
-  NumpadDivide: 0x54,
+  Numpad6: 0x5e,  // and Right Arrow
+  Numpad7: 0x5f,  // and Home
+  Numpad8: 0x60,  // and Up Arrow
+  Numpad9: 0x61,  // and Page Up
+  NumpadPlus: 0x57,
+  NumpadSlash: 0x54,
   NumpadEnter: 0x58,
   NumpadEqual: 0x67,
-  NumpadMultiply: 0x55,
-  NumpadSubtract: 0x56,
-  NumpadDecimal: 0x63,
+  NumpadAsterisk: 0x55,
+  NumpadMinus: 0x56,
+  NumpadDecimal: 0x63,  // aka NumpadDot and Delete
+  Overflow: 0x01,
   PageDown: 0x4e,
   PageUp: 0x4b,
-  Period: 0x37,
+  Period: 0x37,     // aka Dot
   PrintScreen: 0x46,
   Pause: 0x48,
-  Quote: 0x34,
+  Power: 0x66,
+  Quote: 0x34,      // aka Apostrophe
   ScrollLock: 0x47,
   Semicolon: 0x33,
   Slash: 0x38,
   Space: 0x2c,
-  SystemRequest: 0x9a,
+  SystemRequest: 0x9a,  // aka Attention
   Tab: 0x2b,
 } as Record<string, number>;
 
@@ -131,23 +164,23 @@ export const keyDisplayMap: Record<string, string> = {
   AltMetaEscape: "Alt + Meta + Escape",
   CtrlAltBackspace: "Ctrl + Alt + Backspace",
   Escape: "esc",
-  Tab: "tab",
-  Backspace: "backspace",
-  "(Backspace)": "backspace",
+  Tab: "tab ⇥",
+  Backspace: "backspace ⌫",
   Enter: "enter",
-  CapsLock: "caps lock",
-  ShiftLeft: "shift",
-  ShiftRight: "shift",
-  ControlLeft: "ctrl",
-  AltLeft: "alt",
-  AltRight: "alt",
-  MetaLeft: "meta",
-  MetaRight: "meta",
+  CapsLock: "caps lock ⇪",
+  ShiftLeft: "shift ⇧",
+  ShiftRight: "⇧ shift",
+  ControlLeft: "ctrl ⌃",
+  ControlRight: "⌃ ctrl",
+  AltLeft: "alt ⌥",
+  AltRight: "⌥ alt",
+  MetaLeft: "meta ⌘", // "meta ⊞" for windows
+  MetaRight: "⌘ meta",// "≣ meta" for windows
   Space: " ",
-  Insert: "insert",
+  Insert: "ins",
   Home: "home",
   PageUp: "page up",
-  Delete: "delete",
+  Delete: "del ⌦",
   End: "end",
   PageDown: "page down",
   ArrowLeft: "←",
@@ -213,13 +246,12 @@ export const keyDisplayMap: Record<string, string> = {
   Numpad0: "Num 0", Numpad1: "Num 1", Numpad2: "Num 2",
   Numpad3: "Num 3", Numpad4: "Num 4", Numpad5: "Num 5",
   Numpad6: "Num 6", Numpad7: "Num 7", Numpad8: "Num 8",
-  Numpad9: "Num 9", NumpadAdd: "Num +", NumpadSubtract: "Num -",
-  NumpadMultiply: "Num *", NumpadDivide: "Num /", NumpadDecimal: "Num .",
+  Numpad9: "Num 9", NumpadPlus: "Num +", NumpadMinus: "Num -",
+  NumpadAsterisk: "Num *", NumpadSlash: "Num /", NumpadDecimal: "Num .",
   NumpadEqual: "Num =", NumpadEnter: "Num Enter",
   NumLock: "Num Lock",
 
   // Modals
   PrintScreen: "prt sc", ScrollLock: "scr lk", Pause: "pause",
-  "(PrintScreen)": "sys rq", "(Pause)": "break",
-  SystemRequest: "sys rq", Break: "break"
+  "(PrintScreen)": "sys rq", "(Pause)": "break"
 };