ui/src/components/WebRTCVideo.tsx

@@ -225,8 +225,11 @@ export default function WebRTCVideo() {
   );
 
   const relMouseMoveHandler = useMemo(
-    () => getRelMouseMoveHandler(),
+    () => getRelMouseMoveHandler({
-    [getRelMouseMoveHandler],
+      isPointerLockActive,
+      isPointerLockPossible,
+    }),
+    [getRelMouseMoveHandler, isPointerLockActive, isPointerLockPossible],
   );
 
   const mouseWheelHandler = useMemo(

ui/src/hooks/stores.ts

@@ -461,6 +461,9 @@ export interface HidState {
   keysDownState: KeysDownState;
   setKeysDownState: (state: KeysDownState) => void;
 
+  keyPressReportApiAvailable: boolean;
+  setkeyPressReportApiAvailable: (available: boolean) => void;
+
   isVirtualKeyboardEnabled: boolean;
   setVirtualKeyboardEnabled: (enabled: boolean) => void;
 
@@ -478,6 +481,9 @@ export const useHidStore = create<HidState>(set => ({
   keysDownState: { modifier: 0, keys: [0,0,0,0,0,0] } as KeysDownState,
   setKeysDownState: (state: KeysDownState): void => set({ keysDownState: state }),
 
+  keyPressReportApiAvailable: true,
+  setkeyPressReportApiAvailable: (available: boolean) => set({ keyPressReportApiAvailable: available }),
+
   isVirtualKeyboardEnabled: false,
   setVirtualKeyboardEnabled: (enabled: boolean): void => set({ isVirtualKeyboardEnabled: enabled }),
 

ui/src/hooks/useHidRpc.ts

@@ -81,11 +81,8 @@ export function useHidRpc(onHidRpcMessage?: (payload: RpcMessage) => void) {
       return;
     }
 
-    if (message.version > HID_RPC_VERSION) {
+    if (message.version < HID_RPC_VERSION) {
-      // we assume that the UI is always using the latest version of the HID RPC protocol
+      console.error("Server is using an older HID RPC version than the client", message);
-      // so we can't support this
-      // TODO: use capabilities to determine rather than version number
-      console.error("Server is using a newer HID RPC version than the client", message);
       return;
     }
 

ui/src/hooks/useKeyboard.ts

@@ -21,19 +21,23 @@ export default function useKeyboard() {
   // dynamically set when the device responds to the first key press event or reports its
   // keysDownState when queried since the keyPressReport was introduced together with the 
   // getKeysDownState API.
+  const { keyPressReportApiAvailable, setkeyPressReportApiAvailable } = useHidStore();
+  const enableKeyPressReport = useCallback((reason: string) => {
+    if (keyPressReportApiAvailable) return;
+    console.debug(`Enable keyPressReport API because ${reason}`);
+    setkeyPressReportApiAvailable(true);
+  }, [setkeyPressReportApiAvailable, keyPressReportApiAvailable]);
 
   // HidRPC is a binary format for exchanging keyboard and mouse events
-  const {
+  const { reportKeyboardEvent, reportKeypressEvent, rpcHidReady } = useHidRpc((message) => {
-    reportKeyboardEvent: sendKeyboardEventHidRpc,
-    reportKeypressEvent: sendKeypressEventHidRpc,
-    rpcHidReady,
-  } = useHidRpc((message) => {
     switch (message.constructor) {
       case KeysDownStateMessage:
         setKeysDownState((message as KeysDownStateMessage).keysDownState);
+        enableKeyPressReport("HidRPC:KeysDownStateMessage received");
         break;
       case KeyboardLedStateMessage:
         setKeyboardLedState((message as KeyboardLedStateMessage).keyboardLedState);
+        enableKeyPressReport("HidRPC:KeyboardLedStateMessage received");
         break;
       default:
         break;
@@ -52,7 +56,8 @@ export default function useKeyboard() {
 
       if (rpcHidReady) {
         console.debug("Sending keyboard report via HidRPC");
-        sendKeyboardEventHidRpc(state.keys, state.modifier);
+        reportKeyboardEvent(state.keys, state.modifier);
+        enableKeyPressReport("HidRPC:KeyboardReport received");
         return;
       }
 
@@ -66,7 +71,28 @@ export default function useKeyboard() {
       rpcDataChannel?.readyState,
       rpcHidReady,
       send,
-      sendKeyboardEventHidRpc,
+      reportKeyboardEvent,
+      enableKeyPressReport,
+    ],
+  );
+
+  // sendKeypressEvent is used to send a single key press/release event to the device.
+  // It sends the key and whether it is pressed or released.
+  // Older device version will not understand this request and will respond with
+  // an error with code -32601, which means that the RPC method name was not recognized.
+  // In that case we will switch to local key handling and update the keysDownState
+  // in client/browser-side code using simulateDeviceSideKeyHandlingForLegacyDevices.
+  const sendKeypressEvent = useCallback(
+    async (key: number, press: boolean) => {
+      console.debug(`Send keypressEvent key: ${key}, press: ${press}`);
+
+      if (!rpcHidReady) return;
+
+      reportKeypressEvent(key, press);
+    },
+    [
+      rpcHidReady,
+      reportKeypressEvent,
     ],
   );
 
@@ -123,13 +149,14 @@ export default function useKeyboard() {
       if ((key || 0) === 0) return; // ignore zero key presses (they are bad mappings)
 
       if (rpcHidReady) {
+        console.debug("Sending keypress event via HidRPC");
+        reportKeypressEvent(key, press);
+        return;
+      }
+
+      if (keyPressReportApiAvailable) {
         // if the keyPress api is available, we can just send the key press event
-        // sendKeypressEvent is used to send a single key press/release event to the device.
+        sendKeypressEvent(key, press);
-        // It sends the key and whether it is pressed or released.
-        // Older device version doesn't support this API, so we will switch to local key handling
-        // In that case we will switch to local key handling and update the keysDownState
-        // in client/browser-side code using simulateDeviceSideKeyHandlingForLegacyDevices.
-        sendKeypressEventHidRpc(key, press);
       } else {
         // if the keyPress api is not available, we need to handle the key locally
         const downState = simulateDeviceSideKeyHandlingForLegacyDevices(keysDownState, key, press);
@@ -142,12 +169,14 @@ export default function useKeyboard() {
       }
     },
     [
-      rpcHidReady,
+      keyPressReportApiAvailable,
       keysDownState,
       resetKeyboardState,
       rpcDataChannel?.readyState,
+      rpcHidReady,
       sendKeyboardEvent,
-      sendKeypressEventHidRpc,
+      sendKeypressEvent,
+      reportKeypressEvent,
     ],
   );
 

ui/src/hooks/useMouse.ts

@@ -13,6 +13,11 @@ export interface AbsMouseMoveHandlerProps {
   videoHeight: number;
 }
 
+export interface RelMouseMoveHandlerProps {
+  isPointerLockActive: boolean;
+  isPointerLockPossible: boolean;
+}
+
 export default function useMouse() {
   // states
   const { setMousePosition, setMouseMove } = useMouseStore();
@@ -50,8 +55,9 @@ export default function useMouse() {
   );
 
   const getRelMouseMoveHandler = useCallback(
-    () => (e: MouseEvent) => {
+    ({ isPointerLockActive, isPointerLockPossible }: RelMouseMoveHandlerProps) => (e: MouseEvent) => {
       if (mouseMode !== "relative") return;
+      if (isPointerLockActive === false && isPointerLockPossible) return;
 
       // Send mouse movement
       const { buttons } = e;

ui/src/routes/devices.$id.tsx

@@ -583,6 +583,7 @@ export default function KvmIdRoute() {
   const { 
     keyboardLedState,  setKeyboardLedState,
     keysDownState, setKeysDownState, setUsbState,
+    setkeyPressReportApiAvailable
   } = useHidStore();
 
   const [hasUpdated, setHasUpdated] = useState(false);
@@ -620,6 +621,7 @@ export default function KvmIdRoute() {
       const downState = resp.params as KeysDownState;
       console.debug("Setting key down state:", downState);
       setKeysDownState(downState);
+      setkeyPressReportApiAvailable(true); // if they returned a keyDownState, we know they also support keyPressReport
     }
 
     if (resp.method === "otaState") {
@@ -696,6 +698,7 @@ export default function KvmIdRoute() {
         if (resp.error.code === -32601) {
           // if we don't support key down state, we know key press is also not available
           console.warn("Failed to get key down state, switching to old-school", resp.error);
+          setkeyPressReportApiAvailable(false);
         } else {
           console.error("Failed to get key down state", resp.error);
         }
@@ -703,10 +706,11 @@ export default function KvmIdRoute() {
         const downState = resp.result as KeysDownState;
         console.debug("Keyboard key down state", downState);
         setKeysDownState(downState);
+        setkeyPressReportApiAvailable(true); // if they returned a keyDownState, we know they also support keyPressReport
       }
       setNeedKeyDownState(false);
     });
-  }, [keysDownState, needKeyDownState, rpcDataChannel?.readyState, send, setKeysDownState]);
+  }, [keysDownState, needKeyDownState, rpcDataChannel?.readyState, send, setkeyPressReportApiAvailable, setKeysDownState]);
 
   // When the update is successful, we need to refresh the client javascript and show a success modal
   useEffect(() => {