feat: relative mouse (#246)

jsonrpc.go

@@ -799,6 +799,7 @@ var rpcHandlers = map[string]RPCHandler{
 	"getCloudState":          {Func: rpcGetCloudState},
 	"keyboardReport":         {Func: rpcKeyboardReport, Params: []string{"modifier", "keys"}},
 	"absMouseReport":         {Func: rpcAbsMouseReport, Params: []string{"x", "y", "buttons"}},
+	"relMouseReport":         {Func: rpcRelMouseReport, Params: []string{"dx", "dy", "buttons"}},
 	"wheelReport":            {Func: rpcWheelReport, Params: []string{"wheelY"}},
 	"getVideoState":          {Func: rpcGetVideoState},
 	"getUSBState":            {Func: rpcGetUSBState},

ui/src/components/InfoBar.tsx

@@ -14,6 +14,7 @@ export default function InfoBar() {
   const activeModifiers = useHidStore(state => state.activeModifiers);
   const mouseX = useMouseStore(state => state.mouseX);
   const mouseY = useMouseStore(state => state.mouseY);
+  const mouseMove = useMouseStore(state => state.mouseMove);
 
   const videoClientSize = useVideoStore(
     state => `${Math.round(state.clientWidth)}x${Math.round(state.clientHeight)}`,
@@ -62,7 +63,7 @@ export default function InfoBar() {
               </div>
             ) : null}
 
-            {settings.debugMode ? (
+            {(settings.debugMode && settings.mouseMode == "absolute") ? (
               <div className="flex w-[118px] items-center gap-x-1">
                 <span className="text-xs font-semibold">Pointer:</span>
                 <span className="text-xs">
@@ -71,6 +72,17 @@ export default function InfoBar() {
               </div>
             ) : null}
 
+            {(settings.debugMode && settings.mouseMode == "relative") ? (
+              <div className="flex w-[118px] items-center gap-x-1">
+                <span className="text-xs font-semibold">Last Move:</span>
+                <span className="text-xs">
+                  {mouseMove ?
+                    `${mouseMove.x},${mouseMove.y} ${mouseMove.buttons ? `(${mouseMove.buttons})` : ""}` :
+                    "N/A"}
+                </span>
+              </div>
+            ) : null}
+
             {settings.debugMode && (
               <div className="flex w-[156px] items-center gap-x-1">
                 <span className="text-xs font-semibold">USB State:</span>

ui/src/components/WebRTCVideo.tsx

@@ -29,6 +29,7 @@ export default function WebRTCVideo() {
   const settings = useSettingsStore();
   const { sendKeyboardEvent, resetKeyboardState } = useKeyboard();
   const setMousePosition = useMouseStore(state => state.setMousePosition);
+  const setMouseMove = useMouseStore(state => state.setMouseMove);
   const {
     setClientSize: setVideoClientSize,
     setSize: setVideoSize,
@@ -93,19 +94,44 @@ export default function WebRTCVideo() {
   );
 
   // Mouse-related
-  const sendMouseMovement = useCallback(
+  const calcDelta = (pos: number) => Math.abs(pos) < 10 ? pos * 2 : pos;
+  const sendRelMouseMovement = useCallback(
     (x: number, y: number, buttons: number) => {
-      send("absMouseReport", { x, y, buttons });
+      if (settings.mouseMode !== "relative") return;
+      // if we ignore the event, double-click will not work
+      // if (x === 0 && y === 0 && buttons === 0) return;
+      send("relMouseReport", { dx: calcDelta(x), dy: calcDelta(y), buttons });
+      setMouseMove({ x, y, buttons });
+    },
+    [send, setMouseMove, settings.mouseMode],
+  );
 
+  const relMouseMoveHandler = useCallback(
+    (e: MouseEvent) => {
+      if (settings.mouseMode !== "relative") return;
+
+      // Send mouse movement
+      const { buttons } = e;
+      sendRelMouseMovement(e.movementX, e.movementY, buttons);
+    },
+    [sendRelMouseMovement, settings.mouseMode],
+  );
+
+  const sendAbsMouseMovement = useCallback(
+    (x: number, y: number, buttons: number) => {
+      if (settings.mouseMode !== "absolute") return;
+      send("absMouseReport", { x, y, buttons });
       // We set that for the debug info bar
       setMousePosition(x, y);
     },
-    [send, setMousePosition],
+    [send, setMousePosition, settings.mouseMode],
   );
 
-  const mouseMoveHandler = useCallback(
+  const absMouseMoveHandler = useCallback(
     (e: MouseEvent) => {
       if (!videoClientWidth || !videoClientHeight) return;
+      if (settings.mouseMode !== "absolute") return;
+
       // Get the aspect ratios of the video element and the video stream
       const videoElementAspectRatio = videoClientWidth / videoClientHeight;
       const videoStreamAspectRatio = videoWidth / videoHeight;
@@ -140,9 +166,9 @@ export default function WebRTCVideo() {
 
       // Send mouse movement
       const { buttons } = e;
-      sendMouseMovement(x, y, buttons);
+      sendAbsMouseMovement(x, y, buttons);
     },
-    [sendMouseMovement, videoClientHeight, videoClientWidth, videoWidth, videoHeight],
+    [sendAbsMouseMovement, videoClientHeight, videoClientWidth, videoWidth, videoHeight, settings.mouseMode],
   );
 
   const trackpadSensitivity = useDeviceSettingsStore(state => state.trackpadSensitivity);
@@ -193,8 +219,8 @@ export default function WebRTCVideo() {
   );
 
   const resetMousePosition = useCallback(() => {
-    sendMouseMovement(0, 0, 0);
+    sendAbsMouseMovement(0, 0, 0);
-  }, [sendMouseMovement]);
+  }, [sendAbsMouseMovement]);
 
   // Keyboard-related
   const handleModifierKeys = useCallback(
@@ -371,9 +397,9 @@ export default function WebRTCVideo() {
       const abortController = new AbortController();
       const signal = abortController.signal;
 
-      videoElmRefValue.addEventListener("mousemove", mouseMoveHandler, { signal });
+      videoElmRefValue.addEventListener("mousemove", absMouseMoveHandler, { signal });
-      videoElmRefValue.addEventListener("pointerdown", mouseMoveHandler, { signal });
+      videoElmRefValue.addEventListener("pointerdown", absMouseMoveHandler, { signal });
-      videoElmRefValue.addEventListener("pointerup", mouseMoveHandler, { signal });
+      videoElmRefValue.addEventListener("pointerup", absMouseMoveHandler, { signal });
       videoElmRefValue.addEventListener("keyup", videoKeyUpHandler, { signal });
       videoElmRefValue.addEventListener("wheel", mouseWheelHandler, {
         signal,
@@ -395,7 +421,7 @@ export default function WebRTCVideo() {
       };
     },
     [
-      mouseMoveHandler,
+      absMouseMoveHandler,
       resetMousePosition,
       onVideoPlaying,
       mouseWheelHandler,
@@ -403,6 +429,31 @@ export default function WebRTCVideo() {
     ],
   );
 
+  useEffect(
+    function setupRelativeMouseEventListeners() {
+      if (settings.mouseMode !== "relative") return;
+
+      const abortController = new AbortController();
+      const signal = abortController.signal;
+
+      // bind to body to capture all mouse events
+      const body = document.querySelector("body");
+      if (!body) return;
+
+      body.addEventListener("mousemove", relMouseMoveHandler, { signal });
+      body.addEventListener("pointerdown", relMouseMoveHandler, { signal });
+      body.addEventListener("pointerup", relMouseMoveHandler, { signal });
+
+      return () => {
+        abortController.abort();
+
+        body.removeEventListener("mousemove", relMouseMoveHandler);
+        body.removeEventListener("pointerdown", relMouseMoveHandler);
+        body.removeEventListener("pointerup", relMouseMoveHandler);
+      };
+    }, [settings.mouseMode, relMouseMoveHandler],
+  )
+
   useEffect(
     function updateVideoStream() {
       if (!mediaStream) return;

ui/src/hooks/stores.ts

@@ -197,15 +197,23 @@ export const useRTCStore = create<RTCState>(set => ({
   setTerminalChannel: channel => set({ terminalChannel: channel }),
 }));
 
+interface MouseMove {
+  x: number;
+  y: number;
+  buttons: number;
+}
 interface MouseState {
   mouseX: number;
   mouseY: number;
+  mouseMove?: MouseMove;
+  setMouseMove: (move?: MouseMove) => void;
   setMousePosition: (x: number, y: number) => void;
 }
 
 export const useMouseStore = create<MouseState>(set => ({
   mouseX: 0,
   mouseY: 0,
+  setMouseMove: (move?: MouseMove) => set({ mouseMove: move }),
   setMousePosition: (x, y) => set({ mouseX: x, mouseY: y }),
 }));
 

ui/src/routes/devices.$id.settings.mouse.tsx

@@ -1,23 +1,27 @@
-import { SettingsPageHeader } from "@components/SettingsPageheader";
+import MouseIcon from "@/assets/mouse-icon.svg";
-import { SettingsItem } from "./devices.$id.settings";
-import { Checkbox } from "@/components/Checkbox";
-import { GridCard } from "@/components/Card";
 import PointingFinger from "@/assets/pointing-finger.svg";
-import { CheckCircleIcon } from "@heroicons/react/16/solid";
+import { GridCard } from "@/components/Card";
+import { Checkbox } from "@/components/Checkbox";
 import { useDeviceSettingsStore, useSettingsStore } from "@/hooks/stores";
-import notifications from "@/notifications";
-import { useCallback, useEffect, useState } from "react";
 import { useJsonRpc } from "@/hooks/useJsonRpc";
-import { cx } from "../cva.config";
+import notifications from "@/notifications";
+import { SettingsPageHeader } from "@components/SettingsPageheader";
+import { CheckCircleIcon } from "@heroicons/react/16/solid";
+import { useCallback, useEffect, useState } from "react";
+import { FeatureFlag } from "../components/FeatureFlag";
 import { SelectMenuBasic } from "../components/SelectMenuBasic";
 import { useFeatureFlag } from "../hooks/useFeatureFlag";
-import { FeatureFlag } from "../components/FeatureFlag";
+import { SettingsItem } from "./devices.$id.settings";
 
 type ScrollSensitivity = "low" | "default" | "high";
 
 export default function SettingsKeyboardMouseRoute() {
   const hideCursor = useSettingsStore(state => state.isCursorHidden);
   const setHideCursor = useSettingsStore(state => state.setCursorVisibility);
+
+  const mouseMode = useSettingsStore(state => state.mouseMode);
+  const setMouseMode = useSettingsStore(state => state.setMouseMode);
+
   const scrollSensitivity = useDeviceSettingsStore(state => state.scrollSensitivity);
   const setScrollSensitivity = useDeviceSettingsStore(
     state => state.setScrollSensitivity,
@@ -122,19 +126,19 @@ export default function SettingsKeyboardMouseRoute() {
         </SettingsItem>
         <div className="space-y-4">
           <SettingsItem title="Modes" description="Choose the mouse input mode" />
-          <div className="flex flex-col items-center gap-4 md:flex-row">
+          <div className="flex items-center gap-4">
             <button
-              className="group block w-full grow"
+              className="block group grow"
-              onClick={() => console.log("Absolute mouse mode clicked")}
+              onClick={() => { setMouseMode("absolute"); }}
             >
               <GridCard>
-                <div className="group flex items-center gap-x-4 px-4 py-3">
+                <div className="flex items-center px-4 py-3 group gap-x-4">
                   <img
                     className="w-6 shrink-0 dark:invert"
                     src={PointingFinger}
                     alt="Finger touching a screen"
                   />
-                  <div className="flex grow items-center justify-between">
+                  <div className="flex items-center justify-between grow">
                     <div className="text-left">
                       <h3 className="text-sm font-semibold text-black dark:text-white">
                         Absolute
@@ -143,41 +147,32 @@ export default function SettingsKeyboardMouseRoute() {
                         Most convenient
                       </p>
                     </div>
-                    <CheckCircleIcon
+                    {mouseMode === "absolute" && (
-                      className={cx(
+                      <CheckCircleIcon className="w-4 h-4 text-blue-700 dark:text-blue-500" />
-                        "h-4 w-4 text-blue-700 transition-opacity duration-300 dark:text-blue-500",
                     )}
-                    />
                   </div>
                 </div>
               </GridCard>
             </button>
             <button
-              className="group block w-full grow cursor-not-allowed opacity-50"
+              className="block group grow"
-              disabled
+              onClick={() => { setMouseMode("relative"); }}
             >
               <GridCard>
-                <div className="group flex items-center gap-x-4 px-4 py-3">
+                <div className="flex items-center px-4 py-3 gap-x-4">
-                  <img
+                  <img className="w-6 shrink-0 dark:invert" src={MouseIcon} alt="Mouse icon" />
-                    className="w-6 shrink-0 dark:invert"
+                  <div className="flex items-center justify-between grow">
-                    src={PointingFinger}
-                    alt="Finger touching a screen"
-                  />
-                  <div className="flex grow items-center justify-between">
                     <div className="text-left">
                       <h3 className="text-sm font-semibold text-black dark:text-white">
                         Relative
                       </h3>
                       <p className="text-xs leading-none text-slate-800 dark:text-slate-300">
-                        Most Compatible
+                        Most Compatible (Beta)
                       </p>
                     </div>
-                    <CheckCircleIcon
+                    {mouseMode === "relative" && (
-                      className={cx(
+                      <CheckCircleIcon className="w-4 h-4 text-blue-700 dark:text-blue-500" />
-                        "hidden",
-                        "h-4 w-4 text-blue-700 transition-opacity duration-300 dark:text-blue-500",
                     )}
-                    />
                   </div>
                 </div>
               </GridCard>

usb.go

@@ -1,8 +1,9 @@
 package kvm
 
 import (
-	"github.com/jetkvm/kvm/internal/usbgadget"
 	"time"
+
+	"github.com/jetkvm/kvm/internal/usbgadget"
 )
 
 var gadget *usbgadget.UsbGadget
@@ -33,6 +34,10 @@ func rpcAbsMouseReport(x, y int, buttons uint8) error {
 	return gadget.AbsMouseReport(x, y, buttons)
 }
 
+func rpcRelMouseReport(dx, dy int8, buttons uint8) error {
+	return gadget.RelMouseReport(dx, dy, buttons)
+}
+
 func rpcWheelReport(wheelY int8) error {
 	return gadget.AbsMouseWheelReport(wheelY)
 }