feat: implement relative mouse

jsonrpc.go

@@ -771,6 +771,7 @@ var rpcHandlers = map[string]RPCHandler{
 	"getCloudState":          {Func: rpcGetCloudState},
 	"keyboardReport":         {Func: rpcKeyboardReport, Params: []string{"modifier", "keys"}},
 	"absMouseReport":         {Func: rpcAbsMouseReport, Params: []string{"x", "y", "buttons"}},
+	"relMouseReport":         {Func: rpcRelMouseReport, Params: []string{"mx", "my", "buttons"}},
 	"wheelReport":            {Func: rpcWheelReport, Params: []string{"wheelY"}},
 	"getVideoState":          {Func: rpcGetVideoState},
 	"getUSBState":            {Func: rpcGetUSBState},

ui/src/components/InfoBar.tsx

@@ -14,6 +14,7 @@ export default function InfoBar() {
   const activeModifiers = useHidStore(state => state.activeModifiers);
   const mouseX = useMouseStore(state => state.mouseX);
   const mouseY = useMouseStore(state => state.mouseY);
+  const mouseMove = useMouseStore(state => state.mouseMove);
 
   const videoClientSize = useVideoStore(
     state => `${Math.round(state.clientWidth)}x${Math.round(state.clientHeight)}`,
@@ -62,7 +63,7 @@ export default function InfoBar() {
               </div>
             ) : null}
 
-            {settings.debugMode ? (
+            {(settings.debugMode && settings.mouseMode == "absolute") ? (
               <div className="flex w-[118px] items-center gap-x-1">
                 <span className="text-xs font-semibold">Pointer:</span>
                 <span className="text-xs">
@@ -71,6 +72,17 @@ export default function InfoBar() {
               </div>
             ) : null}
 
+            {(settings.debugMode && settings.mouseMode == "relative") ? (
+              <div className="flex w-[118px] items-center gap-x-1">
+                <span className="text-xs font-semibold">Last Move:</span>
+                <span className="text-xs">
+                  {mouseMove ?
+                    `${mouseMove.x},${mouseMove.y} ${mouseMove.buttons ? `(${mouseMove.buttons})` : ""}` :
+                    "N/A"}
+                </span>
+              </div>
+            ) : null}
+
             {settings.debugMode && (
               <div className="flex w-[156px] items-center gap-x-1">
                 <span className="text-xs font-semibold">USB State:</span>

ui/src/components/WebRTCVideo.tsx

@@ -28,6 +28,7 @@ export default function WebRTCVideo() {
   const settings = useSettingsStore();
   const { sendKeyboardEvent, resetKeyboardState } = useKeyboard();
   const setMousePosition = useMouseStore(state => state.setMousePosition);
+  const setMouseMove = useMouseStore(state => state.setMouseMove);
   const {
     setClientSize: setVideoClientSize,
     setSize: setVideoSize,
@@ -92,14 +93,22 @@ export default function WebRTCVideo() {
   );
 
   // Mouse-related
+  const calcDelta = (pos: number) => Math.abs(pos) < 10 ? pos * 2 : pos;
+
   const sendMouseMovement = useCallback(
     (x: number, y: number, buttons: number) => {
+      if (settings.mouseMode === "relative") {
+        // if we ignore the event, double-click will not work
+        // if (x === 0 && y === 0 && buttons === 0) return;
+        send("relMouseReport", { mx: calcDelta(x), my: calcDelta(y), buttons });
+        setMouseMove({ x, y, buttons });
+      } else if (settings.mouseMode === "absolute") {
         send("absMouseReport", { x, y, buttons });
-
         // We set that for the debug info bar
         setMousePosition(x, y);
+      }
     },
-    [send, setMousePosition],
+    [send, setMousePosition, setMouseMove, settings.mouseMode],
   );
 
   const mouseMoveHandler = useCallback(
@@ -109,6 +118,13 @@ export default function WebRTCVideo() {
       const videoElementAspectRatio = videoClientWidth / videoClientHeight;
       const videoStreamAspectRatio = videoWidth / videoHeight;
 
+      const { buttons } = e;
+      // Send mouse movement events to the server
+      if (settings.mouseMode == "relative") {
+          sendMouseMovement(e.movementX, e.movementY, buttons);
+          return;
+      }
+
       // Calculate the effective video display area
       let effectiveWidth = videoClientWidth;
       let effectiveHeight = videoClientHeight;
@@ -137,11 +153,9 @@ export default function WebRTCVideo() {
       const x = Math.round(relativeX * 32767);
       const y = Math.round(relativeY * 32767);
 
-      // Send mouse movement
-      const { buttons } = e;
       sendMouseMovement(x, y, buttons);
     },
-    [sendMouseMovement, videoClientHeight, videoClientWidth, videoWidth, videoHeight],
+    [sendMouseMovement, videoClientHeight, videoClientWidth, videoWidth, videoHeight, settings.mouseMode],
   );
 
   const mouseWheelHandler = useCallback(

ui/src/hooks/stores.ts

@@ -197,15 +197,23 @@ export const useRTCStore = create<RTCState>(set => ({
   setTerminalChannel: channel => set({ terminalChannel: channel }),
 }));
 
+interface MouseMove {
+  x: number;
+  y: number;
+  buttons: number;
+}
 interface MouseState {
   mouseX: number;
   mouseY: number;
+  mouseMove?: MouseMove;
+  setMouseMove: (move?: MouseMove) => void;
   setMousePosition: (x: number, y: number) => void;
 }
 
 export const useMouseStore = create<MouseState>(set => ({
   mouseX: 0,
   mouseY: 0,
+  setMouseMove: (move?: MouseMove) => set({ mouseMove: move }),
   setMousePosition: (x, y) => set({ mouseX: x, mouseY: y }),
 }));
 

ui/src/routes/devices.$id.settings.mouse.tsx

@@ -16,6 +16,9 @@ export default function SettingsKeyboardMouseRoute() {
 
   const [jiggler, setJiggler] = useState(false);
 
+  const mouseMode = useSettingsStore(state => state.mouseMode);
+  const setMouseMode = useSettingsStore(state => state.setMouseMode);
+
   const [send] = useJsonRpc();
 
   useEffect(() => {
@@ -68,7 +71,7 @@ export default function SettingsKeyboardMouseRoute() {
           <div className="flex flex-col items-center gap-4 md:flex-row">
             <button
               className="group block w-full grow"
-              onClick={() => console.log("Absolute mouse mode clicked")}
+              onClick={() => setMouseMode("absolute")}
             >
               <GridCard>
                 <div className="group flex items-center gap-x-4 px-4 py-3">
@@ -88,6 +91,7 @@ export default function SettingsKeyboardMouseRoute() {
                     </div>
                     <CheckCircleIcon
                       className={cx(
+                        mouseMode == "absolute" ? "" : "hidden",
                         "h-4 w-4 text-blue-700 transition-opacity duration-300 dark:text-blue-500",
                       )}
                     />
@@ -96,15 +100,15 @@ export default function SettingsKeyboardMouseRoute() {
               </GridCard>
             </button>
             <button
-              className="group block w-full grow cursor-not-allowed opacity-50"
+              className="group block w-full grow"
-              disabled
+              onClick={() => setMouseMode("relative")}
             >
               <GridCard>
                 <div className="group flex items-center gap-x-4 px-4 py-3">
                   <img
                     className="w-6 shrink-0 dark:invert"
                     src={PointingFinger}
-                    alt="Finger touching a screen"
+                    alt="Relative mouse mode"
                   />
                   <div className="flex grow items-center justify-between">
                     <div className="text-left">
@@ -117,7 +121,7 @@ export default function SettingsKeyboardMouseRoute() {
                     </div>
                     <CheckCircleIcon
                       className={cx(
-                        "hidden",
+                        mouseMode == "relative" ? "" : "hidden",
                         "h-4 w-4 text-blue-700 transition-opacity duration-300 dark:text-blue-500",
                       )}
                     />

usb.go

@@ -74,7 +74,18 @@ var gadgetConfig = map[string]gadgetConfigItem{
 			"subclass":      "1",
 			"report_length": "6",
 		},
-		reportDesc: CombinedMouseReportDesc,
+		reportDesc: CombinedAbsoluteMouseReportDesc,
+	},
+	// relative mouse HID
+	"relative_mouse": {
+		path:       []string{"functions", "hid.usb2"},
+		configPath: path.Join(configC1Path, "hid.usb2"),
+		attrs: gadgetAttributes{
+			"protocol":      "2",
+			"subclass":      "1",
+			"report_length": "4",
+		},
+		reportDesc: CombinedRelativeMouseReportDesc,
 	},
 	// mass storage
 	"mass_storage_base": {
@@ -311,10 +322,14 @@ func rebindUsb(ignoreUnbindError bool) error {
 	return nil
 }
 
-var keyboardHidFile *os.File
+var (
-var keyboardLock = sync.Mutex{}
+	keyboardHidFile *os.File
-var mouseHidFile *os.File
+	keyboardLock    = sync.Mutex{}
-var mouseLock = sync.Mutex{}
+	mouseHidFile    *os.File
+	mouseLock       = sync.Mutex{}
+	relMouseHidFile *os.File
+	relMouseLock    = sync.Mutex{}
+)
 
 func rpcKeyboardReport(modifier uint8, keys []uint8) error {
 	keyboardLock.Lock()
@@ -406,6 +421,32 @@ func abs(x float64) float64 {
 	return x
 }
 
+func rpcRelMouseReport(mx, my int8, buttons uint8) error {
+	relMouseLock.Lock()
+	defer relMouseLock.Unlock()
+	if relMouseHidFile == nil {
+		var err error
+		relMouseHidFile, err = os.OpenFile("/dev/hidg2", os.O_RDWR, 0666)
+		if err != nil {
+			return fmt.Errorf("failed to open hidg2: %w", err)
+		}
+	}
+	resetUserInputTime()
+	_, err := relMouseHidFile.Write([]byte{
+		buttons,   // Buttons
+		uint8(mx), // X
+		uint8(my), // Y
+		0,         // Wheel
+	})
+	if err != nil {
+		logger.Errorf("failed to write to hidg2: %v", err)
+		relMouseHidFile.Close()
+		relMouseHidFile = nil
+		return err
+	}
+	return nil
+}
+
 var usbState = "unknown"
 
 func rpcGetUSBState() (state string) {
@@ -482,7 +523,7 @@ var KeyboardReportDesc = []byte{
 }
 
 // Combined absolute and relative mouse report descriptor with report ID
-var CombinedMouseReportDesc = []byte{
+var CombinedAbsoluteMouseReportDesc = []byte{
 	0x05, 0x01, // Usage Page (Generic Desktop Ctrls)
 	0x09, 0x02, // Usage (Mouse)
 	0xA1, 0x01, // Collection (Application)
@@ -525,3 +566,39 @@ var CombinedMouseReportDesc = []byte{
 
 	0xC0, // End Collection
 }
+
+var CombinedRelativeMouseReportDesc = []byte{
+	// from: https://github.com/NicoHood/HID/blob/b16be57caef4295c6cd382a7e4c64db5073647f7/src/SingleReport/BootMouse.cpp#L26
+	0x05, 0x01, // USAGE_PAGE (Generic Desktop)	  54
+	0x09, 0x02, // USAGE (Mouse)
+	0xa1, 0x01, // COLLECTION (Application)
+
+	// Pointer and Physical are required by Apple Recovery
+	0x09, 0x01, // USAGE (Pointer)
+	0xa1, 0x00, // COLLECTION (Physical)
+
+	// 8 Buttons
+	0x05, 0x09, // USAGE_PAGE (Button)
+	0x19, 0x01, // USAGE_MINIMUM (Button 1)
+	0x29, 0x08, // USAGE_MAXIMUM (Button 8)
+	0x15, 0x00, // LOGICAL_MINIMUM (0)
+	0x25, 0x01, // LOGICAL_MAXIMUM (1)
+	0x95, 0x08, // REPORT_COUNT (8)
+	0x75, 0x01, // REPORT_SIZE (1)
+	0x81, 0x02, // INPUT (Data,Var,Abs)
+
+	// X, Y, Wheel
+	0x05, 0x01, // USAGE_PAGE (Generic Desktop)
+	0x09, 0x30, // USAGE (X)
+	0x09, 0x31, // USAGE (Y)
+	0x09, 0x38, // USAGE (Wheel)
+	0x15, 0x81, // LOGICAL_MINIMUM (-127)
+	0x25, 0x7f, // LOGICAL_MAXIMUM (127)
+	0x75, 0x08, // REPORT_SIZE (8)
+	0x95, 0x03, // REPORT_COUNT (3)
+	0x81, 0x06, // INPUT (Data,Var,Rel)
+
+	// End
+	0xc0, //       End Collection (Physical)
+	0xc0, //       End Collection
+}