Superminaren
/
kvm
mirror of https://github.com/jetkvm/kvm.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

feat: implement relative mouse

This commit is contained in:
Siyuan Miao 2025-03-07 19:22:43 +01:00
parent 4f347dc47f
commit 4997153bee
6 changed files with 136 additions and 20 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
jsonrpc.go
View File

@ -771,6 +771,7 @@ var rpcHandlers = map[string]RPCHandler{
"getCloudState": {Func: rpcGetCloudState},
"keyboardReport": {Func: rpcKeyboardReport, Params: []string{"modifier", "keys"}},
"absMouseReport": {Func: rpcAbsMouseReport, Params: []string{"x", "y", "buttons"}},
"relMouseReport": {Func: rpcRelMouseReport, Params: []string{"mx", "my", "buttons"}},
"wheelReport": {Func: rpcWheelReport, Params: []string{"wheelY"}},
"getVideoState": {Func: rpcGetVideoState},
"getUSBState": {Func: rpcGetUSBState},

14
ui/src/components/InfoBar.tsx
View File

@ -14,6 +14,7 @@ export default function InfoBar() {
const activeModifiers = useHidStore(state => state.activeModifiers);
const mouseX = useMouseStore(state => state.mouseX);
const mouseY = useMouseStore(state => state.mouseY);
const mouseMove = useMouseStore(state => state.mouseMove);
const videoClientSize = useVideoStore(
state => `${Math.round(state.clientWidth)}x${Math.round(state.clientHeight)}`,
@ -62,7 +63,7 @@ export default function InfoBar() {
</div>
) : null}
{settings.debugMode ? (
{(settings.debugMode && settings.mouseMode == "absolute") ? (
<div className="flex w-[118px] items-center gap-x-1">
<span className="text-xs font-semibold">Pointer:</span>
<span className="text-xs">
@ -71,6 +72,17 @@ export default function InfoBar() {
</div>
) : null}
{(settings.debugMode && settings.mouseMode == "relative") ? (
<div className="flex w-[118px] items-center gap-x-1">
<span className="text-xs font-semibold">Last Move:</span>
<span className="text-xs">
{mouseMove ?
`${mouseMove.x},${mouseMove.y} ${mouseMove.buttons ? `(${mouseMove.buttons})` : ""}` :
"N/A"}
</span>
</div>
) : null}
{settings.debugMode && (
<div className="flex w-[156px] items-center gap-x-1">
<span className="text-xs font-semibold">USB State:</span>

30
ui/src/components/WebRTCVideo.tsx
View File

@ -28,6 +28,7 @@ export default function WebRTCVideo() {
const settings = useSettingsStore();
const { sendKeyboardEvent, resetKeyboardState } = useKeyboard();
const setMousePosition = useMouseStore(state => state.setMousePosition);
const setMouseMove = useMouseStore(state => state.setMouseMove);
const {
setClientSize: setVideoClientSize,
setSize: setVideoSize,
@ -92,14 +93,22 @@ export default function WebRTCVideo() {
);
// Mouse-related
const calcDelta = (pos: number) => Math.abs(pos) < 10 ? pos * 2 : pos;
const sendMouseMovement = useCallback(
(x: number, y: number, buttons: number) => {
send("absMouseReport", { x, y, buttons });
// We set that for the debug info bar
setMousePosition(x, y);
if (settings.mouseMode === "relative") {
// if we ignore the event, double-click will not work
// if (x === 0 && y === 0 && buttons === 0) return;
send("relMouseReport", { mx: calcDelta(x), my: calcDelta(y), buttons });
setMouseMove({ x, y, buttons });
} else if (settings.mouseMode === "absolute") {
send("absMouseReport", { x, y, buttons });
// We set that for the debug info bar
setMousePosition(x, y);
}
},
[send, setMousePosition],
[send, setMousePosition, setMouseMove, settings.mouseMode],
);
const mouseMoveHandler = useCallback(
@ -109,6 +118,13 @@ export default function WebRTCVideo() {
const videoElementAspectRatio = videoClientWidth / videoClientHeight;
const videoStreamAspectRatio = videoWidth / videoHeight;
const { buttons } = e;
// Send mouse movement events to the server
if (settings.mouseMode == "relative") {
sendMouseMovement(e.movementX, e.movementY, buttons);
return;
}
// Calculate the effective video display area
let effectiveWidth = videoClientWidth;
let effectiveHeight = videoClientHeight;
@ -137,11 +153,9 @@ export default function WebRTCVideo() {
const x = Math.round(relativeX * 32767);
const y = Math.round(relativeY * 32767);
// Send mouse movement
const { buttons } = e;
sendMouseMovement(x, y, buttons);
},
[sendMouseMovement, videoClientHeight, videoClientWidth, videoWidth, videoHeight],
[sendMouseMovement, videoClientHeight, videoClientWidth, videoWidth, videoHeight, settings.mouseMode],
);
const mouseWheelHandler = useCallback(

8
ui/src/hooks/stores.ts
View File

@ -197,15 +197,23 @@ export const useRTCStore = create<RTCState>(set => ({
setTerminalChannel: channel => set({ terminalChannel: channel }),
}));
interface MouseMove {
x: number;
y: number;
buttons: number;
}
interface MouseState {
mouseX: number;
mouseY: number;
mouseMove?: MouseMove;
setMouseMove: (move?: MouseMove) => void;
setMousePosition: (x: number, y: number) => void;
}
export const useMouseStore = create<MouseState>(set => ({
mouseX: 0,
mouseY: 0,
setMouseMove: (move?: MouseMove) => set({ mouseMove: move }),
setMousePosition: (x, y) => set({ mouseX: x, mouseY: y }),
}));

14
ui/src/routes/devices.$id.settings.mouse.tsx
View File

@ -16,6 +16,9 @@ export default function SettingsKeyboardMouseRoute() {
const [jiggler, setJiggler] = useState(false);
const mouseMode = useSettingsStore(state => state.mouseMode);
const setMouseMode = useSettingsStore(state => state.setMouseMode);
const [send] = useJsonRpc();
useEffect(() => {
@ -68,7 +71,7 @@ export default function SettingsKeyboardMouseRoute() {
<div className="flex flex-col items-center gap-4 md:flex-row">
<button
className="group block w-full grow"
onClick={() => console.log("Absolute mouse mode clicked")}
onClick={() => setMouseMode("absolute")}
>
<GridCard>
<div className="group flex items-center gap-x-4 px-4 py-3">
@ -88,6 +91,7 @@ export default function SettingsKeyboardMouseRoute() {
</div>
<CheckCircleIcon
className={cx(
mouseMode == "absolute" ? "" : "hidden",
"h-4 w-4 text-blue-700 transition-opacity duration-300 dark:text-blue-500",
)}
/>
@ -96,15 +100,15 @@ export default function SettingsKeyboardMouseRoute() {
</GridCard>
</button>
<button
className="group block w-full grow cursor-not-allowed opacity-50"
disabled
className="group block w-full grow"
onClick={() => setMouseMode("relative")}
>
<GridCard>
<div className="group flex items-center gap-x-4 px-4 py-3">
<img
className="w-6 shrink-0 dark:invert"
src={PointingFinger}
alt="Finger touching a screen"
alt="Relative mouse mode"
/>
<div className="flex grow items-center justify-between">
<div className="text-left">
@ -117,7 +121,7 @@ export default function SettingsKeyboardMouseRoute() {
</div>
<CheckCircleIcon
className={cx(
"hidden",
mouseMode == "relative" ? "" : "hidden",
"h-4 w-4 text-blue-700 transition-opacity duration-300 dark:text-blue-500",
)}
/>

89
usb.go
View File

@ -74,7 +74,18 @@ var gadgetConfig = map[string]gadgetConfigItem{
"subclass": "1",
"report_length": "6",
},
reportDesc: CombinedMouseReportDesc,
reportDesc: CombinedAbsoluteMouseReportDesc,
},
// relative mouse HID
"relative_mouse": {
path: []string{"functions", "hid.usb2"},
configPath: path.Join(configC1Path, "hid.usb2"),
attrs: gadgetAttributes{
"protocol": "2",
"subclass": "1",
"report_length": "4",
},
reportDesc: CombinedRelativeMouseReportDesc,
},
// mass storage
"mass_storage_base": {
@ -311,10 +322,14 @@ func rebindUsb(ignoreUnbindError bool) error {
return nil
}
var keyboardHidFile *os.File
var keyboardLock = sync.Mutex{}
var mouseHidFile *os.File
var mouseLock = sync.Mutex{}
var (
keyboardHidFile *os.File
keyboardLock = sync.Mutex{}
mouseHidFile *os.File
mouseLock = sync.Mutex{}
relMouseHidFile *os.File
relMouseLock = sync.Mutex{}
)
func rpcKeyboardReport(modifier uint8, keys []uint8) error {
keyboardLock.Lock()
@ -406,6 +421,32 @@ func abs(x float64) float64 {
return x
}
func rpcRelMouseReport(mx, my int8, buttons uint8) error {
relMouseLock.Lock()
defer relMouseLock.Unlock()
if relMouseHidFile == nil {
var err error
relMouseHidFile, err = os.OpenFile("/dev/hidg2", os.O_RDWR, 0666)
if err != nil {
return fmt.Errorf("failed to open hidg2: %w", err)
}
}
resetUserInputTime()
_, err := relMouseHidFile.Write([]byte{
buttons, // Buttons
uint8(mx), // X
uint8(my), // Y
0, // Wheel
})
if err != nil {
logger.Errorf("failed to write to hidg2: %v", err)
relMouseHidFile.Close()
relMouseHidFile = nil
return err
}
return nil
}
var usbState = "unknown"
func rpcGetUSBState() (state string) {
@ -482,7 +523,7 @@ var KeyboardReportDesc = []byte{
}
// Combined absolute and relative mouse report descriptor with report ID
var CombinedMouseReportDesc = []byte{
var CombinedAbsoluteMouseReportDesc = []byte{
0x05, 0x01, // Usage Page (Generic Desktop Ctrls)
0x09, 0x02, // Usage (Mouse)
0xA1, 0x01, // Collection (Application)
@ -525,3 +566,39 @@ var CombinedMouseReportDesc = []byte{
0xC0, // End Collection
}
var CombinedRelativeMouseReportDesc = []byte{
// from: https://github.com/NicoHood/HID/blob/b16be57caef4295c6cd382a7e4c64db5073647f7/src/SingleReport/BootMouse.cpp#L26
0x05, 0x01, // USAGE_PAGE (Generic Desktop) 54
0x09, 0x02, // USAGE (Mouse)
0xa1, 0x01, // COLLECTION (Application)
// Pointer and Physical are required by Apple Recovery
0x09, 0x01, // USAGE (Pointer)
0xa1, 0x00, // COLLECTION (Physical)
// 8 Buttons
0x05, 0x09, // USAGE_PAGE (Button)
0x19, 0x01, // USAGE_MINIMUM (Button 1)
0x29, 0x08, // USAGE_MAXIMUM (Button 8)
0x15, 0x00, // LOGICAL_MINIMUM (0)
0x25, 0x01, // LOGICAL_MAXIMUM (1)
0x95, 0x08, // REPORT_COUNT (8)
0x75, 0x01, // REPORT_SIZE (1)
0x81, 0x02, // INPUT (Data,Var,Abs)
// X, Y, Wheel
0x05, 0x01, // USAGE_PAGE (Generic Desktop)
0x09, 0x30, // USAGE (X)
0x09, 0x31, // USAGE (Y)
0x09, 0x38, // USAGE (Wheel)
0x15, 0x81, // LOGICAL_MINIMUM (-127)
0x25, 0x7f, // LOGICAL_MAXIMUM (127)
0x75, 0x08, // REPORT_SIZE (8)
0x95, 0x03, // REPORT_COUNT (3)
0x81, 0x06, // INPUT (Data,Var,Rel)
// End
0xc0, // End Collection (Physical)
0xc0, // End Collection
}