Merge 2ff3dffd61 into c8dd84c6b7

ui/package.json

@@ -4,7 +4,7 @@
   "version": "2025.09.03.2100",
   "type": "module",
   "engines": {
-    "node": "22.15.0"
+    "node": "^22.15.0"
   },
   "scripts": {
     "dev": "./dev_device.sh",
@@ -78,6 +78,7 @@
     "prettier": "^3.6.2",
     "prettier-plugin-tailwindcss": "^0.6.14",
     "tailwindcss": "^4.1.12",
+    "tesseract.js": "6.0.1",
     "typescript": "^5.9.2",
     "vite": "^7.1.4",
     "vite-tsconfig-paths": "^5.1.4"

ui/src/components/ActionBar.tsx

@@ -1,4 +1,4 @@
-import { MdOutlineContentPasteGo } from "react-icons/md";
+import { MdOutlineContentPasteGo, MdOutlineDocumentScanner } from "react-icons/md";
 import { LuCable, LuHardDrive, LuMaximize, LuSettings, LuSignal } from "react-icons/lu";
 import { FaKeyboard } from "react-icons/fa6";
 import { Popover, PopoverButton, PopoverPanel } from "@headlessui/react";
@@ -20,10 +20,14 @@ import MountPopopover from "@/components/popovers/MountPopover";
 import ExtensionPopover from "@/components/popovers/ExtensionPopover";
 import { useDeviceUiNavigation } from "@/hooks/useAppNavigation";
 
+import OCRModal from "./popovers/OCRModal";
+
 export default function Actionbar({
   requestFullscreen,
+  videoElmRef,
 }: {
   requestFullscreen: () => Promise<void>;
+  videoElmRef?: React.RefObject<HTMLVideoElement | null>;
 }) {
   const { navigateTo } = useDeviceUiNavigation();
   const { isVirtualKeyboardEnabled, setVirtualKeyboardEnabled } = useHidStore();
@@ -99,6 +103,36 @@ export default function Actionbar({
               }}
             </PopoverPanel>
           </Popover>
+          <Popover>
+            <PopoverButton as={Fragment}>
+              <Button
+                size="XS"
+                theme="light"
+                text="OCR"
+                LeadingIcon={MdOutlineDocumentScanner}
+                onClick={() => {
+                  setDisableVideoFocusTrap(true);
+                }}
+              />
+            </PopoverButton>
+            <PopoverPanel
+              anchor="bottom start"
+              transition
+              className={cx(
+                "z-10 flex w-[420px] origin-top flex-col overflow-visible!",
+                "flex origin-top flex-col transition duration-300 ease-out data-closed:translate-y-8 data-closed:opacity-0",
+              )}
+            >
+              {({ open }) => {
+                checkIfStateChanged(open);
+                return (
+                  <div className="mx-auto w-full max-w-xl">
+                    <OCRModal videoElmRef={videoElmRef} />
+                  </div>
+                );
+              }}
+            </PopoverPanel>
+          </Popover>
           <div className="relative">
             <Popover>
               <PopoverButton as={Fragment}>

ui/src/components/WebRTCVideo.tsx

@@ -487,7 +487,7 @@ export default function WebRTCVideo() {
             disabled={peerConnection?.connectionState !== "connected"}
             className="contents"
           >
-            <Actionbar requestFullscreen={requestFullscreen} />
+            <Actionbar requestFullscreen={requestFullscreen} videoElmRef={videoElm} />
             <MacroBar />
           </fieldset>
         </div>

ui/src/components/popovers/OCRModal.tsx

@@ -0,0 +1,168 @@
+import { useCallback, useMemo, useState } from "react";
+import { LuCornerDownLeft } from "react-icons/lu";
+import { ExclamationCircleIcon } from "@heroicons/react/16/solid";
+import { type LoggerMessage } from "tesseract.js";
+
+import { Button } from "@components/Button";
+import { GridCard } from "@components/Card";
+import { TextAreaWithLabel } from "@components/TextArea";
+import { SettingsPageHeader } from "@components/SettingsPageheader";
+import { useUiStore } from "@/hooks/stores";
+import useOCR from "@/hooks/useOCR";
+import { cx } from "@/cva.config";
+
+export default function OCRModal({ videoElmRef }: { videoElmRef?: React.RefObject<HTMLVideoElement | null> }) {
+  const { setDisableVideoFocusTrap } = useUiStore();
+  const [ocrStatus, setOcrStatus] = useState<LoggerMessage>();
+  const [ocrError, setOcrError] = useState<string | null>(null);
+  const handleOcrError = useCallback((error: any) => { // eslint-disable-line @typescript-eslint/no-explicit-any
+    if (typeof error === "string") {
+      setOcrError(error);
+    } else {
+      setOcrError(error.message);
+    }
+  }, [setOcrError]);
+  const [ocrText, setOcrText] = useState<string | null>(null);
+
+  const { ocrImage } = useOCR();
+
+  const onConfirmOCR = useCallback(async () => {
+    setDisableVideoFocusTrap(true);
+
+    setOcrText(null);
+    setOcrError(null);
+    setOcrStatus(undefined);
+
+    if (!videoElmRef?.current) {
+      setOcrError("Video element not found");
+      return;
+    }
+
+    setOcrStatus({
+      status: "Capturing image",
+      progress: 0,
+      jobId: "",
+      userJobId: "",
+      workerId: "",
+    });
+
+    // create a canvas from the video element then capture the image from the canvas
+    const video = videoElmRef?.current;
+    const canvas = document.createElement("canvas");
+    canvas.width = video.videoWidth;
+    canvas.height = video.videoHeight;
+
+    const ctx = canvas.getContext("2d");
+    ctx?.drawImage(video, 0, 0, canvas.width, canvas.height);
+
+    const text = await ocrImage(["eng"], canvas, { logger: setOcrStatus, errorHandler: handleOcrError });
+    setOcrText(text);
+
+    setOcrStatus(undefined);
+    setOcrError(null);
+  }, [
+    videoElmRef,
+    setDisableVideoFocusTrap,
+    ocrImage,
+    setOcrStatus,
+    setOcrError,
+    handleOcrError,
+  ]);
+
+  const ocrProgress = useMemo(() => {
+    if (!ocrStatus?.progress) return 0;
+    return Math.round(ocrStatus?.progress * 100);
+  }, [ocrStatus]);
+
+  return (
+    <GridCard>
+      <div className="space-y-4 p-4 py-3">
+        <div className="grid h-full grid-rows-(--grid-headerBody)">
+          <div className="h-full space-y-4">
+            <div className="space-y-4">
+              <SettingsPageHeader
+                title="OCR"
+                description="OCR text from the video"
+              />
+
+              <div
+                className={cx("animate-fadeIn space-y-2 opacity-0", ocrText === null ? "hidden" : "")}
+                style={{
+                  animationDuration: "0.7s",
+                  animationDelay: "0.1s",
+                }}
+              >
+                <div>
+                  <div
+                    className="w-full"
+                    onKeyUp={e => e.stopPropagation()}
+                    onKeyDown={e => e.stopPropagation()}
+                  >
+                    <TextAreaWithLabel
+                      value={ocrText || ""}
+                      label="Text"
+                      rows={4}
+                      readOnly
+                      spellCheck={false}
+                      data-lt="false"
+                      data-gram="false"
+                      className="font-mono"
+                    />
+                  </div>
+                </div>
+              </div>
+
+              {ocrStatus && <div className={cx("animate-fadeIn space-y-2 opacity-0")}
+                style={{
+                  animationDuration: "0.7s",
+                  animationDelay: "0.1s",
+                }}
+              >
+                <div className="space-y-1 flex justify-between">
+                  <p className="text-xs text-slate-600 dark:text-slate-400 capitalize">
+                    {ocrStatus?.status}
+                  </p>
+                  <span className="text-xs text-slate-600 dark:text-slate-400">{ocrProgress}%</span>
+                </div>
+                <div className="h-2.5 w-full overflow-hidden rounded-full bg-slate-300">
+                  <div
+                    style={{ width: ocrProgress + "%" }}
+                    className="h-2.5 bg-blue-700 transition-all duration-1000 ease-in-out"
+                  ></div>
+                </div>
+              </div>}
+
+              {ocrError && (
+                <div className="flex items-center gap-x-2">
+                  <ExclamationCircleIcon className="h-4 w-4 text-red-500 dark:text-red-400" />
+                  <span className="text-xs text-red-500 dark:text-red-400">
+                    {ocrError}
+                  </span>
+                </div>
+              )}
+            </div>
+          </div>
+        </div>
+        <div className="gap-y-4">
+          <p className="text-xs text-slate-600 dark:text-slate-400">
+            Internet connectivity might be required to download Tesseract OCR trained data.
+          </p>
+        </div>
+        <div className="flex animate-fadeIn items-center justify-end gap-x-2 opacity-0"
+          style={{
+            animationDuration: "0.7s",
+            animationDelay: "0.2s",
+          }}
+        >
+          <Button
+            size="SM"
+            theme="primary"
+            text="Start OCR"
+            onClick={onConfirmOCR}
+            LeadingIcon={LuCornerDownLeft}
+          />
+        </div>
+      </div>
+    </GridCard>
+  );
+}

ui/src/hooks/useOCR.ts

@@ -0,0 +1,25 @@
+import { type WorkerOptions } from "tesseract.js";
+
+export type ImageLike = string | HTMLImageElement | HTMLCanvasElement | HTMLVideoElement
+  | CanvasRenderingContext2D | File | Blob | OffscreenCanvas;
+
+// tesseract.js is h
+async function ocrImage(
+  language: string | string[],
+  image: ImageLike,
+  options?: Partial<WorkerOptions>,
+) {
+  const tesseract = await import('tesseract.js')
+  const createWorker = tesseract.createWorker || tesseract.default.createWorker
+
+  const worker = await createWorker(language, undefined, options)
+  const { data: { text } } = await worker.recognize(image)
+  await worker.terminate()
+  return text
+}
+
+export default function useOCR() {
+  return {
+    ocrImage,
+  }
+}

ui/vite.config.ts

@@ -31,20 +31,30 @@ export default defineConfig(({ mode, command }) => {
     esbuild: {
       pure: ["console.debug"],
     },
-    build: { outDir: isCloud ? "dist" : "../static" },
+    build: {
+      outDir: isCloud ? "dist" : "../static",
+      rollupOptions: {
+        external: ["tesseract.js"],
+        output: {
+          paths: {
+            "tesseract.js": "https://cdn.jsdelivr.net/npm/tesseract.js@6.0.1/dist/tesseract.esm.min.js",
+          },
+        },
+      },
+    },
     server: {
       host: "0.0.0.0",
       https: useSSL,
       proxy: JETKVM_PROXY_URL
         ? {
-            "/me": JETKVM_PROXY_URL,
+          "/me": JETKVM_PROXY_URL,
-            "/device": JETKVM_PROXY_URL,
+          "/device": JETKVM_PROXY_URL,
-            "/webrtc": JETKVM_PROXY_URL,
+          "/webrtc": JETKVM_PROXY_URL,
-            "/auth": JETKVM_PROXY_URL,
+          "/auth": JETKVM_PROXY_URL,
-            "/storage": JETKVM_PROXY_URL,
+          "/storage": JETKVM_PROXY_URL,
-            "/cloud": JETKVM_PROXY_URL,
+          "/cloud": JETKVM_PROXY_URL,
-            "/developer": JETKVM_PROXY_URL,
+          "/developer": JETKVM_PROXY_URL,
-          }
+        }
         : undefined,
     },
     base: onDevice && command === "build" ? "/static" : "/",