diff --git a/frontend/package-lock.json b/frontend/package-lock.json index e1e1ddd5f..7437786f7 100644 --- a/frontend/package-lock.json +++ b/frontend/package-lock.json @@ -51,6 +51,7 @@ "react-zoom-pan-pinch": "^3.7.0", "tailwind-merge": "^3.3.0", "tailwindcss": "^4.1.8", + "tesseract.js": "^5.1.0", "ts-node": "^10.9.2", "uuid": "^11.1.0", "vite-plugin-environment": "^1.1.3" @@ -6597,6 +6598,12 @@ "baseline-browser-mapping": "dist/cli.js" } }, + "node_modules/bmp-js": { + "version": "0.1.0", + "resolved": "https://registry.npmjs.org/bmp-js/-/bmp-js-0.1.0.tgz", + "integrity": "sha512-vHdS19CnY3hwiNdkaqk93DvjVLfbEcI8mys4UjuWrlX1haDmroo8o4xCzh4wD6DGV6HxRCyauwhHRqMTfERtjw==", + "license": "MIT" + }, "node_modules/brace-expansion": { "version": "1.1.12", "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.12.tgz", @@ -9044,6 +9051,12 @@ "node": ">=0.10.0" } }, + "node_modules/idb-keyval": { + "version": "6.2.2", + "resolved": "https://registry.npmjs.org/idb-keyval/-/idb-keyval-6.2.2.tgz", + "integrity": "sha512-yjD9nARJ/jb1g+CvD0tlhUHOrJ9Sy0P8T9MF3YaLlHnSRpwPfpTX0XIvpmw3gAJUmEu3FiICLBDPXVwyEvrleg==", + "license": "Apache-2.0" + }, "node_modules/identity-obj-proxy": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/identity-obj-proxy/-/identity-obj-proxy-3.0.0.tgz", @@ -9337,6 +9350,12 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/is-electron": { + "version": "2.2.2", + "resolved": "https://registry.npmjs.org/is-electron/-/is-electron-2.2.2.tgz", + "integrity": "sha512-FO/Rhvz5tuw4MCWkpMzHFKWD2LsfHzIb7i6MdPYZ/KW7AlxawyLkqdy+jPZP1WubqEADE3O4FUENlJHDfQASRg==", + "license": "MIT" + }, "node_modules/is-extglob": { "version": "2.1.1", "resolved": "https://registry.npmjs.org/is-extglob/-/is-extglob-2.1.1.tgz", @@ -9598,6 +9617,12 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/is-url": { + "version": "1.2.4", + "resolved": "https://registry.npmjs.org/is-url/-/is-url-1.2.4.tgz", + "integrity": "sha512-ITvGim8FhRiYe4IQ5uHSkj7pVaPDrCTkNd3yq3cV7iZAcJdHTUMPMEHcqSOy9xZ9qFenQCvi+2wjH9a1nXqHww==", + "license": "MIT" + }, "node_modules/is-weakmap": { "version": "2.0.2", "resolved": "https://registry.npmjs.org/is-weakmap/-/is-weakmap-2.0.2.tgz", @@ -11461,6 +11486,48 @@ "dev": true, "license": "MIT" }, + "node_modules/node-fetch": { + "version": "2.7.0", + "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.7.0.tgz", + "integrity": "sha512-c4FRfUm/dbcWZ7U+1Wq0AwCyFL+3nt2bEw05wfxSz+DWpWsitgmSgYmy2dQdWyKC1694ELPqMs/YzUSNozLt8A==", + "license": "MIT", + "dependencies": { + "whatwg-url": "^5.0.0" + }, + "engines": { + "node": "4.x || >=6.0.0" + }, + "peerDependencies": { + "encoding": "^0.1.0" + }, + "peerDependenciesMeta": { + "encoding": { + "optional": true + } + } + }, + "node_modules/node-fetch/node_modules/tr46": { + "version": "0.0.3", + "resolved": "https://registry.npmjs.org/tr46/-/tr46-0.0.3.tgz", + "integrity": "sha512-N3WMsuqV66lT30CrXNbEjx4GEwlow3v6rr4mCcv6prnfwhS01rkgyFdjPNBYd9br7LpXV1+Emh01fHnq2Gdgrw==", + "license": "MIT" + }, + "node_modules/node-fetch/node_modules/webidl-conversions": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-3.0.1.tgz", + "integrity": "sha512-2JAn3z8AR6rjK8Sm8orRC0h/bcl/DqL7tRPdGZ4I1CjdF+EaMLmYxBHyXuKL849eucPFhvBoxMsflfOb8kxaeQ==", + "license": "BSD-2-Clause" + }, + "node_modules/node-fetch/node_modules/whatwg-url": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/whatwg-url/-/whatwg-url-5.0.0.tgz", + "integrity": "sha512-saE57nupxk6v3HY35+jzBwYa0rKSy0XR8JSxZPwgLr7ys0IBzhGviA1/TUGJLmSVqs8pb9AnvICXEuOHLprYTw==", + "license": "MIT", + "dependencies": { + "tr46": "~0.0.3", + "webidl-conversions": "^3.0.0" + } + }, "node_modules/node-int64": { "version": "0.4.0", "resolved": "https://registry.npmjs.org/node-int64/-/node-int64-0.4.0.tgz", @@ -11670,6 +11737,15 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/opencollective-postinstall": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/opencollective-postinstall/-/opencollective-postinstall-2.0.3.tgz", + "integrity": "sha512-8AV/sCtuzUeTo8gQK5qDZzARrulB3egtLzFgteqB2tcT4Mw7B8Kt7JcDHmltjz6FOAHsvTevk70gZEbhM4ZS9Q==", + "license": "MIT", + "bin": { + "opencollective-postinstall": "index.js" + } + }, "node_modules/optionator": { "version": "0.9.4", "resolved": "https://registry.npmjs.org/optionator/-/optionator-0.9.4.tgz", @@ -12511,6 +12587,12 @@ "node": ">=4" } }, + "node_modules/regenerator-runtime": { + "version": "0.13.11", + "resolved": "https://registry.npmjs.org/regenerator-runtime/-/regenerator-runtime-0.13.11.tgz", + "integrity": "sha512-kY1AZVr2Ra+t+piVaJ4gxaFaReZVH40AKNo7UCX6W+dEwBo/2oZJzqfuN1qLq1oL45o56cPaTXELwrTh8Fpggg==", + "license": "MIT" + }, "node_modules/regexp.prototype.flags": { "version": "1.5.4", "resolved": "https://registry.npmjs.org/regexp.prototype.flags/-/regexp.prototype.flags-1.5.4.tgz", @@ -13360,6 +13442,31 @@ "url": "https://opencollective.com/webpack" } }, + "node_modules/tesseract.js": { + "version": "5.1.0", + "resolved": "https://registry.npmjs.org/tesseract.js/-/tesseract.js-5.1.0.tgz", + "integrity": "sha512-2fH9pqWdS2C6ue/3OoGg91Wtv7Rt/1atYu/g0Q1SGFrowEW/kIBkG361hLienHsWe4KWEjxOJBrCQYpIBWG6WA==", + "hasInstallScript": true, + "license": "Apache-2.0", + "dependencies": { + "bmp-js": "^0.1.0", + "idb-keyval": "^6.2.0", + "is-electron": "^2.2.2", + "is-url": "^1.2.4", + "node-fetch": "^2.6.9", + "opencollective-postinstall": "^2.0.3", + "regenerator-runtime": "^0.13.3", + "tesseract.js-core": "^5.1.0", + "wasm-feature-detect": "^1.2.11", + "zlibjs": "^0.3.1" + } + }, + "node_modules/tesseract.js-core": { + "version": "5.1.1", + "resolved": "https://registry.npmjs.org/tesseract.js-core/-/tesseract.js-core-5.1.1.tgz", + "integrity": "sha512-KX3bYSU5iGcO1XJa+QGPbi+Zjo2qq6eBhNjSGR5E5q0JtzkoipJKOUQD7ph8kFyteCEfEQ0maWLu8MCXtvX5uQ==", + "license": "Apache-2.0" + }, "node_modules/test-exclude": { "version": "6.0.0", "resolved": "https://registry.npmjs.org/test-exclude/-/test-exclude-6.0.0.tgz", @@ -14212,6 +14319,12 @@ "makeerror": "1.0.12" } }, + "node_modules/wasm-feature-detect": { + "version": "1.8.0", + "resolved": "https://registry.npmjs.org/wasm-feature-detect/-/wasm-feature-detect-1.8.0.tgz", + "integrity": "sha512-zksaLKM2fVlnB5jQQDqKXXwYHLQUVH9es+5TOOHwGOVJOCeRBCiPjwSg+3tN2AdTCzjgli4jijCH290kXb/zWQ==", + "license": "Apache-2.0" + }, "node_modules/webidl-conversions": { "version": "7.0.0", "resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-7.0.0.tgz", @@ -14540,6 +14653,15 @@ "funding": { "url": "https://github.com/sponsors/sindresorhus" } + }, + "node_modules/zlibjs": { + "version": "0.3.1", + "resolved": "https://registry.npmjs.org/zlibjs/-/zlibjs-0.3.1.tgz", + "integrity": "sha512-+J9RrgTKOmlxFSDHo0pI1xM6BLVUv+o0ZT9ANtCxGkjIVCCUdx9alUF8Gm+dGLKbkkkidWIHFDZHDMpfITt4+w==", + "license": "MIT", + "engines": { + "node": "*" + } } } } diff --git a/frontend/package.json b/frontend/package.json index 0a53f1b8d..89d1f524a 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -66,6 +66,7 @@ "react-zoom-pan-pinch": "^3.7.0", "tailwind-merge": "^3.3.0", "tailwindcss": "^4.1.8", + "tesseract.js": "^5.1.0", "ts-node": "^10.9.2", "uuid": "^11.1.0", "vite-plugin-environment": "^1.1.3" diff --git a/frontend/src/components/Media/ImageViewer.tsx b/frontend/src/components/Media/ImageViewer.tsx index 704b65eda..2872a7433 100644 --- a/frontend/src/components/Media/ImageViewer.tsx +++ b/frontend/src/components/Media/ImageViewer.tsx @@ -1,6 +1,10 @@ -import React, { useRef, useImperativeHandle, forwardRef } from 'react'; +import { useRef, useImperativeHandle, forwardRef, useState, useEffect } from 'react'; import { TransformWrapper, TransformComponent } from 'react-zoom-pan-pinch'; import { convertFileSrc } from '@tauri-apps/api/core'; +import { ocrService } from '../../services/OCRService'; +import { TextOverlay } from './TextOverlay'; +import { Page } from 'tesseract.js'; +import { Loader2 } from 'lucide-react'; interface ImageViewerProps { imagePath: string; @@ -18,6 +22,11 @@ export interface ImageViewerRef { export const ImageViewer = forwardRef( ({ imagePath, alt, rotation, resetSignal }, ref) => { const transformRef = useRef(null); + const imgRef = useRef(null); + const [isOCRActive, setIsOCRActive] = useState(false); + const [ocrData, setOcrData] = useState(null); + const [isOCRLoading, setIsOCRLoading] = useState(false); + const [imageScale, setImageScale] = useState(1); // Expose zoom functions to parent useImperativeHandle(ref, () => ({ @@ -27,53 +36,151 @@ export const ImageViewer = forwardRef( })); // Reset on signal change - React.useEffect(() => { + useEffect(() => { transformRef.current?.resetTransform(); - }, [resetSignal]); + // Reset OCR when image changes + setIsOCRActive(false); + setOcrData(null); + }, [resetSignal, imagePath]); + + // Update scale when image loads or resizes + useEffect(() => { + const updateScale = () => { + if (imgRef.current) { + const { width, naturalWidth } = imgRef.current; + if (naturalWidth > 0) { + setImageScale(width / naturalWidth); + } + } + }; + + const img = imgRef.current; + if (img) { + // Initial update + if (img.complete) updateScale(); + + // Listen for load + img.addEventListener('load', updateScale); + + // Listen for resize + const resizeObserver = new ResizeObserver(updateScale); + resizeObserver.observe(img); + + return () => { + img.removeEventListener('load', updateScale); + resizeObserver.disconnect(); + }; + } + }, [imagePath]); // Re-run when image path changes + + // Handle Ctrl+T to toggle OCR + useEffect(() => { + const handleKeyDown = async (e: KeyboardEvent) => { + if (e.ctrlKey && e.key.toLowerCase() === 't') { + e.preventDefault(); + + if (isOCRActive) { + // Deactivate + setIsOCRActive(false); + } else { + // Activate + setIsOCRActive(true); + if (!ocrData && !isOCRLoading) { + setIsOCRLoading(true); + try { + const src = convertFileSrc(imagePath); + const data = await ocrService.recognize(src); + setOcrData(data); + } catch (error) { + console.error('Failed to perform OCR', error); + setIsOCRActive(false); // Revert if failed + } finally { + setIsOCRLoading(false); + } + } + } + } + }; + + window.addEventListener('keydown', handleKeyDown); + return () => window.removeEventListener('keydown', handleKeyDown); + }, [imagePath, isOCRActive, ocrData, isOCRLoading]); return ( - - + {isOCRLoading && ( +
+ + Processing Text... +
+ )} + + {isOCRActive && !isOCRLoading && ocrData && ( +
+ + + + + Text Selection Active +
+ )} + + - {alt} { - const img = e.target as HTMLImageElement; - img.onerror = null; - img.src = '/placeholder.svg'; + - - + > +
+ {alt} { + const img = e.target as HTMLImageElement; + img.onerror = null; + img.src = '/placeholder.svg'; + }} + style={{ + maxWidth: '100%', + maxHeight: '100%', + objectFit: 'contain', + zIndex: 50, + }} + /> + {isOCRActive && ocrData && ( + + )} +
+ + + ); }, ); diff --git a/frontend/src/components/Media/TextOverlay.tsx b/frontend/src/components/Media/TextOverlay.tsx new file mode 100644 index 000000000..7ad271f84 --- /dev/null +++ b/frontend/src/components/Media/TextOverlay.tsx @@ -0,0 +1,129 @@ +import React, { useEffect, useState } from 'react'; +import { Page } from 'tesseract.js'; +import { Check } from 'lucide-react'; + +interface TextOverlayProps { + ocrData: Page | null; + scale?: number; +} + +export const TextOverlay: React.FC = ({ ocrData, scale = 1 }) => { + const [showCopyFeedback, setShowCopyFeedback] = useState(false); + + useEffect(() => { + const handleKeyDown = async (e: KeyboardEvent) => { + if (e.ctrlKey && e.key.toLowerCase() === 'c') { + const selection = window.getSelection(); + const text = selection?.toString().trim(); + + if (text && text.length > 0) { + // We manually write to clipboard to ensure it works even with transparent text + try { + await navigator.clipboard.writeText(text); + setShowCopyFeedback(true); + setTimeout(() => setShowCopyFeedback(false), 2000); + } catch (err) { + console.error('Failed to copy text:', err); + } + } + } + }; + + window.addEventListener('keydown', handleKeyDown); + return () => window.removeEventListener('keydown', handleKeyDown); + }, []); + + if (!ocrData) return null; + + // Use lines instead of words for better sentence selection + const lines = (ocrData as any).lines || []; + + return ( + <> + {showCopyFeedback && ( +
+
+ +
+ Text copied to clipboard +
+ )} + +
e.stopPropagation()} + style={{ + position: 'absolute', + top: 0, + left: 0, + width: '100%', + height: '100%', + pointerEvents: 'auto', + zIndex: 60, + userSelect: 'text', + WebkitUserSelect: 'text', + opacity: 0, + animation: 'fadeIn 0.3s ease-out forwards', + }} + > + {lines.map((line: any, index: number) => { + const { bbox, text } = line; + const width = (bbox.x1 - bbox.x0) * scale; + const height = (bbox.y1 - bbox.y0) * scale; + const left = bbox.x0 * scale; + const top = bbox.y0 * scale; + + return ( + + {text} + + ); + })} + +
+ + ); +}; diff --git a/frontend/src/services/OCRService.ts b/frontend/src/services/OCRService.ts new file mode 100644 index 000000000..55b3f3bac --- /dev/null +++ b/frontend/src/services/OCRService.ts @@ -0,0 +1,54 @@ +import { createWorker, Worker, PSM } from 'tesseract.js'; + +class OCRService { + private worker: Worker | null = null; + private workerPromise: Promise | null = null; + + private async getWorker(): Promise { + if (this.worker) return this.worker; + + if (!this.workerPromise) { + this.workerPromise = (async () => { + try { + // Initialize with default OEM + const worker = await createWorker('eng', undefined); + + // Set Page Segmentation Mode to AUTO to ensure we get blocks/words + await worker.setParameters({ + tessedit_pageseg_mode: PSM.AUTO, + }); + + this.worker = worker; + return worker; + } catch (error) { + console.error('Failed to initialize Tesseract worker:', error); + this.workerPromise = null; + throw error; + } + })(); + } + + return this.workerPromise; + } + + async recognize(imagePath: string) { + try { + const worker = await this.getWorker(); + const result = await worker.recognize(imagePath); + return result.data; + } catch (error) { + console.error('OCR Error:', error); + throw error; + } + } + + async terminate() { + if (this.worker) { + await this.worker.terminate(); + this.worker = null; + this.workerPromise = null; + } + } +} + +export const ocrService = new OCRService();