diff --git a/backend/app/routes/face_clusters.py b/backend/app/routes/face_clusters.py index 9987b49d8..9805b16ac 100644 --- a/backend/app/routes/face_clusters.py +++ b/backend/app/routes/face_clusters.py @@ -1,19 +1,16 @@ import logging +from binascii import Error as Base64Error +import base64 +from typing import Annotated import uuid import os -from typing import Optional, List, Dict, Any -from pydantic import BaseModel -from app.config.settings import CONFIDENCE_PERCENT, DEFAULT_FACENET_MODEL -from fastapi import APIRouter, HTTPException, status +from fastapi import APIRouter, HTTPException, Query, status from app.database.face_clusters import ( db_get_cluster_by_id, db_update_cluster, db_get_all_clusters_with_face_counts, db_get_images_by_cluster_id, # Add this import ) -from app.database.faces import get_all_face_embeddings -from app.models.FaceDetector import FaceDetector -from app.models.FaceNet import FaceNet from app.schemas.face_clusters import ( RenameClusterRequest, RenameClusterResponse, @@ -26,32 +23,8 @@ GetClusterImagesData, ImageInCluster, ) -from app.schemas.images import AddSingleImageRequest -from app.utils.FaceNet import FaceNet_util_cosine_similarity - - -class BoundingBox(BaseModel): - x: float - y: float - width: float - height: float - - -class ImageData(BaseModel): - id: str - path: str - folder_id: str - thumbnailPath: str - metadata: Dict[str, Any] - isTagged: bool - tags: Optional[List[str]] = None - bboxes: BoundingBox - - -class GetAllImagesResponse(BaseModel): - success: bool - message: str - data: List[ImageData] +from app.schemas.images import FaceSearchRequest, InputType +from app.utils.faceSearch import perform_face_search logger = logging.getLogger(__name__) @@ -236,67 +209,91 @@ def get_cluster_images(cluster_id: str): "/face-search", responses={code: {"model": ErrorResponse} for code in [400, 500]}, ) -def face_tagging(payload: AddSingleImageRequest): - image_path = payload.path - if not os.path.isfile(image_path): - raise HTTPException( - status_code=status.HTTP_400_BAD_REQUEST, - detail=ErrorResponse( - success=False, - error="Invalid file path", - message="The provided path is not a valid file", - ).model_dump(), - ) - - fd = FaceDetector() - fn = FaceNet(DEFAULT_FACENET_MODEL) - try: - matches = [] - image_id = str(uuid.uuid4()) - result = fd.detect_faces(image_id, image_path, forSearch=True) - if not result or result["num_faces"] == 0: - return GetAllImagesResponse( - success=True, - message=f"Successfully retrieved {len(matches)} images", - data=[], +def face_tagging( + payload: FaceSearchRequest, + input_type: Annotated[ + InputType, Query(description="Choose input type: 'path' or 'base64'") + ] = InputType.path, +): + image_path = None + + if input_type == InputType.path: + local_file_path = payload.path + + if not local_file_path: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail=ErrorResponse( + success=False, + error="No Image path provided ", + message="image path is required.", + ).model_dump(), ) + if not os.path.isfile(local_file_path): + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail=ErrorResponse( + success=False, + error="Invalid file path", + message="The provided path is not a valid file", + ).model_dump(), + ) + image_path = payload.path - process_face = result["processed_faces"][0] - new_embedding = fn.get_embedding(process_face) - - images = get_all_face_embeddings() - if len(images) == 0: - return GetAllImagesResponse( - success=True, - message=f"Successfully retrieved {len(matches)} images", - data=[], + elif input_type == InputType.base64: + base64_data = payload.base64_data + if not base64_data: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail=ErrorResponse( + success=False, + error="No base64 data", + message="Base64 image data is required.", + ).model_dump(), ) - else: - for image in images: - max_similarity = 0 - similarity = FaceNet_util_cosine_similarity( - new_embedding, image["embeddings"] - ) - max_similarity = max(max_similarity, similarity) - if max_similarity >= CONFIDENCE_PERCENT: - matches.append( - ImageData( - id=image["id"], - path=image["path"], - folder_id=image["folder_id"], - thumbnailPath=image["thumbnailPath"], - metadata=image["metadata"], - isTagged=image["isTagged"], - tags=image["tags"], - bboxes=image["bbox"], - ) - ) - return GetAllImagesResponse( - success=True, - message=f"Successfully retrieved {len(matches)} images", - data=matches, + MAX_B64_LEN = 14_000_000 # 10MB + if len(base64_data) > MAX_B64_LEN: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail=ErrorResponse( + success=False, + error="Payload too large", + message="Base64 image exceeds maximum allowed size.", + ).model_dump(), + ) + try: + image_bytes = base64.b64decode(base64_data.split(",")[-1]) + except (Base64Error, ValueError): + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail=ErrorResponse( + success=False, + error="Invalid base64 data", + message="The provided base64 image data is malformed or invalid.", + ).model_dump(), ) + + format_match = ( + base64_data.split(";")[0].split("/")[-1] if ";" in base64_data else "jpeg" + ) + extension = ( + format_match + if format_match in ["jpeg", "jpg", "png", "gif", "webp"] + else "jpeg" + ) + image_id = str(uuid.uuid4())[:8] + temp_dir = "temp_uploads" + os.makedirs(temp_dir, exist_ok=True) + local_image_path = os.path.join(temp_dir, f"{image_id}.{extension}") + + with open(local_image_path, "wb") as f: + f.write(image_bytes) + + image_path = local_image_path + + try: + return perform_face_search(image_path) finally: - fd.close() - fn.close() + if input_type == InputType.base64 and image_path and os.path.exists(image_path): + os.remove(image_path) diff --git a/backend/app/schemas/images.py b/backend/app/schemas/images.py index ac1400159..47f939c4f 100644 --- a/backend/app/schemas/images.py +++ b/backend/app/schemas/images.py @@ -1,10 +1,16 @@ +from enum import Enum from pydantic import BaseModel from typing import Optional, List, Union -# Request Model -class AddSingleImageRequest(BaseModel): - path: str +class InputType(str, Enum): + path = "path" + base64 = "base64" + + +class FaceSearchRequest(BaseModel): + path: Optional[str] = None + base64_data: Optional[str] = None class AddMultipleImagesRequest(BaseModel): diff --git a/backend/app/utils/faceSearch.py b/backend/app/utils/faceSearch.py new file mode 100644 index 000000000..385cce908 --- /dev/null +++ b/backend/app/utils/faceSearch.py @@ -0,0 +1,106 @@ +import uuid +from typing import Optional, List, Dict, Any +from pydantic import BaseModel +from app.config.settings import CONFIDENCE_PERCENT, DEFAULT_FACENET_MODEL +from app.database.faces import get_all_face_embeddings +from app.models.FaceDetector import FaceDetector +from app.models.FaceNet import FaceNet +from app.utils.FaceNet import FaceNet_util_cosine_similarity + + +class BoundingBox(BaseModel): + x: float + y: float + width: float + height: float + + +class ImageData(BaseModel): + id: str + path: str + folder_id: str + thumbnailPath: str + metadata: Dict[str, Any] + isTagged: bool + tags: Optional[List[str]] = None + bboxes: BoundingBox + + +class GetAllImagesResponse(BaseModel): + success: bool + message: str + data: List[ImageData] + + +def perform_face_search(image_path: str) -> GetAllImagesResponse: + """ + Performs face detection, embedding generation, and similarity search. + + Args: + image_path (str): Path to the image file to process. + + Returns: + GetAllImagesResponse: Search result containing matched images. + """ + fd = FaceDetector() + fn = FaceNet(DEFAULT_FACENET_MODEL) + + try: + matches = [] + image_id = str(uuid.uuid4()) + + try: + result = fd.detect_faces(image_id, image_path, forSearch=True) + except Exception as e: + return GetAllImagesResponse( + success=False, + message=f"Failed to process image: {str(e)}", + data=[], + ) + if not result or result["num_faces"] == 0: + return GetAllImagesResponse( + success=True, + message="No faces detected in the image.", + data=[], + ) + + process_face = result["processed_faces"][0] + new_embedding = fn.get_embedding(process_face) + + images = get_all_face_embeddings() + if not images: + return GetAllImagesResponse( + success=True, + message="No face embeddings available for comparison.", + data=[], + ) + + for image in images: + similarity = FaceNet_util_cosine_similarity( + new_embedding, image["embeddings"] + ) + if similarity >= CONFIDENCE_PERCENT: + matches.append( + ImageData( + id=image["id"], + path=image["path"], + folder_id=image["folder_id"], + thumbnailPath=image["thumbnailPath"], + metadata=image["metadata"], + isTagged=image["isTagged"], + tags=image["tags"], + bboxes=image["bbox"], + ) + ) + + return GetAllImagesResponse( + success=True, + message=f"Successfully retrieved {len(matches)} matching images.", + data=matches, + ) + + finally: + if "fd" in locals() and fd is not None: + fd.close() + if "fn" in locals() and fn is not None: + fn.close() diff --git a/docs/backend/backend_python/openapi.json b/docs/backend/backend_python/openapi.json index 84caf5d37..b643a08d6 100644 --- a/docs/backend/backend_python/openapi.json +++ b/docs/backend/backend_python/openapi.json @@ -1042,15 +1042,28 @@ ], "summary": "Face Tagging", "operationId": "face_tagging_face_clusters_face_search_post", + "parameters": [ + { + "name": "input_type", + "in": "query", + "required": false, + "schema": { + "$ref": "#/components/schemas/InputType", + "description": "Choose input type: 'path' or 'base64'", + "default": "path" + }, + "description": "Choose input type: 'path' or 'base64'" + } + ], "requestBody": { + "required": true, "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/AddSingleImageRequest" + "$ref": "#/components/schemas/FaceSearchRequest" } } - }, - "required": true + } }, "responses": { "200": { @@ -1062,24 +1075,24 @@ } }, "400": { - "description": "Bad Request", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/app__schemas__face_clusters__ErrorResponse" } } - } + }, + "description": "Bad Request" }, "500": { - "description": "Internal Server Error", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/app__schemas__face_clusters__ErrorResponse" } } - } + }, + "description": "Internal Server Error" }, "422": { "description": "Validation Error", @@ -1288,19 +1301,6 @@ ], "title": "AddFolderResponse" }, - "AddSingleImageRequest": { - "properties": { - "path": { - "type": "string", - "title": "Path" - } - }, - "type": "object", - "required": [ - "path" - ], - "title": "AddSingleImageRequest" - }, "Album": { "properties": { "album_id": { @@ -1513,6 +1513,34 @@ ], "title": "DeleteFoldersResponse" }, + "FaceSearchRequest": { + "properties": { + "path": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Path" + }, + "base64_data": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Base64 Data" + } + }, + "type": "object", + "title": "FaceSearchRequest" + }, "FolderDetails": { "properties": { "folder_id": { @@ -2058,6 +2086,14 @@ "title": "ImageInCluster", "description": "Represents an image that contains faces from a specific cluster." }, + "InputType": { + "type": "string", + "enum": [ + "path", + "base64" + ], + "title": "InputType" + }, "MetadataModel": { "properties": { "name": { diff --git a/frontend/package-lock.json b/frontend/package-lock.json index 0ed88812a..3cc7636ac 100644 --- a/frontend/package-lock.json +++ b/frontend/package-lock.json @@ -46,6 +46,7 @@ "react-image-crop": "^11.0.7", "react-redux": "^9.2.0", "react-router": "^7.6.2", + "react-webcam": "^7.2.0", "react-zoom-pan-pinch": "^3.7.0", "tailwind-merge": "^3.3.0", "tailwindcss": "^4.1.8", @@ -12647,6 +12648,14 @@ } } }, + "node_modules/react-webcam": { + "version": "7.2.0", + "resolved": "https://registry.npmjs.org/react-webcam/-/react-webcam-7.2.0.tgz", + "integrity": "sha512-xkrzYPqa1ag2DP+2Q/kLKBmCIfEx49bVdgCCCcZf88oF+0NPEbkwYk3/s/C7Zy0mhM8k+hpdNkBLzxg8H0aWcg==", + "license": "MIT", + "peerDependencies": { + "react": ">=16.2.0", + "react-dom": ">=16.2.0" "node_modules/react-zoom-pan-pinch": { "version": "3.7.0", "resolved": "https://registry.npmjs.org/react-zoom-pan-pinch/-/react-zoom-pan-pinch-3.7.0.tgz", diff --git a/frontend/package.json b/frontend/package.json index 65f12f633..43fb922c9 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -61,6 +61,7 @@ "react-image-crop": "^11.0.7", "react-redux": "^9.2.0", "react-router": "^7.6.2", + "react-webcam": "^7.2.0", "react-zoom-pan-pinch": "^3.7.0", "tailwind-merge": "^3.3.0", "tailwindcss": "^4.1.8", diff --git a/frontend/src-tauri/Entitlements.plist b/frontend/src-tauri/Entitlements.plist new file mode 100644 index 000000000..13cb114cf --- /dev/null +++ b/frontend/src-tauri/Entitlements.plist @@ -0,0 +1,8 @@ + + + + + com.apple.security.app-sandbox + + + diff --git a/frontend/src-tauri/Info.plist b/frontend/src-tauri/Info.plist new file mode 100644 index 000000000..97feb3a7c --- /dev/null +++ b/frontend/src-tauri/Info.plist @@ -0,0 +1,8 @@ + + + + + NSCameraUsageDescription + Request camera access for WebRTC + + diff --git a/frontend/src-tauri/tauri.conf.json b/frontend/src-tauri/tauri.conf.json index 86bfa0113..ef1f87801 100644 --- a/frontend/src-tauri/tauri.conf.json +++ b/frontend/src-tauri/tauri.conf.json @@ -26,7 +26,8 @@ "../../sync-microservice/dist/": "resources/sync-microservice" }, "macOS": { - "signingIdentity": "-" + "signingIdentity": "-", + "entitlements": "./Entitlements.plist" } }, "productName": "PictoPy", @@ -62,7 +63,7 @@ "scope": ["**"], "enable": true }, - "csp": "default-src 'self' ipc: http://ipc.localhost; img-src 'self' data: asset: http://asset.localhost; connect-src 'self' http://localhost:8000 ws://localhost:8000 http://localhost:8001 ws://localhost:8001" + "csp": "default-src 'self'; img-src 'self' data: asset: http://asset.localhost; media-src 'self' blob: data:; connect-src 'self' ipc: http://ipc.localhost http://localhost:8000 ws://localhost:8000 http://localhost:8001 ws://localhost:8001" } } } diff --git a/frontend/src/api/api-functions/face_clusters.ts b/frontend/src/api/api-functions/face_clusters.ts index 5a079eb9e..14268c16a 100644 --- a/frontend/src/api/api-functions/face_clusters.ts +++ b/frontend/src/api/api-functions/face_clusters.ts @@ -15,6 +15,10 @@ export interface FetchSearchedFacesRequest { path: string; } +export interface FetchSearchedFacesBase64Request { + base64_data: string; +} + export const fetchAllClusters = async (): Promise => { const response = await apiClient.get( faceClustersEndpoints.getAllClusters, @@ -50,3 +54,13 @@ export const fetchSearchedFaces = async ( ); return response.data; }; + +export const fetchSearchedFacesBase64 = async ( + request: FetchSearchedFacesBase64Request, +): Promise => { + const response = await apiClient.post( + faceClustersEndpoints.searchForFacesBase64, + request, + ); + return response.data; +}; diff --git a/frontend/src/api/apiEndpoints.ts b/frontend/src/api/apiEndpoints.ts index 3ff86e5bb..cdab7ccb7 100644 --- a/frontend/src/api/apiEndpoints.ts +++ b/frontend/src/api/apiEndpoints.ts @@ -4,7 +4,8 @@ export const imagesEndpoints = { export const faceClustersEndpoints = { getAllClusters: '/face-clusters/', - searchForFaces: '/face-clusters/face-search', + searchForFaces: '/face-clusters/face-search?input_type=path', + searchForFacesBase64: '/face-clusters/face-search?input_type=base64', renameCluster: (clusterId: string) => `/face-clusters/${clusterId}`, getClusterImages: (clusterId: string) => `/face-clusters/${clusterId}/images`, }; diff --git a/frontend/src/components/Dialog/FaceSearchDialog.tsx b/frontend/src/components/Dialog/FaceSearchDialog.tsx index b61642ab1..3a9d6d667 100644 --- a/frontend/src/components/Dialog/FaceSearchDialog.tsx +++ b/frontend/src/components/Dialog/FaceSearchDialog.tsx @@ -1,5 +1,5 @@ import { useState } from 'react'; -import { Upload, Camera, ScanFace } from 'lucide-react'; +import { Camera, ScanFace } from 'lucide-react'; import { Button } from '@/components/ui/button'; import { Dialog, @@ -19,8 +19,11 @@ import { fetchSearchedFaces } from '@/api/api-functions'; import { showInfoDialog } from '@/features/infoDialogSlice'; import { useNavigate } from 'react-router'; import { ROUTES } from '@/constants/routes'; +import WebcamComponent from '../WebCam/WebCamComponent'; + export function FaceSearchDialog() { const [isDialogOpen, setIsDialogOpen] = useState(false); + const [showCamera, setShowCamera] = useState(false); const { pickSingleFile } = useFile({ title: 'Select File' }); const navigate = useNavigate(); const dispatch = useDispatch(); @@ -56,72 +59,94 @@ export function FaceSearchDialog() { ); }, }); - + const handleWebCam = async () => { + try { + const stream = await navigator.mediaDevices.getUserMedia({ video: true }); + stream.getTracks().forEach((track) => track.stop()); + navigate(`/${ROUTES.HOME}`); + setIsDialogOpen(false); + setShowCamera(true); + } catch (error) { + dispatch( + showInfoDialog({ + title: 'Webcam Not Supported', + message: + 'Webcam is not supported or access was denied on this device.', + variant: 'error', + }), + ); + } + }; const handlePickFile = async () => { navigate(`/${ROUTES.HOME}`); - setIsDialogOpen(false); const filePath = await pickSingleFile(); if (filePath) { + setIsDialogOpen(false); dispatch(startSearch(filePath)); dispatch(showLoader('Searching faces...')); getSearchImages(filePath); } }; return ( - - - - - - - Face Detection Search - - Search for images containing specific faces by uploading a photo or - using your webcam. - - - -
+ <> + + + - -
+ + + Face Detection Search + + Search for images containing specific faces by uploading a photo + or using your webcam. + + + +
+ {/* Upload Button */} + + + {/* Webcam Button */} + +
+ + -

- PictoPy will analyze the face and find matching images in your - gallery. -

- - + setShowCamera(false)} + /> + ); } diff --git a/frontend/src/components/Navigation/Navbar/Navbar.tsx b/frontend/src/components/Navigation/Navbar/Navbar.tsx index 51fcb6a4c..66e7f4348 100644 --- a/frontend/src/components/Navigation/Navbar/Navbar.tsx +++ b/frontend/src/components/Navigation/Navbar/Navbar.tsx @@ -33,7 +33,11 @@ export function Navbar() { {queryImage && (
Query diff --git a/frontend/src/components/WebCam/WebCamComponent.tsx b/frontend/src/components/WebCam/WebCamComponent.tsx new file mode 100644 index 000000000..784eab3bd --- /dev/null +++ b/frontend/src/components/WebCam/WebCamComponent.tsx @@ -0,0 +1,176 @@ +import { useState, useRef, useCallback } from 'react'; +import { useMutationFeedback } from '../../hooks/useMutationFeedback.tsx'; +import Webcam from 'react-webcam'; +import { X, RotateCcw, Search } from 'lucide-react'; +import { Button } from '@/components/ui/button'; +import { + Dialog, + DialogContent, + DialogDescription, + DialogHeader, + DialogTitle, +} from '@/components/ui/dialog'; +import { useDispatch } from 'react-redux'; +import { startSearch, setResults, clearSearch } from '@/features/searchSlice'; +import type { Image } from '@/types/Media'; +import { usePictoMutation } from '@/hooks/useQueryExtension'; +import { fetchSearchedFacesBase64 } from '@/api/api-functions'; +import { showInfoDialog } from '@/features/infoDialogSlice'; + +const videoConstraints = { + facingMode: 'user', +}; + +interface WebcamComponentProps { + isOpen: boolean; + onClose: () => void; +} + +function WebcamComponent({ isOpen, onClose }: WebcamComponentProps) { + const [showCamera, setShowCamera] = useState(true); + const [capturedImageUrl, setCapturedImageUrl] = useState(null); + const webcamRef = useRef(null); + const dispatch = useDispatch(); + + const getSearchImagesBase64 = usePictoMutation({ + mutationFn: async (base64_data: string) => + fetchSearchedFacesBase64({ base64_data }), + }); + + useMutationFeedback(getSearchImagesBase64, { + showLoading: true, + loadingMessage: 'Searching for images...', + successTitle: 'Search Complete', + successMessage: 'Images matching your search have been found.', + errorTitle: 'Search Error', + errorMessage: 'Failed to search images. Please try again.', + onSuccess: () => { + const result = getSearchImagesBase64.data?.data as Image[]; + if (result && result.length > 0) { + dispatch(setResults(result)); + } else { + dispatch( + showInfoDialog({ + title: 'No Match Found', + message: + 'We couldn’t find any matching faces in your gallery for this photo.', + variant: 'info', + }), + ); + dispatch(setResults([])); + dispatch(clearSearch()); + } + getSearchImagesBase64.reset(); + }, + }); + + const capture = useCallback(() => { + if (webcamRef.current) { + const imageSrc = webcamRef.current.getScreenshot(); + setCapturedImageUrl(imageSrc); + setShowCamera(false); + } + }, [webcamRef]); + + const handleRetake = () => { + setCapturedImageUrl(null); + setShowCamera(true); + }; + + const handleSearchCapturedImage = () => { + onClose(); + if (capturedImageUrl) { + dispatch(startSearch(capturedImageUrl)); + getSearchImagesBase64.mutate(capturedImageUrl); + } else { + dispatch( + showInfoDialog({ + title: 'Capture Failed', + message: 'An unexpected error occurred during capture.', + variant: 'error', + }), + ); + handleClose(); + } + }; + + const handleClose = () => { + setShowCamera(true); + setCapturedImageUrl(null); + onClose(); + }; + + return ( + { + if (!open) handleClose(); + }} + > + + + + {capturedImageUrl ? 'Captured Photo' : 'Take a Photo'} + + + {capturedImageUrl + ? 'Review your photo and search for matching faces' + : 'Position your face in the frame and capture'} + + + +
+ {showCamera && !capturedImageUrl ? ( +
+ + +
+ ) : capturedImageUrl ? ( +
+ Captured +
+ + + +
+
+ ) : null} +
+ + + ); +} + +export default WebcamComponent; diff --git a/package.json b/package.json index c298b862e..61a8a383a 100644 --- a/package.json +++ b/package.json @@ -17,4 +17,4 @@ "devDependencies": { "husky": "^9.1.7" } -} \ No newline at end of file +}