From 046353552a67d2b67143c2952f5efe082dd93dd2 Mon Sep 17 00:00:00 2001 From: Akash Shrivastav Date: Sun, 23 Nov 2025 17:42:50 +0530 Subject: [PATCH] Fix: Show images incrementally while adding large folder (Issue #591) --- backend/app/utils/images.py | 102 +++++++++++---------- frontend/src/api/api-functions/folders.ts | 2 +- frontend/src/hooks/useFolder.ts | 59 +++++++++++- frontend/src/hooks/useFolderOperations.tsx | 2 +- 4 files changed, 114 insertions(+), 51 deletions(-) diff --git a/backend/app/utils/images.py b/backend/app/utils/images.py index c3b202205..e54d02dc2 100644 --- a/backend/app/utils/images.py +++ b/backend/app/utils/images.py @@ -38,12 +38,13 @@ def image_util_process_folder_images(folder_data: List[Tuple[str, int, bool]]) - Returns: bool: True if all folders processed successfully, False otherwise """ + BATCH_SIZE = 50 try: # Ensure thumbnail directory exists os.makedirs(THUMBNAIL_IMAGES_PATH, exist_ok=True) - all_image_records = [] all_folder_ids = [] + total_processed = 0 # Process each folder in the provided data for folder_path, folder_id, recursive in folder_data: @@ -60,11 +61,26 @@ def image_util_process_folder_images(folder_data: List[Tuple[str, int, bool]]) - # Step 2: Create folder path mapping for this folder folder_path_to_id = {os.path.abspath(folder_path): folder_id} - # Step 3: Prepare image records for this folder - folder_image_records = image_util_prepare_image_records( - image_files, folder_path_to_id - ) - all_image_records.extend(folder_image_records) + # Step 3: Prepare image records for this folder in batches + batch_records = [] + for i, image_file in enumerate(image_files, 1): + # prepare single image record + image_records = image_util_prepare_single_image_records( + image_file, folder_path_to_id + ) + if image_records: + batch_records.append(image_records) + + # commit batch when it reaches BATCH_SIZE or at the end + if len(batch_records) >= BATCH_SIZE or i == len(image_files): + if batch_records: + db_bulk_insert_images(batch_records) + total_processed += len(batch_records) + logger.info( + f"Committed batch of {len(batch_records)} images from folder {folder_path} " + f"Total processed so far: {total_processed}" + ) + batch_records = [] # reset batch records except Exception as e: logger.error(f"Error processing folder {folder_path}: {e}") @@ -74,13 +90,12 @@ def image_util_process_folder_images(folder_data: List[Tuple[str, int, bool]]) - if all_folder_ids: image_util_remove_obsolete_images(all_folder_ids) - # Step 5: Bulk insert all new records if any exist - if all_image_records: - return db_bulk_insert_images(all_image_records) - - return True # No images to process is not an error + logger.info( + f"Finished processing folders. Total images processed: {total_processed}" + ) + return True except Exception as e: - logger.error(f"Error processing folders: {e}") + logger.error(f"Error in image_util_process_folder_images: {e}") return False @@ -136,48 +151,41 @@ def image_util_classify_and_face_detect_images( face_detector.close() -def image_util_prepare_image_records( - image_files: List[str], folder_path_to_id: Dict[str, int] -) -> List[Dict]: +def image_util_prepare_single_image_records( + image_path: str, folder_path_to_id: Dict[str, int] +) -> Dict | None: """ - Prepare image records with thumbnails for database insertion. + Prepare a single image record with thumbnail for database insertion. Args: - image_files: List of image file paths + image_path: Path to the image file folder_path_to_id: Dictionary mapping folder paths to IDs Returns: - List of image record dictionaries ready for database insertion + Image record dictionary ready for database insertion, or None if preparation fails """ - image_records = [] - for image_path in image_files: - folder_id = image_util_find_folder_id_for_image(image_path, folder_path_to_id) - - if not folder_id: - continue # Skip if no matching folder ID found - - image_id = str(uuid.uuid4()) - thumbnail_name = f"thumbnail_{image_id}.jpg" - thumbnail_path = os.path.abspath( - os.path.join(THUMBNAIL_IMAGES_PATH, thumbnail_name) - ) - - # Generate thumbnail - if image_util_generate_thumbnail(image_path, thumbnail_path): - metadata = image_util_extract_metadata(image_path) - logger.debug(f"Extracted metadata for {image_path}: {metadata}") - image_records.append( - { - "id": image_id, - "path": image_path, - "folder_id": folder_id, - "thumbnailPath": thumbnail_path, - "metadata": json.dumps(metadata), - "isTagged": False, - } - ) - - return image_records + folder_id = image_util_find_folder_id_for_image(image_path, folder_path_to_id) + + if not folder_id: + return None # Skip if no matching folder ID found + image_id = str(uuid.uuid4()) + thumbnail_name = f"thumbnail_{image_id}.jpg" + thumbnail_path = os.path.abspath( + os.path.join(THUMBNAIL_IMAGES_PATH, thumbnail_name) + ) + # Generate thumbnail + if image_util_generate_thumbnail(image_path, thumbnail_path): + metadata = image_util_extract_metadata(image_path) + logger.debug(f"Extracted metadata for {image_path}: {metadata}") + return { + "id": image_id, + "path": image_path, + "folder_id": folder_id, + "thumbnailPath": thumbnail_path, + "metadata": json.dumps(metadata), + "isTagged": False, + } + return None def image_util_get_images_from_folder( diff --git a/frontend/src/api/api-functions/folders.ts b/frontend/src/api/api-functions/folders.ts index d4fc619bc..4f291d9c4 100644 --- a/frontend/src/api/api-functions/folders.ts +++ b/frontend/src/api/api-functions/folders.ts @@ -81,4 +81,4 @@ export const getFoldersTaggingStatus = async (): Promise => { success: res.status === 'success', message: res.message, }; -}; +}; \ No newline at end of file diff --git a/frontend/src/hooks/useFolder.ts b/frontend/src/hooks/useFolder.ts index 4be59fbcd..3e47c0a45 100644 --- a/frontend/src/hooks/useFolder.ts +++ b/frontend/src/hooks/useFolder.ts @@ -2,6 +2,8 @@ import { useCallback, useEffect } from 'react'; import { open } from '@tauri-apps/plugin-dialog'; import { usePictoMutation } from './useQueryExtension'; import { addFolder } from '@/api/api-functions'; +import { useQueryClient } from '@tanstack/react-query'; + interface UseFolderPickerOptions { title?: string; } @@ -9,12 +11,20 @@ interface UseFolderPickerOptions { interface UseFolderPickerReturn { pickSingleFolder: () => Promise; addFolderMutate: (folderPath: string) => void; + isAddingFolder: boolean; } +//polling refs outside component to persist across remounts +const pollIntervalRef: { current: NodeJS.Timeout | null } = { current: null }; +const previousImageCountRef: { current: number } = { current: 0 }; +const stableCountIterationsRef: { current: number } = { current: 0 }; + export const useFolder = ( options: UseFolderPickerOptions = {}, ): UseFolderPickerReturn => { const { title = 'Select folder' } = options; + const queryClient = useQueryClient(); + const { mutate: addFolderMutate, isSuccess: addFolderSuccess, @@ -23,15 +33,59 @@ export const useFolder = ( } = usePictoMutation({ mutationFn: async (folder_path: string) => addFolder({ folder_path }), autoInvalidateTags: ['folders'], + onSuccess: () => { + // console.log('onSuccess called - starting polling setup'); + // Reset Counters + previousImageCountRef.current = 0; + stableCountIterationsRef.current = 0; + + // Immediately invalidate images to show first batch + queryClient.invalidateQueries({ queryKey: ['images'] }); + + // Clear any existing polling interval + if (pollIntervalRef.current) { + clearInterval(pollIntervalRef.current); + // console.log('Cleared existing interval'); + } + // Start polling for incremental updates every 2 seconds; + pollIntervalRef.current = setInterval(() => { + queryClient.invalidateQueries({ queryKey: ['images'] }); + // Get current image count + const imagesData = queryClient.getQueryData(['images']) as any; + const currentImageCount = imagesData?.data?.length || 0; + // Check if count has stabilized + if (currentImageCount === previousImageCountRef.current && currentImageCount > 0) { + stableCountIterationsRef.current += 1; + + if (stableCountIterationsRef.current >= 3) { + if (pollIntervalRef.current) { + clearInterval(pollIntervalRef.current); + pollIntervalRef.current = null; + //console.log('✓ Processing complete! Total images:', currentImageCount); + } + } + } else { + // Count changed, reset stability counter + stableCountIterationsRef.current = 0; + previousImageCountRef.current = currentImageCount; + //console.log('Image count changed to:', currentImageCount); + } + }, 2000); // Poll every 2 seconds + }, }); useEffect(() => { if (addFolderPending) { console.log('Adding folder...'); } else if (addFolderSuccess) { - console.log('Folder added successfully'); + console.log('Folder added successfully - starting to poll for images'); } else if (addFolderError) { console.error('Error adding folder'); + // Clear polling on error + if (pollIntervalRef.current) { + clearInterval(pollIntervalRef.current); + pollIntervalRef.current = null; + } } }, [addFolderSuccess, addFolderError, addFolderPending]); @@ -56,5 +110,6 @@ export const useFolder = ( return { addFolderMutate, pickSingleFolder, + isAddingFolder: addFolderPending, }; -}; +}; \ No newline at end of file diff --git a/frontend/src/hooks/useFolderOperations.tsx b/frontend/src/hooks/useFolderOperations.tsx index 0c0fcc559..015390103 100644 --- a/frontend/src/hooks/useFolderOperations.tsx +++ b/frontend/src/hooks/useFolderOperations.tsx @@ -176,4 +176,4 @@ export const useFolderOperations = () => { }; }; -export default useFolderOperations; +export default useFolderOperations; \ No newline at end of file