feat: Implement infinite scroll pagination for images

.gitignore

@@ -30,4 +30,8 @@ videos_cache.txt
 images_cache.txt
 videos_cache.txt
 venv/
+.venv/
 frontend/dist
+
+*.onnx
+backend/models/

backend/app/config/pagination.py

@@ -0,0 +1,9 @@
+"""Pagination configuration constants for the PictoPy backend."""
+
+MAX_PAGE_SIZE = 1000
+DEFAULT_PAGE_SIZE = 50
+MAX_OFFSET_VALUE = 1_000_000
+MIN_PAGE_SIZE = 1
+
+DEFAULT_RETRY_COUNT = 2
+DEFAULT_RETRY_DELAY_MS = 500

backend/app/database/images.py

@@ -120,23 +120,61 @@ def db_bulk_insert_images(image_records: List[ImageRecord]) -> bool:
         conn.close()
 
 
-def db_get_all_images(tagged: Union[bool, None] = None) -> List[dict]:
+def db_get_all_images(
+    tagged: Union[bool, None] = None,
+    limit: Union[int, None] = None,
+    offset: Union[int, None] = None,
+) -> dict:
     """
-    Get all images from the database with their tags.
+    Retrieve images with tags and optional pagination.
+
+    Uses a CTE (Common Table Expression) to efficiently paginate before joining,
+    preventing unnecessary data loading and SQL injection risks.
 
     Args:
-        tagged: Optional filter for tagged status. If None, returns all images.
-                If True, returns only tagged images. If False, returns only untagged images.
+        tagged: Filter by tagged status (True/False/None for all)
+        limit: Maximum images to return
+        offset: Number of images to skip
 
     Returns:
-        List of dictionaries containing all image data including tags
+        {"images": list of image dicts, "total": total count}
     """
     conn = _connect()
     cursor = conn.cursor()
 
     try:
-        # Build the query with optional WHERE clause
-        query = """
+        count_query = "SELECT COUNT(*) FROM images"
+        count_params: List[Union[bool, int]] = []
+
+        if tagged is not None:
+            count_query += " WHERE isTagged = ?"
+            count_params.append(tagged)
+
+        cursor.execute(count_query, count_params)
+        total_count = cursor.fetchone()[0]
+
+        if total_count == 0:
+            return {"images": [], "total": 0}
+
+        query_parts = ["WITH paginated_images AS ("]
+        query_parts.append("  SELECT id FROM images")
+        query_params: List[Union[bool, int]] = []
+
+        if tagged is not None:
+            query_parts.append("  WHERE isTagged = ?")
+            query_params.append(tagged)
+
+        query_parts.append("  ORDER BY path")
+
+        if limit is not None and limit > 0:
+            query_parts.append("  LIMIT ?")
+            query_params.append(limit)
+            if offset is not None and offset > 0:
+                query_parts.append("  OFFSET ?")
+                query_params.append(offset)
+
+        query_parts.append(")")
+        query_parts.append("""
             SELECT 
                 i.id, 
                 i.path, 
@@ -147,69 +185,57 @@ def db_get_all_images(tagged: Union[bool, None] = None) -> List[dict]:
                 i.isFavourite,
                 m.name as tag_name
             FROM images i
+            INNER JOIN paginated_images pi ON i.id = pi.id
             LEFT JOIN image_classes ic ON i.id = ic.image_id
             LEFT JOIN mappings m ON ic.class_id = m.class_id
-        """
-
-        params = []
-        if tagged is not None:
-            query += " WHERE i.isTagged = ?"
-            params.append(tagged)
-
-        query += " ORDER BY i.path, m.name"
-
-        cursor.execute(query, params)
-
+            ORDER BY i.path, m.name
+        """)
+
+        cursor.execute("\n".join(query_parts), query_params)
         results = cursor.fetchall()
 
-        # Group results by image ID
-        images_dict = {}
-        for (
-            image_id,
-            path,
-            folder_id,
-            thumbnail_path,
-            metadata,
-            is_tagged,
-            is_favourite,
-            tag_name,
-        ) in results:
+        images_dict: dict[str, dict] = {}
+        for row in results:
+            (
+                image_id,
+                path,
+                folder_id,
+                thumbnail_path,
+                metadata,
+                is_tagged,
+                is_favourite,
+                tag_name,
+            ) = row
+
             if image_id not in images_dict:
-                # Safely parse metadata JSON -> dict
                 from app.utils.images import image_util_parse_metadata
 
-                metadata_dict = image_util_parse_metadata(metadata)
-
                 images_dict[image_id] = {
                     "id": image_id,
                     "path": path,
                     "folder_id": str(folder_id),
                     "thumbnailPath": thumbnail_path,
-                    "metadata": metadata_dict,
+                    "metadata": image_util_parse_metadata(metadata),
                     "isTagged": bool(is_tagged),
                     "isFavourite": bool(is_favourite),
                     "tags": [],
                 }
 
-            # Add tag if it exists (avoid duplicates)
             if tag_name and tag_name not in images_dict[image_id]["tags"]:
                 images_dict[image_id]["tags"].append(tag_name)
 
-        # Convert to list and set tags to None if empty
         images = []
         for image_data in images_dict.values():
             if not image_data["tags"]:
                 image_data["tags"] = None
             images.append(image_data)
 
-        # Sort by path
         images.sort(key=lambda x: x["path"])
-
-        return images
+        return {"images": images, "total": total_count}
 
     except Exception as e:
-        logger.error(f"Error getting all images: {e}")
-        return []
+        logger.error(f"Error getting images: {e}", exc_info=True)
+        return {"images": [], "total": 0}
     finally:
         conn.close()
 

backend/app/routes/images.py

@@ -2,7 +2,11 @@
 from typing import List, Optional
 from app.database.images import db_get_all_images
 from app.schemas.images import ErrorResponse
-from app.utils.images import image_util_parse_metadata
+from app.config.pagination import (
+    MAX_PAGE_SIZE,
+    MAX_OFFSET_VALUE,
+    MIN_PAGE_SIZE,
+)
 from pydantic import BaseModel
 from app.database.images import db_toggle_image_favourite_status
 from app.logging.setup_logging import get_logger
@@ -41,29 +45,56 @@ class GetAllImagesResponse(BaseModel):
     success: bool
     message: str
     data: List[ImageData]
+    total: Optional[int] = None
+    limit: Optional[int] = None
+    offset: Optional[int] = None
 
 
 @router.get(
     "/",
     response_model=GetAllImagesResponse,
-    responses={500: {"model": ErrorResponse}},
+    responses={
+        400: {"model": ErrorResponse},
+        500: {"model": ErrorResponse},
+    },
 )
 def get_all_images(
-    tagged: Optional[bool] = Query(None, description="Filter images by tagged status")
+    tagged: Optional[bool] = Query(None, description="Filter images by tagged status"),
+    limit: Optional[int] = Query(
+        None,
+        description="Number of images per page",
+        ge=MIN_PAGE_SIZE,
+        le=MAX_PAGE_SIZE,
+    ),
+    offset: Optional[int] = Query(None, description="Number of images to skip", ge=0),
 ):
-    """Get all images from the database."""
+    """
+    Retrieve images with optional filtering and pagination.
+
+    Returns paginated results with total count metadata.
+    """
     try:
-        # Get all images with tags from database (single query with optional filter)
-        images = db_get_all_images(tagged=tagged)
+        if offset is not None and offset > MAX_OFFSET_VALUE:
+            raise HTTPException(
+                status_code=status.HTTP_400_BAD_REQUEST,
+                detail=ErrorResponse(
+                    success=False,
+                    error="Invalid offset",
+                    message=f"Offset exceeds maximum allowed value ({MAX_OFFSET_VALUE})",
+                ).model_dump(),
+            )
+
+        result = db_get_all_images(tagged=tagged, limit=limit, offset=offset)
+        images = result["images"]
+        total_count = result["total"]
 
-        # Convert to response format
         image_data = [
             ImageData(
                 id=image["id"],
                 path=image["path"],
                 folder_id=image["folder_id"],
                 thumbnailPath=image["thumbnailPath"],
-                metadata=image_util_parse_metadata(image["metadata"]),
+                metadata=image["metadata"],
                 isTagged=image["isTagged"],
                 isFavourite=image.get("isFavourite", False),
                 tags=image["tags"],
@@ -73,10 +104,15 @@ def get_all_images(
 
         return GetAllImagesResponse(
             success=True,
-            message=f"Successfully retrieved {len(image_data)} images",
+            message=f"Successfully retrieved {len(image_data)} of {total_count} images",
             data=image_data,
+            total=total_count,
+            limit=limit,
+            offset=offset,
         )
 
+    except HTTPException:
+        raise
     except Exception as e:
         raise HTTPException(
             status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,

backend/main.py

@@ -44,14 +44,17 @@
 async def lifespan(app: FastAPI):
     # Create tables and initialize systems
     generate_openapi_json()
-    db_create_folders_table()
-    db_create_images_table()
-    db_create_YOLO_classes_table()
-    db_create_clusters_table()  # Create clusters table first since faces references it
-    db_create_faces_table()
-    db_create_albums_table()
-    db_create_album_images_table()
-    db_create_metadata_table()
+
+    # Create tables in the correct order (respecting foreign key dependencies)
+    db_create_YOLO_classes_table()  # No dependencies
+    db_create_clusters_table()  # No dependencies
+    db_create_folders_table()  # No dependencies
+    db_create_albums_table()  # No dependencies
+    db_create_images_table()  # Depends on folders and mappings (YOLO classes)
+    db_create_faces_table()  # Depends on clusters and images
+    db_create_album_images_table()  # Depends on albums and images
+    db_create_metadata_table()  # Depends on images
+
     microservice_util_start_sync_service()
     # Create ProcessPoolExecutor and attach it to app.state
     app.state.executor = ProcessPoolExecutor(max_workers=1)

backend/tests/conftest.py

@@ -22,13 +22,13 @@ def setup_before_all_tests():
     print("Creating database tables...")
     try:
         db_create_YOLO_classes_table()
-        db_create_clusters_table()  # Create clusters table first since faces references it
-        db_create_faces_table()
+        db_create_clusters_table()
         db_create_folders_table()
         db_create_albums_table()
-        db_create_album_images_table()
-        db_create_images_table()
-        db_create_metadata_table()
+        db_create_images_table()  # Must come before faces, album_images, and metadata
+        db_create_faces_table()  # Depends on clusters and images
+        db_create_album_images_table()  # Depends on albums and images
+        db_create_metadata_table()  # Depends on images
         print("All database tables created successfully")
     except Exception as e:
         print(f"Error creating database tables: {e}")