Fixed face count inconsistency after folder removal

.github/workflows/build-and-release.yml

@@ -319,6 +319,11 @@ jobs:
           unzip -o *.zip
           rm *.zip
           ls -l
+      - name: Copy utils folder
+        shell: bash
+        run: |
+          cp -r utils backend/dist/utils
+          cp -r utils sync-microservice/dist/utils
       - name: Setup Node
         uses: actions/setup-node@v4
         with:

.pre-commit-config.yaml

@@ -1,12 +1,12 @@
 repos:
-  - repo: https://github.com/charliermarsh/ruff-pre-commit
-    rev: "v0.0.241"
+  - repo: https://github.com/astral-sh/ruff-pre-commit
+    rev: v0.4.10
     hooks:
       - id: ruff
         args: [--fix]
 
   - repo: https://github.com/psf/black
-    rev: "22.3.0"
+    rev: 24.4.2
     hooks:
       - id: black
         language_version: python3

COPYRIGHT.md

@@ -0,0 +1,8 @@
+Copyright © 2025 AOSSIE <br />
+All rights reserved.
+
+All works in this repository may be used according to the conditions 
+stated in the LICENSE.md file available in this repository.
+
+These works are WITHOUT ANY WARRANTY, without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.

app.py

@@ -1,6 +1,7 @@
 # ...existing code...
 from utils.cache import invalidate_cache
 
+
 # Add cache reset option when application starts
 def initialize_app():
     # ...existing code...

backend/app/config/settings.py

@@ -2,7 +2,7 @@
 MODEL_EXPORTS_PATH = "app/models/ONNX_Exports"
 
 # Microservice URLs
-SYNC_MICROSERVICE_URL = "http://localhost:8001/api/v1"
+SYNC_MICROSERVICE_URL = "http://localhost:52124"
 
 CONFIDENCE_PERCENT = 0.6
 # Object Detection Models:

backend/app/database/face_clusters.py

@@ -1,7 +1,11 @@
 import sqlite3
-from typing import Optional, List, Dict, TypedDict, Union
+from typing import Dict, List, Optional, TypedDict, Union
+
 from app.config.settings import DATABASE_PATH
 
+from app.logging.setup_logging import get_logger
+
+logger = get_logger(__name__)
 # Type definitions
 ClusterId = str
 ClusterName = str
@@ -349,3 +353,28 @@ def db_get_images_by_cluster_id(
         return images
     finally:
         conn.close()
+
+
+def db_delete_empty_clusters() -> int:
+    """
+    Delete all clusters that have no faces associated with them.
+    """
+    conn = sqlite3.connect(DATABASE_PATH)
+    cursor = conn.cursor()
+
+    try:
+        cursor.execute(
+            """
+            DELETE FROM face_clusters
+            WHERE cluster_id NOT IN (
+                SELECT DISTINCT cluster_id FROM faces WHERE cluster_id IS NOT NULL
+            )
+            """
+        )
+
+        deleted_count = cursor.rowcount
+        conn.commit()
+        logger.info(f"Deleted {deleted_count} empty clusters.")
+        return deleted_count
+    finally:
+        conn.close()

backend/app/logging/setup_logging.py

@@ -243,11 +243,16 @@ def emit(self, record: logging.LogRecord) -> None:
         # Create a message that includes the original module in the format
         msg = record.getMessage()
 
-        # Find the appropriate logger
-        logger = get_logger(module_name)
-
-        # Log the message with our custom formatting
-        logger.log(record.levelno, f"[uvicorn] {msg}")
+        record.msg = f"[{module_name}] {msg}"
+        record.args = ()
+        # Clear exception / stack info to avoid duplicate traces
+        record.exc_info = None
+        record.stack_info = None
+
+        root_logger = logging.getLogger()
+        for handler in root_logger.handlers:
+            if handler is not self:
+                handler.handle(record)
 
 
 def configure_uvicorn_logging(component_name: str) -> None:

backend/app/routes/folders.py

@@ -1,49 +1,54 @@
-from fastapi import APIRouter, HTTPException, status, Depends, Request
+import os
+from concurrent.futures import ProcessPoolExecutor
 from typing import List, Tuple
+
+from fastapi import APIRouter, Depends, HTTPException, Request, status
+
 from app.database.folders import (
-    db_update_parent_ids_for_subtree,
-    db_folder_exists,
-    db_find_parent_folder_id,
-    db_enable_ai_tagging_batch,
-    db_disable_ai_tagging_batch,
     db_delete_folders_batch,
+    db_disable_ai_tagging_batch,
+    db_enable_ai_tagging_batch,
+    db_find_parent_folder_id,
+    db_folder_exists,
+    db_get_all_folder_details,
     db_get_direct_child_folders,
     db_get_folder_ids_by_path_prefix,
-    db_get_all_folder_details,
+    db_update_parent_ids_for_subtree,
 )
 from app.logging.setup_logging import get_logger
 from app.schemas.folders import (
+    AddFolderData,
     AddFolderRequest,
     AddFolderResponse,
-    AddFolderData,
-    ErrorResponse,
-    UpdateAITaggingRequest,
-    UpdateAITaggingResponse,
-    UpdateAITaggingData,
+    DeleteFoldersData,
     DeleteFoldersRequest,
     DeleteFoldersResponse,
-    DeleteFoldersData,
+    ErrorResponse,
+    FolderDetails,
+    GetAllFoldersData,
+    GetAllFoldersResponse,
+    SyncFolderData,
     SyncFolderRequest,
     SyncFolderResponse,
-    SyncFolderData,
-    GetAllFoldersResponse,
-    GetAllFoldersData,
-    FolderDetails,
+    UpdateAITaggingData,
+    UpdateAITaggingRequest,
+    UpdateAITaggingResponse,
+)
+from app.utils.API import API_util_restart_sync_microservice_watcher
+from app.utils.face_clusters import (
+    cluster_util_delete_empty_clusters,
+    cluster_util_face_clusters_sync,
 )
-import os
 from app.utils.folders import (
     folder_util_add_folder_tree,
     folder_util_add_multiple_folder_trees,
     folder_util_delete_obsolete_folders,
     folder_util_get_filesystem_direct_child_folders,
 )
-from concurrent.futures import ProcessPoolExecutor
 from app.utils.images import (
     image_util_process_folder_images,
     image_util_process_untagged_images,
 )
-from app.utils.face_clusters import cluster_util_face_clusters_sync
-from app.utils.API import API_util_restart_sync_microservice_watcher
 
 # Initialize logger
 logger = get_logger(__name__)
@@ -72,7 +77,8 @@ def post_folder_add_sequence(folder_path: str, folder_id: int):
 
         # Restart sync microservice watcher after processing images
         API_util_restart_sync_microservice_watcher()
-
+        # Delete empty clusters after folder addition
+        cluster_util_delete_empty_clusters()
     except Exception as e:
         logger.error(
             f"Error in post processing after folder {folder_path} was added: {e}"
@@ -121,6 +127,9 @@ def post_sync_folder_sequence(
 
         # Restart sync microservice watcher after processing images
         API_util_restart_sync_microservice_watcher()
+
+        # Delete empty clusters after folder sync
+        cluster_util_delete_empty_clusters()
     except Exception as e:
         logger.error(
             f"Error in post processing after folder {folder_path} was synced: {e}"
@@ -322,7 +331,7 @@ def delete_folders(request: DeleteFoldersRequest):
             raise ValueError("No folder IDs provided")
 
         deleted_count = db_delete_folders_batch(request.folder_ids)
-
+        cluster_util_delete_empty_clusters()
         return DeleteFoldersResponse(
             data=DeleteFoldersData(
                 deleted_count=deleted_count, folder_ids=request.folder_ids

backend/app/utils/face_clusters.py

@@ -862,3 +862,38 @@ def _determine_cluster_name(faces_in_cluster: List[Dict]) -> Optional[str]:
     most_common_name, _ = name_counts.most_common(1)[0]
 
     return most_common_name
+
+
+def cluster_util_delete_empty_clusters() -> int:
+    """
+    Delete all clusters that have no faces associated with them.
+
+    Returns:
+        int: Number of clusters deleted
+    """
+    import sqlite3
+    from app.config.settings import DATABASE_PATH
+
+    conn = sqlite3.connect(DATABASE_PATH)
+    cursor = conn.cursor()
+
+    try:
+        cursor.execute(
+            """
+            DELETE FROM face_clusters
+            WHERE cluster_id NOT IN (
+                SELECT DISTINCT cluster_id FROM faces WHERE cluster_id IS NOT NULL
+            )
+            """
+        )
+
+        deleted_count = cursor.rowcount
+        conn.commit()
+        logger.info(f"Deleted {deleted_count} empty clusters.")
+        return deleted_count
+    except Exception as e:
+        logger.error(f"Error deleting empty clusters: {e}")
+        conn.rollback()
+        return 0
+    finally:
+        conn.close()