Implement deletion policy

sketch_map_tool/__init__.py

@@ -40,6 +40,13 @@
     "worker_send_task_events": True,  # send task-related events to be monitored
     # Avoid errors due to cached db connections going stale through inactivity
     "database_short_lived_sessions": True,
+    # Cleanup map frames and uploaded files stored in the database
+    "beat_schedule": {
+        "cleanup": {
+            "task": "sketch_map_tool.tasks.cleanup_map_frames",
+            "schedule": timedelta(hours=1),
+        },
+    },
 }
 
 

sketch_map_tool/database/client_celery.py

@@ -1,12 +1,19 @@
+import logging
 from io import BytesIO
 from uuid import UUID
 
 import psycopg2
+from psycopg2.errors import UndefinedTable
 from psycopg2.extensions import connection
 
+from sketch_map_tool import __version__
 from sketch_map_tool.config import get_config_value
-from sketch_map_tool.exceptions import CustomFileNotFoundError
+from sketch_map_tool.exceptions import (
+    CustomFileDoesNotExistAnymoreError,
+    CustomFileNotFoundError,
+)
 from sketch_map_tool.helpers import N_
+from sketch_map_tool.models import Bbox, Layer, PaperFormat
 
 db_conn: connection | None = None
 
@@ -25,22 +32,68 @@ def close_connection():
         db_conn.close()
 
 
-def insert_map_frame(file: BytesIO, uuid: UUID):
-    """Insert map frame as blob into the database with the uuid as primary key.
+def insert_map_frame(
+    file: BytesIO,
+    uuid: UUID,
+    bbox: Bbox,
+    format_: PaperFormat,
+    orientation: str,
+    layer: Layer,
+):
+    """Insert map frame alongside map generation parameters into the database.
 
-    The map frame is later on needed for georeferencing the uploaded photo or scan of
-    a sketch map.
+    The UUID is the primary key.
+    The map frame is needed for georeferencing the uploaded files (sketch maps).
     """
     create_query = """
-    CREATE TABLE IF NOT EXISTS map_frame(
-        uuid UUID PRIMARY KEY,
-        file BYTEA
-        )
-        """
-    insert_query = "INSERT INTO map_frame(uuid, file) VALUES (%s, %s)"
+        CREATE TABLE IF NOT EXISTS map_frame(
+            uuid UUID PRIMARY KEY,
+            file BYTEA,
+            bbox VARCHAR,
+            centroid VARCHAR,
+            format VARCHAR,
+            orientation VARCHAR,
+            layer VARCHAR,
+            version VARCHAR,
+            ts TIMESTAMP WITH TIME ZONE DEFAULT now()
+            )
+    """
+    insert_query = """
+        INSERT INTO map_frame (
+            uuid,
+            file,
+            bbox,
+            centroid,
+            format,
+            orientation,
+            layer,
+            version
+            )
+        VALUES (
+            %s,
+            %s,
+            %s,
+            %s,
+            %s,
+            %s,
+            %s,
+            %s)
+    """
     with db_conn.cursor() as curs:
         curs.execute(create_query)
-        curs.execute(insert_query, (str(uuid), file.read()))
+        curs.execute(
+            insert_query,
+            (
+                str(uuid),
+                file.read(),
+                str(bbox),
+                ",".join([str(bbox.centroid[0]), str(bbox.centroid[1])]),
+                str(format_),
+                orientation,
+                layer,
+                __version__,
+            ),
+        )
 
 
 def delete_map_frame(uuid: UUID):
@@ -50,13 +103,67 @@ def delete_map_frame(uuid: UUID):
         curs.execute(query, [str(uuid)])
 
 
+def cleanup_map_frames():
+    """Cleanup map frames which are old and without consent.
+
+    Only set file to null. Keep metadata.
+    """
+    query = """
+    UPDATE
+        map_frame
+    SET
+        file = NULL
+    WHERE
+        ts < NOW() - INTERVAL '6 months'
+        AND NOT EXISTS (
+            SELECT
+                *
+            FROM
+                blob
+            WHERE
+                map_frame.uuid = blob.uuid
+                AND consent = TRUE);
+    """
+    with db_conn.cursor() as curs:
+        try:
+            curs.execute(query)
+        except UndefinedTable:
+            logging.info("Table `map_frame` does not exist yet. Nothing todo.")
+
+
+def cleanup_blobs():
+    """Cleanup uploaded files (sketch maps) without consent.
+
+    Only set file and name to null. Keep metadata.
+    """
+    # TODO: Wait one day until deletion or check celery task status?
+    query = """
+    UPDATE
+        blob
+    SET
+        file = NULL,
+        file_name = NULL
+    WHERE
+        consent = FALSE;
+    """
+    with db_conn.cursor() as curs:
+        try:
+            curs.execute(query)
+        except UndefinedTable:
+            logging.info("Table `blob` does not exist yet. Nothing todo.")
+
+
 def select_file(id_: int) -> bytes:
     """Get an uploaded file stored in the database by ID."""
     query = "SELECT file FROM blob WHERE id = %s"
     with db_conn.cursor() as curs:
         curs.execute(query, [id_])
         raw = curs.fetchone()
         if raw:
+            if raw[0] is None:
+                raise CustomFileDoesNotExistAnymoreError(
+                    N_("The file with the id: {ID} does not exist anymore"), {"ID", id_}
+                )
             return raw[0]
         else:
             raise CustomFileNotFoundError(

sketch_map_tool/database/client_flask.py

@@ -3,16 +3,19 @@
 
 import psycopg2
 from flask import g
+from psycopg2.errors import UndefinedTable
 from psycopg2.extensions import connection
 from werkzeug.utils import secure_filename
 
 from sketch_map_tool.config import get_config_value
 from sketch_map_tool.definitions import REQUEST_TYPES
 from sketch_map_tool.exceptions import (
+    CustomFileDoesNotExistAnymoreError,
     CustomFileNotFoundError,
     UUIDNotFoundError,
 )
-from sketch_map_tool.helpers import N_
+from sketch_map_tool.helpers import N_, to_array
+from sketch_map_tool.upload_processing import read_qr_code
 
 
 def open_connection():
@@ -85,35 +88,62 @@ def set_async_result_ids(request_uuid, map_: dict[REQUEST_TYPES, str]):
     _insert_id_map(request_uuid, map_)
 
 
-def insert_files(files, consent: bool) -> list[int]:
-    """Insert uploaded files as blob into the database and return primary keys"""
+def insert_files(files, consent: bool) -> tuple[list[int], list[str], list[str]]:
+    """Insert uploaded files as blob into the database and return ID, UUID and name.
+
+    UUID is derived from decoding the qr-code.
+    """
     create_query = """
     CREATE TABLE IF NOT EXISTS blob(
         id SERIAL PRIMARY KEY,
+        uuid UUID,
         file_name VARCHAR,
         file BYTEA,
         consent BOOLEAN,
         ts TIMESTAMP WITH TIME ZONE DEFAULT now()
         )
     """
-    insert_query = (
-        "INSERT INTO blob(file_name, file, consent) VALUES (%s, %s, %s) RETURNING id"
-    )
+    insert_query = """
+    INSERT INTO blob (
+        uuid,
+        file_name,
+        file,
+        consent)
+    VALUES (
+        %s,
+        %s,
+        %s,
+        %s)
+    RETURNING
+        id,
+        uuid,
+        file_name
+    """
     db_conn = open_connection()
     with db_conn.cursor() as curs:
         curs.execute(create_query)
-        ids = []
+        file_ids = []
+        uuids = []
+        file_names = []
         for file in files:
+            file_content = file.read()
+            qr_code_content = read_qr_code(to_array(file_content))
             curs.execute(
                 insert_query,
                 (
+                    qr_code_content["uuid"],
                     secure_filename(file.filename),
-                    file.read(),
+                    file_content,
                     consent,
                 ),
             )
-            ids.append(curs.fetchone()[0])
-    return ids
+            result = curs.fetchone()
+            if result is None:
+                raise ValueError()
+            file_ids.append(result[0])
+            uuids.append(result[1])
+            file_names.append(result[2])
+    return file_ids, uuids, file_names
 
 
 def select_file(id_: int) -> bytes:
@@ -153,14 +183,14 @@ def select_file_name(id_: int) -> str:
             )
 
 
-def select_map_frame(uuid: UUID) -> bytes:
+def select_map_frame(uuid: UUID) -> tuple[bytes, str, str]:
     """Select map frame of the associated UUID."""
-    query = "SELECT file FROM map_frame WHERE uuid = %s"
+    query = "SELECT file, bbox, layer FROM map_frame WHERE uuid = %s"
     db_conn = open_connection()
     with db_conn.cursor() as curs:
         try:
             curs.execute(query, [str(uuid)])
-        except psycopg2.errors.UndefinedTable:
+        except UndefinedTable:
             raise CustomFileNotFoundError(
                 N_(
                     "In this Sketch Map Tool instance no sketch map has been "
@@ -170,7 +200,12 @@ def select_map_frame(uuid: UUID) -> bytes:
             )
         raw = curs.fetchone()
         if raw:
-            return raw[0]
+            if raw[0] is None:
+                raise CustomFileDoesNotExistAnymoreError(
+                    N_("The file with the id: {UUID} does not exist anymore"),
+                    {"UUID", uuid},
+                )
+            return raw
         else:
             raise CustomFileNotFoundError(
                 N_(

sketch_map_tool/exceptions.py

@@ -61,3 +61,7 @@ class UUIDNotFoundError(TranslatableError):
 
 class CustomFileNotFoundError(TranslatableError):
     pass
+
+
+class CustomFileDoesNotExistAnymoreError(TranslatableError):
+    pass

sketch_map_tool/models.py

@@ -17,6 +17,16 @@ class Bbox:
     lon_max: float
     lat_max: float
 
+    @property
+    def centroid(self) -> tuple:
+        """The coordinates of the centroid."""
+        lon_centroid = (self.lon_min + self.lon_max) / 2
+        lat_centroid = (self.lat_min + self.lat_max) / 2
+        return (lon_centroid, lat_centroid)
+
+    def __str__(self):
+        return f"{self.lon_min},{self.lat_min},{self.lon_max},{self.lat_max}"
+
 
 @dataclass(frozen=True, kw_only=True)
 class Size:
@@ -85,6 +95,9 @@ class PaperFormat:
     qr_contents_distances_not_rotated: tuple[int, int]
     qr_contents_distance_rotated: int
 
+    def __str__(self):
+        return self.title
+
 
 @dataclass()
 class LiteratureReference: