Merge pull request #457 from geoadmin/develop

New Release v1.26.0 - #minor

Pipfile

@@ -43,7 +43,7 @@ py-multihash = "~=2.0.1"
 django-prometheus = "~=2.3.1"
 django-admin-autocomplete-filter = "~=0.7.1"
 django-pgtrigger = "~=4.11.1"
-logging-utilities = "~=4.4.1"
+logging-utilities = "~=4.5.0"
 django-environ = "*"
 
 [requires]

app/config/logging-cfg-local.yml

@@ -38,6 +38,11 @@ filters:
     (): logging_utilities.filters.TimeAttribute
     isotime: False
     utc_isotime: True
+  add_request:
+    (): logging_utilities.filters.add_thread_context_filter.AddThreadContextFilter
+    contexts:
+    - logger_key: request
+      context_key: request
   django:
     (): logging_utilities.filters.django_request.JsonDjangoRequest
     attr_name: request
@@ -133,6 +138,7 @@ handlers:
       # handler, they will affect every handler
       - type_filter
       - isotime
+      - add_request
       - django
       # This filter only applies to the current handler (It does not modify the record in-place, but
       # instead selects which logs to display)

app/config/settings_prod.py

@@ -81,6 +81,9 @@
 # last, put everything else in between
 MIDDLEWARE = [
     'django_prometheus.middleware.PrometheusBeforeMiddleware',
+    # Middleware to add request to thread variables, this should be far up in the chain so request
+    # information can be added to as many logs as possible.
+    'logging_utilities.django_middlewares.add_request_context.AddToThreadContextMiddleware',
     'middleware.logging.RequestResponseLoggingMiddleware',
     'django.middleware.security.SecurityMiddleware',
     'middleware.cors.CORSHeadersMiddleware',
@@ -192,6 +195,8 @@
     '.yml': 'application/vnd.oai.openapi+yaml;version=3.0'
 }
 
+DELETE_EXPIRED_ITEMS_OLDER_THAN_HOURS = 24
+
 # Media files (i.e. uploaded content=assets in this project)
 UPLOAD_FILE_CHUNK_SIZE = 1024 * 1024  # Size in Bytes
 STORAGES = {

app/stac_api/management/commands/remove_expired_items.py

@@ -0,0 +1,66 @@
+from datetime import timedelta
+
+from django.conf import settings
+from django.core.management.base import CommandParser
+from django.utils import timezone
+
+from stac_api.models import Item
+from stac_api.utils import CommandHandler
+from stac_api.utils import CustomBaseCommand
+
+
+class Handler(CommandHandler):
+
+    def delete(self, instance, object_type):
+        if self.options['dry_run']:
+            self.print_success(f'skipping deletion of {object_type} {instance}')
+        else:
+            instance.delete()
+
+    def run(self):
+        self.print_success('running command to remove expired items')
+        min_age_hours = self.options['min_age_hours']
+        self.print_warning(f"deleting all items expired longer than {min_age_hours} hours")
+        items = Item.objects.filter(
+            properties_expires__lte=timezone.now() - timedelta(hours=min_age_hours)
+        ).all()
+        for item in items:
+            assets = item.assets.all()
+            assets_length = len(assets)
+            self.delete(assets, 'assets')
+            self.delete(item, 'item')
+            if not self.options['dry_run']:
+                self.print_success(
+                    f"deleted item {item.name} and {assets_length}" + " assets belonging to it.",
+                    extra={"item": item.name}
+                )
+
+        if self.options['dry_run']:
+            self.print_success(f'[dry run] would have removed {len(items)} expired items')
+        else:
+            self.print_success(f'successfully removed {len(items)} expired items')
+
+
+class Command(CustomBaseCommand):
+    help = """Remove items and their assets that have expired more than
+    DELETE_EXPIRED_ITEMS_OLDER_THAN_HOURS hours ago.
+    This command is thought to be scheduled as cron job.
+    """
+
+    def add_arguments(self, parser: CommandParser) -> None:
+        super().add_arguments(parser)
+        parser.add_argument(
+            '--dry-run',
+            action='store_true',
+            help='Simulate deleting items, without actually deleting them'
+        )
+        default_min_age = settings.DELETE_EXPIRED_ITEMS_OLDER_THAN_HOURS
+        parser.add_argument(
+            '--min-age-hours',
+            type=int,
+            default=default_min_age,
+            help=f"Minimum hours the item must have been expired for (default {default_min_age})"
+        )
+
+    def handle(self, *args, **options):
+        Handler(self, options).run()

app/stac_api/utils.py

@@ -458,6 +458,8 @@ def geometry_from_bbox(bbox):
     # if large values, SRID is LV95. The default SRID is 4326
     if list_bbox_values[0] > 360:
         bbox_geometry.srid = 2056
+    else:
+        bbox_geometry.srid = 4326
 
     if not bbox_geometry.valid:
         raise ValueError(f'{bbox_geometry.valid_reason} for bbox with {bbox_geometry.wkt}')

app/stac_api/validators.py

@@ -317,7 +317,7 @@ def validate_text_to_geometry(text_geometry):
     # is the input WKT
     try:
         geos_geometry = GEOSGeometry(text_geometry)
-        validate_geometry(geos_geometry)
+        validate_geometry(geos_geometry, apply_transform=True)
         return geos_geometry
     except (ValueError, ValidationError, IndexError, GDALException, GEOSException) as error:
         message = "The text as WKT could not be transformed into a geometry: %(error)s"
@@ -328,7 +328,7 @@ def validate_text_to_geometry(text_geometry):
     try:
         text_geometry = text_geometry.replace('(', '')
         text_geometry = text_geometry.replace(')', '')
-        return validate_geometry(geometry_from_bbox(text_geometry))
+        return validate_geometry(geometry_from_bbox(text_geometry), apply_transform=True)
     except (ValueError, ValidationError, IndexError, GDALException) as error:
         message = "The text as bbox could not be transformed into a geometry: %(error)s"
         params = {'error': error}
@@ -337,7 +337,7 @@ def validate_text_to_geometry(text_geometry):
         raise ValidationError(errors) from None
 
 
-def validate_geometry(geometry):
+def validate_geometry(geometry, apply_transform=False):
     '''
     A validator function that ensures, that only valid
     geometries are stored.
@@ -351,6 +351,7 @@ def validate_geometry(geometry):
         ValidateionError: About that the geometry is not valid
     '''
     geos_geometry = GEOSGeometry(geometry)
+    bbox_ch = GEOSGeometry("POLYGON ((3 44,3 50,14 50,14 44,3 44))")
     if geos_geometry.empty:
         message = "The geometry is empty: %(error)s"
         params = {'error': geos_geometry.wkt}
@@ -361,6 +362,37 @@ def validate_geometry(geometry):
         params = {'error': geos_geometry.valid_reason}
         logger.error(message, params)
         raise ValidationError(_(message), params=params, code='invalid')
+    if not geos_geometry.srid:
+        message = "No projection provided: SRID=%(error)s"
+        params = {'error': geos_geometry.srid}
+        logger.error(message, params)
+        raise ValidationError(_(message), params=params, code='invalid')
+
+    # transform geometry from textfield input if necessary
+    if apply_transform and geos_geometry.srid != 4326:
+        geos_geometry.transform(4326)
+    elif geos_geometry.srid != 4326:
+        message = 'Non permitted Projection. Projection must be wgs84 (SRID=4326) instead of ' \
+            'SRID=%(error)s'
+        params = {'error': geos_geometry.srid}
+        logger.error(message, params)
+        raise ValidationError(_(message), params=params, code='invalid')
+
+    extent = geos_geometry.extent
+    if abs(extent[1]) > 90 or abs(extent[-1]) > 90:
+        message = "Latitude exceeds permitted value: %(error)s"
+        params = {'error': (extent[1], extent[-1])}
+        logger.error(message, params)
+        raise ValidationError(_(message), params=params, code='invalid')
+    if abs(extent[0]) > 180 or abs(extent[-2]) > 180:
+        message = "Longitude exceeds usual value range: %(warning)s"
+        params = {'warning': (extent[0], extent[-2])}
+        logger.warning(message, params)
+
+    if not geos_geometry.within(bbox_ch):
+        message = "Location of asset is (partially) outside of Switzerland"
+        params = {'warning': geos_geometry.wkt}
+        logger.warning(message, params)
     return geometry
 
 

app/stac_api/validators_view.py

@@ -1,6 +1,8 @@
 import logging
 
+from django.db.models import Q
 from django.http import Http404
+from django.utils import timezone
 from django.utils.translation import gettext_lazy as _
 
 from rest_framework import serializers
@@ -28,7 +30,7 @@ def validate_collection(kwargs):
 
 
 def validate_item(kwargs):
-    '''Validate that the item given in request kwargs exists
+    '''Validate that the item given in request kwargs exists and is not expired
 
     Args:
         kwargs: dict
@@ -38,7 +40,9 @@ def validate_item(kwargs):
         Http404: when the item doesn't exists
     '''
     if not Item.objects.filter(
-        name=kwargs['item_name'], collection__name=kwargs['collection_name']
+        Q(properties_expires=None) | Q(properties_expires__gte=timezone.now()),
+        name=kwargs['item_name'],
+        collection__name=kwargs['collection_name']
     ).exists():
         logger.error(
             "The item %s is not part of the collection %s",

app/stac_api/views.py

@@ -8,6 +8,8 @@
 from django.db import transaction
 from django.db.models import Min
 from django.db.models import Prefetch
+from django.db.models import Q
+from django.utils import timezone
 from django.utils.translation import gettext_lazy as _
 
 from rest_framework import generics
@@ -364,6 +366,8 @@ class ItemsList(generics.GenericAPIView):
     def get_queryset(self):
         # filter based on the url
         queryset = Item.objects.filter(
+            # filter expired items
+            Q(properties_expires__gte=timezone.now()) | Q(properties_expires=None),
             collection__name=self.kwargs['collection_name']
         ).prefetch_related(Prefetch('assets', queryset=Asset.objects.order_by('name')), 'links')
         bbox = self.request.query_params.get('bbox', None)
@@ -428,6 +432,8 @@ class ItemDetail(
     def get_queryset(self):
         # filter based on the url
         queryset = Item.objects.filter(
+            # filter expired items
+            Q(properties_expires__gte=timezone.now()) | Q(properties_expires=None),
             collection__name=self.kwargs['collection_name']
         ).prefetch_related(Prefetch('assets', queryset=Asset.objects.order_by('name')), 'links')
 
@@ -536,6 +542,7 @@ class AssetDetail(
     def get_queryset(self):
         # filter based on the url
         return Asset.objects.filter(
+            Q(item__properties_expires=None) | Q(item__properties_expires__gte=timezone.now()),
             item__collection__name=self.kwargs['collection_name'],
             item__name=self.kwargs['item_name']
         )

app/tests/test_admin_page.py

@@ -118,24 +118,27 @@ def _create_collection(
 
         return collection, data, link, provider
 
-    def _create_item(self, collection, with_link=False, extra=None):
+    def _create_item(self, collection, with_link=False, extra=None, data=None):
 
         # Post data to create a new item
         # Note: the *-*_FORMS fields are necessary management form fields
         # originating from the AdminInline and must be present
-        data = {
-            "collection": collection.id,
-            "name": "test_item",
-            "geometry":
-                "SRID=4326;POLYGON((5.96 45.82, 5.96 47.81, 10.49 47.81, 10.49 45.82, 5.96 45.82))",
-            "text_geometry":
-                "SRID=4326;POLYGON((5.96 45.82, 5.96 47.81, 10.49 47.81, 10.49 45.82, 5.96 45.82))",
-            "properties_datetime_0": "2020-12-01",
-            "properties_datetime_1": "13:15:39",
-            "properties_title": "test",
-            "links-TOTAL_FORMS": "0",
-            "links-INITIAL_FORMS": "0",
-        }
+        if not data:
+            data = {
+                "collection": collection.id,
+                "name": "test_item",
+                "geometry":
+                    "SRID=4326;POLYGON((5.96 45.82, 5.96 47.81, 10.49 47.81, 10.49 45.82, "\
+                        "5.96 45.82))",
+                "text_geometry":
+                    "SRID=4326;POLYGON((5.96 45.82, 5.96 47.81, 10.49 47.81, 10.49 45.82, "\
+                        "5.96 45.82))",
+                "properties_datetime_0": "2020-12-01",
+                "properties_datetime_1": "13:15:39",
+                "properties_title": "test",
+                "links-TOTAL_FORMS": "0",
+                "links-INITIAL_FORMS": "0",
+            }
         if with_link:
             data.update({
                 "links-TOTAL_FORMS": "1",
@@ -672,6 +675,26 @@ def test_add_update_item(self):
             msg="Admin page item properties_title update did not work"
         )
 
+    def test_add_item_with_non_standard_projection(self):
+        geometry = "SRID=4326;POLYGON ((6.146799690987942 46.04410910398307, "\
+            "7.438647976247294 46.05153158188484, 7.438632420871813 46.951082771871064, "\
+            "6.125143650928986 46.94353699772178, 6.146799690987942 46.04410910398307))"
+        text_geometry = "SRID=2056;POLYGON ((2500000 1100000, 2600000 1100000, 2600000 1200000, "\
+            "2500000 1200000, 2500000 1100000))"
+        post_data = {
+            "collection": self.collection.id,
+            "name": "test_item",
+            "geometry": geometry,
+            "text_geometry": text_geometry,
+            "properties_datetime_0": "2020-12-01",
+            "properties_datetime_1": "13:15:39",
+            "properties_title": "test",
+            "links-TOTAL_FORMS": "0",
+            "links-INITIAL_FORMS": "0",
+        }
+        #if transformed text_geometry does not match the geometry provided the creation will fail
+        self._create_item(self.collection, data=post_data)[:2]  # pylint: disable=expression-not-assigned
+
     def test_add_update_item_remove_title(self):
         item, data = self._create_item(self.collection)[:2]