Merge pull request #486 from geoadmin/develop

New Release v1.29.0 - #minor

Pipfile

@@ -45,6 +45,7 @@ django-admin-autocomplete-filter = "~=0.7.1"
 django-pgtrigger = "~=4.11.1"
 logging-utilities = "~=4.5.0"
 django-environ = "*"
+language-tags = "*"
 
 [requires]
 python_version = "3.12"

app/config/settings_dev.py

@@ -79,9 +79,14 @@
 # By default sessions expire after two weeks.
 # Sessions are only useful for user tracking in the admin UI. For security
 # reason we should expire these sessions as soon as possible. Given the use
-# case, it seems reasonable to log out users after 8h of inactivity or whenever
-# they restart their browser.
+# case, it seems reasonable to log users out after 8h of inactivity.
 SESSION_COOKIE_AGE = 60 * 60 * 8
-SESSION_EXPIRE_AT_BROWSER_CLOSE = True
+# Setting sessions to expire when closing the browser means the age expiration
+# is ignored. We could do age-based expiration on the server side but that's
+# more work than it's worth so we just persist the session across browser
+# restarts and expire it only based on age. Furthermore, close-based expiration
+# is sometimes ignored depending on the browser and how it's configured. So
+# let's just turn that off (as it is by default).
+SESSION_EXPIRE_AT_BROWSER_CLOSE = False
 SESSION_COOKIE_SAMESITE = "Strict"
 SESSION_COOKIE_SECURE = True

app/middleware/apigw.py

@@ -2,4 +2,31 @@
 
 
 class ApiGatewayMiddleware(PersistentRemoteUserMiddleware):
+    """Persist user authentication based on the API Gateway headers."""
     header = "HTTP_GEOADMIN_USERNAME"
+
+    def process_request(self, request):
+        """Before processing the request, drop the Geoadmin-Username header if it's invalid.
+
+        API Gateway always sends the Geoadmin-Username header regardless of
+        whether it was able to authenticate the user. If it could not
+        authenticate the user, the value of the header as seen on the wire is a
+        single whitespace. An hexdump looks like this:
+    
+            47 65 6f 61 64 6d 69 6e 5f 75 73 65 72 6e 61 6d 65 3a 20 0d 0a
+            Geoadmin-Username:...
+    
+        This doesn't seem possible to reproduce with curl. It is possible to
+        reproduce with wget. It is unclear whether that technically counts as an
+        empty value or a whitespace. It is also possible that AWS change their
+        implementation later to send something slightly different. Regardless,
+        we already have a separate signal to tell us whether that value is
+        valid: Geoadmin-Authenticated. So we only consider Geoadmin-Username if
+        Geoadmin-Authenticated is set to "true".
+
+        Based on discussion in https://code.djangoproject.com/ticket/35971
+        """
+        apigw_auth = request.META.get("HTTP_GEOADMIN_AUTHENTICATED", "false").lower() == "true"
+        if not apigw_auth and self.header in request.META:
+            del request.META[self.header]
+        return super().process_request(request)

app/stac_api/admin.py

@@ -66,7 +66,18 @@ class ProviderInline(admin.TabularInline):
     }
 
 
-class CollectionLinkInline(admin.TabularInline):
+class LinkInline(admin.TabularInline):
+
+    def formfield_for_dbfield(self, db_field, request, **kwargs):
+        # make the hreflang field a bit shorter so that the inline
+        # will not be rendered too wide
+        if db_field.attname == 'hreflang':
+            attrs = {'size': 10}
+            kwargs['widget'] = forms.TextInput(attrs=attrs)
+        return super().formfield_for_dbfield(db_field, request, **kwargs)
+
+
+class CollectionLinkInline(LinkInline):
     model = CollectionLink
     extra = 0
 
@@ -154,7 +165,7 @@ def get_readonly_fields(self, request, obj=None):
         return self.readonly_fields
 
 
-class ItemLinkInline(admin.TabularInline):
+class ItemLinkInline(LinkInline):
     model = ItemLink
     extra = 0
 
@@ -233,6 +244,18 @@ class Media:
                 )
             }
         ),
+        (
+            'Forecast',
+            {
+                'fields': (
+                    'forecast_reference_datetime',
+                    'forecast_horizon',
+                    'forecast_duration',
+                    'forecast_param',
+                    'forecast_mode',
+                )
+            }
+        ),
     )
 
     list_display = ['name', 'collection', 'collection_published']

app/stac_api/management/commands/update_asset_file_size.py

@@ -4,6 +4,7 @@
 
 from django.conf import settings
 from django.core.management.base import BaseCommand
+from django.core.management.base import CommandParser
 
 from stac_api.models import Asset
 from stac_api.models import CollectionAsset
@@ -14,48 +15,83 @@
 
 logger = logging.getLogger(__name__)
 
+# increase the log level so boto3 doesn't spam the output
+logging.getLogger('boto3').setLevel(logging.WARNING)
+logging.getLogger('botocore').setLevel(logging.WARNING)
+
 
 class Handler(CommandHandler):
 
     def update(self):
-        self.print_success('running command to update file size')
+        self.print_success('Running command to update file size')
+
+        asset_limit = self.options['count']
+
+        asset_qs = Asset.objects.filter(file_size=0, is_external=False)
+        total_asset_count = asset_qs.count()
+        assets = asset_qs.all()[:asset_limit]
+
+        self.print_success(f'Update file size for {len(assets)} assets out of {total_asset_count}')
 
-        self.print_success('update file size for assets')
-        assets = Asset.objects.filter(file_size=0).all()
         for asset in assets:
             selected_bucket = select_s3_bucket(asset.item.collection.name)
             s3 = get_s3_client(selected_bucket)
             bucket = settings.AWS_SETTINGS[selected_bucket.name]['S3_BUCKET_NAME']
             key = SharedAssetUploadBase.get_path(None, asset)
+
             try:
                 file_size = s3.head_object(Bucket=bucket, Key=key)['ContentLength']
                 asset.file_size = file_size
                 asset.save()
+                print(".", end="", flush=True)
             except ClientError:
-                logger.error('file size could not be read from s3 bucket for asset %s', key)
+                logger.error(
+                    'file size could not be read from s3 bucket [%s] for asset %s', bucket, key
+                )
+        print()
+
+        collection_asset_qs = CollectionAsset.objects.filter(file_size=0)
+        total_asset_count = collection_asset_qs.count()
+        collection_assets = collection_asset_qs.all()[:asset_limit]
+
+        self.print_success(
+            f"Update file size for {len(collection_assets)} collection assets out of "
+            "{total_asset_count}"
+        )
 
-        self.print_success('update file size for collection assets')
-        collection_assets = CollectionAsset.objects.filter(file_size=0).all()
         for collection_asset in collection_assets:
             selected_bucket = select_s3_bucket(collection_asset.collection.name)
             s3 = get_s3_client(selected_bucket)
             bucket = settings.AWS_SETTINGS[selected_bucket.name]['S3_BUCKET_NAME']
             key = SharedAssetUploadBase.get_path(None, collection_asset)
+
             try:
                 file_size = s3.head_object(Bucket=bucket, Key=key)['ContentLength']
                 collection_asset.file_size = file_size
                 collection_asset.save()
+                print(".", end="", flush=True)
             except ClientError:
                 logger.error(
-                    'file size could not be read from s3 bucket for collection asset %s', key
+                    'file size could not be read from s3 bucket [%s] for collection asset %s'
                 )
 
+        print()
         self.print_success('Update completed')
 
 
 class Command(BaseCommand):
     help = """Requests the file size of every asset / collection asset from the s3 bucket and
         updates the value in the database"""
 
+    def add_arguments(self, parser: CommandParser) -> None:
+        super().add_arguments(parser)
+        parser.add_argument(
+            '-c',
+            '--count',
+            help="The amount of assets to process at once",
+            required=True,
+            type=int
+        )
+
     def handle(self, *args, **options):
         Handler(self, options).update()

app/stac_api/migrations/0057_item_forecast_duration_item_forecast_horizon_and_more.py

@@ -0,0 +1,66 @@
+# Generated by Django 5.0.8 on 2024-11-28 13:20
+
+from django.db import migrations
+from django.db import models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('stac_api', '0056_alter_collection_total_data_size_and_more'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='item',
+            name='forecast_duration',
+            field=models.DurationField(
+                blank=True,
+                help_text=
+                "If the forecast is not only for a specific instance in time but instead is for a certain period, you can specify the length here. Formatted as ISO 8601 duration, e.g. 'PT3H' for a 3-hour accumulation. If not given, assumes that the forecast is for an instance in time as if this was set to PT0S (0 seconds).",
+                null=True
+            ),
+        ),
+        migrations.AddField(
+            model_name='item',
+            name='forecast_horizon',
+            field=models.DurationField(
+                blank=True,
+                help_text=
+                "The time between the reference datetime and the forecast datetime.Formatted as ISO 8601 duration, e.g. 'PT6H' for a 6-hour forecast.",
+                null=True
+            ),
+        ),
+        migrations.AddField(
+            model_name='item',
+            name='forecast_mode',
+            field=models.CharField(
+                blank=True,
+                choices=[('ctrl', 'Control run'), ('perturb', 'Perturbed run')],
+                default=None,
+                help_text=
+                'Denotes whether the data corresponds to the control run or perturbed runs.',
+                null=True
+            ),
+        ),
+        migrations.AddField(
+            model_name='item',
+            name='forecast_param',
+            field=models.CharField(
+                blank=True,
+                help_text=
+                'Name of the model parameter that corresponds to the data, e.g. `T` (temperature), `P` (pressure), `U`/`V`/`W` (windspeed in three directions).',
+                null=True
+            ),
+        ),
+        migrations.AddField(
+            model_name='item',
+            name='forecast_reference_datetime',
+            field=models.DateTimeField(
+                blank=True,
+                help_text=
+                "The reference datetime: i.e. predictions for times after this point occur in the future. Predictions prior to this time represent 'hindcasts', predicting states that have already occurred. This must be in UTC. It is formatted like '2022-08-12T00:00:00Z'.",
+                null=True
+            ),
+        ),
+    ]

app/stac_api/migrations/0058_collectionlink_hreflang_itemlink_hreflang_and_more.py

@@ -0,0 +1,29 @@
+# Generated by Django 5.0.8 on 2024-12-05 10:23
+
+from django.db import migrations
+from django.db import models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('stac_api', '0057_item_forecast_duration_item_forecast_horizon_and_more'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='collectionlink',
+            name='hreflang',
+            field=models.CharField(blank=True, max_length=32, null=True),
+        ),
+        migrations.AddField(
+            model_name='itemlink',
+            name='hreflang',
+            field=models.CharField(blank=True, max_length=32, null=True),
+        ),
+        migrations.AddField(
+            model_name='landingpagelink',
+            name='hreflang',
+            field=models.CharField(blank=True, max_length=32, null=True),
+        ),
+    ]