To save disk space and import time, we only generate/update materiali…

…zed views when necessary

assets/ts/components/ObservationsMap.vue

@@ -63,14 +63,18 @@ export default defineComponent({
     baseLayerName: String,
     dataLayerOpacity: Number,
     areasToShow: {
-      type: Array as PropType<Array<number>>, // Array of area ids
+      type: Array as PropType<Array<Number>>, // Array of area ids
       default: [],
     },
     layerSwitchZoomLevel: {
       // At which zoom level do we switch from the aggregated hexagons to the "individual observation" layer
       type: Number,
       default: 13,
     },
+    zoomLevelMinMaxQuery: {
+      type: Number,
+      required: true,
+    },
   },
   data: function () {
     return {
@@ -242,7 +246,7 @@ export default defineComponent({
         if (this.aggregatedDataLayer) {
           this.map.removeLayer(this.aggregatedDataLayer as VectorTileLayer<Feature>);
         }
-        this.loadOccMinMax(this.initialPosition.initialZoom, this.filters);
+        this.loadOccMinMax(this.zoomLevelMinMaxQuery, this.filters);
         this.aggregatedDataLayer = this.createAggregatedDataLayer();
         this.map.addLayer(this.aggregatedDataLayer as VectorTileLayer<Feature>);
       }

assets/ts/components/OuterObservationsMap.vue

@@ -44,6 +44,7 @@
       :base-layer-name="mapBaseLayer"
       :data-layer-opacity="dataLayerOpacity"
       :areas-to-show="filters.areaIds"
+      :zoom-level-min-max-query="frontendConfig.zoomLevelMinMaxQuery"
     ></Observations-Map>
   </div>
 </template>

assets/ts/interfaces.ts

@@ -101,6 +101,7 @@ export interface FrontEndConfig {
   authenticatedUser: boolean;
   userId?: number; // Only set if authenticatedUser is true
   mainMapConfig: MapConfig;
+  zoomLevelMinMaxQuery: number;
 }
 
 // Keep in sync with Models.Observation.as_dict()

dashboard/management/commands/import_observations.py

@@ -19,7 +19,9 @@
 
 from dashboard.management.commands.helpers import get_dataset_name_from_gbif_api
 from dashboard.models import Species, Observation, DataImport, Dataset
-from dashboard.views.helpers import create_or_refresh_all_materialized_views
+from dashboard.views.helpers import (
+    create_or_refresh_materialized_views,
+)
 
 BULK_CREATE_CHUNK_SIZE = 10000
 
@@ -377,8 +379,10 @@ def handle(self, *args, **options) -> None:
                 "We'll now create or refresh the materialized views. This can take a while."
             )
 
-            # 7. Create or refresh the materialized views (for the map)
-            create_or_refresh_all_materialized_views()
+            # 7. Create or refresh the materialized view (for the map)
+            create_or_refresh_materialized_views(
+                zoom_levels=[settings.ZOOM_LEVEL_FOR_MIN_MAX_QUERY]
+            )
 
             # 8. Remove unused Dataset entries (and edit related alerts)
             for dataset in Dataset.objects.all():

dashboard/templatetags/gbif-alert_extras.py

@@ -108,6 +108,7 @@ def js_config_object(context):
             ).replace("1", "{id}"),
         },
         "mainMapConfig": settings.GBIF_ALERT["MAIN_MAP_CONFIG"],
+        "zoomLevelMinMaxQuery": settings.ZOOM_LEVEL_FOR_MIN_MAX_QUERY,
     }
     if context.request.user.is_authenticated:
         conf["userId"] = context.request.user.pk

dashboard/views/helpers.py

@@ -164,16 +164,18 @@ def model_to_json_list(Model) -> JsonResponse:
 
 def create_or_refresh_all_materialized_views():
     for hex_size in set(settings.ZOOM_TO_HEX_SIZE.values()):  # set to remove duplicates
-        create_or_refresh_materialized_view(hex_size)
+        create_or_refresh_single_materialized_view(hex_size)
 
 
 def create_or_refresh_materialized_views(zoom_levels: list[int]):
     """Create or refresh a bunch of materialized views for a list of zoom levels"""
     for zoom_level in zoom_levels:
-        create_or_refresh_materialized_view(settings.ZOOM_TO_HEX_SIZE[zoom_level])
+        create_or_refresh_single_materialized_view(
+            settings.ZOOM_TO_HEX_SIZE[zoom_level]
+        )
 
 
-def create_or_refresh_materialized_view(hex_size_meters: int):
+def create_or_refresh_single_materialized_view(hex_size_meters: int):
     """Create or refresh a single materialized view for a specific hex size in meters"""
     logger.info(
         f"Creating or refreshing materialized view for hex size {hex_size_meters}"

djangoproject/settings.py

@@ -298,3 +298,13 @@
     key: value * ZOOM_TO_HEX_SIZE_MULTIPLIER
     for key, value in ZOOM_TO_HEX_SIZE_BASELINE.items()
 }
+
+# The zoom level at which the minimum and maximum values are queried
+# That's the only zoom level where this calculation is done
+# By consequence, we generate a materialized view for this zoom level, so
+# the endpoint has good performances. We initially generated those views at each zoom
+# level, but it's lengthy (during import), eats disk space and is unnecessary.
+# The test suite also make sure the endpoint works at different zoom levels, so it
+# will need to generate more materialized views (so the endpoint works), but it's only
+# when running tests and with a tiny amount of data.
+ZOOM_LEVEL_FOR_MIN_MAX_QUERY = 8