Merge branch 'dev-tabular' into issue_#118_upgrade_to_GeoNode_4.3.1

geonode/base/models.py

@@ -1844,7 +1844,7 @@ def get_legend_url(self, style_name=None):
     def get_ows_url(self):
         """Return URL for OGC WMS server None if it does not exist."""
         try:
-            ows_link = self.link_set.get(name="OGC:WMS")
+            ows_link = self.link_set.get(name="OGC:WMS") or self.link_set.get(name="OGC:WFS")
         except Link.DoesNotExist:
             return None
         else:

geonode/br/management/commands/create_tile_layers.py

@@ -116,7 +116,7 @@ def handle(self, **options):
             curl -v -u admin:geoserver -XGET \
                 "http://<host>:<port>/geoserver/gwc/rest/layers/geonode:tasmania_roads.xml"
             """
-            layers = Dataset.objects.all()
+            layers = Dataset.objects.filter(subtype__not="tabular")
             tot = len(layers)
             logger.info(f"Total layers in GeoNode: {tot}")
             i = 0

geonode/geoserver/helpers.py

@@ -103,6 +103,7 @@
     ("application/wfs-collection-1.1", "vector"),
     ("application/zip", "vector"),
     ("text/csv", "vector"),
+    ("text/csv", "tabular"),
 ]
 
 DEFAULT_STYLE_NAME = ["generic", "line", "point", "polygon", "raster"]
@@ -1014,7 +1015,7 @@ def set_attributes_from_geoserver(layer, overwrite=False):
             tb = traceback.format_exc()
             logger.debug(tb)
             attribute_map = []
-    elif layer.subtype in {"vector", "tileStore", "remote", "wmsStore", "vector_time"}:
+    elif layer.subtype in {"vector", "tileStore", "remote", "wmsStore", "vector_time", "tabular"}:
         typename = layer.alternate if layer.alternate else layer.typename
         dft_url_path = re.sub(r"\/wms\/?$", "/", server_url)
         dft_query = urlencode(
@@ -1126,14 +1127,18 @@ def clean_styles(layer, gs_catalog: Catalog):
         style = None
         gs_catalog.reset()
         gs_dataset = get_dataset(layer, gs_catalog)
-        if gs_dataset is not None:
+        if gs_dataset is None:
+            if gs_dataset.default_style is None:
+                # ignore dataset without style
+                pass
+
             logger.debug(f'clean_styles: Retrieving style "{gs_dataset.default_style.name}" for cleanup')
             style = gs_catalog.get_style(name=gs_dataset.default_style.name, workspace=None, recursive=True)
-        if style:
-            gs_catalog.delete(style, purge=True, recurse=False)
-            logger.debug(f"clean_styles: Style removed: {gs_dataset.default_style.name}")
-        else:
-            logger.debug(f"clean_styles: Style does not exist: {gs_dataset.default_style.name}")
+            if style:
+                gs_catalog.delete(style, purge=True, recurse=False)
+                logger.debug(f"clean_styles: Style removed: {gs_dataset.default_style.name}")
+            else:
+                logger.debug(f"clean_styles: Style does not exist: {gs_dataset.default_style.name}")
     except Exception as e:
         logger.warning(f"Could not clean style for layer {layer.name}", exc_info=e)
         logger.debug(f"Could not clean style for layer {layer.name} - STACK INFO", stack_info=True)
@@ -1213,7 +1218,7 @@ def set_styles(layer, gs_catalog: Catalog):
                 layer.default_style,
             ]
         ):
-            if style:
+            if style and layer.subtype != "tabular":
                 style_name = os.path.basename(urlparse(style.sld_url).path).split(".")[0]
                 legend_url = get_legend_url(layer, style_name)
                 if dataset_legends.filter(resource=layer.resourcebase_ptr, name="Legend", url=legend_url).count() < 2:
@@ -1566,7 +1571,7 @@ def fetch_gs_resource(instance, values, tries):
         else:
             values = {}
 
-        _subtype = gs_resource.store.resource_type
+        _subtype = "tabular" if instance.subtype == "tabular" else gs_resource.store.resource_type 
         if (
             getattr(gs_resource, "metadata", None)
             and gs_resource.metadata.get("time", False)
@@ -1774,7 +1779,7 @@ def style_update(request, url, workspace=None):
 
         # Invalidate GeoWebCache so it doesn't retain old style in tiles
         try:
-            if dataset_name:
+            if dataset_name and layer.subtype != "tabular":
                 _stylefilterparams_geowebcache_dataset(dataset_name)
                 _invalidate_geowebcache_dataset(dataset_name)
         except Exception:
@@ -2181,12 +2186,13 @@ def sync_instance_with_geoserver(instance_id, *args, **kwargs):
                     except Exception as e:
                         logger.warning(e)
 
-                    # Invalidate GeoWebCache for the updated resource
-                    try:
-                        _stylefilterparams_geowebcache_dataset(instance.alternate)
-                        _invalidate_geowebcache_dataset(instance.alternate)
-                    except Exception as e:
-                        logger.warning(e)
+                    if instance.subtype != "tabular":
+                        # Invalidate GeoWebCache for the updated resource
+                        try:
+                            _stylefilterparams_geowebcache_dataset(instance.alternate)
+                            _invalidate_geowebcache_dataset(instance.alternate)
+                        except Exception as e:
+                            logger.warning(e)
 
         # Refreshing dataset links
         logger.debug(f"... Creating Default Resource Links for Dataset {instance.title}")

geonode/geoserver/management/commands/set_default_gridsets.py

@@ -43,7 +43,7 @@ def handle(self, **options):
             curl -v -u admin:geoserver -XGET \
                 "http://<host>:<port>/geoserver/gwc/rest/layers/geonode:tasmania_roads.xml"
             """
-            layers = Dataset.objects.all()
+            layers = Dataset.objects.filter(subtype__not="tabular")
             print(f"Total layers to be updated: {layers.count()}")
             for layer in layers:
                 print(f"Processing layer: {layer.typename}")

geonode/geoserver/manager.py

@@ -532,7 +532,7 @@ def set_permissions(
                                     create_geofence_rules(_resource, DOWNLOAD_PERMISSIONS, None, None, batch)
                                     exist_geolimits = exist_geolimits or has_geolimits(_resource, None, None)
 
-                            if exist_geolimits is not None:
+                            if exist_geolimits is not None and _resource.subtype != "tabular":
                                 filters, formats = _get_gwc_filters_and_formats(exist_geolimits)
                                 try:
                                     _dataset_workspace = get_dataset_workspace(_resource)

geonode/layers/download_handler.py

@@ -75,7 +75,7 @@ def download_url(self):
         resource = self.get_resource()
         if not resource:
             return None
-        if resource.subtype not in ["vector", "raster", "vector_time"]:
+        if resource.subtype not in ["vector", "raster", "vector_time", "tabular"]:
             logger.info("Download URL is available only for datasets that have been harvested and copied locally")
             return None
 
@@ -118,6 +118,8 @@ def process_dowload(self, resource=None):
             return JsonResponse({"error": "The format provided is not valid for the selected resource"}, status=500)
 
         _format = "application/zip" if resource.is_vector() else "image/tiff"
+        if resource.subtype == "tabular":
+            _format = "text/csv"
         # getting default payload
         tpl = get_template("geoserver/dataset_download.xml")
         ctx = {"alternate": resource.alternate, "download_format": download_format or _format}

geonode/layers/migrations/0045_alter_attribute_description.py

@@ -0,0 +1,23 @@
+# Generated by Django 4.2.9 on 2024-07-04 16:07
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("layers", "0044_alter_dataset_unique_together"),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name="attribute",
+            name="description",
+            field=models.TextField(
+                blank=True,
+                help_text="description of attribute to be used in metadata",
+                null=True,
+                verbose_name="attribute description",
+            ),
+        ),
+    ]

geonode/layers/models.py

@@ -328,7 +328,7 @@ def get_linked_resources(self, as_target: bool = False):
 
     @property
     def download_url(self):
-        if self.subtype not in ["vector", "raster", "vector_time"]:
+        if self.subtype not in ["vector", "raster", "vector_time", "tabular"]:
             logger.info("Download URL is available only for datasets that have been harvested and copied locally")
             return None
         return build_absolute_uri(reverse("dataset_download", args=(self.alternate,)))
@@ -400,10 +400,9 @@ class Attribute(models.Model):
         null=True,
         unique=False,
     )
-    description = models.CharField(
+    description = models.TextField(
         _("attribute description"),
         help_text=_("description of attribute to be used in metadata"),
-        max_length=255,
         blank=True,
         null=True,
     )

geonode/maps/api/views.py

@@ -117,10 +117,14 @@ def datasets(self, request, pk=None, *args, **kwargs):
     def perform_create(self, serializer):
         # Thumbnail will be handled later
         post_creation_data = {"thumbnail": serializer.validated_data.pop("thumbnail_url", "")}
+        map_layers = serializer.validated_data.get("maplayers", [])
+        tabular_collection = all(layer.dataset and ("tabular" in layer.dataset.subtype) for layer in map_layers)
+        subtype = "tabular-collection" if len(map_layers) > 0 and tabular_collection else None
 
         instance = serializer.save(
             owner=self.request.user,
             resource_type="map",
+            subtype=subtype,
             uuid=str(uuid4()),
         )
 
@@ -149,7 +153,12 @@ def perform_update(self, serializer):
             "dataset_names_before_changes": [lyr.alternate for lyr in instance.datasets],
         }
 
-        instance = serializer.save()
+        map_layers = serializer.validated_data.get("maplayers", [])
+        tabular_collection = all(("tabular" in layer.dataset.subtype) for layer in map_layers)
+
+        instance = serializer.save(
+            subtype="tabular-collection" if tabular_collection else None,
+        )
 
         # thumbnail, events and resouce routines
         self._post_change_routines(

geonode/utils.py

@@ -1486,7 +1486,7 @@ def set_resource_default_links(instance, layer, prune=False, **kwargs):
         # Set download links for WMS, WCS or WFS and KML
         logger.debug(" -- Resource Links[Set download links for WMS, WCS or WFS and KML]...")
         instance_ows_url = f"{instance.ows_url}?" if instance.ows_url else f"{ogc_server_settings.public_url}ows?"
-        links = wms_links(instance_ows_url, instance.alternate, bbox, srid, height, width)
+        links = wms_links(instance_ows_url, instance.alternate, bbox, srid, height, width) if instance.subtype != "tabular" else []
 
         for ext, name, mime, wms_url in links:
             try:
@@ -1506,7 +1506,7 @@ def set_resource_default_links(instance, layer, prune=False, **kwargs):
                     resource=instance.resourcebase_ptr, name=gettext_lazy(name), link_type="image"
                 ).update(**_d)
 
-        if instance.subtype == "vector":
+        if instance.subtype == "vector" or instance.subtype == "tabular":
             links = wfs_links(
                 instance_ows_url,
                 instance.alternate,
@@ -1592,7 +1592,7 @@ def set_resource_default_links(instance, layer, prune=False, **kwargs):
                         instance.default_style,
                     ]
                 ):
-                    if style:
+                    if style and instance.subtype != "tabular":
                         style_name = os.path.basename(urlparse(style.sld_url).path).split(".")[0]
                         legend_url = get_legend_url(instance, style_name)
                         if Link.objects.filter(resource=instance.resourcebase_ptr, url=legend_url).count() < 2:
@@ -1650,6 +1650,7 @@ def set_resource_default_links(instance, layer, prune=False, **kwargs):
                 ogc_wms_url = instance.ows_url or urljoin(ogc_server_settings.public_url, "ows")
                 ogc_wms_name = f"OGC WMS: {instance.workspace} Service"
                 if (
+                    instance.subtype != "tabular" and
                     Link.objects.filter(resource=instance.resourcebase_ptr, name=ogc_wms_name, url=ogc_wms_url).count()
                     < 2
                 ):
@@ -1665,7 +1666,7 @@ def set_resource_default_links(instance, layer, prune=False, **kwargs):
                         ),
                     )
 
-                if instance.subtype == "vector":
+                if instance.subtype == "vector" or instance.subtype == "tabular":
                     ogc_wfs_url = instance.ows_url or urljoin(ogc_server_settings.public_url, "ows")
                     ogc_wfs_name = f"OGC WFS: {instance.workspace} Service"
                     if (