[Fixes #12657] Refactor supported types

geonode/resource/enumerator.py

@@ -22,6 +22,7 @@
 
 class ExecutionRequestAction(enum.Enum):
     IMPORT = _("import")
+    UPLOAD = _("upload")
     CREATE = _("create")
     COPY = _("copy")
     DELETE = _("delete")

geonode/settings.py

@@ -2240,56 +2240,6 @@ def get_geonode_catalogue_service():
     "document_upload",
 )
 
-SUPPORTED_DATASET_FILE_TYPES = [
-    {
-        "id": "shp",
-        "label": "ESRI Shapefile",
-        "format": "vector",
-        "ext": ["shp"],
-        "requires": ["shp", "prj", "dbf", "shx"],
-        "optional": ["xml", "sld"],
-    },
-    {
-        "id": "tiff",
-        "label": "GeoTIFF",
-        "format": "raster",
-        "ext": ["tiff", "tif", "geotiff", "geotif"],
-        "mimeType": ["image/tiff"],
-        "optional": ["xml", "sld"],
-    },
-    {
-        "id": "csv",
-        "label": "Comma Separated Value (CSV)",
-        "format": "vector",
-        "ext": ["csv"],
-        "mimeType": ["text/csv"],
-        "optional": ["xml", "sld"],
-    },
-    {
-        "id": "zip",
-        "label": "Zip Archive",
-        "format": "archive",
-        "ext": ["zip"],
-        "mimeType": ["application/zip"],
-        "optional": ["xml", "sld"],
-    },
-    {
-        "id": "xml",
-        "label": "XML Metadata File",
-        "format": "metadata",
-        "ext": ["xml"],
-        "mimeType": ["application/json"],
-        "needsFiles": ["shp", "prj", "dbf", "shx", "csv", "tiff", "zip", "sld"],
-    },
-    {
-        "id": "sld",
-        "label": "Styled Layer Descriptor (SLD)",
-        "format": "metadata",
-        "ext": ["sld"],
-        "mimeType": ["application/json"],
-        "needsFiles": ["shp", "prj", "dbf", "shx", "csv", "tiff", "zip", "xml"],
-    },
-]
 INSTALLED_APPS += (
     "dynamic_models",
     # "importer",

geonode/storage/tests.py

@@ -573,23 +573,6 @@ def test_zip_file_should_correctly_index_file_extensions(self):
         # extensions found more than once get indexed
         self.assertIsNotNone(_files.get("csv_file_1"))
 
-    @override_settings(
-        SUPPORTED_DATASET_FILE_TYPES=[
-            {"id": "kmz", "label": "kmz", "format": "vector", "ext": ["kmz"]},
-            {"id": "kml", "label": "kml", "format": "vector", "ext": ["kml"]},
-        ]
-    )
-    def test_zip_file_should_correctly_recognize_main_extension_with_kmz(self):
-        # reinitiate the storage manager with the zip file
-        storage_manager = self.sut(
-            remote_files={"base_file": os.path.join(f"{self.project_root}", "tests/data/Italy.kmz")}
-        )
-        storage_manager.clone_remote_files()
-
-        self.assertIsNotNone(storage_manager.data_retriever.temporary_folder)
-        _files = storage_manager.get_retrieved_paths()
-        self.assertTrue("doc.kml" in _files.get("base_file"), msg=f"files available: {_files}")
-
     def test_zip_file_should_correctly_recognize_main_extension_with_shp(self):
         # zipping files
         storage_manager = self.sut(remote_files=self.local_files_paths)

geonode/tests/test_utils.py

@@ -18,7 +18,6 @@
 #########################################################################
 import copy
 from unittest import TestCase
-from django.test import override_settings
 
 from unittest.mock import patch
 from datetime import datetime, timedelta
@@ -32,8 +31,7 @@
 from geonode.geoserver.helpers import set_attributes
 from geonode.tests.base import GeoNodeBaseTestSupport
 from geonode.br.management.commands.utils.utils import ignore_time
-from geonode.utils import copy_tree, get_supported_datasets_file_types, bbox_to_wkt
-from geonode import settings
+from geonode.utils import copy_tree, bbox_to_wkt
 
 
 class TestCopyTree(GeoNodeBaseTestSupport):
@@ -205,39 +203,6 @@ def setUp(self):
             },
         ]
 
-    @override_settings(
-        ADDITIONAL_DATASET_FILE_TYPES=[
-            {"id": "dummy_type", "label": "Dummy Type", "format": "dummy", "ext": ["dummy"]},
-        ]
-    )
-    def test_should_append_additional_type_if_config_is_provided(self):
-        prev_count = len(settings.SUPPORTED_DATASET_FILE_TYPES)
-        supported_types = get_supported_datasets_file_types()
-        supported_keys = [t.get("id") for t in supported_types]
-        self.assertIn("dummy_type", supported_keys)
-        self.assertEqual(len(supported_keys), prev_count + 1)
-
-    @override_settings(
-        ADDITIONAL_DATASET_FILE_TYPES=[
-            {
-                "id": "shp",
-                "label": "Replaced type",
-                "format": "vector",
-                "ext": ["shp"],
-                "requires": ["shp", "prj", "dbf", "shx"],
-                "optional": ["xml", "sld"],
-            },
-        ]
-    )
-    def test_should_replace_the_type_id_if_already_exists(self):
-        prev_count = len(settings.SUPPORTED_DATASET_FILE_TYPES)
-        supported_types = get_supported_datasets_file_types()
-        supported_keys = [t.get("id") for t in supported_types]
-        self.assertIn("shp", supported_keys)
-        self.assertEqual(len(supported_keys), prev_count)
-        shp_type = [t for t in supported_types if t["id"] == "shp"][0]
-        self.assertEqual(shp_type["label"], "Replaced type")
-
 
 class TestRegionsCrossingDateLine(TestCase):
     def setUp(self):

geonode/upload/api/views.py

@@ -191,7 +191,7 @@ def create(self, request, *args, **kwargs):
 
                     self.validate_upload(request, storage_manager)
 
-                action = ExecutionRequestAction.IMPORT.value
+                action = ExecutionRequestAction.UPLOAD.value
 
                 input_params = {
                     **{"files": files, "handler_module_path": str(handler)},

geonode/upload/celery_tasks.py

@@ -90,7 +90,7 @@ def import_orchestrator(
     step="start_import",
     layer_name=None,
     alternate=None,
-    action=exa.IMPORT.value,
+    action=exa.UPLOAD.value,
     **kwargs,
 ):
     """
@@ -179,7 +179,7 @@ def import_resource(self, execution_id, /, handler_module_path, action, **kwargs
         call_rollback_function(
             execution_id,
             handlers_module_path=handler_module_path,
-            prev_action=exa.IMPORT.value,
+            prev_action=exa.UPLOAD.value,
             layer=None,
             alternate=None,
             error=e,
@@ -309,7 +309,7 @@ def create_geonode_resource(
     layer_name: Optional[str] = None,
     alternate: Optional[str] = None,
     handler_module_path: str = None,
-    action: str = exa.IMPORT.value,
+    action: str = exa.UPLOAD.value,
     **kwargs,
 ):
     """

geonode/upload/handlers/apps.py

@@ -41,21 +41,3 @@ def run_setup_hooks(*args, **kwargs):
     for item in _handlers:
         item.register()
     logger.info(f"The following handlers have been registered: {', '.join(available_handlers)}")
-
-    _available_settings = [
-        import_string(module_path)().supported_file_extension_config
-        for module_path in available_handlers
-        if import_string(module_path)().supported_file_extension_config
-    ]
-    # injecting the new config required for FE
-    supported_type = []
-    supported_type.extend(_available_settings)
-    if not getattr(settings, "ADDITIONAL_DATASET_FILE_TYPES", None):
-        setattr(settings, "ADDITIONAL_DATASET_FILE_TYPES", supported_type)
-    elif "gpkg" not in [x.get("id") for x in settings.ADDITIONAL_DATASET_FILE_TYPES]:
-        settings.ADDITIONAL_DATASET_FILE_TYPES.extend(supported_type)
-        setattr(
-            settings,
-            "ADDITIONAL_DATASET_FILE_TYPES",
-            settings.ADDITIONAL_DATASET_FILE_TYPES,
-        )

geonode/upload/handlers/base.py

@@ -47,7 +47,7 @@ class BaseHandler(ABC):
     REGISTRY = []
 
     TASKS = {
-        exa.IMPORT.value: (),
+        exa.UPLOAD.value: (),
         exa.COPY.value: (),
         exa.DELETE.value: (),
         exa.UPDATE.value: (),

geonode/upload/handlers/common/metadata.py

@@ -17,7 +17,6 @@
 #
 #########################################################################
 import logging
-from geonode.resource.enumerator import ExecutionRequestAction as exa
 from geonode.upload.handlers.base import BaseHandler
 from geonode.upload.handlers.utils import UploadSourcesEnum
 from geonode.upload.models import ResourceHandlerInfo
@@ -37,7 +36,7 @@ class MetadataFileHandler(BaseHandler):
     """
 
     TASKS = {
-        exa.IMPORT.value: ("start_import", "geonode.upload.import_resource"),
+        ira.RESOURCE_METADATA_UPLOAD.value: ("start_import", "geonode.upload.import_resource"),
         ira.ROLLBACK.value: (
             "start_rollback",
             "geonode.upload.rollback",

geonode/upload/handlers/common/raster.py

@@ -310,7 +310,7 @@ def import_resource(self, files: dict, execution_id: str, **kwargs) -> str:
                         "geonode.upload.import_resource",
                         layer_name,
                         alternate,
-                        exa.IMPORT.value,
+                        exa.UPLOAD.value,
                     )
                 )
                 return layer_name, alternate, execution_id

geonode/upload/handlers/common/remote.py

@@ -48,7 +48,7 @@ class BaseRemoteResourceHandler(BaseHandler):
     """
 
     TASKS = {
-        exa.IMPORT.value: (
+        exa.UPLOAD.value: (
             "start_import",
             "geonode.upload.import_resource",
             "geonode.upload.create_geonode_resource",
@@ -163,7 +163,7 @@ def import_resource(self, files: dict, execution_id: str, **kwargs) -> str:
                     "geonode.upload.import_resource",
                     layer_name,
                     alternate,
-                    exa.IMPORT.value,
+                    exa.UPLOAD.value,
                 )
             )
             return layer_name, alternate, execution_id

geonode/upload/handlers/common/vector.py

@@ -851,15 +851,15 @@ def import_next_step(
             actual_step,
             layer_name,
             alternate,
-            exa.IMPORT.value,
+            exa.UPLOAD.value,
         )
 
         import_orchestrator.apply_async(task_params, kwargs)
     except Exception as e:
         call_rollback_function(
             execution_id,
             handlers_module_path=handlers_module_path,
-            prev_action=exa.IMPORT.value,
+            prev_action=exa.UPLOAD.value,
             layer=layer_name,
             alternate=alternate,
             error=e,
@@ -927,7 +927,7 @@ def import_with_ogr2ogr(
         call_rollback_function(
             execution_id,
             handlers_module_path=handler_module_path,
-            prev_action=exa.IMPORT.value,
+            prev_action=exa.UPLOAD.value,
             layer=original_name,
             alternate=alternate,
             error=e,

geonode/upload/handlers/csv/handler.py

@@ -41,7 +41,7 @@ class CSVFileHandler(BaseVectorFileHandler):
     """
 
     TASKS = {
-        exa.IMPORT.value: (
+        exa.UPLOAD.value: (
             "start_import",
             "geonode.upload.import_resource",
             "geonode.upload.publish_resource",
@@ -58,6 +58,12 @@ class CSVFileHandler(BaseVectorFileHandler):
             "start_rollback",
             "geonode.upload.rollback",
         ),
+        ira.REPLACE.value: (
+            "start_import",
+            "geonode.upload.import_resource",
+            "geonode.upload.publish_resource",
+            "geonode.upload.create_geonode_resource",
+        ),
     }
 
     possible_geometry_column_name = ["geom", "geometry", "wkt_geom", "the_geom"]
@@ -69,11 +75,15 @@ class CSVFileHandler(BaseVectorFileHandler):
     def supported_file_extension_config(self):
         return {
             "id": "csv",
-            "label": "CSV",
-            "format": "vector",
-            "mimeType": ["text/csv"],
-            "ext": ["csv"],
-            "optional": ["sld", "xml"],
+            "formats": [
+                {
+                    "label": "CSV",
+                    "required_ext": ["csv"],
+                    "optional_ext": ["sld", "xml"],
+                }
+            ],
+            "actions": list(self.TASKS.keys()),
+            "type": "vector",
         }
 
     @staticmethod

geonode/upload/handlers/geojson/handler.py

@@ -37,7 +37,7 @@ class GeoJsonFileHandler(BaseVectorFileHandler):
     """
 
     TASKS = {
-        exa.IMPORT.value: (
+        exa.UPLOAD.value: (
             "start_import",
             "geonode.upload.import_resource",
             "geonode.upload.publish_resource",
@@ -54,16 +54,32 @@ class GeoJsonFileHandler(BaseVectorFileHandler):
             "start_rollback",
             "geonode.upload.rollback",
         ),
+        ira.REPLACE.value: (
+            "start_import",
+            "geonode.upload.import_resource",
+            "geonode.upload.publish_resource",
+            "geonode.upload.create_geonode_resource",
+        ),
     }
 
     @property
     def supported_file_extension_config(self):
         return {
             "id": "geojson",
-            "label": "GeoJSON",
-            "format": "vector",
-            "ext": ["json", "geojson"],
-            "optional": ["xml", "sld"],
+            "formats": [
+                {
+                    "label": "GeoJSON",
+                    "required_ext": ["geojson"],
+                    "optional_ext": ["sld", "xml"],
+                },
+                {
+                    "label": "GeoJSON",
+                    "required_ext": ["json"],
+                    "optional_ext": ["sld", "xml"],
+                },
+            ],
+            "actions": list(self.TASKS.keys()),
+            "type": "vector",
         }
 
     @staticmethod