refactor: Address PR comments

backend/check_stac_metadata/utils.py

@@ -15,13 +15,13 @@
 from ..s3 import S3_URL_PREFIX
 from ..stac_format import (
     STAC_ASSETS_KEY,
-    STAC_CATALOG_TYPE,
-    STAC_COLLECTION_TYPE,
     STAC_FILE_CHECKSUM_KEY,
     STAC_HREF_KEY,
-    STAC_ITEM_TYPE,
     STAC_LINKS_KEY,
-    STAC_TYPE_KEY,
+    STAC_MEDIA_TYPE_KEY,
+    STAC_TYPE_CATALOG,
+    STAC_TYPE_COLLECTION,
+    STAC_TYPE_ITEM,
 )
 from ..types import JsonObject
 from ..validation_results_model import ValidationResult, ValidationResultFactory
@@ -41,9 +41,9 @@
         Type[STACItemSchemaValidator],
     ],
 ] = {
-    STAC_COLLECTION_TYPE: STACCollectionSchemaValidator,
-    STAC_CATALOG_TYPE: STACCatalogSchemaValidator,
-    STAC_ITEM_TYPE: STACItemSchemaValidator,
+    STAC_TYPE_COLLECTION: STACCollectionSchemaValidator,
+    STAC_TYPE_CATALOG: STACCatalogSchemaValidator,
+    STAC_TYPE_ITEM: STACItemSchemaValidator,
 }
 
 PROCESSING_ASSET_ASSET_KEY = "asset"
@@ -111,7 +111,7 @@ def validate(self, url: str) -> None:  # pylint: disable=too-complex
         self.traversed_urls.append(url)
         object_json = self.get_object(url)
 
-        stac_type = object_json[STAC_TYPE_KEY]
+        stac_type = object_json[STAC_MEDIA_TYPE_KEY]
         validator = STAC_TYPE_VALIDATION_MAP[stac_type]()
 
         try:

backend/populate_catalog/task.py

@@ -1,5 +1,6 @@
-import os
 from json import dumps
+from os.path import join
+from urllib.parse import urlparse
 
 import boto3
 from pystac import STAC_IO, Catalog, CatalogType, Collection, Item  # type: ignore[import]
@@ -63,32 +64,29 @@ class UnhandledSQSMessageException(Exception):
 
 class GeostoreSTACLayoutStrategy(HrefLayoutStrategy):
     def get_catalog_href(self, cat: Catalog, parent_dir: str, is_root: bool) -> str:
-        original_path = cat.get_self_href().split("/")
+        original_path = urlparse(cat.get_self_href()).path.rsplit("/", maxsplit=2)
         if is_root:
             cat_root = parent_dir
         else:
-            cat_root = os.path.join(parent_dir, original_path[-2])
+            cat_root = join(parent_dir, original_path[-2])
 
-        return os.path.join(cat_root, original_path[-1])
+        return join(cat_root, original_path[-1])
 
     def get_collection_href(self, col: Collection, parent_dir: str, is_root: bool) -> str:
-        original_path = col.get_self_href().split("/")
-        if is_root:
-            col_root = parent_dir
-        else:
-            col_root = os.path.join(parent_dir, original_path[-2])
-
-        return os.path.join(col_root, original_path[-1])
+        original_path = urlparse(col.get_self_href()).path.rsplit("/", maxsplit=2)
+        assert not is_root
+        col_root = join(parent_dir, original_path[-2])
+        return join(col_root, original_path[-1])
 
     def get_item_href(self, item: Item, parent_dir: str) -> str:
         original_path = item.get_self_href().split("/")
-        return os.path.join(parent_dir, original_path[-1])
+        return join(parent_dir, original_path[-1])
 
 
 def handle_dataset(version_metadata_key: str) -> None:
     """Handle writing a new dataset version to the dataset catalog"""
     storage_bucket_path = f"{S3_URL_PREFIX}{ResourceName.STORAGE_BUCKET_NAME.value}"
-    dataset_prefix = version_metadata_key.split("/", maxsplit=2)[0]
+    dataset_prefix = version_metadata_key.split("/", maxsplit=1)[0]
 
     dataset_catalog = Catalog.from_file(f"{storage_bucket_path}/{dataset_prefix}/{CATALOG_KEY}")
 

backend/sqs_message_attributes.py

@@ -1,4 +1,9 @@
 def decapitalize(key: str) -> str:
+    """
+    This method will be used to lower case the first character of SQS
+    message attributes being received by Lambda to resolve inconsistencies.
+    Issue outlined here: https://github.com/boto/boto3/issues/2582
+    """
     return f"{key[:1].lower()}{key[1:]}"
 
 

backend/stac_format.py

@@ -10,19 +10,20 @@
 STAC_HREF_KEY = "href"
 STAC_ID_KEY = "id"
 STAC_LICENSE_KEY = "license"
+STAC_LINKS_KEY = "links"
+STAC_MEDIA_TYPE_GEOJSON = "application/geo+json"
+STAC_MEDIA_TYPE_JSON = "application/json"
+STAC_MEDIA_TYPE_KEY = "type"
 STAC_PROPERTIES_DATETIME_KEY = "datetime"
 STAC_PROPERTIES_KEY = "properties"
-STAC_TITLE_KEY = "title"
-STAC_TYPE_KEY = "type"
-STAC_VERSION_KEY = "stac_version"
-STAC_LINKS_KEY = "links"
 STAC_REL_CHILD = "child"
 STAC_REL_ITEM = "item"
 STAC_REL_KEY = "rel"
 STAC_REL_PARENT = "parent"
 STAC_REL_ROOT = "root"
 STAC_REL_SELF = "self"
-
-STAC_COLLECTION_TYPE = "Collection"
-STAC_ITEM_TYPE = "Feature"
-STAC_CATALOG_TYPE = "Catalog"
+STAC_TITLE_KEY = "title"
+STAC_TYPE_COLLECTION = "Collection"
+STAC_TYPE_ITEM = "Feature"
+STAC_TYPE_CATALOG = "Catalog"
+STAC_VERSION_KEY = "stac_version"

tests/stac_objects.py

@@ -2,8 +2,6 @@
 
 from backend.stac_format import (
     STAC_ASSETS_KEY,
-    STAC_CATALOG_TYPE,
-    STAC_COLLECTION_TYPE,
     STAC_DESCRIPTION_KEY,
     STAC_EXTENT_BBOX_KEY,
     STAC_EXTENT_KEY,
@@ -14,12 +12,14 @@
     STAC_GEOMETRY_KEY,
     STAC_HREF_KEY,
     STAC_ID_KEY,
-    STAC_ITEM_TYPE,
     STAC_LICENSE_KEY,
     STAC_LINKS_KEY,
+    STAC_MEDIA_TYPE_KEY,
     STAC_PROPERTIES_DATETIME_KEY,
     STAC_PROPERTIES_KEY,
-    STAC_TYPE_KEY,
+    STAC_TYPE_CATALOG,
+    STAC_TYPE_COLLECTION,
+    STAC_TYPE_ITEM,
     STAC_VERSION_KEY,
 )
 
@@ -46,7 +46,7 @@
     STAC_LICENSE_KEY: "MIT",
     STAC_LINKS_KEY: [],
     STAC_VERSION_KEY: STAC_VERSION,
-    STAC_TYPE_KEY: STAC_COLLECTION_TYPE,
+    STAC_MEDIA_TYPE_KEY: STAC_TYPE_COLLECTION,
 }
 
 MINIMAL_VALID_STAC_ITEM_OBJECT: Dict[str, Any] = {
@@ -59,13 +59,13 @@
     STAC_LINKS_KEY: [],
     STAC_PROPERTIES_KEY: {STAC_PROPERTIES_DATETIME_KEY: any_past_datetime_string()},
     STAC_VERSION_KEY: STAC_VERSION,
-    STAC_TYPE_KEY: STAC_ITEM_TYPE,
+    STAC_MEDIA_TYPE_KEY: STAC_TYPE_ITEM,
 }
 
 MINIMAL_VALID_STAC_CATALOG_OBJECT: Dict[str, Any] = {
     STAC_DESCRIPTION_KEY: any_dataset_description(),
     STAC_ID_KEY: any_dataset_id(),
     STAC_LINKS_KEY: [],
     STAC_VERSION_KEY: STAC_VERSION,
-    STAC_TYPE_KEY: STAC_CATALOG_TYPE,
+    STAC_MEDIA_TYPE_KEY: STAC_TYPE_CATALOG,
 }

tests/test_check_stac_metadata.py

@@ -34,7 +34,6 @@
 from backend.s3 import S3_URL_PREFIX
 from backend.stac_format import (
     STAC_ASSETS_KEY,
-    STAC_COLLECTION_TYPE,
     STAC_DESCRIPTION_KEY,
     STAC_EXTENT_BBOX_KEY,
     STAC_EXTENT_KEY,
@@ -46,7 +45,8 @@
     STAC_ID_KEY,
     STAC_LICENSE_KEY,
     STAC_LINKS_KEY,
-    STAC_TYPE_KEY,
+    STAC_MEDIA_TYPE_KEY,
+    STAC_TYPE_COLLECTION,
     STAC_VERSION_KEY,
 )
 from backend.step_function import DATASET_ID_KEY, METADATA_URL_KEY, VERSION_ID_KEY
@@ -154,7 +154,7 @@ def should_report_duplicate_asset_names(validation_results_factory_mock: MagicMo
         f' "{STAC_LICENSE_KEY}": "MIT",'
         f' "{STAC_LINKS_KEY}": [],'
         f' "{STAC_VERSION_KEY}": "{STAC_VERSION}",'
-        f' "{STAC_TYPE_KEY}": "{STAC_COLLECTION_TYPE}"'
+        f' "{STAC_MEDIA_TYPE_KEY}": "{STAC_TYPE_COLLECTION}"'
         "}"
     )
     metadata_url = any_s3_url()
@@ -433,7 +433,7 @@ def should_treat_any_missing_top_level_key_as_invalid(subtests: SubTests) -> Non
         MINIMAL_VALID_STAC_CATALOG_OBJECT,
     ]:
         for key in stac_object:
-            with subtests.test(msg=f"{stac_object[STAC_TYPE_KEY]} {key}"):
+            with subtests.test(msg=f"{stac_object[STAC_MEDIA_TYPE_KEY]} {key}"):
                 stac_object = deepcopy(stac_object)
                 stac_object.pop(key)