Features/update asset roles (#73)

* Modified the way assets are catalogued so that the asset type is no longer used a key to group assets. Instead an asset role is a piece of metadata for an asset and the fileame of the asset is used as the key.

* Updated changelog and version number.

* We now use the href and not just the filename for the asset key to ensure keys are unique.

* Modified how to handle CMR assets that don't have an asset role specified as metadata key/value pair. In these situations we use the key as the asset role.

* Updated changelog.

* Fixed description in changelog

* added correct client ids for dev, prod

* Updated many methods in Collection to avoid conflating the values needing to be stored for the properties type and roles. Also added new property "roles" to Datafile to assist with the previously mentioned improvement.

* Fixed bug with to_stac method.

* Fixed pluralization with roles key in data_service

---------

Co-authored-by: mike-gangl <michael.e.gangl@jpl.nasa.gov>

Author: anilnatha

CHANGELOG.md

@@ -7,17 +7,17 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 
 
-## Unreleased 
+## Unreleased [0.3.0]
 ### Added
 ### Fixed
 * fixed an issue with encoding a json deploy request twice [71](https://github.com/unity-sds/unity-py/issues/71)
 ### Changed
+* We now use the asset URI/HREF as the key into the assets, and use the "metadata" and "data" types as asset-roles: [69](https://github.com/unity-sds/unity-py/issues/69)
 ### Removed
 ### Security
 ### Deprecated
 
 
---------
 ## [0.2.2] - 2024-01-03 
 ### Added
 * Added project/venue support [5](https://github.com/unity-sds/unity-py/issues/58)

pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "unity-sds-client"
-version = "0.2.2"
+version = "0.3.0"
 description = "Unity-Py is a Python client to simplify interactions with NASA's Unity Platform."
 authors = ["Anil Natha, Mike Gangl"]
 readme = "README.md"

tests/test_files/SNDR.SS1330.CHIRP.20160829T2317.m06.g233.L1_AQ.std.v02_48.G.200425130422.json

@@ -24,13 +24,21 @@
     }
   ],
   "assets": {
-    "data": {
+    "SNDR.SS1330.CHIRP.20160829T2317.m06.g233.L1_AQ.std.v02_48.G.200425130422.nc": {
       "href": "/unity/ads/sounder_sips/chirp_test_data/SNDR.SS1330.CHIRP.20160829T2317.m06.g233.L1_AQ.std.v02_48.G.200425130422.nc",
-      "title": "Main Data File"
+      "title": "Main Data File",
+      "description": "",
+      "roles": [
+        "data"
+      ]
     },
-    "metadata_stac": {
+    "SNDR.SS1330.CHIRP.20160829T2317.m06.g233.L1_AQ.std.v02_48.G.200425130422.json": {
       "href": "/unity/ads/sounder_sips/chirp_test_data/SNDR.SS1330.CHIRP.20160829T2317.m06.g233.L1_AQ.std.v02_48.G.200425130422.json",
-      "title": "Metadata STAC File"
+      "title": "Metadata STAC File",
+      "description": "",
+      "roles": [
+        "metadata"
+      ]
     }
   },
   "bbox": "",

tests/test_unity_stac.py

@@ -26,30 +26,34 @@ def test_read_stac():
     # Added 8/10/23 to check the STAC collection information
     assert datasets[1].collection_id == 'C2011289787-GES_DISC'
 
-    data_files = collection.data_locations()
+    data_files = collection.data_files()
     assert len(data_files) == 6
-    data_files = collection.data_locations(["data","opendap"])
+    data_files = collection.data_files(["data","opendap"])
+    assert len(data_files) == 2
+    data_files = collection.data_files(["data","opendap","metadata"])
     assert len(data_files) == 4
-    data_files = collection.data_locations(["data","opendap","metadata"])
-    assert len(data_files) == 6
     data_files = collection.data_locations(["data"])
     assert len(data_files) == 2
-    for x in data_files:
+    data_locations = collection.data_locations(["data"])
+    for x in data_locations:
         assert x in ['https://data.gesdisc.earthdata.nasa.gov/data/CHIRP/SNDR13CHRP1.2/2016/235/SNDR.SS1330.CHIRP.20160822T0005.m06.g001.L1_AQ.std.v02_48.G.200425095850.nc', 'https://data.gesdisc.earthdata.nasa.gov/data/CHIRP/SNDR13CHRP1.2/2016/235/SNDR.SS1330.CHIRP.20160822T0011.m06.g002.L1_AQ.std.v02_48.G.200425095901.nc']
 
-        #Try a "classic" catalog + item files stac catalog
+    #Try a "classic" catalog + item files stac catalog
     collection = Collection.from_stac("tests/test_files/catalog_01.json")
     datasets = collection.datasets
     # Added 8/10/23 to check the STAC collection information
     assert datasets[0].collection_id == 'collection_test'
     assert len(datasets) == 1
-    data_files = collection.data_locations()
+    data_files = collection.data_files()
     assert len(data_files) == 2
-    data_files = collection.data_locations(["data"])
+    data_files = collection.data_files(["data", "metadata"])
+    assert len(data_files) == 2
+    data_files = collection.data_files(["data"])
     assert len(data_files) == 1
-    data_files = collection.data_locations(["metadata_stac"])
+    assert data_files[0].roles == ["data"]
+    data_files = collection.data_files(["metadata"])
     assert len(data_files) == 1
-    assert data_files[0] == "/unity/ads/sounder_sips/chirp_test_data/SNDR.SS1330.CHIRP.20160829T2317.m06.g233.L1_AQ.std.v02_48.G.200425130422.json"
+    assert data_files[0].roles == ["metadata"]
 
 
 def test_write_stac():

unity_sds_client/envs/environments.cfg

@@ -1,15 +1,15 @@
 [DEV]
-client_id = 2phr7qsbf1k3i9l38288n35ale
+client_id = 40c2s0ulbhp9i0fmaph3su9jch
 auth_endpoint = https://cognito-idp.us-west-2.amazonaws.com
-unity_href = d3vc8w9zcq658.cloudfront.net
+unity_href = https://d3vc8w9zcq658.cloudfront.net/
 
 [TEST]
 client_id = 71894molftjtie4dvvkbjeard0
 auth_endpoint = https://cognito-idp.us-west-2.amazonaws.com
 unity_href = https://dxebrgu0bc9w7.cloudfront.net/
 
 [PROD]
-client_id = 
+client_id = 7vehllplbone6p4usqgutqun35
 auth_endpoint = https://cognito-idp.us-west-2.amazonaws.com
 unity_href = https://d2zjsabg0fonik.cloudfront.net/
 
@@ -20,4 +20,4 @@ unity_href = https://d2zjsabg0fonik.cloudfront.net/
 ;dapa_endpoint = https://58nbcawrvb.execute-api.us-west-2.amazonaws.com/test/
 ;
 ;[SPS]
-;sps_endpoint = http://a22b9d7b66df24e6fb3326ecd4cb0614-676486270.us-west-2.elb.amazonaws.com:5001/
+;sps_endpoint = http://a22b9d7b66df24e6fb3326ecd4cb0614-676486270.us-west-2.elb.amazonaws.com:5001/

unity_sds_client/resources/collection.py

@@ -42,24 +42,42 @@ def datasets(self):
                 List of dataset objects
         """
         return self._datasets
+    
+    def data_files(self, roles=[]):
+        """
+            A method to list all assets (data, metdata, etc)
+            Parameters
+            ----------
+            type : List of Strings
+                List of "stac asset roles" to filter on. commonly ["data"] is of most importance
 
-    def data_locations(self, type=[]):
+            Returns
+            -------
+            files
+                List of returned datafiles
+        """
+        if len(roles) == 0:
+            return [file for files in [x.datafiles for x in self._datasets] for file in files]
+        else:
+            return [file for files in [x.datafiles for x in self._datasets] for file in files if set(file.roles).intersection(set(roles))]
+
+    def data_locations(self, roles=[]):
         """
             A method to list all asset locations (data, metdata, etc)
             Parameters
             ----------
             type : List of Strings
-                List of "stac asset keys" to filter on. commonly ["data"] is of most importance
+                List of "stac asset roles" to filter on. commonly ["data"] is of most importance
 
             Returns
             -------
             locations
                 List of returned asset locations
         """
-        if len(type) == 0:
+        if len(roles) == 0:
             return [file.location for files in [x.datafiles for x in self._datasets] for file in files]
         else:
-            return [file.location for files in [x.datafiles for x in self._datasets] for file in files if file.type in type  ]
+            return [file.location for files in [x.datafiles for x in self._datasets] for file in files if set(file.roles).intersection(set(roles))]
 
     def is_uri(path):
         if(path.startswith(tuple(["http:","https:","s3:"]))):
@@ -85,32 +103,38 @@ def to_stac(collection, data_dir):
         for dataset in collection._datasets:
             updated = datetime.now(timezone.utc).isoformat().replace('+00:00', 'Z')
             item = Item(
-            id=dataset.id,
-            geometry=dataset.geometry,
-            bbox=dataset.bbox,
-            collection=dataset.collection_id,
-            datetime = date_parser.parse(dataset.data_begin_time),
-            properties={
-                "datetime": dataset.data_begin_time,
-                "start_datetime": dataset.data_begin_time,
-                "end_datetime":dataset.data_end_time,
-                "created": dataset.data_create_time if dataset.data_create_time!= None else updated,
-                "updated": updated
-            },
-
+                id=dataset.id,
+                geometry=dataset.geometry,
+                bbox=dataset.bbox,
+                collection=dataset.collection_id,
+                datetime = date_parser.parse(dataset.data_begin_time),
+                properties={
+                    "datetime": dataset.data_begin_time,
+                    "start_datetime": dataset.data_begin_time,
+                    "end_datetime":dataset.data_end_time,
+                    "created": dataset.data_create_time if dataset.data_create_time!= None else updated,
+                    "updated": updated
+                },
             )
             item.properties.update(dataset.properties)
             catalog.add_item(item)
 
             for df in dataset.datafiles:
+                
                 if(Collection.is_uri(df.location)):
                     item_location = df.location
                 else:
                     item_location = df.location.replace(data_dir,".")
+
                 item.add_asset(
-                    # key="data", asset=pystac.Asset(href=f,title="Main Data File", media_type=pystac.MediaType.HDF5)
-                    key=df.type, asset=Asset(href=item_location,title="{} file".format(df.type))
+                    key = item_location,
+                    asset = Asset(
+                        href = item_location,
+                        title = "{} file".format(df.type),
+                        description = "",
+                        roles = [df.roles]
                     )
+                )
 
         from pystac.layout import TemplateLayoutStrategy
         write_dir = data_dir
@@ -182,13 +206,22 @@ def from_stac(stac_file):
                 ds.properties.update(item.properties)
 
                 for asset_key in item.assets:
-                    asset = item.assets[asset_key]
+
+                    asset:Asset = item.assets[asset_key]
+                    asset_type = asset.media_type if asset.media_type else ''
+                    asset_roles = asset.roles if asset.roles is not None else []
+                    asset_title = asset.title if asset.title is not None else ''
+                    asset_description = asset.description if asset.description is not None else ''
+
+                    if len(asset_roles) == 0 and asset_key in ["data", "metadata"]:
+                        asset_roles = [asset_key]
+                    
                     if(Collection.is_uri(asset.href)):
-                        ds.add_data_file(DataFile(asset_key ,asset.href))
+                        ds.add_data_file(DataFile(asset_type, asset.href, roles=asset_roles, title=asset_title, description=asset_description))
                     elif(os.path.isabs(asset.href)):
-                        ds.add_data_file(DataFile(asset_key ,asset.href))
+                        ds.add_data_file(DataFile(asset_type, asset.href, roles=asset_roles, title=asset_title, description=asset_description))
                     else:
-                        ds.add_data_file(DataFile(asset_key ,os.path.join(stac_dir, asset.href)))
+                        ds.add_data_file(DataFile(asset_type, os.path.join(stac_dir, asset.href), roles=asset_roles, title=asset_title, description=asset_description))
 
                 collection._datasets.append(ds)
             return collection

unity_sds_client/resources/data_file.py

@@ -8,6 +8,9 @@ def __str__(self):
     def __repr__(self):
         return self.__str__()
 
-    def __init__(self, type, location ):
-        self.type = type
+    def __init__(self, type, location, roles = [], title = "", description = "" ):
+        self.description = description
         self.location = location
+        self.roles = roles
+        self.title = title
+        self.type = type

unity_sds_client/services/data_service.py

@@ -63,11 +63,18 @@ def get_collection_data(self, collection: type= Collection):
         token = self._session.get_auth().get_token()
         response = requests.get(url, headers={"Authorization": "Bearer " + token})
         results = response.json()['features']
-
+        
         for dataset in results:
             ds = Dataset(dataset['id'], collection.collection_id, dataset['properties']['start_datetime'], dataset['properties']['end_datetime'], dataset['properties']['created'])
-            ds.add_data_file(DataFile("data" ,dataset['assets']['data']['href']))
-            ds.add_data_file(DataFile("metadata" ,dataset['assets']['metadata__data']['href']))
+            
+            for asset_key in dataset['assets']:
+                location = dataset['assets'][asset_key]['href']
+                file_type = dataset['assets'][asset_key].get('type', "")
+                title = dataset['assets'][asset_key].get('title', "")
+                description = dataset['assets'][asset_key].get('description', "")
+                roles = dataset['assets'][asset_key]["roles"] if "roles" in dataset['assets'][asset_key] else ["metadata"] if asset_key in ['metadata__cmr','metadata__data'] else [asset_key]
+                ds.add_data_file(DataFile(file_type, location, roles=roles, title=title, description=description))
+
             datasets.append(ds)
 
         return datasets