diff --git a/mdps_ds_lib/ds_client/ds_client_base.py b/mdps_ds_lib/ds_client/ds_client_base.py index 3f4fd2a..439c2cb 100644 --- a/mdps_ds_lib/ds_client/ds_client_base.py +++ b/mdps_ds_lib/ds_client/ds_client_base.py @@ -22,7 +22,8 @@ def __init__(self, token_retriever: TokenAbstract, ds_url: str, ds_stage: str): self.__default_collection_venue = '001' def get_complete_collection(self): - return f'{self.collection}{self.__default_collection_venue}' if self.collection_venue is None else f'{self.collection}___{self.collection_venue}' + collection_venue = self.__default_collection_venue if self.collection_venue is None else self.collection_venue + return f'{self.collection}___{collection_venue}' @property def granule(self): diff --git a/mdps_ds_lib/ds_client/ds_client_user.py b/mdps_ds_lib/ds_client/ds_client_user.py index d579fa2..6e5126a 100644 --- a/mdps_ds_lib/ds_client/ds_client_user.py +++ b/mdps_ds_lib/ds_client/ds_client_user.py @@ -59,17 +59,13 @@ def query_single_collection(self, limit= 10, offset= None, datetime = None, filt def create_new_granule(self, granule_stac: Item): request_url = f'{self._uds_url}collections/' - temp_collection_id = f"{self.collection}___001" if self.collection_venue is None else f"{self.collection}___{self.collection_venue}" - temp_collection_id = [ - self.urn, self.org, self.project, self.tenant, self.tenant_venue, temp_collection_id - ] + temp_collection_id = [self.urn, self.org, self.project, self.tenant, self.tenant_venue, self.get_complete_collection()] temp_granule_id = temp_collection_id + [self.granule] temp_collection_id = ':'.join(temp_collection_id) temp_granule_id = ':'.join(temp_granule_id) granule_stac.id = temp_granule_id - granule_stac.collection = temp_collection_id - + granule_stac.collection_id = temp_collection_id request_url = f'{request_url}{temp_collection_id}/items/{temp_granule_id}' s = requests.session() s.trust_env = self._trust_env diff --git a/tests/mdps_ds_lib/ds_client/test_ds_client_local.py b/tests/mdps_ds_lib/ds_client/test_ds_client_local.py index 20bd874..d3a8aeb 100644 --- a/tests/mdps_ds_lib/ds_client/test_ds_client_local.py +++ b/tests/mdps_ds_lib/ds_client/test_ds_client_local.py @@ -1,5 +1,6 @@ import base64 import json +import math import os from unittest import TestCase @@ -9,6 +10,7 @@ from mdps_ds_lib.ds_client.auth_token.token_factory import TokenFactory from mdps_ds_lib.ds_client.ds_client_admin import DsClientAdmin from mdps_ds_lib.ds_client.ds_client_user import DsClientUser +from mdps_ds_lib.lib.utils.file_utils import FileUtils class TestDsClientAdmin(TestCase): @@ -42,12 +44,10 @@ def test_01_admin(self): client.urn = 'URN' client.org = 'NASA' client.project = 'GEMX' - client.tenant = 'AVIRIS' - client.tenant_venue = 'OPS' + client.tenant = 'DEMO' + client.tenant_venue = 'DEV' client.add_admin_group(['CREATE', 'READ', 'DELETE'], 'Test_User') - - gemx_custom_properties = { "acquisition_date_l1b": {"type": "date", "format": "yyyy-MM-dd HH:mm:ss||yyyy-MM-dd'T'HH:mm:ssZ||yyyy-MM-dd'T'HH:mm:ss'Z'||yyyy-MM-dd'T'HH:mm:ss.SSSSSSZ||yyyy-MM-dd'T'HH:mm:ss.SSSSSS'Z'||yyyy-MM-dd||epoch_millis"}, @@ -68,14 +68,6 @@ def test_01_admin(self): "scan_lines": {"type": "long"}, } client.add_tenant_database_index(gemx_custom_properties) - # client.collection = '' - - client.urn = 'URN' - client.org = 'NASA' - client.project = 'GEMX' - client.tenant = 'MASTERS' - client.tenant_venue = 'OPS' - client.add_admin_group(['CREATE', 'READ', 'DELETE'], 'Test_User') return def test_02_user_add_collection(self): @@ -91,20 +83,15 @@ def test_02_user_add_collection(self): client.urn = 'URN' client.org = 'NASA' client.project = 'GEMX' - client.tenant = 'AVIRIS' - client.tenant_venue = 'OPS' - client.collection = 'GEMX_2024' - client.collection_venue = '001' + client.tenant = 'DEMO' + client.tenant_venue = 'DEV' + client.collection = 'AVIRIS' result = client.create_new_collection(True) print(result) - client.tenant = 'MASTERS' - client.tenant_venue = 'OPS' - client.collection = '2024' - client.collection_venue = '001' + client.collection = 'MASTER' result = client.create_new_collection(True) print(result) - return def test_03_user_add_granule(self): @@ -120,65 +107,67 @@ def test_03_user_add_granule(self): client.urn = 'URN' client.org = 'NASA' client.project = 'GEMX' - client.tenant = 'AVIRIS' - client.tenant_venue = 'OPS' - client.collection = 'GEMX_2024' + client.tenant = 'DEMO' + client.tenant_venue = 'DEV' + client.collection = 'GEMX' client.collection_venue = '001' - client.granule = 'f240424t01_p00_r10' - - temp_granule = { - "type": "Feature", - "stac_version": "1.0.0", - "id": "URN:NASA:AVIRIS:f240424t01:p00_r10", - "properties": { - "start_datetime": "2024-04-24T20:37:00.000000Z", - "end_datetime": "2024-04-24T20:50:00.000000Z", - "site_name": "x001(orthocorrected)", - "nasa_log": 232016.0, - "investigator": "Raymond Kokaly", - "comments": "x001 s-)n; LN2 refill 2042", - "site_info": "GEMx - RFLY02", - "datetime": "1970-01-01T00:00:00Z" - }, - "geometry": { - "type": "Point", - "coordinates": [ - 0.0, - 0.0 - ] + # client.granule = 'f240424t01_p00_r10' + temp_granule = { + "type": "Feature", + "stac_version": "1.0.0", + "id": "f230331t01-p00_r09", + "properties": { + "start_datetime": "2023-03-31T18:52:00.000000Z", + "end_datetime": "2023-03-31T19:07:00.000000Z", + "site_name": "Yosemite-NEON Box 4 (YN38)(orthocorrected)", + "nasa_log": "232017", + "investigator": "Robert Green", + "comments": "s-)nclr", + "site_info": "WDTS - Yosemite", + "datetime": "1970-01-01T00:00:00Z" + }, + "geometry": { + "type": "Point", + "coordinates": [ + 0.0, + 0.0 + ] + }, + "links": [], + "assets": { + "l1b": { + "href": "https://popo.jpl.nasa.gov/gemx/data_products/l1b/f230331t01p00r09rdn_g.tar.gz", + "title": "f230331t01p00r09rdn_g.tar.gz", + "description": "2024-11-05 09:22", + "file:size": 3435973836.8 }, - "links": [], - "assets": { - "l1b": { - "href": "https://popo.jpl.nasa.gov/gemx/data_products/l1b/f240424t01p00r10rdn_g.tar.gz", - "title": "f240424t01p00r10rdn_g.tar.gz", - "description": "2024-10-08 15:16", - "file:size": 2362232012.8 - }, - "l2": { - "href": "https://popo.jpl.nasa.gov/gemx/data_products/l2/f240424t01p00r10rfl.tar.gz", - "title": "f240424t01p00r10rfl.tar.gz", - "description": "2024-10-18 08:22", - "file:size": 4187593113.6 - }, - "quicklook": { - "href": "http://aviris.jpl.nasa.gov/ql/24qlook/f240424t01p00r10_geo.jpeg", - "title": "f240424t01p00r10_geo.jpeg", - "description": "Quicklook Link" - } + "l2": { + "href": "https://popo.jpl.nasa.gov/gemx/data_products/l2/f230331t01p00r09rfl.tar.gz", + "title": "f230331t01p00r09rfl.tar.gz", + "description": "2024-11-05 09:09", + "file:size": 6442450944.0 }, - "bbox": [ - -114.3144070, - 32.5, - -114.3144078, - 33.5, - ], - "stac_extensions": [ - "https://stac-extensions.github.io/file/v2.1.0/schema.json" - ], - "collection": "URN:NASA:AVIRIS:f240424t01" - } + "quicklook": { + "href": "http://aviris.jpl.nasa.gov/ql/23qlook/f230331t01p00r09_geo.jpeg", + "title": "f230331t01p00r09_geo.jpeg", + "description": "Quicklook Link" + } + }, + "bbox": [ + -119.2020323, + 36.6539295, + -119.1875901, + 38.1015035 + ], + "stac_extensions": [ + "https://stac-extensions.github.io/file/v2.1.0/schema.json" + ], + "collection": "MASTER" + } + + temp_granule = Item.from_dict(temp_granule) + client.granule = temp_granule.id result = client.create_new_granule(temp_granule) print(result) client.tenant = 'MASTERS' @@ -187,7 +176,41 @@ def test_03_user_add_granule(self): client.collection_venue = '001' # result = client.create_new_granule() print(result) + return + + def test_03_user_add_many_granules(self): + encoded_token = self.__get_dummy_token() + os.environ['TOKEN_FACTORY'] = 'DUMMY' + os.environ['DS_TOKEN'] = encoded_token + # os.environ['DS_URL'] = encoded_token + # os.environ['DS_STAGE'] = encoded_token + token_retriever: TokenAbstract = TokenFactory().get_instance(os.getenv('TOKEN_FACTORY')) + client = DsClientUser(token_retriever, 'http://localhost:8005', 'data') + # client = DsClientAdmin(token_retriever, 'http://localhost:8005', 'data') + client.urn = 'URN' + client.org = 'NASA' + client.project = 'GEMX' + client.tenant = 'DEMO' + client.tenant_venue = 'DEV' + client.collection = 'MASTER' + result = FileUtils.read_json('master_stac.json.corrected.json') + # client.collection = 'AVIRIS' + # result = FileUtils.read_json('aviris_2023.stac.json.corrected.json') + # result = FileUtils.read_json('aviris_2024.stac.json.corrected.json') + + for temp_granule in result: + temp_granule = Item.from_dict(temp_granule) + if math.isnan(temp_granule.bbox[0]): + print(temp_granule) + continue + client.granule = temp_granule.id + try: + result = client.create_new_granule(temp_granule) + except Exception as e: + print(client.granule) + raise e + print(result) return def test_query_granules(self): @@ -207,6 +230,7 @@ def test_query_granules(self): client.tenant_venue = 'OPS' client.collection = 'GEMX_2024' client.collection_venue = '001' + # result = client.query_granules(bbox='-114,32.5,-113,33.5') result = client.query_granules(bbox='-114,32.5,-113,33.5') print(result) print(client.query_granules_next()) @@ -225,9 +249,12 @@ def test_query_granules_across_collections(self): client.urn = 'URN' client.org = 'NASA' client.project = 'GEMX' - client.tenant = 'AVIRIS' - client.tenant_venue = 'OPS' - result = client.query_granules_across_collections(bbox='-114.314407,32.5,-114.3144078,33.5') + client.tenant = 'DEMO' + client.tenant_venue = 'DEV' + # result = client.query_granules_across_collections(bbox='-114.314407,32.5,-114.3144078,32.55') + result = client.query_granules_across_collections(sort_keys='-collection') print(result) - print(client.query_granules_next()) + while len(result['features']) > 0: + result = client.query_granules_next() + print(result) return