diff --git a/mdps_ds_lib/ds_client/ds_client_user.py b/mdps_ds_lib/ds_client/ds_client_user.py index 9060f88..494a590 100644 --- a/mdps_ds_lib/ds_client/ds_client_user.py +++ b/mdps_ds_lib/ds_client/ds_client_user.py @@ -52,6 +52,22 @@ def create_new_collection(self, is_actual_execution=False): response.raise_for_status() return response.text + def delete_collection(self): + request_url = f'{self._uds_url}collections/' + s = requests.session() + s.trust_env = self._trust_env + temp_collection_id = [ + self.urn, self.org, self.project, self.tenant, self.tenant_venue, self.get_complete_collection() + ] + temp_collection_id = ':'.join(temp_collection_id) + request_url = f'{request_url}{temp_collection_id}/' + + response = s.delete(url=request_url, headers={ + 'Authorization': f'Bearer {self._token_retriever.get_token()}', + }, verify=self._trust_env) + response.raise_for_status() + return response.text + def query_custom_properties(self): if self.tenant is None or self.tenant_venue is None or self.collection is None: raise ValueError(f'require to set tenant & tenant_venue & collection & granule') @@ -257,6 +273,23 @@ def delete_single_granule(self): response = json.loads(response.text) return response + def add_archive_config(self, daac_config: dict): + if self.tenant is None or self.tenant_venue is None or self.collection is None: + raise ValueError(f'require to set tenant & tenant_venue & collection') + collection_id = ':'.join([self.urn, self.org, self.project, self.tenant, self.tenant_venue, self.get_complete_collection()]) + + request_url = f'{self._uds_url}collections/{collection_id}/archive/' + s = requests.session() + s.trust_env = self._trust_env + response = s.put(url=request_url, headers={ + 'Authorization': f'Bearer {self._token_retriever.get_token()}', + 'Content-Type': 'application/json', + + }, verify=self._trust_env, data = json.dumps(daac_config)) + response.raise_for_status() + response = json.loads(response.text) + return response + def archive_granule(self): if self.tenant is None or self.tenant_venue is None or self.collection is None or self.granule is None: raise ValueError(f'require to set tenant & tenant_venue & collection & granule') diff --git a/tests/mdps_ds_lib/ds_client/test_ds_client_unity.py b/tests/mdps_ds_lib/ds_client/test_ds_client_unity.py index dd25835..325c7f9 100644 --- a/tests/mdps_ds_lib/ds_client/test_ds_client_unity.py +++ b/tests/mdps_ds_lib/ds_client/test_ds_client_unity.py @@ -1,8 +1,11 @@ import json import os +from datetime import datetime +from time import sleep from unittest import TestCase from dotenv import load_dotenv +from requests import HTTPError from mdps_ds_lib.ds_client.auth_token.token_abstract import TokenAbstract from mdps_ds_lib.ds_client.auth_token.token_factory import TokenFactory @@ -16,26 +19,67 @@ def setUp(self) -> None: load_dotenv() def test_01_admin(self): os.environ['TRUST_ENV'] = 'TRUE' - os.environ['PASSWORD_TYPE'] = 'PARAM_STORE' - os.environ['USERNAME'] = '/unity/uds/user/wphyo/username' - os.environ['PASSWORD'] = '/unity/uds/user/wphyo/dwssap' - os.environ['CLIENT_ID'] = '71g0c73jl77gsqhtlfg2ht388c' - os.environ['COGNITO_URL'] = 'https://cognito-idp.us-west-2.amazonaws.com' - os.environ['TOKEN_FACTORY'] = 'COGNITO' token_retriever: TokenAbstract = TokenFactory().get_instance(os.getenv('TOKEN_FACTORY')) client = DsClientAdmin(token_retriever, 'https://d3vc8w9zcq658.cloudfront.net', 'data-sbx') # client.setup_database() - client.urn = 'urn' - client.org = 'nasa' - client.project = 'unity' - client.tenant = 'UDS_LOCAL_TEST' + client.urn = 'URN' + client.org = 'NASA' + client.project = 'UNITY' + client.tenant = 'UDS_LOCAL_TEST_3' client.tenant_venue = 'DEV' client.add_admin_group(['CREATE', 'READ', 'DELETE'], 'Unity_Viewer') return + def test_1001_temp(self): + """ + curl -v -L -X POST 'https://www.dev.mdps.mcp.nasa.gov:4443/stac_fast_api/collections' \ + -H 'Content-Type:application/json' \ + -H 'cookie: mod_auth_openidc_session=01dc038a-9e14-4f88-b034-4762603729d3' \ +-d @/tmp/sample.json + :return: + """ + import requests + from mdps_ds_lib.lib.utils.file_utils import FileUtils + my_collection = 'URN:NASA:UNITY:UDS_LOCAL_TEST_3:DEV:DDD-01___001' + my_session = 'af448dc3-936c-4e28-af89-9ec086535452' + for i in range(1, 21): + tempt1 = False + granules = FileUtils.read_json(f'/tmp/sample_granules_{i}.json') + while tempt1 is False: + s = requests.session() + s.trust_env = True + response = s.post(url=f'https://www.dev.mdps.mcp.nasa.gov:4443/stac_fast_api/collections/{my_collection}/items', headers={ + 'cookie': f'mod_auth_openidc_session={my_session}', + 'Content-Type': 'application/json', + }, verify=True, data=json.dumps(granules)) + tempt1 = response.status_code != 404 + print(f'{i} = {response.status_code} : {response.text}') + return + + + def test_02_custom_metadata(self): + os.environ['TRUST_ENV'] = 'TRUE' + os.environ['TOKEN_FACTORY'] = 'COGNITO' + token_retriever: TokenAbstract = TokenFactory().get_instance(os.getenv('TOKEN_FACTORY')) + client = DsClientAdmin(token_retriever, 'https://d3vc8w9zcq658.cloudfront.net', 'data-sbx') + + client.urn = 'URN' + client.org = 'NASA' + client.project = 'UNITY' + client.tenant = 'UDS_LOCAL_TEST_3' + client.tenant_venue = 'DEV' + + client.add_tenant_database_index({ + 'tag': {'type': 'keyword'}, + 'c_data1': {'type': 'long'}, + 'c_data2': {'type': 'boolean'}, + 'c_data3': {'type': 'keyword'}, + }) + return + def test_query_granules_across_collections(self): os.environ['TRUST_ENV'] = 'TRUE' os.environ['TOKEN_FACTORY'] = 'COGNITO' @@ -77,6 +121,47 @@ def test_query_single_collection(self): print(client.query_single_collection()) return + def test_delete_collection(self): + os.environ['TRUST_ENV'] = 'TRUE' + os.environ['TOKEN_FACTORY'] = 'COGNITO' + token_retriever: TokenAbstract = TokenFactory().get_instance(os.getenv('TOKEN_FACTORY')) + client = DsClientUser(token_retriever, 'https://d3vc8w9zcq658.cloudfront.net', 'data-sbx') + + client.urn = 'URN' + client.org = 'NASA' + client.project = 'UNITY' + client.tenant = 'UDS_LOCAL_TEST_3' + client.tenant_venue = 'DEV' + client.collection = 'DDD-01' + client.collection_venue = '001' + + with self.assertRaises(HTTPError) as context: + result = client.delete_collection() # bbox='-114,32.5,-113,33.5' + print(context) + self.assertEqual(context.exception.response.status_code, 409) + return + + def test_create_delete_empty_collection(self): + os.environ['TRUST_ENV'] = 'TRUE' + os.environ['TOKEN_FACTORY'] = 'COGNITO' + token_retriever: TokenAbstract = TokenFactory().get_instance(os.getenv('TOKEN_FACTORY')) + client = DsClientUser(token_retriever, 'https://d3vc8w9zcq658.cloudfront.net', 'data-sbx') + + client.urn = 'URN' + client.org = 'NASA' + client.project = 'UNITY' + client.tenant = 'UDS_LOCAL_TEST_3' + client.tenant_venue = 'DEV' + client.collection = 'DDD-01' + client.collection_venue = datetime.now().strftime("%Y%m%d%H%M%S") + + result = client.create_new_collection(False) + print(result) + sleep(70) + result = client.delete_collection() # bbox='-114,32.5,-113,33.5' + print(result) + return + def test_query_granules(self): os.environ['TRUST_ENV'] = 'TRUE' os.environ['TOKEN_FACTORY'] = 'COGNITO' @@ -84,6 +169,7 @@ def test_query_granules(self): client = DsClientUser(token_retriever, 'https://d3vc8w9zcq658.cloudfront.net', 'data-sbx') # client = DsClientAdmin(token_retriever, 'http://localhost:8005', 'data') + # URN:NASA:UNITY:UDS_LOCAL_TEST_3:DEV:DDD-01___001 client.urn = 'URN' client.org = 'NASA' client.project = 'UNITY' @@ -92,8 +178,19 @@ def test_query_granules(self): client.collection = 'DDD-01' client.collection_venue = '001' result = client.query_granules(sort_keys='+properties.datetime,-id') # bbox='-114,32.5,-113,33.5' - print(result) - print(client.query_granules_next()) + + i = 1 + for each in result['features']: + from mdps_ds_lib.lib.utils.file_utils import FileUtils + FileUtils.write_json(f'/tmp/sample_granules_{i}.json', each, overwrite=True, prettify=True) + i += 1 + for each in client.query_granules_next()['features']: + from mdps_ds_lib.lib.utils.file_utils import FileUtils + FileUtils.write_json(f'/tmp/sample_granules_{i}.json', each, overwrite=True, prettify=True) + i += 1 + + # print(json.dumps(result, indent=4)) + # print(json.dumps(client.query_granules_next(), indent=4)) return def test_query_granules02(self): @@ -225,6 +322,23 @@ def test_query_granules01(self): print(json.dumps(result, indent=4)) return + def test_query_single_granule01(self): + os.environ['TRUST_ENV'] = 'TRUE' + os.environ['TOKEN_FACTORY'] = 'COGNITO' + token_retriever: TokenAbstract = TokenFactory().get_instance(os.getenv('TOKEN_FACTORY')) + client = DsClientUser(token_retriever, 'https://api.test.mdps.mcp.nasa.gov', 'am-uds-dapa') + client.urn = 'URN' + client.org = 'NASA' + client.project = 'UNITY' + client.tenant = 'unity' + client.tenant_venue = 'test' + client.collection = 'TRPSDL2ALLCRS1MGLOS' + client.collection_venue = '2' + client.granule = 'TROPESS_CrIS-JPSS1_L2_Standard_TATM_20250108_MUSES_R1p23_megacity_los_angeles_MGLOS_F2p5_J0' + # urn:nasa:unity:uds_local_test:DEV1:CHRP_16_DAY_REBIN___10:SNDR.SS1330.CHIRP.20230101T0000.m06.g001.L1_J1.std.v02_48.G.200101070318_REBIN + print(client.query_single_granule()) + return + def test_archive_one(self): os.environ['TRUST_ENV'] = 'TRUE' # https://api.test.mdps.mcp.nasa.gov/am-uds-dapa/collections/URN:NASA:UNITY:unity:test:TRPSDL2ALLCRS1MGLOS___2/items @@ -241,6 +355,37 @@ def test_archive_one(self): client.collection = 'TRPSDL2ALLCRS1MGLOS' client.collection_venue = '2' client.granule = 'TROPESS_CrIS-JPSS1_L2_Standard_NH3_20250108_MUSES_R1p23_megacity_los_angeles_MGLOS_F2p5_J0' - + client.granule = 'TROPESS_CrIS-JPSS1_L2_Standard_TATM_20250108_MUSES_R1p23_megacity_los_angeles_MGLOS_F2p5_J0' print(client.archive_granule()) - return \ No newline at end of file + return + + def test_add_archive_config(self): + os.environ['TRUST_ENV'] = 'TRUE' + # https://api.test.mdps.mcp.nasa.gov/am-uds-dapa/collections/URN:NASA:UNITY:unity:test:TRPSDL2ALLCRS1MGLOS___2/items + os.environ['TOKEN_FACTORY'] = 'COGNITO' + token_retriever: TokenAbstract = TokenFactory().get_instance(os.getenv('TOKEN_FACTORY')) + client = DsClientUser(token_retriever, 'https://d3vc8w9zcq658.cloudfront.net', 'data-sbx') + # client = DsClientAdmin(token_retriever, 'http://localhost:8005', 'data') + + client.urn = 'URN' + client.org = 'NASA' + client.project = 'UNITY' + client.tenant = 'UDS_LOCAL_TEST_3' # 'uds_local_test' # 'uds_sandbox' + client.tenant_venue = 'DEV' + client.collection = 'TRPSDL2ALLCRS1MGLOS' + client.collection_venue = '2' + daac_config = { + "daac_collection_id": f"daac-mock-collection", + "daac_provider": f"daac-provider--mock-collection", + "daac_sns_topic_arn": "arn:aws:sns:us-west-2:561555463819:uds-test-cumulus-mock_daac_cnm_sns", + "daac_role_arn": "mock", + "daac_role_session_name": "mock", + "daac_data_version": "123", + "archiving_types": [ + {"data_type": "data", "file_extension": [".json", ".nc"]}, + {"data_type": "metadata", "file_extension": [".xml"]}, + {"data_type": "browse"} + ] + } + print(client.add_archive_config(daac_config)) + return