diff --git a/mdps_ds_lib/ds_client/ds_client_user.py b/mdps_ds_lib/ds_client/ds_client_user.py index 6e5126a..16f4f14 100644 --- a/mdps_ds_lib/ds_client/ds_client_user.py +++ b/mdps_ds_lib/ds_client/ds_client_user.py @@ -17,6 +17,7 @@ class DsClientUser(DsClient): def __init__(self, token_retriever: TokenAbstract, ds_url: str, ds_stage: str): super().__init__(token_retriever, ds_url, ds_stage) self.__granule_query_next_page = None + self.__collection_query_next_page = None def create_new_collection(self, is_actual_execution=False): request_url = f'{self._uds_url}collections/' @@ -51,11 +52,75 @@ def create_new_collection(self, is_actual_execution=False): response.raise_for_status() return response.text - def query_collections(self): - return + def query_custom_properties(self): + if self.tenant is None or self.tenant_venue is None or self.collection is None: + raise ValueError(f'require to set tenant & tenant_venue & collection & granule') + collection_id_for_granules = ':'.join([self.urn, self.org, self.project, self.tenant, self.tenant_venue, self.get_complete_collection()]) + request_url = f'{self._uds_url}collections/' + request_url = f'{request_url}{collection_id_for_granules}/variables' + print(request_url) + s = requests.session() + s.trust_env = self._trust_env + response = s.get(url=request_url, headers={ + 'Authorization': f'Bearer {self._token_retriever.get_token()}', + }, verify=self._trust_env) + response.raise_for_status() + response = json.loads(response.text) + return response + + def query_collections(self, limit=10): + query_params = { + # 'datetime': datetime, + 'limit': limit, + # 'filter': filter, + # 'bbox': bbox, + # 'sortby': sort_keys, + } + query_params = {k: v for k, v in query_params.items() if v is not None} + request_url = f'{self._uds_url}collections/' + query_params = '&'.join([f'{k}={v}' for k, v in query_params.items()]) + request_url = f'{request_url}?{query_params}' + s = requests.session() + s.trust_env = self._trust_env + response = s.get(url=request_url, headers={ + 'Authorization': f'Bearer {self._token_retriever.get_token()}', + }, verify=self._trust_env) + response.raise_for_status() + response = json.loads(response.text) + self.__collection_query_next_page = [k['href'] for k in response['links'] if k['rel'] == 'next'] + self.__collection_query_next_page = None if len(self.__collection_query_next_page) < 1 else \ + self.__collection_query_next_page[0] + + return response + + def query_collections_next(self): + if self.__collection_query_next_page is None: + return None + s = requests.session() + s.trust_env = self._trust_env + response = s.get(url=self.__collection_query_next_page, headers={ + 'Authorization': f'Bearer {self._token_retriever.get_token()}', + }, verify=self._trust_env) + response.raise_for_status() + response = json.loads(response.text) + self.__collection_query_next_page = [k['href'] for k in response['links'] if k['rel'] == 'next'] + self.__collection_query_next_page = None if len(self.__collection_query_next_page) < 1 else self.__collection_query_next_page[0] + return response - def query_single_collection(self, limit= 10, offset= None, datetime = None, filter = None): - return + def query_single_collection(self): + if self.tenant is None or self.tenant_venue is None or self.collection is None: + raise ValueError(f'require to set tenant & tenant_venue & collection') + collection_id_for_granules = ':'.join([self.urn, self.org, self.project, self.tenant, self.tenant_venue, self.get_complete_collection()]) + request_url = f'{self._uds_url}collections/' + request_url = f'{request_url}{collection_id_for_granules}/' + s = requests.session() + s.trust_env = self._trust_env + response = s.get(url=request_url, headers={ + 'Authorization': f'Bearer {self._token_retriever.get_token()}', + }, verify=self._trust_env) + response.raise_for_status() + response = json.loads(response.text) + return response def create_new_granule(self, granule_stac: Item): request_url = f'{self._uds_url}collections/' @@ -76,7 +141,6 @@ def create_new_granule(self, granule_stac: Item): response.raise_for_status() return response.text - def query_granules_across_collections(self, limit= 10, datetime = None, filter = None, bbox= None, sort_keys=None): if self.tenant is None or self.tenant_venue is None: raise ValueError(f'require to set tenant & tenant_venue') @@ -147,37 +211,37 @@ def query_granules(self, limit= 10, datetime = None, filter = None, bbox= None, self.__granule_query_next_page = None if len(self.__granule_query_next_page) < 1 else \ self.__granule_query_next_page[0] return response - return def query_single_granule(self): - return + if self.tenant is None or self.tenant_venue is None or self.collection is None or self.granule is None: + raise ValueError(f'require to set tenant & tenant_venue & collection & granule') + collection_id_for_granules = ':'.join([self.urn, self.org, self.project, self.tenant, self.tenant_venue, self.get_complete_collection()]) + granule_id_complete = ':'.join([collection_id_for_granules, self.granule]) + request_url = f'{self._uds_url}collections/' + request_url = f'{request_url}{collection_id_for_granules}/items/{granule_id_complete}' + print(request_url) + s = requests.session() + s.trust_env = self._trust_env + response = s.get(url=request_url, headers={ + 'Authorization': f'Bearer {self._token_retriever.get_token()}', + }, verify=self._trust_env) + response.raise_for_status() + response = json.loads(response.text) + return response def delete_single_granule(self): - return - - def add_admin_group(self, crud_actions: list, group_name: str): - request_url = f'{self._uds_url}admin/auth' - LOGGER.debug(f'request_url: {request_url}') - collection_complete_name = '.*' if self.collection is None else \ - f'{self.collection}.*' if self.collection_venue is None else f'{self.collection}___{self.collection_venue}' - resources = [self.urn, self.org, self.project, - self.tenant if self.tenant is not None else '*', - self.tenant_venue if self.tenant_venue is not None else '*', - collection_complete_name - ] - admin_add_body = { - "actions": [k.upper() for k in crud_actions], - "resources": [','.join(resources)], - "tenant": self.tenant, - "venue": self.tenant_venue, - "group_name": group_name - } - LOGGER.debug(f"admin_add_body: {admin_add_body}") + if self.tenant is None or self.tenant_venue is None or self.collection is None or self.granule is None: + raise ValueError(f'require to set tenant & tenant_venue & collection & granule') + collection_id_for_granules = ':'.join([self.urn, self.org, self.project, self.tenant, self.tenant_venue, self.get_complete_collection()]) + granule_id_complete = ':'.join([collection_id_for_granules, self.granule]) + request_url = f'{self._uds_url}collections/' + request_url = f'{request_url}{collection_id_for_granules}/items/{granule_id_complete}' + print(request_url) s = requests.session() s.trust_env = self._trust_env - response = s.put(url=request_url, headers={ + response = s.delete(url=request_url, headers={ 'Authorization': f'Bearer {self._token_retriever.get_token()}', - 'Content-Type': 'application/json', - }, verify=self._trust_env, data=json.dumps(admin_add_body)) + }, verify=self._trust_env) response.raise_for_status() - return response.text + response = json.loads(response.text) + return response diff --git a/tests/mdps_ds_lib/ds_client/test_ds_client_unity.py b/tests/mdps_ds_lib/ds_client/test_ds_client_unity.py index 1cd5ecd..72ab337 100644 --- a/tests/mdps_ds_lib/ds_client/test_ds_client_unity.py +++ b/tests/mdps_ds_lib/ds_client/test_ds_client_unity.py @@ -1,8 +1,10 @@ +import json import os from unittest import TestCase from mdps_ds_lib.ds_client.auth_token.token_abstract import TokenAbstract from mdps_ds_lib.ds_client.auth_token.token_factory import TokenFactory +from mdps_ds_lib.ds_client.ds_client_admin import DsClientAdmin from mdps_ds_lib.ds_client.ds_client_user import DsClientUser @@ -29,6 +31,43 @@ def test_query_granules_across_collections(self): print(client.query_granules_next()) return + def test_query_collections(self): + os.environ['TRUST_ENV'] = 'TRUE' + os.environ['PASSWORD_TYPE'] = 'PARAM_STORE' + os.environ['USERNAME'] = '/unity/uds/user/wphyo/username' + os.environ['PASSWORD'] = '/unity/uds/user/wphyo/dwssap' + os.environ['CLIENT_ID'] = '71g0c73jl77gsqhtlfg2ht388c' + os.environ['COGNITO_URL'] = 'https://cognito-idp.us-west-2.amazonaws.com' + + os.environ['TOKEN_FACTORY'] = 'COGNITO' + token_retriever: TokenAbstract = TokenFactory().get_instance(os.getenv('TOKEN_FACTORY')) + client = DsClientUser(token_retriever, 'https://d3vc8w9zcq658.cloudfront.net', 'data-sbx') + print(client.query_collections(10)) + print(client.query_collections_next()) + return + + def test_query_single_collection(self): + os.environ['TRUST_ENV'] = 'TRUE' + os.environ['PASSWORD_TYPE'] = 'PARAM_STORE' + os.environ['USERNAME'] = '/unity/uds/user/wphyo/username' + os.environ['PASSWORD'] = '/unity/uds/user/wphyo/dwssap' + os.environ['CLIENT_ID'] = '71g0c73jl77gsqhtlfg2ht388c' + os.environ['COGNITO_URL'] = 'https://cognito-idp.us-west-2.amazonaws.com' + + os.environ['TOKEN_FACTORY'] = 'COGNITO' + token_retriever: TokenAbstract = TokenFactory().get_instance(os.getenv('TOKEN_FACTORY')) + client = DsClientUser(token_retriever, 'https://d3vc8w9zcq658.cloudfront.net', 'data-sbx') + client.urn = 'URN' + client.org = 'NASA' + client.project = 'UNITY' + client.tenant = 'UDS_LOCAL_TEST_3' + client.tenant_venue = 'DEV' + client.collection = 'DDD-01' + client.collection_venue = '001' + + print(client.query_single_collection()) + return + def test_query_granules(self): os.environ['TRUST_ENV'] = 'TRUE' os.environ['PASSWORD_TYPE'] = 'PARAM_STORE' @@ -53,3 +92,136 @@ def test_query_granules(self): print(result) print(client.query_granules_next()) return + + def test_query_granules02(self): + os.environ['TRUST_ENV'] = 'TRUE' + os.environ['PASSWORD_TYPE'] = 'PARAM_STORE' + os.environ['USERNAME'] = '/unity/uds/user/wphyo/username' + os.environ['PASSWORD'] = '/unity/uds/user/wphyo/dwssap' + os.environ['CLIENT_ID'] = '71g0c73jl77gsqhtlfg2ht388c' + os.environ['COGNITO_URL'] = 'https://cognito-idp.us-west-2.amazonaws.com' + + os.environ['TOKEN_FACTORY'] = 'COGNITO' + token_retriever: TokenAbstract = TokenFactory().get_instance(os.getenv('TOKEN_FACTORY')) + client = DsClientUser(token_retriever, 'https://d3vc8w9zcq658.cloudfront.net', 'am-uds-dapa') + # client = DsClientAdmin(token_retriever, 'http://localhost:8005', 'data') + 'uds_local_test:DEV1:CHRP_16_DAY_REBIN___5/items?limit=1000' + client.urn = 'urn' + client.org = 'nasa' + client.project = 'unity' + client.tenant = 'uds_local_test' + client.tenant_venue = 'DEV1' + client.collection = 'CHRP_16_DAY_REBIN' + client.collection_venue = '5' + + client.urn = 'urn' + client.org = 'nasa' + client.project = 'unity' + client.tenant = 'uds_local_test' + client.tenant_venue = 'DEV1' + client.collection = 'CHRP_16_DAY_REBIN' + client.collection_venue = '10' + # 3749 + result = client.query_granules(limit=50) # bbox='-114,32.5,-113,33.5' + print(json.dumps(result, indent=4)) + # print(json.dumps(client.query_granules_next(), indent=4)) + return + + def test_query_custom_properties(self): + os.environ['TRUST_ENV'] = 'TRUE' + os.environ['PASSWORD_TYPE'] = 'PARAM_STORE' + os.environ['USERNAME'] = '/unity/uds/user/wphyo/username' + os.environ['PASSWORD'] = '/unity/uds/user/wphyo/dwssap' + os.environ['CLIENT_ID'] = '71g0c73jl77gsqhtlfg2ht388c' + os.environ['COGNITO_URL'] = 'https://cognito-idp.us-west-2.amazonaws.com' + + os.environ['TOKEN_FACTORY'] = 'COGNITO' + token_retriever: TokenAbstract = TokenFactory().get_instance(os.getenv('TOKEN_FACTORY')) + client = DsClientUser(token_retriever, 'https://d3vc8w9zcq658.cloudfront.net', 'am-uds-dapa') + # client = DsClientAdmin(token_retriever, 'http://localhost:8005', 'data') + 'uds_local_test:DEV1:CHRP_16_DAY_REBIN___5/items?limit=1000' + client.urn = 'urn' + client.org = 'nasa' + client.project = 'unity' + client.tenant = 'uds_local_test' + client.tenant_venue = 'DEV1' + client.collection = 'CHRP_16_DAY_REBIN' + client.collection_venue = '5' + + # 3749 + result = client.query_custom_properties() # bbox='-114,32.5,-113,33.5' + print(json.dumps(result, indent=4)) + # print(json.dumps(client.query_granules_next(), indent=4)) + return + + def test_query_single_granule(self): + os.environ['TRUST_ENV'] = 'TRUE' + os.environ['PASSWORD_TYPE'] = 'PARAM_STORE' + os.environ['USERNAME'] = '/unity/uds/user/wphyo/username' + os.environ['PASSWORD'] = '/unity/uds/user/wphyo/dwssap' + os.environ['CLIENT_ID'] = '71g0c73jl77gsqhtlfg2ht388c' + os.environ['COGNITO_URL'] = 'https://cognito-idp.us-west-2.amazonaws.com' + + os.environ['TOKEN_FACTORY'] = 'COGNITO' + token_retriever: TokenAbstract = TokenFactory().get_instance(os.getenv('TOKEN_FACTORY')) + client = DsClientUser(token_retriever, 'https://d3vc8w9zcq658.cloudfront.net', 'am-uds-dapa') + client.urn = 'urn' + client.org = 'nasa' + client.project = 'unity' + client.tenant = 'uds_local_test' + client.tenant_venue = 'DEV1' + client.collection = 'CHRP_16_DAY_REBIN' + client.collection_venue = '10' + client.granule = 'SNDR.SS1330.CHIRP.20230101T0000.m06.g001.L1_J1.std.v02_48.G.200101070318_REBIN' + # urn:nasa:unity:uds_local_test:DEV1:CHRP_16_DAY_REBIN___10:SNDR.SS1330.CHIRP.20230101T0000.m06.g001.L1_J1.std.v02_48.G.200101070318_REBIN + print(client.query_single_granule()) + return + + + def test_update_admin(self): + os.environ['TRUST_ENV'] = 'TRUE' + os.environ['PASSWORD_TYPE'] = 'PARAM_STORE' + os.environ['USERNAME'] = '/unity/uds/user/wphyo/username' + os.environ['PASSWORD'] = '/unity/uds/user/wphyo/dwssap' + os.environ['CLIENT_ID'] = '71g0c73jl77gsqhtlfg2ht388c' + os.environ['COGNITO_URL'] = 'https://cognito-idp.us-west-2.amazonaws.com' + + os.environ['TOKEN_FACTORY'] = 'COGNITO' + token_retriever: TokenAbstract = TokenFactory().get_instance(os.getenv('TOKEN_FACTORY')) + client = DsClientAdmin(token_retriever, 'https://d3vc8w9zcq658.cloudfront.net', 'data-sbx') + + # URN:NASA:UNITY:UDS_LOCAL_TEST_3:DEV:DDD-01___001:test_file10 + client.urn = 'URN' + client.org = 'NASA' + client.project = 'UNITY' + client.tenant = 'UDS_LOCAL_TEST_3' + client.tenant_venue = 'DEV' + client.collection = 'DDD-01' + client.collection_venue = '001' + client.add_admin_group(['CREATE', 'READ', 'DELETE'], 'wphyo') + return + + def test_delete_single_granule(self): + os.environ['TRUST_ENV'] = 'TRUE' + os.environ['PASSWORD_TYPE'] = 'PARAM_STORE' + os.environ['USERNAME'] = '/unity/uds/user/wphyo/username' + os.environ['PASSWORD'] = '/unity/uds/user/wphyo/dwssap' + os.environ['CLIENT_ID'] = '71g0c73jl77gsqhtlfg2ht388c' + os.environ['COGNITO_URL'] = 'https://cognito-idp.us-west-2.amazonaws.com' + + os.environ['TOKEN_FACTORY'] = 'COGNITO' + token_retriever: TokenAbstract = TokenFactory().get_instance(os.getenv('TOKEN_FACTORY')) + client = DsClientUser(token_retriever, 'https://d3vc8w9zcq658.cloudfront.net', 'data-sbx') + + # URN:NASA:UNITY:UDS_LOCAL_TEST_3:DEV:DDD-01___001:test_file10 + client.urn = 'URN' + client.org = 'NASA' + client.project = 'UNITY' + client.tenant = 'UDS_LOCAL_TEST_3' + client.tenant_venue = 'DEV' + client.collection = 'DDD-01' + client.collection_venue = '001' + client.granule = 'test_file10' + # urn:nasa:unity:uds_local_test:DEV1:CHRP_16_DAY_REBIN___10:SNDR.SS1330.CHIRP.20230101T0000.m06.g001.L1_J1.std.v02_48.G.200101070318_REBIN + print(client.delete_single_granule()) + return