feat: DS Client Library (#94)

* feat: add AUDIT log level for upload

* chore: update outdated tests

* fix: allow empty str as RESULT_PATH_PREFIX & replace w/ default val

* fix: add option to run no ssl es for demo purposes

* feat: DS python client

* feat: add method to query granules

* fix: client example / test case + passing sort keys

Author: wphyojpl

mdps_ds_lib/ds_client/__init__.py

@@ -0,0 +1,7 @@
+from abc import ABC, abstractmethod
+
+
+class TokenAbstract(ABC):
+    @abstractmethod
+    def get_token(self):
+        return ''

mdps_ds_lib/ds_client/auth_token/__init__.py

@@ -0,0 +1,12 @@
+import os
+
+from mdps_ds_lib.ds_client.auth_token.token_abstract import TokenAbstract
+from mdps_ds_lib.lib.cognito_login.cognito_token_retriever import CognitoTokenRetriever
+
+
+class TokenCognito(TokenAbstract):
+    def get_token(self):
+        token_retriever = CognitoTokenRetriever()
+        token = token_retriever.start()
+        header = {'Authorization': f'Bearer {token}'}
+        return token

mdps_ds_lib/ds_client/auth_token/token_abstract.py

@@ -0,0 +1,11 @@
+import os
+
+from mdps_ds_lib.ds_client.auth_token.token_abstract import TokenAbstract
+
+
+class TokenDummy(TokenAbstract):
+    def get_token(self):
+        env_token = os.getenv('DS_TOKEN', None)
+        if env_token is None:
+            raise ValueError(f'missing env_token DS_TOKEN')
+        return env_token

mdps_ds_lib/ds_client/auth_token/token_cognito.py

@@ -0,0 +1,16 @@
+from mdps_ds_lib.lib.utils.factory_abstract import FactoryAbstract
+
+
+class TokenFactory(FactoryAbstract):
+    DUMMY = 'DUMMY'
+    COGNITO = 'COGNITO'
+
+    def get_instance(self, class_type, **kwargs):
+        ct = class_type.upper()
+        if ct == self.DUMMY:
+            from mdps_ds_lib.ds_client.auth_token.token_dummy import TokenDummy
+            return TokenDummy()
+        if ct == self.COGNITO:
+            from mdps_ds_lib.ds_client.auth_token.token_cognito import TokenCognito
+            return TokenCognito()
+        raise ModuleNotFoundError(f'cannot find ES class for {ct}')

mdps_ds_lib/ds_client/auth_token/token_dummy.py

@@ -0,0 +1,148 @@
+import json
+import logging
+
+import requests
+
+from mdps_ds_lib.ds_client.auth_token.token_abstract import TokenAbstract
+from mdps_ds_lib.ds_client.ds_client_base import DsClient
+LOGGER = logging.getLogger(__name__)
+
+
+class DsClientAdmin(DsClient):
+
+
+    def __init__(self, token_retriever: TokenAbstract, ds_url: str, ds_stage: str):
+        super().__init__(token_retriever, ds_url, ds_stage)
+
+    def add_admin_group(self, crud_actions: list, group_name: str):
+        request_url = f'{self._uds_url}admin/auth'
+        LOGGER.debug(f'request_url: {request_url}')
+        collection_complete_name = '.*' if self.collection is None else \
+            f'{self.collection}.*' if self.collection_venue is None else f'{self.collection}___{self.collection_venue}'
+        resources = [self.urn, self.org, self.project,
+                     self.tenant if self.tenant is not None else '*',
+                     self.tenant_venue if self.tenant_venue is not None else '*',
+                     collection_complete_name
+                     ]
+        admin_add_body = {
+            "actions": [k.upper() for k in crud_actions],
+            "resources": [':'.join(resources)],
+            "tenant": self.tenant,
+            "venue": self.tenant_venue,
+            "group_name": group_name
+        }
+        LOGGER.debug(f"admin_add_body: {admin_add_body}")
+        s = requests.session()
+        s.trust_env = self._trust_env
+        response = s.put(url=request_url, headers={
+            'Authorization': f'Bearer {self._token_retriever.get_token()}',
+            'Content-Type': 'application/json',
+        }, verify=self._trust_env, data=json.dumps(admin_add_body))
+        response.raise_for_status()
+        return response.text
+
+    def delete_admin_group(self, group_name: str):
+        request_url = f'{self._uds_url}admin/auth'
+        LOGGER.debug(f'request_url: {request_url}')
+        admin_delete_body = {
+            "tenant": self.tenant,
+            "venue": self.tenant_venue,
+            "group_name": group_name
+        }
+        LOGGER.debug(f"admin_add_body: {admin_delete_body}")
+        s = requests.session()
+        s.trust_env = self._trust_env
+        response = s.delete(url=request_url, headers={
+            'Authorization': f'Bearer {self._token_retriever.get_token()}',
+            'Content-Type': 'application/json',
+        }, verify=self._trust_env, data=json.dumps(admin_delete_body))
+        response.raise_for_status()
+        return response.text
+
+    def list_admin_group(self, group_names: list=[]):
+        request_url = f'{self._uds_url}admin/auth'
+        params = [
+            f'tenant={self.tenant}',
+            f'venue={self.tenant_venue}',
+        ]
+        if len(group_names) < 1:
+            params.append(f'group_names={",".join(group_names)}')
+        params = '&'.join(params)
+        request_url = f'{request_url}?{params}'
+        LOGGER.debug(f'request_url: {request_url}')
+        s = requests.session()
+        s.trust_env = self._trust_env
+        response = s.get(url=request_url, headers={
+            'Authorization': f'Bearer {self._token_retriever.get_token()}',
+            'Content-Type': 'application/json',
+        }, verify=self._trust_env)
+        response.raise_for_status()
+        return json.loads(response.text)
+
+    def update_admin_group(self, crud_actions: list, group_name: str):
+        raise NotImplementedError(f'TO DO')
+
+    def setup_database(self):
+        es_setup_url = f'{self._uds_url}admin/system/es_setup/'
+        print(f'es_setup_url: {es_setup_url}')
+        s = requests.session()
+        s.trust_env = self._trust_env
+        response = s.put(url=es_setup_url, headers={'Authorization': f'Bearer {self._token_retriever.get_token()}'}, verify=self._trust_env)
+        response.raise_for_status()
+        return response.text
+
+    def add_tenant_database_index(self, custom_metadata_es_index: dict):
+        request_url = f'{self._uds_url}admin/custom_metadata/{self.tenant}/'
+        if self.tenant_venue is not None:
+            request_url = f'{request_url}?venue={self.tenant_venue}'
+        print(f'request_url: {request_url}')
+        s = requests.session()
+        s.trust_env = self._trust_env
+        response = s.put(url=request_url, headers={
+            'Authorization': f'Bearer {self._token_retriever.get_token()}',
+            'Content-Type': 'application/json',
+        }, verify=self._trust_env, data=json.dumps(custom_metadata_es_index))
+        response.raise_for_status()
+        return response.text
+
+    def get_tenant_database_index(self):
+        request_url = f'{self._uds_url}admin/custom_metadata/{self.tenant}/'
+        if self.tenant_venue is not None:
+            request_url = f'{request_url}?venue={self.tenant_venue}'
+        print(f'request_url: {request_url}')
+        s = requests.session()
+        s.trust_env = self._trust_env
+        response = s.get(url=request_url, headers={
+            'Authorization': f'Bearer {self._token_retriever.get_token()}',
+            'Content-Type': 'application/json',
+        }, verify=self._trust_env)
+        response.raise_for_status()
+        return json.loads(response.text)
+
+    def delete_tenant_database_index(self):
+        request_url = f'{self._uds_url}admin/custom_metadata/{self.tenant}/'
+        if self.tenant_venue is not None:
+            request_url = f'{request_url}?venue={self.tenant_venue}'
+        print(f'request_url: {request_url}')
+        s = requests.session()
+        s.trust_env = self._trust_env
+        response = s.delete(url=request_url, headers={
+            'Authorization': f'Bearer {self._token_retriever.get_token()}',
+            'Content-Type': 'application/json',
+        }, verify=self._trust_env)
+        response.raise_for_status()
+        return response.text
+
+    def destroy_tenant_database_index(self):
+        request_url = f'{self._uds_url}admin/custom_metadata/{self.tenant}/destroy/'
+        if self.tenant_venue is not None:
+            request_url = f'{request_url}?venue={self.tenant_venue}'
+        print(f'request_url: {request_url}')
+        s = requests.session()
+        s.trust_env = self._trust_env
+        response = s.delete(url=request_url, headers={
+            'Authorization': f'Bearer {self._token_retriever.get_token()}',
+            'Content-Type': 'application/json',
+        }, verify=self._trust_env)
+        response.raise_for_status()
+        return response.text

mdps_ds_lib/ds_client/auth_token/token_factory.py

@@ -0,0 +1,125 @@
+import json
+import logging
+import os
+
+import requests
+
+from dotenv import load_dotenv
+from mdps_ds_lib.ds_client.auth_token.token_abstract import TokenAbstract
+from mdps_ds_lib.ds_client.auth_token.token_factory import TokenFactory
+LOGGER = logging.getLogger(__name__)
+
+
+class DsClient:
+    def __init__(self, token_retriever: TokenAbstract, ds_url: str, ds_stage: str):
+        load_dotenv()
+        self._trust_env = os.getenv('TRUST_ENV', 'FALSE').upper().strip() == 'TRUE'
+        self._token_retriever = token_retriever
+        self._uds_url = f'{ds_url}/{ds_stage}/'
+        self.__urn, self.__org, self.__project = 'URN', 'NASA', 'UNITY'
+        self.__tenant, self.__tenant_venue, self.__collection, self.__collection_venue = None, None, None, None
+        self.__granule = None
+        self.__default_collection_venue = '001'
+
+    def get_complete_collection(self):
+        return f'{self.collection}{self.__default_collection_venue}' if self.collection_venue is None else f'{self.collection}___{self.collection_venue}'
+
+    @property
+    def granule(self):
+        return self.__granule
+
+    @granule.setter
+    def granule(self, val):
+        """
+        :param val:
+        :return: None
+        """
+        self.__granule = val
+        return
+
+    @property
+    def project(self):
+        return self.__project
+
+    @project.setter
+    def project(self, val):
+        """
+        :param val:
+        :return: None
+        """
+        self.__project = val
+        return
+    @property
+    def urn(self):
+        return self.__urn
+
+    @urn.setter
+    def urn(self, val):
+        """
+        :param val:
+        :return: None
+        """
+        self.__urn = val
+        return
+    @property
+    def org(self):
+        return self.__org
+
+    @org.setter
+    def org(self, val):
+        """
+        :param val:
+        :return: None
+        """
+        self.__org = val
+        return
+    @property
+    def tenant(self):
+        return self.__tenant
+
+    @tenant.setter
+    def tenant(self, val):
+        """
+        :param val:
+        :return: None
+        """
+        self.__tenant = val
+        return
+    @property
+    def tenant_venue(self):
+        return self.__tenant_venue
+
+    @tenant_venue.setter
+    def tenant_venue(self, val):
+        """
+        :param val:
+        :return: None
+        """
+        self.__tenant_venue = val
+        return
+    @property
+    def collection(self):
+        return self.__collection
+
+    @collection.setter
+    def collection(self, val):
+        """
+        :param val:
+        :return: None
+        """
+        self.__collection = val
+        return
+    @property
+    def collection_venue(self):
+        return self.__collection_venue
+
+    @collection_venue.setter
+    def collection_venue(self, val):
+        """
+        :param val:
+        :return: None
+        """
+        self.__collection_venue = val
+        return
+
+