Merge pull request #545 from unity-sds/develop

release/9.8.1

Author: ngachung

CHANGELOG.md

@@ -5,6 +5,18 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## [9.8.1] - 2025-02-19
+### Fixed
+- [#535](https://github.com/unity-sds/unity-data-services/pull/535) fix: sorting key order was not kept previously
+
+## [9.8.0] - 2025-02-10
+### Changed
+- [#532](https://github.com/unity-sds/unity-data-services/pull/532) feat: granules sort by time or other properties
+
+## [9.7.0] - 2025-02-10
+### Changed
+- [#521](https://github.com/unity-sds/unity-data-services/pull/521) feat: gemx branch
+
 ## [9.6.0] - 2025-02-03
 ### Changed
 - [#517](https://github.com/unity-sds/unity-data-services/pull/517) feat: stac browser oidc cookie

README.md

@@ -111,7 +111,7 @@ Questions about our project? Please see our: [FAQ]([INSERT LINK TO FAQ / DISCUSS
 1. Question 1
    - Answer to question 1.
 2. Question 2
-   - Answer to question 2
+   - Answer to question 2.
 -->
 
 <!-- example FAQ inline with no questions yet>

cumulus_lambda_functions/cumulus_es_setup/es_setup.py

@@ -16,9 +16,10 @@ def __init__(self):
         required_env = ['ES_URL']
         if not all([k in os.environ for k in required_env]):
             raise EnvironmentError(f'one or more missing env: {required_env}')
-        self.__es: ESAbstract = ESFactory().get_instance('AWS',
+        self.__es: ESAbstract = ESFactory().get_instance(os.getenv('ES_TYPE', 'AWS'),
                                                          index=DBConstants.collections_index,
                                                          base_url=os.getenv('ES_URL'),
+                                                         use_ssl=os.getenv('ES_USE_SSL', 'TRUE').strip() is True,
                                                          port=int(os.getenv('ES_PORT', '443'))
                                                          )
 

cumulus_lambda_functions/cumulus_wrapper/query_granules.py

@@ -132,6 +132,40 @@ def query_direct_to_private_api(self, private_api_prefix: str, transform=True):
             return {'server_error': f'error while invoking:{str(e)}'}
         return {'results': stac_list}
 
+    def add_entry(self, private_api_prefix: str, new_granule: dict):
+        raise NotImplementedError(f'Please implement adding granules to Cumulus')
+        # https://nasa.github.io/cumulus-api/v18.4.0/#create-granule
+        # payload = {
+        #     'httpMethod': 'POST',
+        #     'resource': '/{proxy+}',
+        #     'path': f'/{self.__collections_key}',
+        #     'headers': {
+        #         'Content-Type': 'application/json',
+        #     },
+        #     'body': json.dumps(new_granule)
+        # }
+        # LOGGER.debug(f'payload: {payload}')
+        # try:
+        #     query_result = self._invoke_api(payload, private_api_prefix)
+        #     """
+        #     {'statusCode': 500, 'body': '', 'headers': {}}
+        #     """
+        #     if query_result['statusCode'] >= 500:
+        #         LOGGER.error(f'server error status code: {query_result["statusCode"]}. details: {query_result}')
+        #         return {'server_error': query_result}
+        #     if query_result['statusCode'] >= 400:
+        #         LOGGER.error(f'client error status code: {query_result["statusCode"]}. details: {query_result}')
+        #         return {'client_error': query_result}
+        #     query_result = json.loads(query_result['body'])
+        #     LOGGER.debug(f'json query_result: {query_result}')
+        #     if 'message' not in query_result:
+        #         return {'server_error': f'invalid response: {query_result}'}
+        # except Exception as e:
+        #     LOGGER.exception('error while invoking')
+        #     return {'server_error': f'error while invoking:{str(e)}'}
+        # return {'status': query_result['message']}
+        return
+
     def delete_entry(self, private_api_prefix: str, granule_id: str):
         payload = {
             'httpMethod': 'DELETE',

cumulus_lambda_functions/daac_archiver/daac_archiver_logic.py

@@ -23,7 +23,7 @@
 class DaacArchiverLogic:
     def __init__(self):
         self.__es_url, self.__es_port = os.getenv('ES_URL'), int(os.getenv('ES_PORT', '443'))
-        self.__archive_index_logic = UdsArchiveConfigIndex(self.__es_url, self.__es_port)
+        self.__archive_index_logic = UdsArchiveConfigIndex(self.__es_url, self.__es_port, os.getenv('ES_TYPE', 'AWS'), os.getenv('ES_USE_SSL', 'TRUE').strip() is True)
         self.__granules_index = GranulesDbIndex()
         self.__sns = AwsSns()
         self.__s3 = AwsS3()

cumulus_lambda_functions/granules_cnm_ingester/granules_cnm_ingester_logic.py

@@ -40,7 +40,7 @@ def __init__(self):
         if 'UNITY_DEFAULT_PROVIDER' not in os.environ:
             raise ValueError(f'missing UNITY_DEFAULT_PROVIDER')
         self.__default_provider = os.environ.get('UNITY_DEFAULT_PROVIDER')
-        self.__uds_collection = UdsCollections(es_url=os.getenv('ES_URL'), es_port=int(os.getenv('ES_PORT', '443')))
+        self.__uds_collection = UdsCollections(es_url=os.getenv('ES_URL'), es_port=int(os.getenv('ES_PORT', '443')), es_type=os.getenv('ES_TYPE', 'AWS'))
 
     @property
     def successful_features_json(self):

cumulus_lambda_functions/lib/authorization/uds_authorizer_es_identity_pool.py

@@ -12,11 +12,12 @@
 
 class UDSAuthorizorEsIdentityPool(UDSAuthorizorAbstract):
 
-    def __init__(self, es_url: str, es_port=443) -> None:
+    def __init__(self, es_url: str, es_port=443, es_type='AWS', use_ssl=True) -> None:
         super().__init__()
-        self.__es: ESAbstract = ESFactory().get_instance('AWS',
+        self.__es: ESAbstract = ESFactory().get_instance(es_type,
                                                          index=DBConstants.authorization_index,
                                                          base_url=es_url,
+                                                         use_ssl=use_ssl,
                                                          port=es_port)
 
     def add_authorized_group(self, action: [str], resource: [str], tenant: str, venue: str, ldap_group_name: str):

cumulus_lambda_functions/lib/authorization/uds_authorizer_factory.py

@@ -1,12 +1,18 @@
+import os
+
 from mdps_ds_lib.lib.utils.factory_abstract import FactoryAbstract
 
 
 class UDSAuthorizerFactory(FactoryAbstract):
     cognito = 'COGNITO'
 
     def get_instance(self, class_type, **kwargs):
+        if 'use_ssl' not in kwargs:
+            kwargs['use_ssl'] = os.getenv('ES_USE_SSL', 'TRUE').strip() is True
+        if 'es_type' not in kwargs:
+            kwargs['es_type'] = os.getenv('ES_TYPE', 'AWS')
         if class_type == self.cognito:
             from cumulus_lambda_functions.lib.authorization.uds_authorizer_es_identity_pool import \
                 UDSAuthorizorEsIdentityPool
-            return UDSAuthorizorEsIdentityPool(kwargs['es_url'], kwargs['es_port'])
+            return UDSAuthorizorEsIdentityPool(kwargs['es_url'], kwargs['es_port'], kwargs['es_type'], kwargs['use_ssl'])
         raise ValueError(f'class_type: {class_type} not implemented')

cumulus_lambda_functions/lib/uds_db/archive_index.py

@@ -25,10 +25,12 @@ class UdsArchiveConfigIndex:
             'archiving_types': {'type': 'array', 'items': {'type': 'object'}},
         }
     }
-    def __init__(self, es_url, es_port=443):
-        self.__es: ESAbstract = ESFactory().get_instance('AWS',
+
+    def __init__(self, es_url, es_port=443, es_type='AWS', use_ssl=True):
+        self.__es: ESAbstract = ESFactory().get_instance(es_type,
                                                          index='TODO',
                                                          base_url=es_url,
+                                                         use_ssl=use_ssl,
                                                          port=es_port)
         self.__tenant, self.__venue = '', ''
 

cumulus_lambda_functions/lib/uds_db/granules_db_index.py

@@ -19,10 +19,11 @@ def __init__(self):
         required_env = ['ES_URL']
         if not all([k in os.environ for k in required_env]):
             raise EnvironmentError(f'one or more missing env: {required_env}')
-        self.__es: ESAbstract = ESFactory().get_instance('AWS',
+        self.__es: ESAbstract = ESFactory().get_instance(os.getenv('ES_TYPE', 'AWS'),
                                                          index=DBConstants.collections_index,
                                                          base_url=os.getenv('ES_URL'),
-                                                         port=int(os.getenv('ES_PORT', '443'))
+                                                         port=int(os.getenv('ES_PORT', '443')),
+                                                         use_ssl=os.getenv('ES_USE_SSL', 'TRUE').strip() is True,
                                                          )
         # self.__default_fields = {
         #     "granule_id": {"type": "keyword"},
@@ -36,8 +37,11 @@ def __init__(self):
     def to_es_bbox(bbox_array):
         # lon = x, lat = y
         # lon, lat, lon, lat
+        # -180, -90, 180, 90
         # x can be 170 to -170
         # 170, 0, -170, 10
+        # latitude must be between -90.0 and 90.0
+        # longitude must be between -180.0 and 180.0
         minX, minY, maxX, maxY = bbox_array
 
         # Ensure the values are properly sorted