chore: merge from develop

Author: wphyojpl

CHANGELOG.md

@@ -5,6 +5,22 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## [9.4.1] - 2024-12-18
+### Fixed
+- [#489](https://github.com/unity-sds/unity-data-services/pull/489) fix: delete bug
+
+## [9.4.0] - 2024-12-16
+### Changed
+- [#485](https://github.com/unity-sds/unity-data-services/pull/485) feat: delete granule endpoint
+
+## [9.3.0] - 2024-12-11
+### Changed
+- [#482](https://github.com/unity-sds/unity-data-services/pull/482) feat: updated name spaces and naming of resources
+
+## [9.2.0] - 2024-12-09
+### Changed
+- [#478](https://github.com/unity-sds/unity-data-services/pull/478) feat: uds lib update
+
 ## [9.1.2] - 2024-12-09
 ### Fixed
 - [#448](https://github.com/unity-sds/unity-data-services/pull/448) fix: wrong location archive keys

ci.cd/Makefile

@@ -1,64 +0,0 @@
-export IMAGE_PREFIX ?= cae-artifactory.jpl.nasa.gov:16001/gov/nasa/jpl/unity
-export NAME ?= uds
-export VERSION ?= latest
-
-
-all: build_lambda upload_lambda update_lambda_function build_docker
-local: build_lambda upload_lambda
-build_docker:
-	docker build --no-cache -t "$(IMAGE_PREFIX)/$(NAME):$(VERSION)" -f docker/Dockerfile_download_granules.jpl .
-
-zip_docker:
-	docker save "$(IMAGE_PREFIX)/$(NAME):$(VERSION)" | gzip > "$(NAME)__$(VERSION).tar.gz"
-
-build_lambda:
-	docker run --rm -v `PWD`:"/usr/src/app/cumulus_lambda_functions":z -w "/usr/src/app/cumulus_lambda_functions" cae-artifactory.jpl.nasa.gov:17001/python:3.9 ci.cd/create_s3_zip.sh
-
-build_lambda_public:
-	docker run --rm -v `PWD`:"/usr/src/app/cumulus_lambda_functions":z -w "/usr/src/app/cumulus_lambda_functions" python:3.9 ci.cd/create_s3_zip.sh
-
-upload_lambda:
-	aws --profile saml-pub s3 cp cumulus_lambda_functions_deployment.zip s3://am-uds-dev-cumulus-tf-state/unity_cumulus_lambda/
-
-move:
-	 mv /Users/wphyo/Downloads/cumulus_lambda_functions-*.zip tf-module/unity-cumulus/build/cumulus_lambda_functions_deployment.zip
-
-upload_lambda_mcp_dev:
-	aws s3 cp tf-module/unity-cumulus/build/cumulus_lambda_functions_deployment.zip s3://uds-dev-cumulus-public/unity_cumulus_lambda/
-update_lambda_function_mcp_dev_6:
-	aws lambda update-function-code --s3-key unity_cumulus_lambda/cumulus_lambda_functions_deployment.zip --s3-bucket uds-dev-cumulus-public --function-name arn:aws:lambda:us-west-2:237868187491:function:uds-dev-cumulus-metadata_s4pa_generate_cmr --publish &>/dev/null
-update_lambda_function_mcp_dev_7:
-	aws lambda update-function-code --s3-key unity_cumulus_lambda/cumulus_lambda_functions_deployment.zip --s3-bucket uds-dev-cumulus-public --function-name arn:aws:lambda:us-west-2:237868187491:function:uds-dev-cumulus-metadata_stac_generate_cmr --publish &>/dev/null
-update_lambda_function_mcp_dev_auth_list:
-	aws lambda update-function-code --s3-key unity_cumulus_lambda/cumulus_lambda_functions_deployment.zip --s3-bucket uds-dev-cumulus-public --function-name arn:aws:lambda:us-west-2:237868187491:function:uds-dev-cumulus-cumulus_auth_list --publish &>/dev/null
-update_lambda_function_mcp_dev_auth_add:
-	aws lambda update-function-code --s3-key unity_cumulus_lambda/cumulus_lambda_functions_deployment.zip --s3-bucket uds-dev-cumulus-public --function-name arn:aws:lambda:us-west-2:237868187491:function:uds-dev-cumulus-cumulus_auth_add --publish &>/dev/null
-update_lambda_function_mcp_dev_auth_delete:
-	aws lambda update-function-code --s3-key unity_cumulus_lambda/cumulus_lambda_functions_deployment.zip --s3-bucket uds-dev-cumulus-public --function-name arn:aws:lambda:us-west-2:237868187491:function:uds-dev-cumulus-cumulus_auth_delete --publish &>/dev/null
-
-update_lambda_function_mcp_dev_uds_api:
-	aws lambda update-function-code --s3-key unity_cumulus_lambda/cumulus_lambda_functions_deployment.zip --s3-bucket uds-dev-cumulus-public --function-name arn:aws:lambda:us-west-2:237868187491:function:uds-dev-cumulus-uds_api_1 --publish &>/dev/null
-
-update_lambda_function_mcp_sbx_uds_api:
-	aws lambda update-function-code --s3-key unity_cumulus_lambda/cumulus_lambda_functions_deployment.zip --s3-bucket uds-dev-cumulus-public --function-name arn:aws:lambda:us-west-2:237868187491:function:uds-sbx-cumulus-uds_api_1 --publish &>/dev/null
-
-mcp_dev: upload_lambda_mcp_dev update_lambda_function_mcp_dev_auth_list update_lambda_function_mcp_dev_auth_add update_lambda_function_mcp_dev_auth_delete
-
-mcp_dev_fastapi: upload_lambda_mcp_dev update_lambda_function_mcp_dev_uds_api
-
-update_lambda_function_mcp_sbx_7:
-	aws lambda update-function-code --s3-key unity_cumulus_lambda/cumulus_lambda_functions_deployment.zip --s3-bucket uds-dev-cumulus-public --function-name arn:aws:lambda:us-west-2:237868187491:function:uds-sbx-cumulus-metadata_stac_generate_cmr --publish &>/dev/null
-
-update_lambda_function_mcp_sbx_8:
-	aws lambda update-function-code --s3-key unity_cumulus_lambda/cumulus_lambda_functions_deployment.zip --s3-bucket uds-dev-cumulus-public --function-name arn:aws:lambda:us-west-2:237868187491:function:uds-sbx-cumulus-granules_to_es --publish &>/dev/null
-
-
-update_lambda_function_uds-sbx-cumulus-granules_cnm_response_writer:
-	aws lambda update-function-code --s3-key unity_cumulus_lambda/cumulus_lambda_functions_deployment.zip --s3-bucket uds-dev-cumulus-public --function-name arn:aws:lambda:us-west-2:237868187491:function:uds-sbx-cumulus-granules_cnm_response_writer --publish &>/dev/null
-
-update_lambda_function_mcp_sbx_ingester:
-	aws lambda update-function-code --s3-key unity_cumulus_lambda/cumulus_lambda_functions_deployment.zip --s3-bucket uds-dev-cumulus-public --function-name arn:aws:lambda:us-west-2:237868187491:function:uds-sbx-cumulus-granules_cnm_ingester --publish &>/dev/null
-
-mcp_sbx: upload_lambda_mcp_dev update_lambda_function_mcp_sbx_7 update_lambda_function_mcp_sbx_8
-
-mcp_sbx_fastapi: upload_lambda_mcp_dev update_lambda_function_mcp_sbx_uds_api

cumulus_lambda_functions/cumulus_wrapper/query_granules.py

@@ -21,6 +21,7 @@ def __init__(self, cumulus_base: str, cumulus_token: str):
         super().__init__(cumulus_base, cumulus_token)
         self._conditions.append('status=completed')
         self._item_transformer = ItemTransformer()
+        self.__collection_id = None
 
     def with_filter(self, filter_key, filter_values: list):
         if len(filter_values) < 1:
@@ -34,6 +35,7 @@ def with_filter(self, filter_key, filter_values: list):
 
     def with_collection_id(self, collection_id: str):
         self._conditions.append(f'{self.__collection_id_key}={collection_id}')
+        self.__collection_id = collection_id
         return self
 
     def with_bbox(self):
@@ -130,6 +132,48 @@ def query_direct_to_private_api(self, private_api_prefix: str, transform=True):
             return {'server_error': f'error while invoking:{str(e)}'}
         return {'results': stac_list}
 
+    def delete_entry(self, private_api_prefix: str, granule_id: str):
+        payload = {
+            'httpMethod': 'DELETE',
+            'resource': '/{proxy+}',
+            'path': f'/{self.__granules_key}/{self.__collection_id}/{granule_id}',
+            'queryStringParameters': {**{k[0]: k[1] for k in [k1.split('=') for k1 in self._conditions]}},
+            # 'queryStringParameters': {'limit': '30'},
+            'headers': {
+                'Content-Type': 'application/json',
+            },
+            # 'body': json.dumps({"action": "removeFromCmr"})
+        }
+        LOGGER.debug(f'payload: {payload}')
+        try:
+            query_result = self._invoke_api(payload, private_api_prefix)
+            """
+        {'statusCode': 200, 'body': '{"meta":{"name":"cumulus-api","stack":"am-uds-dev-cumulus","table":"granule","limit":3,"page":1,"count":0},"results":[]}', 'headers': {'x-powered-by': 'Express', 'access-control-allow-origin': '*', 'strict-transport-security': 'max-age=31536000; includeSubDomains', 'content-type': 'application/json; charset=utf-8', 'content-length': '120', 'etag': 'W/"78-YdHqDNIH4LuOJMR39jGNA/23yOQ"', 'date': 'Tue, 07 Jun 2022 22:30:44 GMT', 'connection': 'close'}, 'isBase64Encoded': False}
+            """
+            LOGGER.debug(f'json query_result: {query_result}')
+            if query_result['statusCode'] >= 500:
+                LOGGER.error(f'server error status code: {query_result["statusCode"]}. details: {query_result}')
+                return {'server_error': query_result}
+            if query_result['statusCode'] >= 400:
+                LOGGER.error(f'client error status code: {query_result["statusCode"]}. details: {query_result}')
+                return {'client_error': query_result}
+            query_result = json.loads(query_result['body'])
+            """
+            {
+              "detail": "Record deleted"
+            }
+            """
+            if 'detail' not in query_result:
+                LOGGER.error(f'missing key: detail. invalid response json: {query_result}')
+                return {'server_error': f'missing key: detail. invalid response json: {query_result}'}
+            if query_result['detail'] != 'Record deleted':
+                LOGGER.error(f'Wrong Message: {query_result}')
+                return {'server_error': f'Wrong Message: {query_result}'}
+        except Exception as e:
+            LOGGER.exception('error while invoking')
+            return {'server_error': f'error while invoking:{str(e)}'}
+        return {}
+
     def query(self, transform=True):
         conditions_str = '&'.join(self._conditions)
         LOGGER.info(f'cumulus_base: {self.cumulus_base}')

cumulus_lambda_functions/granules_to_es/granules_indexer.py

@@ -73,6 +73,9 @@ def start(self):
         result = JsonValidator(self.CUMULUS_SCHEMA).validate(incoming_msg)
         if result is not None:
             raise ValueError(f'input json has CUMULUS validation errors: {result}')
+        if 'event' not in incoming_msg or incoming_msg['event'].upper() == 'DELETE':
+            LOGGER.debug(f'missing event or it is DELETE event. Not inserting to ES')
+            return
         self.__cumulus_record = incoming_msg['record']
         if len(self.__cumulus_record['files']) < 1:
             # TODO ingest updating stage?

cumulus_lambda_functions/lib/uds_db/granules_db_index.py

@@ -217,6 +217,24 @@ def __query_by_id_local(self, tenant: str, tenant_venue: str, doc_id: str, ):
             return None
         return result['hits']['hits'][0]
 
+    def delete_entry(self, tenant: str, tenant_venue: str, doc_id: str, ):
+        read_alias_name = f'{DBConstants.granules_read_alias_prefix}_{tenant}_{tenant_venue}'.lower().strip()
+        result = self.__es.query({
+            'size': 9999,
+            'query': {'term': {'_id': doc_id}}
+        }, read_alias_name)
+        if result is None:
+            raise ValueError(f"no such granule: {doc_id}")
+        for each_granule in result['hits']['hits']:
+            LOGGER.debug(f"deleting {each_granule['_id']} from {each_granule['_index']}")
+            delete_result = self.__es.delete_by_query({
+                'query': {'term': {'id': each_granule['_id']}}
+            }, each_granule['_index'])
+            LOGGER.debug(f'delete_result: {delete_result}')
+            if delete_result is None:
+                raise ValueError(f"error deleting {each_granule}")
+        return result
+
     def update_entry(self, tenant: str, tenant_venue: str, json_body: dict, doc_id: str, ):
         write_alias_name = f'{DBConstants.granules_write_alias_prefix}_{tenant}_{tenant_venue}'.lower().strip()
         json_body['event_time'] = TimeUtils.get_current_unix_milli()

cumulus_lambda_functions/uds_api/dapa/granules_dapa_query_es.py

@@ -1,6 +1,10 @@
+import json
+import os
 from copy import deepcopy
 
+from mdps_ds_lib.lib.aws.aws_lambda import AwsLambda
 from pystac import Link
+from starlette.datastructures import URL
 
 from cumulus_lambda_functions.daac_archiver.daac_archiver_logic import DaacArchiverLogic
 from cumulus_lambda_functions.granules_to_es.granules_index_mapping import GranulesIndexMapping
@@ -17,6 +21,7 @@
 
 class GranulesDapaQueryEs:
     def __init__(self, collection_id, limit, offset, input_datetime, filter_input, pagination_link_obj: PaginationLinksGenerator, base_url):
+        self.__collection_cnm_lambda_name = os.environ.get('COLLECTION_CREATION_LAMBDA_NAME', '').strip()
         self.__pagination_link_obj = pagination_link_obj
         self.__input_datetime = input_datetime
         self.__collection_id = collection_id
@@ -149,6 +154,64 @@ def get_single_granule(self, granule_id):
         self.__restructure_each_granule_result(each_granules_query_result_stripped)
         return each_granules_query_result_stripped
 
+    def delete_facade(self, current_url: URL, bearer_token: str):
+        actual_path = current_url.path
+        actual_path = actual_path if actual_path.endswith('/') else f'{actual_path}/'
+        actual_path = f'{actual_path}actual'
+        LOGGER.info(f'sanity_check')
+
+        actual_event = {
+            'resource': actual_path,
+            'path': actual_path,
+            'httpMethod': 'DELETE',
+            'headers': {
+                'Accept': '*/*', 'Accept-Encoding': 'gzip, deflate', 'Authorization': bearer_token,
+                'Host': current_url.hostname, 'User-Agent': 'python-requests/2.28.2',
+                'X-Amzn-Trace-Id': 'Root=1-64a66e90-6fa8b7a64449014639d4f5b4', 'X-Forwarded-For': '44.236.15.58',
+                'X-Forwarded-Port': '443', 'X-Forwarded-Proto': 'https'},
+            'multiValueHeaders': {
+                'Accept': ['*/*'], 'Accept-Encoding': ['gzip, deflate'], 'Authorization': [bearer_token],
+                'Host': [current_url.hostname], 'User-Agent': ['python-requests/2.28.2'],
+                'X-Amzn-Trace-Id': ['Root=1-64a66e90-6fa8b7a64449014639d4f5b4'],
+                'X-Forwarded-For': ['127.0.0.1'], 'X-Forwarded-Port': ['443'], 'X-Forwarded-Proto': ['https']
+            },
+            'queryStringParameters': {},
+            'multiValueQueryStringParameters': {},
+            'pathParameters': {},
+            'stageVariables': None,
+            'requestContext': {
+                'resourceId': '',
+                'authorizer': {'principalId': '', 'integrationLatency': 0},
+                'resourcePath': actual_path, 'httpMethod': 'PUT',
+                'extendedRequestId': '', 'requestTime': '',
+                'path': actual_path, 'accountId': '',
+                'protocol': 'HTTP/1.1', 'stage': '', 'domainPrefix': '', 'requestTimeEpoch': 0,
+                'requestId': '',
+                'identity': {
+                    'cognitoIdentityPoolId': None, 'accountId': None, 'cognitoIdentityId': None, 'caller': None,
+                    'sourceIp': '127.0.0.1', 'principalOrgId': None, 'accessKey': None,
+                    'cognitoAuthenticationType': None,
+                    'cognitoAuthenticationProvider': None, 'userArn': None, 'userAgent': 'python-requests/2.28.2',
+                    'user': None
+                },
+                'domainName': current_url.hostname, 'apiId': ''
+            },
+            'body': json.dumps({}),
+            'isBase64Encoded': False
+        }
+        LOGGER.info(f'actual_event: {actual_event}')
+        response = AwsLambda().invoke_function(
+            function_name=self.__collection_cnm_lambda_name,
+            payload=actual_event,
+        )
+        LOGGER.debug(f'async function started: {response}')
+        return {
+            'statusCode': 202,
+            'body': {
+                'message': 'processing'
+            }
+        }
+
     def start(self):
         try:
             granules_query_dsl = self.__generate_es_dsl()
@@ -164,7 +227,8 @@ def start(self):
                 self_link = Link(rel='self', target=f'{self.__base_url}/{WebServiceConstants.COLLECTIONS}/{self.__collection_id}/items/{each_granules_query_result_stripped["id"]}', media_type='application/json', title=each_granules_query_result_stripped["id"]).to_dict(False)
                 each_granules_query_result_stripped['links'].append(self_link)
                 self.__restructure_each_granule_result(each_granules_query_result_stripped)
-            pagination_link = '' if len(granules_query_result['hits']['hits']) < self.__limit else ','.join(granules_query_result['hits']['hits'][-1]['sort'])
+
+            pagination_link = '' if len(granules_query_result['hits']['hits']) < self.__limit else ','.join([k if isinstance(k, str) else str(k) for k in granules_query_result['hits']['hits'][-1]['sort']])
             return {
                 'statusCode': 200,
                 'body': {