Merge branch 'develop' of github.com:unity-sds/unity-data-services into case-insensitive

Author: wphyojpl

CHANGELOG.md

@@ -5,6 +5,14 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## [9.11.3] - 2025-04-21
+### Fixed
+- [#565](https://github.com/unity-sds/unity-data-services/pull/565) fix: data service version endpoint
+
+## [9.11.2] - 2025-04-15
+### Fixed
+- [#564](https://github.com/unity-sds/unity-data-services/pull/564) fix: check collection when querying single granule
+
 ## [9.11.1] - 2025-04-10
 ### Fixed
 - [#561](https://github.com/unity-sds/unity-data-services/pull/561) fix: docker error

ci.cd/create_aws_lambda_zip.sh

@@ -2,13 +2,28 @@
 
 apt-get update -y && apt-get install zip -y
 
+# github.job
+github_branch=${GITHUB_REF##*/}
+software_version_trailing=""
+main_branch="main"
+if [ "$github_branch" = "$main_branch" ];
+then
+  software_version=""
+else
+  software_version_trailing="-${github_branch}-${GITHUB_RUN_ID}"
+fi
+
 ZIP_NAME='cumulus_lambda_functions_deployment.zip'
 TERRAFORM_ZIP_NAME='terraform_cumulus_lambda_functions_deployment.zip'
 TERRAFORM_MARKETPLACE_ZIP_NAME='terraform_marketplace_deployment.zip'
 TERRAFORM_ECR_ZIP_NAME='terraform_ecr_deployment.zip'
 TERRAFORM_STAC_BR_ZIP_NAME='terraform_stac_br_deployment.zip'
 
 project_root_dir=${GITHUB_WORKSPACE}
+
+software_version=`python3 ${project_root_dir}/setup.py --version`
+echo "software_version=${software_version}${software_version_trailing}" >> ${GITHUB_ENV}
+
 zip_file="${project_root_dir}/$ZIP_NAME" ; # save the result file in current working directory
 terraform_zip_file="${project_root_dir}/$TERRAFORM_ZIP_NAME" ; # save the result file in current working directory
 terraform_marketplace_zip_file="${project_root_dir}/$TERRAFORM_MARKETPLACE_ZIP_NAME" ; # save the result file in current working directory
@@ -18,6 +33,13 @@ terraform_stac_br_zip_file="${project_root_dir}/$TERRAFORM_STAC_BR_ZIP_NAME" ; #
 source_dir=`python3 -c 'import sysconfig; print(sysconfig.get_paths()["purelib"])'`
 
 cd ${source_dir}
+built_time=$(date -u +"%Y-%m-%dT%H:%M:%SZ")
+cat <<EOF > ds_version.json
+{
+  "version": "$software_version",
+  "built": "$built_time"
+}
+EOF
 rm -rf ${zip_file} && \
 zip -r9 ${zip_file} . && \
 echo "zipped to ${zip_file}"
@@ -38,16 +60,5 @@ zip -9 ${terraform_ecr_zip_file} * **/*
 cd $project_root_dir/tf-module/stac_browser
 zip -9 ${terraform_stac_br_zip_file} * **/*
 
-# github.job
-github_branch=${GITHUB_REF##*/}
-software_version_trailing=""
-main_branch="main"
-if [ "$github_branch" = "$main_branch" ];
-then
-  software_version=""
-else
-  software_version_trailing="-${github_branch}-${GITHUB_RUN_ID}"
-fi
-software_version=`python3 ${project_root_dir}/setup.py --version`
-echo "software_version=${software_version}${software_version_trailing}" >> ${GITHUB_ENV}
+
 cat ${GITHUB_ENV}

cumulus_lambda_functions/uds_api/dapa/granules_dapa_query_es.py

@@ -172,7 +172,8 @@ def get_single_granule(self, granule_id):
             'size': 1,
             'sort': [{'id': {'order': 'asc'}}],
             'query': {'bool': {'must': [{
-                'term': {'id': granule_id}
+                'term': {'id': granule_id}}, {
+                'term': {'collection': self.__collection_id},
             }]}}
         }
         LOGGER.debug(f'granules_query_dsl: {granules_query_dsl}')
@@ -182,7 +183,7 @@ def get_single_granule(self, granule_id):
                                                              granules_query_dsl)
         LOGGER.debug(f'granules_query_result: {granules_query_result}')
         if len(granules_query_result['hits']['hits']) < 1:
-            raise ValueError(f'cannot find granule for : {granule_id}')
+            return None
 
         each_granules_query_result_stripped = granules_query_result['hits']['hits'][0]['_source']
         self_link = Link(rel='self', target=f'{self.__base_url}/{WebServiceConstants.COLLECTIONS}/{self.__collection_id}/items/{each_granules_query_result_stripped["id"]}', media_type='application/json', title=each_granules_query_result_stripped["id"]).to_dict(False)

cumulus_lambda_functions/uds_api/granules_api.py

@@ -136,6 +136,8 @@ async def get_single_granule_dapa(request: Request, collection_id: str, granule_
     except Exception as e:
         LOGGER.exception('failed during get_granules_dapa')
         raise HTTPException(status_code=500, detail=str(e))
+    if granules_result is None:
+        raise HTTPException(status_code=404, detail={'message': f'no granule with id: {granule_id} in collection: {collection_id}'})
     return granules_result
 
 @router.delete("/{collection_id}/items/{granule_id}")

cumulus_lambda_functions/uds_api/misc_api.py

@@ -1,8 +1,10 @@
 import json
 import os
+from glob import glob
 from time import time
 from typing import Union
 
+from mdps_ds_lib.lib.utils.file_utils import FileUtils
 from starlette.responses import Response, RedirectResponse
 from cumulus_lambda_functions.uds_api.fast_api_utils import FastApiUtils
 
@@ -50,6 +52,7 @@ async def catalog_list(request: Request, response: Response):
     }]
     return stac_browser_expecting_result
 
+
 @router.get(f'/stac_entry')
 @router.get(f'/stac_entry/')
 async def stac_entry(request: Request, response: Response):
@@ -79,3 +82,29 @@ async def stac_entry(request: Request, response: Response):
         redirect_response.set_cookie(key="unity_token", value=request_headers['oidc_access_token'], httponly=False, secure=False, samesite='strict')  # missing , domain=base_url
         redirect_response.set_cookie(key="test1", value=f"{time()}", httponly=False, secure=False, samesite='strict')  # missing , domain=base_url
     return redirect_response
+
+
+@router.get(f'/version')
+@router.get(f'/version/')
+async def ds_version(request: Request, response: Response):
+    """
+    This is to list all catalogs for STAC Browser.
+    This doesn't require any authorization token.
+    :param request:
+    :param response:
+    :return:
+    """
+    version_details_unknown = {
+        'version': 'unknown',
+        'built': 'unknown'
+    }
+    if not FileUtils.file_exist('/var/task/ds_version.json'):
+        print(f'missing file : {[k for k in glob("/var/task/*.json")]}')
+        return version_details_unknown
+    version_details = FileUtils.read_json('/var/task/ds_version.json')
+
+    version_details = {
+        **version_details_unknown,
+        **version_details,
+    }
+    return version_details

cumulus_lambda_functions/uds_api/web_service.py

@@ -64,7 +64,6 @@ async def get_open_api(request: Request):
         default_open_api_doc['paths'].pop(k)
     return app.openapi()
 
-
 # to make it work with Amazon Lambda, we create a handler object
 handler = Mangum(app=app)
 

setup.py

@@ -12,7 +12,7 @@
 
 setup(
     name="cumulus_lambda_functions",
-    version="9.11.1",
+    version="9.11.3",
     packages=find_packages(),
     install_requires=install_requires,
     package_data={

tests/integration_tests/test_uds_api.py

@@ -233,6 +233,40 @@ def test_single_granule_get(self):
 
         return
 
+    def test_version_get(self):
+        post_url = f'{self.uds_url}misc/version/'
+        headers = {
+            'Authorization': f'Bearer {self.bearer_token}',
+        }
+        print(post_url)
+        query_result = requests.get(url=post_url,
+                                    headers=headers,
+                                    )
+        # self.assertEqual(query_result.status_code, 200, f'wrong status code. {query_result.text}')
+        response_json = json.loads(query_result.text)
+        print(json.dumps(response_json))
+        return
+
+      
+    def test_single_granule_get_wrong_collection(self):
+        # post_url = f'{self.uds_url}collections/urn:nasa:unity:uds_local_test:DEV1:CHRP_16_DAY_REBIN___10/items/urn:nasa:unity:uds_local_test:DEV1:CHRP_16_DAY_REBIN___10:SNDR.SS1330.CHIRP.20230101T0000.m06.g001.L1_J1.std.v02_48.G.200101070318_REBIN'
+        post_url = f'{self.uds_url}collections/URN:NASA:UNITY:UDS_LOCAL_TEST_3:DEV:DDD-02___001/items/URN:NASA:UNITY:UDS_LOCAL_TEST_3:DEV:DDD-01___001:test_file10'
+        headers = {
+            'Authorization': f'Bearer {self.bearer_token}',
+        }
+        print(post_url)
+        query_result = requests.get(url=post_url, headers=headers,)
+        self.assertEqual(query_result.status_code, 404, f'wrong status code. {query_result.text}')
+        response_json = json.loads(query_result.text)
+        print(json.dumps(response_json))
+
+        post_url = f'{self.uds_url}collections/URN:NASA:UNITY:UDS_LOCAL_TEST_3:DEV:DDD-01___001/items/URN:NASA:UNITY:UDS_LOCAL_TEST_3:DEV:DDD-01___001:test_file10'
+        query_result = requests.get(url=post_url, headers=headers,)
+        self.assertEqual(query_result.status_code, 200, f'wrong status code. {query_result.text}')
+        response_json = json.loads(query_result.text)
+        print(json.dumps(response_json))
+        return
+
     def test_create_new_collection(self):
         post_url = f'{self.uds_url}collections/'  # MCP Dev
         headers = {

tf-module/unity-cumulus/api_gateway.tf

@@ -54,10 +54,12 @@ resource "aws_api_gateway_deployment" "shared_services_api_gateway_deployment" {
 
     aws_api_gateway_integration.misc_catalog_list_lambda_integration,
     aws_api_gateway_integration.misc_stac_entry_lambda_integration,
+    aws_api_gateway_integration.misc_version_lambda_integration,
 
     aws_api_gateway_integration.stac_browser_lambda_integration,
     aws_api_gateway_integration.stac_browser_proxy_lambda_integration,
 
+
     module.uds_all_cors_method.options_integration_object,
     module.uds_all_any_to_lambda_module.lambda_integration_object,
 

tf-module/unity-cumulus/api_gateway_01_misc_base_00_version.tf

@@ -0,0 +1,47 @@
+resource "aws_api_gateway_resource" "misc_version_resource" {
+  rest_api_id = data.aws_api_gateway_rest_api.rest_api.id
+  parent_id   = aws_api_gateway_resource.misc_base_resource.id
+  path_part   = "version"
+}
+
+resource "aws_api_gateway_method" "misc_version_method" {
+  rest_api_id   = data.aws_api_gateway_rest_api.rest_api.id
+  resource_id   = aws_api_gateway_resource.misc_version_resource.id
+  http_method   = "GET"
+  authorization = "NONE"
+  request_parameters = {
+    "method.request.path.proxy" = true
+  }
+}
+
+resource "aws_api_gateway_method_response" "misc_version_method_response" {
+    rest_api_id   = data.aws_api_gateway_rest_api.rest_api.id
+    resource_id   = aws_api_gateway_resource.misc_version_resource.id
+    http_method   = aws_api_gateway_method.misc_version_method.http_method
+    status_code   = 200
+    response_models = {
+        "application/json" = "Empty"
+    }
+    response_parameters = {
+        "method.response.header.Access-Control-Allow-Origin" = true
+    }
+    depends_on = ["aws_api_gateway_method.misc_version_method"]
+}
+
+resource "aws_api_gateway_integration" "misc_version_lambda_integration" {
+  rest_api_id   = data.aws_api_gateway_rest_api.rest_api.id
+  resource_id          = aws_api_gateway_resource.misc_version_resource.id
+  http_method          = aws_api_gateway_method.misc_version_method.http_method
+  type                 = "AWS_PROXY"
+  uri = aws_lambda_function.uds_api_1.invoke_arn
+  integration_http_method = "POST"
+
+#  cache_key_parameters = ["method.request.path.proxy"]
+
+  timeout_milliseconds = 29000
+#  request_parameters = {
+#    "integration.request.path.proxy" = "method.request.path.proxy"
+#  }
+}
+
+##########################################################################################################################