Use ECS logging in S3 to JSON job

Sage-Bionetworks · Oct 20, 2023 · 2cbc80a · 2cbc80a
1 parent f8aa14d
commit 2cbc80a
Show file tree

Hide file tree

Showing 7 changed files with 204 additions and 33 deletions.
diff --git a/.github/workflows/upload-and-deploy.yaml b/.github/workflows/upload-and-deploy.yaml
@@ -78,6 +78,10 @@ jobs:
           role_session_name: ${{ github.event.repository.name }}-${{ github.run_id }}-nonglue-unit-tests
           python_version: ${{ env.PYTHON_VERSION }}
 
+      - name: Install additional python dependency
+        run: |
+          pipenv install ecs_logging~=2.0
+
       - name: Test lambda scripts with pytest
         run: |
           pipenv run python -m pytest tests/test_s3_event_config_lambda.py -v

diff --git a/Pipfile b/Pipfile
@@ -17,3 +17,4 @@ pandas = "<1.5"
 moto = "~=4.1"
 datacompy = "~=0.8"
 docker = "~=6.1"
+ecs_logging = "~=2.0"
diff --git a/config/develop/namespaced/glue-job-S3ToJsonS3.yaml b/config/develop/namespaced/glue-job-S3ToJsonS3.yaml
@@ -11,5 +11,6 @@ parameters:
   S3ScriptKey: '{{ stack_group_config.namespace }}/src/glue/jobs/s3_to_json.py'
   GluePythonShellVersion: "{{ stack_group_config.glue_python_shell_python_version }}"
   GlueVersion: "{{ stack_group_config.glue_python_shell_glue_version }}"
+  AdditionalPythonModules: ecs_logging~=2.0
 stack_tags:
   {{ stack_group_config.default_stack_tags }}
diff --git a/config/prod/namespaced/glue-job-S3ToJsonS3.yaml b/config/prod/namespaced/glue-job-S3ToJsonS3.yaml
@@ -11,5 +11,6 @@ parameters:
   S3ScriptKey: '{{ stack_group_config.namespace }}/src/glue/jobs/s3_to_json.py'
   GluePythonShellVersion: "{{ stack_group_config.glue_python_shell_python_version }}"
   GlueVersion: "{{ stack_group_config.glue_python_shell_glue_version }}"
+  AdditionalPythonModules: ecs_logging~=2.0
 stack_tags:
   {{ stack_group_config.default_stack_tags }}