Merge pull request #149 from PermanentOrg/account_space_lambda_deploy…

…_test PER-9562: Deploy new account space update job to a lambda
PermanentOrg · Dec 4, 2024 · 8130f22 · 8130f22
2 parents a783d7c + 46e4a38
commit 8130f22
Show file tree

Hide file tree

Showing 19 changed files with 750 additions and 92 deletions.
diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
@@ -104,3 +104,23 @@ jobs:
           AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
       - name: Publish Image to ECR
         run: docker push $ACCESS_COPY_LAMBDA_IMAGE_TAG
+  build_account_space_updater:
+    needs:
+      - generate_image_tags
+    runs-on: ubuntu-20.04
+    env:
+      ACCOUNT_SPACE_UPDATER_IMAGE_TAG: ${{ needs.generate_image_tags.outputs.ACCOUNT_SPACE_UPDATER_IMAGE_TAG }}
+    steps:
+      - uses: actions/checkout@v3
+      - name: Build Image
+        run: docker build -t $ACCOUNT_SPACE_UPDATER_IMAGE_TAG --build-arg="AWS_RDS_CERT_BUNDLE=$AWS_RDS_CERT_BUNDLE" -f Dockerfile.account_space_updater .
+        env:
+          AWS_RDS_CERT_BUNDLE: ${{ secrets.AWS_RDS_CERT_BUNDLE }}
+      - name: AWS Login
+        run: aws ecr get-login-password --region $AWS_REGION | docker login --username AWS --password-stdin 364159549467.dkr.ecr.$AWS_REGION.amazonaws.com
+        env:
+          AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
+          AWS_REGION: ${{ secrets.AWS_DEFAULT_REGION }}
+          AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+      - name: Publish Image to ECR
+        run: docker push $ACCOUNT_SPACE_UPDATER_IMAGE_TAG
diff --git a/.github/workflows/dev_deploy.yml b/.github/workflows/dev_deploy.yml
@@ -29,6 +29,7 @@ jobs:
       RECORD_THUMBNAIL_LAMBDA_IMAGE_TAG: ${{ needs.generate_image_tags.outputs.RECORD_THUMBNAIL_LAMBDA_IMAGE_TAG }}
       THUMBNAIL_REFRESH_IMAGE_TAG: ${{ needs.generate_image_tags.outputs.THUMBNAIL_REFRESH_IMAGE_TAG }}
       ACCESS_COPY_LAMBDA_IMAGE_TAG: ${{ needs.generate_image_tags.outputs.ACCESS_COPY_LAMBDA_IMAGE_TAG }}
+      ACCOUNT_SPACE_UPDATER_IMAGE_TAG: ${{ needs.generate_image_tags.outputs.ACCOUNT_SPACE_UPDATER_IMAGE_TAG }}
     defaults:
       run:
         working-directory: ./terraform/test_cluster
@@ -60,11 +61,14 @@ jobs:
           -var="thumbnail_refresh_staging_image=$THUMBNAIL_REFRESH_IMAGE_TAG" \
           -var="access_copy_dev_lambda_image=$ACCESS_COPY_LAMBDA_IMAGE_TAG" \
           -var="access_copy_staging_lambda_image=$ACCESS_COPY_LAMBDA_IMAGE_TAG" \
+          -var="account_space_updater_dev_lambda_image=$ACCOUNT_SPACE_UPDATER_IMAGE_TAG" \
+          -var="account_space_updater_staging_lambda_image=$ACCOUNT_SPACE_UPDATER_IMAGE_TAG" \
           -target=kubernetes_deployment.stela_dev \
           -target=kubernetes_cron_job_v1.archivematica_cleanup_dev \
           -target=aws_lambda_function.record_thumbnail_lambda \
           -target=kubernetes_cron_job_v1.thumbnail_refresh_dev \
-          -target=aws_lambda_function.access_copy_dev_lambda
+          -target=aws_lambda_function.access_copy_dev_lambda \
+          -target=aws_lambda_function.account_space_update_dev_lambda
       - name: Terraform Apply
         run: |
           terraform apply -auto-approve -input=false \
@@ -78,8 +82,11 @@ jobs:
           -var="thumbnail_refresh_staging_image=$THUMBNAIL_REFRESH_IMAGE_TAG" \
           -var="access_copy_dev_lambda_image=$ACCESS_COPY_LAMBDA_IMAGE_TAG" \
           -var="access_copy_staging_lambda_image=$ACCESS_COPY_LAMBDA_IMAGE_TAG" \
+          -var="account_space_updater_dev_lambda_image=$ACCOUNT_SPACE_UPDATER_IMAGE_TAG" \
+          -var="account_space_updater_staging_lambda_image=$ACCOUNT_SPACE_UPDATER_IMAGE_TAG" \
           -target=kubernetes_deployment.stela_dev \
           -target=kubernetes_cron_job_v1.archivematica_cleanup_dev \
           -target=aws_lambda_function.record_thumbnail_lambda \
           -target=kubernetes_cron_job_v1.thumbnail_refresh_dev \
-          -target=aws_lambda_function.access_copy_dev_lambda
+          -target=aws_lambda_function.access_copy_dev_lambda \
+          -target=aws_lambda_function.account_space_update_dev_lambda
diff --git a/.github/workflows/full_test_deploy.yml b/.github/workflows/full_test_deploy.yml
@@ -26,6 +26,7 @@ jobs:
       RECORD_THUMBNAIL_LAMBDA_IMAGE_TAG: ${{ needs.generate_image_tags.outputs.RECORD_THUMBNAIL_LAMBDA_IMAGE_TAG }}
       THUMBNAIL_REFRESH_IMAGE_TAG: ${{ needs.generate_image_tags.outputs.THUMBNAIL_REFRESH_IMAGE_TAG }}
       ACCESS_COPY_LAMBDA_IMAGE_TAG: ${{ needs.generate_image_tags.outputs.ACCESS_COPY_LAMBDA_IMAGE_TAG }}
+      ACCOUNT_SPACE_UPDATER_IMAGE_TAG: ${{ needs.generate_image_tags.outputs.ACCOUNT_SPACE_UPDATER_IMAGE_TAG }}
     defaults:
       run:
         working-directory: ./terraform/test_cluster
@@ -54,7 +55,9 @@ jobs:
           -var="thumbnail_refresh_dev_image=$THUMBNAIL_REFRESH_IMAGE_TAG" \
           -var="thumbnail_refresh_staging_image=$THUMBNAIL_REFRESH_IMAGE_TAG" \
           -var="access_copy_dev_lambda_image=$ACCESS_COPY_LAMBDA_IMAGE_TAG" \
-          -var="access_copy_staging_lambda_image=$ACCESS_COPY_LAMBDA_IMAGE_TAG"
+          -var="access_copy_staging_lambda_image=$ACCESS_COPY_LAMBDA_IMAGE_TAG" \
+          -var="account_space_updater_dev_lambda_image=$ACCOUNT_SPACE_UPDATER_IMAGE_TAG" \
+          -var="account_space_updater_staging_lambda_image=$ACCOUNT_SPACE_UPDATER_IMAGE_TAG"
       - name: Terraform Apply
         run: |
           terraform apply -auto-approve -input=false \
@@ -67,4 +70,6 @@ jobs:
           -var="thumbnail_refresh_dev_image=$THUMBNAIL_REFRESH_IMAGE_TAG" \
           -var="thumbnail_refresh_staging_image=$THUMBNAIL_REFRESH_IMAGE_TAG" \
           -var="access_copy_dev_lambda_image=$ACCESS_COPY_LAMBDA_IMAGE_TAG" \
-          -var="access_copy_staging_lambda_image=$ACCESS_COPY_LAMBDA_IMAGE_TAG"
+          -var="access_copy_staging_lambda_image=$ACCESS_COPY_LAMBDA_IMAGE_TAG" \
+          -var="account_space_updater_dev_lambda_image=$ACCOUNT_SPACE_UPDATER_IMAGE_TAG" \
+          -var="account_space_updater_staging_lambda_image=$ACCOUNT_SPACE_UPDATER_IMAGE_TAG"
diff --git a/.github/workflows/generate_image_tags.yml b/.github/workflows/generate_image_tags.yml
@@ -12,6 +12,8 @@ on:
         value: ${{ jobs.generate_image_tags.outputs.THUMBNAIL_REFRESH_IMAGE_TAG }}
       ACCESS_COPY_LAMBDA_IMAGE_TAG:
         value: ${{ jobs.generate_image_tags.outputs.ACCESS_COPY_LAMBDA_IMAGE_TAG }}
+      ACCOUNT_SPACE_UPDATER_IMAGE_TAG:
+        value: ${{ jobs.generate_image_tags.outputs.ACCOUNT_SPACE_UPDATER_IMAGE_TAG }}
 jobs:
   generate_image_tags:
     runs-on: ubuntu-20.04
@@ -21,6 +23,7 @@ jobs:
       RECORD_THUMBNAIL_LAMBDA_IMAGE_TAG: ${{ steps.generate_record_thumbnail_lambda_image_tag.outputs.RECORD_THUMBNAIL_LAMBDA_IMAGE_TAG }}
       THUMBNAIL_REFRESH_IMAGE_TAG: ${{ steps.generate_thumbnail_refresh_image_tag.outputs.THUMBNAIL_REFRESH_IMAGE_TAG }}
       ACCESS_COPY_LAMBDA_IMAGE_TAG: ${{ steps.generate_access_copy_lambda_image_tag.outputs.ACCESS_COPY_LAMBDA_IMAGE_TAG }}
+      ACCOUNT_SPACE_UPDATER_IMAGE_TAG: ${{ steps.generate_account_space_updater_image_tag.outputs.ACCOUNT_SPACE_UPDATER_IMAGE_TAG }}
     steps:
       - uses: actions/checkout@v3
       - name: Set ECR domain env var
@@ -49,3 +52,6 @@ jobs:
       - name: Generate Access Copy Lambda Image Tag
         id: generate_access_copy_lambda_image_tag
         run: echo "ACCESS_COPY_LAMBDA_IMAGE_TAG=$ECR_DOMAIN/stela:access_copy_lambda-$BRANCH_TYPE-$ABBREVIATED_COMMIT_HASH" >> "$GITHUB_OUTPUT"
+      - name: Generate Account Space Updater Image Tag
+        id: generate_account_space_updater_image_tag
+        run: echo "ACCOUNT_SPACE_UPDATER_IMAGE_TAG=$ECR_DOMAIN/stela:account_space_updater-$BRANCH_TYPE-$ABBREVIATED_COMMIT_HASH" >> "$GITHUB_OUTPUT"
diff --git a/.github/workflows/prod_deploy.yml b/.github/workflows/prod_deploy.yml
@@ -24,6 +24,7 @@ jobs:
       RECORD_THUMBNAIL_LAMBDA_IMAGE_TAG: ${{ needs.generate_image_tags.outputs.RECORD_THUMBNAIL_LAMBDA_IMAGE_TAG }}
       THUMBNAIL_REFRESH_IMAGE_TAG: ${{ needs.generate_image_tags.outputs.THUMBNAIL_REFRESH_IMAGE_TAG }}
       ACCESS_COPY_LAMBDA_IMAGE_TAG: ${{ needs.generate_image_tags.outputs.ACCESS_COPY_LAMBDA_IMAGE_TAG }}
+      ACCOUNT_SPACE_UPDATER_IMAGE_TAG: ${{ needs.generate_image_tags.outputs.ACCOUNT_SPACE_UPDATER_IMAGE_TAG }}
     defaults:
       run:
         working-directory: ./terraform/prod_cluster
@@ -47,12 +48,14 @@ jobs:
           -var="archivematica_cleanup_image=$AM_CLEANUP_IMAGE_TAG" \
           -var="record_thumbnail_lambda_image=$RECORD_THUMBNAIL_LAMBDA_IMAGE_TAG" \
           -var="thumbnail_refresh_image=$THUMBNAIL_REFRESH_IMAGE_TAG" \
-          -var="access_copy_lambda_image=$ACCESS_COPY_LAMBDA_IMAGE_TAG"
+          -var="access_copy_lambda_image=$ACCESS_COPY_LAMBDA_IMAGE_TAG" \
+          -var="account_space_updater_lambda_image=$ACCOUNT_SPACE_UPDATER_IMAGE_TAG"
       - name: Terraform Apply
         run: |
           terraform apply -auto-approve -input=false \
           -var="stela_image=$API_IMAGE_TAG" \
           -var="archivematica_cleanup_image=$AM_CLEANUP_IMAGE_TAG" \
           -var="record_thumbnail_lambda_image=$RECORD_THUMBNAIL_LAMBDA_IMAGE_TAG" \
           -var="thumbnail_refresh_image=$THUMBNAIL_REFRESH_IMAGE_TAG" \
-          -var="access_copy_lambda_image=$ACCESS_COPY_LAMBDA_IMAGE_TAG"
+          -var="access_copy_lambda_image=$ACCESS_COPY_LAMBDA_IMAGE_TAG" \
+          -var="account_space_updater_lambda_image=$ACCOUNT_SPACE_UPDATER_IMAGE_TAG"
diff --git a/.github/workflows/staging_deploy.yml b/.github/workflows/staging_deploy.yml
@@ -27,6 +27,7 @@ jobs:
       RECORD_THUMBNAIL_LAMBDA_IMAGE_TAG: ${{ needs.generate_image_tags.outputs.RECORD_THUMBNAIL_LAMBDA_IMAGE_TAG }}
       THUMBNAIL_REFRESH_IMAGE_TAG: ${{ needs.generate_image_tags.outputs.THUMBNAIL_REFRESH_IMAGE_TAG }}
       ACCESS_COPY_LAMBDA_IMAGE_TAG: ${{ needs.generate_image_tags.outputs.ACCESS_COPY_LAMBDA_IMAGE_TAG }}
+      ACCOUNT_SPACE_UPDATER_IMAGE_TAG: ${{ needs.generate_image_tags.outputs.ACCOUNT_SPACE_UPDATER_IMAGE_TAG }}
     defaults:
       run:
         working-directory: ./terraform/test_cluster
@@ -58,11 +59,14 @@ jobs:
           -var="thumbnail_refresh_staging_image=$THUMBNAIL_REFRESH_IMAGE_TAG" \
           -var="access_copy_dev_lambda_image=$ACCESS_COPY_LAMBDA_IMAGE_TAG" \
           -var="access_copy_staging_lambda_image=$ACCESS_COPY_LAMBDA_IMAGE_TAG" \
+          -var="account_space_updater_dev_lambda_image=$ACCOUNT_SPACE_UPDATER_IMAGE_TAG" \
+          -var="account_space_updater_staging_lambda_image=$ACCOUNT_SPACE_UPDATER_IMAGE_TAG" \
           -target=kubernetes_deployment.stela_staging \
           -target=kubernetes_cron_job_v1.archivematica_cleanup_staging \
           -target=aws_lambda_function.record_thumbnail_lambda_staging \
           -target=kubernetes_cron_job_v1.thumbnail_refresh_staging \
-          -target=aws_lambda_function.access_copy_lambda_staging
+          -target=aws_lambda_function.access_copy_lambda_staging \
+          -target=aws_lambda_function.account_space_update_staging_lambda
       - name: Terraform Apply
         run: |
           terraform apply -auto-approve -input=false \
@@ -74,8 +78,13 @@ jobs:
           -var="record_thumbnail_staging_lambda_image=$RECORD_THUMBNAIL_LAMBDA_IMAGE_TAG" \
           -var="thumbnail_refresh_dev_image=$THUMBNAIL_REFRESH_IMAGE_TAG" \
           -var="thumbnail_refresh_staging_image=$THUMBNAIL_REFRESH_IMAGE_TAG" \
+          -var="access_copy_dev_lambda_image=$ACCESS_COPY_LAMBDA_IMAGE_TAG" \
+          -var="access_copy_staging_lambda_image=$ACCESS_COPY_LAMBDA_IMAGE_TAG" \
+          -var="account_space_updater_dev_lambda_image=$ACCOUNT_SPACE_UPDATER_IMAGE_TAG" \
+          -var="account_space_updater_staging_lambda_image=$ACCOUNT_SPACE_UPDATER_IMAGE_TAG" \
           -target=kubernetes_deployment.stela_staging \
           -target=kubernetes_cron_job_v1.archivematica_cleanup_staging \
           -target=aws_lambda_function.record_thumbnail_lambda_staging \
           -target=kubernetes_cron_job_v1.thumbnail_refresh_staging \
-          -target=aws_lambda_function.access_copy_lambda_staging
+          -target=aws_lambda_function.access_copy_lambda_staging \
+          -target=aws_lambda_function.account_space_update_staging_lambda
diff --git a/Dockerfile.account_space_updater b/Dockerfile.account_space_updater
@@ -14,12 +14,20 @@ RUN npm run build -ws
 
 
 FROM public.ecr.aws/lambda/nodejs:18 as final
+
+ARG AWS_RDS_CERT_BUNDLE
+
 WORKDIR ${LAMBDA_TASK_ROOT}
 
+RUN mkdir /etc/ca-certificates
+RUN echo -e $AWS_RDS_CERT_BUNDLE > /etc/ca-certificates/rds-us-west-2-ca-bundle.pem
+
 COPY --from=builder /usr/local/apps/stela/packages/account_space_updater/dist ./packages/account_space_updater/dist
 COPY --from=builder /usr/local/apps/stela/packages/account_space_updater/package.json ./packages/account_space_updater/package.json
 COPY --from=builder /usr/local/apps/stela/packages/logger/dist ./packages/logger/dist
 COPY --from=builder /usr/local/apps/stela/packages/logger/package.json ./packages/logger/package.json
+COPY --from=builder /usr/local/apps/stela/packages/s3-utils/dist ./packages/s3-utils/dist
+COPY --from=builder /usr/local/apps/stela/packages/s3-utils/package.json ./packages/s3-utils/package.json
 COPY --from=builder /usr/local/apps/stela/package.json ./package.json
 COPY --from=builder /usr/local/apps/stela/package-lock.json ./package-lock.json
 

diff --git a/package-lock.json b/package-lock.json
diff --git a/packages/account_space_updater/package.json b/packages/account_space_updater/package.json
@@ -32,6 +32,7 @@
   "dependencies": {
     "@sentry/node": "^7.57.0",
     "@stela/logger": "^1.0.0",
+    "@stela/s3-utils": "^1.0.0",
     "ajv": "^8.16.0",
     "dotenv": "^8.2.0",
     "require-env-variable": "^3.1.2",