From 4b300e3c01c2cdbd6f8a588cd49b31ba14e1fbea Mon Sep 17 00:00:00 2001
From: BryanFauble <17128019+BryanFauble@users.noreply.github.com>
Date: Fri, 12 Jul 2024 17:00:04 -0700
Subject: [PATCH 1/3] Adjust cleanup job to run always, but skip steps if
 `main`

---
 .github/workflows/upload-and-deploy.yaml | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/upload-and-deploy.yaml b/.github/workflows/upload-and-deploy.yaml
index d5653069..457f353f 100755
--- a/.github/workflows/upload-and-deploy.yaml
+++ b/.github/workflows/upload-and-deploy.yaml
@@ -262,7 +262,6 @@ jobs:
     name: Cleanup non-main branch data before integration tests
     runs-on: ubuntu-latest
     needs: sceptre-deploy-develop
-    if: github.ref_name != 'main'
     environment: develop
     # These permissions are needed to interact with GitHub's OIDC Token endpoint.
     permissions:
@@ -270,6 +269,7 @@ jobs:
       contents: read
     steps:
       - name: Setup code, pipenv, aws
+        if: github.ref_name != 'main'
         uses: Sage-Bionetworks/action-pipenv-aws-setup@v3
         with:
           role_to_assume: ${{ vars.AWS_CREDENTIALS_IAM_ROLE }}
@@ -277,15 +277,18 @@ jobs:
           python_version: ${{ env.PYTHON_VERSION }}
 
       - name: Set namespace for non-default branch
+        if: github.ref_name != 'main'
         run: echo "NAMESPACE=$GITHUB_REF_NAME" >> $GITHUB_ENV
 
       - name: Clean input data bucket
+        if: github.ref_name != 'main'
         run: >
           pipenv run python src/scripts/manage_artifacts/clean_for_integration_test.py
           --bucket $DEV_INPUT_BUCKET
           --bucket_prefix "${{ env.NAMESPACE }}/"
 
       - name: Clean intermediate data bucket
+        if: github.ref_name != 'main'
         run: >
           pipenv run python src/scripts/manage_artifacts/clean_for_integration_test.py
           --bucket $DEV_INTERMEDIATE_BUCKET

From 4430539f813f3f24f548715909f2a93632fbf6b3 Mon Sep 17 00:00:00 2001
From: BryanFauble <17128019+BryanFauble@users.noreply.github.com>
Date: Mon, 15 Jul 2024 11:05:25 -0700
Subject: [PATCH 2/3] Update when cleanup is ran

---
 .github/workflows/upload-and-deploy.yaml                   | 3 ---
 src/scripts/manage_artifacts/clean_for_integration_test.py | 5 -----
 2 files changed, 8 deletions(-)

diff --git a/.github/workflows/upload-and-deploy.yaml b/.github/workflows/upload-and-deploy.yaml
index 457f353f..8749870a 100755
--- a/.github/workflows/upload-and-deploy.yaml
+++ b/.github/workflows/upload-and-deploy.yaml
@@ -269,7 +269,6 @@ jobs:
       contents: read
     steps:
       - name: Setup code, pipenv, aws
-        if: github.ref_name != 'main'
         uses: Sage-Bionetworks/action-pipenv-aws-setup@v3
         with:
           role_to_assume: ${{ vars.AWS_CREDENTIALS_IAM_ROLE }}
@@ -281,14 +280,12 @@ jobs:
         run: echo "NAMESPACE=$GITHUB_REF_NAME" >> $GITHUB_ENV
 
       - name: Clean input data bucket
-        if: github.ref_name != 'main'
         run: >
           pipenv run python src/scripts/manage_artifacts/clean_for_integration_test.py
           --bucket $DEV_INPUT_BUCKET
           --bucket_prefix "${{ env.NAMESPACE }}/"
 
       - name: Clean intermediate data bucket
-        if: github.ref_name != 'main'
         run: >
           pipenv run python src/scripts/manage_artifacts/clean_for_integration_test.py
           --bucket $DEV_INTERMEDIATE_BUCKET
diff --git a/src/scripts/manage_artifacts/clean_for_integration_test.py b/src/scripts/manage_artifacts/clean_for_integration_test.py
index cb9668a4..e016f2ff 100755
--- a/src/scripts/manage_artifacts/clean_for_integration_test.py
+++ b/src/scripts/manage_artifacts/clean_for_integration_test.py
@@ -42,8 +42,6 @@ def delete_objects(bucket_prefix: str, bucket: str) -> None:
             # Skip the owner.txt file so it does not need to be re-created
             if object_key.endswith("owner.txt"):
                 continue
-            elif "main" in object_key:
-                raise ValueError("Cannot delete objects in the main directory")
 
             s3_client.delete_object(Bucket=bucket, Key=object_key)
 
@@ -56,9 +54,6 @@ def main() -> None:
     if not args.bucket_prefix or args.bucket_prefix[-1] != "/":
         raise ValueError("Bucket prefix must be provided and end with a '/'")
 
-    if "main" in args.bucket_prefix:
-        raise ValueError("Cannot delete objects in the main directory")
-
     try:
         delete_objects(bucket_prefix=args.bucket_prefix, bucket=args.bucket)
     except Exception as ex:

From 6e366260fdaa97dffded87acc1ecadde13787c1b Mon Sep 17 00:00:00 2001
From: BryanFauble <17128019+BryanFauble@users.noreply.github.com>
Date: Mon, 15 Jul 2024 11:31:41 -0700
Subject: [PATCH 3/3] Add guard condition for main and non-dev bucket

---
 src/scripts/manage_artifacts/clean_for_integration_test.py | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/src/scripts/manage_artifacts/clean_for_integration_test.py b/src/scripts/manage_artifacts/clean_for_integration_test.py
index e016f2ff..04196fcb 100755
--- a/src/scripts/manage_artifacts/clean_for_integration_test.py
+++ b/src/scripts/manage_artifacts/clean_for_integration_test.py
@@ -54,6 +54,11 @@ def main() -> None:
     if not args.bucket_prefix or args.bucket_prefix[-1] != "/":
         raise ValueError("Bucket prefix must be provided and end with a '/'")
 
+    if "main" in args.bucket_prefix and "recover-dev" not in args.bucket:
+        raise ValueError(
+            "Cannot delete objects in the main directory of a non-dev bucket"
+        )
+
     try:
         delete_objects(bucket_prefix=args.bucket_prefix, bucket=args.bucket)
     except Exception as ex: