From 40097d93f03a9f50bb9d965d2319f8b72ab428f8 Mon Sep 17 00:00:00 2001 From: Wenjin Situ Date: Thu, 13 Apr 2023 13:19:51 -0700 Subject: [PATCH 1/3] remove github workflow for terraform --- .github/workflows/terraform-cognito-test.yaml | 69 ------------------- .github/workflows/terraform-vanilla-test.yaml | 68 ------------------ 2 files changed, 137 deletions(-) delete mode 100644 .github/workflows/terraform-cognito-test.yaml delete mode 100644 .github/workflows/terraform-vanilla-test.yaml diff --git a/.github/workflows/terraform-cognito-test.yaml b/.github/workflows/terraform-cognito-test.yaml deleted file mode 100644 index af881b5d0e..0000000000 --- a/.github/workflows/terraform-cognito-test.yaml +++ /dev/null @@ -1,69 +0,0 @@ -name: Terraform Cognito Test -on: - # Run on PR configuration - push: - paths: - - deployments/cognito/terraform/** - - iaac/terraform/aws-infra/cognito/** - - iaac/terraform/aws-infra/subdomain/** - # TODO: Add relevant helm chart path - branches: - - main - - # Run on schedule (every 4 hours) - schedule: - - cron: '0 0/4 * * *' - - workflow_dispatch: - -# Ensure that only a single workflow which deploy cognito related stack will be run at a time. TODO: enable parallel runs by creating separate hosted zones -concurrency: cognito - -jobs: - build: - runs-on: ubuntu-latest - # environment enables protection rules (e.g. need approval to run the workflow) - environment: gh-actions-test - permissions: - # needed to interact with GitHub's OIDC Token endpoint - id-token: write - # needed to checkout repository - contents: read - env: - ROOT_DOMAIN_NAME: ${{ secrets.ROOT_HOSTED_ZONE_NAME }} - CLUSTER_REGION: ${{ github.event_name == 'schedule' && 'us-west-2' || secrets.AWS_REGION }} - steps: - - name: Checkout - uses: actions/checkout@v3 - - - name: Configure AWS credentials from Test account - uses: aws-actions/configure-aws-credentials@v1 - with: - role-to-assume: ${{ secrets.PR_BUILD_ROLE }} - role-session-name: prrolesession-${{ github.run_id }}-${{ github.run_attempt }} - aws-region: ${{ secrets.AWS_REGION }} - role-duration-seconds: 14400 - - - name: Install Terraform - uses: hashicorp/setup-terraform@v2 - with: - terraform_version: 1.2.7 - terraform_wrapper: false - - - name: Install Python - uses: actions/setup-python@v4 - with: - python-version: '3.8' - - - name: Plan and apply terraform - run: | - cd tests/e2e - pip install -r requirements.txt - pytest tests/terraform/test_cognito.py -s -q --region $CLUSTER_REGION --root-domain-name $ROOT_DOMAIN_NAME - - - name: Clean up terraform - if: failure() - # retry delete if flakiness present - run: | - cd deployments/cognito/terraform - make delete || make delete || make delete diff --git a/.github/workflows/terraform-vanilla-test.yaml b/.github/workflows/terraform-vanilla-test.yaml deleted file mode 100644 index 93fff44543..0000000000 --- a/.github/workflows/terraform-vanilla-test.yaml +++ /dev/null @@ -1,68 +0,0 @@ -name: Terraform Vanilla Test -on: - # Run on PR configuration - push: - paths: - - deployments/vanilla/terraform/** - - iaac/terraform/apps/** - - iaac/terraform/common/** - - iaac/terraform/utils/** - branches: - - main - - # Run on schedule (every 4 hours) - schedule: - - cron: '0 0/4 * * *' - - workflow_dispatch: - -# Ensure that only a single workflow which deploy tf build related stack will be run at a time. TODO: enable parallel runs -concurrency: tf-build - -jobs: - build: - runs-on: ubuntu-latest - # environment enables protection rules (e.g. need approval to run the workflow) - environment: gh-actions-test - permissions: - # needed to interact with GitHub's OIDC Token endpoint - id-token: write - # needed to checkout repository - contents: read - env: - CLUSTER_REGION: ${{ github.event_name == 'schedule' && 'us-west-2' || secrets.AWS_REGION }} - steps: - - name: Checkout - uses: actions/checkout@v3 - - - name: Configure AWS credentials from Test account - uses: aws-actions/configure-aws-credentials@v1 - with: - role-to-assume: ${{ secrets.PR_BUILD_ROLE }} - role-session-name: prrolesession-${{ github.run_id }}-${{ github.run_attempt }} - aws-region: ${{ github.event_name == 'schedule' && 'us-west-2' || secrets.AWS_REGION }} - role-duration-seconds: 14400 - - - name: Install Terraform - uses: hashicorp/setup-terraform@v2 - with: - terraform_version: 1.2.7 - terraform_wrapper: false - - - name: Install Python - uses: actions/setup-python@v4 - with: - python-version: '3.8' - - - name: Plan and apply terraform - run: | - cd tests/e2e - pip install -r requirements.txt - pytest tests/terraform/test_vanilla.py -s -q --region $CLUSTER_REGION - - - name: Clean up terraform - if: failure() - # retry delete if flakiness present - run: | - cd deployments/vanilla/terraform - make delete || make delete || make delete From 8b0a1254fc97b04063ec8f9b34451ff968316f07 Mon Sep 17 00:00:00 2001 From: Wenjin Situ Date: Tue, 25 Apr 2023 08:32:09 -0700 Subject: [PATCH 2/3] fix s3-static comma --- awsconfigs/apps/pipeline-static/s3/config | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/awsconfigs/apps/pipeline-static/s3/config b/awsconfigs/apps/pipeline-static/s3/config index a345224df6..9bb11706ae 100644 --- a/awsconfigs/apps/pipeline-static/s3/config +++ b/awsconfigs/apps/pipeline-static/s3/config @@ -5,7 +5,7 @@ artifactRepository: bucket: $(kfp-artifact-bucket-name), endpoint: $(kfp-artifact-storage-endpoint), insecure: false, - keyFormat: "artifacts/{{workflow.name}}/{{workflow.creationTimestamp.Y}}/{{workflow.creationTimestamp.m}}/{{workflow.creationTimestamp.d}}/{{pod.name}}"， + keyFormat: "artifacts/{{workflow.name}}/{{workflow.creationTimestamp.Y}}/{{workflow.creationTimestamp.m}}/{{workflow.creationTimestamp.d}}/{{pod.name}}", accessKeySecret: { name: mlpipeline-minio-artifact, key: accesskey From 37d4fb62a8e42ef101287d31581c34bb180536d7 Mon Sep 17 00:00:00 2001 From: Wenjin Situ Date: Tue, 25 Apr 2023 08:33:42 -0700 Subject: [PATCH 3/3] add back workflows --- .github/workflows/terraform-cognito-test.yaml | 69 +++++++++++++++++++ .github/workflows/terraform-vanilla-test.yaml | 68 ++++++++++++++++++ 2 files changed, 137 insertions(+) create mode 100644 .github/workflows/terraform-cognito-test.yaml create mode 100644 .github/workflows/terraform-vanilla-test.yaml diff --git a/.github/workflows/terraform-cognito-test.yaml b/.github/workflows/terraform-cognito-test.yaml new file mode 100644 index 0000000000..af881b5d0e --- /dev/null +++ b/.github/workflows/terraform-cognito-test.yaml @@ -0,0 +1,69 @@ +name: Terraform Cognito Test +on: + # Run on PR configuration + push: + paths: + - deployments/cognito/terraform/** + - iaac/terraform/aws-infra/cognito/** + - iaac/terraform/aws-infra/subdomain/** + # TODO: Add relevant helm chart path + branches: + - main + + # Run on schedule (every 4 hours) + schedule: + - cron: '0 0/4 * * *' + + workflow_dispatch: + +# Ensure that only a single workflow which deploy cognito related stack will be run at a time. TODO: enable parallel runs by creating separate hosted zones +concurrency: cognito + +jobs: + build: + runs-on: ubuntu-latest + # environment enables protection rules (e.g. need approval to run the workflow) + environment: gh-actions-test + permissions: + # needed to interact with GitHub's OIDC Token endpoint + id-token: write + # needed to checkout repository + contents: read + env: + ROOT_DOMAIN_NAME: ${{ secrets.ROOT_HOSTED_ZONE_NAME }} + CLUSTER_REGION: ${{ github.event_name == 'schedule' && 'us-west-2' || secrets.AWS_REGION }} + steps: + - name: Checkout + uses: actions/checkout@v3 + + - name: Configure AWS credentials from Test account + uses: aws-actions/configure-aws-credentials@v1 + with: + role-to-assume: ${{ secrets.PR_BUILD_ROLE }} + role-session-name: prrolesession-${{ github.run_id }}-${{ github.run_attempt }} + aws-region: ${{ secrets.AWS_REGION }} + role-duration-seconds: 14400 + + - name: Install Terraform + uses: hashicorp/setup-terraform@v2 + with: + terraform_version: 1.2.7 + terraform_wrapper: false + + - name: Install Python + uses: actions/setup-python@v4 + with: + python-version: '3.8' + + - name: Plan and apply terraform + run: | + cd tests/e2e + pip install -r requirements.txt + pytest tests/terraform/test_cognito.py -s -q --region $CLUSTER_REGION --root-domain-name $ROOT_DOMAIN_NAME + + - name: Clean up terraform + if: failure() + # retry delete if flakiness present + run: | + cd deployments/cognito/terraform + make delete || make delete || make delete diff --git a/.github/workflows/terraform-vanilla-test.yaml b/.github/workflows/terraform-vanilla-test.yaml new file mode 100644 index 0000000000..93fff44543 --- /dev/null +++ b/.github/workflows/terraform-vanilla-test.yaml @@ -0,0 +1,68 @@ +name: Terraform Vanilla Test +on: + # Run on PR configuration + push: + paths: + - deployments/vanilla/terraform/** + - iaac/terraform/apps/** + - iaac/terraform/common/** + - iaac/terraform/utils/** + branches: + - main + + # Run on schedule (every 4 hours) + schedule: + - cron: '0 0/4 * * *' + + workflow_dispatch: + +# Ensure that only a single workflow which deploy tf build related stack will be run at a time. TODO: enable parallel runs +concurrency: tf-build + +jobs: + build: + runs-on: ubuntu-latest + # environment enables protection rules (e.g. need approval to run the workflow) + environment: gh-actions-test + permissions: + # needed to interact with GitHub's OIDC Token endpoint + id-token: write + # needed to checkout repository + contents: read + env: + CLUSTER_REGION: ${{ github.event_name == 'schedule' && 'us-west-2' || secrets.AWS_REGION }} + steps: + - name: Checkout + uses: actions/checkout@v3 + + - name: Configure AWS credentials from Test account + uses: aws-actions/configure-aws-credentials@v1 + with: + role-to-assume: ${{ secrets.PR_BUILD_ROLE }} + role-session-name: prrolesession-${{ github.run_id }}-${{ github.run_attempt }} + aws-region: ${{ github.event_name == 'schedule' && 'us-west-2' || secrets.AWS_REGION }} + role-duration-seconds: 14400 + + - name: Install Terraform + uses: hashicorp/setup-terraform@v2 + with: + terraform_version: 1.2.7 + terraform_wrapper: false + + - name: Install Python + uses: actions/setup-python@v4 + with: + python-version: '3.8' + + - name: Plan and apply terraform + run: | + cd tests/e2e + pip install -r requirements.txt + pytest tests/terraform/test_vanilla.py -s -q --region $CLUSTER_REGION + + - name: Clean up terraform + if: failure() + # retry delete if flakiness present + run: | + cd deployments/vanilla/terraform + make delete || make delete || make delete