From f5b2390341f79bf843e194770d06f4882741c39b Mon Sep 17 00:00:00 2001 From: Kyryl Skobylko Date: Mon, 12 Dec 2022 15:50:06 +0200 Subject: [PATCH 01/24] add fluent-bit logging to k3s setup --- .github/workflows/gradle.yml | 14 ++++---------- tools/bin/acceptance_test_kube_helm.sh | 8 ++++++++ tools/bin/fluent_values.yaml | 10 ++++++++++ 3 files changed, 22 insertions(+), 10 deletions(-) create mode 100644 tools/bin/fluent_values.yaml diff --git a/.github/workflows/gradle.yml b/.github/workflows/gradle.yml index 4c2f0b4a9e4b..982927306af3 100644 --- a/.github/workflows/gradle.yml +++ b/.github/workflows/gradle.yml @@ -1023,12 +1023,8 @@ jobs: uses: Wandalen/wretry.action@master with: command: | - sudo rm /var/lib/dpkg/lock - sudo rm /var/lib/dpkg/lock-frontend sudo apt-get remove docker.io || sudo apt-get remove docker curl -fsSL https://get.docker.com | bash - - sudo rm -f /var/lib/dpkg/lock - sudo rm -f /var/lib/dpkg/lock-frontend attempt_limit: 3 attempt_delay: 2000 # in ms @@ -1061,20 +1057,16 @@ jobs: version: 'v1.21.2' # default is latest stable - name: Install tmate and it's dependencies - if: inputs.debug_mode == 'true' + # if: inputs.debug_mode == 'true' uses: Wandalen/wretry.action@master with: command: | - sudo rm /var/lib/dpkg/lock - sudo rm /var/lib/dpkg/lock-frontend sudo apt-get -y install tmate - sudo rm -f /var/lib/dpkg/lock - sudo rm -f /var/lib/dpkg/lock-frontend attempt_limit: 3 attempt_delay: 2000 # in ms - name: Start tmate session in background - if: inputs.debug_mode == 'true' + # if: inputs.debug_mode == 'true' shell: bash run: | tmate -S /tmp/tmate.sock new-session -d # Launch tmate in a headless mode @@ -1124,6 +1116,8 @@ jobs: # HOME: /home/runner # AWS_S3_INTEGRATION_TEST_CREDS can be found in LastPass as AWS_S3_INTEGRATION_TEST_CREDS AWS_S3_INTEGRATION_TEST_CREDS: ${{ secrets.AWS_S3_INTEGRATION_TEST_CREDS }} + WORKFLOW_RUN_ID: ${{ github.run_id }} + WORKFLOW_RUN_NUMBER: ${{ github.run_number }} SECRET_STORE_GCP_CREDENTIALS: ${{ secrets.SECRET_STORE_GCP_CREDENTIALS }} SECRET_STORE_GCP_PROJECT_ID: ${{ secrets.SECRET_STORE_GCP_PROJECT_ID }} timeout-minutes: 20 diff --git a/tools/bin/acceptance_test_kube_helm.sh b/tools/bin/acceptance_test_kube_helm.sh index d3445766f605..87796ffe5a23 100755 --- a/tools/bin/acceptance_test_kube_helm.sh +++ b/tools/bin/acceptance_test_kube_helm.sh @@ -16,6 +16,14 @@ kubectl patch configmap/coredns \ --type merge \ -p '{"data":{"NodeHosts": "${DOCKER_HOST_IP} host.docker.internal" }}' +echo "Getting key value" +AWS_ACCESS_KEY_ID="$(echo "$AWS_S3_INTEGRATION_TEST_CREDS" | jq -r .aws_access_key_id)" +AWS_SECRET_ACCESS_KEY="$(echo "$AWS_S3_INTEGRATION_TEST_CREDS" | jq -r .aws_secret_access_key)" + +echo "Deploying fluentbit" +helm repo add fluent https://fluent.github.io/helm-charts +helm repo update fluent +helm install --values tools/bin/fluent_values.yaml --set env.AWS_ACCESS_KEY_ID=$(echo "$AWS_S3_INTEGRATION_TEST_CREDS" | jq -r .aws_access_key_id),env.AWS_SECRET_ACCESS_KEY=$(echo "$AWS_S3_INTEGRATION_TEST_CREDS" | jq -r .aws_secret_access_key),env.WORKFLOW_RUN_ID=${WORKFLOW_RUN_ID},env.WORKFLOW_RUN_NUBMER=${WORKFLOW_RUN_NUBMER} fluent-bit fluent/fluent-bit echo "Replacing default Chart.yaml and values.yaml with a test one" mv charts/airbyte/Chart.yaml charts/airbyte/Chart.yaml.old diff --git a/tools/bin/fluent_values.yaml b/tools/bin/fluent_values.yaml new file mode 100644 index 000000000000..2641f808167a --- /dev/null +++ b/tools/bin/fluent_values.yaml @@ -0,0 +1,10 @@ +outputs: | + [OUTPUT] + Name s3 + Match kube.* + bucket ${AWS_S3_BUCKET} + region us-east-1 + total_file_size 1M + upload_timeout 1m + use_put_object On + s3_key_format /k3s-logs/${WORKFLOW_RUN_ID}/${WORKFLOW_RUN_NUMBER}/$TAG/%Y/%m/%d/%H/%M From 0a89af791bdea4af91423bebf6e2d0239a75d305 Mon Sep 17 00:00:00 2001 From: Kyryl Skobylko Date: Mon, 12 Dec 2022 16:33:39 +0200 Subject: [PATCH 02/24] Fix --set for helm --- tools/bin/acceptance_test_kube_helm.sh | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/tools/bin/acceptance_test_kube_helm.sh b/tools/bin/acceptance_test_kube_helm.sh index 87796ffe5a23..9ebc20ac88d3 100755 --- a/tools/bin/acceptance_test_kube_helm.sh +++ b/tools/bin/acceptance_test_kube_helm.sh @@ -23,7 +23,11 @@ AWS_SECRET_ACCESS_KEY="$(echo "$AWS_S3_INTEGRATION_TEST_CREDS" | jq -r .aws_secr echo "Deploying fluentbit" helm repo add fluent https://fluent.github.io/helm-charts helm repo update fluent -helm install --values tools/bin/fluent_values.yaml --set env.AWS_ACCESS_KEY_ID=$(echo "$AWS_S3_INTEGRATION_TEST_CREDS" | jq -r .aws_access_key_id),env.AWS_SECRET_ACCESS_KEY=$(echo "$AWS_S3_INTEGRATION_TEST_CREDS" | jq -r .aws_secret_access_key),env.WORKFLOW_RUN_ID=${WORKFLOW_RUN_ID},env.WORKFLOW_RUN_NUBMER=${WORKFLOW_RUN_NUBMER} fluent-bit fluent/fluent-bit +helm install --values tools/bin/fluent_values.yaml --set env[0].AWS_ACCESS_KEY_ID=$(echo "$AWS_S3_INTEGRATION_TEST_CREDS" | jq -r .aws_access_key_id) \\ + --set env[1].AWS_SECRET_ACCESS_KEY=$(echo "$AWS_S3_INTEGRATION_TEST_CREDS" | jq -r .aws_secret_access_key) \\ + --set env[2].WORKFLOW_RUN_ID=${WORKFLOW_RUN_ID} \\ + --set env[3].WORKFLOW_RUN_NUBMER=${WORKFLOW_RUN_NUBMER} \\ + fluent-bit fluent/fluent-bit echo "Replacing default Chart.yaml and values.yaml with a test one" mv charts/airbyte/Chart.yaml charts/airbyte/Chart.yaml.old From 74f038d349e85c7e99e20a779c52836dc7706683 Mon Sep 17 00:00:00 2001 From: Kyryl Skobylko Date: Mon, 12 Dec 2022 17:01:52 +0200 Subject: [PATCH 03/24] add --generate-name --- tools/bin/acceptance_test_kube_helm.sh | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/tools/bin/acceptance_test_kube_helm.sh b/tools/bin/acceptance_test_kube_helm.sh index 9ebc20ac88d3..80e8e4018dd1 100755 --- a/tools/bin/acceptance_test_kube_helm.sh +++ b/tools/bin/acceptance_test_kube_helm.sh @@ -16,9 +16,6 @@ kubectl patch configmap/coredns \ --type merge \ -p '{"data":{"NodeHosts": "${DOCKER_HOST_IP} host.docker.internal" }}' -echo "Getting key value" -AWS_ACCESS_KEY_ID="$(echo "$AWS_S3_INTEGRATION_TEST_CREDS" | jq -r .aws_access_key_id)" -AWS_SECRET_ACCESS_KEY="$(echo "$AWS_S3_INTEGRATION_TEST_CREDS" | jq -r .aws_secret_access_key)" echo "Deploying fluentbit" helm repo add fluent https://fluent.github.io/helm-charts @@ -27,7 +24,7 @@ helm install --values tools/bin/fluent_values.yaml --set env[0].AWS_ACCESS_KEY_I --set env[1].AWS_SECRET_ACCESS_KEY=$(echo "$AWS_S3_INTEGRATION_TEST_CREDS" | jq -r .aws_secret_access_key) \\ --set env[2].WORKFLOW_RUN_ID=${WORKFLOW_RUN_ID} \\ --set env[3].WORKFLOW_RUN_NUBMER=${WORKFLOW_RUN_NUBMER} \\ - fluent-bit fluent/fluent-bit + --generate-name fluent/fluent-bit echo "Replacing default Chart.yaml and values.yaml with a test one" mv charts/airbyte/Chart.yaml charts/airbyte/Chart.yaml.old From 264342bbd407b6f1795a5ca7c75b999f07cedb37 Mon Sep 17 00:00:00 2001 From: Kyryl Skobylko Date: Mon, 12 Dec 2022 17:35:20 +0200 Subject: [PATCH 04/24] fix slash in multiline bash --- tools/bin/acceptance_test_kube_helm.sh | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/tools/bin/acceptance_test_kube_helm.sh b/tools/bin/acceptance_test_kube_helm.sh index 80e8e4018dd1..2e413c486fab 100755 --- a/tools/bin/acceptance_test_kube_helm.sh +++ b/tools/bin/acceptance_test_kube_helm.sh @@ -20,11 +20,11 @@ kubectl patch configmap/coredns \ echo "Deploying fluentbit" helm repo add fluent https://fluent.github.io/helm-charts helm repo update fluent -helm install --values tools/bin/fluent_values.yaml --set env[0].AWS_ACCESS_KEY_ID=$(echo "$AWS_S3_INTEGRATION_TEST_CREDS" | jq -r .aws_access_key_id) \\ - --set env[1].AWS_SECRET_ACCESS_KEY=$(echo "$AWS_S3_INTEGRATION_TEST_CREDS" | jq -r .aws_secret_access_key) \\ - --set env[2].WORKFLOW_RUN_ID=${WORKFLOW_RUN_ID} \\ - --set env[3].WORKFLOW_RUN_NUBMER=${WORKFLOW_RUN_NUBMER} \\ - --generate-name fluent/fluent-bit +helm install --values tools/bin/fluent_values.yaml --set env[0].AWS_ACCESS_KEY_ID=$(echo "$AWS_S3_INTEGRATION_TEST_CREDS" | jq -r .aws_access_key_id) \ + --set env[1].AWS_SECRET_ACCESS_KEY=$(echo "$AWS_S3_INTEGRATION_TEST_CREDS" | jq -r .aws_secret_access_key) \ + --set env[2].WORKFLOW_RUN_ID=${WORKFLOW_RUN_ID} \ + --set env[3].WORKFLOW_RUN_NUBMER=${WORKFLOW_RUN_NUBMER} \ + fluent-bit fluent/fluent-bit echo "Replacing default Chart.yaml and values.yaml with a test one" mv charts/airbyte/Chart.yaml charts/airbyte/Chart.yaml.old From b08d2f2a085da3f7597fbcbbe3a93c2cb87db280 Mon Sep 17 00:00:00 2001 From: Kyryl Skobylko Date: Tue, 13 Dec 2022 10:21:01 +0200 Subject: [PATCH 05/24] Update --set for helm --- .github/workflows/gradle.yml | 1 + tools/bin/acceptance_test_kube_helm.sh | 9 +++++---- tools/bin/fluent_values.yaml | 2 +- 3 files changed, 7 insertions(+), 5 deletions(-) diff --git a/.github/workflows/gradle.yml b/.github/workflows/gradle.yml index 982927306af3..2c7d6300340b 100644 --- a/.github/workflows/gradle.yml +++ b/.github/workflows/gradle.yml @@ -1118,6 +1118,7 @@ jobs: AWS_S3_INTEGRATION_TEST_CREDS: ${{ secrets.AWS_S3_INTEGRATION_TEST_CREDS }} WORKFLOW_RUN_ID: ${{ github.run_id }} WORKFLOW_RUN_NUMBER: ${{ github.run_number }} + AWS_S3_BUCKET: airbyte-buildcache SECRET_STORE_GCP_CREDENTIALS: ${{ secrets.SECRET_STORE_GCP_CREDENTIALS }} SECRET_STORE_GCP_PROJECT_ID: ${{ secrets.SECRET_STORE_GCP_PROJECT_ID }} timeout-minutes: 20 diff --git a/tools/bin/acceptance_test_kube_helm.sh b/tools/bin/acceptance_test_kube_helm.sh index 2e413c486fab..b99434d5b58b 100755 --- a/tools/bin/acceptance_test_kube_helm.sh +++ b/tools/bin/acceptance_test_kube_helm.sh @@ -20,10 +20,11 @@ kubectl patch configmap/coredns \ echo "Deploying fluentbit" helm repo add fluent https://fluent.github.io/helm-charts helm repo update fluent -helm install --values tools/bin/fluent_values.yaml --set env[0].AWS_ACCESS_KEY_ID=$(echo "$AWS_S3_INTEGRATION_TEST_CREDS" | jq -r .aws_access_key_id) \ - --set env[1].AWS_SECRET_ACCESS_KEY=$(echo "$AWS_S3_INTEGRATION_TEST_CREDS" | jq -r .aws_secret_access_key) \ - --set env[2].WORKFLOW_RUN_ID=${WORKFLOW_RUN_ID} \ - --set env[3].WORKFLOW_RUN_NUBMER=${WORKFLOW_RUN_NUBMER} \ +helm install --values tools/bin/fluent_values.yaml --set env[0].name="AWS_ACCESS_KEY_ID" --set env[0].value=$(echo "$AWS_S3_INTEGRATION_TEST_CREDS" | jq -r .aws_access_key_id) \ + --set env[1].name="AWS_SECRET_ACCESS_KEY" --set env[1].value=$(echo "$AWS_S3_INTEGRATION_TEST_CREDS" | jq -r .aws_secret_access_key) \ + --set env[2].name="WORKFLOW_RUN_ID" --set env[2].value=${WORKFLOW_RUN_ID} \ + --set env[3].name="WORKFLOW_RUN_NUBMER" --set env[3].value=${WORKFLOW_RUN_NUBMER} \ + --set env[4].name="AWS_S3_BUCKET" --set env[4].value=${AWS_S3_BUCKET} \ fluent-bit fluent/fluent-bit echo "Replacing default Chart.yaml and values.yaml with a test one" diff --git a/tools/bin/fluent_values.yaml b/tools/bin/fluent_values.yaml index 2641f808167a..b6f88a9eca60 100644 --- a/tools/bin/fluent_values.yaml +++ b/tools/bin/fluent_values.yaml @@ -3,7 +3,7 @@ outputs: | Name s3 Match kube.* bucket ${AWS_S3_BUCKET} - region us-east-1 + region us-east-2 total_file_size 1M upload_timeout 1m use_put_object On From 45d0d9f9a8450f98336806bbf804bbe4ff18d5e8 Mon Sep 17 00:00:00 2001 From: Kyryl Skobylko Date: Tue, 13 Dec 2022 10:34:08 +0200 Subject: [PATCH 06/24] Use generate-name instead --- tools/bin/acceptance_test_kube_helm.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/bin/acceptance_test_kube_helm.sh b/tools/bin/acceptance_test_kube_helm.sh index b99434d5b58b..e8c6591167b1 100755 --- a/tools/bin/acceptance_test_kube_helm.sh +++ b/tools/bin/acceptance_test_kube_helm.sh @@ -25,7 +25,7 @@ helm install --values tools/bin/fluent_values.yaml --set env[0].name="AWS_ACCESS --set env[2].name="WORKFLOW_RUN_ID" --set env[2].value=${WORKFLOW_RUN_ID} \ --set env[3].name="WORKFLOW_RUN_NUBMER" --set env[3].value=${WORKFLOW_RUN_NUBMER} \ --set env[4].name="AWS_S3_BUCKET" --set env[4].value=${AWS_S3_BUCKET} \ - fluent-bit fluent/fluent-bit + --generate-name fluent/fluent-bit echo "Replacing default Chart.yaml and values.yaml with a test one" mv charts/airbyte/Chart.yaml charts/airbyte/Chart.yaml.old From e2ac39063daa1dfb3ef35d61540522c70bfe6b09 Mon Sep 17 00:00:00 2001 From: Kyryl Skobylko Date: Tue, 13 Dec 2022 10:50:17 +0200 Subject: [PATCH 07/24] Add debug --- tools/bin/acceptance_test_kube_helm.sh | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/tools/bin/acceptance_test_kube_helm.sh b/tools/bin/acceptance_test_kube_helm.sh index e8c6591167b1..ca86c7c33f09 100755 --- a/tools/bin/acceptance_test_kube_helm.sh +++ b/tools/bin/acceptance_test_kube_helm.sh @@ -20,6 +20,13 @@ kubectl patch configmap/coredns \ echo "Deploying fluentbit" helm repo add fluent https://fluent.github.io/helm-charts helm repo update fluent +helm template --values tools/bin/fluent_values.yaml --set env[0].name="AWS_ACCESS_KEY_ID" --set env[0].value=$(echo "$AWS_S3_INTEGRATION_TEST_CREDS" | jq -r .aws_access_key_id) \ + --set env[1].name="AWS_SECRET_ACCESS_KEY" --set env[1].value=$(echo "$AWS_S3_INTEGRATION_TEST_CREDS" | jq -r .aws_secret_access_key) \ + --set env[2].name="WORKFLOW_RUN_ID" --set env[2].value=${WORKFLOW_RUN_ID} \ + --set env[3].name="WORKFLOW_RUN_NUBMER" --set env[3].value=${WORKFLOW_RUN_NUBMER} \ + --set env[4].name="AWS_S3_BUCKET" --set env[4].value=${AWS_S3_BUCKET} \ + --generate-name fluent/fluent-bit + helm install --values tools/bin/fluent_values.yaml --set env[0].name="AWS_ACCESS_KEY_ID" --set env[0].value=$(echo "$AWS_S3_INTEGRATION_TEST_CREDS" | jq -r .aws_access_key_id) \ --set env[1].name="AWS_SECRET_ACCESS_KEY" --set env[1].value=$(echo "$AWS_S3_INTEGRATION_TEST_CREDS" | jq -r .aws_secret_access_key) \ --set env[2].name="WORKFLOW_RUN_ID" --set env[2].value=${WORKFLOW_RUN_ID} \ From 0d468ce27b022aafa150d581fe2b013d46c14a39 Mon Sep 17 00:00:00 2001 From: Kyryl Skobylko Date: Tue, 13 Dec 2022 11:12:06 +0200 Subject: [PATCH 08/24] Remove deubg for helm --- .github/workflows/gradle.yml | 1 - tools/bin/acceptance_test_kube_helm.sh | 8 -------- tools/bin/fluent_values.yaml | 2 +- 3 files changed, 1 insertion(+), 10 deletions(-) diff --git a/.github/workflows/gradle.yml b/.github/workflows/gradle.yml index 2c7d6300340b..625dd578c587 100644 --- a/.github/workflows/gradle.yml +++ b/.github/workflows/gradle.yml @@ -1117,7 +1117,6 @@ jobs: # AWS_S3_INTEGRATION_TEST_CREDS can be found in LastPass as AWS_S3_INTEGRATION_TEST_CREDS AWS_S3_INTEGRATION_TEST_CREDS: ${{ secrets.AWS_S3_INTEGRATION_TEST_CREDS }} WORKFLOW_RUN_ID: ${{ github.run_id }} - WORKFLOW_RUN_NUMBER: ${{ github.run_number }} AWS_S3_BUCKET: airbyte-buildcache SECRET_STORE_GCP_CREDENTIALS: ${{ secrets.SECRET_STORE_GCP_CREDENTIALS }} SECRET_STORE_GCP_PROJECT_ID: ${{ secrets.SECRET_STORE_GCP_PROJECT_ID }} diff --git a/tools/bin/acceptance_test_kube_helm.sh b/tools/bin/acceptance_test_kube_helm.sh index ca86c7c33f09..514ea2b07f04 100755 --- a/tools/bin/acceptance_test_kube_helm.sh +++ b/tools/bin/acceptance_test_kube_helm.sh @@ -20,17 +20,9 @@ kubectl patch configmap/coredns \ echo "Deploying fluentbit" helm repo add fluent https://fluent.github.io/helm-charts helm repo update fluent -helm template --values tools/bin/fluent_values.yaml --set env[0].name="AWS_ACCESS_KEY_ID" --set env[0].value=$(echo "$AWS_S3_INTEGRATION_TEST_CREDS" | jq -r .aws_access_key_id) \ - --set env[1].name="AWS_SECRET_ACCESS_KEY" --set env[1].value=$(echo "$AWS_S3_INTEGRATION_TEST_CREDS" | jq -r .aws_secret_access_key) \ - --set env[2].name="WORKFLOW_RUN_ID" --set env[2].value=${WORKFLOW_RUN_ID} \ - --set env[3].name="WORKFLOW_RUN_NUBMER" --set env[3].value=${WORKFLOW_RUN_NUBMER} \ - --set env[4].name="AWS_S3_BUCKET" --set env[4].value=${AWS_S3_BUCKET} \ - --generate-name fluent/fluent-bit - helm install --values tools/bin/fluent_values.yaml --set env[0].name="AWS_ACCESS_KEY_ID" --set env[0].value=$(echo "$AWS_S3_INTEGRATION_TEST_CREDS" | jq -r .aws_access_key_id) \ --set env[1].name="AWS_SECRET_ACCESS_KEY" --set env[1].value=$(echo "$AWS_S3_INTEGRATION_TEST_CREDS" | jq -r .aws_secret_access_key) \ --set env[2].name="WORKFLOW_RUN_ID" --set env[2].value=${WORKFLOW_RUN_ID} \ - --set env[3].name="WORKFLOW_RUN_NUBMER" --set env[3].value=${WORKFLOW_RUN_NUBMER} \ --set env[4].name="AWS_S3_BUCKET" --set env[4].value=${AWS_S3_BUCKET} \ --generate-name fluent/fluent-bit diff --git a/tools/bin/fluent_values.yaml b/tools/bin/fluent_values.yaml index b6f88a9eca60..6ab69bc0d730 100644 --- a/tools/bin/fluent_values.yaml +++ b/tools/bin/fluent_values.yaml @@ -7,4 +7,4 @@ outputs: | total_file_size 1M upload_timeout 1m use_put_object On - s3_key_format /k3s-logs/${WORKFLOW_RUN_ID}/${WORKFLOW_RUN_NUMBER}/$TAG/%Y/%m/%d/%H/%M + s3_key_format /k3s-logs/${WORKFLOW_RUN_ID}/$TAG/%Y/%m/%d/%H/%M From 9763bb2d9f817c6210b163dda767fa328028bcab Mon Sep 17 00:00:00 2001 From: Kyryl Skobylko Date: Tue, 13 Dec 2022 11:24:51 +0200 Subject: [PATCH 09/24] Fix env again --- tools/bin/acceptance_test_kube_helm.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/bin/acceptance_test_kube_helm.sh b/tools/bin/acceptance_test_kube_helm.sh index 514ea2b07f04..8dc34ebbba27 100755 --- a/tools/bin/acceptance_test_kube_helm.sh +++ b/tools/bin/acceptance_test_kube_helm.sh @@ -23,7 +23,7 @@ helm repo update fluent helm install --values tools/bin/fluent_values.yaml --set env[0].name="AWS_ACCESS_KEY_ID" --set env[0].value=$(echo "$AWS_S3_INTEGRATION_TEST_CREDS" | jq -r .aws_access_key_id) \ --set env[1].name="AWS_SECRET_ACCESS_KEY" --set env[1].value=$(echo "$AWS_S3_INTEGRATION_TEST_CREDS" | jq -r .aws_secret_access_key) \ --set env[2].name="WORKFLOW_RUN_ID" --set env[2].value=${WORKFLOW_RUN_ID} \ - --set env[4].name="AWS_S3_BUCKET" --set env[4].value=${AWS_S3_BUCKET} \ + --set env[3].name="AWS_S3_BUCKET" --set env[3].value=${AWS_S3_BUCKET} \ --generate-name fluent/fluent-bit echo "Replacing default Chart.yaml and values.yaml with a test one" From 1fbfd9fba46f594ac4ba2adc387a235e443dab26 Mon Sep 17 00:00:00 2001 From: Kyryl Skobylko Date: Tue, 13 Dec 2022 11:25:32 +0200 Subject: [PATCH 10/24] Disable build --- .github/workflows/gradle.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/gradle.yml b/.github/workflows/gradle.yml index 625dd578c587..e14f920886b6 100644 --- a/.github/workflows/gradle.yml +++ b/.github/workflows/gradle.yml @@ -1103,12 +1103,12 @@ jobs: org.gradle.vfs.watch=false EOF - - name: Build Platform Docker Images - uses: Wandalen/wretry.action@master - with: - command: pwd && SUB_BUILD=PLATFORM ./gradlew assemble -x test --scan - attempt_limit: 3 - attempt_delay: 5000 # in ms + # - name: Build Platform Docker Images + # uses: Wandalen/wretry.action@master + # with: + # command: pwd && SUB_BUILD=PLATFORM ./gradlew assemble -x test --scan + # attempt_limit: 3 + # attempt_delay: 5000 # in ms - name: Run Helm End-to-End Acceptance Tests env: From e7efe75720fab1220d6ce30a8bbe331f554f71b9 Mon Sep 17 00:00:00 2001 From: Kyryl Skobylko Date: Tue, 13 Dec 2022 14:12:14 +0200 Subject: [PATCH 11/24] Remove everything --- .github/workflows/gradle.yml | 1720 ++++++++++++------------ tools/bin/acceptance_test_kube_helm.sh | 6 + 2 files changed, 866 insertions(+), 860 deletions(-) diff --git a/.github/workflows/gradle.yml b/.github/workflows/gradle.yml index e14f920886b6..3bc4f5546ec9 100644 --- a/.github/workflows/gradle.yml +++ b/.github/workflows/gradle.yml @@ -93,872 +93,872 @@ jobs: # echo '${{ toJSON(needs) }}' ## BUILDS - octavia-cli-build: - needs: changes - runs-on: ubuntu-latest - # Because scheduled builds on master require us to skip the changes job. Use always() to force this to run on master. - if: needs.changes.outputs.cli == 'true' || needs.changes.outputs.build == 'true' || (always() && github.ref == 'refs/heads/master') - name: "Octavia CLI: Build" - timeout-minutes: 90 - steps: - - name: Checkout Airbyte - uses: actions/checkout@v3 - - - name: Cache Build Artifacts - uses: ./.github/actions/cache-build-artifacts - with: - cache-key: ${{ secrets.CACHE_VERSION }} - cache-python: "false" - - - uses: actions/setup-java@v3 - with: - distribution: "zulu" - java-version: "17" - - - uses: actions/setup-python@v4 - with: - python-version: "3.9" - - - name: Set up CI Gradle Properties - run: | - mkdir -p ~/.gradle/ - cat > ~/.gradle/gradle.properties < ~/.gradle/gradle.properties < ~/.gradle/gradle.properties < ~/.gradle/gradle.properties < ~/.gradle/gradle.properties < ~/.gradle/gradle.properties < ~/.gradle/gradle.properties < ~/.gradle/gradle.properties < ~/.gradle/gradle.properties < /tmp/kind-config.yaml < ~/.gradle/gradle.properties < ~/.gradle/gradle.properties < ~/.gradle/gradle.properties < /tmp/kind-config.yaml < Date: Tue, 13 Dec 2022 14:12:53 +0200 Subject: [PATCH 12/24] Disable everything again --- .github/workflows/gradle.yml | 116 +++++++++++++++++------------------ 1 file changed, 58 insertions(+), 58 deletions(-) diff --git a/.github/workflows/gradle.yml b/.github/workflows/gradle.yml index 3bc4f5546ec9..d4030eea49cb 100644 --- a/.github/workflows/gradle.yml +++ b/.github/workflows/gradle.yml @@ -1195,61 +1195,61 @@ jobs: - notify-failure-slack-channel: - name: "Notify Slack Channel on Build Failures" - runs-on: ubuntu-latest - needs: - - build-connectors-base - - frontend-build - - octavia-cli-build - - platform-build - - kube-acceptance-test - # Todo: Kyryl turn this on. - # - helm-acceptance-test - if: ${{ failure() && github.ref == 'refs/heads/master' }} - steps: - - name: Publish to OSS Build Failure Slack Channel - uses: abinoda/slack-action@master - env: - SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN_AIRBYTE_TEAM }} - with: - args: >- - {\"channel\":\"C03BEADRPNY\", \"blocks\":[ - {\"type\":\"divider\"}, - {\"type\":\"section\",\"text\":{\"type\":\"mrkdwn\",\"text\":\" Merge to OSS Master failed! :bangbang: \n\n\"}}, - {\"type\":\"section\",\"text\":{\"type\":\"mrkdwn\",\"text\":\"_merged by_: *${{ github.actor }}* \n\"}}, - {\"type\":\"section\",\"text\":{\"type\":\"mrkdwn\",\"text\":\" :octavia-shocked: :octavia-shocked: \n\"}}, - {\"type\":\"divider\"}]} - - notify-failure-slack-channel-fixed-broken-build: - name: "Notify Slack Channel on Build Fixes" - runs-on: ubuntu-latest - needs: - - build-connectors-base - - frontend-build - - octavia-cli-build - - platform-build - - kube-acceptance-test - # Todo: Kyryl turn this on. - # - helm-acceptance-test - if: success() - steps: - - name: Get Previous Workflow Status - uses: Mercymeilya/last-workflow-status@v0.3 - id: last_status - with: - github_token: ${{ secrets.GITHUB_TOKEN }} - # To avoid clogging up the channel, only publish build success if the previous build was a failure since this means the build was fixed. - - name: Publish Build Fixed Message to OSS Build Failure Slack Channel - if: ${{ steps.last_status.outputs.last_status == 'failure' }} - uses: abinoda/slack-action@master - env: - SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN_AIRBYTE_TEAM }} - with: - args: >- - {\"channel\":\"C03BEADRPNY\", \"blocks\":[ - {\"type\":\"divider\"}, - {\"type\":\"section\",\"text\":{\"type\":\"mrkdwn\",\"text\":\" OSS Master Fixed! :white_check_mark: \n\n\"}}, - {\"type\":\"section\",\"text\":{\"type\":\"mrkdwn\",\"text\":\"_merged by_: *${{ github.actor }}* \n\"}}, - {\"type\":\"section\",\"text\":{\"type\":\"mrkdwn\",\"text\":\" :octavia-rocket: :octavia-rocket: \n\"}}, - {\"type\":\"divider\"}]} + # notify-failure-slack-channel: + # name: "Notify Slack Channel on Build Failures" + # runs-on: ubuntu-latest + # needs: + # - build-connectors-base + # - frontend-build + # - octavia-cli-build + # - platform-build + # - kube-acceptance-test + # # Todo: Kyryl turn this on. + # # - helm-acceptance-test + # if: ${{ failure() && github.ref == 'refs/heads/master' }} + # steps: + # - name: Publish to OSS Build Failure Slack Channel + # uses: abinoda/slack-action@master + # env: + # SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN_AIRBYTE_TEAM }} + # with: + # args: >- + # {\"channel\":\"C03BEADRPNY\", \"blocks\":[ + # {\"type\":\"divider\"}, + # {\"type\":\"section\",\"text\":{\"type\":\"mrkdwn\",\"text\":\" Merge to OSS Master failed! :bangbang: \n\n\"}}, + # {\"type\":\"section\",\"text\":{\"type\":\"mrkdwn\",\"text\":\"_merged by_: *${{ github.actor }}* \n\"}}, + # {\"type\":\"section\",\"text\":{\"type\":\"mrkdwn\",\"text\":\" :octavia-shocked: :octavia-shocked: \n\"}}, + # {\"type\":\"divider\"}]} + + # notify-failure-slack-channel-fixed-broken-build: + # name: "Notify Slack Channel on Build Fixes" + # runs-on: ubuntu-latest + # needs: + # - build-connectors-base + # - frontend-build + # - octavia-cli-build + # - platform-build + # - kube-acceptance-test + # # Todo: Kyryl turn this on. + # # - helm-acceptance-test + # if: success() + # steps: + # - name: Get Previous Workflow Status + # uses: Mercymeilya/last-workflow-status@v0.3 + # id: last_status + # with: + # github_token: ${{ secrets.GITHUB_TOKEN }} + # # To avoid clogging up the channel, only publish build success if the previous build was a failure since this means the build was fixed. + # - name: Publish Build Fixed Message to OSS Build Failure Slack Channel + # if: ${{ steps.last_status.outputs.last_status == 'failure' }} + # uses: abinoda/slack-action@master + # env: + # SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN_AIRBYTE_TEAM }} + # with: + # args: >- + # {\"channel\":\"C03BEADRPNY\", \"blocks\":[ + # {\"type\":\"divider\"}, + # {\"type\":\"section\",\"text\":{\"type\":\"mrkdwn\",\"text\":\" OSS Master Fixed! :white_check_mark: \n\n\"}}, + # {\"type\":\"section\",\"text\":{\"type\":\"mrkdwn\",\"text\":\"_merged by_: *${{ github.actor }}* \n\"}}, + # {\"type\":\"section\",\"text\":{\"type\":\"mrkdwn\",\"text\":\" :octavia-rocket: :octavia-rocket: \n\"}}, + # {\"type\":\"divider\"}]} From 77483283bfdf5fbe07ede5ac946233cbceef1b1d Mon Sep 17 00:00:00 2001 From: Kyryl Skobylko Date: Tue, 13 Dec 2022 14:27:28 +0200 Subject: [PATCH 13/24] Snap back to reallity --- .github/workflows/gradle.yml | 1850 ++++++++++++------------ tools/bin/acceptance_test_kube_helm.sh | 10 +- 2 files changed, 927 insertions(+), 933 deletions(-) diff --git a/.github/workflows/gradle.yml b/.github/workflows/gradle.yml index d4030eea49cb..625dd578c587 100644 --- a/.github/workflows/gradle.yml +++ b/.github/workflows/gradle.yml @@ -93,872 +93,872 @@ jobs: # echo '${{ toJSON(needs) }}' ## BUILDS - # octavia-cli-build: - # needs: changes - # runs-on: ubuntu-latest - # # Because scheduled builds on master require us to skip the changes job. Use always() to force this to run on master. - # if: needs.changes.outputs.cli == 'true' || needs.changes.outputs.build == 'true' || (always() && github.ref == 'refs/heads/master') - # name: "Octavia CLI: Build" - # timeout-minutes: 90 - # steps: - # - name: Checkout Airbyte - # uses: actions/checkout@v3 - - # - name: Cache Build Artifacts - # uses: ./.github/actions/cache-build-artifacts - # with: - # cache-key: ${{ secrets.CACHE_VERSION }} - # cache-python: "false" - - # - uses: actions/setup-java@v3 - # with: - # distribution: "zulu" - # java-version: "17" - - # - uses: actions/setup-python@v4 - # with: - # python-version: "3.9" - - # - name: Set up CI Gradle Properties - # run: | - # mkdir -p ~/.gradle/ - # cat > ~/.gradle/gradle.properties < ~/.gradle/gradle.properties < ~/.gradle/gradle.properties < ~/.gradle/gradle.properties < ~/.gradle/gradle.properties < ~/.gradle/gradle.properties < ~/.gradle/gradle.properties < ~/.gradle/gradle.properties < ~/.gradle/gradle.properties < ~/.gradle/gradle.properties < ~/.gradle/gradle.properties < /tmp/kind-config.yaml < ~/.gradle/gradle.properties < /tmp/kind-config.yaml <- - # {\"channel\":\"C03BEADRPNY\", \"blocks\":[ - # {\"type\":\"divider\"}, - # {\"type\":\"section\",\"text\":{\"type\":\"mrkdwn\",\"text\":\" Merge to OSS Master failed! :bangbang: \n\n\"}}, - # {\"type\":\"section\",\"text\":{\"type\":\"mrkdwn\",\"text\":\"_merged by_: *${{ github.actor }}* \n\"}}, - # {\"type\":\"section\",\"text\":{\"type\":\"mrkdwn\",\"text\":\" :octavia-shocked: :octavia-shocked: \n\"}}, - # {\"type\":\"divider\"}]} - - # notify-failure-slack-channel-fixed-broken-build: - # name: "Notify Slack Channel on Build Fixes" - # runs-on: ubuntu-latest - # needs: - # - build-connectors-base - # - frontend-build - # - octavia-cli-build - # - platform-build - # - kube-acceptance-test - # # Todo: Kyryl turn this on. - # # - helm-acceptance-test - # if: success() - # steps: - # - name: Get Previous Workflow Status - # uses: Mercymeilya/last-workflow-status@v0.3 - # id: last_status - # with: - # github_token: ${{ secrets.GITHUB_TOKEN }} - # # To avoid clogging up the channel, only publish build success if the previous build was a failure since this means the build was fixed. - # - name: Publish Build Fixed Message to OSS Build Failure Slack Channel - # if: ${{ steps.last_status.outputs.last_status == 'failure' }} - # uses: abinoda/slack-action@master - # env: - # SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN_AIRBYTE_TEAM }} - # with: - # args: >- - # {\"channel\":\"C03BEADRPNY\", \"blocks\":[ - # {\"type\":\"divider\"}, - # {\"type\":\"section\",\"text\":{\"type\":\"mrkdwn\",\"text\":\" OSS Master Fixed! :white_check_mark: \n\n\"}}, - # {\"type\":\"section\",\"text\":{\"type\":\"mrkdwn\",\"text\":\"_merged by_: *${{ github.actor }}* \n\"}}, - # {\"type\":\"section\",\"text\":{\"type\":\"mrkdwn\",\"text\":\" :octavia-rocket: :octavia-rocket: \n\"}}, - # {\"type\":\"divider\"}]} + notify-failure-slack-channel: + name: "Notify Slack Channel on Build Failures" + runs-on: ubuntu-latest + needs: + - build-connectors-base + - frontend-build + - octavia-cli-build + - platform-build + - kube-acceptance-test + # Todo: Kyryl turn this on. + # - helm-acceptance-test + if: ${{ failure() && github.ref == 'refs/heads/master' }} + steps: + - name: Publish to OSS Build Failure Slack Channel + uses: abinoda/slack-action@master + env: + SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN_AIRBYTE_TEAM }} + with: + args: >- + {\"channel\":\"C03BEADRPNY\", \"blocks\":[ + {\"type\":\"divider\"}, + {\"type\":\"section\",\"text\":{\"type\":\"mrkdwn\",\"text\":\" Merge to OSS Master failed! :bangbang: \n\n\"}}, + {\"type\":\"section\",\"text\":{\"type\":\"mrkdwn\",\"text\":\"_merged by_: *${{ github.actor }}* \n\"}}, + {\"type\":\"section\",\"text\":{\"type\":\"mrkdwn\",\"text\":\" :octavia-shocked: :octavia-shocked: \n\"}}, + {\"type\":\"divider\"}]} + + notify-failure-slack-channel-fixed-broken-build: + name: "Notify Slack Channel on Build Fixes" + runs-on: ubuntu-latest + needs: + - build-connectors-base + - frontend-build + - octavia-cli-build + - platform-build + - kube-acceptance-test + # Todo: Kyryl turn this on. + # - helm-acceptance-test + if: success() + steps: + - name: Get Previous Workflow Status + uses: Mercymeilya/last-workflow-status@v0.3 + id: last_status + with: + github_token: ${{ secrets.GITHUB_TOKEN }} + # To avoid clogging up the channel, only publish build success if the previous build was a failure since this means the build was fixed. + - name: Publish Build Fixed Message to OSS Build Failure Slack Channel + if: ${{ steps.last_status.outputs.last_status == 'failure' }} + uses: abinoda/slack-action@master + env: + SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN_AIRBYTE_TEAM }} + with: + args: >- + {\"channel\":\"C03BEADRPNY\", \"blocks\":[ + {\"type\":\"divider\"}, + {\"type\":\"section\",\"text\":{\"type\":\"mrkdwn\",\"text\":\" OSS Master Fixed! :white_check_mark: \n\n\"}}, + {\"type\":\"section\",\"text\":{\"type\":\"mrkdwn\",\"text\":\"_merged by_: *${{ github.actor }}* \n\"}}, + {\"type\":\"section\",\"text\":{\"type\":\"mrkdwn\",\"text\":\" :octavia-rocket: :octavia-rocket: \n\"}}, + {\"type\":\"divider\"}]} diff --git a/tools/bin/acceptance_test_kube_helm.sh b/tools/bin/acceptance_test_kube_helm.sh index 7085a1a880a9..e7bee6930990 100755 --- a/tools/bin/acceptance_test_kube_helm.sh +++ b/tools/bin/acceptance_test_kube_helm.sh @@ -20,16 +20,10 @@ kubectl patch configmap/coredns \ echo "Deploying fluentbit" helm repo add fluent https://fluent.github.io/helm-charts helm repo update fluent -helm template --values tools/bin/fluent_values.yaml --set env[0].name="AWS_ACCESS_KEY_ID" --set env[0].value=$(echo "$AWS_S3_INTEGRATION_TEST_CREDS" | jq -r .aws_access_key_id) \ - --set env[1].name="AWS_SECRET_ACCESS_KEY" --set env[1].value=$(echo "$AWS_S3_INTEGRATION_TEST_CREDS" | jq -r .aws_secret_access_key) \ - --set env[2].name="WORKFLOW_RUN_ID" --set env[2].value=${WORKFLOW_RUN_ID} \ - --set env[3].name="AWS_S3_BUCKET" --set env[3].value=${AWS_S3_BUCKET} \ - --generate-name fluent/fluent-bit - helm install --values tools/bin/fluent_values.yaml --set env[0].name="AWS_ACCESS_KEY_ID" --set env[0].value=$(echo "$AWS_S3_INTEGRATION_TEST_CREDS" | jq -r .aws_access_key_id) \ --set env[1].name="AWS_SECRET_ACCESS_KEY" --set env[1].value=$(echo "$AWS_S3_INTEGRATION_TEST_CREDS" | jq -r .aws_secret_access_key) \ - --set env[2].name="WORKFLOW_RUN_ID" --set env[2].value=${WORKFLOW_RUN_ID} \ - --set env[3].name="AWS_S3_BUCKET" --set env[3].value=${AWS_S3_BUCKET} \ + --set env[2].name="WORKFLOW_RUN_ID" --set env[2].value=\"${WORKFLOW_RUN_ID}\" \ + --set env[3].name="AWS_S3_BUCKET" --set env[3].value=\"${AWS_S3_BUCKET}\" \ --generate-name fluent/fluent-bit echo "Replacing default Chart.yaml and values.yaml with a test one" From df991004eb33e75c71cb35db1d1b4a793b1fd59a Mon Sep 17 00:00:00 2001 From: Kyryl Skobylko Date: Wed, 14 Dec 2022 11:58:55 +0200 Subject: [PATCH 14/24] Update fluent_values.yaml --- tools/bin/fluent_values.yaml | 21 +++++++++++---------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/tools/bin/fluent_values.yaml b/tools/bin/fluent_values.yaml index 6ab69bc0d730..b237f945ff42 100644 --- a/tools/bin/fluent_values.yaml +++ b/tools/bin/fluent_values.yaml @@ -1,10 +1,11 @@ -outputs: | - [OUTPUT] - Name s3 - Match kube.* - bucket ${AWS_S3_BUCKET} - region us-east-2 - total_file_size 1M - upload_timeout 1m - use_put_object On - s3_key_format /k3s-logs/${WORKFLOW_RUN_ID}/$TAG/%Y/%m/%d/%H/%M +config: + outputs: | + [OUTPUT] + Name s3 + Match kube.* + bucket ${AWS_S3_BUCKET} + region us-east-2 + total_file_size 1M + upload_timeout 1m + use_put_object On + s3_key_format /k3s-logs/${WORKFLOW_RUN_ID}/$TAG/%Y/%m/%d/%H/%M From 7d12f9b214c3cc7d138e67a8c195df46d3584ac5 Mon Sep 17 00:00:00 2001 From: Kyryl Skobylko Date: Wed, 14 Dec 2022 16:48:12 +0200 Subject: [PATCH 15/24] Use role instead of AWS creds --- .github/actions/start-aws-runner/action.yml | 5 +++++ .github/workflows/gradle.yml | 12 +++++++++++- tools/bin/acceptance_test_kube_helm.sh | 9 +++++---- 3 files changed, 21 insertions(+), 5 deletions(-) diff --git a/.github/actions/start-aws-runner/action.yml b/.github/actions/start-aws-runner/action.yml index 15bc9d6c4e75..25d2a6910a14 100644 --- a/.github/actions/start-aws-runner/action.yml +++ b/.github/actions/start-aws-runner/action.yml @@ -24,6 +24,10 @@ inputs: required: false ec2-instance-id: required: false + iam-role-name: + required: false + default: "" + outputs: label: value: ${{ steps.start-ec2-runner.outputs.label }} @@ -49,6 +53,7 @@ runs: ec2-instance-type: ${{ inputs.ec2-instance-type }} subnet-id: ${{ inputs.subnet-id }} security-group-id: ${{ inputs.security-group-id }} + iam-role-name: ${{ inputs.iam-role-name }} # this adds a label to group any EC2 runners spun up within the same action run # this enables creating a pool of runners to run multiple/matrix jobs on in parallel label: runner-pool-${{ github.run_id }} diff --git a/.github/workflows/gradle.yml b/.github/workflows/gradle.yml index 625dd578c587..9f2332991091 100644 --- a/.github/workflows/gradle.yml +++ b/.github/workflows/gradle.yml @@ -993,6 +993,7 @@ jobs: aws-access-key-id: ${{ secrets.SELF_RUNNER_AWS_ACCESS_KEY_ID }} aws-secret-access-key: ${{ secrets.SELF_RUNNER_AWS_SECRET_ACCESS_KEY }} github-token: ${{ env.PAT }} + iam-role-name: ${{ secrets.AWS_ASSUME_ROLE_NAME }} helm-acceptance-test: name: "Platform: Acceptance Tests (Helm)" @@ -1110,12 +1111,21 @@ jobs: attempt_limit: 3 attempt_delay: 5000 # in ms + - name: Configure AWS Credentials + uses: aws-actions/configure-aws-credentials@v1 + id: logs-creds + with: + role-to-assume: arn:aws:iam::${{ secrets.AWS_ACCOUNT_ID }}:role/${{ secrets.AWS_ASSUME_ROLE_NAME }} + aws-region: us-east-2 + + + - name: Run Helm End-to-End Acceptance Tests env: # USER: root # HOME: /home/runner # AWS_S3_INTEGRATION_TEST_CREDS can be found in LastPass as AWS_S3_INTEGRATION_TEST_CREDS - AWS_S3_INTEGRATION_TEST_CREDS: ${{ secrets.AWS_S3_INTEGRATION_TEST_CREDS }} + # AWS_S3_INTEGRATION_TEST_CREDS: ${{ secrets.AWS_S3_INTEGRATION_TEST_CREDS }} WORKFLOW_RUN_ID: ${{ github.run_id }} AWS_S3_BUCKET: airbyte-buildcache SECRET_STORE_GCP_CREDENTIALS: ${{ secrets.SECRET_STORE_GCP_CREDENTIALS }} diff --git a/tools/bin/acceptance_test_kube_helm.sh b/tools/bin/acceptance_test_kube_helm.sh index e7bee6930990..d4c746ced48e 100755 --- a/tools/bin/acceptance_test_kube_helm.sh +++ b/tools/bin/acceptance_test_kube_helm.sh @@ -20,10 +20,11 @@ kubectl patch configmap/coredns \ echo "Deploying fluentbit" helm repo add fluent https://fluent.github.io/helm-charts helm repo update fluent -helm install --values tools/bin/fluent_values.yaml --set env[0].name="AWS_ACCESS_KEY_ID" --set env[0].value=$(echo "$AWS_S3_INTEGRATION_TEST_CREDS" | jq -r .aws_access_key_id) \ - --set env[1].name="AWS_SECRET_ACCESS_KEY" --set env[1].value=$(echo "$AWS_S3_INTEGRATION_TEST_CREDS" | jq -r .aws_secret_access_key) \ - --set env[2].name="WORKFLOW_RUN_ID" --set env[2].value=\"${WORKFLOW_RUN_ID}\" \ - --set env[3].name="AWS_S3_BUCKET" --set env[3].value=\"${AWS_S3_BUCKET}\" \ +helm install --values tools/bin/fluent_values.yaml --set env[0].name="AWS_ACCESS_KEY_ID" --set env[0].value=\"${AWS_ACCESS_KEY_ID}\" \ + --set env[1].name="AWS_SECRET_ACCESS_KEY" --set env[1].value=\"${AWS_SECRET_ACCESS_KEY}\" \ + --set env[2].name="AWS_SESSION_TOKEN" --set env[2].value=\"${AWS_SESSION_TOKEN}\" \ + --set env[4].name="WORKFLOW_RUN_ID" --set env[4].value=\"${WORKFLOW_RUN_ID}\" \ + --set env[4].name="AWS_S3_BUCKET" --set env[4].value=\"${AWS_S3_BUCKET}\" \ --generate-name fluent/fluent-bit echo "Replacing default Chart.yaml and values.yaml with a test one" From ded553e9a13e15c37d03a85f7094bf7a212dc26d Mon Sep 17 00:00:00 2001 From: Kyryl Skobylko Date: Thu, 15 Dec 2022 12:41:52 +0200 Subject: [PATCH 16/24] Use new bucket --- .github/workflows/gradle.yml | 2 +- tools/bin/fluent_values.yaml | 16 ++++++++-------- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/.github/workflows/gradle.yml b/.github/workflows/gradle.yml index 625dd578c587..8d44e8c7b762 100644 --- a/.github/workflows/gradle.yml +++ b/.github/workflows/gradle.yml @@ -1117,7 +1117,7 @@ jobs: # AWS_S3_INTEGRATION_TEST_CREDS can be found in LastPass as AWS_S3_INTEGRATION_TEST_CREDS AWS_S3_INTEGRATION_TEST_CREDS: ${{ secrets.AWS_S3_INTEGRATION_TEST_CREDS }} WORKFLOW_RUN_ID: ${{ github.run_id }} - AWS_S3_BUCKET: airbyte-buildcache + AWS_S3_BUCKET: airbyte-acceptance-test-source-s3 SECRET_STORE_GCP_CREDENTIALS: ${{ secrets.SECRET_STORE_GCP_CREDENTIALS }} SECRET_STORE_GCP_PROJECT_ID: ${{ secrets.SECRET_STORE_GCP_PROJECT_ID }} timeout-minutes: 20 diff --git a/tools/bin/fluent_values.yaml b/tools/bin/fluent_values.yaml index b237f945ff42..4778d6516955 100644 --- a/tools/bin/fluent_values.yaml +++ b/tools/bin/fluent_values.yaml @@ -1,11 +1,11 @@ config: outputs: | [OUTPUT] - Name s3 - Match kube.* - bucket ${AWS_S3_BUCKET} - region us-east-2 - total_file_size 1M - upload_timeout 1m - use_put_object On - s3_key_format /k3s-logs/${WORKFLOW_RUN_ID}/$TAG/%Y/%m/%d/%H/%M + Name s3 + Match kube.* + bucket ${AWS_S3_BUCKET} + region us-east-2 + total_file_size 1M + upload_timeout 1m + use_put_object On + s3_key_format /k3s-logs/${WORKFLOW_RUN_ID}/$TAG/%Y/%m/%d/%H/%M From df7b55bb6cbb364b58c9404b19b2ae7fddc24cff Mon Sep 17 00:00:00 2001 From: Kyryl Skobylko Date: Thu, 15 Dec 2022 12:43:58 +0200 Subject: [PATCH 17/24] Revert changes --- .github/actions/start-aws-runner/action.yml | 4 ---- .github/workflows/gradle.yml | 11 +---------- tools/bin/acceptance_test_kube_helm.sh | 9 ++++----- 3 files changed, 5 insertions(+), 19 deletions(-) diff --git a/.github/actions/start-aws-runner/action.yml b/.github/actions/start-aws-runner/action.yml index 25d2a6910a14..da3f04b790f2 100644 --- a/.github/actions/start-aws-runner/action.yml +++ b/.github/actions/start-aws-runner/action.yml @@ -24,9 +24,6 @@ inputs: required: false ec2-instance-id: required: false - iam-role-name: - required: false - default: "" outputs: label: @@ -53,7 +50,6 @@ runs: ec2-instance-type: ${{ inputs.ec2-instance-type }} subnet-id: ${{ inputs.subnet-id }} security-group-id: ${{ inputs.security-group-id }} - iam-role-name: ${{ inputs.iam-role-name }} # this adds a label to group any EC2 runners spun up within the same action run # this enables creating a pool of runners to run multiple/matrix jobs on in parallel label: runner-pool-${{ github.run_id }} diff --git a/.github/workflows/gradle.yml b/.github/workflows/gradle.yml index 9518cd1a18b9..8fe72063cb43 100644 --- a/.github/workflows/gradle.yml +++ b/.github/workflows/gradle.yml @@ -1111,21 +1111,12 @@ jobs: attempt_limit: 3 attempt_delay: 5000 # in ms - - name: Configure AWS Credentials - uses: aws-actions/configure-aws-credentials@v1 - id: logs-creds - with: - role-to-assume: arn:aws:iam::${{ secrets.AWS_ACCOUNT_ID }}:role/${{ secrets.AWS_ASSUME_ROLE_NAME }} - aws-region: us-east-2 - - - - name: Run Helm End-to-End Acceptance Tests env: # USER: root # HOME: /home/runner # AWS_S3_INTEGRATION_TEST_CREDS can be found in LastPass as AWS_S3_INTEGRATION_TEST_CREDS - # AWS_S3_INTEGRATION_TEST_CREDS: ${{ secrets.AWS_S3_INTEGRATION_TEST_CREDS }} + AWS_S3_INTEGRATION_TEST_CREDS: ${{ secrets.AWS_S3_INTEGRATION_TEST_CREDS }} WORKFLOW_RUN_ID: ${{ github.run_id }} AWS_S3_BUCKET: airbyte-acceptance-test-source-s3 SECRET_STORE_GCP_CREDENTIALS: ${{ secrets.SECRET_STORE_GCP_CREDENTIALS }} diff --git a/tools/bin/acceptance_test_kube_helm.sh b/tools/bin/acceptance_test_kube_helm.sh index d4c746ced48e..e7bee6930990 100755 --- a/tools/bin/acceptance_test_kube_helm.sh +++ b/tools/bin/acceptance_test_kube_helm.sh @@ -20,11 +20,10 @@ kubectl patch configmap/coredns \ echo "Deploying fluentbit" helm repo add fluent https://fluent.github.io/helm-charts helm repo update fluent -helm install --values tools/bin/fluent_values.yaml --set env[0].name="AWS_ACCESS_KEY_ID" --set env[0].value=\"${AWS_ACCESS_KEY_ID}\" \ - --set env[1].name="AWS_SECRET_ACCESS_KEY" --set env[1].value=\"${AWS_SECRET_ACCESS_KEY}\" \ - --set env[2].name="AWS_SESSION_TOKEN" --set env[2].value=\"${AWS_SESSION_TOKEN}\" \ - --set env[4].name="WORKFLOW_RUN_ID" --set env[4].value=\"${WORKFLOW_RUN_ID}\" \ - --set env[4].name="AWS_S3_BUCKET" --set env[4].value=\"${AWS_S3_BUCKET}\" \ +helm install --values tools/bin/fluent_values.yaml --set env[0].name="AWS_ACCESS_KEY_ID" --set env[0].value=$(echo "$AWS_S3_INTEGRATION_TEST_CREDS" | jq -r .aws_access_key_id) \ + --set env[1].name="AWS_SECRET_ACCESS_KEY" --set env[1].value=$(echo "$AWS_S3_INTEGRATION_TEST_CREDS" | jq -r .aws_secret_access_key) \ + --set env[2].name="WORKFLOW_RUN_ID" --set env[2].value=\"${WORKFLOW_RUN_ID}\" \ + --set env[3].name="AWS_S3_BUCKET" --set env[3].value=\"${AWS_S3_BUCKET}\" \ --generate-name fluent/fluent-bit echo "Replacing default Chart.yaml and values.yaml with a test one" From ead967d79e6dc1b3ef3b34da9ecd84cc8886c8b5 Mon Sep 17 00:00:00 2001 From: Kyryl Skobylko Date: Thu, 15 Dec 2022 13:02:54 +0200 Subject: [PATCH 18/24] Update bucket name --- .github/workflows/gradle.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/gradle.yml b/.github/workflows/gradle.yml index 8fe72063cb43..067ccd287825 100644 --- a/.github/workflows/gradle.yml +++ b/.github/workflows/gradle.yml @@ -1118,7 +1118,7 @@ jobs: # AWS_S3_INTEGRATION_TEST_CREDS can be found in LastPass as AWS_S3_INTEGRATION_TEST_CREDS AWS_S3_INTEGRATION_TEST_CREDS: ${{ secrets.AWS_S3_INTEGRATION_TEST_CREDS }} WORKFLOW_RUN_ID: ${{ github.run_id }} - AWS_S3_BUCKET: airbyte-acceptance-test-source-s3 + AWS_S3_BUCKET: airbyte-ci-ac-tests-logs SECRET_STORE_GCP_CREDENTIALS: ${{ secrets.SECRET_STORE_GCP_CREDENTIALS }} SECRET_STORE_GCP_PROJECT_ID: ${{ secrets.SECRET_STORE_GCP_PROJECT_ID }} timeout-minutes: 20 From 3e0d0a60c31d8da823c0f9a183e305a2295a3eea Mon Sep 17 00:00:00 2001 From: Kyryl Skobylko Date: Thu, 15 Dec 2022 14:12:52 +0200 Subject: [PATCH 19/24] Play around with fluent-bit config --- tools/bin/acceptance_test_kube_helm.sh | 4 ++-- tools/bin/fluent_values.yaml | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/tools/bin/acceptance_test_kube_helm.sh b/tools/bin/acceptance_test_kube_helm.sh index e7bee6930990..8dc34ebbba27 100755 --- a/tools/bin/acceptance_test_kube_helm.sh +++ b/tools/bin/acceptance_test_kube_helm.sh @@ -22,8 +22,8 @@ helm repo add fluent https://fluent.github.io/helm-charts helm repo update fluent helm install --values tools/bin/fluent_values.yaml --set env[0].name="AWS_ACCESS_KEY_ID" --set env[0].value=$(echo "$AWS_S3_INTEGRATION_TEST_CREDS" | jq -r .aws_access_key_id) \ --set env[1].name="AWS_SECRET_ACCESS_KEY" --set env[1].value=$(echo "$AWS_S3_INTEGRATION_TEST_CREDS" | jq -r .aws_secret_access_key) \ - --set env[2].name="WORKFLOW_RUN_ID" --set env[2].value=\"${WORKFLOW_RUN_ID}\" \ - --set env[3].name="AWS_S3_BUCKET" --set env[3].value=\"${AWS_S3_BUCKET}\" \ + --set env[2].name="WORKFLOW_RUN_ID" --set env[2].value=${WORKFLOW_RUN_ID} \ + --set env[3].name="AWS_S3_BUCKET" --set env[3].value=${AWS_S3_BUCKET} \ --generate-name fluent/fluent-bit echo "Replacing default Chart.yaml and values.yaml with a test one" diff --git a/tools/bin/fluent_values.yaml b/tools/bin/fluent_values.yaml index 4778d6516955..26607e82daf7 100644 --- a/tools/bin/fluent_values.yaml +++ b/tools/bin/fluent_values.yaml @@ -6,6 +6,6 @@ config: bucket ${AWS_S3_BUCKET} region us-east-2 total_file_size 1M - upload_timeout 1m + upload_timeout 2m use_put_object On - s3_key_format /k3s-logs/${WORKFLOW_RUN_ID}/$TAG/%Y/%m/%d/%H/%M + s3_key_format /k3s-logs/${WORKFLOW_RUN_ID}/$TAG[4]/$UUID.log From 20a85857a5b184ff573e6d21b60b01066c5fb881 Mon Sep 17 00:00:00 2001 From: Kyryl Skobylko Date: Thu, 15 Dec 2022 14:27:47 +0200 Subject: [PATCH 20/24] Escape string in helm --set arg --- tools/bin/acceptance_test_kube_helm.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/bin/acceptance_test_kube_helm.sh b/tools/bin/acceptance_test_kube_helm.sh index 8dc34ebbba27..da5a3071f8a2 100755 --- a/tools/bin/acceptance_test_kube_helm.sh +++ b/tools/bin/acceptance_test_kube_helm.sh @@ -22,7 +22,7 @@ helm repo add fluent https://fluent.github.io/helm-charts helm repo update fluent helm install --values tools/bin/fluent_values.yaml --set env[0].name="AWS_ACCESS_KEY_ID" --set env[0].value=$(echo "$AWS_S3_INTEGRATION_TEST_CREDS" | jq -r .aws_access_key_id) \ --set env[1].name="AWS_SECRET_ACCESS_KEY" --set env[1].value=$(echo "$AWS_S3_INTEGRATION_TEST_CREDS" | jq -r .aws_secret_access_key) \ - --set env[2].name="WORKFLOW_RUN_ID" --set env[2].value=${WORKFLOW_RUN_ID} \ + --set env[2].name="WORKFLOW_RUN_ID" --set env[2].value=\'${WORKFLOW_RUN_ID}\' \ --set env[3].name="AWS_S3_BUCKET" --set env[3].value=${AWS_S3_BUCKET} \ --generate-name fluent/fluent-bit From 8c7536ee9862b53ca1f13e76ff0e09c6bac9281e Mon Sep 17 00:00:00 2001 From: Kyryl Skobylko Date: Thu, 15 Dec 2022 14:46:05 +0200 Subject: [PATCH 21/24] Update fluent-bit config + update --set argument --- tools/bin/acceptance_test_kube_helm.sh | 2 +- tools/bin/fluent_values.yaml | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/tools/bin/acceptance_test_kube_helm.sh b/tools/bin/acceptance_test_kube_helm.sh index da5a3071f8a2..ea0017fe2f19 100755 --- a/tools/bin/acceptance_test_kube_helm.sh +++ b/tools/bin/acceptance_test_kube_helm.sh @@ -22,7 +22,7 @@ helm repo add fluent https://fluent.github.io/helm-charts helm repo update fluent helm install --values tools/bin/fluent_values.yaml --set env[0].name="AWS_ACCESS_KEY_ID" --set env[0].value=$(echo "$AWS_S3_INTEGRATION_TEST_CREDS" | jq -r .aws_access_key_id) \ --set env[1].name="AWS_SECRET_ACCESS_KEY" --set env[1].value=$(echo "$AWS_S3_INTEGRATION_TEST_CREDS" | jq -r .aws_secret_access_key) \ - --set env[2].name="WORKFLOW_RUN_ID" --set env[2].value=\'${WORKFLOW_RUN_ID}\' \ + --set env[2].name="WORKFLOW_RUN_ID" --set env[2].value=\\'${WORKFLOW_RUN_ID}\' \ --set env[3].name="AWS_S3_BUCKET" --set env[3].value=${AWS_S3_BUCKET} \ --generate-name fluent/fluent-bit diff --git a/tools/bin/fluent_values.yaml b/tools/bin/fluent_values.yaml index 26607e82daf7..5cab86a24f36 100644 --- a/tools/bin/fluent_values.yaml +++ b/tools/bin/fluent_values.yaml @@ -8,4 +8,5 @@ config: total_file_size 1M upload_timeout 2m use_put_object On + log_key log s3_key_format /k3s-logs/${WORKFLOW_RUN_ID}/$TAG[4]/$UUID.log From 2dfd1a4519571dc8d43208c3420cd1aae2a37cc0 Mon Sep 17 00:00:00 2001 From: Kyryl Skobylko Date: Thu, 15 Dec 2022 15:50:28 +0200 Subject: [PATCH 22/24] Update fluentbit config so it"ll create s3 path correctly --- tools/bin/acceptance_test_kube_helm.sh | 6 +++--- tools/bin/fluent_values.yaml | 3 +++ 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/tools/bin/acceptance_test_kube_helm.sh b/tools/bin/acceptance_test_kube_helm.sh index ea0017fe2f19..c8a7c1a10067 100755 --- a/tools/bin/acceptance_test_kube_helm.sh +++ b/tools/bin/acceptance_test_kube_helm.sh @@ -20,9 +20,9 @@ kubectl patch configmap/coredns \ echo "Deploying fluentbit" helm repo add fluent https://fluent.github.io/helm-charts helm repo update fluent -helm install --values tools/bin/fluent_values.yaml --set env[0].name="AWS_ACCESS_KEY_ID" --set env[0].value=$(echo "$AWS_S3_INTEGRATION_TEST_CREDS" | jq -r .aws_access_key_id) \ - --set env[1].name="AWS_SECRET_ACCESS_KEY" --set env[1].value=$(echo "$AWS_S3_INTEGRATION_TEST_CREDS" | jq -r .aws_secret_access_key) \ - --set env[2].name="WORKFLOW_RUN_ID" --set env[2].value=\\'${WORKFLOW_RUN_ID}\' \ +sed -i "s/PLACEHOLDER/${WORKFLOW_RUN_ID}/" tools/bin/fluent_values.yaml +helm install --values tools/bin/fluent_values.yaml --set env[1].name="AWS_ACCESS_KEY_ID" --set env[1].value=$(echo "$AWS_S3_INTEGRATION_TEST_CREDS" | jq -r .aws_access_key_id) \ + --set env[2].name="AWS_SECRET_ACCESS_KEY" --set env[2].value=$(echo "$AWS_S3_INTEGRATION_TEST_CREDS" | jq -r .aws_secret_access_key) \ --set env[3].name="AWS_S3_BUCKET" --set env[3].value=${AWS_S3_BUCKET} \ --generate-name fluent/fluent-bit diff --git a/tools/bin/fluent_values.yaml b/tools/bin/fluent_values.yaml index 5cab86a24f36..62805e245f17 100644 --- a/tools/bin/fluent_values.yaml +++ b/tools/bin/fluent_values.yaml @@ -10,3 +10,6 @@ config: use_put_object On log_key log s3_key_format /k3s-logs/${WORKFLOW_RUN_ID}/$TAG[4]/$UUID.log +env: + - name: WORKFLOW_RUN_ID + value: 'PLACEHOLDER' \ No newline at end of file From acb17a629e860456cc294d183b5fa6a23c2ac841 Mon Sep 17 00:00:00 2001 From: Kyryl Skobylko Date: Thu, 15 Dec 2022 16:15:18 +0200 Subject: [PATCH 23/24] Add logging to kustomize --- .github/workflows/gradle.yml | 12 ++++++++++++ tools/bin/acceptance_test_kube.sh | 11 +++++++++++ tools/bin/acceptance_test_kube_helm.sh | 1 + tools/bin/fluent_values.yaml | 2 +- 4 files changed, 25 insertions(+), 1 deletion(-) diff --git a/.github/workflows/gradle.yml b/.github/workflows/gradle.yml index 067ccd287825..bd356949ba0c 100644 --- a/.github/workflows/gradle.yml +++ b/.github/workflows/gradle.yml @@ -806,6 +806,16 @@ jobs: with: python-version: "3.9" + - name: Install unzip + shell: bash + run: sudo apt-get update && sudo apt-get install -y unzip + + - uses: azure/setup-helm@v3 + with: + version: 'latest' + token: ${{ secrets.GITHUB_TOKEN }} + id: install + - name: Fix EC-2 Runner run: | mkdir -p /home/runner @@ -857,6 +867,8 @@ jobs: HOME: /home/runner # AWS_S3_INTEGRATION_TEST_CREDS can be found in LastPass as AWS_S3_INTEGRATION_TEST_CREDS AWS_S3_INTEGRATION_TEST_CREDS: ${{ secrets.AWS_S3_INTEGRATION_TEST_CREDS }} + WORKFLOW_RUN_ID: ${{ github.run_id }} + AWS_S3_BUCKET: airbyte-ci-ac-tests-logs SECRET_STORE_GCP_CREDENTIALS: ${{ secrets.SECRET_STORE_GCP_CREDENTIALS }} SECRET_STORE_GCP_PROJECT_ID: ${{ secrets.SECRET_STORE_GCP_PROJECT_ID }} uses: Wandalen/wretry.action@master diff --git a/tools/bin/acceptance_test_kube.sh b/tools/bin/acceptance_test_kube.sh index eb1d86de442d..097f673307c9 100755 --- a/tools/bin/acceptance_test_kube.sh +++ b/tools/bin/acceptance_test_kube.sh @@ -20,6 +20,17 @@ if [ -n "$CI" ]; then wait fi +echo "Deploying fluentbit" +helm repo add fluent https://fluent.github.io/helm-charts +helm repo update fluent +sed -i "s/PLACEHOLDER/${WORKFLOW_RUN_ID}/" tools/bin/fluent_values.yaml +helm install --values tools/bin/fluent_values.yaml --set env[1].name="AWS_ACCESS_KEY_ID" --set env[1].value=$(echo "$AWS_S3_INTEGRATION_TEST_CREDS" | jq -r .aws_access_key_id) \ + --set env[2].name="AWS_SECRET_ACCESS_KEY" --set env[2].value=$(echo "$AWS_S3_INTEGRATION_TEST_CREDS" | jq -r .aws_secret_access_key) \ + --set env[3].name="AWS_S3_BUCKET" --set env[3].value=${AWS_S3_BUCKET} \ + --set env[4].name="SUITE_TYPE" --set env[4].value="kustomize-logs" \ + --generate-name fluent/fluent-bit + + echo "Starting app..." echo "Applying dev-integration-test manifests to kubernetes..." diff --git a/tools/bin/acceptance_test_kube_helm.sh b/tools/bin/acceptance_test_kube_helm.sh index c8a7c1a10067..97a6e95b1316 100755 --- a/tools/bin/acceptance_test_kube_helm.sh +++ b/tools/bin/acceptance_test_kube_helm.sh @@ -24,6 +24,7 @@ sed -i "s/PLACEHOLDER/${WORKFLOW_RUN_ID}/" tools/bin/fluent_values.yaml helm install --values tools/bin/fluent_values.yaml --set env[1].name="AWS_ACCESS_KEY_ID" --set env[1].value=$(echo "$AWS_S3_INTEGRATION_TEST_CREDS" | jq -r .aws_access_key_id) \ --set env[2].name="AWS_SECRET_ACCESS_KEY" --set env[2].value=$(echo "$AWS_S3_INTEGRATION_TEST_CREDS" | jq -r .aws_secret_access_key) \ --set env[3].name="AWS_S3_BUCKET" --set env[3].value=${AWS_S3_BUCKET} \ + --set env[4].name="SUITE_TYPE" --set env[4].value="helm-logs" \ --generate-name fluent/fluent-bit echo "Replacing default Chart.yaml and values.yaml with a test one" diff --git a/tools/bin/fluent_values.yaml b/tools/bin/fluent_values.yaml index 62805e245f17..1ec639291ab8 100644 --- a/tools/bin/fluent_values.yaml +++ b/tools/bin/fluent_values.yaml @@ -9,7 +9,7 @@ config: upload_timeout 2m use_put_object On log_key log - s3_key_format /k3s-logs/${WORKFLOW_RUN_ID}/$TAG[4]/$UUID.log + s3_key_format /${SUITE_TYPE}/${WORKFLOW_RUN_ID}/$TAG[4]/$UUID.log env: - name: WORKFLOW_RUN_ID value: 'PLACEHOLDER' \ No newline at end of file From e92b9a8c057b3d95e3a3bf3fcc092406083c0cdc Mon Sep 17 00:00:00 2001 From: Kyryl Skobylko Date: Thu, 15 Dec 2022 16:18:10 +0200 Subject: [PATCH 24/24] Revert debug mode --- .github/workflows/gradle.yml | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/.github/workflows/gradle.yml b/.github/workflows/gradle.yml index bd356949ba0c..d5e5dab790d4 100644 --- a/.github/workflows/gradle.yml +++ b/.github/workflows/gradle.yml @@ -10,7 +10,11 @@ on: inputs: debug_mode: description: "Enable or disable tmate session for debug during helm ac tests" + type: choice default: 'false' + options: + - 'true' + - 'false' required: false schedule: - cron: "0 */1 * * *" @@ -1070,7 +1074,7 @@ jobs: version: 'v1.21.2' # default is latest stable - name: Install tmate and it's dependencies - # if: inputs.debug_mode == 'true' + if: inputs.debug_mode == 'true' uses: Wandalen/wretry.action@master with: command: | @@ -1079,7 +1083,7 @@ jobs: attempt_delay: 2000 # in ms - name: Start tmate session in background - # if: inputs.debug_mode == 'true' + if: inputs.debug_mode == 'true' shell: bash run: | tmate -S /tmp/tmate.sock new-session -d # Launch tmate in a headless mode