fix: Improve Airbyte CI logging (#20517)

* add fluent-bit logging to k3s setup * Fix --set for helm * add --generate-name * fix slash in multiline bash * Update --set for helm * Use generate-name instead * Add debug * Remove deubg for helm * Fix env again * Disable build * Remove everything * Disable everything again * Snap back to reallity * Update fluent_values.yaml * Use role instead of AWS creds * Use new bucket * Revert changes * Update bucket name * Play around with fluent-bit config * Escape string in helm --set arg * Update fluent-bit config + update --set argument * Update fluentbit config so it"ll create s3 path correctly * Add logging to kustomize * Revert debug mode
airbytehq · Dec 27, 2022 · 9da0de1 · 9da0de1
1 parent c3c5a41
commit 9da0de1
Show file tree

Hide file tree

Showing 5 changed files with 56 additions and 8 deletions.
diff --git a/.github/actions/start-aws-runner/action.yml b/.github/actions/start-aws-runner/action.yml
@@ -24,6 +24,7 @@ inputs:
     required: false
   ec2-instance-id:
     required: false
+
 outputs:
   label:
     value: ${{ steps.start-ec2-runner.outputs.label }}

diff --git a/.github/workflows/gradle.yml b/.github/workflows/gradle.yml
@@ -10,7 +10,11 @@ on:
     inputs:
       debug_mode:
         description: "Enable or disable tmate session for debug during helm ac tests"
+        type: choice
         default: 'false'
+        options: 
+          - 'true'
+          - 'false'
         required: false
   schedule:
     - cron: "0 */1 * * *"
@@ -811,6 +815,16 @@ jobs:
         with:
           python-version: "3.9"
 
+      - name: Install unzip
+        shell: bash
+        run: sudo apt-get update && sudo apt-get install -y unzip
+
+      - uses: azure/setup-helm@v3
+        with:
+          version: 'latest'
+          token: ${{ secrets.GITHUB_TOKEN }}
+        id: install
+
       - name: Fix EC-2 Runner
         run: |
           mkdir -p /home/runner
@@ -862,6 +876,8 @@ jobs:
           HOME: /home/runner
           # AWS_S3_INTEGRATION_TEST_CREDS can be found in LastPass as AWS_S3_INTEGRATION_TEST_CREDS
           AWS_S3_INTEGRATION_TEST_CREDS: ${{ secrets.AWS_S3_INTEGRATION_TEST_CREDS }}
+          WORKFLOW_RUN_ID: ${{ github.run_id }}
+          AWS_S3_BUCKET: airbyte-ci-ac-tests-logs
           SECRET_STORE_GCP_CREDENTIALS: ${{ secrets.SECRET_STORE_GCP_CREDENTIALS }}
           SECRET_STORE_GCP_PROJECT_ID: ${{ secrets.SECRET_STORE_GCP_PROJECT_ID }}
         uses: Wandalen/wretry.action@master
@@ -998,6 +1014,7 @@ jobs:
           aws-access-key-id: ${{ secrets.SELF_RUNNER_AWS_ACCESS_KEY_ID }}
           aws-secret-access-key: ${{ secrets.SELF_RUNNER_AWS_SECRET_ACCESS_KEY }}
           github-token: ${{ env.PAT }}
+          iam-role-name: ${{ secrets.AWS_ASSUME_ROLE_NAME }}
 
   helm-acceptance-test:
     name: "Platform: Acceptance Tests (Helm)"
@@ -1028,12 +1045,8 @@ jobs:
         uses: Wandalen/wretry.action@master
         with:
           command: |
-            sudo rm /var/lib/dpkg/lock
-            sudo rm /var/lib/dpkg/lock-frontend
             sudo apt-get remove docker.io || sudo apt-get remove docker
             curl -fsSL https://get.docker.com | bash -
-            sudo rm -f /var/lib/dpkg/lock
-            sudo rm -f /var/lib/dpkg/lock-frontend
           attempt_limit: 3
           attempt_delay: 2000 # in ms
 
@@ -1070,11 +1083,7 @@ jobs:
         uses: Wandalen/wretry.action@master
         with:
           command: |
-            sudo rm /var/lib/dpkg/lock
-            sudo rm /var/lib/dpkg/lock-frontend
             sudo apt-get -y install tmate
-            sudo rm -f /var/lib/dpkg/lock
-            sudo rm -f /var/lib/dpkg/lock-frontend
           attempt_limit: 3
           attempt_delay: 2000 # in ms
 
@@ -1129,6 +1138,8 @@ jobs:
           # HOME: /home/runner
           # AWS_S3_INTEGRATION_TEST_CREDS can be found in LastPass as AWS_S3_INTEGRATION_TEST_CREDS
           AWS_S3_INTEGRATION_TEST_CREDS: ${{ secrets.AWS_S3_INTEGRATION_TEST_CREDS }}
+          WORKFLOW_RUN_ID: ${{ github.run_id }}
+          AWS_S3_BUCKET: airbyte-ci-ac-tests-logs
           SECRET_STORE_GCP_CREDENTIALS: ${{ secrets.SECRET_STORE_GCP_CREDENTIALS }}
           SECRET_STORE_GCP_PROJECT_ID: ${{ secrets.SECRET_STORE_GCP_PROJECT_ID }}
         timeout-minutes: 20

diff --git a/tools/bin/acceptance_test_kube.sh b/tools/bin/acceptance_test_kube.sh
@@ -20,6 +20,17 @@ if [ -n "$CI" ]; then
   wait
 fi
 
+echo "Deploying fluentbit"
+helm repo add fluent https://fluent.github.io/helm-charts
+helm repo update fluent
+sed -i "s/PLACEHOLDER/${WORKFLOW_RUN_ID}/" tools/bin/fluent_values.yaml  
+helm install --values tools/bin/fluent_values.yaml --set env[1].name="AWS_ACCESS_KEY_ID" --set env[1].value=$(echo "$AWS_S3_INTEGRATION_TEST_CREDS" | jq -r .aws_access_key_id) \
+ --set env[2].name="AWS_SECRET_ACCESS_KEY" --set env[2].value=$(echo "$AWS_S3_INTEGRATION_TEST_CREDS" | jq -r .aws_secret_access_key) \
+ --set env[3].name="AWS_S3_BUCKET" --set env[3].value=${AWS_S3_BUCKET} \
+ --set env[4].name="SUITE_TYPE" --set env[4].value="kustomize-logs" \
+ --generate-name fluent/fluent-bit
+
+
 echo "Starting app..."
 
 echo "Applying dev-integration-test manifests to kubernetes..."

diff --git a/tools/bin/acceptance_test_kube_helm.sh b/tools/bin/acceptance_test_kube_helm.sh
@@ -17,6 +17,16 @@ kubectl patch configmap/coredns \
   -p '{"data":{"NodeHosts": "${DOCKER_HOST_IP} host.docker.internal" }}'
 
 
+echo "Deploying fluentbit"
+helm repo add fluent https://fluent.github.io/helm-charts
+helm repo update fluent
+sed -i "s/PLACEHOLDER/${WORKFLOW_RUN_ID}/" tools/bin/fluent_values.yaml  
+helm install --values tools/bin/fluent_values.yaml --set env[1].name="AWS_ACCESS_KEY_ID" --set env[1].value=$(echo "$AWS_S3_INTEGRATION_TEST_CREDS" | jq -r .aws_access_key_id) \
+ --set env[2].name="AWS_SECRET_ACCESS_KEY" --set env[2].value=$(echo "$AWS_S3_INTEGRATION_TEST_CREDS" | jq -r .aws_secret_access_key) \
+ --set env[3].name="AWS_S3_BUCKET" --set env[3].value=${AWS_S3_BUCKET} \
+ --set env[4].name="SUITE_TYPE" --set env[4].value="helm-logs" \
+ --generate-name fluent/fluent-bit
+
 echo "Replacing default Chart.yaml and values.yaml with a test one"
 mv charts/airbyte/Chart.yaml charts/airbyte/Chart.yaml.old
 mv charts/airbyte/Chart.yaml.test charts/airbyte/Chart.yaml

diff --git a/tools/bin/fluent_values.yaml b/tools/bin/fluent_values.yaml
@@ -0,0 +1,15 @@
+config:
+  outputs: |
+    [OUTPUT]
+        Name s3
+        Match kube.*
+        bucket ${AWS_S3_BUCKET}
+        region us-east-2
+        total_file_size 1M
+        upload_timeout 2m
+        use_put_object On
+        log_key log
+        s3_key_format /${SUITE_TYPE}/${WORKFLOW_RUN_ID}/$TAG[4]/$UUID.log
+env:
+  - name: WORKFLOW_RUN_ID
+    value: 'PLACEHOLDER'