fix: Improve Airbyte CI logging

.github/actions/start-aws-runner/action.yml

@@ -24,6 +24,7 @@ inputs:
     required: false
   ec2-instance-id:
     required: false
+
 outputs:
   label:
     value: ${{ steps.start-ec2-runner.outputs.label }}

.github/workflows/gradle.yml

@@ -10,7 +10,11 @@ on:
     inputs:
       debug_mode:
         description: "Enable or disable tmate session for debug during helm ac tests"
+        type: choice
         default: 'false'
+        options: 
+          - 'true'
+          - 'false'
         required: false
   schedule:
     - cron: "0 */1 * * *"
@@ -806,6 +810,16 @@ jobs:
         with:
           python-version: "3.9"
 
+      - name: Install unzip
+        shell: bash
+        run: sudo apt-get update && sudo apt-get install -y unzip
+
+      - uses: azure/setup-helm@v3
+        with:
+          version: 'latest'
+          token: ${{ secrets.GITHUB_TOKEN }}
+        id: install
+
       - name: Fix EC-2 Runner
         run: |
           mkdir -p /home/runner
@@ -857,6 +871,8 @@ jobs:
           HOME: /home/runner
           # AWS_S3_INTEGRATION_TEST_CREDS can be found in LastPass as AWS_S3_INTEGRATION_TEST_CREDS
           AWS_S3_INTEGRATION_TEST_CREDS: ${{ secrets.AWS_S3_INTEGRATION_TEST_CREDS }}
+          WORKFLOW_RUN_ID: ${{ github.run_id }}
+          AWS_S3_BUCKET: airbyte-ci-ac-tests-logs
           SECRET_STORE_GCP_CREDENTIALS: ${{ secrets.SECRET_STORE_GCP_CREDENTIALS }}
           SECRET_STORE_GCP_PROJECT_ID: ${{ secrets.SECRET_STORE_GCP_PROJECT_ID }}
         uses: Wandalen/wretry.action@master
@@ -993,6 +1009,7 @@ jobs:
           aws-access-key-id: ${{ secrets.SELF_RUNNER_AWS_ACCESS_KEY_ID }}
           aws-secret-access-key: ${{ secrets.SELF_RUNNER_AWS_SECRET_ACCESS_KEY }}
           github-token: ${{ env.PAT }}
+          iam-role-name: ${{ secrets.AWS_ASSUME_ROLE_NAME }}
 
   helm-acceptance-test:
     name: "Platform: Acceptance Tests (Helm)"
@@ -1023,12 +1040,8 @@ jobs:
         uses: Wandalen/wretry.action@master
         with:
           command: |
-            sudo rm /var/lib/dpkg/lock
-            sudo rm /var/lib/dpkg/lock-frontend
             sudo apt-get remove docker.io || sudo apt-get remove docker
             curl -fsSL https://get.docker.com | bash -
-            sudo rm -f /var/lib/dpkg/lock
-            sudo rm -f /var/lib/dpkg/lock-frontend
           attempt_limit: 3
           attempt_delay: 2000 # in ms
 
@@ -1065,11 +1078,7 @@ jobs:
         uses: Wandalen/wretry.action@master
         with:
           command: |
-            sudo rm /var/lib/dpkg/lock
-            sudo rm /var/lib/dpkg/lock-frontend
             sudo apt-get -y install tmate
-            sudo rm -f /var/lib/dpkg/lock
-            sudo rm -f /var/lib/dpkg/lock-frontend
           attempt_limit: 3
           attempt_delay: 2000 # in ms
 
@@ -1124,6 +1133,8 @@ jobs:
           # HOME: /home/runner
           # AWS_S3_INTEGRATION_TEST_CREDS can be found in LastPass as AWS_S3_INTEGRATION_TEST_CREDS
           AWS_S3_INTEGRATION_TEST_CREDS: ${{ secrets.AWS_S3_INTEGRATION_TEST_CREDS }}
+          WORKFLOW_RUN_ID: ${{ github.run_id }}
+          AWS_S3_BUCKET: airbyte-ci-ac-tests-logs
           SECRET_STORE_GCP_CREDENTIALS: ${{ secrets.SECRET_STORE_GCP_CREDENTIALS }}
           SECRET_STORE_GCP_PROJECT_ID: ${{ secrets.SECRET_STORE_GCP_PROJECT_ID }}
         timeout-minutes: 20

tools/bin/acceptance_test_kube.sh

@@ -20,6 +20,17 @@ if [ -n "$CI" ]; then
   wait
 fi
 
+echo "Deploying fluentbit"
+helm repo add fluent https://fluent.github.io/helm-charts
+helm repo update fluent
+sed -i "s/PLACEHOLDER/${WORKFLOW_RUN_ID}/" tools/bin/fluent_values.yaml  
+helm install --values tools/bin/fluent_values.yaml --set env[1].name="AWS_ACCESS_KEY_ID" --set env[1].value=$(echo "$AWS_S3_INTEGRATION_TEST_CREDS" | jq -r .aws_access_key_id) \
+ --set env[2].name="AWS_SECRET_ACCESS_KEY" --set env[2].value=$(echo "$AWS_S3_INTEGRATION_TEST_CREDS" | jq -r .aws_secret_access_key) \
+ --set env[3].name="AWS_S3_BUCKET" --set env[3].value=${AWS_S3_BUCKET} \
+ --set env[4].name="SUITE_TYPE" --set env[4].value="kustomize-logs" \
+ --generate-name fluent/fluent-bit
+
+
 echo "Starting app..."
 
 echo "Applying dev-integration-test manifests to kubernetes..."

tools/bin/acceptance_test_kube_helm.sh

@@ -17,6 +17,16 @@ kubectl patch configmap/coredns \
   -p '{"data":{"NodeHosts": "${DOCKER_HOST_IP} host.docker.internal" }}'
 
 
+echo "Deploying fluentbit"
+helm repo add fluent https://fluent.github.io/helm-charts
+helm repo update fluent
+sed -i "s/PLACEHOLDER/${WORKFLOW_RUN_ID}/" tools/bin/fluent_values.yaml  
+helm install --values tools/bin/fluent_values.yaml --set env[1].name="AWS_ACCESS_KEY_ID" --set env[1].value=$(echo "$AWS_S3_INTEGRATION_TEST_CREDS" | jq -r .aws_access_key_id) \
+ --set env[2].name="AWS_SECRET_ACCESS_KEY" --set env[2].value=$(echo "$AWS_S3_INTEGRATION_TEST_CREDS" | jq -r .aws_secret_access_key) \
+ --set env[3].name="AWS_S3_BUCKET" --set env[3].value=${AWS_S3_BUCKET} \
+ --set env[4].name="SUITE_TYPE" --set env[4].value="helm-logs" \
+ --generate-name fluent/fluent-bit
+
 echo "Replacing default Chart.yaml and values.yaml with a test one"
 mv charts/airbyte/Chart.yaml charts/airbyte/Chart.yaml.old
 mv charts/airbyte/Chart.yaml.test charts/airbyte/Chart.yaml

tools/bin/fluent_values.yaml

@@ -0,0 +1,15 @@
+config:
+  outputs: |
+    [OUTPUT]
+        Name s3
+        Match kube.*
+        bucket ${AWS_S3_BUCKET}
+        region us-east-2
+        total_file_size 1M
+        upload_timeout 2m
+        use_put_object On
+        log_key log
+        s3_key_format /${SUITE_TYPE}/${WORKFLOW_RUN_ID}/$TAG[4]/$UUID.log
+env:
+  - name: WORKFLOW_RUN_ID
+    value: 'PLACEHOLDER'