diff --git a/.github/actions/start-aws-runner/action.yml b/.github/actions/start-aws-runner/action.yml index 526a71dea781..1d45df604d77 100644 --- a/.github/actions/start-aws-runner/action.yml +++ b/.github/actions/start-aws-runner/action.yml @@ -24,6 +24,7 @@ inputs: required: false ec2-instance-id: required: false + outputs: label: value: ${{ steps.start-ec2-runner.outputs.label }} diff --git a/.github/workflows/gradle.yml b/.github/workflows/gradle.yml index 52e6b923a1e6..03d8f1da3613 100644 --- a/.github/workflows/gradle.yml +++ b/.github/workflows/gradle.yml @@ -10,7 +10,11 @@ on: inputs: debug_mode: description: "Enable or disable tmate session for debug during helm ac tests" + type: choice default: 'false' + options: + - 'true' + - 'false' required: false schedule: - cron: "0 */1 * * *" @@ -811,6 +815,16 @@ jobs: with: python-version: "3.9" + - name: Install unzip + shell: bash + run: sudo apt-get update && sudo apt-get install -y unzip + + - uses: azure/setup-helm@v3 + with: + version: 'latest' + token: ${{ secrets.GITHUB_TOKEN }} + id: install + - name: Fix EC-2 Runner run: | mkdir -p /home/runner @@ -862,6 +876,8 @@ jobs: HOME: /home/runner # AWS_S3_INTEGRATION_TEST_CREDS can be found in LastPass as AWS_S3_INTEGRATION_TEST_CREDS AWS_S3_INTEGRATION_TEST_CREDS: ${{ secrets.AWS_S3_INTEGRATION_TEST_CREDS }} + WORKFLOW_RUN_ID: ${{ github.run_id }} + AWS_S3_BUCKET: airbyte-ci-ac-tests-logs SECRET_STORE_GCP_CREDENTIALS: ${{ secrets.SECRET_STORE_GCP_CREDENTIALS }} SECRET_STORE_GCP_PROJECT_ID: ${{ secrets.SECRET_STORE_GCP_PROJECT_ID }} uses: Wandalen/wretry.action@master @@ -998,6 +1014,7 @@ jobs: aws-access-key-id: ${{ secrets.SELF_RUNNER_AWS_ACCESS_KEY_ID }} aws-secret-access-key: ${{ secrets.SELF_RUNNER_AWS_SECRET_ACCESS_KEY }} github-token: ${{ env.PAT }} + iam-role-name: ${{ secrets.AWS_ASSUME_ROLE_NAME }} helm-acceptance-test: name: "Platform: Acceptance Tests (Helm)" @@ -1028,12 +1045,8 @@ jobs: uses: Wandalen/wretry.action@master with: command: | - sudo rm /var/lib/dpkg/lock - sudo rm /var/lib/dpkg/lock-frontend sudo apt-get remove docker.io || sudo apt-get remove docker curl -fsSL https://get.docker.com | bash - - sudo rm -f /var/lib/dpkg/lock - sudo rm -f /var/lib/dpkg/lock-frontend attempt_limit: 3 attempt_delay: 2000 # in ms @@ -1070,11 +1083,7 @@ jobs: uses: Wandalen/wretry.action@master with: command: | - sudo rm /var/lib/dpkg/lock - sudo rm /var/lib/dpkg/lock-frontend sudo apt-get -y install tmate - sudo rm -f /var/lib/dpkg/lock - sudo rm -f /var/lib/dpkg/lock-frontend attempt_limit: 3 attempt_delay: 2000 # in ms @@ -1129,6 +1138,8 @@ jobs: # HOME: /home/runner # AWS_S3_INTEGRATION_TEST_CREDS can be found in LastPass as AWS_S3_INTEGRATION_TEST_CREDS AWS_S3_INTEGRATION_TEST_CREDS: ${{ secrets.AWS_S3_INTEGRATION_TEST_CREDS }} + WORKFLOW_RUN_ID: ${{ github.run_id }} + AWS_S3_BUCKET: airbyte-ci-ac-tests-logs SECRET_STORE_GCP_CREDENTIALS: ${{ secrets.SECRET_STORE_GCP_CREDENTIALS }} SECRET_STORE_GCP_PROJECT_ID: ${{ secrets.SECRET_STORE_GCP_PROJECT_ID }} timeout-minutes: 20 diff --git a/tools/bin/acceptance_test_kube.sh b/tools/bin/acceptance_test_kube.sh index eb1d86de442d..097f673307c9 100755 --- a/tools/bin/acceptance_test_kube.sh +++ b/tools/bin/acceptance_test_kube.sh @@ -20,6 +20,17 @@ if [ -n "$CI" ]; then wait fi +echo "Deploying fluentbit" +helm repo add fluent https://fluent.github.io/helm-charts +helm repo update fluent +sed -i "s/PLACEHOLDER/${WORKFLOW_RUN_ID}/" tools/bin/fluent_values.yaml +helm install --values tools/bin/fluent_values.yaml --set env[1].name="AWS_ACCESS_KEY_ID" --set env[1].value=$(echo "$AWS_S3_INTEGRATION_TEST_CREDS" | jq -r .aws_access_key_id) \ + --set env[2].name="AWS_SECRET_ACCESS_KEY" --set env[2].value=$(echo "$AWS_S3_INTEGRATION_TEST_CREDS" | jq -r .aws_secret_access_key) \ + --set env[3].name="AWS_S3_BUCKET" --set env[3].value=${AWS_S3_BUCKET} \ + --set env[4].name="SUITE_TYPE" --set env[4].value="kustomize-logs" \ + --generate-name fluent/fluent-bit + + echo "Starting app..." echo "Applying dev-integration-test manifests to kubernetes..." diff --git a/tools/bin/acceptance_test_kube_helm.sh b/tools/bin/acceptance_test_kube_helm.sh index d3445766f605..97a6e95b1316 100755 --- a/tools/bin/acceptance_test_kube_helm.sh +++ b/tools/bin/acceptance_test_kube_helm.sh @@ -17,6 +17,16 @@ kubectl patch configmap/coredns \ -p '{"data":{"NodeHosts": "${DOCKER_HOST_IP} host.docker.internal" }}' +echo "Deploying fluentbit" +helm repo add fluent https://fluent.github.io/helm-charts +helm repo update fluent +sed -i "s/PLACEHOLDER/${WORKFLOW_RUN_ID}/" tools/bin/fluent_values.yaml +helm install --values tools/bin/fluent_values.yaml --set env[1].name="AWS_ACCESS_KEY_ID" --set env[1].value=$(echo "$AWS_S3_INTEGRATION_TEST_CREDS" | jq -r .aws_access_key_id) \ + --set env[2].name="AWS_SECRET_ACCESS_KEY" --set env[2].value=$(echo "$AWS_S3_INTEGRATION_TEST_CREDS" | jq -r .aws_secret_access_key) \ + --set env[3].name="AWS_S3_BUCKET" --set env[3].value=${AWS_S3_BUCKET} \ + --set env[4].name="SUITE_TYPE" --set env[4].value="helm-logs" \ + --generate-name fluent/fluent-bit + echo "Replacing default Chart.yaml and values.yaml with a test one" mv charts/airbyte/Chart.yaml charts/airbyte/Chart.yaml.old mv charts/airbyte/Chart.yaml.test charts/airbyte/Chart.yaml diff --git a/tools/bin/fluent_values.yaml b/tools/bin/fluent_values.yaml new file mode 100644 index 000000000000..1ec639291ab8 --- /dev/null +++ b/tools/bin/fluent_values.yaml @@ -0,0 +1,15 @@ +config: + outputs: | + [OUTPUT] + Name s3 + Match kube.* + bucket ${AWS_S3_BUCKET} + region us-east-2 + total_file_size 1M + upload_timeout 2m + use_put_object On + log_key log + s3_key_format /${SUITE_TYPE}/${WORKFLOW_RUN_ID}/$TAG[4]/$UUID.log +env: + - name: WORKFLOW_RUN_ID + value: 'PLACEHOLDER' \ No newline at end of file