From fc29dc16a8ce44807918ff21050e4800c72086d7 Mon Sep 17 00:00:00 2001 From: cpdeethree Date: Tue, 1 Nov 2022 11:13:50 -0500 Subject: [PATCH 1/5] ci: use custom action to upload job results --- .github/workflows/gradle.yml | 130 ++++++++++++++++++++----- tools/bin/prep_test_results_for_gcs.py | 74 ++++++++++++++ 2 files changed, 179 insertions(+), 25 deletions(-) create mode 100644 tools/bin/prep_test_results_for_gcs.py diff --git a/.github/workflows/gradle.yml b/.github/workflows/gradle.yml index 1682e4fc1f1e..95dc0d2af408 100644 --- a/.github/workflows/gradle.yml +++ b/.github/workflows/gradle.yml @@ -84,6 +84,7 @@ jobs: ${{ secrets.SUPERTOPHER_PAT }} \ ${{ secrets.DAVINCHIA_PAT }} + # Uncomment to debug. # changes-output: # name: "Debug Change Detection Logic" @@ -235,6 +236,37 @@ jobs: - name: Ensure no file change run: git --no-pager diff && test -z "$(git --no-pager diff)" + - name: Publish Connectors Base Test Results + uses: EnricoMi/publish-unit-test-result-action@v2 + id: connectors-test-results + if: always() + with: + junit_files: "/actions-runner/_work/airbyte/airbyte/*/build/test-results/*/*.xml\n/actions-runner/_work/airbyte/airbyte/*/*/build/test-results/*/*.xml" + comment_mode: off + json_file: connectors_base_results.json + json_test_case_results: true + check_name: "Connectors Base Test Results" + + - name: Setup Cloud SDK + if: always() + uses: google-github-actions/setup-gcloud@v0 + with: + service_account_key: ${{ secrets.GKE_TEST_SA_KEY }} + export_default_credentials: true + + - name: Prep Test Results For GCS + if: always() + run: | + python tools/bin/prep_test_results_for_gcs.py --json connectors_base_results.json + + - name: Upload Test Results to GCS + if: always() + run: | + gcs_bucket_name="dev-ab-ci-run-results" + filename=$(echo "${{ fromJSON( steps.connectors-test-results.outputs.json ).check_url }}" | sed 's@.*/@@') + echo "$filename" + gsutil -h "Cache-Control:public" cp connectors_base_results.jsonl "gs://$gcs_bucket_name/$filename.jsonl" + - name: Generate Test Report uses: dorny/test-reporter@v1 if: always() @@ -522,6 +554,42 @@ jobs: - name: Automatic Migration Acceptance Test run: SUB_BUILD=PLATFORM ./gradlew :airbyte-tests:automaticMigrationAcceptanceTest --scan -i + - uses: actions/setup-python@v2 + if: always() + with: + python-version: "3.9" + + - name: Publish Platform Test Results + uses: EnricoMi/publish-unit-test-result-action@v2 + id: platform-results + if: always() + with: + junit_files: "/actions-runner/_work/airbyte/airbyte/*/build/test-results/*/*.xml\n/actions-runner/_work/airbyte/airbyte/*/*/build/test-results/*/*.xml" + comment_mode: off + json_file: platform_results.json + json_test_case_results: true + check_name: "Platform Test Results" + + - name: Setup Cloud SDK + if: always() + uses: google-github-actions/setup-gcloud@v0 + with: + service_account_key: ${{ secrets.GKE_TEST_SA_KEY }} + export_default_credentials: true + + - name: Prep Test Results For GCS + if: always() + run: | + python tools/bin/prep_test_results_for_gcs.py --json platform_results.json + + - name: Upload Test Results to GCS + if: always() + run: | + gcs_bucket_name="dev-ab-ci-run-results" + filename=$(echo "${{ fromJSON( steps.platform-results.outputs.json ).check_url }}" | sed 's@.*/@@') + echo "$filename" + gsutil -h "Cache-Control:public" cp platform_results.jsonl "gs://$gcs_bucket_name/$filename.jsonl" + - name: Generate Test Report uses: dorny/test-reporter@v1 if: always() # run this step even if previous step failed @@ -541,15 +609,6 @@ jobs: key: ${{ secrets.BUILDPULSE_ACCESS_KEY_ID }} secret: ${{ secrets.BUILDPULSE_SECRET_ACCESS_KEY }} - - name: Upload test results to Github for analysis - if: '!cancelled()' # Run this step even when the tests fail. Skip if the workflow is cancelled. - uses: actions/upload-artifact@v3 - with: - path: | - /actions-runner/_work/airbyte/airbyte/*/build/test-results/*/*.xml - /actions-runner/_work/airbyte/airbyte/*/*/build/test-results/*/*.xml - name: test-results-build - # In case of self-hosted EC2 errors, remove this block. stop-platform-build-runner: name: "Platform: Stop Build EC2 Runner" @@ -681,6 +740,41 @@ jobs: run: | CI=true IS_MINIKUBE=true ./tools/bin/acceptance_test_kube.sh + - uses: actions/setup-python@v2 + with: + python-version: "3.9" + + - name: Publish Kube Test Results + id: kube-results + uses: EnricoMi/publish-unit-test-result-action@v2 + if: always() + with: + junit_files: "/actions-runner/_work/airbyte/airbyte/*/build/test-results/*/*.xml\n/actions-runner/_work/airbyte/airbyte/*/*/build/test-results/*/*.xml" + comment_mode: off + json_file: kube_results.json + json_test_case_results: true + check_name: "Kube Test Results" + + - name: Setup Cloud SDK + if: always() + uses: google-github-actions/setup-gcloud@v0 + with: + service_account_key: ${{ secrets.GKE_TEST_SA_KEY }} + export_default_credentials: true + + - name: Prep Test Results For GCS + if: always() + run: | + python tools/bin/prep_test_results_for_gcs.py --json kube_results.json + + - name: Upload Test Results to GCS + if: always() + run: | + gcs_bucket_name="dev-ab-ci-run-results" + filename=$(echo "${{ fromJSON( steps.kube-results.outputs.json ).check_url }}" | sed 's@.*/@@') + echo "$filename" + gsutil -h "Cache-Control:public" cp kube_results.jsonl "gs://$gcs_bucket_name/$filename.jsonl" + - name: Generate Test Report uses: dorny/test-reporter@v1 if: always() # run this step even if previous step failed @@ -699,20 +793,13 @@ jobs: key: ${{ secrets.BUILDPULSE_ACCESS_KEY_ID }} secret: ${{ secrets.BUILDPULSE_SECRET_ACCESS_KEY }} - - name: Upload test results to Github for analysis - if: '!cancelled()' # Run this step even when the tests fail. Skip if the workflow is cancelled. - uses: actions/upload-artifact@v3 - with: - path: | - /actions-runner/_work/airbyte/airbyte/*/build/test-results/*/*.xml - /actions-runner/_work/airbyte/airbyte/*/*/build/test-results/*/*.xml - name: test-results-kube - - uses: actions/upload-artifact@v2 if: failure() with: name: Kubernetes Logs path: /tmp/kubernetes_logs/* + + # In case of self-hosted EC2 errors, remove this block. stop-kube-acceptance-test-runner: name: "Platform: Stop Kube Acceptance Test EC2 Runner" @@ -859,13 +946,6 @@ jobs: # SECRET_STORE_GCP_PROJECT_ID: ${{ secrets.SECRET_STORE_GCP_PROJECT_ID }} # run: | # CI=true IS_MINIKUBE=true ./tools/bin/acceptance_test_kube_helm.sh -# - name: Generate Test Report -# uses: dorny/test-reporter@v1 -# if: always() # run this step even if previous step failed -# with: -# name: Platform Helm E2E Test Report -# path: '/actions-runner/_work/airbyte/airbyte/*/build/test-results/*/*.xml' -# reporter: java-junit # # - uses: actions/upload-artifact@v2 # if: failure() diff --git a/tools/bin/prep_test_results_for_gcs.py b/tools/bin/prep_test_results_for_gcs.py new file mode 100644 index 000000000000..9ebd33e9d9e3 --- /dev/null +++ b/tools/bin/prep_test_results_for_gcs.py @@ -0,0 +1,74 @@ +import argparse +import json +import os + + +''' + +This script is intended to be run in conjuction with https://github.com/EnricoMi/publish-unit-test-result-action to upload trimmed +test results from the output to a GCS bucket for further analysis. + +The script takes as input the filename of the json output by the aforementioned action, trims it, and uploads it to GCS with a ".jsonl" filename + +''' + +# Initiate the parser +parser = argparse.ArgumentParser() + +# Add long and short argument +parser.add_argument("--json", "-j", help="Path to the result json output by https://github.com/EnricoMi/publish-unit-test-result-action") + +def main(): + # Read arguments from the command line + args = parser.parse_args() + + token = os.getenv('GITHUB_TOKEN') + + f = open(args.json) + d = json.load(f) + out = [] + + check_run_id = int(d["check_url"].split("/")[-1]) + for elem in d['cases']: + if 'success' in elem['states']: + for i in range(len(elem['states']['success'])): + output = { + "test_name": elem['states']['success'][i]['test_name'], + "class_name": elem['states']['success'][i]['class_name'], + "result_file": elem['states']['success'][i]['result_file'], + "time": elem['states']['success'][i]['time'], + "state": "success", + "check_run_id": check_run_id, + } + out.append(output) + if 'failure' in elem['states']: + for i in range(len(elem['states']['failure'])): + output = { + "test_name": elem['states']['failure'][i]['test_name'], + "class_name": elem['states']['failure'][i]['class_name'], + "result_file": elem['states']['failure'][i]['result_file'], + "time": elem['states']['failure'][i]['time'], + "state": "failure", + "check_run_id": check_run_id, + } + out.append(output) + if 'skipped' in elem['states']: + for i in range(len(elem['states']['skipped'])): + output = { + "test_name": elem['states']['skipped'][i]['test_name'], + "class_name": elem['states']['skipped'][i]['class_name'], + "result_file": elem['states']['skipped'][i]['result_file'], + "time": elem['states']['skipped'][i]['time'], + "state": "skipped", + "check_run_id": check_run_id, + } + out.append(output) + + with open(args.json + "l", 'w') as f: + for o in out: + json.dump(o, f) + f.write('\n') + + +if __name__ == '__main__': + main() \ No newline at end of file From 81d48e6998c75ea5469d5638a39a887f339336b3 Mon Sep 17 00:00:00 2001 From: cpdeethree Date: Tue, 1 Nov 2022 12:08:39 -0500 Subject: [PATCH 2/5] fix: pr comments --- .github/workflows/gradle.yml | 12 ++++---- tools/bin/prep_test_results_for_gcs.py | 39 +++++++------------------- 2 files changed, 16 insertions(+), 35 deletions(-) diff --git a/.github/workflows/gradle.yml b/.github/workflows/gradle.yml index 1f1421cf94fe..1fa220e4421e 100644 --- a/.github/workflows/gradle.yml +++ b/.github/workflows/gradle.yml @@ -249,7 +249,7 @@ jobs: json_test_case_results: true check_name: "Connectors Base Test Results" - - name: Setup Cloud SDK + - name: Setup Google Cloud SDK if: always() uses: google-github-actions/setup-gcloud@v0 with: @@ -267,7 +267,7 @@ jobs: gcs_bucket_name="dev-ab-ci-run-results" filename=$(echo "${{ fromJSON( steps.connectors-test-results.outputs.json ).check_url }}" | sed 's@.*/@@') echo "$filename" - gsutil -h "Cache-Control:public" cp connectors_base_results.jsonl "gs://$gcs_bucket_name/$filename.jsonl" + gsutil -h "Cache-Control:public" cp connectors_base_results.jsonl "gs://$gcs_bucket_name/oss/$filename.jsonl" - name: Generate Test Report uses: dorny/test-reporter@v1 @@ -572,7 +572,7 @@ jobs: json_test_case_results: true check_name: "Platform Test Results" - - name: Setup Cloud SDK + - name: Setup Google Cloud SDK if: always() uses: google-github-actions/setup-gcloud@v0 with: @@ -590,7 +590,7 @@ jobs: gcs_bucket_name="dev-ab-ci-run-results" filename=$(echo "${{ fromJSON( steps.platform-results.outputs.json ).check_url }}" | sed 's@.*/@@') echo "$filename" - gsutil -h "Cache-Control:public" cp platform_results.jsonl "gs://$gcs_bucket_name/$filename.jsonl" + gsutil -h "Cache-Control:public" cp platform_results.jsonl "gs://$gcs_bucket_name/oss/$filename.jsonl" - name: Generate Test Report uses: dorny/test-reporter@v1 @@ -757,7 +757,7 @@ jobs: json_test_case_results: true check_name: "Kube Test Results" - - name: Setup Cloud SDK + - name: Setup Google Cloud SDK if: always() uses: google-github-actions/setup-gcloud@v0 with: @@ -775,7 +775,7 @@ jobs: gcs_bucket_name="dev-ab-ci-run-results" filename=$(echo "${{ fromJSON( steps.kube-results.outputs.json ).check_url }}" | sed 's@.*/@@') echo "$filename" - gsutil -h "Cache-Control:public" cp kube_results.jsonl "gs://$gcs_bucket_name/$filename.jsonl" + gsutil -h "Cache-Control:public" cp kube_results.jsonl "gs://$gcs_bucket_name/oss/$filename.jsonl" - name: Generate Test Report uses: dorny/test-reporter@v1 diff --git a/tools/bin/prep_test_results_for_gcs.py b/tools/bin/prep_test_results_for_gcs.py index 9ebd33e9d9e3..72cc4a249f93 100644 --- a/tools/bin/prep_test_results_for_gcs.py +++ b/tools/bin/prep_test_results_for_gcs.py @@ -29,37 +29,18 @@ def main(): out = [] check_run_id = int(d["check_url"].split("/")[-1]) + for elem in d['cases']: - if 'success' in elem['states']: - for i in range(len(elem['states']['success'])): - output = { - "test_name": elem['states']['success'][i]['test_name'], - "class_name": elem['states']['success'][i]['class_name'], - "result_file": elem['states']['success'][i]['result_file'], - "time": elem['states']['success'][i]['time'], - "state": "success", - "check_run_id": check_run_id, - } - out.append(output) - if 'failure' in elem['states']: - for i in range(len(elem['states']['failure'])): - output = { - "test_name": elem['states']['failure'][i]['test_name'], - "class_name": elem['states']['failure'][i]['class_name'], - "result_file": elem['states']['failure'][i]['result_file'], - "time": elem['states']['failure'][i]['time'], - "state": "failure", - "check_run_id": check_run_id, - } - out.append(output) - if 'skipped' in elem['states']: - for i in range(len(elem['states']['skipped'])): + for conclusion in ('success', 'failure', 'skipped'): + if conclusion not in elem['states']: + continue + for i in range(len(elem['states'][conclusion])): output = { - "test_name": elem['states']['skipped'][i]['test_name'], - "class_name": elem['states']['skipped'][i]['class_name'], - "result_file": elem['states']['skipped'][i]['result_file'], - "time": elem['states']['skipped'][i]['time'], - "state": "skipped", + "test_name": elem['states'][conclusion][i]['test_name'], + "class_name": elem['states'][conclusion][i]['class_name'], + "result_file": elem['states'][conclusion][i]['result_file'], + "time": elem['states'][conclusion][i]['time'], + "state": conclusion, "check_run_id": check_run_id, } out.append(output) From ac45c49f79cce03f192967469a7717082a420e48 Mon Sep 17 00:00:00 2001 From: cpdeethree Date: Tue, 1 Nov 2022 12:21:35 -0500 Subject: [PATCH 3/5] fix: pr comments --- .github/workflows/gradle.yml | 1 + tools/bin/prep_test_results_for_gcs.py | 4 +--- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/.github/workflows/gradle.yml b/.github/workflows/gradle.yml index 1fa220e4421e..fdf484ba6a9f 100644 --- a/.github/workflows/gradle.yml +++ b/.github/workflows/gradle.yml @@ -743,6 +743,7 @@ jobs: CI=true IS_MINIKUBE=true ./tools/bin/acceptance_test_kube.sh - uses: actions/setup-python@v2 + if: always() with: python-version: "3.9" diff --git a/tools/bin/prep_test_results_for_gcs.py b/tools/bin/prep_test_results_for_gcs.py index 72cc4a249f93..96c5e716e155 100644 --- a/tools/bin/prep_test_results_for_gcs.py +++ b/tools/bin/prep_test_results_for_gcs.py @@ -8,7 +8,7 @@ This script is intended to be run in conjuction with https://github.com/EnricoMi/publish-unit-test-result-action to upload trimmed test results from the output to a GCS bucket for further analysis. -The script takes as input the filename of the json output by the aforementioned action, trims it, and uploads it to GCS with a ".jsonl" filename +The script takes as input the filename of the json output by the aforementioned action, trims it, and writes it out in jsonl format with ".jsonl" filename ''' @@ -22,8 +22,6 @@ def main(): # Read arguments from the command line args = parser.parse_args() - token = os.getenv('GITHUB_TOKEN') - f = open(args.json) d = json.load(f) out = [] From d000f073f309ad8fa7c3b2a6c815445377ee8393 Mon Sep 17 00:00:00 2001 From: cpdeethree Date: Tue, 1 Nov 2022 13:04:41 -0500 Subject: [PATCH 4/5] ci: add repo to test result records --- tools/bin/prep_test_results_for_gcs.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tools/bin/prep_test_results_for_gcs.py b/tools/bin/prep_test_results_for_gcs.py index 96c5e716e155..bfb6e8b59b87 100644 --- a/tools/bin/prep_test_results_for_gcs.py +++ b/tools/bin/prep_test_results_for_gcs.py @@ -40,6 +40,7 @@ def main(): "time": elem['states'][conclusion][i]['time'], "state": conclusion, "check_run_id": check_run_id, + "repo": "oss" } out.append(output) From 5c078249b0094f8ac39b622b500149517baf9e80 Mon Sep 17 00:00:00 2001 From: cpdeethree Date: Tue, 1 Nov 2022 13:10:02 -0500 Subject: [PATCH 5/5] ci: update repo data --- tools/bin/prep_test_results_for_gcs.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/bin/prep_test_results_for_gcs.py b/tools/bin/prep_test_results_for_gcs.py index bfb6e8b59b87..7f658eb381fc 100644 --- a/tools/bin/prep_test_results_for_gcs.py +++ b/tools/bin/prep_test_results_for_gcs.py @@ -40,7 +40,7 @@ def main(): "time": elem['states'][conclusion][i]['time'], "state": conclusion, "check_run_id": check_run_id, - "repo": "oss" + "repo": "airbytehq/airbyte" } out.append(output)