From b31fa7e253cc96c0da01c9cd76d2574c8612d4b5 Mon Sep 17 00:00:00 2001 From: Mateusz Bojanowski Date: Fri, 6 Dec 2024 14:19:07 +0100 Subject: [PATCH 1/4] [BSE-4337] Trigger BodoSQL customer tests when changes in BodoSQL/calcite_sql/ --- .github/workflows/java_tests.yml | 66 -------------------------------- .github/workflows/pr_ci.yml | 46 +++++++++++++++++++++- 2 files changed, 44 insertions(+), 68 deletions(-) delete mode 100644 .github/workflows/java_tests.yml diff --git a/.github/workflows/java_tests.yml b/.github/workflows/java_tests.yml deleted file mode 100644 index a1999cdf27..0000000000 --- a/.github/workflows/java_tests.yml +++ /dev/null @@ -1,66 +0,0 @@ -name: BodoSQL Java Tests -on: - # Pull Request that Modifies BodoSQL Java - pull_request: - paths: - - 'BodoSQL/calcite_sql/**' - -# Limit CI to cancel previous runs in the same PR -concurrency: - group: ${{ github.workflow }}-${{ github.head_ref || github.ref_name }} - cancel-in-progress: true - -jobs: - get-msg: - name: Get Commit Message - runs-on: ubuntu-latest - outputs: - commitMsg: ${{ steps.get_msg.outputs.commitMsg }} - steps: - # Fetch the branch for the history only - - uses: actions/checkout@v4 - with: - ref: ${{ github.event.pull_request.head.sha }} - sparse-checkout: . - - name: Get Commit Message - id: get_msg - run: | - set -xe pipefail - echo "commitMsg=$(git log -1 --pretty=format:'%s')" >> $GITHUB_OUTPUT - - run-java: - needs: [get-msg] - name: Run BodoSQL Java Tests - runs-on: [self-hosted, large] - # https://docs.github.com/en/actions/learn-github-actions/expressions#contains - # contains is case-insensitive - if: contains(needs.get-msg.outputs.commitMsg, '[run ci]') - steps: - - uses: actions/checkout@v4 - - name: Install Java - uses: actions/setup-java@v4 - with: - distribution: 'temurin' - java-version: '11' - cache: 'maven' - cache-dependency-path: 'BodoSQL/calcite_sql/bodosql-calcite-application/pom.xml' - - name: Install Maven - run: sudo yum install maven -y - - name: Run Maven Tests - run: | - which mvn - cd BodoSQL/calcite_sql/ - mvn -B -U test - - # Publishing Test Results - # Always run both even if tests fail - - name: Publish Test Result Failures - uses: mikepenz/action-junit-report@v4 - if: success() || failure() - with: - report_paths: 'BodoSQL/calcite_sql/bodosql-calcite-application/target/surefire-reports/TEST-*.xml' - - name: Publish Test Results in Comment - uses: EnricoMi/publish-unit-test-result-action@v2 - if: success() || failure() - with: - files: 'BodoSQL/calcite_sql/bodosql-calcite-application/target/surefire-reports/TEST-*.xml' diff --git a/.github/workflows/pr_ci.yml b/.github/workflows/pr_ci.yml index f8b9e88f5d..f2202929f5 100644 --- a/.github/workflows/pr_ci.yml +++ b/.github/workflows/pr_ci.yml @@ -8,7 +8,31 @@ concurrency: cancel-in-progress: true jobs: - # 1) Check if commit has [run ci] before continuing + + # 1) Validate changes in PR + validate: + name: Validate + runs-on: ubuntu-latest + outputs: + run_bodosql_customer_tests: ${{ steps.changes.outputs.bodosql_customer_tests }} + + steps: + - uses: actions/checkout@v4 + with: + ref: ${{ github.ref }} + fetch-depth: "0" + + - name: Validate Changes + id: changes + uses: dorny/paths-filter@v3 + with: + filters: | + bodosql_customer_tests: + - 'BodoSQL/calcite_sql/**' + + + # 2) Check if commit has [run ci] before continuing + # TODO: We will need to probably find different way for that commit-prefix-ci: name: Get Commit Message runs-on: ubuntu-latest @@ -16,19 +40,37 @@ jobs: # https://docs.github.com/en/actions/learn-github-actions/expressions#contains # contains is case-insensitive runCI: ${{ contains(steps.check_msg.outputs.commitMsg, '[run ci]') }} + steps: # Fetch the PR branch for the commit history - uses: actions/checkout@v4 with: ref: ${{ github.event.pull_request.head.sha }} sparse-checkout: . + - name: Check Commit Message id: check_msg run: | set -xe pipefail echo "commitMsg=$(git log -1 --pretty=format:'%s')" >> $GITHUB_OUTPUT + + bodosql-customer-tests: + needs: [validate] + name: BodoSQL Customer Tests + runs-on: ubuntu-latest + steps: + - uses: convictional/trigger-workflow-and-wait@v1.6.1 + with: + owner: bodo-ai + repo: customer-sample-code + github_token: ${{ secrets.BOT_HERMAN_GITHUB_TOKEN }} + github_user: ${{ secrets.BOT_GITHUB_USERNAME }} + workflow_file_name: customer_bodosql_ci.yaml + ref: master + client_payload: '{ "branch" : "${{ github.head_ref || github.ref_name }}" }' + - # 2) Pre-Build Bodo to save build artifacts to sccache + # 3) Pre-Build Bodo to save build artifacts to sccache compile-bodo: needs: [commit-prefix-ci] name: Pre-Build Bodo for Cache From 921ee4220e357fc4923fce977f42388fa2ab1d21 Mon Sep 17 00:00:00 2001 From: Mateusz Bojanowski Date: Fri, 6 Dec 2024 14:47:12 +0100 Subject: [PATCH 2/4] Update --- .github/workflows/pr_ci.yml | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/.github/workflows/pr_ci.yml b/.github/workflows/pr_ci.yml index f2202929f5..010f5680e8 100644 --- a/.github/workflows/pr_ci.yml +++ b/.github/workflows/pr_ci.yml @@ -64,13 +64,12 @@ jobs: owner: bodo-ai repo: customer-sample-code github_token: ${{ secrets.BOT_HERMAN_GITHUB_TOKEN }} - github_user: ${{ secrets.BOT_GITHUB_USERNAME }} workflow_file_name: customer_bodosql_ci.yaml ref: master client_payload: '{ "branch" : "${{ github.head_ref || github.ref_name }}" }' - # 3) Pre-Build Bodo to save build artifacts to sccache + # 4) Pre-Build Bodo to save build artifacts to sccache compile-bodo: needs: [commit-prefix-ci] name: Pre-Build Bodo for Cache @@ -93,7 +92,7 @@ jobs: run: | wget -O hadoop.tar.gz "https://www.apache.org/dyn/mirrors/mirrors.cgi?action=download&filename=hadoop/common/hadoop-3.3.2/hadoop-3.3.2.tar.gz" - # 3) Actually run tests + # 5) Actually run tests pr-ci: needs: [compile-bodo] name: Test @@ -121,7 +120,7 @@ jobs: collect-coverage: false secrets: inherit - # 4) Collect and combine any results from runs + # 6) Collect and combine any results from runs collect-results: needs: [pr-ci] name: Collect Results From 6664222d3b4f0072ca2526625ec44c9f2f84afea Mon Sep 17 00:00:00 2001 From: Mateusz Bojanowski Date: Fri, 6 Dec 2024 15:16:23 +0100 Subject: [PATCH 3/4] Update pipeline --- .github/workflows/pr_ci.yml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.github/workflows/pr_ci.yml b/.github/workflows/pr_ci.yml index 010f5680e8..1570b5608c 100644 --- a/.github/workflows/pr_ci.yml +++ b/.github/workflows/pr_ci.yml @@ -55,9 +55,13 @@ jobs: echo "commitMsg=$(git log -1 --pretty=format:'%s')" >> $GITHUB_OUTPUT bodosql-customer-tests: - needs: [validate] + needs: [validate, commit-prefix-ci] name: BodoSQL Customer Tests runs-on: ubuntu-latest + if: | + needs.commit-prefix-ci.outputs.runCI == 'true' && + needs.validate.outputs.run_bodosql_customer_tests == 'true' + steps: - uses: convictional/trigger-workflow-and-wait@v1.6.1 with: From 4905e0b59ce7abcf7838d606859d207d87c48b8e Mon Sep 17 00:00:00 2001 From: Mateusz Bojanowski Date: Mon, 9 Dec 2024 11:59:45 +0100 Subject: [PATCH 4/4] Small improvment --- .github/workflows/pr_ci.yml | 34 +++++++++++++--------------------- 1 file changed, 13 insertions(+), 21 deletions(-) diff --git a/.github/workflows/pr_ci.yml b/.github/workflows/pr_ci.yml index 1570b5608c..8a882bb1fd 100644 --- a/.github/workflows/pr_ci.yml +++ b/.github/workflows/pr_ci.yml @@ -9,11 +9,14 @@ concurrency: jobs: - # 1) Validate changes in PR + # 1) Validate changes and commit message validate: name: Validate runs-on: ubuntu-latest outputs: + # https://docs.github.com/en/actions/learn-github-actions/expressions#contains + # contains is case-insensitive + run_tests: ${{ contains(steps.check_msg.outputs.commit_message, '[run ci]') }} run_bodosql_customer_tests: ${{ steps.changes.outputs.bodosql_customer_tests }} steps: @@ -30,18 +33,6 @@ jobs: bodosql_customer_tests: - 'BodoSQL/calcite_sql/**' - - # 2) Check if commit has [run ci] before continuing - # TODO: We will need to probably find different way for that - commit-prefix-ci: - name: Get Commit Message - runs-on: ubuntu-latest - outputs: - # https://docs.github.com/en/actions/learn-github-actions/expressions#contains - # contains is case-insensitive - runCI: ${{ contains(steps.check_msg.outputs.commitMsg, '[run ci]') }} - - steps: # Fetch the PR branch for the commit history - uses: actions/checkout@v4 with: @@ -52,14 +43,15 @@ jobs: id: check_msg run: | set -xe pipefail - echo "commitMsg=$(git log -1 --pretty=format:'%s')" >> $GITHUB_OUTPUT + echo "commit_message=$(git log -1 --pretty=format:'%s')" >> $GITHUB_OUTPUT + # 2) Trigger BodoSQL Customer tests bodosql-customer-tests: - needs: [validate, commit-prefix-ci] + needs: [validate] name: BodoSQL Customer Tests runs-on: ubuntu-latest if: | - needs.commit-prefix-ci.outputs.runCI == 'true' && + needs.validate.outputs.run_tests == 'true' && needs.validate.outputs.run_bodosql_customer_tests == 'true' steps: @@ -73,12 +65,12 @@ jobs: client_payload: '{ "branch" : "${{ github.head_ref || github.ref_name }}" }' - # 4) Pre-Build Bodo to save build artifacts to sccache + # 3) Pre-Build Bodo to save build artifacts to sccache compile-bodo: - needs: [commit-prefix-ci] + needs: [validate] name: Pre-Build Bodo for Cache runs-on: [self-hosted, large] - if: needs.commit-prefix-ci.outputs.runCI == 'true' + if: needs.validate.outputs.run_tests == 'true' steps: - uses: actions/checkout@v4 - name: Build from Source @@ -96,7 +88,7 @@ jobs: run: | wget -O hadoop.tar.gz "https://www.apache.org/dyn/mirrors/mirrors.cgi?action=download&filename=hadoop/common/hadoop-3.3.2/hadoop-3.3.2.tar.gz" - # 5) Actually run tests + # 4) Actually run tests pr-ci: needs: [compile-bodo] name: Test @@ -124,7 +116,7 @@ jobs: collect-coverage: false secrets: inherit - # 6) Collect and combine any results from runs + # 5) Collect and combine any results from runs collect-results: needs: [pr-ci] name: Collect Results