From ad3317a20012bcfdec990395c30dbaecf45f460d Mon Sep 17 00:00:00 2001 From: Manan Gupta <35839558+GuptaManan100@users.noreply.github.com> Date: Wed, 20 Dec 2023 09:21:16 +0530 Subject: [PATCH 1/5] Refactor Upgrade downgrade tests (#14782) Signed-off-by: Manan Gupta --- .../upgrade_downgrade_test_backups_e2e.yml | 19 ++++--- ...owngrade_test_backups_e2e_next_release.yml | 46 ++++++---------- .../upgrade_downgrade_test_backups_manual.yml | 38 +++++-------- ...grade_test_backups_manual_next_release.yml | 54 +++++++------------ ...e_downgrade_test_query_serving_queries.yml | 46 ++++++---------- ...est_query_serving_queries_next_release.yml | 52 +++++++----------- ...de_downgrade_test_query_serving_schema.yml | 46 ++++++---------- ...test_query_serving_schema_next_release.yml | 52 +++++++----------- ...rade_downgrade_test_reparent_new_vtctl.yml | 52 +++++++----------- ...e_downgrade_test_reparent_new_vttablet.yml | 52 +++++++----------- ...rade_downgrade_test_reparent_old_vtctl.yml | 46 ++++++---------- ...e_downgrade_test_reparent_old_vttablet.yml | 46 ++++++---------- test/config.json | 20 +++---- 13 files changed, 216 insertions(+), 353 deletions(-) diff --git a/.github/workflows/upgrade_downgrade_test_backups_e2e.yml b/.github/workflows/upgrade_downgrade_test_backups_e2e.yml index 8769c59a30c..e915644b4e5 100644 --- a/.github/workflows/upgrade_downgrade_test_backups_e2e.yml +++ b/.github/workflows/upgrade_downgrade_test_backups_e2e.yml @@ -1,4 +1,4 @@ -name: Upgrade Downgrade Testing - Backups - E2E +name: Backups - E2E - Upgrade Downgrade Testing on: push: pull_request: @@ -32,11 +32,8 @@ jobs: upgrade_downgrade_test_e2e: timeout-minutes: 60 - if: always() && needs.get_previous_release.result == 'success' name: Run Upgrade Downgrade Test - Backups - E2E runs-on: ubuntu-22.04 - needs: - - get_previous_release steps: - name: Skip CI @@ -59,6 +56,16 @@ jobs: - name: Check out commit's code if: steps.skip-workflow.outputs.skip-workflow == 'false' uses: actions/checkout@v3 + with: + fetch-depth: 0 + + - name: Set output with latest release branch + if: steps.skip-workflow.outputs.skip-workflow == 'false' + id: output-previous-release-ref + run: | + previous_release_ref=$(./tools/get_previous_release.sh ${{github.base_ref}} ${{github.ref}}) + echo $previous_release_ref + echo "previous_release_ref=${previous_release_ref}" >> $GITHUB_OUTPUT - name: Check for changes in relevant files if: steps.skip-workflow.outputs.skip-workflow == 'false' @@ -117,11 +124,11 @@ jobs: sudo apt-get install percona-xtrabackup-24 # Checkout to the last release of Vitess - - name: Check out other version's code (${{ needs.get_previous_release.outputs.previous_release }}) + - name: Check out other version's code (${{ steps.output-previous-release-ref.outputs.previous_release_ref }}) if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' uses: actions/checkout@v3 with: - ref: ${{ needs.get_previous_release.outputs.previous_release }} + ref: ${{ steps.output-previous-release-ref.outputs.previous_release_ref }} - name: Get dependencies for the last release if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' diff --git a/.github/workflows/upgrade_downgrade_test_backups_e2e_next_release.yml b/.github/workflows/upgrade_downgrade_test_backups_e2e_next_release.yml index 6e610a142c2..2f1d6a4641b 100644 --- a/.github/workflows/upgrade_downgrade_test_backups_e2e_next_release.yml +++ b/.github/workflows/upgrade_downgrade_test_backups_e2e_next_release.yml @@ -1,4 +1,4 @@ -name: Upgrade Downgrade Testing - Backups - E2E - Next Release +name: Backups - E2E - Next Release - Upgrade Downgrade Testing on: push: pull_request: @@ -10,33 +10,11 @@ concurrency: permissions: read-all jobs: - get_next_release: - if: always() - name: Get Latest Release - Backups - E2E - Next Release - runs-on: ubuntu-22.04 - outputs: - next_release: ${{ steps.output-next-release-ref.outputs.next_release_ref }} - - steps: - - name: Check out to HEAD - uses: actions/checkout@v3 - with: - fetch-depth: 0 - - - name: Set output with latest release branch - id: output-next-release-ref - run: | - next_release_ref=$(./tools/get_next_release.sh ${{github.base_ref}} ${{github.ref}}) - echo $next_release_ref - echo "next_release_ref=${next_release_ref}" >> $GITHUB_OUTPUT upgrade_downgrade_test_e2e: timeout-minutes: 60 - if: always() && needs.get_next_release.result == 'success' name: Run Upgrade Downgrade Test - Backups - E2E - Next Release runs-on: ubuntu-22.04 - needs: - - get_next_release steps: - name: Skip CI @@ -46,6 +24,18 @@ jobs: exit 1 fi + - name: Check out commit's code + uses: actions/checkout@v3 + with: + fetch-depth: 0 + + - name: Set output with latest release branch + id: output-next-release-ref + run: | + next_release_ref=$(./tools/get_next_release.sh ${{github.base_ref}} ${{github.ref}}) + echo $next_release_ref + echo "next_release_ref=${next_release_ref}" >> $GITHUB_OUTPUT + - name: Check if workflow needs to be skipped id: skip-workflow run: | @@ -53,16 +43,12 @@ jobs: if [[ "${{github.event.pull_request}}" == "" ]] && [[ "${{github.ref}}" != "refs/heads/main" ]] && [[ ! "${{github.ref}}" =~ ^refs/heads/release-[0-9]+\.[0-9]$ ]] && [[ ! "${{github.ref}}" =~ "refs/tags/.*" ]]; then skip='true' fi - if [[ "${{needs.get_next_release.outputs.next_release}}" == "" ]]; then + if [[ "${{steps.output-next-release-ref.outputs.next_release_ref}}" == "" ]]; then skip='true' fi echo Skip ${skip} echo "skip-workflow=${skip}" >> $GITHUB_OUTPUT - - name: Check out commit's code - if: steps.skip-workflow.outputs.skip-workflow == 'false' - uses: actions/checkout@v3 - - name: Check for changes in relevant files if: steps.skip-workflow.outputs.skip-workflow == 'false' uses: frouioui/paths-filter@main @@ -120,11 +106,11 @@ jobs: sudo apt-get install percona-xtrabackup-24 # Checkout to the next release of Vitess - - name: Check out other version's code (${{ needs.get_next_release.outputs.next_release }}) + - name: Check out other version's code (${{ steps.output-next-release-ref.outputs.next_release_ref }}) if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' uses: actions/checkout@v3 with: - ref: ${{ needs.get_next_release.outputs.next_release }} + ref: ${{ steps.output-next-release-ref.outputs.next_release_ref }} - name: Get dependencies for the next release if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' diff --git a/.github/workflows/upgrade_downgrade_test_backups_manual.yml b/.github/workflows/upgrade_downgrade_test_backups_manual.yml index 6b415cf2283..20bd31c7dc7 100644 --- a/.github/workflows/upgrade_downgrade_test_backups_manual.yml +++ b/.github/workflows/upgrade_downgrade_test_backups_manual.yml @@ -1,4 +1,4 @@ -name: Upgrade Downgrade Testing - Backups - Manual +name: Backups - Manual - Upgrade Downgrade Testing on: push: pull_request: @@ -10,34 +10,12 @@ concurrency: permissions: read-all jobs: - get_previous_release: - if: always() - name: Get Previous Release - Backups - Manual - runs-on: ubuntu-22.04 - outputs: - previous_release: ${{ steps.output-previous-release-ref.outputs.previous_release_ref }} - - steps: - - name: Check out to HEAD - uses: actions/checkout@v3 - with: - fetch-depth: 0 - - - name: Set output with latest release branch - id: output-previous-release-ref - run: | - previous_release_ref=$(./tools/get_previous_release.sh ${{github.base_ref}} ${{github.ref}}) - echo $previous_release_ref - echo "previous_release_ref=${previous_release_ref}" >> $GITHUB_OUTPUT # This job usually execute in ± 20 minutes upgrade_downgrade_test_manual: timeout-minutes: 40 - if: always() && (needs.get_previous_release.result == 'success') name: Run Upgrade Downgrade Test - Backups - Manual runs-on: ubuntu-22.04 - needs: - - get_previous_release steps: - name: Skip CI @@ -61,6 +39,16 @@ jobs: - name: Checkout to commit's code if: steps.skip-workflow.outputs.skip-workflow == 'false' uses: actions/checkout@v3 + with: + fetch-depth: 0 + + - name: Set output with latest release branch + id: output-previous-release-ref + if: steps.skip-workflow.outputs.skip-workflow == 'false' + run: | + previous_release_ref=$(./tools/get_previous_release.sh ${{github.base_ref}} ${{github.ref}}) + echo $previous_release_ref + echo "previous_release_ref=${previous_release_ref}" >> $GITHUB_OUTPUT - name: Check for changes in relevant files if: steps.skip-workflow.outputs.skip-workflow == 'false' @@ -138,11 +126,11 @@ jobs: sudo apt-get install percona-xtrabackup-24 # Checkout to the last release of Vitess - - name: Checkout to the other version's code (${{ needs.get_previous_release.outputs.previous_release }}) + - name: Checkout to the other version's code (${{ steps.output-previous-release-ref.outputs.previous_release_ref }}) if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' uses: actions/checkout@v3 with: - ref: ${{ needs.get_previous_release.outputs.previous_release }} + ref: ${{ steps.output-previous-release-ref.outputs.previous_release_ref }} - name: Get dependencies for the last release if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' diff --git a/.github/workflows/upgrade_downgrade_test_backups_manual_next_release.yml b/.github/workflows/upgrade_downgrade_test_backups_manual_next_release.yml index b6b84cac2db..f73345ddf39 100644 --- a/.github/workflows/upgrade_downgrade_test_backups_manual_next_release.yml +++ b/.github/workflows/upgrade_downgrade_test_backups_manual_next_release.yml @@ -1,4 +1,4 @@ -name: Upgrade Downgrade Testing - Backups - Manual - Next Release +name: Backups - Manual - Next Release - Upgrade Downgrade Testing on: push: pull_request: @@ -10,34 +10,12 @@ concurrency: permissions: read-all jobs: - get_next_release: - if: always() - name: Get Previous Release - Backups - Manual - Next Release - runs-on: ubuntu-22.04 - outputs: - next_release: ${{ steps.output-next-release-ref.outputs.next_release_ref }} - - steps: - - name: Check out to HEAD - uses: actions/checkout@v3 - with: - fetch-depth: 0 - - - name: Set output with latest release branch - id: output-next-release-ref - run: | - next_release_ref=$(./tools/get_next_release.sh ${{github.base_ref}} ${{github.ref}}) - echo $next_release_ref - echo "next_release_ref=${next_release_ref}" >> $GITHUB_OUTPUT # This job usually execute in ± 20 minutes upgrade_downgrade_test_manual: timeout-minutes: 40 - if: always() && (needs.get_next_release.result == 'success') name: Run Upgrade Downgrade Test - Backups - Manual - Next Release runs-on: ubuntu-22.04 - needs: - - get_next_release steps: - name: Skip CI @@ -47,6 +25,19 @@ jobs: exit 1 fi + # Checkout to this build's commit + - name: Checkout to commit's code + uses: actions/checkout@v3 + with: + fetch-depth: 0 + + - name: Set output with latest release branch + id: output-next-release-ref + run: | + next_release_ref=$(./tools/get_next_release.sh ${{github.base_ref}} ${{github.ref}}) + echo $next_release_ref + echo "next_release_ref=${next_release_ref}" >> $GITHUB_OUTPUT + - name: Check if workflow needs to be skipped id: skip-workflow run: | @@ -54,17 +45,12 @@ jobs: if [[ "${{github.event.pull_request}}" == "" ]] && [[ "${{github.ref}}" != "refs/heads/main" ]] && [[ ! "${{github.ref}}" =~ ^refs/heads/release-[0-9]+\.[0-9]$ ]] && [[ ! "${{github.ref}}" =~ "refs/tags/.*" ]]; then skip='true' fi - if [[ "${{needs.get_next_release.outputs.next_release}}" == "" ]]; then + if [[ "${{steps.output-next-release-ref.outputs.next_release_ref}}" == "" ]]; then skip='true' fi echo Skip ${skip} echo "skip-workflow=${skip}" >> $GITHUB_OUTPUT - # Checkout to this build's commit - - name: Checkout to commit's code - if: steps.skip-workflow.outputs.skip-workflow == 'false' - uses: actions/checkout@v3 - - name: Check for changes in relevant files if: steps.skip-workflow.outputs.skip-workflow == 'false' uses: frouioui/paths-filter@main @@ -138,14 +124,14 @@ jobs: sudo apt-get install -y gnupg2 sudo dpkg -i percona-release_latest.$(lsb_release -sc)_all.deb sudo apt-get update - sudo apt-get install percona-xtrabackup-24 + sudo apt-get install -y percona-xtrabackup-24 # Checkout to the next release of Vitess - - name: Checkout to the other version's code (${{ needs.get_next_release.outputs.next_release }}) + - name: Checkout to the other version's code (${{ steps.output-next-release-ref.outputs.next_release_ref }}) if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' uses: actions/checkout@v3 with: - ref: ${{ needs.get_next_release.outputs.next_release }} + ref: ${{ steps.output-next-release-ref.outputs.next_release_ref }} - name: Get dependencies for the next release if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' @@ -157,7 +143,7 @@ jobs: timeout-minutes: 5 run: | source build.env - make build + NOVTADMINBUILD=1 make build mkdir -p /tmp/vitess-build-other/ cp -R bin /tmp/vitess-build-other/ rm -Rf bin/* @@ -182,7 +168,7 @@ jobs: timeout-minutes: 5 run: | source build.env - make build + NOVTADMINBUILD=1 make build mkdir -p /tmp/vitess-build-current/ cp -R bin /tmp/vitess-build-current/ diff --git a/.github/workflows/upgrade_downgrade_test_query_serving_queries.yml b/.github/workflows/upgrade_downgrade_test_query_serving_queries.yml index 8723307e316..fcfe36623d5 100644 --- a/.github/workflows/upgrade_downgrade_test_query_serving_queries.yml +++ b/.github/workflows/upgrade_downgrade_test_query_serving_queries.yml @@ -1,4 +1,4 @@ -name: Upgrade Downgrade Testing Query Serving (Queries) +name: Query Serving (Queries) - Upgrade Downgrade Testing on: push: pull_request: @@ -13,32 +13,10 @@ permissions: read-all # (vtgate, vttablet, etc) built on different versions. jobs: - get_previous_release: - if: always() - name: Get Previous Release - Query Serving (Queries) - runs-on: ubuntu-22.04 - outputs: - previous_release: ${{ steps.output-previous-release-ref.outputs.previous_release_ref }} - - steps: - - name: Check out to HEAD - uses: actions/checkout@v3 - with: - fetch-depth: 0 - - - name: Set output with latest release branch - id: output-previous-release-ref - run: | - previous_release_ref=$(./tools/get_previous_release.sh ${{github.base_ref}} ${{github.ref}}) - echo $previous_release_ref - echo "previous_release_ref=${previous_release_ref}" >> $GITHUB_OUTPUT upgrade_downgrade_test: - if: always() && (needs.get_previous_release.result == 'success') name: Run Upgrade Downgrade Test - Query Serving (Queries) runs-on: ubuntu-22.04 - needs: - - get_previous_release steps: - name: Skip CI @@ -61,6 +39,16 @@ jobs: - name: Check out commit's code if: steps.skip-workflow.outputs.skip-workflow == 'false' uses: actions/checkout@v3 + with: + fetch-depth: 0 + + - name: Set output with latest release branch + id: output-previous-release-ref + if: steps.skip-workflow.outputs.skip-workflow == 'false' + run: | + previous_release_ref=$(./tools/get_previous_release.sh ${{github.base_ref}} ${{github.ref}}) + echo $previous_release_ref + echo "previous_release_ref=${previous_release_ref}" >> $GITHUB_OUTPUT - name: Check for changes in relevant files if: steps.skip-workflow.outputs.skip-workflow == 'false' @@ -87,7 +75,7 @@ jobs: if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' uses: actions/setup-go@v4 with: - go-version: 1.20.12 + go-version: 1.21.5 - name: Set up python if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' @@ -132,14 +120,14 @@ jobs: sudo apt-get install -y gnupg2 sudo dpkg -i percona-release_latest.$(lsb_release -sc)_all.deb sudo apt-get update - sudo apt-get install percona-xtrabackup-24 + sudo apt-get install -y percona-xtrabackup-24 # Checkout to the last release of Vitess - - name: Check out other version's code (${{ needs.get_previous_release.outputs.previous_release }}) + - name: Check out other version's code (${{ steps.output-previous-release-ref.outputs.previous_release_ref }}) if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' uses: actions/checkout@v3 with: - ref: ${{ needs.get_previous_release.outputs.previous_release }} + ref: ${{ steps.output-previous-release-ref.outputs.previous_release_ref }} - name: Get dependencies for the last release if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' @@ -151,7 +139,7 @@ jobs: timeout-minutes: 10 run: | source build.env - make build + NOVTADMINBUILD=1 make build mkdir -p /tmp/vitess-build-other/ cp -R bin /tmp/vitess-build-other/ rm -Rf bin/* @@ -171,7 +159,7 @@ jobs: timeout-minutes: 10 run: | source build.env - make build + NOVTADMINBUILD=1 make build mkdir -p /tmp/vitess-build-current/ cp -R bin /tmp/vitess-build-current/ diff --git a/.github/workflows/upgrade_downgrade_test_query_serving_queries_next_release.yml b/.github/workflows/upgrade_downgrade_test_query_serving_queries_next_release.yml index 7ac232a06d7..a37fe112829 100644 --- a/.github/workflows/upgrade_downgrade_test_query_serving_queries_next_release.yml +++ b/.github/workflows/upgrade_downgrade_test_query_serving_queries_next_release.yml @@ -1,4 +1,4 @@ -name: Upgrade Downgrade Testing Query Serving (Queries) Next Release +name: Query Serving (Queries) Next Release - Upgrade Downgrade Testing on: push: pull_request: @@ -13,32 +13,10 @@ permissions: read-all # (vtgate, vttablet, etc) built on different versions. jobs: - get_next_release: - if: always() - name: Get Latest Release - Query Serving (Queries) Next Release - runs-on: ubuntu-22.04 - outputs: - next_release: ${{ steps.output-next-release-ref.outputs.next_release_ref }} - - steps: - - name: Check out to HEAD - uses: actions/checkout@v3 - with: - fetch-depth: 0 - - - name: Set output with latest release branch - id: output-next-release-ref - run: | - next_release_ref=$(./tools/get_next_release.sh ${{github.base_ref}} ${{github.ref}}) - echo $next_release_ref - echo "next_release_ref=${next_release_ref}" >> $GITHUB_OUTPUT upgrade_downgrade_test: - if: always() && (needs.get_next_release.result == 'success') name: Run Upgrade Downgrade Test - Query Serving (Queries) Next Release runs-on: ubuntu-22.04 - needs: - - get_next_release steps: - name: Skip CI @@ -48,6 +26,18 @@ jobs: exit 1 fi + - name: Check out commit's code + uses: actions/checkout@v3 + with: + fetch-depth: 0 + + - name: Set output with latest release branch + id: output-next-release-ref + run: | + next_release_ref=$(./tools/get_next_release.sh ${{github.base_ref}} ${{github.ref}}) + echo $next_release_ref + echo "next_release_ref=${next_release_ref}" >> $GITHUB_OUTPUT + - name: Check if workflow needs to be skipped id: skip-workflow run: | @@ -55,16 +45,12 @@ jobs: if [[ "${{github.event.pull_request}}" == "" ]] && [[ "${{github.ref}}" != "refs/heads/main" ]] && [[ ! "${{github.ref}}" =~ ^refs/heads/release-[0-9]+\.[0-9]$ ]] && [[ ! "${{github.ref}}" =~ "refs/tags/.*" ]]; then skip='true' fi - if [[ "${{needs.get_next_release.outputs.next_release}}" == "" ]]; then + if [[ "${{steps.output-next-release-ref.outputs.next_release_ref}}" == "" ]]; then skip='true' fi echo Skip ${skip} echo "skip-workflow=${skip}" >> $GITHUB_OUTPUT - - name: Check out commit's code - if: steps.skip-workflow.outputs.skip-workflow == 'false' - uses: actions/checkout@v3 - - name: Check for changes in relevant files if: steps.skip-workflow.outputs.skip-workflow == 'false' uses: frouioui/paths-filter@main @@ -135,14 +121,14 @@ jobs: sudo apt-get install -y gnupg2 sudo dpkg -i percona-release_latest.$(lsb_release -sc)_all.deb sudo apt-get update - sudo apt-get install percona-xtrabackup-24 + sudo apt-get install -y percona-xtrabackup-24 # Checkout to the next release of Vitess - - name: Check out other version's code (${{ needs.get_next_release.outputs.next_release }}) + - name: Check out other version's code (${{ steps.output-next-release-ref.outputs.next_release_ref }}) if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' uses: actions/checkout@v3 with: - ref: ${{ needs.get_next_release.outputs.next_release }} + ref: ${{ steps.output-next-release-ref.outputs.next_release_ref }} - name: Get dependencies for the next release if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' @@ -154,7 +140,7 @@ jobs: timeout-minutes: 10 run: | source build.env - make build + NOVTADMINBUILD=1 make build mkdir -p /tmp/vitess-build-other/ cp -R bin /tmp/vitess-build-other/ rm -Rf bin/* @@ -174,7 +160,7 @@ jobs: timeout-minutes: 10 run: | source build.env - make build + NOVTADMINBUILD=1 make build mkdir -p /tmp/vitess-build-current/ cp -R bin /tmp/vitess-build-current/ diff --git a/.github/workflows/upgrade_downgrade_test_query_serving_schema.yml b/.github/workflows/upgrade_downgrade_test_query_serving_schema.yml index c19fd392e31..1e2b5ad6a74 100644 --- a/.github/workflows/upgrade_downgrade_test_query_serving_schema.yml +++ b/.github/workflows/upgrade_downgrade_test_query_serving_schema.yml @@ -1,4 +1,4 @@ -name: Upgrade Downgrade Testing Query Serving (Schema) +name: Query Serving (Schema) - Upgrade Downgrade Testing on: push: pull_request: @@ -13,32 +13,10 @@ permissions: read-all # (vtgate, vttablet, etc) built on different versions. jobs: - get_previous_release: - if: always() - name: Get Previous Release - Query Serving (Schema) - runs-on: ubuntu-22.04 - outputs: - previous_release: ${{ steps.output-previous-release-ref.outputs.previous_release_ref }} - - steps: - - name: Check out to HEAD - uses: actions/checkout@v3 - with: - fetch-depth: 0 - - - name: Set output with latest release branch - id: output-previous-release-ref - run: | - previous_release_ref=$(./tools/get_previous_release.sh ${{github.base_ref}} ${{github.ref}}) - echo $previous_release_ref - echo "previous_release_ref=${previous_release_ref}" >> $GITHUB_OUTPUT upgrade_downgrade_test: - if: always() && (needs.get_previous_release.result == 'success') name: Run Upgrade Downgrade Test - Query Serving (Schema) runs-on: ubuntu-22.04 - needs: - - get_previous_release steps: - name: Skip CI @@ -61,6 +39,16 @@ jobs: - name: Check out commit's code if: steps.skip-workflow.outputs.skip-workflow == 'false' uses: actions/checkout@v3 + with: + fetch-depth: 0 + + - name: Set output with latest release branch + id: output-previous-release-ref + if: steps.skip-workflow.outputs.skip-workflow == 'false' + run: | + previous_release_ref=$(./tools/get_previous_release.sh ${{github.base_ref}} ${{github.ref}}) + echo $previous_release_ref + echo "previous_release_ref=${previous_release_ref}" >> $GITHUB_OUTPUT - name: Check for changes in relevant files if: steps.skip-workflow.outputs.skip-workflow == 'false' @@ -87,7 +75,7 @@ jobs: if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' uses: actions/setup-go@v4 with: - go-version: 1.20.12 + go-version: 1.21.5 - name: Set up python if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' @@ -132,14 +120,14 @@ jobs: sudo apt-get install -y gnupg2 sudo dpkg -i percona-release_latest.$(lsb_release -sc)_all.deb sudo apt-get update - sudo apt-get install percona-xtrabackup-24 + sudo apt-get install -y percona-xtrabackup-24 # Checkout to the last release of Vitess - - name: Check out other version's code (${{ needs.get_previous_release.outputs.previous_release }}) + - name: Check out other version's code (${{ steps.output-previous-release-ref.outputs.previous_release_ref }}) if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' uses: actions/checkout@v3 with: - ref: ${{ needs.get_previous_release.outputs.previous_release }} + ref: ${{ steps.output-previous-release-ref.outputs.previous_release_ref }} - name: Get dependencies for the last release if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' @@ -151,7 +139,7 @@ jobs: timeout-minutes: 10 run: | source build.env - make build + NOVTADMINBUILD=1 make build mkdir -p /tmp/vitess-build-other/ cp -R bin /tmp/vitess-build-other/ rm -Rf bin/* @@ -171,7 +159,7 @@ jobs: timeout-minutes: 10 run: | source build.env - make build + NOVTADMINBUILD=1 make build mkdir -p /tmp/vitess-build-current/ cp -R bin /tmp/vitess-build-current/ diff --git a/.github/workflows/upgrade_downgrade_test_query_serving_schema_next_release.yml b/.github/workflows/upgrade_downgrade_test_query_serving_schema_next_release.yml index 1c6beff78dc..b45b6ce4456 100644 --- a/.github/workflows/upgrade_downgrade_test_query_serving_schema_next_release.yml +++ b/.github/workflows/upgrade_downgrade_test_query_serving_schema_next_release.yml @@ -1,4 +1,4 @@ -name: Upgrade Downgrade Testing Query Serving (Schema) Next Release +name: Query Serving (Schema) Next Release - Upgrade Downgrade Testing on: push: pull_request: @@ -13,32 +13,10 @@ permissions: read-all # (vtgate, vttablet, etc) built on different versions. jobs: - get_next_release: - if: always() - name: Get Latest Release - Query Serving (Schema) Next Release - runs-on: ubuntu-22.04 - outputs: - next_release: ${{ steps.output-next-release-ref.outputs.next_release_ref }} - - steps: - - name: Check out to HEAD - uses: actions/checkout@v3 - with: - fetch-depth: 0 - - - name: Set output with latest release branch - id: output-next-release-ref - run: | - next_release_ref=$(./tools/get_next_release.sh ${{github.base_ref}} ${{github.ref}}) - echo $next_release_ref - echo "next_release_ref=${next_release_ref}" >> $GITHUB_OUTPUT upgrade_downgrade_test: - if: always() && (needs.get_next_release.result == 'success') name: Run Upgrade Downgrade Test - Query Serving (Schema) Next Release runs-on: ubuntu-22.04 - needs: - - get_next_release steps: - name: Skip CI @@ -48,6 +26,18 @@ jobs: exit 1 fi + - name: Check out commit's code + uses: actions/checkout@v3 + with: + fetch-depth: 0 + + - name: Set output with latest release branch + id: output-next-release-ref + run: | + next_release_ref=$(./tools/get_next_release.sh ${{github.base_ref}} ${{github.ref}}) + echo $next_release_ref + echo "next_release_ref=${next_release_ref}" >> $GITHUB_OUTPUT + - name: Check if workflow needs to be skipped id: skip-workflow run: | @@ -55,16 +45,12 @@ jobs: if [[ "${{github.event.pull_request}}" == "" ]] && [[ "${{github.ref}}" != "refs/heads/main" ]] && [[ ! "${{github.ref}}" =~ ^refs/heads/release-[0-9]+\.[0-9]$ ]] && [[ ! "${{github.ref}}" =~ "refs/tags/.*" ]]; then skip='true' fi - if [[ "${{needs.get_next_release.outputs.next_release}}" == "" ]]; then + if [[ "${{steps.output-next-release-ref.outputs.next_release_ref}}" == "" ]]; then skip='true' fi echo Skip ${skip} echo "skip-workflow=${skip}" >> $GITHUB_OUTPUT - - name: Check out commit's code - if: steps.skip-workflow.outputs.skip-workflow == 'false' - uses: actions/checkout@v3 - - name: Check for changes in relevant files if: steps.skip-workflow.outputs.skip-workflow == 'false' uses: frouioui/paths-filter@main @@ -135,14 +121,14 @@ jobs: sudo apt-get install -y gnupg2 sudo dpkg -i percona-release_latest.$(lsb_release -sc)_all.deb sudo apt-get update - sudo apt-get install percona-xtrabackup-24 + sudo apt-get install -y percona-xtrabackup-24 # Checkout to the next release of Vitess - - name: Check out other version's code (${{ needs.get_next_release.outputs.next_release }}) + - name: Check out other version's code (${{ steps.output-next-release-ref.outputs.next_release_ref }}) if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' uses: actions/checkout@v3 with: - ref: ${{ needs.get_next_release.outputs.next_release }} + ref: ${{ steps.output-next-release-ref.outputs.next_release_ref }} - name: Get dependencies for the next release if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' @@ -154,7 +140,7 @@ jobs: timeout-minutes: 10 run: | source build.env - make build + NOVTADMINBUILD=1 make build mkdir -p /tmp/vitess-build-other/ cp -R bin /tmp/vitess-build-other/ rm -Rf bin/* @@ -174,7 +160,7 @@ jobs: timeout-minutes: 10 run: | source build.env - make build + NOVTADMINBUILD=1 make build mkdir -p /tmp/vitess-build-current/ cp -R bin /tmp/vitess-build-current/ diff --git a/.github/workflows/upgrade_downgrade_test_reparent_new_vtctl.yml b/.github/workflows/upgrade_downgrade_test_reparent_new_vtctl.yml index 30b358be83d..7a86a605560 100644 --- a/.github/workflows/upgrade_downgrade_test_reparent_new_vtctl.yml +++ b/.github/workflows/upgrade_downgrade_test_reparent_new_vtctl.yml @@ -1,4 +1,4 @@ -name: Upgrade Downgrade Testing Reparent New Vtctl +name: Reparent New Vtctl - Upgrade Downgrade Testing on: push: pull_request: @@ -13,32 +13,10 @@ permissions: read-all # (vtctl, vttablet, etc) built on different versions. jobs: - get_next_release: - if: always() - name: Get Latest Release - Reparent New Vtctl - runs-on: ubuntu-22.04 - outputs: - next_release: ${{ steps.output-next-release-ref.outputs.next_release_ref }} - - steps: - - name: Check out to HEAD - uses: actions/checkout@v3 - with: - fetch-depth: 0 - - - name: Set output with latest release branch - id: output-next-release-ref - run: | - next_release_ref=$(./tools/get_next_release.sh ${{github.base_ref}} ${{github.ref}}) - echo $next_release_ref - echo "next_release_ref=${next_release_ref}" >> $GITHUB_OUTPUT upgrade_downgrade_test: - if: always() && (needs.get_next_release.result == 'success') name: Run Upgrade Downgrade Test - Reparent New Vtctl runs-on: ubuntu-22.04 - needs: - - get_next_release steps: - name: Skip CI @@ -48,6 +26,18 @@ jobs: exit 1 fi + - name: Check out commit's code + uses: actions/checkout@v3 + with: + fetch-depth: 0 + + - name: Set output with latest release branch + id: output-next-release-ref + run: | + next_release_ref=$(./tools/get_next_release.sh ${{github.base_ref}} ${{github.ref}}) + echo $next_release_ref + echo "next_release_ref=${next_release_ref}" >> $GITHUB_OUTPUT + - name: Check if workflow needs to be skipped id: skip-workflow run: | @@ -55,16 +45,12 @@ jobs: if [[ "${{github.event.pull_request}}" == "" ]] && [[ "${{github.ref}}" != "refs/heads/main" ]] && [[ ! "${{github.ref}}" =~ ^refs/heads/release-[0-9]+\.[0-9]$ ]] && [[ ! "${{github.ref}}" =~ "refs/tags/.*" ]]; then skip='true' fi - if [[ "${{needs.get_next_release.outputs.next_release}}" == "" ]]; then + if [[ "${{steps.output-next-release-ref.outputs.next_release_ref}}" == "" ]]; then skip='true' fi echo Skip ${skip} echo "skip-workflow=${skip}" >> $GITHUB_OUTPUT - - name: Check out commit's code - if: steps.skip-workflow.outputs.skip-workflow == 'false' - uses: actions/checkout@v3 - - name: Check for changes in relevant files if: steps.skip-workflow.outputs.skip-workflow == 'false' uses: frouioui/paths-filter@main @@ -135,14 +121,14 @@ jobs: sudo apt-get install -y gnupg2 sudo dpkg -i percona-release_latest.$(lsb_release -sc)_all.deb sudo apt-get update - sudo apt-get install percona-xtrabackup-24 + sudo apt-get install -y percona-xtrabackup-24 # Checkout to the next release of Vitess - - name: Check out other version's code (${{ needs.get_next_release.outputs.next_release }}) + - name: Check out other version's code (${{ steps.output-next-release-ref.outputs.next_release_ref }}) if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' uses: actions/checkout@v3 with: - ref: ${{ needs.get_next_release.outputs.next_release }} + ref: ${{ steps.output-next-release-ref.outputs.next_release_ref }} - name: Get dependencies for the next release if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' @@ -154,7 +140,7 @@ jobs: timeout-minutes: 10 run: | source build.env - make build + NOVTADMINBUILD=1 make build mkdir -p /tmp/vitess-build-other/ cp -R bin /tmp/vitess-build-other/ rm -Rf bin/* @@ -174,7 +160,7 @@ jobs: timeout-minutes: 10 run: | source build.env - make build + NOVTADMINBUILD=1 make build mkdir -p /tmp/vitess-build-current/ cp -R bin /tmp/vitess-build-current/ diff --git a/.github/workflows/upgrade_downgrade_test_reparent_new_vttablet.yml b/.github/workflows/upgrade_downgrade_test_reparent_new_vttablet.yml index 7122845fb22..13413770784 100644 --- a/.github/workflows/upgrade_downgrade_test_reparent_new_vttablet.yml +++ b/.github/workflows/upgrade_downgrade_test_reparent_new_vttablet.yml @@ -1,4 +1,4 @@ -name: Upgrade Downgrade Testing Reparent New VTTablet +name: Reparent New VTTablet - Upgrade Downgrade Testing on: push: pull_request: @@ -13,32 +13,10 @@ permissions: read-all # (vtctl, vttablet, etc) built on different versions. jobs: - get_next_release: - if: always() - name: Get Latest Release - Reparent New VTTablet - runs-on: ubuntu-22.04 - outputs: - next_release: ${{ steps.output-next-release-ref.outputs.next_release_ref }} - - steps: - - name: Check out to HEAD - uses: actions/checkout@v3 - with: - fetch-depth: 0 - - - name: Set output with latest release branch - id: output-next-release-ref - run: | - next_release_ref=$(./tools/get_next_release.sh ${{github.base_ref}} ${{github.ref}}) - echo $next_release_ref - echo "next_release_ref=${next_release_ref}" >> $GITHUB_OUTPUT upgrade_downgrade_test: - if: always() && (needs.get_next_release.result == 'success') name: Run Upgrade Downgrade Test - Reparent New VTTablet runs-on: ubuntu-22.04 - needs: - - get_next_release steps: - name: Skip CI @@ -48,6 +26,18 @@ jobs: exit 1 fi + - name: Check out commit's code + uses: actions/checkout@v3 + with: + fetch-depth: 0 + + - name: Set output with latest release branch + id: output-next-release-ref + run: | + next_release_ref=$(./tools/get_next_release.sh ${{github.base_ref}} ${{github.ref}}) + echo $next_release_ref + echo "next_release_ref=${next_release_ref}" >> $GITHUB_OUTPUT + - name: Check if workflow needs to be skipped id: skip-workflow run: | @@ -55,16 +45,12 @@ jobs: if [[ "${{github.event.pull_request}}" == "" ]] && [[ "${{github.ref}}" != "refs/heads/main" ]] && [[ ! "${{github.ref}}" =~ ^refs/heads/release-[0-9]+\.[0-9]$ ]] && [[ ! "${{github.ref}}" =~ "refs/tags/.*" ]]; then skip='true' fi - if [[ "${{needs.get_next_release.outputs.next_release}}" == "" ]]; then + if [[ "${{steps.output-next-release-ref.outputs.next_release_ref}}" == "" ]]; then skip='true' fi echo Skip ${skip} echo "skip-workflow=${skip}" >> $GITHUB_OUTPUT - - name: Check out commit's code - if: steps.skip-workflow.outputs.skip-workflow == 'false' - uses: actions/checkout@v3 - - name: Check for changes in relevant files if: steps.skip-workflow.outputs.skip-workflow == 'false' uses: frouioui/paths-filter@main @@ -135,14 +121,14 @@ jobs: sudo apt-get install -y gnupg2 sudo dpkg -i percona-release_latest.$(lsb_release -sc)_all.deb sudo apt-get update - sudo apt-get install percona-xtrabackup-24 + sudo apt-get install -y percona-xtrabackup-24 # Checkout to the next release of Vitess - - name: Check out other version's code (${{ needs.get_next_release.outputs.next_release }}) + - name: Check out other version's code (${{ steps.output-next-release-ref.outputs.next_release_ref }}) if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' uses: actions/checkout@v3 with: - ref: ${{ needs.get_next_release.outputs.next_release }} + ref: ${{ steps.output-next-release-ref.outputs.next_release_ref }} - name: Get dependencies for the next release if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' @@ -154,7 +140,7 @@ jobs: timeout-minutes: 10 run: | source build.env - make build + NOVTADMINBUILD=1 make build mkdir -p /tmp/vitess-build-other/ cp -R bin /tmp/vitess-build-other/ rm -Rf bin/* @@ -174,7 +160,7 @@ jobs: timeout-minutes: 10 run: | source build.env - make build + NOVTADMINBUILD=1 make build mkdir -p /tmp/vitess-build-current/ cp -R bin /tmp/vitess-build-current/ diff --git a/.github/workflows/upgrade_downgrade_test_reparent_old_vtctl.yml b/.github/workflows/upgrade_downgrade_test_reparent_old_vtctl.yml index 7d3dd30a728..32dd015ecab 100644 --- a/.github/workflows/upgrade_downgrade_test_reparent_old_vtctl.yml +++ b/.github/workflows/upgrade_downgrade_test_reparent_old_vtctl.yml @@ -1,4 +1,4 @@ -name: Upgrade Downgrade Testing Reparent Old Vtctl +name: Reparent Old Vtctl - Upgrade Downgrade Testing on: push: pull_request: @@ -13,32 +13,10 @@ permissions: read-all # (vtctl, vttablet, etc) built on different versions. jobs: - get_previous_release: - if: always() - name: Get Previous Release - Reparent Old Vtctl - runs-on: ubuntu-22.04 - outputs: - previous_release: ${{ steps.output-previous-release-ref.outputs.previous_release_ref }} - - steps: - - name: Check out to HEAD - uses: actions/checkout@v3 - with: - fetch-depth: 0 - - - name: Set output with latest release branch - id: output-previous-release-ref - run: | - previous_release_ref=$(./tools/get_previous_release.sh ${{github.base_ref}} ${{github.ref}}) - echo $previous_release_ref - echo "previous_release_ref=${previous_release_ref}" >> $GITHUB_OUTPUT upgrade_downgrade_test: - if: always() && (needs.get_previous_release.result == 'success') name: Run Upgrade Downgrade Test - Reparent Old Vtctl runs-on: ubuntu-22.04 - needs: - - get_previous_release steps: - name: Skip CI @@ -61,6 +39,16 @@ jobs: - name: Check out commit's code if: steps.skip-workflow.outputs.skip-workflow == 'false' uses: actions/checkout@v3 + with: + fetch-depth: 0 + + - name: Set output with latest release branch + id: output-previous-release-ref + if: steps.skip-workflow.outputs.skip-workflow == 'false' + run: | + previous_release_ref=$(./tools/get_previous_release.sh ${{github.base_ref}} ${{github.ref}}) + echo $previous_release_ref + echo "previous_release_ref=${previous_release_ref}" >> $GITHUB_OUTPUT - name: Check for changes in relevant files if: steps.skip-workflow.outputs.skip-workflow == 'false' @@ -87,7 +75,7 @@ jobs: if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' uses: actions/setup-go@v4 with: - go-version: 1.20.12 + go-version: 1.21.5 - name: Set up python if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' @@ -132,14 +120,14 @@ jobs: sudo apt-get install -y gnupg2 sudo dpkg -i percona-release_latest.$(lsb_release -sc)_all.deb sudo apt-get update - sudo apt-get install percona-xtrabackup-24 + sudo apt-get install -y percona-xtrabackup-24 # Checkout to the last release of Vitess - - name: Check out other version's code (${{ needs.get_previous_release.outputs.previous_release }}) + - name: Check out other version's code (${{ steps.output-previous-release-ref.outputs.previous_release_ref }}) if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' uses: actions/checkout@v3 with: - ref: ${{ needs.get_previous_release.outputs.previous_release }} + ref: ${{ steps.output-previous-release-ref.outputs.previous_release_ref }} - name: Get dependencies for the last release if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' @@ -151,7 +139,7 @@ jobs: timeout-minutes: 10 run: | source build.env - make build + NOVTADMINBUILD=1 make build mkdir -p /tmp/vitess-build-other/ cp -R bin /tmp/vitess-build-other/ rm -Rf bin/* @@ -171,7 +159,7 @@ jobs: timeout-minutes: 10 run: | source build.env - make build + NOVTADMINBUILD=1 make build mkdir -p /tmp/vitess-build-current/ cp -R bin /tmp/vitess-build-current/ diff --git a/.github/workflows/upgrade_downgrade_test_reparent_old_vttablet.yml b/.github/workflows/upgrade_downgrade_test_reparent_old_vttablet.yml index afb70620d4b..4b54876bf45 100644 --- a/.github/workflows/upgrade_downgrade_test_reparent_old_vttablet.yml +++ b/.github/workflows/upgrade_downgrade_test_reparent_old_vttablet.yml @@ -1,4 +1,4 @@ -name: Upgrade Downgrade Testing Reparent Old VTTablet +name: Reparent Old VTTablet - Upgrade Downgrade Testing on: push: pull_request: @@ -13,32 +13,10 @@ permissions: read-all # (vtctl, vttablet, etc) built on different versions. jobs: - get_previous_release: - if: always() - name: Get Previous Release - Reparent Old VTTablet - runs-on: ubuntu-22.04 - outputs: - previous_release: ${{ steps.output-previous-release-ref.outputs.previous_release_ref }} - - steps: - - name: Check out to HEAD - uses: actions/checkout@v3 - with: - fetch-depth: 0 - - - name: Set output with latest release branch - id: output-previous-release-ref - run: | - previous_release_ref=$(./tools/get_previous_release.sh ${{github.base_ref}} ${{github.ref}}) - echo $previous_release_ref - echo "previous_release_ref=${previous_release_ref}" >> $GITHUB_OUTPUT upgrade_downgrade_test: - if: always() && (needs.get_previous_release.result == 'success') name: Run Upgrade Downgrade Test - Reparent Old VTTablet runs-on: ubuntu-22.04 - needs: - - get_previous_release steps: - name: Skip CI @@ -61,6 +39,16 @@ jobs: - name: Check out commit's code if: steps.skip-workflow.outputs.skip-workflow == 'false' uses: actions/checkout@v3 + with: + fetch-depth: 0 + + - name: Set output with latest release branch + id: output-previous-release-ref + if: steps.skip-workflow.outputs.skip-workflow == 'false' + run: | + previous_release_ref=$(./tools/get_previous_release.sh ${{github.base_ref}} ${{github.ref}}) + echo $previous_release_ref + echo "previous_release_ref=${previous_release_ref}" >> $GITHUB_OUTPUT - name: Check for changes in relevant files if: steps.skip-workflow.outputs.skip-workflow == 'false' @@ -87,7 +75,7 @@ jobs: if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' uses: actions/setup-go@v4 with: - go-version: 1.20.12 + go-version: 1.21.5 - name: Set up python if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' @@ -132,14 +120,14 @@ jobs: sudo apt-get install -y gnupg2 sudo dpkg -i percona-release_latest.$(lsb_release -sc)_all.deb sudo apt-get update - sudo apt-get install percona-xtrabackup-24 + sudo apt-get install -y percona-xtrabackup-24 # Checkout to the last release of Vitess - - name: Check out other version's code (${{ needs.get_previous_release.outputs.previous_release }}) + - name: Check out other version's code (${{ steps.output-previous-release-ref.outputs.previous_release_ref }}) if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' uses: actions/checkout@v3 with: - ref: ${{ needs.get_previous_release.outputs.previous_release }} + ref: ${{ steps.output-previous-release-ref.outputs.previous_release_ref }} - name: Get dependencies for the last release if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' @@ -151,7 +139,7 @@ jobs: timeout-minutes: 10 run: | source build.env - make build + NOVTADMINBUILD=1 make build mkdir -p /tmp/vitess-build-other/ cp -R bin /tmp/vitess-build-other/ rm -Rf bin/* @@ -171,7 +159,7 @@ jobs: timeout-minutes: 10 run: | source build.env - make build + NOVTADMINBUILD=1 make build mkdir -p /tmp/vitess-build-current/ cp -R bin /tmp/vitess-build-current/ diff --git a/test/config.json b/test/config.json index dd37ecc3ce8..a71e9486336 100644 --- a/test/config.json +++ b/test/config.json @@ -109,7 +109,7 @@ "RetryMax": 1, "Tags": [] }, - "backup_mysqlctld": { + "backup_mysqlctld": { "File": "unused.go", "Args": ["vitess.io/vitess/go/test/endtoend/backup/mysqlctld", "-timeout", "30m"], "Command": [], @@ -520,7 +520,7 @@ "Manual": false, "Shard": "vtgate_queries", "RetryMax": 1, - "Tags": [] + "Tags": ["upgrade_downgrade_query_serving_queries"] }, "vtgate_queries_aggregation": { "File": "unused.go", @@ -529,7 +529,7 @@ "Manual": false, "Shard": "vtgate_queries", "RetryMax": 2, - "Tags": [] + "Tags": ["upgrade_downgrade_query_serving_queries"] }, "vtgate_queries_foundrows": { "File": "unused.go", @@ -538,7 +538,7 @@ "Manual": false, "Shard": "vtgate_queries", "RetryMax": 2, - "Tags": [] + "Tags": ["upgrade_downgrade_query_serving_queries"] }, "vtgate_queries_informationschema": { "File": "unused.go", @@ -547,7 +547,7 @@ "Manual": false, "Shard": "vtgate_queries", "RetryMax": 2, - "Tags": [] + "Tags": ["upgrade_downgrade_query_serving_queries"] }, "vtgate_queries_misc": { "File": "unused.go", @@ -565,7 +565,7 @@ "Manual": false, "Shard": "vtgate_queries", "RetryMax": 1, - "Tags": [] + "Tags": ["upgrade_downgrade_query_serving_queries"] }, "vtgate_queries_normalize": { "File": "unused.go", @@ -574,7 +574,7 @@ "Manual": false, "Shard": "vtgate_queries", "RetryMax": 2, - "Tags": [] + "Tags": ["upgrade_downgrade_query_serving_queries"] }, "vtgate_queries_orderby": { "File": "unused.go", @@ -610,7 +610,7 @@ "Manual": false, "Shard": "vtgate_queries", "RetryMax": 2, - "Tags": [] + "Tags": ["upgrade_downgrade_query_serving_queries"] }, "vtgate_queries_vexplain": { "File": "unused.go", @@ -619,7 +619,7 @@ "Manual": false, "Shard": "vtgate_queries", "RetryMax": 2, - "Tags": [] + "Tags": ["upgrade_downgrade_query_serving_queries"] }, "vtgate_queries_reference": { "File": "unused.go", @@ -628,7 +628,7 @@ "Manual": false, "Shard": "vtgate_queries", "RetryMax": 1, - "Tags": [] + "Tags": ["upgrade_downgrade_query_serving_queries"] }, "vtgate_concurrentdml": { "File": "unused.go", From 5b4e543dd50b7d089a759ef01411e0234e9e4fd6 Mon Sep 17 00:00:00 2001 From: Manan Gupta Date: Thu, 21 Dec 2023 18:26:26 +0530 Subject: [PATCH 2/5] empty commit to kick CI Signed-off-by: Manan Gupta From 7d62aef09068d031623db9139e7da402373640b1 Mon Sep 17 00:00:00 2001 From: Manan Gupta Date: Mon, 25 Dec 2023 19:41:08 +0530 Subject: [PATCH 3/5] test: fix upgrade downgrade tests Signed-off-by: Manan Gupta --- .../queries/aggregation/aggregation_test.go | 19 +++++++++++++++---- .../vtgate/queries/aggregation/fuzz_test.go | 4 +++- .../queries/normalize/normalize_test.go | 10 ++++++++++ 3 files changed, 28 insertions(+), 5 deletions(-) diff --git a/go/test/endtoend/vtgate/queries/aggregation/aggregation_test.go b/go/test/endtoend/vtgate/queries/aggregation/aggregation_test.go index 3323a541606..2f54eedfaa0 100644 --- a/go/test/endtoend/vtgate/queries/aggregation/aggregation_test.go +++ b/go/test/endtoend/vtgate/queries/aggregation/aggregation_test.go @@ -68,6 +68,7 @@ func TestAggregateTypes(t *testing.T) { } func TestGroupBy(t *testing.T) { + utils.SkipIfBinaryIsBelowVersion(t, 17, "vtgate") mcmp, closer := start(t) defer closer() mcmp.Exec("insert into t3(id5, id6, id7) values(1,1,2), (2,2,4), (3,2,4), (4,1,2), (5,1,2), (6,3,6)") @@ -314,8 +315,14 @@ func TestAggOnTopOfLimit(t *testing.T) { mcmp.AssertMatches(" select /*vt+ PLANNER=gen4 */ val1, count(*) from (select id, val1 from aggr_test where val2 < 4 order by val1 limit 2) as x group by val1", `[[NULL INT64(1)] [VARCHAR("a") INT64(1)]]`) mcmp.AssertMatchesNoOrder(" select /*vt+ PLANNER=gen4 */ val1, count(val2) from (select val1, val2 from aggr_test limit 8) as x group by val1", `[[NULL INT64(1)] [VARCHAR("a") INT64(2)] [VARCHAR("b") INT64(1)] [VARCHAR("c") INT64(2)]]`) - // mysql returns FLOAT64(0), vitess returns DECIMAL(0) - mcmp.AssertMatchesNoCompare(" select /*vt+ PLANNER=gen4 */ count(*), sum(val1) from (select id, val1 from aggr_test where val2 < 4 order by val1 desc limit 2) as x", "[[INT64(2) FLOAT64(0)]]", "[[INT64(2) DECIMAL(0)]]") + version, err := cluster.GetMajorVersion("vtgate") + require.NoError(t, err) + if version >= 18 { + mcmp.AssertMatchesNoCompare(" select /*vt+ PLANNER=gen4 */ count(*), sum(val1) from (select id, val1 from aggr_test where val2 < 4 order by val1 desc limit 2) as x", "[[INT64(2) FLOAT64(0)]]", "[[INT64(2) FLOAT64(0)]]") + } else { + // mysql returns FLOAT64(0), vitess returns DECIMAL(0) + mcmp.AssertMatchesNoCompare(" select /*vt+ PLANNER=gen4 */ count(*), sum(val1) from (select id, val1 from aggr_test where val2 < 4 order by val1 desc limit 2) as x", "[[INT64(2) FLOAT64(0)]]", "[[INT64(2) DECIMAL(0)]]") + } mcmp.AssertMatches(" select /*vt+ PLANNER=gen4 */ count(val1), sum(id) from (select id, val1 from aggr_test where val2 < 4 order by val1 desc limit 2) as x", "[[INT64(2) DECIMAL(7)]]") mcmp.AssertMatches(" select /*vt+ PLANNER=gen4 */ count(*), sum(id) from (select id, val1 from aggr_test where val2 is null limit 2) as x", "[[INT64(2) DECIMAL(14)]]") mcmp.AssertMatches(" select /*vt+ PLANNER=gen4 */ count(val1), sum(id) from (select id, val1 from aggr_test where val2 is null limit 2) as x", "[[INT64(1) DECIMAL(14)]]") @@ -386,11 +393,14 @@ func TestAggregateLeftJoin(t *testing.T) { mcmp.AssertMatchesNoOrder("SELECT t1.shardkey FROM t1 LEFT JOIN t2 ON t1.t1_id = t2.id", `[[INT64(1)] [INT64(0)]]`) mcmp.AssertMatches("SELECT count(t1.shardkey) FROM t1 LEFT JOIN t2 ON t1.t1_id = t2.id", `[[INT64(2)]]`) - mcmp.AssertMatches("SELECT count(t2.shardkey) FROM t1 LEFT JOIN t2 ON t1.t1_id = t2.id", `[[INT64(1)]]`) mcmp.AssertMatches("SELECT count(*) FROM t1 LEFT JOIN t2 ON t1.t1_id = t2.id", `[[INT64(2)]]`) mcmp.AssertMatches("SELECT sum(t1.shardkey) FROM t1 LEFT JOIN t2 ON t1.t1_id = t2.id", `[[DECIMAL(1)]]`) - mcmp.AssertMatches("SELECT sum(t2.shardkey) FROM t1 LEFT JOIN t2 ON t1.t1_id = t2.id", `[[DECIMAL(1)]]`) mcmp.AssertMatches("SELECT count(*) FROM t2 LEFT JOIN t1 ON t1.t1_id = t2.id WHERE IFNULL(t1.name, 'NOTSET') = 'r'", `[[INT64(1)]]`) + t.Run("Wrap aggregation", func(t *testing.T) { + utils.SkipIfBinaryIsBelowVersion(t, 17, "vtgate") + mcmp.AssertMatches("SELECT count(t2.shardkey) FROM t1 LEFT JOIN t2 ON t1.t1_id = t2.id", `[[INT64(1)]]`) + mcmp.AssertMatches("SELECT sum(t2.shardkey) FROM t1 LEFT JOIN t2 ON t1.t1_id = t2.id", `[[DECIMAL(1)]]`) + }) } // TestScalarAggregate tests validates that only count is returned and no additional field is returned.gst @@ -431,6 +441,7 @@ func TestAggregationRandomOnAnAggregatedValue(t *testing.T) { } func TestBuggyQueries(t *testing.T) { + utils.SkipIfBinaryIsBelowVersion(t, 17, "vtgate") // These queries have been found to be producing the wrong results by the query fuzzer // Adding them as end2end tests to make sure we never get them wrong again mcmp, closer := start(t) diff --git a/go/test/endtoend/vtgate/queries/aggregation/fuzz_test.go b/go/test/endtoend/vtgate/queries/aggregation/fuzz_test.go index 985116c488b..23d5c359571 100644 --- a/go/test/endtoend/vtgate/queries/aggregation/fuzz_test.go +++ b/go/test/endtoend/vtgate/queries/aggregation/fuzz_test.go @@ -25,6 +25,7 @@ import ( "golang.org/x/exp/maps" + "vitess.io/vitess/go/test/endtoend/utils" "vitess.io/vitess/go/vt/log" ) @@ -40,6 +41,7 @@ type ( ) func TestFuzzAggregations(t *testing.T) { + utils.SkipIfBinaryIsBelowVersion(t, 17, "vtgate") // This test randomizes values and queries, and checks that mysql returns the same values that Vitess does mcmp, closer := start(t) defer closer() @@ -51,7 +53,7 @@ func TestFuzzAggregations(t *testing.T) { } t1Insert := fmt.Sprintf("insert into t1 (t1_id, name, value, shardKey) values %s;", strings.Join(values, ",")) values = nil - noOfRows = rand.Intn(20) + noOfRows = rand.Intn(20) + 1 for i := 0; i < noOfRows; i++ { values = append(values, fmt.Sprintf("(%d, %d)", i, i)) } diff --git a/go/test/endtoend/vtgate/queries/normalize/normalize_test.go b/go/test/endtoend/vtgate/queries/normalize/normalize_test.go index 4fa4313e76c..3fc301e64f2 100644 --- a/go/test/endtoend/vtgate/queries/normalize/normalize_test.go +++ b/go/test/endtoend/vtgate/queries/normalize/normalize_test.go @@ -25,6 +25,7 @@ import ( "testing" "time" + "vitess.io/vitess/go/test/endtoend/cluster" "vitess.io/vitess/go/test/endtoend/utils" "github.com/stretchr/testify/assert" @@ -40,11 +41,20 @@ func TestNormalizeAllFields(t *testing.T) { insertQuery := `insert into t1 values (1, "chars", "variable chars", x'73757265', 0x676F, 0.33, 9.99, 1, "1976-06-08", "small", "b", "{\"key\":\"value\"}", point(1,5), b'011', 0b0101)` normalizedInsertQuery := `insert into t1 values (:vtg1 /* INT64 */, :vtg2 /* VARCHAR */, :vtg3 /* VARCHAR */, :vtg4 /* HEXVAL */, :vtg5 /* HEXNUM */, :vtg6 /* DECIMAL */, :vtg7 /* DECIMAL */, :vtg8 /* INT64 */, :vtg9 /* VARCHAR */, :vtg10 /* VARCHAR */, :vtg11 /* VARCHAR */, :vtg12 /* VARCHAR */, point(:vtg13 /* INT64 */, :vtg14 /* INT64 */), :vtg15 /* HEXNUM */, :vtg16 /* HEXNUM */)` + version, err := cluster.GetMajorVersion("vtgate") + require.NoError(t, err) + if version < 17 { + normalizedInsertQuery = "insert into t1 values (:vtg1, :vtg2, :vtg3, :vtg4, :vtg5, :vtg6, :vtg7, :vtg8, :vtg9, :vtg10, :vtg11, :vtg12, point(:vtg13, :vtg14), :vtg15, :vtg16)" + } selectQuery := "select * from t1" utils.Exec(t, conn, insertQuery) qr := utils.Exec(t, conn, selectQuery) assert.Equal(t, 1, len(qr.Rows), "wrong number of table rows, expected 1 but had %d. Results: %v", len(qr.Rows), qr.Rows) + // How we read the plan cache changed in v18, so we won't run this test for upgrade downgrade. + if version >= 18 { + return + } // Now need to figure out the best way to check the normalized query in the planner cache... results, err := getPlanCache(fmt.Sprintf("%s:%d", vtParams.Host, clusterInstance.VtgateProcess.Port)) require.Nil(t, err) From e74d79beb3242b75a45aac4d716830de172ac856 Mon Sep 17 00:00:00 2001 From: Manan Gupta Date: Tue, 26 Dec 2023 09:12:21 +0530 Subject: [PATCH 4/5] test: fix test Signed-off-by: Manan Gupta --- go/test/endtoend/vtgate/queries/aggregation/aggregation_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/go/test/endtoend/vtgate/queries/aggregation/aggregation_test.go b/go/test/endtoend/vtgate/queries/aggregation/aggregation_test.go index 2f54eedfaa0..6afd9d82214 100644 --- a/go/test/endtoend/vtgate/queries/aggregation/aggregation_test.go +++ b/go/test/endtoend/vtgate/queries/aggregation/aggregation_test.go @@ -395,9 +395,9 @@ func TestAggregateLeftJoin(t *testing.T) { mcmp.AssertMatches("SELECT count(t1.shardkey) FROM t1 LEFT JOIN t2 ON t1.t1_id = t2.id", `[[INT64(2)]]`) mcmp.AssertMatches("SELECT count(*) FROM t1 LEFT JOIN t2 ON t1.t1_id = t2.id", `[[INT64(2)]]`) mcmp.AssertMatches("SELECT sum(t1.shardkey) FROM t1 LEFT JOIN t2 ON t1.t1_id = t2.id", `[[DECIMAL(1)]]`) - mcmp.AssertMatches("SELECT count(*) FROM t2 LEFT JOIN t1 ON t1.t1_id = t2.id WHERE IFNULL(t1.name, 'NOTSET') = 'r'", `[[INT64(1)]]`) t.Run("Wrap aggregation", func(t *testing.T) { utils.SkipIfBinaryIsBelowVersion(t, 17, "vtgate") + mcmp.AssertMatches("SELECT count(*) FROM t2 LEFT JOIN t1 ON t1.t1_id = t2.id WHERE IFNULL(t1.name, 'NOTSET') = 'r'", `[[INT64(1)]]`) mcmp.AssertMatches("SELECT count(t2.shardkey) FROM t1 LEFT JOIN t2 ON t1.t1_id = t2.id", `[[INT64(1)]]`) mcmp.AssertMatches("SELECT sum(t2.shardkey) FROM t1 LEFT JOIN t2 ON t1.t1_id = t2.id", `[[DECIMAL(1)]]`) }) From a5a4fc28b42d57f7c06f9f926313376757f0c2d4 Mon Sep 17 00:00:00 2001 From: Manan Gupta Date: Tue, 26 Dec 2023 11:04:25 +0530 Subject: [PATCH 5/5] empty commit to kick CI Signed-off-by: Manan Gupta