Refactor Upgrade downgrade tests (#14782)

Signed-off-by: Manan Gupta <manan@planetscale.com>
vitessio · Dec 21, 2023 · 268913d · 268913d
1 parent ad1bd65
commit 268913d
Show file tree

Hide file tree

Showing 13 changed files with 299 additions and 410 deletions.
diff --git a/.github/workflows/upgrade_downgrade_test_backups_e2e.yml b/.github/workflows/upgrade_downgrade_test_backups_e2e.yml
@@ -1,4 +1,4 @@
-name: Upgrade Downgrade Testing - Backups - E2E
+name: Backups - E2E - Upgrade Downgrade Testing
 on:
   push:
   pull_request:
@@ -8,33 +8,10 @@ concurrency:
   cancel-in-progress: true
 
 jobs:
-  get_previous_release:
-    if: always()
-    name: Get Previous Release - Backups - E2E
-    runs-on: ubuntu-22.04
-    outputs:
-      previous_release: ${{ steps.output-previous-release-ref.outputs.previous_release_ref }}
-
-    steps:
-      - name: Check out to HEAD
-        uses: actions/checkout@v3
-        with:
-          fetch-depth: 0
-
-      - name: Set output with latest release branch
-        id: output-previous-release-ref
-        run: |
-          previous_release_ref=$(./tools/get_previous_release.sh ${{github.base_ref}} ${{github.ref}})
-          echo $previous_release_ref
-          echo "previous_release_ref=${previous_release_ref}" >> $GITHUB_OUTPUT
-
   upgrade_downgrade_test_e2e:
     timeout-minutes: 60
-    if: always() && needs.get_previous_release.result == 'success'
     name: Run Upgrade Downgrade Test - Backups - E2E
     runs-on: ubuntu-22.04
-    needs:
-      - get_previous_release
 
     steps:
     - name: Skip CI
@@ -57,6 +34,16 @@ jobs:
     - name: Check out commit's code
       if: steps.skip-workflow.outputs.skip-workflow == 'false'
       uses: actions/checkout@v3
+      with:
+        fetch-depth: 0
+
+    - name: Set output with latest release branch
+      if: steps.skip-workflow.outputs.skip-workflow == 'false'
+      id: output-previous-release-ref
+      run: |
+        previous_release_ref=$(./tools/get_previous_release.sh ${{github.base_ref}} ${{github.ref}})
+        echo $previous_release_ref
+        echo "previous_release_ref=${previous_release_ref}" >> $GITHUB_OUTPUT
 
     - name: Check for changes in relevant files
       if: steps.skip-workflow.outputs.skip-workflow == 'false'
@@ -115,11 +102,11 @@ jobs:
         sudo apt-get install percona-xtrabackup-24
 
     # Checkout to the last release of Vitess
-    - name: Check out other version's code (${{ needs.get_previous_release.outputs.previous_release }})
+    - name: Check out other version's code (${{ steps.output-previous-release-ref.outputs.previous_release_ref }})
       if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true'
       uses: actions/checkout@v3
       with:
-        ref: ${{ needs.get_previous_release.outputs.previous_release }}
+        ref: ${{ steps.output-previous-release-ref.outputs.previous_release_ref }}
 
     - name: Get dependencies for the last release
       if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true'

diff --git a/.github/workflows/upgrade_downgrade_test_backups_e2e_next_release.yml b/.github/workflows/upgrade_downgrade_test_backups_e2e_next_release.yml
@@ -1,4 +1,4 @@
-name: Upgrade Downgrade Testing - Backups - E2E - Next Release
+name: Backups - E2E - Next Release - Upgrade Downgrade Testing
 on:
   push:
   pull_request:
@@ -7,34 +7,14 @@ concurrency:
   group: format('{0}-{1}', ${{ github.ref }}, 'Upgrade Downgrade Testing - Backups - E2E - Next Release')
   cancel-in-progress: true
 
-jobs:
-  get_next_release:
-    if: always()
-    name: Get Latest Release - Backups - E2E - Next Release
-    runs-on: ubuntu-22.04
-    outputs:
-      next_release: ${{ steps.output-next-release-ref.outputs.next_release_ref }}
+permissions: read-all
 
-    steps:
-      - name: Check out to HEAD
-        uses: actions/checkout@v3
-        with:
-          fetch-depth: 0
-
-      - name: Set output with latest release branch
-        id: output-next-release-ref
-        run: |
-          next_release_ref=$(./tools/get_next_release.sh ${{github.base_ref}} ${{github.ref}})
-          echo $next_release_ref
-          echo "next_release_ref=${next_release_ref}" >> $GITHUB_OUTPUT
+jobs:
 
   upgrade_downgrade_test_e2e:
     timeout-minutes: 60
-    if: always() && needs.get_next_release.result == 'success'
     name: Run Upgrade Downgrade Test - Backups - E2E - Next Release
     runs-on: ubuntu-22.04
-    needs:
-      - get_next_release
 
     steps:
     - name: Skip CI
@@ -44,23 +24,31 @@ jobs:
           exit 1
         fi
 
+    - name: Check out commit's code
+      uses: actions/checkout@v3
+      with:
+        fetch-depth: 0
+
+    - name: Set output with latest release branch
+      id: output-next-release-ref
+      run: |
+        next_release_ref=$(./tools/get_next_release.sh ${{github.base_ref}} ${{github.ref}})
+        echo $next_release_ref
+        echo "next_release_ref=${next_release_ref}" >> $GITHUB_OUTPUT
+
     - name: Check if workflow needs to be skipped
       id: skip-workflow
       run: |
         skip='false'
         if [[ "${{github.event.pull_request}}" ==  "" ]] && [[ "${{github.ref}}" != "refs/heads/main" ]] && [[ ! "${{github.ref}}" =~ ^refs/heads/release-[0-9]+\.[0-9]$ ]] && [[ ! "${{github.ref}}" =~ "refs/tags/.*" ]]; then
           skip='true'
         fi
-        if [[ "${{needs.get_next_release.outputs.next_release}}" == "" ]]; then
+        if [[ "${{steps.output-next-release-ref.outputs.next_release_ref}}" == "" ]]; then
           skip='true'
         fi
         echo Skip ${skip}
         echo "skip-workflow=${skip}" >> $GITHUB_OUTPUT
 
-    - name: Check out commit's code
-      if: steps.skip-workflow.outputs.skip-workflow == 'false'
-      uses: actions/checkout@v3
-
     - name: Check for changes in relevant files
       if: steps.skip-workflow.outputs.skip-workflow == 'false'
       uses: frouioui/paths-filter@main
@@ -84,9 +72,9 @@ jobs:
 
     - name: Set up Go
       if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true'
-      uses: actions/setup-go@v3
+      uses: actions/setup-go@v4
       with:
-        go-version: 1.20.12
+        go-version: 1.21.5
 
     - name: Set up python
       if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true'
@@ -115,14 +103,14 @@ jobs:
         sudo apt-get install -y gnupg2
         sudo dpkg -i percona-release_latest.$(lsb_release -sc)_all.deb
         sudo apt-get update
-        sudo apt-get install percona-xtrabackup-24
+        sudo apt-get install -y percona-xtrabackup-24
 
     # Checkout to the next release of Vitess
-    - name: Check out other version's code (${{ needs.get_next_release.outputs.next_release }})
+    - name: Check out other version's code (${{ steps.output-next-release-ref.outputs.next_release_ref }})
       if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true'
       uses: actions/checkout@v3
       with:
-        ref: ${{ needs.get_next_release.outputs.next_release }}
+        ref: ${{ steps.output-next-release-ref.outputs.next_release_ref }}
 
     - name: Get dependencies for the next release
       if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true'
@@ -134,7 +122,7 @@ jobs:
       timeout-minutes: 10
       run: |
         source build.env
-        make build
+        NOVTADMINBUILD=1 make build
         mkdir -p /tmp/vitess-build-other/
         cp -R bin /tmp/vitess-build-other/
         rm -Rf bin/*
@@ -154,7 +142,7 @@ jobs:
       timeout-minutes: 10
       run: |
         source build.env
-        make build
+        NOVTADMINBUILD=1 make build
         mkdir -p /tmp/vitess-build-current/
         cp -R bin /tmp/vitess-build-current/
 
@@ -164,8 +152,10 @@ jobs:
       run: |
         source build.env
 
-        rm -f $PWD/bin/vttablet
+        rm -f $PWD/bin/vttablet $PWD/bin/mysqlctl $PWD/bin/mysqlctld
         cp /tmp/vitess-build-other/bin/vttablet $PWD/bin/vttablet
+        cp /tmp/vitess-build-other/bin/mysqlctl $PWD/bin/mysqlctl
+        cp /tmp/vitess-build-other/bin/mysqlctld $PWD/bin/mysqlctld
         vttablet --version
 
     # Run test with VTTablet at version N+1 and VTBackup at version N
@@ -184,9 +174,11 @@ jobs:
       run: |
         source build.env
 
-        rm -f $PWD/bin/vtbackup $PWD/bin/vttablet
+        rm -f $PWD/bin/vtbackup $PWD/bin/vttablet $PWD/bin/mysqlctl $PWD/bin/mysqlctld
         cp /tmp/vitess-build-other/bin/vtbackup $PWD/bin/vtbackup
         cp /tmp/vitess-build-current/bin/vttablet $PWD/bin/vttablet
+        cp /tmp/vitess-build-current/bin/mysqlctl $PWD/bin/mysqlctl
+        cp /tmp/vitess-build-current/bin/mysqlctld $PWD/bin/mysqlctld
         vtbackup --version
         vttablet --version
 

diff --git a/.github/workflows/upgrade_downgrade_test_backups_manual.yml b/.github/workflows/upgrade_downgrade_test_backups_manual.yml
@@ -1,4 +1,4 @@
-name: Upgrade Downgrade Testing - Backups - Manual
+name: Backups - Manual - Upgrade Downgrade Testing
 on:
   push:
   pull_request:
@@ -7,35 +7,15 @@ concurrency:
   group: format('{0}-{1}', ${{ github.ref }}, 'Upgrade Downgrade Testing - Backups - Manual')
   cancel-in-progress: true
 
-jobs:
-  get_previous_release:
-    if: always()
-    name: Get Previous Release - Backups - Manual
-    runs-on: ubuntu-22.04
-    outputs:
-      previous_release: ${{ steps.output-previous-release-ref.outputs.previous_release_ref }}
+permissions: read-all
 
-    steps:
-      - name: Check out to HEAD
-        uses: actions/checkout@v3
-        with:
-          fetch-depth: 0
-
-      - name: Set output with latest release branch
-        id: output-previous-release-ref
-        run: |
-          previous_release_ref=$(./tools/get_previous_release.sh ${{github.base_ref}} ${{github.ref}})
-          echo $previous_release_ref
-          echo "previous_release_ref=${previous_release_ref}" >> $GITHUB_OUTPUT
+jobs:
 
   # This job usually execute in ± 20 minutes
   upgrade_downgrade_test_manual:
     timeout-minutes: 40
-    if: always() && (needs.get_previous_release.result == 'success')
     name: Run Upgrade Downgrade Test - Backups - Manual
     runs-on: ubuntu-22.04
-    needs:
-      - get_previous_release
 
     steps:
     - name: Skip CI
@@ -59,6 +39,16 @@ jobs:
     - name: Checkout to commit's code
       if: steps.skip-workflow.outputs.skip-workflow == 'false'
       uses: actions/checkout@v3
+      with:
+        fetch-depth: 0
+
+    - name: Set output with latest release branch
+      id: output-previous-release-ref
+      if: steps.skip-workflow.outputs.skip-workflow == 'false'
+      run: |
+        previous_release_ref=$(./tools/get_previous_release.sh ${{github.base_ref}} ${{github.ref}})
+        echo $previous_release_ref
+        echo "previous_release_ref=${previous_release_ref}" >> $GITHUB_OUTPUT
 
     - name: Check for changes in relevant files
       if: steps.skip-workflow.outputs.skip-workflow == 'false'
@@ -84,9 +74,9 @@ jobs:
 
     - name: Set up Go
       if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true'
-      uses: actions/setup-go@v3
+      uses: actions/setup-go@v4
       with:
-        go-version: 1.20.12
+        go-version: 1.21.5
 
     - name: Set up python
       if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true'
@@ -133,14 +123,14 @@ jobs:
         sudo apt-get install -y gnupg2
         sudo dpkg -i percona-release_latest.$(lsb_release -sc)_all.deb
         sudo apt-get update
-        sudo apt-get install percona-xtrabackup-24
+        sudo apt-get install -y percona-xtrabackup-24
 
     # Checkout to the last release of Vitess
-    - name: Checkout to the other version's code (${{ needs.get_previous_release.outputs.previous_release }})
+    - name: Checkout to the other version's code (${{ steps.output-previous-release-ref.outputs.previous_release_ref }})
       if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true'
       uses: actions/checkout@v3
       with:
-        ref: ${{ needs.get_previous_release.outputs.previous_release }}
+        ref: ${{ steps.output-previous-release-ref.outputs.previous_release_ref }}
 
     - name: Get dependencies for the last release
       if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true'
@@ -152,7 +142,7 @@ jobs:
       timeout-minutes: 5
       run: |
         source build.env
-        make build
+        NOVTADMINBUILD=1 make build
         mkdir -p /tmp/vitess-build-other/
         cp -R bin /tmp/vitess-build-other/
         rm -Rf bin/*
@@ -177,7 +167,7 @@ jobs:
       timeout-minutes: 5
       run: |
         source build.env
-        make build
+        NOVTADMINBUILD=1 make build
         mkdir -p /tmp/vitess-build-current/
         cp -R bin /tmp/vitess-build-current/
 
@@ -228,8 +218,10 @@ jobs:
       run: |
         source build.env
 
-        rm -f $PWD/bin/vttablet
+        rm -f $PWD/bin/vttablet $PWD/bin/mysqlctl $PWD/bin/mysqlctld
         cp /tmp/vitess-build-other/bin/vttablet $PWD/bin/vttablet
+        cp /tmp/vitess-build-other/bin/mysqlctl $PWD/bin/mysqlctl
+        cp /tmp/vitess-build-other/bin/mysqlctld $PWD/bin/mysqlctld
         vttablet --version
 
     # Starting the tablets again, they will automatically start restoring the last backup.
@@ -276,8 +268,10 @@ jobs:
       run: |
         source build.env
 
-        rm -f $PWD/bin/vttablet
+        rm -f $PWD/bin/vttablet $PWD/bin/mysqlctl $PWD/bin/mysqlctld
         cp /tmp/vitess-build-current/bin/vttablet $PWD/bin/vttablet
+        cp /tmp/vitess-build-current/bin/mysqlctl $PWD/bin/mysqlctl
+        cp /tmp/vitess-build-current/bin/mysqlctld $PWD/bin/mysqlctld
         vttablet --version
 
     # Starting the tablets again and restoring the previous backup.