Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[release-16.0] Refactor Upgrade downgrade tests (#14782) #14830

Merged
merged 8 commits into from
Dec 27, 2023
Merged
Show file tree
Hide file tree
Changes from 7 commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
39 changes: 13 additions & 26 deletions .github/workflows/upgrade_downgrade_test_backups_e2e.yml
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
name: Upgrade Downgrade Testing - Backups - E2E
name: Backups - E2E - Upgrade Downgrade Testing
on:
push:
pull_request:
Expand All @@ -8,33 +8,10 @@ concurrency:
cancel-in-progress: true

jobs:
get_previous_release:
if: always()
name: Get Previous Release - Backups - E2E
runs-on: ubuntu-22.04
outputs:
previous_release: ${{ steps.output-previous-release-ref.outputs.previous_release_ref }}

steps:
- name: Check out to HEAD
uses: actions/checkout@v3
with:
fetch-depth: 0

- name: Set output with latest release branch
id: output-previous-release-ref
run: |
previous_release_ref=$(./tools/get_previous_release.sh ${{github.base_ref}} ${{github.ref}})
echo $previous_release_ref
echo "previous_release_ref=${previous_release_ref}" >> $GITHUB_OUTPUT

upgrade_downgrade_test_e2e:
timeout-minutes: 60
if: always() && needs.get_previous_release.result == 'success'
name: Run Upgrade Downgrade Test - Backups - E2E
runs-on: ubuntu-22.04
needs:
- get_previous_release

steps:
- name: Skip CI
Expand All @@ -57,6 +34,16 @@ jobs:
- name: Check out commit's code
if: steps.skip-workflow.outputs.skip-workflow == 'false'
uses: actions/checkout@v3
with:
fetch-depth: 0

- name: Set output with latest release branch
if: steps.skip-workflow.outputs.skip-workflow == 'false'
id: output-previous-release-ref
run: |
previous_release_ref=$(./tools/get_previous_release.sh ${{github.base_ref}} ${{github.ref}})
echo $previous_release_ref
echo "previous_release_ref=${previous_release_ref}" >> $GITHUB_OUTPUT

- name: Check for changes in relevant files
if: steps.skip-workflow.outputs.skip-workflow == 'false'
Expand Down Expand Up @@ -115,11 +102,11 @@ jobs:
sudo apt-get install percona-xtrabackup-24

# Checkout to the last release of Vitess
- name: Check out other version's code (${{ needs.get_previous_release.outputs.previous_release }})
- name: Check out other version's code (${{ steps.output-previous-release-ref.outputs.previous_release_ref }})
if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true'
uses: actions/checkout@v3
with:
ref: ${{ needs.get_previous_release.outputs.previous_release }}
ref: ${{ steps.output-previous-release-ref.outputs.previous_release_ref }}

- name: Get dependencies for the last release
if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true'
Expand Down
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
name: Upgrade Downgrade Testing - Backups - E2E - Next Release
name: Backups - E2E - Next Release - Upgrade Downgrade Testing
on:
push:
pull_request:
Expand All @@ -7,34 +7,14 @@ concurrency:
group: format('{0}-{1}', ${{ github.ref }}, 'Upgrade Downgrade Testing - Backups - E2E - Next Release')
cancel-in-progress: true

jobs:
get_next_release:
if: always()
name: Get Latest Release - Backups - E2E - Next Release
runs-on: ubuntu-22.04
outputs:
next_release: ${{ steps.output-next-release-ref.outputs.next_release_ref }}
permissions: read-all

steps:
- name: Check out to HEAD
uses: actions/checkout@v3
with:
fetch-depth: 0

- name: Set output with latest release branch
id: output-next-release-ref
run: |
next_release_ref=$(./tools/get_next_release.sh ${{github.base_ref}} ${{github.ref}})
echo $next_release_ref
echo "next_release_ref=${next_release_ref}" >> $GITHUB_OUTPUT
jobs:

upgrade_downgrade_test_e2e:
timeout-minutes: 60
if: always() && needs.get_next_release.result == 'success'
name: Run Upgrade Downgrade Test - Backups - E2E - Next Release
runs-on: ubuntu-22.04
needs:
- get_next_release

steps:
- name: Skip CI
Expand All @@ -44,23 +24,31 @@ jobs:
exit 1
fi

- name: Check out commit's code
uses: actions/checkout@v3
with:
fetch-depth: 0

- name: Set output with latest release branch
id: output-next-release-ref
run: |
next_release_ref=$(./tools/get_next_release.sh ${{github.base_ref}} ${{github.ref}})
echo $next_release_ref
echo "next_release_ref=${next_release_ref}" >> $GITHUB_OUTPUT

- name: Check if workflow needs to be skipped
id: skip-workflow
run: |
skip='false'
if [[ "${{github.event.pull_request}}" == "" ]] && [[ "${{github.ref}}" != "refs/heads/main" ]] && [[ ! "${{github.ref}}" =~ ^refs/heads/release-[0-9]+\.[0-9]$ ]] && [[ ! "${{github.ref}}" =~ "refs/tags/.*" ]]; then
skip='true'
fi
if [[ "${{needs.get_next_release.outputs.next_release}}" == "" ]]; then
if [[ "${{steps.output-next-release-ref.outputs.next_release_ref}}" == "" ]]; then
skip='true'
fi
echo Skip ${skip}
echo "skip-workflow=${skip}" >> $GITHUB_OUTPUT

- name: Check out commit's code
if: steps.skip-workflow.outputs.skip-workflow == 'false'
uses: actions/checkout@v3

- name: Check for changes in relevant files
if: steps.skip-workflow.outputs.skip-workflow == 'false'
uses: frouioui/paths-filter@main
Expand All @@ -84,9 +72,9 @@ jobs:

- name: Set up Go
if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true'
uses: actions/setup-go@v3
uses: actions/setup-go@v4
with:
go-version: 1.20.12
go-version: 1.21.5

- name: Set up python
if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true'
Expand Down Expand Up @@ -115,14 +103,14 @@ jobs:
sudo apt-get install -y gnupg2
sudo dpkg -i percona-release_latest.$(lsb_release -sc)_all.deb
sudo apt-get update
sudo apt-get install percona-xtrabackup-24
sudo apt-get install -y percona-xtrabackup-24

# Checkout to the next release of Vitess
- name: Check out other version's code (${{ needs.get_next_release.outputs.next_release }})
- name: Check out other version's code (${{ steps.output-next-release-ref.outputs.next_release_ref }})
if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true'
uses: actions/checkout@v3
with:
ref: ${{ needs.get_next_release.outputs.next_release }}
ref: ${{ steps.output-next-release-ref.outputs.next_release_ref }}

- name: Get dependencies for the next release
if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true'
Expand All @@ -134,7 +122,7 @@ jobs:
timeout-minutes: 10
run: |
source build.env
make build
NOVTADMINBUILD=1 make build
mkdir -p /tmp/vitess-build-other/
cp -R bin /tmp/vitess-build-other/
rm -Rf bin/*
Expand All @@ -154,7 +142,7 @@ jobs:
timeout-minutes: 10
run: |
source build.env
make build
NOVTADMINBUILD=1 make build
mkdir -p /tmp/vitess-build-current/
cp -R bin /tmp/vitess-build-current/

Expand All @@ -164,8 +152,10 @@ jobs:
run: |
source build.env

rm -f $PWD/bin/vttablet
rm -f $PWD/bin/vttablet $PWD/bin/mysqlctl $PWD/bin/mysqlctld
cp /tmp/vitess-build-other/bin/vttablet $PWD/bin/vttablet
cp /tmp/vitess-build-other/bin/mysqlctl $PWD/bin/mysqlctl
cp /tmp/vitess-build-other/bin/mysqlctld $PWD/bin/mysqlctld
vttablet --version

# Run test with VTTablet at version N+1 and VTBackup at version N
Expand All @@ -184,9 +174,11 @@ jobs:
run: |
source build.env

rm -f $PWD/bin/vtbackup $PWD/bin/vttablet
rm -f $PWD/bin/vtbackup $PWD/bin/vttablet $PWD/bin/mysqlctl $PWD/bin/mysqlctld
cp /tmp/vitess-build-other/bin/vtbackup $PWD/bin/vtbackup
cp /tmp/vitess-build-current/bin/vttablet $PWD/bin/vttablet
cp /tmp/vitess-build-current/bin/mysqlctl $PWD/bin/mysqlctl
cp /tmp/vitess-build-current/bin/mysqlctld $PWD/bin/mysqlctld
vtbackup --version
vttablet --version

Expand Down
58 changes: 26 additions & 32 deletions .github/workflows/upgrade_downgrade_test_backups_manual.yml
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
name: Upgrade Downgrade Testing - Backups - Manual
name: Backups - Manual - Upgrade Downgrade Testing
on:
push:
pull_request:
Expand All @@ -7,35 +7,15 @@ concurrency:
group: format('{0}-{1}', ${{ github.ref }}, 'Upgrade Downgrade Testing - Backups - Manual')
cancel-in-progress: true

jobs:
get_previous_release:
if: always()
name: Get Previous Release - Backups - Manual
runs-on: ubuntu-22.04
outputs:
previous_release: ${{ steps.output-previous-release-ref.outputs.previous_release_ref }}
permissions: read-all

steps:
- name: Check out to HEAD
uses: actions/checkout@v3
with:
fetch-depth: 0

- name: Set output with latest release branch
id: output-previous-release-ref
run: |
previous_release_ref=$(./tools/get_previous_release.sh ${{github.base_ref}} ${{github.ref}})
echo $previous_release_ref
echo "previous_release_ref=${previous_release_ref}" >> $GITHUB_OUTPUT
jobs:

# This job usually execute in ± 20 minutes
upgrade_downgrade_test_manual:
timeout-minutes: 40
if: always() && (needs.get_previous_release.result == 'success')
name: Run Upgrade Downgrade Test - Backups - Manual
runs-on: ubuntu-22.04
needs:
- get_previous_release

steps:
- name: Skip CI
Expand All @@ -59,6 +39,16 @@ jobs:
- name: Checkout to commit's code
if: steps.skip-workflow.outputs.skip-workflow == 'false'
uses: actions/checkout@v3
with:
fetch-depth: 0

- name: Set output with latest release branch
id: output-previous-release-ref
if: steps.skip-workflow.outputs.skip-workflow == 'false'
run: |
previous_release_ref=$(./tools/get_previous_release.sh ${{github.base_ref}} ${{github.ref}})
echo $previous_release_ref
echo "previous_release_ref=${previous_release_ref}" >> $GITHUB_OUTPUT

- name: Check for changes in relevant files
if: steps.skip-workflow.outputs.skip-workflow == 'false'
Expand All @@ -84,9 +74,9 @@ jobs:

- name: Set up Go
if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true'
uses: actions/setup-go@v3
uses: actions/setup-go@v4
with:
go-version: 1.20.12
go-version: 1.21.5

- name: Set up python
if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true'
Expand Down Expand Up @@ -133,14 +123,14 @@ jobs:
sudo apt-get install -y gnupg2
sudo dpkg -i percona-release_latest.$(lsb_release -sc)_all.deb
sudo apt-get update
sudo apt-get install percona-xtrabackup-24
sudo apt-get install -y percona-xtrabackup-24

# Checkout to the last release of Vitess
- name: Checkout to the other version's code (${{ needs.get_previous_release.outputs.previous_release }})
- name: Checkout to the other version's code (${{ steps.output-previous-release-ref.outputs.previous_release_ref }})
if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true'
uses: actions/checkout@v3
with:
ref: ${{ needs.get_previous_release.outputs.previous_release }}
ref: ${{ steps.output-previous-release-ref.outputs.previous_release_ref }}

- name: Get dependencies for the last release
if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true'
Expand All @@ -152,7 +142,7 @@ jobs:
timeout-minutes: 5
run: |
source build.env
make build
NOVTADMINBUILD=1 make build
mkdir -p /tmp/vitess-build-other/
cp -R bin /tmp/vitess-build-other/
rm -Rf bin/*
Expand All @@ -177,7 +167,7 @@ jobs:
timeout-minutes: 5
run: |
source build.env
make build
NOVTADMINBUILD=1 make build
mkdir -p /tmp/vitess-build-current/
cp -R bin /tmp/vitess-build-current/

Expand Down Expand Up @@ -228,8 +218,10 @@ jobs:
run: |
source build.env

rm -f $PWD/bin/vttablet
rm -f $PWD/bin/vttablet $PWD/bin/mysqlctl $PWD/bin/mysqlctld
cp /tmp/vitess-build-other/bin/vttablet $PWD/bin/vttablet
cp /tmp/vitess-build-other/bin/mysqlctl $PWD/bin/mysqlctl
cp /tmp/vitess-build-other/bin/mysqlctld $PWD/bin/mysqlctld
vttablet --version

# Starting the tablets again, they will automatically start restoring the last backup.
Expand Down Expand Up @@ -276,8 +268,10 @@ jobs:
run: |
source build.env

rm -f $PWD/bin/vttablet
rm -f $PWD/bin/vttablet $PWD/bin/mysqlctl $PWD/bin/mysqlctld
cp /tmp/vitess-build-current/bin/vttablet $PWD/bin/vttablet
cp /tmp/vitess-build-current/bin/mysqlctl $PWD/bin/mysqlctl
cp /tmp/vitess-build-current/bin/mysqlctld $PWD/bin/mysqlctld
vttablet --version

# Starting the tablets again and restoring the previous backup.
Expand Down
Loading
Loading