From 4d98be58106e052c860f93d8b83eb6bac7a94ff1 Mon Sep 17 00:00:00 2001 From: Razvan-Liviu Varzaru Date: Mon, 14 Oct 2024 16:44:39 +0300 Subject: [PATCH 01/10] Make version_name from os_info actually be useful for searching the RPM repo on {archive/mirror}.mariadb.org Exception: centos >= 9 which should be sorted out! (relative path on mirrors centos/$version/$platform) Build_arch is used as $arch in bash_lib.sh, rpm_setup_mariadb_mirror{} to calculate mirror/archive URL. --- master-libvirt/master.cfg | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/master-libvirt/master.cfg b/master-libvirt/master.cfg index f8b2b78b..c30078d4 100644 --- a/master-libvirt/master.cfg +++ b/master-libvirt/master.cfg @@ -301,7 +301,14 @@ for builder_name in builders_install: elif builder_type == "rpm": factory_install = f_rpm_install factory_upgrade = f_rpm_upgrade - build_arch = os_name + os_version + "-" + platform + build_arch = os_name + os_info[os_info_name]["version_name"] + "-" + platform + + # FIXME - all RPM's should follow the same conventions! + if os_name == "centos" and os_info[os_info_name]["version_name"] >= 9: + if platform == "amd64": + platform = "x86_64" + build_arch = f"centos/{os_info[os_info_name]["version_name"]}/{platform}" + c["builders"].append( util.BuilderConfig( From c34873e9643a485b387bcd14956f901c79888dcd Mon Sep 17 00:00:00 2001 From: Razvan-Liviu Varzaru Date: Mon, 14 Oct 2024 16:46:00 +0300 Subject: [PATCH 02/10] Fix OpenSuse path on mirror/archive {opensuse155-amd64,opensuse156-amd64} on https://mirror.mariadb.org/yum/$branch --- os_info.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/os_info.yaml b/os_info.yaml index 6caba015..2de78f57 100644 --- a/os_info.yaml +++ b/os_info.yaml @@ -72,12 +72,12 @@ openeuler-2403: - aarch64 type: rpm opensuse-1505: - version_name: 15.5 + version_name: 155 arch: - amd64 type: rpm opensuse-1506: - version_name: 15.6 + version_name: 156 arch: - amd64 type: rpm From 8990ed9e3738787bb266137096236e163b91f9f4 Mon Sep 17 00:00:00 2001 From: Razvan-Liviu Varzaru Date: Mon, 14 Oct 2024 16:53:39 +0300 Subject: [PATCH 03/10] libmecab-dev is required by mariadb-plugin-mroonga Fixing minor upgrades in test_mode = 'ALL". See: https://buildbot.dev.mariadb.org/#/builders/326/builds/14/steps/3/logs/stdio --- ci_build_images/debian.Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/ci_build_images/debian.Dockerfile b/ci_build_images/debian.Dockerfile index c4d9448d..78923dca 100644 --- a/ci_build_images/debian.Dockerfile +++ b/ci_build_images/debian.Dockerfile @@ -73,6 +73,7 @@ RUN . /etc/os-release \ libeigen3-dev \ libffi-dev \ libio-socket-ssl-perl \ + libmecab-dev \ libnet-ssleay-perl \ libssl-dev \ lsof \ From b26b2678ffe961cfa62fbae759894905fc0e57e6 Mon Sep 17 00:00:00 2001 From: Razvan-Liviu Varzaru Date: Mon, 14 Oct 2024 17:51:29 +0300 Subject: [PATCH 04/10] lsof has been demoted to a weak dependecy. The Q3 2024 release lists it as REQUIRED hence minor upgrades (all) will reflect the change during verifications. > 10.5 MariaDB releases use liburing. Q3 release was linked against libaio which accounts for the differences found during testing. --- scripts/bash_lib.sh | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/scripts/bash_lib.sh b/scripts/bash_lib.sh index 56e6ba33..1189f11a 100644 --- a/scripts/bash_lib.sh +++ b/scripts/bash_lib.sh @@ -571,6 +571,12 @@ check_upgraded_versions() { # when legitimate changes in dependencies happen between minor versions. # The adjustments should be done to .cmp files, and removed after the release # + + # Remove after Q4 2024 release + sed -i '/libaio.so/d;liburing.so/d' ./reqs-*.cmp + sed -i '/libaio.so/d;liburing.so/d' ./ldd-*.cmp + sed -i '/lsof/d' ./reqs-*.cmp + # End of temporary adjustments set -o pipefail From eaf6a45c7197cc27d267a53feb8f1353545b78e5 Mon Sep 17 00:00:00 2001 From: Razvan-Liviu Varzaru Date: Wed, 16 Oct 2024 11:21:54 +0300 Subject: [PATCH 05/10] fix syntax errors in build_arch --- master-libvirt/master.cfg | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/master-libvirt/master.cfg b/master-libvirt/master.cfg index c30078d4..6dd74eee 100644 --- a/master-libvirt/master.cfg +++ b/master-libvirt/master.cfg @@ -301,13 +301,13 @@ for builder_name in builders_install: elif builder_type == "rpm": factory_install = f_rpm_install factory_upgrade = f_rpm_upgrade - build_arch = os_name + os_info[os_info_name]["version_name"] + "-" + platform + build_arch = os_name + str(os_info[os_info_name]["version_name"]) + "-" + platform # FIXME - all RPM's should follow the same conventions! if os_name == "centos" and os_info[os_info_name]["version_name"] >= 9: if platform == "amd64": platform = "x86_64" - build_arch = f"centos/{os_info[os_info_name]["version_name"]}/{platform}" + build_arch = f"centos/{os_info[os_info_name]['version_name']}/{platform}" c["builders"].append( From 001142019224055d616e279154f34ad1dc50fd76 Mon Sep 17 00:00:00 2001 From: Razvan-Liviu Varzaru Date: Wed, 16 Oct 2024 11:28:19 +0300 Subject: [PATCH 06/10] Better sort uniq syntax, more verbose get_packages_file_mirror() --- scripts/deb-upgrade.sh | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/scripts/deb-upgrade.sh b/scripts/deb-upgrade.sh index cb04f208..c54ec7ab 100755 --- a/scripts/deb-upgrade.sh +++ b/scripts/deb-upgrade.sh @@ -37,7 +37,7 @@ get_packages_file_mirror() { set -u if ! wget "$baseurl/$dist_name/dists/$version_name/main/binary-$(deb_arch)/Packages" then - bb_log_err "Could not find the 'Packages' file for a previous version." + bb_log_err "Could not find the 'Packages' file for $dist_name $version_name on deb|archive.mariadb.org." exit 1 fi set +u @@ -50,10 +50,10 @@ case $test_mode in if grep -qi columnstore Packages; then bb_log_warn "due to MCOL-4120 (Columnstore leaves the server shut down) and other bugs Columnstore upgrade is tested separately" fi - package_list=$(grep "^Package:" Packages | grep -vE 'galera|spider|columnstore' | awk '{print $2}' | sort | uniq | xargs) + package_list=$(grep "^Package:" Packages | grep -vE 'galera|spider|columnstore' | awk '{print $2}' | sort -u | xargs) if grep -qi spider Packages; then bb_log_warn "due to MDEV-14622 Spider will be installed separately after the server" - spider_package_list=$(grep "^Package:" Packages | grep 'spider' | awk '{print $2}' | sort | uniq | xargs) + spider_package_list=$(grep "^Package:" Packages | grep 'spider' | awk '{print $2}' | sort -u | xargs) fi if grep -si tokudb Packages; then # For the sake of installing TokuDB, disable hugepages @@ -75,7 +75,7 @@ case $test_mode in bb_log_warn "Columnstore isn't necessarily built on Sid, the test will be skipped" exit fi - package_list="mariadb-server "$(grep "^Package:" Packages | grep 'columnstore' | awk '{print $2}' | sort | uniq | xargs) + package_list="mariadb-server "$(grep "^Package:" Packages | grep 'columnstore' | awk '{print $2}' | sort -u | xargs) ;; *) bb_log_err "unknown test mode: $test_mode" @@ -233,7 +233,7 @@ store_mariadb_server_info new # //TEMP what needs to be done here? # # Dependency information for new binaries/libraries # set +x -# for i in $(sudo which mysqld | sed -e 's/mysqld$/mysql\*/') $(which mysql | sed -e 's/mysql$/mysql\*/') $(dpkg-query -L $(dpkg -l | grep mariadb | awk '{print $2}' | xargs) | grep -v 'mysql-test' | grep -v '/debug/' | grep '/plugin/' | sed -e 's/[^\/]*$/\*/' | sort | uniq | xargs); do +# for i in $(sudo which mysqld | sed -e 's/mysqld$/mysql\*/') $(which mysql | sed -e 's/mysql$/mysql\*/') $(dpkg-query -L $(dpkg -l | grep mariadb | awk '{print $2}' | xargs) | grep -v 'mysql-test' | grep -v '/debug/' | grep '/plugin/' | sed -e 's/[^\/]*$/\*/' | sort -u | xargs); do # echo "=== $i" # ldd $i | sort | sed 's/(.*)//' # done >/home/buildbot/ldd.new From 9db192c95fa30c9faf97ad560a0d0be92c354d66 Mon Sep 17 00:00:00 2001 From: Razvan-Liviu Varzaru Date: Wed, 16 Oct 2024 12:28:43 +0300 Subject: [PATCH 07/10] Account for mariadb-plugin-mroonga diffs in Debian-11 --- scripts/bash_lib.sh | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/scripts/bash_lib.sh b/scripts/bash_lib.sh index 1189f11a..b85a3c1a 100644 --- a/scripts/bash_lib.sh +++ b/scripts/bash_lib.sh @@ -577,6 +577,10 @@ check_upgraded_versions() { sed -i '/libaio.so/d;liburing.so/d' ./ldd-*.cmp sed -i '/lsof/d' ./reqs-*.cmp + #Account for mariadb-plugin-mroonga diffs in Debian-11 + sed -i '/liblz4-1/d' ./reqs-*.cmp + sed -i '/liblz4.so.1/d' ./ldd-*.cmp + # End of temporary adjustments set -o pipefail From 46c23d68fc7dc02648d7befa9f322d6644e856fc Mon Sep 17 00:00:00 2001 From: Razvan-Liviu Varzaru Date: Thu, 17 Oct 2024 14:52:31 +0300 Subject: [PATCH 08/10] Fix OpenSuse upgrade script - cache location is zypp - "all" does not exist for clearing cache, instead zypper uses --all - when MariaDB.repo is first added the key should be trusted in a non-interactive manner, hence --gpg-auto-import-keys - first zypper packages -r goes to /dev/null because there is a lot of noise from trusting the key. Sudo is needed to make the cache. - second zypper packages -r runs as a normal user with no issues. --- scripts/bash_lib.sh | 12 +++++++++--- scripts/rpm-upgrade.sh | 16 +++++++++++----- 2 files changed, 20 insertions(+), 8 deletions(-) diff --git a/scripts/bash_lib.sh b/scripts/bash_lib.sh index b85a3c1a..fd9ed283 100644 --- a/scripts/bash_lib.sh +++ b/scripts/bash_lib.sh @@ -167,8 +167,13 @@ rpm_pkg_makecache() { # Try several times, to avoid sporadic "The requested URL returned error: 404" made_cache=0 for i in {1..5}; do - sudo rm -rf "/var/cache/$pkg_cmd/*" - sudo "$pkg_cmd" clean all + if [[ $ID_LIKE =~ ^suse* ]]; then + sudo rm -rf "/var/cache/zypp/*" + sudo "$pkg_cmd" clean --all + else + sudo rm -rf "/var/cache/$pkg_cmd/*" + sudo "$pkg_cmd" clean all + fi source /etc/os-release if [[ $ID == "rhel" ]]; then sudo subscription-manager refresh @@ -210,7 +215,8 @@ rpm_repoquery() { set +u # return full package list from repository if [[ $ID_LIKE =~ ^suse* ]]; then - zypper packages -r "${repo_name}" | grep "MariaDB" | awk '{print $8}' + sudo zypper --gpg-auto-import-keys packages -r "${repo_name}">/dev/null + zypper packages -r "${repo_name}" | grep "MariaDB" | awk '{print $4}' #After cache is made, no need for sudo else repoquery --disablerepo=* --enablerepo="${repo_name}" -a -q | cut -d ":" -f1 | sort -u | sed 's/-0//' diff --git a/scripts/rpm-upgrade.sh b/scripts/rpm-upgrade.sh index 24988344..0c66d582 100755 --- a/scripts/rpm-upgrade.sh +++ b/scripts/rpm-upgrade.sh @@ -148,10 +148,16 @@ bb_log_info "Package_list: $package_list" # sudo rm -rf # /etc/zypp/repos.d/SUSE_Linux_Enterprise_Server_12_SP3_x86_64:SLES12-SP3-Updates.repo # /etc/zypp/repos.d/SUSE_Linux_Enterprise_Server_12_SP3_x86_64:SLES12-SP3-Pool.repo -sudo "$pkg_cmd" clean all +if [[ $ID_LIKE =~ ^suse* ]]; then + sudo "$pkg_cmd" clean --all + pkg_cmd_nonint="-n" +else + sudo "$pkg_cmd" clean all + pkg_cmd_nonint="-y" +fi # Install previous release -echo "$package_list" | xargs sudo "$pkg_cmd" -y install || +echo "$package_list" | xargs sudo "$pkg_cmd" "$pkg_cmd_nonint" install || bb_log_err "installation of a previous release failed, see the output above" #fi @@ -246,7 +252,7 @@ fi if [[ $test_type == "major" ]]; then bb_log_info "remove old packages for major upgrade" packages_to_remove=$(rpm -qa | grep 'MariaDB-' | awk -F'-' '{print $1"-"$2}') - echo "$packages_to_remove" | xargs sudo "$pkg_cmd" -y remove + echo "$packages_to_remove" | xargs sudo "$pkg_cmd" "$pkg_cmd_nonint" remove rpm -qa | grep -iE 'maria|mysql' || true fi @@ -254,10 +260,10 @@ rpm_setup_bb_galera_artifacts_mirror rpm_setup_bb_artifacts_mirror if [[ $test_type == "major" ]]; then # major upgrade (remove then install) - echo "$package_list" | xargs sudo "$pkg_cmd" -y install + echo "$package_list" | xargs sudo "$pkg_cmd" "$pkg_cmd_nonint" install else # minor upgrade (upgrade works) - echo "$package_list" | xargs sudo "$pkg_cmd" -y upgrade + echo "$package_list" | xargs sudo "$pkg_cmd" "$pkg_cmd_nonint" upgrade fi # set +e From 39390f736716f2334da1af5af65231e9f2408b77 Mon Sep 17 00:00:00 2001 From: Razvan-Liviu Varzaru Date: Thu, 17 Oct 2024 15:20:25 +0300 Subject: [PATCH 09/10] Fix don't build compat for OpenSUSE/SLES 15.x --- utils.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/utils.py b/utils.py index 69450764..b2e78344 100644 --- a/utils.py +++ b/utils.py @@ -477,7 +477,15 @@ def hasCompat(step): # For s390x and the listed distros there are no compat files if any( builderName.find(sub) != -1 - for sub in {"s390x", "fedora", "alma", "rocky", "openeuler", "suse15", "sles15"} + for sub in { + "s390x", + "fedora", + "alma", + "rocky", + "openeuler", + "suse-15", + "sles-15", + } ): return False if "rhel" in builderName or "centos" in builderName: From e6b5f9f6baf2b4e6aba1db2a8f3adafab6ce165e Mon Sep 17 00:00:00 2001 From: Razvan-Liviu Varzaru Date: Fri, 18 Oct 2024 10:36:43 +0300 Subject: [PATCH 10/10] Trust mariadb key after setting up the repo --- scripts/bash_lib.sh | 4 +++- scripts/rpm-upgrade.sh | 12 ++++++------ 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/scripts/bash_lib.sh b/scripts/bash_lib.sh index fd9ed283..5e3403eb 100644 --- a/scripts/bash_lib.sh +++ b/scripts/bash_lib.sh @@ -215,7 +215,6 @@ rpm_repoquery() { set +u # return full package list from repository if [[ $ID_LIKE =~ ^suse* ]]; then - sudo zypper --gpg-auto-import-keys packages -r "${repo_name}">/dev/null zypper packages -r "${repo_name}" | grep "MariaDB" | awk '{print $4}' #After cache is made, no need for sudo else repoquery --disablerepo=* --enablerepo="${repo_name}" -a -q | @@ -315,6 +314,9 @@ module_hotfixes = 1 gpgkey=https://rpm.mariadb.org/RPM-GPG-KEY-MariaDB gpgcheck=1 EOF + if [[ $ID_LIKE =~ ^suse* ]]; then + sudo zypper --gpg-auto-import-keys refresh mariadb + fi set +u } diff --git a/scripts/rpm-upgrade.sh b/scripts/rpm-upgrade.sh index 0c66d582..0460a4d1 100755 --- a/scripts/rpm-upgrade.sh +++ b/scripts/rpm-upgrade.sh @@ -150,14 +150,14 @@ bb_log_info "Package_list: $package_list" # /etc/zypp/repos.d/SUSE_Linux_Enterprise_Server_12_SP3_x86_64:SLES12-SP3-Pool.repo if [[ $ID_LIKE =~ ^suse* ]]; then sudo "$pkg_cmd" clean --all - pkg_cmd_nonint="-n" + pkg_cmd_options="-n" else sudo "$pkg_cmd" clean all - pkg_cmd_nonint="-y" + pkg_cmd_options="-y" fi # Install previous release -echo "$package_list" | xargs sudo "$pkg_cmd" "$pkg_cmd_nonint" install || +echo "$package_list" | xargs sudo "$pkg_cmd" "$pkg_cmd_options" install || bb_log_err "installation of a previous release failed, see the output above" #fi @@ -252,7 +252,7 @@ fi if [[ $test_type == "major" ]]; then bb_log_info "remove old packages for major upgrade" packages_to_remove=$(rpm -qa | grep 'MariaDB-' | awk -F'-' '{print $1"-"$2}') - echo "$packages_to_remove" | xargs sudo "$pkg_cmd" "$pkg_cmd_nonint" remove + echo "$packages_to_remove" | xargs sudo "$pkg_cmd" "$pkg_cmd_options" remove rpm -qa | grep -iE 'maria|mysql' || true fi @@ -260,10 +260,10 @@ rpm_setup_bb_galera_artifacts_mirror rpm_setup_bb_artifacts_mirror if [[ $test_type == "major" ]]; then # major upgrade (remove then install) - echo "$package_list" | xargs sudo "$pkg_cmd" "$pkg_cmd_nonint" install + echo "$package_list" | xargs sudo "$pkg_cmd" "$pkg_cmd_options" install else # minor upgrade (upgrade works) - echo "$package_list" | xargs sudo "$pkg_cmd" "$pkg_cmd_nonint" upgrade + echo "$package_list" | xargs sudo "$pkg_cmd" "$pkg_cmd_options" upgrade fi # set +e