Bump mysql to 8.0.31 and 5.7.41, fixes ddev#4619, fixes ddev#4504 (ddev#4644)

Author: rfay

.vscode/launch.json

@@ -7,11 +7,23 @@
             "request": "launch",
             "mode": "debug",
             "program": "${workspaceRoot}/cmd/ddev",
-            "cwd": "${workspaceRoot}/../d9",
+            "cwd": "${workspaceRoot}/../d9simple",
             "env": {"DDEV_DEBUG": "true"},
             "args": ["start", "-y"],
             "showLog": true
         },
+        {
+            "name": "ddev snapshot",
+            "type": "go",
+            "request": "launch",
+            "mode": "debug",
+            "program": "${workspaceRoot}/cmd/ddev",
+            "cwd": "${workspaceRoot}/../d9simple",
+            "env": {"DDEV_DEBUG": "true"},
+            "args": ["snapshot"],
+            "showLog": true
+        },
+
         {
             "name": "debug ddev describe",
             "type": "go",

containers/ddev-dbserver/Dockerfile

@@ -60,6 +60,8 @@ RUN ln -s -f /dev/stderr /var/log/mysqld.err
 RUN rm -rf /var/lib/mysql/*
 RUN chmod -R ugo+rw /var/lib/mysql /etc/mysql/conf.d /mysqlbase && find /mysqlbase -type d | xargs chmod ugo+rwx
 
+RUN mkdir -p /var/run/mysqld && chmod 777 /var/run/mysqld
+
 RUN /sbin/mkhomedir_helper www-data
 
 # Remove the /etc/apt entry so that if they don't renew the key

containers/ddev-dbserver/Makefile

@@ -10,7 +10,7 @@ CURRENT_ARCH=$(shell ../get_arch.sh)
 # So has to explicitly declare anything it might need from there (like SHELL)
 SHELL = /bin/bash
 
-BUILD_TARGETS=mariadb_5.5_amd64 mariadb_10.0_amd64 mariadb_10.1_both mariadb_10.2_both mariadb_10.3_both mariadb_10.4_both mariadb_10.5_both mariadb_10.6_both mariadb_10.7_both mariadb_10.8_both mysql_5.5_amd64 mysql_5.6_amd64 mysql_5.7_both mysql_8.0_both_8.0.26
+BUILD_TARGETS=mariadb_5.5_amd64 mariadb_10.0_amd64 mariadb_10.1_both mariadb_10.2_both mariadb_10.3_both mariadb_10.4_both mariadb_10.5_both mariadb_10.6_both mariadb_10.7_both mariadb_10.8_both mysql_5.5_amd64 mysql_5.6_amd64 mysql_5.7_both mysql_8.0_both_8.0.31
 TEST_TARGETS=$(shell if [ "$(CURRENT_ARCH)" = "amd64" ] ; then \
 	echo "mariadb_5.5_test mariadb_10.0_test mariadb_10.1_test mariadb_10.2_test mariadb_10.3_test mariadb_10.4_test mariadb_10.5_test mariadb_10.6_test mariadb_10.7_test mariadb_10.8_test mysql_5.5_test mysql_5.6_test mysql_5.7_test mysql_8.0_test"; \
 	else \
@@ -37,13 +37,13 @@ mysql_5.7: mysql_5.7_both
 
 # Mysql 8.0 often must be pinned because xtrabackup is not ready for latest 8.0
 # So check whether xtrabackup is available for latest 8.0 before changing pin
-mysql_8.0: mysql_8.0_both_8.0.26
+mysql_8.0: mysql_8.0_both_8.0.31
 
 
 # Examples:
 # make <dbtype>_<dbmajor>_[both|amd64]_<pin>  # pin is optional, often needed for mysql 8.0
 # make mariadb_10.3_both VERSION=someversion PUSH=true
-# make mysql_8.0_amd64_8.0.26 VERSION=someversion
+# make mysql_8.0_amd64_8.0.31 VERSION=someversion
 $(BUILD_TARGETS):
 	@echo "building $@";
 	export DB_TYPE=$(word 1, $(subst _, ,$@)) && \

pkg/ddevapp/app_compose_template.yaml

@@ -12,6 +12,8 @@ services:
         uid: '{{ .UID }}'
         gid: {{ if ne .DBType "postgres" }} {{ .GID }} {{ else }} "999" {{ end }}
     image: ${DDEV_DBIMAGE}-${DDEV_SITENAME}-built
+    cap_add:
+      - SYS_NICE
     stop_grace_period: 60s
     working_dir: "{{ .DBWorkingDir }}"
     volumes:

pkg/ddevapp/ddevapp.go

@@ -2106,6 +2106,41 @@ func (app *DdevApp) Snapshot(snapshotName string) (string, error) {
 		return "", fmt.Errorf("unable to snapshot database, \nyour db container in project %v is not running. \nPlease start the project if you want to snapshot it. \nIf deleting project, you can delete without a snapshot using \n'ddev delete --omit-snapshot --yes', \nwhich will destroy your database", app.Name)
 	}
 
+	// For versions less than 8.0.32, we have to OPTIMIZE TABLES to make xtrabackup work
+	// See https://docs.percona.com/percona-xtrabackup/8.0/em/instant.html and
+	// https://www.percona.com/blog/percona-xtrabackup-8-0-29-and-instant-add-drop-columns/
+	if app.Database.Type == "mysql" && app.Database.Version == nodeps.MySQL80 {
+		stdout, stderr, err := app.Exec(&ExecOpts{
+			Service: "db",
+			Cmd:     `set -eu -o pipefail; MYSQL_PWD=root mysql -e 'SET SQL_NOTES=0'; mysql -N -uroot -e 'SELECT NAME FROM INFORMATION_SCHEMA.INNODB_TABLES WHERE TOTAL_ROW_VERSIONS > 0;'`,
+		})
+		if err != nil {
+			util.Warning("could not check for tables to optimize (mysql 8.0): %v (stdout='%s', stderr='%s')", err, stdout, stderr)
+		} else {
+			stdout = strings.Trim(stdout, "\n\t ")
+			tables := strings.Split(stdout, "\n")
+			// util.Success("tables=%v len(tables)=%d stdout was '%s'", tables, len(tables), stdout)
+			if len(stdout) > 0 && len(tables) > 0 {
+				for _, t := range tables {
+					r := strings.Split(t, `/`)
+					if len(r) != 2 {
+						util.Warning("unable to get database/table from %s", r)
+						continue
+					}
+					d := r[0]
+					t := r[1]
+					stdout, stderr, err := app.Exec(&ExecOpts{
+						Service: "db",
+						Cmd:     fmt.Sprintf(`set -eu -o pipefail; MYSQL_PWD=root mysql -uroot -D %s -e 'OPTIMIZE TABLES %s';`, d, t),
+					})
+					if err != nil {
+						util.Warning("unable to optimize table %s (mysql 8.0): %v (stdout='%s', stderr='%s')", t, err, stdout, stderr)
+					}
+				}
+				util.Success("Optimized mysql 8.0 tables '%s' in preparation for snapshot", strings.Join(tables, `,'`))
+			}
+		}
+	}
 	util.Success("Creating database snapshot %s", snapshotName)
 
 	c := getBackupCommand(app, path.Join(containerSnapshotDir, snapshotFile))

pkg/versionconstants/versionconstants.go

@@ -23,7 +23,7 @@ var WebTag = "20230207_fix_nvm" // Note that this can be overridden by make
 var DBImg = "drud/ddev-dbserver"
 
 // BaseDBTag is the main tag, DBTag is constructed from it
-var BaseDBTag = "v1.21.4"
+var BaseDBTag = "v1.21.5"
 
 // DBAImg defines the default phpmyadmin image tag used for applications.
 var DBAImg = "phpmyadmin"