Release 4.1.0 - See CHANGELOG.md

CHANGELOG.md

@@ -1,3 +1,13 @@
+## 4.1.0 2024-05-25 <dave at tiredofit dot ca>
+
+ ### Added
+  - Introduce DEFAULT/DBXX_MYSQL_CLIENT option to use mariadb or mysql for client dumping to solve incompatibility issues
+  - Alpine 3.20 Base
+  - MariaDB 10.11.8 Client
+  - AWS Client 1.32.113
+  - MySQL Client 8.4.0
+
+
 ## 4.0.35 2024-01-14 <dave at tiredofit dot ca>
 
  ### Changed

Dockerfile

@@ -1,5 +1,5 @@
 ARG DISTRO=alpine
-ARG DISTRO_VARIANT=3.19
+ARG DISTRO_VARIANT=3.20
 
 FROM docker.io/tiredofit/${DISTRO}:${DISTRO_VARIANT}
 LABEL maintainer="Dave Conroy (github.com/tiredofit)"
@@ -9,7 +9,9 @@ ENV INFLUX1_CLIENT_VERSION=1.8.0 \
  INFLUX2_CLIENT_VERSION=2.7.3 \
  MSODBC_VERSION=18.3.2.1-1 \
  MSSQL_VERSION=18.3.1.1-1 \
- AWS_CLI_VERSION=1.31.5 \
+ MYSQL_VERSION=mysql-8.4.0 \
+ MYSQL_REPO_URL=https://github.com/mysql/mysql-server \
+ AWS_CLI_VERSION=1.32.113 \
  CONTAINER_ENABLE_MESSAGING=TRUE \
  CONTAINER_ENABLE_MONITORING=TRUE \
  IMAGE_NAME="tiredofit/db-backup" \
@@ -27,11 +29,13 @@ RUN source /assets/functions/00-container && \
  build-base \
  bzip2-dev \
  cargo \
+ cmake \
  git \
  go \
  libarchive-dev \
  openssl-dev \
  libffi-dev \
+ ncurses-dev \
  python3-dev \
  py3-pip \
  xz-dev \
@@ -47,8 +51,10 @@ RUN source /assets/functions/00-container && \
  mariadb-client \
  mariadb-connector-c \
  mongodb-tools \
+ ncurses \
  openssl \
  pigz \
+ pixz \
  postgresql16 \
  postgresql16-client \
  pv \
@@ -81,24 +87,24 @@ RUN source /assets/functions/00-container && \
  clone_git_repo https://github.com/influxdata/influxdb "${INFLUX1_CLIENT_VERSION}" && \
  go build -o /usr/sbin/influxd ./cmd/influxd && \
  strip /usr/sbin/influxd && \
+ \
+ clone_git_repo "${MYSQL_REPO_URL}" "${MYSQL_VERSION}" && \
+ cmake \
+ -DCMAKE_BUILD_TYPE=MinSizeRel \
+ -DCMAKE_INSTALL_PREFIX=/opt/mysql \
+ -DFORCE_INSOURCE_BUILD=1 \
+ -DWITHOUT_SERVER:BOOL=ON \
+ && \
+ make -j$(nproc) install && \
+ \
+ pip3 install --break-system-packages awscli==${AWS_CLI_VERSION} && \
+ pip3 install --break-system-packages blobxfer && \
+ \
  mkdir -p /usr/src/pbzip2 && \
  curl -sSL https://launchpad.net/pbzip2/1.1/1.1.13/+download/pbzip2-1.1.13.tar.gz | tar xvfz - --strip=1 -C /usr/src/pbzip2 && \
  cd /usr/src/pbzip2 && \
  make && \
  make install && \
- mkdir -p /usr/src/pixz && \
- curl -sSL https://github.com/vasi/pixz/releases/download/v1.0.7/pixz-1.0.7.tar.xz | tar xvfJ - --strip 1 -C /usr/src/pixz && \
- cd /usr/src/pixz && \
- ./configure \
- --prefix=/usr \
- --sysconfdir=/etc \
- --localstatedir=/var \
- && \
- make && \
- make install && \
- \
- pip3 install --break-system-packages awscli==${AWS_CLI_VERSION} && \
- pip3 install --break-system-packages blobxfer && \
  \
  package remove .db-backup-build-deps && \
  package cleanup && \

README.md

@@ -267,6 +267,7 @@ Encryption occurs after compression and the encrypted filename will have a `.gpg
 | `DEFAULT_EXTRA_BACKUP_OPTS` | Pass extra arguments to the backup command only, add them here e.g. `--extra-command` | | |
 | `DEFAULT_EXTRA_ENUMERATION_OPTS` | Pass extra arguments to the database enumeration command only, add them here e.g. `--extra-command` | | |
 | `DEFAULT_EXTRA_OPTS` | Pass extra arguments to the backup and database enumeration command, add them here e.g. `--extra-command` | | |
+| `DEFAULT_MYSQL_CLIENT` | Choose between `mariadb` or `mysql` client to perform dump operations for compatibility purposes | `mariadb` | |
 | `DEFAULT_MYSQL_EVENTS` | Backup Events | `TRUE` | |
 | `DEFAULT_MYSQL_MAX_ALLOWED_PACKET` | Max allowed packet | `512M` | |
 | `DEFAULT_MYSQL_SINGLE_TRANSACTION` | Backup in a single transaction | `TRUE` | |

install/assets/defaults/10-db-backup

@@ -6,7 +6,6 @@ DBBACKUP_USER=${DBBACKUP_USER:-"dbbackup"}
 DBBACKUP_GROUP=${DBBACKUP_GROUP:-"${DBBACKUP_USER}"} # Must go after DBBACKUP_USER
 DEFAULT_BACKUP_BEGIN=${DEFAULT_BACKUP_BEGIN:-+0}
 DEFAULT_BACKUP_INTERVAL=${DEFAULT_BACKUP_INTERVAL:-1440}
-DEFAULT_BACKUP_INTERVAL=${DEFAULT_BACKUP_INTERVAL:-1440}
 DEFAULT_BACKUP_LOCATION=${DEFAULT_BACKUP_LOCATION:-"FILESYSTEM"}
 DEFAULT_BLOBXFER_REMOTE_PATH=${DEFAULT_BLOBXFER_REMOTE_PATH:-"/docker-db-backup"}
 DEFAULT_CHECKSUM=${DEFAULT_CHECKSUM:-"MD5"}
@@ -20,6 +19,7 @@ DEFAULT_FILESYSTEM_PATH_PERMISSION=${DEFAULT_FILESYSTEM_PATH_PERMISSION:-"700"}
 DEFAULT_FILESYSTEM_PERMISSION=${DEFAULT_FILESYSTEM_PERMISSION:-"600"}
 DEFAULT_FILESYSTEM_ARCHIVE_PATH=${DEFAULT_FILESYSTEM_ARCHIVE_PATH:-"${DEFAULT_FILESYSTEM_PATH}/archive/"}
 DEFAULT_LOG_LEVEL=${DEFAULT_LOG_LEVEL:-"notice"}
+DEFAULT_MYSQL_CLIENT=${DEFAULT_MYSQL_CLIENT:-"mariadb"}
 DEFAULT_MYSQL_ENABLE_TLS=${DEFAULT_MYSQL_ENABLE_TLS:-"FALSE"}
 DEFAULT_MYSQL_EVENTS=${DEFAULT_MYSQL_EVENTS:-"TRUE"}
 DEFAULT_MYSQL_MAX_ALLOWED_PACKET=${DEFAULT_MYSQL_MAX_ALLOWED_PACKET:-"512M"}

install/assets/functions/10-db-backup

@@ -221,6 +221,7 @@ bootstrap_variables() {
  transform_backup_instance_variable "${backup_instance_number}" INFLUX_VERSION backup_job_influx_version
  transform_backup_instance_variable "${backup_instance_number}" LOG_LEVEL backup_job_log_level
  transform_backup_instance_variable "${backup_instance_number}" MONGO_CUSTOM_URI backup_job_mongo_custom_uri
+ transform_backup_instance_variable "${backup_instance_number}" MYSQL_CLIENT backup_job_mysql_client
  transform_backup_instance_variable "${backup_instance_number}" MYSQL_ENABLE_TLS backup_job_mysql_enable_tls
  transform_backup_instance_variable "${backup_instance_number}" MYSQL_EVENTS backup_job_mysql_events
  transform_backup_instance_variable "${backup_instance_number}" MYSQL_MAX_ALLOWED_PACKET backup_job_mysql_max_allowed_packet
@@ -401,6 +402,20 @@ EOF
  dbtype=mysql
  backup_job_db_port=${backup_job_db_port:-3306}
  check_var backup_job_db_name DB"${v_instance}"_NAME "database name. Seperate multiple with commas"
+ case "${backup_job_mysql_client,,}" in
+ mariadb )
+ _mysql_prefix=/usr/bin/
+ ;;
+ mysql )
+ _mysql_prefix=/opt/mysql/bin/
+ ;;
+ * )
+ print_error "I don't understand '${backup_job_mysql_client,,}' as a client. Exiting.."
+ exit 99
+ ;;
+ esac
+
+ print_debug "Using '${backup_job_mysql_client,,}' as client"
 
  if [ -n "${backup_job_db_pass}" ] ; then export MYSQL_PWD=${backup_job_db_pass} ; fi
  if var_true "${backup_job_mysql_enable_tls}" ; then
@@ -665,7 +680,7 @@ backup_mysql() {
 
  if [ "${backup_job_db_name,,}" = "all" ] ; then
  write_log debug "Preparing to back up everything except for information_schema and _* prefixes"
- db_names=$(run_as_user mysql -h ${backup_job_db_host} -P ${backup_job_db_port} -u${backup_job_db_user} ${mysql_tls_args} ${backup_job_extra_opts} ${backup_job_extra_enumeration_opts} --batch -e "SHOW DATABASES;" | grep -v Database | grep -v schema )
+ db_names=$(run_as_user ${_mysql_prefix}mysql -h ${backup_job_db_host} -P ${backup_job_db_port} -u${backup_job_db_user} ${mysql_tls_args} ${backup_job_extra_opts} ${backup_job_extra_enumeration_opts} --batch -e "SHOW DATABASES;" | grep -v Database | grep -v schema )
  if [ -n "${backup_job_db_name_exclude}" ] ; then
  db_names_exclusions=$(echo "${backup_job_db_name_exclude}" | tr ',' '\n')
  for db_exclude in ${db_names_exclusions} ; do
@@ -682,13 +697,13 @@ backup_mysql() {
  if var_true "${backup_job_split_db}" ; then
  for db in ${db_names} ; do
  prepare_dbbackup
- backup_job_filename=mysql_${db}_${backup_job_db_host,,}_${now}.sql
- backup_job_filename_base=mysql_${db}_${backup_job_db_host,,}
+ backup_job_filename=${backup_job_mysql_client,,}_${db}_${backup_job_db_host,,}_${now}.sql
+ backup_job_filename_base=${backup_job_mysql_client,,}_${db}_${backup_job_db_host,,}
  compression
  pre_dbbackup "${db}"
  write_log notice "Dumping MySQL/MariaDB database: '${db}' ${compression_string}"
  if var_true "${DEBUG_BACKUP_MYSQL}" ; then debug on; fi
- run_as_user ${play_fair} mysqldump --max-allowed-packet=${backup_job_mysql_max_allowed_packet} -h ${backup_job_db_host} -P ${backup_job_db_port} -u${backup_job_db_user} ${events} ${single_transaction} ${stored_procedures} ${mysql_tls_args} ${backup_job_extra_opts} ${backup_job_extra_backup_opts} $db | ${compress_cmd} | run_as_user tee "${temporary_directory}"/"${backup_job_filename}" > /dev/null
+ run_as_user ${play_fair} ${_mysql_prefix}mysqldump --max-allowed-packet=${backup_job_mysql_max_allowed_packet} -h ${backup_job_db_host} -P ${backup_job_db_port} -u${backup_job_db_user} ${events} ${single_transaction} ${stored_procedures} ${mysql_tls_args} ${backup_job_extra_opts} ${backup_job_extra_backup_opts} $db | ${compress_cmd} | run_as_user tee "${temporary_directory}"/"${backup_job_filename}" > /dev/null
  exit_code=$?
  if var_true "${DEBUG_BACKUP_MYSQL}" ; then debug off; fi
  check_exit_code backup "${backup_job_filename}"
@@ -703,13 +718,13 @@ backup_mysql() {
  else
  write_log debug "Not splitting database dumps into their own files"
  prepare_dbbackup
- backup_job_filename=mysql_all_${backup_job_db_host,,}_${now}.sql
- backup_job_filename_base=mysql_all_${backup_job_db_host,,}
+ backup_job_filename=${backup_job_mysql_client,,}_all_${backup_job_db_host,,}_${now}.sql
+ backup_job_filename_base=${backup_job_mysql_client,,}_all_${backup_job_db_host,,}
  compression
  pre_dbbackup all
  write_log notice "Dumping all MySQL / MariaDB databases: '$(echo ${db_names} | xargs | tr ' ' ',')' ${compression_string}"
  if var_true "${DEBUG_BACKUP_MYSQL}" ; then debug on; fi
- run_as_user ${play_fair} mysqldump --max-allowed-packet=${backup_job_mysql_max_allowed_packet} -h ${backup_job_db_host} -P ${backup_job_db_port} -u${backup_job_db_user} ${events} ${single_transaction} ${stored_procedures} ${mysql_tls_args} ${backup_job_extra_opts} ${backup_job_extra_backup_opts} --databases $(echo ${db_names} | xargs) | ${compress_cmd} | run_as_user tee "${temporary_directory}"/"${backup_job_filename}" > /dev/null
+ run_as_user ${play_fair} ${_mysql_prefix}mysqldump --max-allowed-packet=${backup_job_mysql_max_allowed_packet} -h ${backup_job_db_host} -P ${backup_job_db_port} -u${backup_job_db_user} ${events} ${single_transaction} ${stored_procedures} ${mysql_tls_args} ${backup_job_extra_opts} ${backup_job_extra_backup_opts} --databases $(echo ${db_names} | xargs) | ${compress_cmd} | run_as_user tee "${temporary_directory}"/"${backup_job_filename}" > /dev/null
  exit_code=$?
  if var_true "${DEBUG_BACKUP_MYSQL}" ; then debug off; fi
  check_exit_code backup "${backup_job_filename}"