update to xenial

Dockerfile

@@ -1,9 +1,48 @@
-FROM ubuntu:trusty
+FROM ubuntu:xenial
 MAINTAINER Tutum Labs <support@tutum.co>
 
-RUN apt-get update && \
-    apt-get install -y --no-install-recommends mysql-client && \
-    mkdir /backup
+# add our user and group first to make sure their IDs get assigned consistently, regardless of whatever dependencies get added
+RUN mkdir -p /var/lib/mysql/ \
+	&& groupadd -r mysql \
+	&& useradd -r -g mysql -d /var/lib/mysql/ mysql \
+	&& chown mysql:mysql /var/lib/mysql \
+	&& chmod 700 /var/lib/mysql
+
+# add gosu for easy step-down from root
+ENV GOSU_VERSION 1.7
+RUN set -x \
+	&& apt-get update && apt-get install -y --no-install-recommends ca-certificates wget && rm -rf /var/lib/apt/lists/* \
+	&& wget -O /usr/local/bin/gosu "https://github.com/tianon/gosu/releases/download/$GOSU_VERSION/gosu-$(dpkg --print-architecture)" \
+	&& wget -O /usr/local/bin/gosu.asc "https://github.com/tianon/gosu/releases/download/$GOSU_VERSION/gosu-$(dpkg --print-architecture).asc" \
+	&& export GNUPGHOME="$(mktemp -d)" \
+	&& gpg --keyserver ha.pool.sks-keyservers.net --recv-keys B42F6819007F00F88E364FD4036A9C25BF357DD4 \
+	&& gpg --batch --verify /usr/local/bin/gosu.asc /usr/local/bin/gosu \
+	&& rm -r "$GNUPGHOME" /usr/local/bin/gosu.asc \
+	&& chmod +x /usr/local/bin/gosu \
+	&& gosu nobody true \
+	&& apt-get purge -y --auto-remove ca-certificates wget
+
+# gpg: key 5072E1F5: public key "MySQL Release Engineering <mysql-build@oss.oracle.com>" imported
+RUN apt-key adv --keyserver ha.pool.sks-keyservers.net --recv-keys A4A9406876FCBD3C456770C88C718D3B5072E1F5
+
+ENV MYSQL_MAJOR 5.7
+ENV MYSQL_VERSION 5.7.17-1ubuntu16.04
+
+RUN echo "deb http://repo.mysql.com/apt/ubuntu/ xenial mysql-${MYSQL_MAJOR}" > /etc/apt/sources.list.d/mysql.list
+
+RUN apt-get update \
+	&& apt-get install -y \
+		mysql-client="${MYSQL_VERSION}" \
+		cron \
+		openssh-client \
+		python-paramiko \
+		python-pexpect \
+		duplicity \
+		python \
+	&& rm -rf /var/lib/apt/lists/*
+
+# timezone
+ENV TIMEZONE="Etc/UTC"
 
 ENV CRON_TIME="0 0 * * *" \
     MYSQL_DB="--all-databases"

README.md

@@ -9,7 +9,8 @@ This image runs mysqldump to backup data using cronjob to folder `/backup`
         --env MYSQL_PORT=27017 \
         --env MYSQL_USER=admin \
         --env MYSQL_PASS=password \
-        --volume host.folder:/backup
+        --volume host.folder:/backup \
+        --name tutum-backup \
         tutum/mysql-backup
 
 Moreover, if you link `tutum/mysql-backup` to a mysql container(e.g. `tutum/mysql`) with an alias named mysql, this image will try to auto load the `host`, `port`, `user`, `pass` if possible.
@@ -19,6 +20,7 @@ Moreover, if you link `tutum/mysql-backup` to a mysql container(e.g. `tutum/mysq
 
 ## Parameters
 
+    TIMEZONE        e.g. Europe/Moscow
     MYSQL_HOST      the host/ip of your mysql database
     MYSQL_PORT      the port number of your mysql database
     MYSQL_USER      the username of your mysql database
@@ -30,12 +32,95 @@ Moreover, if you link `tutum/mysql-backup` to a mysql container(e.g. `tutum/mysq
     INIT_BACKUP     if set, create a backup when the container starts
     INIT_RESTORE_LATEST if set, restores latest backup
 
+## Duplicity over sftp parameters
+
+	SFTP_USER                        user to connect over sftp
+	SFTP_HOST                        host to connect to by sftp
+	SFTP_PORT                        port to connect to by sftp
+	SFTP_DIR                         remote directory to place files over sftp
+	DUPLICITY_EXTRA_OPTS             usefull value: --full-if-older-than 1M --allow-source-mismatch
+	DUPLICITY_ENCRYPT_PASSPHRASE     the encryption passphrase. keep it in a secret
+	DUPLICITY_SCHEME                 the scheme for duplicity to work with backup server. the default value is sftp. pexpect+sftp is also supported.
+
+## Usage with duplicity over sftp
+
+    mkdir -p /root/.cache/duplicity
+    docker run -d \
+        --env MYSQL_HOST=mysql.host \
+        --env MYSQL_PORT=27017 \
+        --env MYSQL_USER=admin \
+        --env MYSQL_PASS=password \
+        --env MYSQL_DB=mydatabase \
+        --env EXTRA_OPTS=--skip-lock-tables --single-transaction --flush-logs --hex-blob --master-data=2 \
+        --env CRON_TIME=0 6 * * * \
+        --env INIT_BACKUP=1 \
+        --env MAX_BACKUPS=30 \
+        --env TIMEZONE=Europe/Moscow \
+        --env SFTP_USER=username \
+        --env SFTP_HOST=12.34.56.78 \
+        --env SFTP_PORT=22 \
+        --env SFTP_DIR=backup/ \
+        --env DUPLICITY_EXTRA_OPTS=--full-if-older-than 1M --allow-source-mismatch \
+        --env DUPLICITY_ENCRYPT_PASSPHRASE=12345676543212345676543234567654 \
+        --volume /backup:/backup \
+        --volume /restore:/restore \
+        --volume /root/.ssh:/root/.ssh \
+        --volume /root/.cache/duplicity:/root/.cache/duplicity \
+        --name tutum-backup \
+        tutum/mysql-backup
+
+You need to connect to the backup server at least once from your host system in order to have a valid record for it in the known_hosts file.
+
+On your host machine run this command and press Enter for all questions:
+
+    ssh-keygen
+
+It would generate `/root/.ssh/id_rsa` and `/root/.ssh/id_rsa.pub` files.
+
+Then copy this pub key to the backup server. Be sure to allow password access first.
+
+    cd /root/.ssh
+    ssh-copy-id -p 22 -i id_rsa.pub username@12.34.56.78
+
+> Not forget to disallow password authentication on the backup server after pub key copying.
+
+It would ask for your username's password. if this command complete, try to connect:
+
+    sftp -P 22 username@12.34.56.78
+
+If this command succeeds and have not asked you for password, then you can be sure this image would function too.
+
+Use this [Guide](www.jscape.com/blog/setting-up-sftp-public-key-authentication-caommand-line) if you need more details about sftp configuration.
+
 ## Restore from a backup
 
 See the list of backups, you can run:
 
     docker exec tutum-backup ls /backup
+
+> Note that `tutum-backup` is a docker container name assigned previously with `--name` option.
 
 To restore database from a certain backup, simply run:
 
     docker exec tutum-backup /restore.sh /backup/2015.08.06.171901
+
+## Restore from a backup if using duplicity
+
+To restore database from a yesterday backup, simply run:
+
+    docker exec tutum-backup /restore.sh 1D
+
+If you want restore only sql dump file without replacing actual database state, do it with:
+
+    docker exec tutum-backup /restore-file-only.sh 1D
+
+if you have used same options for backup container startup as in the example above, 
+after executing it you would find file named /restore/mydatabase-1D.sql restored.
+
+Instead of 1D you can use 1h for hours, 15m for minutes, 1W for weeks or 1Y for years.
+You can use an exact date in a format like YYYY/MM/DD. See the full list of formats there:
+[Duplicity Time Formats](http://duplicity.nongnu.org/duplicity.1.html#sect8)
+
+If you have used your backup server for backup from several servers, e.g. `master` and `slave`, and defined the environment variable `SFTP_DIR` as `backup/1d` on `master` and to `backup/15m` on `slave`, you can use this comand to restore master backup on a slave:
+
+     docker exec tutum-backup /bin/bash -c "export SFTP_DIR=backup/1d && export MYSQL_USER=root && export MYSQL_PASS=123456 && /restore.sh 1D"

run.sh

@@ -16,32 +16,73 @@ MYSQL_PASS=${MYSQL_PASS:-${MYSQL_ENV_MYSQL_PASS}}
 [ -z "${MYSQL_USER}" ] && { echo "=> MYSQL_USER cannot be empty" && exit 1; }
 [ -z "${MYSQL_PASS}" ] && { echo "=> MYSQL_PASS cannot be empty" && exit 1; }
 
-BACKUP_CMD="mysqldump -h${MYSQL_HOST} -P${MYSQL_PORT} -u${MYSQL_USER} -p${MYSQL_PASS} ${EXTRA_OPTS} ${MYSQL_DB} > /backup/"'${BACKUP_NAME}'
+echo ${TIMEZONE} > /etc/timezone
+ln -sf /usr/share/zoneinfo/${TIMEZONE} /etc/localtime
 
 echo "=> Creating backup script"
 rm -f /backup.sh
 cat <<EOF >> /backup.sh
 #!/bin/bash
+
+# Setting the pass phrase to encrypt the backup files.
+export PASSPHRASE=$DUPLICITY_ENCRYPT_PASSPHRASE
+
 MAX_BACKUPS=${MAX_BACKUPS}
 
-BACKUP_NAME=\$(date +\%Y.\%m.\%d.\%H\%M\%S).sql
+BACKUP_NAME_NOEXT=\$(date +\%Y.\%m.\%d.\%H\%M\%S)
+BACKUP_GZ_NAME=\${BACKUP_NAME_NOEXT}.gz
+
+SFTP_USER=${SFTP_USER}
+SFTP_HOST=${SFTP_HOST}
+SFTP_PORT=${SFTP_PORT}
+SFTP_DIR="${SFTP_DIR}"
+DUPLICITY_EXTRA_OPTS="${DUPLICITY_EXTRA_OPTS}"
+DUPLICITY_SCHEME="${DUPLICITY_SCHEME:-sftp}"
+
+MYSQL_HOST=${MYSQL_HOST}
+MYSQL_PORT=${MYSQL_PORT}
+MYSQL_USER=${MYSQL_USER}
+MYSQL_PASS="${MYSQL_PASS}"
+MYSQL_DB=${MYSQL_DB}
+EXTRA_OPTS="${EXTRA_OPTS}"
 
-echo "=> Backup started: \${BACKUP_NAME}"
-if ${BACKUP_CMD} ;then
-    echo "   Backup succeeded"
+if [[ ! -z \${SFTP_USER} && ! -z \${SFTP_HOST} && ! -z \${SFTP_DIR} ]]; then
+	echo "=> Backup started: \${MYSQL_DB}.sql"
+
+	# using pexpect+sftp because of a bug in paramiko backend. 
+	# @see https://lists.gnu.org/archive/html/duplicity-talk/2016-10/msg00010.html
+	if flock -x -n /root/.cache/duplicity/backup.lock -c "/usr/local/bin/gosu mysql mysqldump -h\${MYSQL_HOST} -P\${MYSQL_PORT} -u\${MYSQL_USER} -p\${MYSQL_PASS} \${EXTRA_OPTS} \${MYSQL_DB} > /backup/\${MYSQL_DB}.sql && duplicity --ssh-options=\"-oProtocol=2 -oIdentityFile=/root/.ssh/id_rsa\" \${DUPLICITY_EXTRA_OPTS} /backup \${DUPLICITY_SCHEME}://\${SFTP_USER}@\${SFTP_HOST}:\${SFTP_PORT}/\${SFTP_DIR}" ;then
+		echo "   Backup succeeded"
+	else
+		echo "   Backup failed"
+	fi
+
+	if [ -n "\${MAX_BACKUPS}" ]; then
+		if flock -x -n /root/.cache/duplicity/backup.lock -c "duplicity remove-older-than \${MAX_BACKUPS} --force --ssh-options=\"-oProtocol=2 -oIdentityFile=/root/.ssh/id_rsa\" \${DUPLICITY_SCHEME}://\${SFTP_USER}@\${SFTP_HOST}:\${SFTP_PORT}/\${SFTP_DIR}" ;then
+			echo "   Backup succeeded"
+		else
+			echo "   Backup failed"
+		fi
+	fi
 else
-    echo "   Backup failed"
-    rm -rf /backup/\${BACKUP_NAME}
-fi
+	echo "=> Backup started: \${BACKUP_GZ_NAME}"
 
-if [ -n "\${MAX_BACKUPS}" ]; then
-    while [ \$(ls /backup -N1 | wc -l) -gt \${MAX_BACKUPS} ];
-    do
-        BACKUP_TO_BE_DELETED=\$(ls /backup -N1 | sort | head -n 1)
-        echo "   Backup \${BACKUP_TO_BE_DELETED} is deleted"
-        rm -rf /backup/\${BACKUP_TO_BE_DELETED}
-    done
+	if flock -x -n /root/.cache/duplicity/backup.lock -c "exec /usr/local/bin/gosu mysql mysqldump -h\${MYSQL_HOST} -P\${MYSQL_PORT} -u\${MYSQL_USER} -p\${MYSQL_PASS} \${EXTRA_OPTS} \${MYSQL_DB} | gzip -c -9 > /backup/\${BACKUP_GZ_NAME}" ;then
+		echo "   Backup succeeded"
+	else
+		echo "   Backup failed"
+	fi
+
+	if [ -n "\${MAX_BACKUPS}" ]; then
+		while [ \$(ls /backup -N1 | wc -l) -gt \${MAX_BACKUPS} ];
+		do
+			BACKUP_TO_BE_DELETED=\$(ls /backup -N1 | sort | head -n 1)
+			echo "   Backup \${BACKUP_TO_BE_DELETED} is deleted"
+			rm -rf /backup/\${BACKUP_TO_BE_DELETED}
+		done
+	fi
 fi
+
 echo "=> Backup done"
 EOF
 chmod +x /backup.sh
@@ -50,16 +91,77 @@ echo "=> Creating restore script"
 rm -f /restore.sh
 cat <<EOF >> /restore.sh
 #!/bin/bash
-echo "=> Restore database from \$1"
-if mysql -h${MYSQL_HOST} -P${MYSQL_PORT} -u${MYSQL_USER} -p${MYSQL_PASS} < \$1 ;then
-    echo "   Restore succeeded"
+
+# Setting the pass phrase to encrypt the backup files.
+export PASSPHRASE=\$DUPLICITY_ENCRYPT_PASSPHRASE
+
+SFTP_USER=${SFTP_USER}
+SFTP_HOST=${SFTP_HOST}
+SFTP_PORT=${SFTP_PORT}
+SFTP_DIR="${SFTP_DIR}"
+DUPLICITY_SCHEME="${DUPLICITY_SCHEME:-sftp}"
+
+MYSQL_HOST=${MYSQL_HOST}
+MYSQL_PORT=${MYSQL_PORT}
+MYSQL_USER=${MYSQL_USER}
+MYSQL_PASS="${MYSQL_PASS}"
+MYSQL_DB=${MYSQL_DB}
+
+if [[ ! -z \${SFTP_USER} && ! -z \${SFTP_HOST} && ! -z \${SFTP_DIR} ]]; then
+	# using pexpect+sftp because of a bug in paramiko backend. 
+	# @see https://lists.gnu.org/archive/html/duplicity-talk/2016-10/msg00010.html
+	if duplicity --allow-source-mismatch --ssh-options="-oProtocol=2 -oIdentityFile=/root/.ssh/id_rsa" -t \${1} --file-to-restore \${MYSQL_DB}.sql \${DUPLICITY_SCHEME}://\${SFTP_USER}@\${SFTP_HOST}:\${SFTP_PORT}/\${SFTP_DIR} /restore/\${MYSQL_DB}-\${1}.sql && gosu mysql mysql -h\${MYSQL_HOST} -P\${MYSQL_PORT} -u\${MYSQL_USER} -p\${MYSQL_PASS} < /restore/\${MYSQL_DB}-\${1}.sql ;then
+		echo "   Restore succeeded"
+	else
+		echo "   Restore failed"
+	fi
 else
-    echo "   Restore failed"
+	echo "=> Restore database from \$1"
+	if gunzip -c \$1 | exec gosu mysql mysql -h\${MYSQL_HOST} -P\${MYSQL_PORT} -u\${MYSQL_USER} -p\${MYSQL_PASS} ;then
+		echo "   Restore succeeded"
+	else
+		echo "   Restore failed"
+	fi
 fi
 echo "=> Done"
 EOF
 chmod +x /restore.sh
 
+rm -f /restore-file-only.sh
+cat <<EOF >> /restore-file-only.sh
+#!/bin/bash
+
+# Setting the pass phrase to encrypt the backup files.
+export PASSPHRASE=\$DUPLICITY_ENCRYPT_PASSPHRASE
+
+SFTP_USER=${SFTP_USER}
+SFTP_HOST=${SFTP_HOST}
+SFTP_PORT=${SFTP_PORT}
+SFTP_DIR="${SFTP_DIR}"
+DUPLICITY_SCHEME="${DUPLICITY_SCHEME:-sftp}"
+
+MYSQL_DB=${MYSQL_DB}
+
+if [[ ! -z \${SFTP_USER} && ! -z \${SFTP_HOST} && ! -z \${SFTP_DIR} ]]; then
+	# using pexpect+sftp because of a bug in paramiko backend. 
+	# @see https://lists.gnu.org/archive/html/duplicity-talk/2016-10/msg00010.html
+	if duplicity --allow-source-mismatch --ssh-options="-oProtocol=2 -oIdentityFile=/root/.ssh/id_rsa" -t \${1} --file-to-restore \${MYSQL_DB}.sql ${DUPLICITY_SCHEME}://\${SFTP_USER}@\${SFTP_HOST}:\${SFTP_PORT}/\${SFTP_DIR} /restore/\${MYSQL_DB}-\${1}.sql ;then
+		echo "   Restore succeeded"
+	else
+		echo "   Restore failed"
+	fi
+else
+	echo "=> Restore database from \$1"
+	if gunzip -c \$1 > /restore/\${1}.sql ;then
+		echo "   Restore succeeded"
+	else
+		echo "   Restore failed"
+	fi
+fi
+echo "=> Done"
+EOF
+chmod +x /restore-file-only.sh
+
 touch /mysql_backup.log
 tail -F /mysql_backup.log &