From d866d0fcaff69a0d2f41ff8d7ab3634515fb7965 Mon Sep 17 00:00:00 2001 From: Kay0u Date: Fri, 21 Jun 2024 20:06:39 +0200 Subject: [PATCH 1/6] add logging conf --- conf/logging.conf | 23 +++++++++++++++++++++++ scripts/install | 5 ++++- scripts/upgrade | 5 ++++- 3 files changed, 31 insertions(+), 2 deletions(-) create mode 100644 conf/logging.conf diff --git a/conf/logging.conf b/conf/logging.conf new file mode 100644 index 0000000..7739e4f --- /dev/null +++ b/conf/logging.conf @@ -0,0 +1,23 @@ +[loggers] +keys=root + +[handlers] +keys=logfile + +[formatters] +keys=logfile + +[logger_root] +level=NOTSET +handlers=logfile + +[handler_logfile] +class=FileHandler +level=INFO +formatter=logfile +args=('/var/log/__APP__/borg.log', 'a') + +[formatter_logfile] +format=%(asctime)s %(levelname)s %(message)s +datefmt= +class=logging.Formatter diff --git a/scripts/install b/scripts/install index 57da313..2adbf74 100755 --- a/scripts/install +++ b/scripts/install @@ -71,6 +71,9 @@ chown "$app:$app" "$install_dir/backup-with-borg" ynh_add_config --template="sudoer" --destination="/etc/sudoers.d/$app" chown root:root "/etc/sudoers.d/$app" +ynh_add_config --template="logging.conf" --destination="$install_dir/logging.conf" +chown "$app:$app" "$install_dir/logging.conf" + #================================================= # SYSTEM CONFIGURATION #================================================= @@ -78,7 +81,7 @@ ynh_script_progression --message="Adding system configurations related to $app.. # Create a dedicated systemd config ynh_add_systemd_config -yunohost service add $app --description="Deduplicating backup program" --test_status="systemctl show $app.service -p ActiveState --value | grep -v failed" +yunohost service add $app --description="Deduplicating backup program" --test_status="systemctl show $app.service -p ActiveState --value | grep -v failed" --log "/var/log/$app/borg.log" # Disable the service, this is to prevent the service from being triggered at boot time systemctl disable $app.service --quiet diff --git a/scripts/upgrade b/scripts/upgrade index 93a6039..1ddafe9 100755 --- a/scripts/upgrade +++ b/scripts/upgrade @@ -93,6 +93,9 @@ chown "$app:$app" "$install_dir/backup-with-borg" ynh_add_config --template="sudoer" --destination="/etc/sudoers.d/$app" chown root:root "/etc/sudoers.d/$app" +ynh_add_config --template="logging.conf" --destination="$install_dir/logging.conf" +chown "$app:$app" "$install_dir/logging.conf" + #================================================= # REAPPLY SYSTEM CONFIGURATIONS #================================================= @@ -100,7 +103,7 @@ ynh_script_progression --message="Upgrading system configurations related to $ap # Create a dedicated systemd config ynh_add_systemd_config -yunohost service add $app --description="Deduplicating backup program" --test_status="systemctl show $app.service -p ActiveState --value | grep -v failed" +yunohost service add $app --description="Deduplicating backup program" --test_status="systemctl show $app.service -p ActiveState --value | grep -v failed" --log "/var/log/$app/borg.log" # Disable the service, this is to prevent the service from being triggered at boot time systemctl disable $app.service --quiet From 9c033bf31f636016d8f0f8499de4c60c3f81b478 Mon Sep 17 00:00:00 2001 From: Kay0u Date: Fri, 21 Jun 2024 20:07:07 +0200 Subject: [PATCH 2/6] backup/restore backup-with-borg and logging files --- scripts/backup | 3 +++ scripts/restore | 7 +++++++ 2 files changed, 10 insertions(+) diff --git a/scripts/backup b/scripts/backup index 964f5d9..1a8036e 100755 --- a/scripts/backup +++ b/scripts/backup @@ -22,6 +22,9 @@ ynh_backup --src_path="/etc/sudoers.d/$app" ynh_backup --src_path="/root/.ssh/id_${app}_ed25519" --not_mandatory ynh_backup --src_path="/root/.ssh/id_${app}_ed25519.pub" --not_mandatory +ynh_backup --src_path="$install_dir/backup-with-borg" +ynh_backup --src_path="$install_dir/logging.conf" + #================================================= # END OF SCRIPT #================================================= diff --git a/scripts/restore b/scripts/restore index 7f7831c..4738f2c 100755 --- a/scripts/restore +++ b/scripts/restore @@ -37,6 +37,13 @@ chmod go=--- "/etc/yunohost/hooks.d/backup_method/05-${app}_app" ynh_restore_file --origin_path="/etc/sudoers.d/$app" chown root:root "/etc/sudoers.d/$app" +ynh_restore_file --origin_path="$install_dir/backup-with-borg" +chmod u+x "$install_dir/backup-with-borg" +chown "$app:$app" "$install_dir/backup-with-borg" + +ynh_restore_file --origin_path="$install_dir/logging.conf" +chown "$app:$app" "$install_dir/logging.conf" + #================================================= # RESTORE SYSTEM CONFIGURATIONS #================================================= From 014972b8109f35a911ec644484ef50bb0823c02c Mon Sep 17 00:00:00 2001 From: Kay0u Date: Fri, 21 Jun 2024 20:09:16 +0200 Subject: [PATCH 3/6] use env var BORG_REPO and BORG_LOGGING_CONF to manage repo and logs --- conf/backup_method | 46 ++++++++++++++++++++++------------------------ 1 file changed, 22 insertions(+), 24 deletions(-) diff --git a/conf/backup_method b/conf/backup_method index 0d9540f..c314628 100644 --- a/conf/backup_method +++ b/conf/backup_method @@ -5,7 +5,8 @@ borg="__INSTALL_DIR__/venv/bin/borg" app="__APP__" BORG_PASSPHRASE="$(yunohost app setting "$app" passphrase)" -repo="$(yunohost app setting "$app" repository)" #$4 +BORG_REPO="$(yunohost app setting "$app" repository)" +BORG_LOGGING_CONF="__INSTALL_DIR__/logging.conf" if ssh-keygen -F "__SERVER__" >/dev/null ; then BORG_RSH="ssh -i /root/.ssh/id_${app}_ed25519 -oStrictHostKeyChecking=yes " @@ -17,22 +18,16 @@ do_need_mount() { true } -LOGFILE=/var/log/backup_borg.err -log_with_timestamp() { - sed -e "s/^/[$(date +"%Y-%m-%d_%H:%M:%S")] /" | tee -a $LOGFILE -} - do_backup() { export BORG_PASSPHRASE + export BORG_REPO export BORG_RSH + export BORG_LOGGING_CONF export BORG_RELOCATED_REPO_ACCESS_IS_OK=yes work_dir="$1" name="$2" - repo="$3" - size="$4" - description="$5" - current_date=$(date +"%Y-%m-%d_%H:%M") - pushd "$work_dir" + size="$3" + description="$4" set +e if "$borg" init -e repokey "$repo" ; then # human_size=`echo $size | awk '{ suffix=" KMGT"; for(i=1; $1>1024 && i < length(suffix); i++) $1/=1024; print int($1) substr(suffix, i, 1), $3; }'` @@ -41,35 +36,38 @@ do_backup() { # evaluated_time=$(($size / ($speed * 1000 / 8) / 3600)) echo "Hello, -Your first backup on $repo is starting. +Your first backup on $BORG_REPO is starting. This is an automated message from your beloved YunoHost server." | /usr/bin/mail.mailutils -a "Content-Type: text/plain; charset=UTF-8" -s "[YNH] First backup is starting" "root" fi set -e - "$borg" create "$repo::_${name}-${current_date}" ./ 2>&1 >/dev/null | log_with_timestamp - popd + # About the {now} placeholder: + # https://borgbackup.readthedocs.io/en/stable/usage/create.html#description + # In the archive name, you may use the following placeholders: {now}, {utcnow}, {fqdn}, {hostname}, {user} and some others. + "$borg" create "::_${name}-{now}" "$work_dir" # About thi _20 it's a crazy fix to avoid pruning wordpress__2 # if you prune wordpress - "$borg" prune "$repo" -P "_${name}-" --keep-hourly 2 --keep-daily=7 --keep-weekly=8 --keep-monthly=12 2>&1 >/dev/null | log_with_timestamp + "$borg" prune -P "_${name}-" --keep-hourly 2 --keep-daily=7 --keep-weekly=8 --keep-monthly=12 # Prune legacy archive name without error on wordpress/wordpress__2 - "$borg" prune "$repo" -P "${name}_" --keep-within 2m --keep-monthly=12 2>&1 >/dev/null | log_with_timestamp + "$borg" prune -P "${name}_" --keep-within 2m --keep-monthly=12 # We prune potential manual backup older than 1 year - "$borg" prune "$repo" --keep-within 1y 2>&1 >/dev/null | log_with_timestamp + "$borg" prune --keep-within 1y } do_mount() { export BORG_PASSPHRASE + export BORG_REPO export BORG_RSH + export BORG_LOGGING_CONF work_dir="$1" name="$2" - repo="$3" - size="$4" - description="$5" - "$borg" mount "$repo::$name" "$work_dir" 2>&1 >/dev/null | log_with_timestamp + size="$3" + description="$4" + "$borg" mount "::$name" "$work_dir" } work_dir="$2" @@ -80,13 +78,13 @@ description="$6" case "$1" in need_mount) - do_need_mount "$work_dir" "$name" "$repo" "$size" "$description" + do_need_mount "$work_dir" "$name" "$size" "$description" ;; backup) - do_backup "$work_dir" "$name" "$repo" "$size" "$description" + do_backup "$work_dir" "$name" "$size" "$description" ;; mount) - do_mount "$work_dir" "$name" "$repo" "$size" "$description" + do_mount "$work_dir" "$name" "$size" "$description" ;; *) echo "hook called with unknown argument \`$1'" >&2 From d3f58d01d01907fa6701f148f38477c9cf420e97 Mon Sep 17 00:00:00 2001 From: Kay0u Date: Fri, 21 Jun 2024 20:12:09 +0200 Subject: [PATCH 4/6] avoid an error about the fact that the repo already exists (I assume that getting the config of a repo is fast, but maybe not?) --- conf/backup_method | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/conf/backup_method b/conf/backup_method index c314628..9103a6e 100644 --- a/conf/backup_method +++ b/conf/backup_method @@ -29,7 +29,8 @@ do_backup() { size="$3" description="$4" set +e - if "$borg" init -e repokey "$repo" ; then + if ! "$borg" config -l > /dev/null 2>&1; then + "$borg" init -e repokey # human_size=`echo $size | awk '{ suffix=" KMGT"; for(i=1; $1>1024 && i < length(suffix); i++) $1/=1024; print int($1) substr(suffix, i, 1), $3; }'` # Speed in Kbps # speed=1000 From 7d36895fcbfcde86b2d10f55ed51b6a271b278d9 Mon Sep 17 00:00:00 2001 From: Kay0u Date: Fri, 21 Jun 2024 20:53:29 +0200 Subject: [PATCH 5/6] add more info in logs --- conf/backup_method | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/conf/backup_method b/conf/backup_method index 9103a6e..10543ea 100644 --- a/conf/backup_method +++ b/conf/backup_method @@ -46,17 +46,17 @@ This is an automated message from your beloved YunoHost server." | /usr/bin/mail # About the {now} placeholder: # https://borgbackup.readthedocs.io/en/stable/usage/create.html#description # In the archive name, you may use the following placeholders: {now}, {utcnow}, {fqdn}, {hostname}, {user} and some others. - "$borg" create "::_${name}-{now}" "$work_dir" + "$borg" create --stats "::_${name}-{now}" "$work_dir" # About thi _20 it's a crazy fix to avoid pruning wordpress__2 # if you prune wordpress - "$borg" prune -P "_${name}-" --keep-hourly 2 --keep-daily=7 --keep-weekly=8 --keep-monthly=12 + "$borg" prune -P "_${name}-" --list --keep-hourly 2 --keep-daily=7 --keep-weekly=8 --keep-monthly=12 # Prune legacy archive name without error on wordpress/wordpress__2 - "$borg" prune -P "${name}_" --keep-within 2m --keep-monthly=12 + "$borg" prune -P "${name}_" --list --keep-within 2m --keep-monthly=12 # We prune potential manual backup older than 1 year - "$borg" prune --keep-within 1y + "$borg" prune --list --keep-within 1y } do_mount() { From 4e15f55b9bda8d84e51aaa59d32a187ac7ff4cdb Mon Sep 17 00:00:00 2001 From: Kay0u Date: Fri, 21 Jun 2024 21:11:16 +0200 Subject: [PATCH 6/6] prune -P is deprecated, use --glob-archives --- conf/backup_method | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/conf/backup_method b/conf/backup_method index 10543ea..2efb46c 100644 --- a/conf/backup_method +++ b/conf/backup_method @@ -46,14 +46,9 @@ This is an automated message from your beloved YunoHost server." | /usr/bin/mail # About the {now} placeholder: # https://borgbackup.readthedocs.io/en/stable/usage/create.html#description # In the archive name, you may use the following placeholders: {now}, {utcnow}, {fqdn}, {hostname}, {user} and some others. - "$borg" create --stats "::_${name}-{now}" "$work_dir" + "$borg" create --stats "::${name}-{now}" "$work_dir" - # About thi _20 it's a crazy fix to avoid pruning wordpress__2 - # if you prune wordpress - "$borg" prune -P "_${name}-" --list --keep-hourly 2 --keep-daily=7 --keep-weekly=8 --keep-monthly=12 - - # Prune legacy archive name without error on wordpress/wordpress__2 - "$borg" prune -P "${name}_" --list --keep-within 2m --keep-monthly=12 + "$borg" prune --glob-archives "${name}-*" --list --keep-hourly 2 --keep-daily=7 --keep-weekly=8 --keep-monthly=12 # We prune potential manual backup older than 1 year "$borg" prune --list --keep-within 1y