Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Misc improvements #163

Merged
merged 6 commits into from
Jun 25, 2024
Merged

Misc improvements #163

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
d866d0f
add logging conf
kay0u Jun 21, 2024
9c033bf
backup/restore backup-with-borg and logging files
kay0u Jun 21, 2024
014972b
use env var BORG_REPO and BORG_LOGGING_CONF to manage repo and logs
kay0u Jun 21, 2024
d3f58d0
avoid an error about the fact that the repo already exists (I assume …
kay0u Jun 21, 2024
7d36895
add more info in logs
kay0u Jun 21, 2024
4e15f55
prune -P is deprecated, use --glob-archives
kay0u Jun 21, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
52 changes: 23 additions & 29 deletions conf/backup_method
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,8 @@ borg="__INSTALL_DIR__/venv/bin/borg"
app="__APP__"

BORG_PASSPHRASE="$(yunohost app setting "$app" passphrase)"
repo="$(yunohost app setting "$app" repository)" #$4
BORG_REPO="$(yunohost app setting "$app" repository)"
BORG_LOGGING_CONF="__INSTALL_DIR__/logging.conf"

if ssh-keygen -F "__SERVER__" >/dev/null ; then
BORG_RSH="ssh -i /root/.ssh/id_${app}_ed25519 -oStrictHostKeyChecking=yes "
Expand All @@ -17,59 +18,52 @@ do_need_mount() {
true
}

LOGFILE=/var/log/backup_borg.err
log_with_timestamp() {
sed -e "s/^/[$(date +"%Y-%m-%d_%H:%M:%S")] /" | tee -a $LOGFILE
}

do_backup() {
export BORG_PASSPHRASE
export BORG_REPO
export BORG_RSH
export BORG_LOGGING_CONF
export BORG_RELOCATED_REPO_ACCESS_IS_OK=yes
work_dir="$1"
name="$2"
repo="$3"
size="$4"
description="$5"
current_date=$(date +"%Y-%m-%d_%H:%M")
pushd "$work_dir"
size="$3"
description="$4"
set +e
if "$borg" init -e repokey "$repo" ; then
if ! "$borg" config -l > /dev/null 2>&1; then
"$borg" init -e repokey
# human_size=`echo $size | awk '{ suffix=" KMGT"; for(i=1; $1>1024 && i < length(suffix); i++) $1/=1024; print int($1) substr(suffix, i, 1), $3; }'`
# Speed in Kbps
# speed=1000
# evaluated_time=$(($size / ($speed * 1000 / 8) / 3600))
echo "Hello,

Your first backup on $repo is starting.
Your first backup on $BORG_REPO is starting.

This is an automated message from your beloved YunoHost server." | /usr/bin/mail.mailutils -a "Content-Type: text/plain; charset=UTF-8" -s "[YNH] First backup is starting" "root"
fi
set -e

"$borg" create "$repo::_${name}-${current_date}" ./ 2>&1 >/dev/null | log_with_timestamp
popd

# About thi _20 it's a crazy fix to avoid pruning wordpress__2
# if you prune wordpress
"$borg" prune "$repo" -P "_${name}-" --keep-hourly 2 --keep-daily=7 --keep-weekly=8 --keep-monthly=12 2>&1 >/dev/null | log_with_timestamp
# About the {now} placeholder:
# https://borgbackup.readthedocs.io/en/stable/usage/create.html#description
# In the archive name, you may use the following placeholders: {now}, {utcnow}, {fqdn}, {hostname}, {user} and some others.
"$borg" create --stats "::${name}-{now}" "$work_dir"

# Prune legacy archive name without error on wordpress/wordpress__2
"$borg" prune "$repo" -P "${name}_" --keep-within 2m --keep-monthly=12 2>&1 >/dev/null | log_with_timestamp
"$borg" prune --glob-archives "${name}-*" --list --keep-hourly 2 --keep-daily=7 --keep-weekly=8 --keep-monthly=12

# We prune potential manual backup older than 1 year
"$borg" prune "$repo" --keep-within 1y 2>&1 >/dev/null | log_with_timestamp
"$borg" prune --list --keep-within 1y
Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@kay0u I'm not very fond of pruning without the user being informed ?
Shouldn't this "1y" be a setting in the config panel, set to "never" by default ?

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This code was already there, it's a discussion for another PR. To me it's very reasonable to have pruning by default because you don't want backup to just pile up and eat space forever. 1 year is a reasonable value ... Most of the time when you need to restore a backup, you need one from say a few days ago ...

Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I understand your point. It should at least be mentionned somewhere in the post-install notes for instance.
I'll try to see if I can work on this small matter.

}

do_mount() {
export BORG_PASSPHRASE
export BORG_REPO
export BORG_RSH
export BORG_LOGGING_CONF
work_dir="$1"
name="$2"
repo="$3"
size="$4"
description="$5"
"$borg" mount "$repo::$name" "$work_dir" 2>&1 >/dev/null | log_with_timestamp
size="$3"
description="$4"
"$borg" mount "::$name" "$work_dir"
}

work_dir="$2"
Expand All @@ -80,13 +74,13 @@ description="$6"

case "$1" in
need_mount)
do_need_mount "$work_dir" "$name" "$repo" "$size" "$description"
do_need_mount "$work_dir" "$name" "$size" "$description"
;;
backup)
do_backup "$work_dir" "$name" "$repo" "$size" "$description"
do_backup "$work_dir" "$name" "$size" "$description"
;;
mount)
do_mount "$work_dir" "$name" "$repo" "$size" "$description"
do_mount "$work_dir" "$name" "$size" "$description"
;;
*)
echo "hook called with unknown argument \`$1'" >&2
Expand Down
23 changes: 23 additions & 0 deletions conf/logging.conf
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,23 @@
[loggers]
keys=root

[handlers]
keys=logfile

[formatters]
keys=logfile

[logger_root]
level=NOTSET
handlers=logfile

[handler_logfile]
class=FileHandler
level=INFO
formatter=logfile
args=('/var/log/__APP__/borg.log', 'a')

[formatter_logfile]
format=%(asctime)s %(levelname)s %(message)s
datefmt=
class=logging.Formatter
3 changes: 3 additions & 0 deletions scripts/backup
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,9 @@ ynh_backup --src_path="/etc/sudoers.d/$app"
ynh_backup --src_path="/root/.ssh/id_${app}_ed25519" --not_mandatory
ynh_backup --src_path="/root/.ssh/id_${app}_ed25519.pub" --not_mandatory

ynh_backup --src_path="$install_dir/backup-with-borg"
ynh_backup --src_path="$install_dir/logging.conf"

#=================================================
# END OF SCRIPT
#=================================================
Expand Down
5 changes: 4 additions & 1 deletion scripts/install
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -71,14 +71,17 @@ chown "$app:$app" "$install_dir/backup-with-borg"
ynh_add_config --template="sudoer" --destination="/etc/sudoers.d/$app"
chown root:root "/etc/sudoers.d/$app"

ynh_add_config --template="logging.conf" --destination="$install_dir/logging.conf"
chown "$app:$app" "$install_dir/logging.conf"

#=================================================
# SYSTEM CONFIGURATION
#=================================================
ynh_script_progression --message="Adding system configurations related to $app..." --weight=1

# Create a dedicated systemd config
ynh_add_systemd_config
yunohost service add $app --description="Deduplicating backup program" --test_status="systemctl show $app.service -p ActiveState --value | grep -v failed"
yunohost service add $app --description="Deduplicating backup program" --test_status="systemctl show $app.service -p ActiveState --value | grep -v failed" --log "/var/log/$app/borg.log"
# Disable the service, this is to prevent the service from being triggered at boot time
systemctl disable $app.service --quiet

Expand Down
7 changes: 7 additions & 0 deletions scripts/restore
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,13 @@ chmod go=--- "/etc/yunohost/hooks.d/backup_method/05-${app}_app"
ynh_restore_file --origin_path="/etc/sudoers.d/$app"
chown root:root "/etc/sudoers.d/$app"

ynh_restore_file --origin_path="$install_dir/backup-with-borg"
chmod u+x "$install_dir/backup-with-borg"
chown "$app:$app" "$install_dir/backup-with-borg"

ynh_restore_file --origin_path="$install_dir/logging.conf"
chown "$app:$app" "$install_dir/logging.conf"

#=================================================
# RESTORE SYSTEM CONFIGURATIONS
#=================================================
Expand Down
5 changes: 4 additions & 1 deletion scripts/upgrade
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -93,14 +93,17 @@ chown "$app:$app" "$install_dir/backup-with-borg"
ynh_add_config --template="sudoer" --destination="/etc/sudoers.d/$app"
chown root:root "/etc/sudoers.d/$app"

ynh_add_config --template="logging.conf" --destination="$install_dir/logging.conf"
chown "$app:$app" "$install_dir/logging.conf"

#=================================================
# REAPPLY SYSTEM CONFIGURATIONS
#=================================================
ynh_script_progression --message="Upgrading system configurations related to $app..." --weight=1

# Create a dedicated systemd config
ynh_add_systemd_config
yunohost service add $app --description="Deduplicating backup program" --test_status="systemctl show $app.service -p ActiveState --value | grep -v failed"
yunohost service add $app --description="Deduplicating backup program" --test_status="systemctl show $app.service -p ActiveState --value | grep -v failed" --log "/var/log/$app/borg.log"
# Disable the service, this is to prevent the service from being triggered at boot time
systemctl disable $app.service --quiet

Expand Down