Skip to content

Commit

Permalink
Hook scripts (#14)
Browse files Browse the repository at this point in the history
* Hook scripts
* #14 Fix typo
  • Loading branch information
melchor629 authored May 17, 2019
1 parent 94497e6 commit 439def4
Show file tree
Hide file tree
Showing 6 changed files with 183 additions and 2 deletions.
20 changes: 19 additions & 1 deletion README.md
Original file line number Diff line number Diff line change
Expand Up @@ -142,7 +142,18 @@ This allows you to auto-complete with the elements available in the configuratio
"bucket": "Name of the bucket",
"password": "(optional) Protects files with passwords"
}
]
],
"hooks": {
"backup:before": "echo $@",
"backup:after": "path/to/script",
"backup:error": "wombo combo $1 $2",
"upload:before": "echo $@",
"upload:after": "echo $@",
"upload:error": "echo $@",
"oldBackup:deleting": "echo $@",
"oldBackup:deleted": "echo $@",
"oldBackup:error": "echo $@"
}
}
```

Expand Down Expand Up @@ -211,6 +222,13 @@ Environment variables are replaced by their values from the path in the KV. As e

For cloud storage providers, the KV in the path should contain the same structure as expected in the provider configuration (as seen in the example json). In this case, no key must be defined, it will take the whole path as configuration.


### Hooks

Hooks are scripts that run when some event is going to happen or just happened. Is useful to define extend the tool with your custom scripts, including one-liner scrips. The hook is run with `sh` so scripts can be defined inlined. If a hook is not defined, won't run anything.

The output of the script is redirected to the logger using the `DEBUG` level. If you have some troubles with your hook script, set the log level to `DEBUG`.

## Creating your first steps

The steps are kept in `steps` folder and must be shell scripts. They will be run in alphabetical order one by one, but these scripts are not full scripts. The utility prepares the script with some environment variables and some functions that will be available in the step script. By default, there's some functions (described later) available, but you can add new functions by defining your own script and adding its path to `customUtilsScript` setting.
Expand Down
52 changes: 52 additions & 0 deletions config/config.schema.json
Original file line number Diff line number Diff line change
Expand Up @@ -388,6 +388,58 @@
}
}
}
},
"hooks": {
"$id": "#/properties/hooks",
"type": "object",
"title": "Run scripts or one-liner scripts before or after things occur",
"properties": {
"backup:before": {
"$id": "#/properties/hooks/properties/backup:before",
"type": "string",
"title": "Run the script when the backup is going to be done"
},
"backup:after": {
"$id": "#/properties/hooks/properties/backup:after",
"type": "string",
"title": "Run the script when the backup is done, but before uploading it"
},
"backup:error": {
"$id": "#/properties/hooks/properties/backup:error",
"type": "string",
"title": "Run the script when the backup has failed for some reason"
},
"upload:before": {
"$id": "#/properties/hooks/properties/upload:before",
"type": "string",
"title": "Run the script when the backup is going to be uploaded using one provider"
},
"upload:after": {
"$id": "#/properties/hooks/properties/upload:after",
"type": "string",
"title": "Run the script when the backup have been uploaded using one provider"
},
"upload:error": {
"$id": "#/properties/hooks/properties/upload:error",
"type": "string",
"title": "Run the script when the backup could not be uploaded using one provider"
},
"oldBackup:deleting": {
"$id": "#/properties/hooks/properties/oldBackup:deleting",
"type": "string",
"title": "Run the script when an old backup is going to be deleted"
},
"oldBackup:deleted": {
"$id": "#/properties/hooks/properties/oldBackup:deleted",
"type": "string",
"title": "Run the script when an old backup is already deleted"
},
"oldBackup:error": {
"$id": "#/properties/hooks/properties/oldBackup:error",
"type": "string",
"title": "Run the script when an old backup failed when it was being deleted"
}
}
}
}
}
16 changes: 15 additions & 1 deletion mdbackup/__main__.py
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,7 @@
)
from .backup import do_backup, get_backup_folders_sorted
from .config import Config, ProviderConfig
from .hooks import define_hook, run_hook
from .storage import create_storage_instance


Expand Down Expand Up @@ -70,6 +71,7 @@ def main_do_backup(logger: logging.Logger, config: Config) -> Path:
**secret_env)
except Exception as e:
logger.error(e)
run_hook('backup:error', str(config.backups_path / '.partial'), str(e))
shutil.rmtree(str(config.backups_path / '.partial'))
sys.exit(1)

Expand Down Expand Up @@ -126,12 +128,15 @@ def main_upload_backup(logger: logging.Logger, config: Config, backup: Path):
final_items, items_to_remove = main_compress_folders(config, backup)
has_compressed = True

run_hook('upload:before', prov_config.type, str(backup))

# Create folder for this backup
try:
logger.info(f'Creating folder {backup_folder_name} in {prov_config.backups_path}')
backup_cloud_folder = storage.create_folder(backup_folder_name, prov_config.backups_path)
except Exception as e:
# If we cannot create it, will continue to the next configured provider
run_hook('upload:error', prov_config.type, str(backup), str(e))
logger.exception(f'Could not create folder {backup_folder_name}', e)
continue

Expand All @@ -142,7 +147,10 @@ def main_upload_backup(logger: logging.Logger, config: Config, backup: Path):
storage.upload(item, backup_cloud_folder)
except Exception as e:
# Log only in case of error (tries to upload as much as it can)
run_hook('upload:error', prov_config.type, str(backup), str(e))
logger.exception(f'Could not upload file {item}: {e}')

run_hook('upload:after', prov_config.type, str(backup), backup_cloud_folder)
else:
# The provider is invalid, show error
logger.error(f'Unknown storage provider "{prov_config.type}", ignoring...')
Expand All @@ -160,10 +168,13 @@ def main_clean_up(logger: logging.Logger, config: Config):
logger.debug('List of folders available:\n{}'.format('\n'.join([str(b) for b in backups_list])))
for old in backups_list[0:max(0, len(backups_list) - max_backups)]:
logger.warning(f'Removing old backup folder {old}')
run_hook('oldBackup:deleting', str(old.absolute()))
try:
shutil.rmtree(str(old.absolute()))
except OSError:
run_hook('oldBackup:deleted', str(old.absolute()))
except OSError as e:
logger.exception(f'Could not completely remove backup {old}')
run_hook('oldBackup:error', str(old.absolute()), str(e))


def main():
Expand Down Expand Up @@ -208,6 +219,9 @@ def main():
level=config.log_level)
logger = logging.getLogger('mdbackup')

# Configure hooks
[define_hook(name, script) for (name, script) in config.hooks.items()]

try:
if args.backup_only:
backup = main_do_backup(logger, config)
Expand Down
6 changes: 6 additions & 0 deletions mdbackup/backup.py
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,8 @@
from threading import Thread
from typing import List, Dict, Union, Callable

from .hooks import run_hook


def generate_backup_path(backups_folder: Path) -> Path:
"""
Expand Down Expand Up @@ -124,6 +126,8 @@ def do_backup(backups_folder: Path, custom_utils: str = None, **kwargs) -> Path:
logger = logging.getLogger(__name__)
tmp_backup = Path(backups_folder, '.partial')

run_hook('backup:before', str(tmp_backup))

logger.info(f'Temporary backup folder is {tmp_backup}')
tmp_backup.mkdir(exist_ok=True, parents=True)
tmp_backup.chmod(0o755)
Expand All @@ -147,6 +151,8 @@ def do_backup(backups_folder: Path, custom_utils: str = None, **kwargs) -> Path:
current_backup.unlink()
os.symlink(backup, current_backup)

run_hook('backup:after', str(backup))

return backup


Expand Down
8 changes: 8 additions & 0 deletions mdbackup/config.py
Original file line number Diff line number Diff line change
Expand Up @@ -107,6 +107,7 @@ def _parse_config(self, conf):
self.__providers = [ProviderConfig(provider_dict) for provider_dict in conf.get('providers', [])]
self.__secrets = [SecretConfig(key, secret_dict.get('env'), secret_dict['config'], secret_dict.get('providers'))
for key, secret_dict in conf.get('secrets', {}).items()]
self.__hooks = conf.get('hooks', {})
if 'compression' in conf:
self.__compression_level = conf['compression'].get('level', 5)
self.__compression_strategy = conf['compression']['strategy']
Expand Down Expand Up @@ -209,3 +210,10 @@ def cypher_params(self) -> Optional[Dict[str, Any]]:
:return: The cypher parameters for the given strategy
"""
return self.__cypher_params

@property
def hooks(self) -> Dict[str, str]:
"""
:return: The hooks dictionary
"""
return self.__hooks
83 changes: 83 additions & 0 deletions mdbackup/hooks.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,83 @@
# Small but customizable utility to create backups and store them in
# cloud storage providers
# Copyright (C) 2019 Melchor Alejo Garau Madrigal / Andrés Mateos García
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <https://www.gnu.org/licenses/>.

import logging
from pathlib import Path
import subprocess
from threading import Thread
from typing import Optional, List, Dict


hooks_config: Dict[str, List[str]] = {}


def run_hook(hook_name: str, *args: str, cwd: Optional[str] = None):
logger = logging.getLogger(__name__)
if hook_name not in hooks_config:
logger.debug(f'The hook {hook_name} is not defined, not running it')
return

if cwd is not None:
cwd_path = Path(cwd)
if not cwd_path.exists():
logger.warning(f'The CWD {cwd} does not exist')
return
if not cwd_path.is_dir():
logger.warning(f'The CWD {cwd} is not a folder')
return

logger.info(f'Running hook {hook_name}')
joins = [(_hook_runner(hook_name, hook, hook_name, *args, cwd=cwd, shell=True), hook)
for hook in hooks_config[hook_name]]
for join, hook in joins:
try:
join()
except Exception:
logger.debug(f'Failed running the hook {hook}')


def _hook_runner(name: str, path: str, *args: str, cwd: Optional[str] = None, shell: bool = False):
logger = logging.getLogger(f'{__name__}:{name}')
logger.debug(f'Running script {path}')
process = subprocess.Popen([path, *args],
stderr=subprocess.PIPE,
stdout=subprocess.PIPE,
cwd=cwd,
bufsize=1,
shell=shell)

# Print stderr lines in a thread
stderr_thread = Thread(target=lambda: [logger.debug(err_line[:-1].decode('UTF-8')) for err_line in process.stderr])
stderr_thread.start()

# Read every line (from stdout) into the logger
for line in process.stdout:
logger.debug(line[0:-1].decode('UTF-8'))

# Join thread and wait to wait process (same as join, but in processes)
def join():
stderr_thread.join()
return process.wait()

return join


def define_hook(hook_name: str, hook_script: str):
if hook_name not in hooks_config:
hooks_config[hook_name] = []

hooks_config[hook_name].append(hook_script)

0 comments on commit 439def4

Please sign in to comment.