Backblaze intergration (#2)

* Backblaze integration * Some bug fixes
MajorcaDevs · Feb 10, 2019 · 569692c · 569692c
1 parent f660f55
commit 569692c
Show file tree

Hide file tree

Showing 7 changed files with 50 additions and 28 deletions.
diff --git a/mdbackup/__main__.py b/mdbackup/__main__.py
@@ -65,15 +65,21 @@ def main():
 
     # Do backups
     backups_path = config.backups_path
-    backup = do_backup(backups_path,
-                       config.custom_utils_script,
-                       **config.env,
-                       compression_strategy=config.compression_strategy,
-                       compression_level=config.compression_level,
-                       cypher_strategy=config.cypher_strategy,
-                       **{f'cypher_{key}': value
-                          for key, value in (config.cypher_params.items() if config.cypher_params is not None else [])},
-                       **secret_env_flatten)
+    try:
+        backup = do_backup(backups_path,
+                           config.custom_utils_script,
+                           **config.env,
+                           compression_strategy=config.compression_strategy,
+                           compression_level=config.compression_level,
+                           cypher_strategy=config.cypher_strategy,
+                           **{f'cypher_{key}': value
+                              for key, value in (config.cypher_params.items() if config.cypher_params is not None else [])},
+                           **secret_env_flatten)
+    except Exception as e:
+        logger.error(e)
+        shutil.rmtree(str(backups_path / '.partial'))
+        sys.exit(1)
+
     final_items = []
     items_to_remove = []
 
@@ -114,6 +120,7 @@ def main():
                 if storage is not None:
                     # Create folder for this backup
                     try:
+                        logger.info(f'Creating folder {backup_folder_name} in {prov_config.backups_path}')
                         backup_cloud_folder = storage.create_folder(backup_folder_name, prov_config.backups_path)
                     except Exception:
                         # If we cannot create it, will continue to the next configured provider

diff --git a/mdbackup/backup.py b/mdbackup/backup.py
@@ -120,6 +120,7 @@ def do_backup(backups_folder: Path, custom_utils: str = None, **kwargs) -> Path:
             logger.error(f'Script returned {output.returncode}')
             if output.stdout: logger.error(f'stdout\n{output.stdout.decode("utf-8")}')
             if output.stderr: logger.error(f'stderr\n{output.stderr.decode("utf-8")}')
+            raise Exception(f'The step {step_script} failed, backup will stop')
         else:
             if output.stdout: logger.debug(f'stdout\n{output.stdout.decode("utf-8")}')
             if output.stderr: logger.debug(f'stderr\n{output.stderr.decode("utf-8")}')

diff --git a/mdbackup/config.py b/mdbackup/config.py
@@ -111,6 +111,9 @@ def _parse_config(self, conf):
             self.__cypher_strategy = conf['cypher']['strategy']
             self.__cypher_params = change_keys(conf['cypher'])
             del self.__cypher_params['strategy']
+        else:
+            self.__cypher_strategy = None
+            self.__cypher_params = None
 
         Config.__check_paths(self.__backups_path)
         Config.__check_paths(self.__custom_utils_script)

diff --git a/mdbackup/storage/backblaze.py b/mdbackup/storage/backblaze.py
@@ -32,29 +32,33 @@ def __init__(self, config):
         self.__b2 = B2(key_id=config['keyId'], application_key=config['appKey'])
         self.__bucket: B2Bucket = self.__b2.buckets.get(config['bucket'])
         self.__password: str = config.get('password')
-        self.__pre = config['backupsPath']
+        self.__pre = config.backups_path if not config.backups_path.endswith('/') else config.backups_path[:-1]
 
     def list_directory(self, path: Union[str, Path, str]) -> List[str]:
-        path = path if isinstance(path, str) else str(path.absolute())
+        path = path if isinstance(path, str) else str(path)
         return [item.file_name for item in self.__bucket.files.all(include_hidden=True)
-                if item.file_name.startswith(self.__pre + path)]
+                if item.file_name.startswith(self.__pre + '/' + path)]
 
     def create_folder(self, name: str, parent: Union[Path, str, str]=None) -> str:
-        key = self.__pre + f'{parent.absolute()}/{name}/'
+        key = f'{parent}/{name}/'
+        if key.startswith('/'):
+            key = key[1:]
         return key
 
     def upload(self, path: Path, parent: Union[Path, str, str]=None):
         if isinstance(parent, Path):
-            key = f'{parent.absolute()}/{path.name}'
+            key = '/'.join(parent.absolute().parts + (path.name,))
         elif isinstance(parent, str):
-            key = f'{parent}/{path.name}'
+            if parent.endswith('/'):
+                key = (parent + path.name)
+            else:
+                key = f'{parent}/{path.name}'
         else:
             key = path.name
-        key = self.__pre + key
+        if key.startswith('/'):
+            key = key[1:]
         self.__log.info(f'Uploading file {key} (from {path})')
         with open(str(path.absolute()), 'rb') as file_to_upload:
             ret = self.__bucket.files.upload(contents=file_to_upload,
-                                             file_name=key,
-                                             bucket_name=self.__bucket,
-                                             password=self.__password)
+                                             file_name=key)
         self.__log.debug(ret)
diff --git a/mdbackup/storage/s3.py b/mdbackup/storage/s3.py
@@ -37,28 +37,34 @@ def __init__(self, config):
         )
 
         self.__bucket: str = config['bucket']
-        self.__pre = config['backupsPath']
+        self.__pre = config.backups_path if not config.backups_path.endswith('/') else config.backups_path[:-1]
 
     def list_directory(self, path: Union[str, Path, str]) -> List[str]:
         path = path if isinstance(path, str) else str(path)
         return [item['Key'] for item in self.__s3.list_objects_v2(Bucket=self.__bucket)['Contents']
-                if item['Key'].startswith(self.__pre + path)]
+                if item['Key'].startswith(self.__pre + '/' + path)]
 
     def create_folder(self, name: str, parent: Union[Path, str, str]=None) -> str:
-        key = self.__pre + f'{parent}/{name}/'
+        key = f'{parent}/{name}/'
+        if key.startswith('/'):
+            key = key[1:]
         self.__log.info(f'Creating folder {key}')
         ret = self.__s3.put_object(Key=key, Bucket=self.__bucket)
         self.__log.debug(ret)
-        return key
+        return f'{parent}/{name}/'
 
     def upload(self, path: Path, parent: Union[Path, str, str]=None):
         if isinstance(parent, Path):
-            key = f'{parent.absolute()}/{path.name}'
+            key = '/'.join(parent.absolute().parts + (path.name,))
         elif isinstance(parent, str):
-            key = f'{parent}/{path.name}'
+            if parent.endswith('/'):
+                key = (parent + path.name)
+            else:
+                key = f'{parent}/{path.name}'
         else:
             key = path.name
-        key = self.__pre + key
+        if key.startswith('/'):
+            key = key[1:]
         self.__log.info(f'Uploading file {key} (from {path})')
         ret = self.__s3.upload_file(str(path.absolute()), self.__bucket, key)
         self.__log.debug(ret)
diff --git a/mdbackup/utils.sh b/mdbackup/utils.sh
@@ -214,7 +214,7 @@ function backup-docker-volume() {
 function backup-docker-volume-physically() {
     if docker volume inspect "$1" > /dev/null; then
         local dst=$(docker volume inspect "$1" --format "{{.Mountpoint}}")
-        backup-folder "$DST" "$1" || return $?
+        backup-folder "$dst" "$1" || return $?
     else
         echo "Volume '$1' does not exist"
         return 1

diff --git a/setup.py b/setup.py
@@ -32,6 +32,7 @@
         'PyDrive',
         'python-magic',
         'requests',
-        'boto3'
+        'boto3',
+        'b2blaze'
     ]
 )