Merge pull request #57 from epfl-idevelop/feature-wwp-53-backup

Manage backups

.env.sample

@@ -22,6 +22,7 @@ WP_PORT_HTTPS=443
 # WP MANAGEMENT
 WP_PORT_PHPMA=8080
 WP_PORT_SSHD=2222
+BACKUP_PATH=/tmp/backups
 
 
 # JAHIA variables

.gitignore

@@ -10,4 +10,5 @@
 log/
 htmlcov/
 volumes/
+data/backups/
 coverage.xml

README.md

@@ -64,6 +64,9 @@ Icons are used to mark the progress as follows: :balloon:, :tada:, :champagne: o
         $ python jahia2wp.py generate-many path/to/source.csv
         ...
 
+        $ python jahia2wp.py backup-many path/to/source.csv
+        ...
+
         $ python jahia2wp.py inventory $WP_ENV /srv/your-env/localhost
         INFO - your-env - inventory - Building inventory...
         path;valid;url;version;db_name;db_user;admins
@@ -77,6 +80,9 @@ Icons are used to mark the progress as follows: :balloon:, :tada:, :champagne: o
         $ python jahia2wp.py clean $WP_ENV http://localhost
         ...
 
+        $ python jahia2wp.py backup $WP_ENV http://localhost
+        ...
+
 We will secondly add support for migration of a simple site:
 
 1. :gift_heart: Export the content of a Jahia website as a zipped package

data/csv/sites_wp.csv

@@ -3,4 +3,3 @@ https://www.epfl.ch/recherche,Recherche,wordpress,dev,GeneralPublic,EPFL,asked,y
 https://www.epfl.ch/education,Education,wordpress,dev,GeneralPublic,EPFL,yes,yes,no,"fr,en",123456,123456,VPE,ce qui me plait
 https://www.epfl.ch/education/master,Master,wordpress,dev,GeneralPublic,EPFL,yes,yes,no,"fr,en",123456,123456,VPE,des infos que l'utilisateur m'a donné par ex
 https://www.epfl.ch/atelierweb,WWW,wordpress,dev,GeneralPublic,EPFL,no,yes,no,"fr,en",123456,123456,VPSI,
-https://dcsl.epfl.ch,DCSL,wordpress,int,Lab,EPFL,to check,yes,no,fr,123456,123456,IC,

docs/CHANGELOG.md

@@ -11,6 +11,7 @@ Table of releases
 
 <!-- TOC depthFrom:2 depthTo:2 orderedList:false -->
 
+- [[0.2.7] - 2017-11-01](#027---2017-11-01)
 - [[0.2.6] - 2017-10-31](#026---2017-10-31)
 - [[0.2.5] - 2017-10-20](#025---2017-10-20)
 - [[0.2.4] - 2017-10-19](#024---2017-10-19)
@@ -22,6 +23,51 @@ Table of releases
 
 <!-- /TOC -->
 
+## [0.2.7] - 2017-11-01
+**[PR #57](https://github.com/epfl-idevelop/jahia2wp/pull/57)**
+
+**high level:**
+
+1. new commands `backup` and `backup-many` to backup one single site or all sites in source of trust. 
+
+        $ python jahia2wp.py backup $WP_ENV http://localhost
+        ...
+        $ python jahia2wp.py backup-many path/to/csv
+        ...
+
+1. Backups are `full` by default, but can be `inc`remental by using option `--backup-type`
+
+        $ python jahia2wp.py backup $WP_ENV http://localhost --backup-type=inc
+        ...
+1. new environment variable `BACKUP_PATH` to define where to store backups (by default: `jahia2wp/data/backups`)
+
+**low level:**
+
+A backup of a WordPress site relies on a `WPConfig`, and is called/used in a similar way as `WPGenerator`. It creates three files:
+- a `.tar` of all WP files (php, assets, media)
+- a `.list` reference file for incremental backups
+- a `.sql` dump of the database
+
+Backups are stored in `BACKUP_PATH` (by default: `jahia2wp/data/backups`), with the following name convention: `<wp_site_name>[_<timestamp>]_fullN[_incM].<tar|list|sql>`
+
+**DISCLAIMER:** next iteration might revise this structure for something more easily exploitable, such as:
+
+    backups
+    ├── site_1
+    │   ├── 2017-10-31
+    │   │   ├── full-201710310945.tar
+    │   │   ├── inc-201711012300.tar
+    │   │   ├── inc-201711022301.tar
+    │   │   └── inc-201711032310.tar
+    │   └── 2017-11-06
+    │       └── full-201711060946.tar
+    ├── site_2
+    │   └── 2017-11-06
+    │       ├── full-201711060947.tar
+    │       └── inc-201711060946.tar
+    └── site_3
+
+
 
 ## [0.2.6] - 2017-10-31
 **[PR #54](https://github.com/epfl-idevelop/jahia2wp/pull/54)**

docs/index.md

@@ -55,6 +55,7 @@ Table of contents
     - [Create a new WordPress site](#create-a-new-wordpress-site)
     - [Information on a WordPress site](#information-on-a-wordpress-site)
     - [Inventory of WordPress sites for a given path (in a given env)](#inventory-of-wordpress-sites-for-a-given-path-in-a-given-env)
+    - [Backup a WordPress site](#backup-a-wordpress-site)
     - [Delete a WordPress site](#delete-a-wordpress-site)
     - [phpMyAdmin (locally)](#phpmyadmin-locally)
 - [Contribution](#contribution)
@@ -272,6 +273,14 @@ To look into the tree structure and list all valid/unvalid WordPress sites, with
     /srv/your-env/localhost/htdocs/unittest;KO;;;;;
     INFO - your-env - inventory - Inventory made for /srv/your-env/localhost
 
+### Backup a WordPress site
+
+To save a WordPress site, you can choose between a full or an incremental backup, and control the location of your backups with the environment variable `BACKUP_PATH`
+
+    python jahia2wp.py backup $WP_ENV http://localhost/folder
+    python jahia2wp.py backup $WP_ENV http://localhost/folder --backup-type=inc
+
+
 ### Delete a WordPress site
 
 To delete the sites created in the previous section, you could do

src/jahia2wp.py

@@ -2,23 +2,27 @@
 jahia2wp: an amazing tool !
 
 Usage:
-  jahia2wp.py download              <site>            [--debug | --quiet]
+  jahia2wp.py download              <site>                          [--debug | --quiet]
     [--username=<USERNAME> --host=<HOST> --zip-path=<ZIP_PATH> --force]
-  jahia2wp.py clean                 <wp_env> <wp_url> [--debug | --quiet]
-  jahia2wp.py check                 <wp_env> <wp_url> [--debug | --quiet]
-  jahia2wp.py generate              <wp_env> <wp_url> [--debug | --quiet]
+  jahia2wp.py clean                 <wp_env> <wp_url>               [--debug | --quiet]
+  jahia2wp.py check                 <wp_env> <wp_url>               [--debug | --quiet]
+  jahia2wp.py generate              <wp_env> <wp_url>               [--debug | --quiet]
     [--wp-title=<WP_TITLE> --admin-password=<ADMIN_PASSWORD>]
     [--owner=<OWNER_ID> --responsible=<RESPONSIBLE_ID>]
-  jahia2wp.py version               <wp_env> <wp_url> [--debug | --quiet]
-  jahia2wp.py admins                <wp_env> <wp_url> [--debug | --quiet]
-  jahia2wp.py check-one             <wp_env> <wp_url> [--debug | --quiet] [DEPRECATED]
-  jahia2wp.py clean-one             <wp_env> <wp_url> [--debug | --quiet] [DEPRECATED]
-  jahia2wp.py generate-one          <wp_env> <wp_url> [--debug | --quiet] [DEPRECATED]
+  jahia2wp.py backup                <wp_env> <wp_url>               [--debug | --quiet]
+    [--backup-type=<BACKUP_TYPE>]
+  jahia2wp.py version               <wp_env> <wp_url>               [--debug | --quiet]
+  jahia2wp.py admins                <wp_env> <wp_url>               [--debug | --quiet]
+  jahia2wp.py check-one             <wp_env> <wp_url>               [--debug | --quiet] [DEPRECATED]
+  jahia2wp.py clean-one             <wp_env> <wp_url>               [--debug | --quiet] [DEPRECATED]
+  jahia2wp.py generate-one          <wp_env> <wp_url>               [--debug | --quiet] [DEPRECATED]
     [--wp-title=<WP_TITLE> --admin-password=<ADMIN_PASSWORD>]
     [--owner=<OWNER_ID> --responsible=<RESPONSIBLE_ID>]
-  jahia2wp.py generate-many         <csv_file>        [--debug | --quiet]
-  jahia2wp.py veritas               <csv_file>        [--debug | --quiet]
-  jahia2wp.py inventory             <wp_env> <path>   [--debug | --quiet]
+  jahia2wp.py generate-many         <csv_file>                      [--debug | --quiet]
+  jahia2wp.py backup-many           <csv_file>                      [--debug | --quiet]
+    [--backup-type=<BACKUP_TYPE>]
+  jahia2wp.py veritas               <csv_file>                      [--debug | --quiet]
+  jahia2wp.py inventory             <wp_env> <path>                 [--debug | --quiet]
   jahia2wp.py extract-plugin-config <wp_env> <wp_url> <output_file> [--debug | --quiet]
 
 Options:
@@ -27,16 +31,14 @@
   --debug                   Set log level to DEBUG (default is INFO)
   --quiet                   Set log level to WARNING (default is INFO)
 """
-
 import logging
 import getpass
 
 from docopt import docopt
 from docopt_dispatch import dispatch
 
 from veritas.veritas import VeritasValidor
-from wordpress import WPSite, WPConfig, WPGenerator
-from wordpress.plugins import WPPluginConfigExtractor
+from wordpress import WPSite, WPConfig, WPGenerator, WPBackup, WPPluginConfigExtractor
 from crawler import JahiaCrawler
 
 from settings import VERSION
@@ -118,6 +120,15 @@ def generate(wp_env, wp_url, wp_title=None, admin_password=None, owner_id=None,
     print("Successfully created new WordPress site at {}".format(wp_generator.wp_site.url))
 
 
+@dispatch.on('backup')
+def backup(wp_env, wp_url, backup_type=None, **kwargs):
+    wp_backup = WPBackup(wp_env, wp_url, backup_type=backup_type)
+    if not wp_backup.backup():
+        raise SystemExit("Backup failed. More info above")
+
+    print("Successfully backed-up WordPress site for {}".format(wp_backup.wp_site.url))
+
+
 @dispatch.on('version')
 def version(wp_env, wp_url, **kwargs):
     wp_config = _check_site(wp_env, wp_url, **kwargs)
@@ -136,13 +147,7 @@ def admins(wp_env, wp_url, **kwargs):
 @dispatch.on('generate-many')
 def generate_many(csv_file, **kwargs):
     # use Veritas to get valid rows
-    validator = VeritasValidor(csv_file)
-    rows = validator.get_valid_rows()
-
-    # print errors
-    if validator.errors:
-        print("The following lines have errors that prevent the generation of the WP site:\n")
-        validator.print_errors()
+    rows = VeritasValidor.filter_valid_rows(csv_file)
 
     # create a new WP site for each row
     print("\n{} websites will now be generated...".format(len(rows)))
@@ -158,6 +163,23 @@ def generate_many(csv_file, **kwargs):
         ).generate()
 
 
+@dispatch.on('backup-many')
+def backup_many(csv_file, backup_type=None, **kwargs):
+    # use Veritas to get valid rows
+    rows = VeritasValidor.filter_valid_rows(csv_file)
+
+    # create a new WP site backup for each row
+    print("\n{} websites will now be backuped...".format(len(rows)))
+    for index, row in rows:
+        logging.debug("%s - row %s: %s", row["wp_site_url"], index, row)
+        WPBackup(
+            openshift_env=row["openshift_env"],
+            wp_site_url=row["wp_site_url"],
+            wp_default_site_title=row["wp_default_site_title"],
+            backup_type=backup_type
+        ).backup()
+
+
 @dispatch.on('inventory')
 def inventory(wp_env, path, **kwargs):
     logging.info("Building inventory...")

src/settings.py

@@ -3,17 +3,14 @@
 
 from utils import Utils
 
-VERSION = "0.2.6"
+VERSION = "0.2.7"
 
 DATA_PATH = os.path.abspath(
-    os.path.sep.join([
-        os.path.dirname(__file__),
-        '..',
-        'data',
-        'wp',
-        ]
-    )
+    os.path.sep.join([os.path.dirname(__file__), '..', 'data'])
 )
+WP_PATH = os.path.join(DATA_PATH, 'wp')
+BACKUP_PATH = Utils.get_optional_env(
+    "BACKUP_PATH", os.path.join(DATA_PATH, 'backups'))
 
 ENV_DIRS = ['logs', 'venv', 'jahia2wp']
 

src/tests/test_jahia2wp.py

@@ -39,6 +39,16 @@ def test_generate_one_success(self):
         assert Utils.run_command('python %s generate %s http://localhost/unittest'
                                  % (SCRIPT_FILE, TEST_ENV)) == expected
 
+    def test_backup_full(self):
+        expected = "Successfully backed-up WordPress site for http://localhost/unittest"
+        assert Utils.run_command('python %s backup %s http://localhost/unittest'
+                                 % (SCRIPT_FILE, TEST_ENV)) == expected
+
+    def test_backup_incremental(self):
+        expected = "Successfully backed-up WordPress site for http://localhost/unittest"
+        assert Utils.run_command('python %s backup %s http://localhost/unittest --backup-type=inc'
+                                 % (SCRIPT_FILE, TEST_ENV)) == expected
+
     def test_deprecated_calls(self):
         expected = "WARNING: Call to deprecated function"
         assert Utils.run_command('python %s check-one %s http://localhost/unittest'

src/tests/test_utils.py

@@ -69,3 +69,27 @@ def test_csv_from_string(self):
 DB_PASSWORD,Rfcua2LKD^vpGy@m*R*Z,constant
 DB_COLLATE,,constant"""
         assert Utils.csv_string_to_dict(text) == EXPECTED_OUTPUT_FROM_CSV
+
+
+class TestTar:
+
+    OUTPUT_TAR = "/tmp/test.tar"
+    OUTPUT_INC = "/tmp/test.inc"
+
+    def test_generate_tar_file(self):
+        # setup
+        for file_name in [self.OUTPUT_TAR, self.OUTPUT_INC]:
+            if os.path.exists(file_name):
+                os.remove(file_name)
+
+        # run command
+        Utils.generate_tar_file(
+            self.OUTPUT_TAR,
+            self.OUTPUT_INC,
+            os.path.join(CURRENT_DIR, TEST_FILE)
+        )
+
+        # check output
+        for file_name in [self.OUTPUT_TAR, self.OUTPUT_INC]:
+            assert os.path.exists(file_name)
+            os.remove(file_name)

src/utils.py

@@ -158,3 +158,15 @@ def generate_password(length, symbols='!@#^&*'):
         random.seed = (os.urandom(1024))
 
         return ''.join(random.choice(chars) for i in range(length))
+
+    @staticmethod
+    def generate_tar_file(backup_file, backup_listed_incremental_file, source_path):
+        """
+        Generate a tar file
+        """
+        command = "tar --create --file={} --listed-incremental={} {}".format(
+            backup_file,
+            backup_listed_incremental_file,
+            source_path
+        )
+        return Utils.run_command(command)

src/veritas/tests/test_veritas.py

@@ -5,6 +5,21 @@
 
 CURRENT_DIR = os.path.dirname(__file__)
 TEST_FILE = 'test_veritas_data.csv'
+VALID_LINE = {
+        'category': 'GeneralPublic',
+        'comment': 'je mets ici',
+        'installs_locked': 'yes',
+        'langs': 'fr,en',
+        'openshift_env': 'dev',
+        'owner_id': '123456',
+        'responsible_id': '123456',
+        'site_type': 'WordPress',
+        'status': 'asked',
+        'theme': 'EPFL',
+        'unit': 'VPR',
+        'updates_automatic': 'no',
+        'wp_default_site_title': 'Recherche',
+        'wp_site_url': 'htt://www.epfl.ch/recherche'}
 
 
 def test_validate():
@@ -29,21 +44,12 @@ def test_validate():
 
 def test_get_valid_rows():
     filename = os.path.join(CURRENT_DIR, TEST_FILE)
+    valid_lines = ((0, VALID_LINE),)
     validator = VeritasValidor(filename)
-    valid_lines = ((0, {
-        'category': 'GeneralPublic',
-        'comment': 'je mets ici',
-        'installs_locked': 'yes',
-        'langs': 'fr,en',
-        'openshift_env': 'dev',
-        'owner_id': '123456',
-        'responsible_id': '123456',
-        'site_type': 'WordPress',
-        'status': 'asked',
-        'theme': 'EPFL',
-        'unit': 'VPR',
-        'updates_automatic': 'no',
-        'wp_default_site_title': 'Recherche',
-        'wp_site_url': 'htt://www.epfl.ch/recherche'}
-    ),)
     assert validator.get_valid_rows() == valid_lines
+
+
+def test_filter_method():
+    filename = os.path.join(CURRENT_DIR, TEST_FILE)
+    valid_lines = ((0, VALID_LINE),)
+    assert valid_lines == VeritasValidor.filter_valid_rows(filename)

src/veritas/validators.py

@@ -76,3 +76,7 @@ def validate_theme(text):
 
 def validate_languages(text):
     return MultipleChoicesValidator(SUPPORTED_LANGUAGES)(text)
+
+
+def validate_backup_type(text):
+    return ChoiceValidator(choices=['inc', 'full'])(text)

src/veritas/veritas.py

@@ -34,6 +34,21 @@ class VeritasValidor:
     # the csv delimiter
     DELIMITER = ","
 
+    @classmethod
+    def filter_valid_rows(cls, csv_file):
+        """Shortcut method to call get_valid_rows, print errors, and only return valid elements"""
+        # use Veritas to get valid rows
+        validator = cls(csv_file)
+        rows = validator.get_valid_rows()
+
+        # print errors
+        if validator.errors:
+            print("The following lines have errors and have been filtered out:\n")
+            validator.print_errors()
+
+        # return valid rows only
+        return rows
+
     def __init__(self, csv_path, columns=JAHIA2WP_COLUMNS):
         """ csv_path: path on file system pointing the CSV file to validate
             columns: description of the validations to make on columns, array of tuple