Merge pull request bcgov#165 from WadeBarnes/backup

Add support for backup and restore.

.dockerignore

@@ -3,4 +3,9 @@ Dockerfile
 LICENSE
 manage
 README.md
+COMPLIANCE.yaml
 .git*
+tmp/
+backup/
+openshift/
+docs/

.gitignore

@@ -3,6 +3,9 @@
 *.key
 *.pem
 
+# Backup files
+backup/
+
 # tmp script files
 *_DeploymentConfig.json
 *_BuildConfig.json

Dockerfile

@@ -10,6 +10,9 @@ RUN pip install --no-cache-dir -r server/requirements.txt
 
 ADD --chown=indy:indy indy_config.py /etc/indy/
 ADD --chown=indy:indy . $HOME
+RUN tar -xzf ./bin/sst_dump.tar.gz -C ./bin \
+    && rm ./bin/sst_dump.tar.gz \
+    && chmod +x ./bin/sst_dump
 
 RUN mkdir -p \
     $HOME/cli-scripts \

bin/read_ledger

@@ -1,24 +1,30 @@
 #! /usr/bin/env python3
 """
 Convenience script for reading a ledger transaction data (stored in leveldb)
+
 """
 import argparse
 import logging
 import os
 import shutil
 
+from pathlib import Path
+
 from common.serializers.json_serializer import JsonSerializer
-from ledger.compact_merkle_tree import CompactMerkleTree
 from plenum.common.ledger import Ledger
-from storage.helper import initHashStore
+from plenum.common.constants import HS_ROCKSDB
 
 from indy_common.config_util import getConfig
-from indy_common.config_helper import ConfigHelper, NodeConfigHelper
+from common.serializers.serialization import ledger_txn_serializer
 
 logging.root.handlers = []
 logger = logging.getLogger()
 logger.propagate = False
 logger.disabled = True
+_DATA = 'data'
+
+# TODO: Replace with constant from config
+postfix = '_transactions'
 
 
 def read_args():
@@ -39,73 +45,69 @@ def read_args():
     parser.add_argument('--count', required=False, action='store_true',
                         help="returns the number of txns in the given ledger")
     parser.add_argument('--node_name', required=False, help="Node's name")
-    parser.add_argument('--client_name', required=False, help="Client's name")
     parser.add_argument('--serializer', required=False, default='json',
                         help="How to represent the data (json by default)")
     parser.add_argument('--network', required=False, type=str,
                         help="Network name to read ledger from")
-    parser.add_argument('--base_dir', required=False, type=str,
-                        help="Specifify a directory to read from")
 
     return parser.parse_args()
 
 
-def get_ledger_dir(node_name, client_name, network, base_dir):
-
-    if node_name and client_name:
-        print("Either 'node_name' or 'client_name' can be specified")
-        exit()
-
+def get_ledger_dir(node_name, network):
     config = getConfig()
     _network = network if network else config.NETWORK_NAME
-    if node_name or client_name:
-        if node_name:
-            config_helper = NodeConfigHelper(node_name, config)
-            ledger_data_dir = config_helper.ledger_dir
-        else:
-            ledger_data_dir = os.path.join(config.CLI_BASE_DIR, _network,
-                                           config.clientDataDir, client_name)
-        if not os.path.isdir(ledger_data_dir):
-            print("Specified Node or Client folder not found: {}".format(
-                ledger_data_dir))
-            exit()
+    ledger_base_dir = config.LEDGER_DIR
+    if node_name:
+        # Build path to data if --node_name was specified
+        ledger_data_dir = os.path.join(ledger_base_dir, _network, _DATA, node_name)
     else:
-
-        if not base_dir:
-            base_dir = config.CLI_BASE_DIR if client_name else config.baseDir
-        data_dir = config.clientDataDir if client_name else 'data'
-        ledger_data_dir = os.path.join(base_dir, _network, data_dir)
-
-        if not os.path.exists(ledger_data_dir):
-            print("Can not find the directory with the ledger: {}".format(
-                ledger_data_dir))
-            exit()
-
-        dirs = os.listdir(ledger_data_dir)
-        ledger_data_dir = os.path.join(ledger_data_dir, dirs[0])
+        ledger_data_dir = os.path.join(ledger_base_dir, _network, _DATA)
+        if os.path.exists(ledger_data_dir):
+            dirs = os.listdir(ledger_data_dir)
+            if len(dirs) == 0:
+                print("Node's 'data' folder not found: {}".format(ledger_data_dir))
+                exit()
+            # --node_name parameter was not set, therefore we can choose first Node name in data dir
+            ledger_data_dir = os.path.join(ledger_data_dir, dirs[0])
+    if not os.path.exists(ledger_data_dir):
+        print("No such file or directory: {}".format(ledger_data_dir))
+        print("Please check, that network: '{}' was used ".format(_network))
+        exit()
 
     return ledger_data_dir
 
 
-def get_ledger(type_, ledger_data_dir):
+def get_storage(type_, ledger_data_dir):
     config = getConfig()
 
-    ledger_name = None
+    storage_name = None
     if type_ == 'pool':
-        ledger_name = config.poolTransactionsFile
+        storage_name = config.poolTransactionsFile
     elif type_ == 'domain':
-        ledger_name = config.domainTransactionsFile
+        storage_name = config.domainTransactionsFile
     elif type_ == 'config':
-        ledger_name = config.configTransactionsFile
+        storage_name = config.configTransactionsFile
+    elif type_ in get_additional_storages(ledger_data_dir):
+        storage_name = type_ + postfix
     else:
         print("Unknown ledger type: {}".format(type_))
         exit()
 
-    hash_store = initHashStore(ledger_data_dir, type_, config)
-    return Ledger(CompactMerkleTree(hashStore=hash_store), dataDir=ledger_data_dir, fileName=ledger_name)
+    return Ledger._defaultStore(dataDir=ledger_data_dir,
+                                logName=storage_name,
+                                ensureDurability=True,
+                                open=True,
+                                config=config,
+                                read_only=True)
+
 
+def get_additional_storages(ledger_data_dir):
+    additional_storages = \
+        [name[:name.find(postfix)] for name in os.listdir(ledger_data_dir) if postfix in name]
+    return additional_storages
 
-def print_txns(ledger, args):
+
+def print_txns(storage, args):
     serializer = None
     if args.serializer == 'json':
         serializer = JsonSerializer()
@@ -116,40 +118,43 @@ def print_txns(ledger, args):
     # --count
     count = args.count
     if count:
-        print_count(ledger)
+        print_count(storage)
         return
 
     # --seq_no
-    seq_no = int(args.seq_no) if args.seq_no is not None else None
+    seq_no = args.seq_no
     if seq_no:
-        print_by_seq_no(ledger, seq_no, serializer)
+        print_by_seq_no(storage, seq_no, serializer)
         return
 
     # print all (--from --to)
-    print_all(ledger, serializer)
+    print_all(storage, serializer)
 
 
-def print_by_seq_no(ledger, seq_no, serializer):
+def print_by_seq_no(storage, seq_no, serializer):
     try:
-        txn = ledger.getBySeqNo(seq_no)
+        print("Getting by seq no:", seq_no)
+        txn = storage.get(seq_no)
     except KeyError:
         print('No transactions found for seq_no={}'.format(seq_no))
         return
+    print("Serializing transactions ...")
     txn = serializer.serialize(txn, toBytes=False)
+    print("Printing transactions ...")
     print(txn)
     return
 
 
-def print_count(ledger):
-    print(ledger.size)
+def print_count(storage):
+    print(storage.size)
 
 
-def print_all(ledger, serializer):
-    frm = int(args.frm) if args.frm is not None else None
-    to = int(args.to) if args.to is not None else None
-    for txn in ledger.getAllTxn(frm=frm, to=to):
-        txn = serializer.serialize(txn, toBytes=False)
-        print(txn)
+def print_all(storage, serializer):
+    frm = int(args.frm) if args.frm else None
+    to = int(args.to) if args.to else None
+    for seqNo, txn in storage.iterator(start=frm, end=to):
+        txn = ledger_txn_serializer.deserialize(txn)
+        print(serializer.serialize(txn, toBytes=False))
 
 
 def make_copy_of_ledger(data_dir):
@@ -162,20 +167,30 @@ def make_copy_of_ledger(data_dir):
 
 if __name__ == '__main__':
     args = read_args()
+    config = getConfig()
 
-    # TODO: works well only for small ledgers,
-    ledger_data_dir = get_ledger_dir(
-        args.node_name,
-        args.client_name,
-        args.network,
-        args.base_dir)
-
+    ledger_data_dir = get_ledger_dir(args.node_name, args.network)
     read_copy_ledger_data_dir = None
     try:
-        read_copy_ledger_data_dir = make_copy_of_ledger(ledger_data_dir)
-        ledger = get_ledger(args.type, read_copy_ledger_data_dir)
-        print_txns(ledger, args)
+        # RocksDB supports real read-only mode and does not need to have a ledger copy.
+        if config.hashStore['type'].lower() != HS_ROCKSDB:
+            config.db_transactions_config = None
+            # NOTE: such approach works well only for small ledgers.
+            tmp = make_copy_of_ledger(ledger_data_dir)
+
+            # Let's be paranoid to avoid removing of ledger instead of its copy.
+            ledger_path = Path(ledger_data_dir)
+            ledger_copy_path = Path(tmp)
+            assert ledger_path != ledger_copy_path
+            assert ledger_copy_path not in ledger_path.parents
+
+            read_copy_ledger_data_dir = tmp
+            ledger_data_dir = read_copy_ledger_data_dir
+        elif config.db_transactions_config is not None:
+            # This allows to avoid debug logs creation on each read_ledger run
+            config.db_transactions_config['db_log_dir'] = '/dev/null'
+        storage = get_storage(args.type, ledger_data_dir)
+        print_txns(storage, args)
     finally:
-        # TODO be careful about removing original the ledger data dir
         if read_copy_ledger_data_dir:
             shutil.rmtree(read_copy_ledger_data_dir)