[MultiDB] mimic redis-dump-load , adding sonic_db_dump_load.py and so…

…nic-db-dump & sonic-db-load (sonic-net#62)

* [MultiDB] mimic redis-dump-load , adding sonic_db_dump_load.py and sonic-db-dump & sonic-db-load
* Add original copyright statements

ThirdPartyLicenses.txt

@@ -1,12 +1,12 @@
 Third Party Notices
 
-This Microsoft Open Source project incorporates material from the project(s) listed below 
-(Third Party Code). Microsoft is not the original author of the Third Party Code. Microsoft 
+This Microsoft Open Source project incorporates material from the project(s) listed below
+(Third Party Code). Microsoft is not the original author of the Third Party Code. Microsoft
 reserves all other rights not expressly granted, whether by implication, estoppel or otherwise.
 
 1. redis-py https://github.com/andymccurdy/redis-py
-2. pyagentx  https://github.com/rayed/pyagentx
-
+2. pyagentx https://github.com/rayed/pyagentx
+3. redis-dump-load https://github.com/p/redis-dump-load
 
 1. redis-py https://github.com/andymccurdy/redis-py
 Copyright (c) 2012 Andy McCurdy
@@ -56,3 +56,29 @@ LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
 ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
 SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+
+3. redis-dump-load https://github.com/p/redis-dump-load
+Copyright (c) 2011-2016 Oleg Pudeyev
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are met:
+
+1. Redistributions of source code must retain the above copyright notice, this
+ list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright notice,
+ this list of conditions and the following disclaimer in the documentation
+ and/or other materials provided with the distribution.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
+FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

setup.py

@@ -25,6 +25,12 @@
  extras_require={
  'high_perf': high_performance_deps
  },
+ entry_points={
+ 'console_scripts': [
+ 'sonic-db-load = swsssdk:sonic_db_dump_load',
+ 'sonic-db-dump = swsssdk:sonic_db_dump_load',
+ ],
+ },
  classifiers=[
  'Intended Audience :: Developers',
  'Operating System :: Linux',

src/swsssdk/__init__.py

@@ -10,6 +10,7 @@
 try:
  from .dbconnector import SonicDBConfig, SonicV2Connector
  from .configdb import ConfigDBConnector
+ from .sonic_db_dump_load import sonic_db_dump_load
 except (KeyError, ValueError):
  msg = "Failed to database connector objects -- incorrect database config schema."
  logger.exception(msg)

src/swsssdk/sonic_db_dump_load.py

@@ -0,0 +1,139 @@
+## ref: https://github.com/p/redis-dump-load/blob/7bbdb1eaea0a51ed4758d3ce6ca01d497a4e7429/redisdl.py
+
+def sonic_db_dump_load():
+ import optparse
+ import os.path
+ import re
+ import sys
+ from redisdl import dump, load
+ from swsssdk import SonicDBConfig
+
+ DUMP = 1
+ LOAD = 2
+
+ def options_to_kwargs(options):
+ args = {}
+ if options.password:
+ args['password'] = options.password
+ if options.encoding:
+ args['encoding'] = options.encoding
+ # dump only
+ if hasattr(options, 'pretty') and options.pretty:
+ args['pretty'] = True
+ if hasattr(options, 'keys') and options.keys:
+ args['keys'] = options.keys
+ # load only
+ if hasattr(options, 'use_expireat') and options.use_expireat:
+ args['use_expireat'] = True
+ if hasattr(options, 'empty') and options.empty:
+ args['empty'] = True
+ if hasattr(options, 'backend') and options.backend:
+ args['streaming_backend'] = options.backend
+ if hasattr(options, 'dbname') and options.dbname:
+ if options.conntype == 'tcp':
+ args['host'] = SonicDBConfig.get_hostname(options.dbname)
+ args['port'] = SonicDBConfig.get_port(options.dbname)
+ args['db'] = SonicDBConfig.get_dbid(options.dbname)
+ args['unix_socket_path'] = None
+ elif options.conntype == "unix_socket":
+ args['host'] = None
+ args['port'] = None
+ args['db'] = SonicDBConfig.get_dbid(options.dbname)
+ args['unix_socket_path'] = SonicDBConfig.get_socket(options.dbname)
+ else:
+ raise TypeError('redis connection type is tcp or unix_socket')
+
+ return args
+
+ def do_dump(options):
+ if options.output:
+ output = open(options.output, 'w')
+ else:
+ output = sys.stdout
+
+ kwargs = options_to_kwargs(options)
+ dump(output, **kwargs)
+
+ if options.output:
+ output.close()
+
+ def do_load(options, args):
+ if len(args) > 0:
+ input = open(args[0], 'rb')
+ else:
+ input = sys.stdin
+
+ kwargs = options_to_kwargs(options)
+ load(input, **kwargs)
+
+ if len(args) > 0:
+ input.close()
+
+ script_name = os.path.basename(sys.argv[0])
+ if re.search(r'load(?:$|\.)', script_name):
+ action = help = LOAD
+ elif re.search(r'dump(?:$|\.)', script_name):
+ action = help = DUMP
+ else:
+ # default is dump, however if dump is specifically requested
+ # we don't show help text for toggling between dumping and loading
+ action = DUMP
+ help = None
+
+ if help == LOAD:
+ usage = "Usage: %prog [options] [FILE]"
+ usage += "\n\nLoad data from FILE (which must be a JSON dump previously created"
+ usage += "\nby redisdl) into specified or default redis."
+ usage += "\n\nIf FILE is omitted standard input is read."
+ elif help == DUMP:
+ usage = "Usage: %prog [options]"
+ usage += "\n\nDump data from specified or default redis."
+ usage += "\n\nIf no output file is specified, dump to standard output."
+ else:
+ usage = "Usage: %prog [options]"
+ usage += "\n %prog -l [options] [FILE]"
+ usage += "\n\nDump data from redis or load data into redis."
+ usage += "\n\nIf input or output file is specified, dump to standard output and load"
+ usage += "\nfrom standard input."
+ parser = optparse.OptionParser(usage=usage)
+ parser.add_option('-w', '--password', help='connect with PASSWORD')
+ if help == DUMP:
+ parser.add_option('-n', '--dbname', help='dump DATABASE (APPL_DB/ASIC_DB...)')
+ parser.add_option('-t', '--conntype', help='indicate redis connection type (tcp[default] or unix_socket)', default='tcp')
+ parser.add_option('-k', '--keys', help='dump only keys matching specified glob-style pattern')
+ parser.add_option('-o', '--output', help='write to OUTPUT instead of stdout')
+ parser.add_option('-y', '--pretty', help='split output on multiple lines and indent it', action='store_true')
+ parser.add_option('-E', '--encoding', help='set encoding to use while decoding data from redis', default='utf-8')
+ elif help == LOAD:
+ parser.add_option('-n', '--dbname', help='dump DATABASE (APPL_DB/ASIC_DB...)')
+ parser.add_option('-t', '--conntype', help='indicate redis connection type (tcp[default] or unix_socket)', default='tcp')
+ parser.add_option('-e', '--empty', help='delete all keys in destination db prior to loading', action='store_true')
+ parser.add_option('-E', '--encoding', help='set encoding to use while encoding data to redis', default='utf-8')
+ parser.add_option('-B', '--backend', help='use specified streaming backend')
+ parser.add_option('-A', '--use-expireat', help='use EXPIREAT rather than TTL/EXPIRE', action='store_true')
+ else:
+ parser.add_option('-l', '--load', help='load data into redis (default is to dump data from redis)', action='store_true')
+ parser.add_option('-n', '--dbname', help='dump DATABASE (APPL_DB/ASIC_DB/COUNTERS_DB/LOGLEVEL_DB/CONFIG_DB...)')
+ parser.add_option('-t', '--conntype', help='indicate redis connection type (tcp[default] or unix_socket)', default='tcp')
+ parser.add_option('-k', '--keys', help='dump only keys matching specified glob-style pattern')
+ parser.add_option('-o', '--output', help='write to OUTPUT instead of stdout (dump mode only)')
+ parser.add_option('-y', '--pretty', help='split output on multiple lines and indent it (dump mode only)', action='store_true')
+ parser.add_option('-e', '--empty', help='delete all keys in destination db prior to loading (load mode only)', action='store_true')
+ parser.add_option('-E', '--encoding', help='set encoding to use while decoding data from redis', default='utf-8')
+ parser.add_option('-A', '--use-expireat', help='use EXPIREAT rather than TTL/EXPIRE', action='store_true')
+ parser.add_option('-B', '--backend', help='use specified streaming backend (load mode only)')
+ options, args = parser.parse_args()
+
+ if hasattr(options, 'load') and options.load:
+ action = LOAD
+
+ if action == DUMP:
+ if len(args) > 0:
+ parser.print_help()
+ exit(4)
+ do_dump(options)
+ else:
+ if len(args) > 1:
+ parser.print_help()
+ exit(4)
+ do_load(options, args)