From df617345d807bc01e3f5740fdfdc4b9799f1cf63 Mon Sep 17 00:00:00 2001 From: Dong Zhang <41927498+dzhangalibaba@users.noreply.github.com> Date: Thu, 20 Feb 2020 11:33:27 -0800 Subject: [PATCH] [MultiDB] mimic redis-dump-load , adding sonic_db_dump_load.py and sonic-db-dump & sonic-db-load (#62) * [MultiDB] mimic redis-dump-load , adding sonic_db_dump_load.py and sonic-db-dump & sonic-db-load * Add original copyright statements --- ThirdPartyLicenses.txt | 34 +++++++- setup.py | 6 ++ src/swsssdk/__init__.py | 1 + src/swsssdk/sonic_db_dump_load.py | 139 ++++++++++++++++++++++++++++++ 4 files changed, 176 insertions(+), 4 deletions(-) create mode 100644 src/swsssdk/sonic_db_dump_load.py diff --git a/ThirdPartyLicenses.txt b/ThirdPartyLicenses.txt index 9d14ac044f76..482caef64b63 100644 --- a/ThirdPartyLicenses.txt +++ b/ThirdPartyLicenses.txt @@ -1,12 +1,12 @@ Third Party Notices -This Microsoft Open Source project incorporates material from the project(s) listed below -(Third Party Code). Microsoft is not the original author of the Third Party Code. Microsoft +This Microsoft Open Source project incorporates material from the project(s) listed below +(Third Party Code). Microsoft is not the original author of the Third Party Code. Microsoft reserves all other rights not expressly granted, whether by implication, estoppel or otherwise. 1. redis-py https://github.com/andymccurdy/redis-py -2. pyagentx https://github.com/rayed/pyagentx - +2. pyagentx https://github.com/rayed/pyagentx +3. redis-dump-load https://github.com/p/redis-dump-load 1. redis-py https://github.com/andymccurdy/redis-py Copyright (c) 2012 Andy McCurdy @@ -56,3 +56,29 @@ LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + +3. redis-dump-load https://github.com/p/redis-dump-load +Copyright (c) 2011-2016 Oleg Pudeyev +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + +1. Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. + +2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/setup.py b/setup.py index 2f3b70c6b572..735ab5b5d5b5 100644 --- a/setup.py +++ b/setup.py @@ -25,6 +25,12 @@ extras_require={ 'high_perf': high_performance_deps }, + entry_points={ + 'console_scripts': [ + 'sonic-db-load = swsssdk:sonic_db_dump_load', + 'sonic-db-dump = swsssdk:sonic_db_dump_load', + ], + }, classifiers=[ 'Intended Audience :: Developers', 'Operating System :: Linux', diff --git a/src/swsssdk/__init__.py b/src/swsssdk/__init__.py index 1722bb70182b..d7cf391f550c 100644 --- a/src/swsssdk/__init__.py +++ b/src/swsssdk/__init__.py @@ -10,6 +10,7 @@ try: from .dbconnector import SonicDBConfig, SonicV2Connector from .configdb import ConfigDBConnector + from .sonic_db_dump_load import sonic_db_dump_load except (KeyError, ValueError): msg = "Failed to database connector objects -- incorrect database config schema." logger.exception(msg) diff --git a/src/swsssdk/sonic_db_dump_load.py b/src/swsssdk/sonic_db_dump_load.py new file mode 100644 index 000000000000..0240c116d8af --- /dev/null +++ b/src/swsssdk/sonic_db_dump_load.py @@ -0,0 +1,139 @@ +## ref: https://github.com/p/redis-dump-load/blob/7bbdb1eaea0a51ed4758d3ce6ca01d497a4e7429/redisdl.py + +def sonic_db_dump_load(): + import optparse + import os.path + import re + import sys + from redisdl import dump, load + from swsssdk import SonicDBConfig + + DUMP = 1 + LOAD = 2 + + def options_to_kwargs(options): + args = {} + if options.password: + args['password'] = options.password + if options.encoding: + args['encoding'] = options.encoding + # dump only + if hasattr(options, 'pretty') and options.pretty: + args['pretty'] = True + if hasattr(options, 'keys') and options.keys: + args['keys'] = options.keys + # load only + if hasattr(options, 'use_expireat') and options.use_expireat: + args['use_expireat'] = True + if hasattr(options, 'empty') and options.empty: + args['empty'] = True + if hasattr(options, 'backend') and options.backend: + args['streaming_backend'] = options.backend + if hasattr(options, 'dbname') and options.dbname: + if options.conntype == 'tcp': + args['host'] = SonicDBConfig.get_hostname(options.dbname) + args['port'] = SonicDBConfig.get_port(options.dbname) + args['db'] = SonicDBConfig.get_dbid(options.dbname) + args['unix_socket_path'] = None + elif options.conntype == "unix_socket": + args['host'] = None + args['port'] = None + args['db'] = SonicDBConfig.get_dbid(options.dbname) + args['unix_socket_path'] = SonicDBConfig.get_socket(options.dbname) + else: + raise TypeError('redis connection type is tcp or unix_socket') + + return args + + def do_dump(options): + if options.output: + output = open(options.output, 'w') + else: + output = sys.stdout + + kwargs = options_to_kwargs(options) + dump(output, **kwargs) + + if options.output: + output.close() + + def do_load(options, args): + if len(args) > 0: + input = open(args[0], 'rb') + else: + input = sys.stdin + + kwargs = options_to_kwargs(options) + load(input, **kwargs) + + if len(args) > 0: + input.close() + + script_name = os.path.basename(sys.argv[0]) + if re.search(r'load(?:$|\.)', script_name): + action = help = LOAD + elif re.search(r'dump(?:$|\.)', script_name): + action = help = DUMP + else: + # default is dump, however if dump is specifically requested + # we don't show help text for toggling between dumping and loading + action = DUMP + help = None + + if help == LOAD: + usage = "Usage: %prog [options] [FILE]" + usage += "\n\nLoad data from FILE (which must be a JSON dump previously created" + usage += "\nby redisdl) into specified or default redis." + usage += "\n\nIf FILE is omitted standard input is read." + elif help == DUMP: + usage = "Usage: %prog [options]" + usage += "\n\nDump data from specified or default redis." + usage += "\n\nIf no output file is specified, dump to standard output." + else: + usage = "Usage: %prog [options]" + usage += "\n %prog -l [options] [FILE]" + usage += "\n\nDump data from redis or load data into redis." + usage += "\n\nIf input or output file is specified, dump to standard output and load" + usage += "\nfrom standard input." + parser = optparse.OptionParser(usage=usage) + parser.add_option('-w', '--password', help='connect with PASSWORD') + if help == DUMP: + parser.add_option('-n', '--dbname', help='dump DATABASE (APPL_DB/ASIC_DB...)') + parser.add_option('-t', '--conntype', help='indicate redis connection type (tcp[default] or unix_socket)', default='tcp') + parser.add_option('-k', '--keys', help='dump only keys matching specified glob-style pattern') + parser.add_option('-o', '--output', help='write to OUTPUT instead of stdout') + parser.add_option('-y', '--pretty', help='split output on multiple lines and indent it', action='store_true') + parser.add_option('-E', '--encoding', help='set encoding to use while decoding data from redis', default='utf-8') + elif help == LOAD: + parser.add_option('-n', '--dbname', help='dump DATABASE (APPL_DB/ASIC_DB...)') + parser.add_option('-t', '--conntype', help='indicate redis connection type (tcp[default] or unix_socket)', default='tcp') + parser.add_option('-e', '--empty', help='delete all keys in destination db prior to loading', action='store_true') + parser.add_option('-E', '--encoding', help='set encoding to use while encoding data to redis', default='utf-8') + parser.add_option('-B', '--backend', help='use specified streaming backend') + parser.add_option('-A', '--use-expireat', help='use EXPIREAT rather than TTL/EXPIRE', action='store_true') + else: + parser.add_option('-l', '--load', help='load data into redis (default is to dump data from redis)', action='store_true') + parser.add_option('-n', '--dbname', help='dump DATABASE (APPL_DB/ASIC_DB/COUNTERS_DB/LOGLEVEL_DB/CONFIG_DB...)') + parser.add_option('-t', '--conntype', help='indicate redis connection type (tcp[default] or unix_socket)', default='tcp') + parser.add_option('-k', '--keys', help='dump only keys matching specified glob-style pattern') + parser.add_option('-o', '--output', help='write to OUTPUT instead of stdout (dump mode only)') + parser.add_option('-y', '--pretty', help='split output on multiple lines and indent it (dump mode only)', action='store_true') + parser.add_option('-e', '--empty', help='delete all keys in destination db prior to loading (load mode only)', action='store_true') + parser.add_option('-E', '--encoding', help='set encoding to use while decoding data from redis', default='utf-8') + parser.add_option('-A', '--use-expireat', help='use EXPIREAT rather than TTL/EXPIRE', action='store_true') + parser.add_option('-B', '--backend', help='use specified streaming backend (load mode only)') + options, args = parser.parse_args() + + if hasattr(options, 'load') and options.load: + action = LOAD + + if action == DUMP: + if len(args) > 0: + parser.print_help() + exit(4) + do_dump(options) + else: + if len(args) > 1: + parser.print_help() + exit(4) + do_load(options, args)