[db migrator] Introduce the DB migration infrastructure (sonic-net#519)

* [db migrator] Introduce the DB migration infrastructure

- Add infrastructure to migrate DB contents from a lower version schema
  to a higher version schema, eventually reach latest version.
- if config load_minigraph is executed, the db schema version is always
  the latest.

Signed-off-by: Ying Xie <ying.xie@microsoft.com>

* [db migrator] address issues found in test

config/main.py

@@ -387,6 +387,12 @@ def reload(filename, yes, load_sysinfo):
     command = "{} -j {} --write-to-db".format(SONIC_CFGGEN_PATH, filename)
     run_command(command, display_cmd=True)
     client.set(config_db.INIT_INDICATOR, 1)
+
+    # Migrate DB contents to latest version
+    db_migrator='/usr/bin/db_migrator.py'
+    if os.path.isfile(db_migrator) and os.access(db_migrator, os.X_OK):
+        run_command(db_migrator + ' -o migrate')
+
     _restart_services()
 
 @config.command()
@@ -437,6 +443,12 @@ def load_minigraph():
     if os.path.isfile('/etc/sonic/acl.json'):
         run_command("acl-loader update full /etc/sonic/acl.json", display_cmd=True)
     run_command("config qos reload", display_cmd=True)
+
+    # Write latest db version string into db
+    db_migrator='/usr/bin/db_migrator.py'
+    if os.path.isfile(db_migrator) and os.access(db_migrator, os.X_OK):
+        run_command(db_migrator + ' -o set_version')
+
     #FIXME: After config DB daemon is implemented, we'll no longer need to restart every service.
     _restart_services()
     click.echo("Please note setting loaded from minigraph will be lost after system reboot. To preserve setting, run `config save`.")

scripts/db_migrator.py

@@ -0,0 +1,138 @@
+#!/usr/bin/env python
+
+import sys
+import argparse
+import syslog
+from swsssdk import ConfigDBConnector
+
+
+SYSLOG_IDENTIFIER = 'db_migrator'
+
+
+def log_info(msg):
+    syslog.openlog(SYSLOG_IDENTIFIER)
+    syslog.syslog(syslog.LOG_INFO, msg)
+    syslog.closelog()
+
+
+def log_error(msg):
+    syslog.openlog(SYSLOG_IDENTIFIER)
+    syslog.syslog(syslog.LOG_ERR, msg)
+    syslog.closelog()
+
+
+class DBMigrator():
+    def __init__(self):
+        """
+        Version string format:
+           version_<major>_<minor>_<build>
+              major: starting from 1, sequentially incrementing in master
+                     branch.
+              minor: in github branches, minor version stays in 0. This minor
+                     version creates space for private branches derived from
+                     github public branches. These private branches shall use
+                     none-zero values.
+              build: sequentially increase within a minor version domain.
+        """
+        self.CURRENT_VERSION = 'version_1_0_1'
+
+        self.TABLE_NAME      = 'VERSIONS'
+        self.TABLE_KEY       = 'DATABASE'
+        self.TABLE_FIELD     = 'VERSION'
+        self.configDB        = ConfigDBConnector()
+        self.configDB.db_connect('CONFIG_DB')
+
+
+    def migrate_pfc_wd_table(self):
+        # Migrate all data entries from table PFC_WD_TABLE to PFC_WD
+        data = self.configDB.get_table('PFC_WD_TABLE')
+        for key in data.keys():
+            self.configDB.set_entry('PFC_WD', key, data[key])
+        self.configDB.delete_table('PFC_WD_TABLE')
+
+
+    def version_unknown(self):
+        """
+        version_unknown tracks all SONiC versions that doesn't have a version
+        string defined in config_DB.
+        Nothing can be assumped when migrating from this version to the next
+        version.
+        Any migration operation needs to test if the DB is in expected format
+        before migrating date to the next version.
+        """
+
+        log_info('Handling version_unknown')
+
+        # NOTE: Uncomment next 3 lines of code when the migration code is in
+        #       place. Note that returning specific string is intentional,
+        #       here we only intended to migrade to DB version 1.0.1.
+        #       If new DB version is added in the future, the incremental
+        #       upgrade will take care of the subsequent migrations.
+        # self.migrate_pfc_wd_table()
+        # self.set_version('version_1_0_1')
+        # return 'version_1_0_1'
+
+
+    def version_1_0_1(self):
+        """
+        Current latest version. Nothing to do here.
+        """
+        log_info('Handling version_1_0_1')
+
+        return None
+
+
+    def get_version(self):
+        version = self.configDB.get_entry(self.TABLE_NAME, self.TABLE_KEY)
+        if version and version[self.TABLE_FIELD]:
+            return version[self.TABLE_FIELD]
+
+        return 'version_unknown'
+
+
+    def set_version(self, version=None):
+        if not version:
+            version = self.CURRENT_VERSION
+        log_info('Setting version to ' + version)
+        entry = { self.TABLE_FIELD : version }
+        self.configDB.set_entry(self.TABLE_NAME, self.TABLE_KEY, entry)
+
+
+    def migrate(self):
+        version = self.get_version()
+        log_info('Upgrading from version ' + version)
+        while version:
+            next_version = getattr(self, version)()
+            if next_version == version:
+                raise Exception('Version migrate from %s stuck in same version' % version)
+            version = next_version
+
+
+def main():
+    try:
+        parser = argparse.ArgumentParser()
+
+        parser.add_argument('-o',
+                            dest='operation',
+                            metavar='operation (migrate, set_version, get_version)',
+                            type = str,
+                            required = False,
+                            choices=['migrate', 'set_version', 'get_version'],
+                            help = 'operation to perform [default: get_version]',
+                            default='get_version')
+        args = parser.parse_args()
+        operation = args.operation
+
+        dbmgtr = DBMigrator()
+        result = getattr(dbmgtr, operation)()
+        if result:
+            print(str(result))
+
+    except Exception as e:
+        log_error('Caught excetion: ' + str(e))
+        parser.print_help()
+        sys.exit(1)
+
+
+if __name__ == "__main__":
+    main()

setup.py

@@ -54,6 +54,7 @@
         'scripts/aclshow',
         'scripts/boot_part',
         'scripts/coredump-compress',
+        'scripts/db_migrator.py',
         'scripts/decode-syseeprom',
         'scripts/dropcheck',
         'scripts/ecnconfig',