Verify port status in APPL_DB after shutting down interfaces.

config/config_mgmt.py

@@ -248,7 +248,77 @@ def __init__(self, source="configDB", debug=False, allowTablesWithoutYang=True):
     def __del__(self):
         pass
 
-    def _checkKeyinAsicDB(self, key, db):
+    def _checkPortsDownAppDb(self, db, ports):
+        '''
+        Check APP DB for PORT oper status down.
+
+        Parameters:
+            db (SonicV2Connector): database.
+            ports (list): List of ports
+
+        Returns:
+            (bool): True, if all ports are oper down.
+        '''
+        try:
+            # connect to APP DB,
+            db.connect(db.APPL_DB)
+            for port in ports:
+                # Key format "PORT_TABLE:Ethernet112"
+                key = 'PORT_TABLE:{}'.format(port)
+                portOperStatus = db.get('APPL_DB', key, 'oper_status')
+                self.sysLog(syslog.LOG_DEBUG, "APPL_DB: {} {}".\
+                    format(key, portOperStatus))
+                if portOperStatus is None or portOperStatus != 'down':
+                    return False
+
+        except Exception as e:
+            self.sysLog(doPrint=True, logLevel=syslog.LOG_ERR, msg=str(e))
+            return False
+
+        return True
+
+    def _verifyAppDB(self, db, ports, timeout):
+        '''
+        Verify in the App DB that ports are oper_status down,
+        Keep on trying till timeout period.
+
+        Parameters:
+            db (SonicV2Connector): database.
+            ports (list): List of ports
+            timeout (int): timeout period.
+
+        Returns:
+            (bool): True, if all ports are oper down.
+        '''
+        self.sysLog(msg="Verify Port oper_status down from App DB, Wait...")
+
+        try:
+            for waitTime in range(timeout):
+                self.sysLog(logLevel=syslog.LOG_DEBUG, \
+                    msg='Check App DB: {} try'.format(waitTime+1))
+                # _checkPortsDownAppDb will return True if all ports are Oper
+                # down in APP DB
+                if self._checkPortsDownAppDb(db, ports):
+                    break
+                tsleep(1)
+
+            # raise if timer expired
+            if waitTime + 1 == timeout:
+                self.sysLog(logLevel=syslog.LOG_CRIT, \
+                    msg="!!! Critical Failure, Ports are not yet Operation \
+                    Down in App DB, Bail Out !!!",  doPrint=True)
+                raise(Exception("Ports are not Operation Down in App DB after {} secs"\
+                    .format(timeout)))
+
+        except Exception as e:
+            self.sysLog(doPrint=True, logLevel=syslog.LOG_ERR, msg=str(e))
+            self.sysLog(doPrint=True, logLevel=syslog.LOG_ERR, \
+                msg="_verifyAppDB failed {}".format(str(e)))
+            raise e
+
+        return True
+
+    def _checkKeyInAsicDB(self, key, db):
         '''
         Check if a key exists in ASIC DB or not.
 
@@ -287,7 +357,7 @@ def _checkNoPortsInAsicDb(self, db, ports, portMap):
             db.connect(db.ASIC_DB)
             for port in ports:
                 key = self.oidKey + portMap[port]
-                if self._checkKeyinAsicDB(key, db) == True:
+                if self._checkKeyInAsicDB(key, db) == True:
                     return False
 
         except Exception as e:
@@ -371,12 +441,13 @@ def breakOutPort(self, delPorts=list(), portJson=dict(), force=False, \
 
             # If we are here, then get ready to update the Config DB as below:
             # -- shutdown the ports,
+            # -- verify App DB for port status,
             # -- Update deletion of ports in Config DB,
             # -- verify Asic DB for port deletion,
             # -- then update addition of ports in config DB.
             self._shutdownIntf(delPorts)
+            self._verifyAppDB(db=dataBase, ports=delPorts, timeout=MAX_WAIT)
             self.writeConfigDB(delConfigToLoad)
-            # Verify in Asic DB,
             self._verifyAsicDB(db=dataBase, ports=delPorts, portMap=if_name_map, \
                 timeout=MAX_WAIT)
             self.writeConfigDB(addConfigtoLoad)

tests/config_dpb_test.py

@@ -170,6 +170,7 @@ def config_mgmt_dpb(cfgdb):
     cmdpb = config_mgmt.ConfigMgmtDPB(source=config_mgmt.CONFIG_DB_JSON_FILE)
     # mock funcs
     cmdpb.writeConfigDB = mock.MagicMock(return_value=True)
+    cmdpb._verifyAppDB  = mock.MagicMock(return_value=True)
     cmdpb._verifyAsicDB = mock.MagicMock(return_value=True)
     return cmdpb
 

tests/config_mgmt_test.py

@@ -6,11 +6,15 @@
 
 import pytest
 from utilities_common.general import load_module_from_source
+from swsssdk import ConfigDBConnector
 
 # Import file under test i.e., config_mgmt.py
 config_mgmt_py_path = os.path.join(os.path.dirname(__file__), '..', 'config', 'config_mgmt.py')
 config_mgmt = load_module_from_source('config_mgmt', config_mgmt_py_path)
 
+# import file sonic_cfggen
+load_module_from_source('sonic_cfggen', '/usr/local/bin/sonic-cfggen')
+from sonic_cfggen import deep_update, FormatConverter
 
 class TestConfigMgmt(TestCase):
     '''
@@ -78,6 +82,71 @@ def test_break_out(self):
         self.dpb_port8_1x100G_1x50G_2x25G_f_l(curConfig)
         # Ethernet4: breakout from 4x25G to 2x50G with -f -l
         self.dpb_port4_4x25G_2x50G_f_l(curConfig)
+
+        return
+
+    def test_verifyAppDB_call(self):
+        '''
+        Verify that verifyAppDB() is called with deleted ports while calling
+        breakOutPort()
+        '''
+        conf = dict(configDbJson)
+        cmdpb = self.config_mgmt_dpb(conf)
+        print(conf['PORT'])
+        # create ARGS
+        dPorts,  pJson = self.generate_args(portIdx=8, laneIdx=73, \
+            curMode='1x50G(2)+2x25G(2)', newMode='2x50G')
+        # Try to breakout and see if verifyAppDB is called
+        cmdpb.breakOutPort(delPorts=dPorts, portJson=pJson, \
+            force=True, loadDefConfig=False)
+        print(conf['PORT'])
+
+        # verify correct function call
+        assert cmdpb._verifyAppDB.call_count == 1
+        print(cmdpb._verifyAppDB.call_args_list[0])
+        (args, kwargs) = cmdpb._verifyAppDB.call_args_list[0]
+        print(kwargs)
+        assert len(kwargs) == 3
+        assert kwargs['ports'] == dPorts
+        return
+
+    def test_verifyAppDB_success(self):
+        '''
+            Verify that verifyAppDB() return False, is any deleted port is up
+            in APPL_DB. And return success when all delPorts are admin down.
+        '''
+        curConfig = dict(configDbJson)
+        self.writeJson(curConfig, config_mgmt.CONFIG_DB_JSON_FILE)
+        cmdpb = config_mgmt.ConfigMgmtDPB(source=config_mgmt.CONFIG_DB_JSON_FILE)
+
+        # Bit of hack here, we use ConfigDBConnector class to connect to APPL_DB
+        # So that we can update oper_status in PORT_TABLE of APPL_DB.
+        db = ConfigDBConnector()
+        db.db_connect('APPL_DB')
+        db.connect = mock.MagicMock(return_value=True)
+        # get dPorts
+        dPorts, _ = self.generate_args(portIdx=8, laneIdx=73, \
+            curMode='4x25G', newMode='2x50G')
+        # Make sure _verifyAppDB raises Exception
+        try:
+            ret = cmdpb._verifyAppDB(db, dPorts, 3)
+        except Exception as e:
+            assert 'Ports are not Operation Down in App DB' in str(e)
+        # set ports 8, 9, 10 admin down
+        self.update_db(db, {'PORT_TABLE': {'Ethernet8': {'oper_status': 'down'}}})
+        self.update_db(db, {'PORT_TABLE': {'Ethernet9': {'oper_status': 'down'}}})
+        self.update_db(db, {'PORT_TABLE': {'Ethernet10': {'oper_status': 'down'}}})
+        # Make sure _verifyAppDB raises Exception even with 1 port up/None
+        try:
+            ret = cmdpb._verifyAppDB(db, dPorts, 3)
+        except Exception as e:
+            assert 'Ports are not Operation Down in App DB' in str(e)
+        # set ports 11 admin down
+        self.update_db(db, {'PORT_TABLE': {'Ethernet11': {'oper_status': 'down'}}})
+        # Make sure _verifyAppDB ret True
+        ret = cmdpb._verifyAppDB(db, dPorts, 3)
+        assert ret == True
+
         return
 
     @pytest.mark.skip(reason="not stable")
@@ -125,6 +194,12 @@ def tearDown(self):
         return
 
     ########### HELPER FUNCS #####################################
+    def update_db(self, db, tables):
+        data = dict()
+        deep_update(data, FormatConverter.to_deserialized(tables))
+        db.mod_config(FormatConverter.output_to_db(data))
+        return
+
     def writeJson(self, d, file):
         with open(file, 'w') as f:
             dump(d, f, indent=4)
@@ -146,6 +221,7 @@ def config_mgmt_dpb(self, curConfig):
         cmdpb = config_mgmt.ConfigMgmtDPB(source=config_mgmt.CONFIG_DB_JSON_FILE)
         # mock funcs
         cmdpb.writeConfigDB = mock.MagicMock(return_value=True)
+        cmdpb._verifyAppDB  = mock.MagicMock(return_value=True)
         cmdpb._verifyAsicDB = mock.MagicMock(return_value=True)
         from .mock_tables import dbconnector
         return cmdpb