diff --git a/config/config_mgmt.py b/config/config_mgmt.py index 9b2021bef0..2360c85e19 100644 --- a/config/config_mgmt.py +++ b/config/config_mgmt.py @@ -248,7 +248,77 @@ def __init__(self, source="configDB", debug=False, allowTablesWithoutYang=True): def __del__(self): pass - def _checkKeyinAsicDB(self, key, db): + def _checkPortsDownAppDb(self, db, ports): + ''' + Check APP DB for PORT oper status down. + + Parameters: + db (SonicV2Connector): database. + ports (list): List of ports + + Returns: + (bool): True, if all ports are oper down. + ''' + try: + # connect to APP DB, + db.connect(db.APPL_DB) + for port in ports: + # Key format "PORT_TABLE:Ethernet112" + key = 'PORT_TABLE:{}'.format(port) + portOperStatus = db.get('APPL_DB', key, 'oper_status') + self.sysLog(syslog.LOG_DEBUG, "APPL_DB: {} {}".\ + format(key, portOperStatus)) + if portOperStatus is None or portOperStatus != 'down': + return False + + except Exception as e: + self.sysLog(doPrint=True, logLevel=syslog.LOG_ERR, msg=str(e)) + return False + + return True + + def _verifyAppDB(self, db, ports, timeout): + ''' + Verify in the App DB that ports are oper_status down, + Keep on trying till timeout period. + + Parameters: + db (SonicV2Connector): database. + ports (list): List of ports + timeout (int): timeout period. + + Returns: + (bool): True, if all ports are oper down. + ''' + self.sysLog(msg="Verify Port oper_status down from App DB, Wait...") + + try: + for waitTime in range(timeout): + self.sysLog(logLevel=syslog.LOG_DEBUG, \ + msg='Check App DB: {} try'.format(waitTime+1)) + # _checkPortsDownAppDb will return True if all ports are Oper + # down in APP DB + if self._checkPortsDownAppDb(db, ports): + break + tsleep(1) + + # raise if timer expired + if waitTime + 1 == timeout: + self.sysLog(logLevel=syslog.LOG_CRIT, \ + msg="!!! Critical Failure, Ports are not yet Operation \ + Down in App DB, Bail Out !!!", doPrint=True) + raise(Exception("Ports are not Operation Down in App DB after {} secs"\ + .format(timeout))) + + except Exception as e: + self.sysLog(doPrint=True, logLevel=syslog.LOG_ERR, msg=str(e)) + self.sysLog(doPrint=True, logLevel=syslog.LOG_ERR, \ + msg="_verifyAppDB failed {}".format(str(e))) + raise e + + return True + + def _checkKeyInAsicDB(self, key, db): ''' Check if a key exists in ASIC DB or not. @@ -287,7 +357,7 @@ def _checkNoPortsInAsicDb(self, db, ports, portMap): db.connect(db.ASIC_DB) for port in ports: key = self.oidKey + portMap[port] - if self._checkKeyinAsicDB(key, db) == True: + if self._checkKeyInAsicDB(key, db) == True: return False except Exception as e: @@ -371,12 +441,13 @@ def breakOutPort(self, delPorts=list(), portJson=dict(), force=False, \ # If we are here, then get ready to update the Config DB as below: # -- shutdown the ports, + # -- verify App DB for port status, # -- Update deletion of ports in Config DB, # -- verify Asic DB for port deletion, # -- then update addition of ports in config DB. self._shutdownIntf(delPorts) + self._verifyAppDB(db=dataBase, ports=delPorts, timeout=MAX_WAIT) self.writeConfigDB(delConfigToLoad) - # Verify in Asic DB, self._verifyAsicDB(db=dataBase, ports=delPorts, portMap=if_name_map, \ timeout=MAX_WAIT) self.writeConfigDB(addConfigtoLoad) diff --git a/tests/config_dpb_test.py b/tests/config_dpb_test.py index 1251f1e02f..8611daa5a6 100644 --- a/tests/config_dpb_test.py +++ b/tests/config_dpb_test.py @@ -170,6 +170,7 @@ def config_mgmt_dpb(cfgdb): cmdpb = config_mgmt.ConfigMgmtDPB(source=config_mgmt.CONFIG_DB_JSON_FILE) # mock funcs cmdpb.writeConfigDB = mock.MagicMock(return_value=True) + cmdpb._verifyAppDB = mock.MagicMock(return_value=True) cmdpb._verifyAsicDB = mock.MagicMock(return_value=True) return cmdpb diff --git a/tests/config_mgmt_test.py b/tests/config_mgmt_test.py index 41b7f5883d..50fe1cab09 100644 --- a/tests/config_mgmt_test.py +++ b/tests/config_mgmt_test.py @@ -6,11 +6,15 @@ import pytest from utilities_common.general import load_module_from_source +from swsssdk import ConfigDBConnector # Import file under test i.e., config_mgmt.py config_mgmt_py_path = os.path.join(os.path.dirname(__file__), '..', 'config', 'config_mgmt.py') config_mgmt = load_module_from_source('config_mgmt', config_mgmt_py_path) +# import file sonic_cfggen +load_module_from_source('sonic_cfggen', '/usr/local/bin/sonic-cfggen') +from sonic_cfggen import deep_update, FormatConverter class TestConfigMgmt(TestCase): ''' @@ -78,6 +82,71 @@ def test_break_out(self): self.dpb_port8_1x100G_1x50G_2x25G_f_l(curConfig) # Ethernet4: breakout from 4x25G to 2x50G with -f -l self.dpb_port4_4x25G_2x50G_f_l(curConfig) + + return + + def test_verifyAppDB_call(self): + ''' + Verify that verifyAppDB() is called with deleted ports while calling + breakOutPort() + ''' + conf = dict(configDbJson) + cmdpb = self.config_mgmt_dpb(conf) + print(conf['PORT']) + # create ARGS + dPorts, pJson = self.generate_args(portIdx=8, laneIdx=73, \ + curMode='1x50G(2)+2x25G(2)', newMode='2x50G') + # Try to breakout and see if verifyAppDB is called + cmdpb.breakOutPort(delPorts=dPorts, portJson=pJson, \ + force=True, loadDefConfig=False) + print(conf['PORT']) + + # verify correct function call + assert cmdpb._verifyAppDB.call_count == 1 + print(cmdpb._verifyAppDB.call_args_list[0]) + (args, kwargs) = cmdpb._verifyAppDB.call_args_list[0] + print(kwargs) + assert len(kwargs) == 3 + assert kwargs['ports'] == dPorts + return + + def test_verifyAppDB_success(self): + ''' + Verify that verifyAppDB() return False, is any deleted port is up + in APPL_DB. And return success when all delPorts are admin down. + ''' + curConfig = dict(configDbJson) + self.writeJson(curConfig, config_mgmt.CONFIG_DB_JSON_FILE) + cmdpb = config_mgmt.ConfigMgmtDPB(source=config_mgmt.CONFIG_DB_JSON_FILE) + + # Bit of hack here, we use ConfigDBConnector class to connect to APPL_DB + # So that we can update oper_status in PORT_TABLE of APPL_DB. + db = ConfigDBConnector() + db.db_connect('APPL_DB') + db.connect = mock.MagicMock(return_value=True) + # get dPorts + dPorts, _ = self.generate_args(portIdx=8, laneIdx=73, \ + curMode='4x25G', newMode='2x50G') + # Make sure _verifyAppDB raises Exception + try: + ret = cmdpb._verifyAppDB(db, dPorts, 3) + except Exception as e: + assert 'Ports are not Operation Down in App DB' in str(e) + # set ports 8, 9, 10 admin down + self.update_db(db, {'PORT_TABLE': {'Ethernet8': {'oper_status': 'down'}}}) + self.update_db(db, {'PORT_TABLE': {'Ethernet9': {'oper_status': 'down'}}}) + self.update_db(db, {'PORT_TABLE': {'Ethernet10': {'oper_status': 'down'}}}) + # Make sure _verifyAppDB raises Exception even with 1 port up/None + try: + ret = cmdpb._verifyAppDB(db, dPorts, 3) + except Exception as e: + assert 'Ports are not Operation Down in App DB' in str(e) + # set ports 11 admin down + self.update_db(db, {'PORT_TABLE': {'Ethernet11': {'oper_status': 'down'}}}) + # Make sure _verifyAppDB ret True + ret = cmdpb._verifyAppDB(db, dPorts, 3) + assert ret == True + return @pytest.mark.skip(reason="not stable") @@ -125,6 +194,12 @@ def tearDown(self): return ########### HELPER FUNCS ##################################### + def update_db(self, db, tables): + data = dict() + deep_update(data, FormatConverter.to_deserialized(tables)) + db.mod_config(FormatConverter.output_to_db(data)) + return + def writeJson(self, d, file): with open(file, 'w') as f: dump(d, f, indent=4) @@ -146,6 +221,7 @@ def config_mgmt_dpb(self, curConfig): cmdpb = config_mgmt.ConfigMgmtDPB(source=config_mgmt.CONFIG_DB_JSON_FILE) # mock funcs cmdpb.writeConfigDB = mock.MagicMock(return_value=True) + cmdpb._verifyAppDB = mock.MagicMock(return_value=True) cmdpb._verifyAsicDB = mock.MagicMock(return_value=True) from .mock_tables import dbconnector return cmdpb