diff --git a/orchagent/aclorch.cpp b/orchagent/aclorch.cpp index 7051f792838..946c952a868 100644 --- a/orchagent/aclorch.cpp +++ b/orchagent/aclorch.cpp @@ -1083,12 +1083,6 @@ bool AclRuleMirror::validateAddAction(string attr_name, string attr_value) m_sessionName = attr_value; - if (!m_pMirrorOrch->sessionExists(m_sessionName)) - { - SWSS_LOG_ERROR("Mirror rule reference mirror session that does not exists %s", m_sessionName.c_str()); - return false; - } - // insert placeholder value, we'll set the session oid in AclRuleMirror::create() m_actions[action] = sai_attribute_value_t{}; @@ -1178,6 +1172,12 @@ bool AclRuleMirror::create() sai_object_id_t oid = SAI_NULL_OBJECT_ID; bool state = false; + if (!m_pMirrorOrch->sessionExists(m_sessionName)) + { + SWSS_LOG_ERROR("Mirror rule references mirror session \"%s\" that does not exist yet", m_sessionName.c_str()); + return false; + } + if (!m_pMirrorOrch->getSessionStatus(m_sessionName, state)) { SWSS_LOG_THROW("Failed to get mirror session state for session %s", m_sessionName.c_str()); @@ -3124,7 +3124,16 @@ void AclOrch::doAclRuleTask(Consumer &consumer) } - newRule = AclRule::makeShared(type, this, m_mirrorOrch, m_dTelOrch, rule_id, table_id, t); + try + { + newRule = AclRule::makeShared(type, this, m_mirrorOrch, m_dTelOrch, rule_id, table_id, t); + } + catch (exception &e) + { + SWSS_LOG_ERROR("Error while creating ACL rule %s: %s", rule_id.c_str(), e.what()); + it = consumer.m_toSync.erase(it); + return; + } for (const auto& itr : kfvFieldsValues(t)) { diff --git a/orchagent/aclorch.h b/orchagent/aclorch.h index f4037d38312..f0230dbb9a7 100644 --- a/orchagent/aclorch.h +++ b/orchagent/aclorch.h @@ -434,6 +434,8 @@ class AclOrch : public Orch, public Observer // Get the OID for the ACL bind point for a given port static bool getAclBindPortId(Port& port, sai_object_id_t& port_id); + using Orch::doTask; // Allow access to the basic doTask + private: SwitchOrch *m_switchOrch; void doTask(Consumer &consumer); diff --git a/orchagent/mirrororch.cpp b/orchagent/mirrororch.cpp index 1abb522e21a..2878a5265d2 100644 --- a/orchagent/mirrororch.cpp +++ b/orchagent/mirrororch.cpp @@ -87,9 +87,6 @@ bool MirrorOrch::bake() { SWSS_LOG_ENTER(); - // Freeze the route update during orchagent restoration - m_freeze = true; - deque entries; vector keys; m_mirrorTable.getKeys(keys); @@ -134,23 +131,6 @@ bool MirrorOrch::bake() return Orch::bake(); } -bool MirrorOrch::postBake() -{ - SWSS_LOG_ENTER(); - - SWSS_LOG_NOTICE("Start MirrorOrch post-baking"); - - // Unfreeze the route update - m_freeze = false; - - Orch::doTask(); - - // Clean up the recovery cache - m_recoverySessionMap.clear(); - - return Orch::postBake(); -} - void MirrorOrch::update(SubjectType type, void *cntx) { SWSS_LOG_ENTER(); @@ -340,7 +320,7 @@ bool MirrorOrch::validateSrcPortList(const string& srcPortList) return true; } -void MirrorOrch::createEntry(const string& key, const vector& data) +task_process_status MirrorOrch::createEntry(const string& key, const vector& data) { SWSS_LOG_ENTER(); @@ -349,7 +329,7 @@ void MirrorOrch::createEntry(const string& key, const vector& d { SWSS_LOG_NOTICE("Failed to create session, session %s already exists", key.c_str()); - return; + return task_process_status::task_duplicated; } string platform = getenv("platform") ? getenv("platform") : ""; @@ -364,7 +344,7 @@ void MirrorOrch::createEntry(const string& key, const vector& d if (!entry.srcIp.isV4()) { SWSS_LOG_ERROR("Unsupported version of sessions %s source IP address", key.c_str()); - return; + return task_process_status::task_invalid_entry; } } else if (fvField(i) == MIRROR_SESSION_DST_IP) @@ -373,7 +353,7 @@ void MirrorOrch::createEntry(const string& key, const vector& d if (!entry.dstIp.isV4()) { SWSS_LOG_ERROR("Unsupported version of sessions %s destination IP address", key.c_str()); - return; + return task_process_status::task_invalid_entry; } } else if (fvField(i) == MIRROR_SESSION_GRE_TYPE) @@ -398,7 +378,7 @@ void MirrorOrch::createEntry(const string& key, const vector& d { SWSS_LOG_ERROR("Failed to get policer %s", fvValue(i).c_str()); - return; + return task_process_status::task_need_retry; } m_policerOrch->increaseRefCount(fvValue(i)); @@ -409,7 +389,7 @@ void MirrorOrch::createEntry(const string& key, const vector& d if (!validateSrcPortList(fvValue(i))) { SWSS_LOG_ERROR("Failed to get valid source port list %s", fvValue(i).c_str()); - return; + return task_process_status::task_invalid_entry; } entry.src_port = fvValue(i); } @@ -418,7 +398,7 @@ void MirrorOrch::createEntry(const string& key, const vector& d if (!validateDstPort(fvValue(i))) { SWSS_LOG_ERROR("Failed to get valid destination port %s", fvValue(i).c_str()); - return; + return task_process_status::task_invalid_entry; } entry.dst_port = fvValue(i); } @@ -428,7 +408,7 @@ void MirrorOrch::createEntry(const string& key, const vector& d || fvValue(i) == MIRROR_BOTH_DIRECTION)) { SWSS_LOG_ERROR("Failed to get valid direction %s", fvValue(i).c_str()); - return; + return task_process_status::task_invalid_entry; } entry.direction = fvValue(i); } @@ -439,18 +419,18 @@ void MirrorOrch::createEntry(const string& key, const vector& d else { SWSS_LOG_ERROR("Failed to parse session %s configuration. Unknown attribute %s", key.c_str(), fvField(i).c_str()); - return; + return task_process_status::task_invalid_entry; } } catch (const exception& e) { SWSS_LOG_ERROR("Failed to parse session %s attribute %s error: %s.", key.c_str(), fvField(i).c_str(), e.what()); - return; + return task_process_status::task_invalid_entry; } catch (...) { SWSS_LOG_ERROR("Failed to parse session %s attribute %s. Unknown error has been occurred", key.c_str(), fvField(i).c_str()); - return; + return task_process_status::task_failed; } } @@ -470,6 +450,8 @@ void MirrorOrch::createEntry(const string& key, const vector& d // Attach the destination IP to the routeOrch m_routeOrch->attach(this, entry.dstIp); } + + return task_process_status::task_success; } task_process_status MirrorOrch::deleteEntry(const string& name) @@ -1412,11 +1394,6 @@ void MirrorOrch::doTask(Consumer& consumer) { SWSS_LOG_ENTER(); - if (m_freeze) - { - return; - } - if (!gPortsOrch->allPortsReady()) { return; @@ -1429,26 +1406,32 @@ void MirrorOrch::doTask(Consumer& consumer) string key = kfvKey(t); string op = kfvOp(t); + task_process_status task_status = task_process_status::task_failed; if (op == SET_COMMAND) { - createEntry(key, kfvFieldsValues(t)); + task_status = createEntry(key, kfvFieldsValues(t)); } else if (op == DEL_COMMAND) { - auto task_status = deleteEntry(key); - // Specifically retry the task when asked - if (task_status == task_process_status::task_need_retry) - { - it++; - continue; - } + task_status = deleteEntry(key); } else { SWSS_LOG_ERROR("Unknown operation type %s", op.c_str()); } - consumer.m_toSync.erase(it++); + // Specifically retry the task when asked + if (task_status == task_process_status::task_need_retry) + { + it++; + } + else + { + consumer.m_toSync.erase(it++); + } } + + // Clear any recovery state that might be leftover from warm reboot + m_recoverySessionMap.clear(); } diff --git a/orchagent/mirrororch.h b/orchagent/mirrororch.h index 5edbf0ee456..1e0b19c1e67 100644 --- a/orchagent/mirrororch.h +++ b/orchagent/mirrororch.h @@ -80,7 +80,6 @@ class MirrorOrch : public Orch, public Observer, public Subject PortsOrch *portOrch, RouteOrch *routeOrch, NeighOrch *neighOrch, FdbOrch *fdbOrch, PolicerOrch *policerOrch); bool bake() override; - bool postBake() override; void update(SubjectType, void *); bool sessionExists(const string&); bool getSessionStatus(const string&, bool&); @@ -88,6 +87,8 @@ class MirrorOrch : public Orch, public Observer, public Subject bool increaseRefCount(const string&); bool decreaseRefCount(const string&); + using Orch::doTask; // Allow access to the basic doTask + private: PortsOrch *m_portsOrch; RouteOrch *m_routeOrch; @@ -101,9 +102,7 @@ class MirrorOrch : public Orch, public Observer, public Subject // session_name -> VLAN | monitor_port_alias | next_hop_ip map m_recoverySessionMap; - bool m_freeze = false; - - void createEntry(const string&, const vector&); + task_process_status createEntry(const string&, const vector&); task_process_status deleteEntry(const string&); bool activateSession(const string&, MirrorEntry&); diff --git a/orchagent/orch.cpp b/orchagent/orch.cpp index 35eaaf7b588..b29f95f563b 100644 --- a/orchagent/orch.cpp +++ b/orchagent/orch.cpp @@ -299,13 +299,6 @@ bool Orch::bake() return true; } -bool Orch::postBake() -{ - SWSS_LOG_ENTER(); - - return true; -} - /* - Validates reference has proper format which is [table_name:object_name] - validates table_name exists diff --git a/orchagent/orch.h b/orchagent/orch.h index 1cdb7bc6200..1a7197dfa97 100644 --- a/orchagent/orch.h +++ b/orchagent/orch.h @@ -50,7 +50,8 @@ typedef enum task_invalid_entry, task_failed, task_need_retry, - task_ignore + task_ignore, + task_duplicated } task_process_status; typedef struct @@ -204,8 +205,6 @@ class Orch // Prepare for warm start if Redis contains valid input data // otherwise fallback to cold start virtual bool bake(); - // Clean up the state set in bake() - virtual bool postBake(); /* Iterate all consumers in m_consumerMap and run doTask(Consumer) */ virtual void doTask(); diff --git a/orchagent/orchdaemon.cpp b/orchagent/orchdaemon.cpp index 480891a1892..7661c0494fc 100644 --- a/orchagent/orchdaemon.cpp +++ b/orchagent/orchdaemon.cpp @@ -32,6 +32,7 @@ NeighOrch *gNeighOrch; RouteOrch *gRouteOrch; FgNhgOrch *gFgNhgOrch; AclOrch *gAclOrch; +MirrorOrch *gMirrorOrch; CrmOrch *gCrmOrch; BufferOrch *gBufferOrch; SwitchOrch *gSwitchOrch; @@ -177,7 +178,7 @@ bool OrchDaemon::init() TableConnector stateDbMirrorSession(m_stateDb, STATE_MIRROR_SESSION_TABLE_NAME); TableConnector confDbMirrorSession(m_configDb, CFG_MIRROR_SESSION_TABLE_NAME); - MirrorOrch *mirror_orch = new MirrorOrch(stateDbMirrorSession, confDbMirrorSession, gPortsOrch, gRouteOrch, gNeighOrch, gFdbOrch, policer_orch); + gMirrorOrch = new MirrorOrch(stateDbMirrorSession, confDbMirrorSession, gPortsOrch, gRouteOrch, gNeighOrch, gFdbOrch, policer_orch); TableConnector confDbAclTable(m_configDb, CFG_ACL_TABLE_TABLE_NAME); TableConnector confDbAclRuleTable(m_configDb, CFG_ACL_RULE_TABLE_NAME); @@ -273,10 +274,10 @@ bool OrchDaemon::init() dtel_orch = new DTelOrch(m_configDb, dtel_tables, gPortsOrch); m_orchList.push_back(dtel_orch); } - gAclOrch = new AclOrch(acl_table_connectors, gSwitchOrch, gPortsOrch, mirror_orch, gNeighOrch, gRouteOrch, dtel_orch); + gAclOrch = new AclOrch(acl_table_connectors, gSwitchOrch, gPortsOrch, gMirrorOrch, gNeighOrch, gRouteOrch, dtel_orch); m_orchList.push_back(gFdbOrch); - m_orchList.push_back(mirror_orch); + m_orchList.push_back(gMirrorOrch); m_orchList.push_back(gAclOrch); m_orchList.push_back(chassis_frontend_orch); m_orchList.push_back(vrf_orch); @@ -548,18 +549,24 @@ bool OrchDaemon::warmRestoreAndSyncUp() for (auto it = 0; it < 3; it++) { - SWSS_LOG_DEBUG("The current iteration is %d", it); + SWSS_LOG_DEBUG("The current doTask iteration is %d", it); for (Orch *o : m_orchList) { + if (o == gMirrorOrch) { + SWSS_LOG_DEBUG("Skipping mirror processing until the end"); + continue; + } + o->doTask(); } } - for (Orch *o : m_orchList) - { - o->postBake(); - } + // MirrorOrch depends on everything else being settled before it can run, + // and mirror ACL rules depend on MirrorOrch, so run these two at the end + // after the rest of the data has been processed. + gMirrorOrch->doTask(); + gAclOrch->doTask(); /* * At this point, all the pre-existing data should have been processed properly, and diff --git a/tests/conftest.py b/tests/conftest.py index 3073b397d08..bc946052774 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1021,16 +1021,25 @@ def remove_neighbor(self, interface, ip): tbl._del(interface + ":" + ip) time.sleep(1) - # deps: mirror_port_erspan + # deps: mirror_port_erspan, warm_reboot def add_route(self, prefix, nexthop): self.runcmd("ip route add " + prefix + " via " + nexthop) time.sleep(1) - # deps: mirror_port_erspan + # deps: mirror_port_erspan, warm_reboot def change_route(self, prefix, nexthop): self.runcmd("ip route change " + prefix + " via " + nexthop) time.sleep(1) + # deps: warm_reboot + def change_route_ecmp(self, prefix, nexthops): + cmd = "" + for nexthop in nexthops: + cmd += " nexthop via " + nexthop + + self.runcmd("ip route change " + prefix + cmd) + time.sleep(1) + # deps: acl, mirror_port_erspan def remove_route(self, prefix): self.runcmd("ip route del " + prefix) diff --git a/tests/dvslib/dvs_acl.py b/tests/dvslib/dvs_acl.py index 914e9ed40a7..41c588e738a 100644 --- a/tests/dvslib/dvs_acl.py +++ b/tests/dvslib/dvs_acl.py @@ -331,6 +331,11 @@ def remove_acl_rule(self, table_name: str, rule_name: str) -> None: """ self.config_db.delete_entry("ACL_RULE", "{}|{}".format(table_name, rule_name)) + def verify_acl_rule_count(self, expected: int) -> None: + """Verify that there are N rules in the ASIC DB.""" + num_keys = len(self.asic_db.default_acl_entries) + self.asic_db.wait_for_n_keys("ASIC_STATE:SAI_OBJECT_TYPE_ACL_ENTRY", num_keys + expected) + def verify_no_acl_rules(self) -> None: """Verify that there are no ACL rules in the ASIC DB.""" num_keys = len(self.asic_db.default_acl_entries) diff --git a/tests/dvslib/dvs_mirror.py b/tests/dvslib/dvs_mirror.py index d5d58b623a6..70c42badb90 100644 --- a/tests/dvslib/dvs_mirror.py +++ b/tests/dvslib/dvs_mirror.py @@ -23,18 +23,21 @@ def create_span_session(self, name, dst_port, src_ports=None, direction="BOTH", self.config_db.create_entry("MIRROR_SESSION", name, mirror_entry) def create_erspan_session(self, name, src, dst, gre, dscp, ttl, queue, policer=None, src_ports=None, direction="BOTH"): - mirror_entry = {} - mirror_entry["src_ip"] = src - mirror_entry["dst_ip"] = dst - mirror_entry["gre_type"] = gre - mirror_entry["dscp"] = dscp - mirror_entry["ttl"] = ttl - mirror_entry["queue"] = queue + mirror_entry = { + "src_ip": src, + "dst_ip": dst, + "gre_type": gre, + "dscp": dscp, + "ttl": ttl, + "queue": queue, + "direction": direction + } + if policer: mirror_entry["policer"] = policer + if src_ports: mirror_entry["src_port"] = src_ports - mirror_entry["direction"] = direction self.config_db.create_entry("MIRROR_SESSION", name, mirror_entry) diff --git a/tests/dvslib/dvs_policer.py b/tests/dvslib/dvs_policer.py index ea4b729329c..153e16d4d2f 100644 --- a/tests/dvslib/dvs_policer.py +++ b/tests/dvslib/dvs_policer.py @@ -3,7 +3,7 @@ def __init__(self, adb, cdb): self.asic_db = adb self.config_db = cdb - def create_policer(self, name, type="packets", cir="600", cbs="600", mode="sr_tcm", red_action="drop" ): + def create_policer(self, name, type="packets", cir="600", cbs="600", mode="sr_tcm", red_action="drop"): policer_entry = {"meter_type": type, "mode": mode, "cir": cir, "cbs": cbs, "red_packet_action": red_action} self.config_db.create_entry("POLICER", name, policer_entry) diff --git a/tests/mock_tests/mock_orchagent_main.cpp b/tests/mock_tests/mock_orchagent_main.cpp index 86651a7c949..15004b756ef 100644 --- a/tests/mock_tests/mock_orchagent_main.cpp +++ b/tests/mock_tests/mock_orchagent_main.cpp @@ -22,7 +22,6 @@ bool gSaiRedisLogRotate = false; ofstream gRecordOfs; string gRecordFile; -MirrorOrch *gMirrorOrch; VRFOrch *gVrfOrch; void syncd_apply_view() {} diff --git a/tests/test_warm_reboot.py b/tests/test_warm_reboot.py index e22104adc33..9cce686c99b 100644 --- a/tests/test_warm_reboot.py +++ b/tests/test_warm_reboot.py @@ -1783,6 +1783,24 @@ def test_routing_WarmRestart(self, dvs, testlog): (addobjs, delobjs) = dvs.GetSubscribedAsicDbObjects(pubsubAsicDB) assert len(addobjs) == 0 and len(delobjs) == 0 + # + # Remove route entries so they don't interfere with later tests + # + dvs.runcmd("ip route del 192.168.1.100/32") + dvs.runcmd("ip route del 192.168.1.200/32") + dvs.runcmd("ip route del 192.168.1.230/32") + dvs.runcmd("ip route del 192.168.1.1/32") + dvs.runcmd("ip route del 192.168.1.2/32") + dvs.runcmd("ip route del 192.168.1.3/32") + dvs.runcmd("ip route del 192.168.100.0/24") + dvs.runcmd("ip -6 route del fc00:11:11::1/128") + dvs.runcmd("ip -6 route del fc00:12:12::1/128") + dvs.runcmd("ip -6 route del fc00:13:13::1/128") + dvs.runcmd("ip -6 route del fc00:1:1::1/128") + dvs.runcmd("ip -6 route del fc00:2:2::1/128") + dvs.runcmd("ip -6 route del fc00:3:3::1/128") + time.sleep(5) + intf_tbl._del("{}|111.0.0.1/24".format(intfs[0])) intf_tbl._del("{}|1110::1/64".format(intfs[0])) intf_tbl._del("{}|122.0.0.1/24".format(intfs[1])) @@ -2191,6 +2209,137 @@ def test_VrfMgrdWarmRestart(self, dvs, testlog): dvs.servers[1].runcmd("ifconfig eth0 0") time.sleep(2) + @pytest.fixture(scope="class") + def setup_erspan_neighbors(self, dvs): + dvs.setup_db() + + dvs.set_interface_status("Ethernet12", "up") + dvs.set_interface_status("Ethernet16", "up") + dvs.set_interface_status("Ethernet20", "up") + + dvs.add_ip_address("Ethernet12", "10.0.0.0/31") + dvs.add_ip_address("Ethernet16", "11.0.0.0/31") + dvs.add_ip_address("Ethernet20", "12.0.0.0/31") + + dvs.add_neighbor("Ethernet12", "10.0.0.1", "02:04:06:08:10:12") + dvs.add_neighbor("Ethernet16", "11.0.0.1", "03:04:06:08:10:12") + dvs.add_neighbor("Ethernet20", "12.0.0.1", "04:04:06:08:10:12") + + dvs.add_route("2.2.2.2", "10.0.0.1") + + yield + + dvs.remove_route("2.2.2.2") + + dvs.remove_neighbor("Ethernet12", "10.0.0.1") + dvs.remove_neighbor("Ethernet16", "11.0.0.1") + dvs.remove_neighbor("Ethernet20", "12.0.0.1") + + dvs.remove_ip_address("Ethernet12", "10.0.0.0/31") + dvs.remove_ip_address("Ethernet16", "11.0.0.0/31") + dvs.remove_ip_address("Ethernet20", "12.0.0.1/31") + + dvs.set_interface_status("Ethernet12", "down") + dvs.set_interface_status("Ethernet16", "down") + dvs.set_interface_status("Ethernet20", "down") + + @pytest.mark.usefixtures("dvs_mirror_manager", "setup_erspan_neighbors") + def test_MirrorSessionWarmReboot(self, dvs): + dvs.setup_db() + + # Setup the mirror session + self.dvs_mirror.create_erspan_session("test_session", "1.1.1.1", "2.2.2.2", "0x6558", "8", "100", "0") + + # Verify the monitor port + state_db = dvs.get_state_db() + state_db.wait_for_field_match("MIRROR_SESSION_TABLE", "test_session", {"monitor_port": "Ethernet12"}) + + # Setup ECMP routes to the session destination + dvs.change_route_ecmp("2.2.2.2", ["12.0.0.1", "11.0.0.1", "10.0.0.1"]) + + # Monitor port should not change b/c routes are ECMP + state_db.wait_for_field_match("MIRROR_SESSION_TABLE", "test_session", {"monitor_port": "Ethernet12"}) + + dvs.runcmd("config warm_restart enable swss") + dvs.stop_swss() + dvs.start_swss() + + dvs.check_swss_ready() + + # Monitor port should not change b/c destination is frozen + state_db.wait_for_field_match("MIRROR_SESSION_TABLE", "test_session", {"monitor_port": "Ethernet12"}) + + self.dvs_mirror.remove_mirror_session("test_session") + + # Revert the route back to the fixture-defined route + dvs.change_route("2.2.2.2", "10.0.0.1") + + # Reset for test cases after this one + dvs.stop_swss() + dvs.start_swss() + dvs.check_swss_ready() + + @pytest.mark.usefixtures("dvs_mirror_manager", "dvs_policer_manager", "setup_erspan_neighbors") + def test_EverflowWarmReboot(self, dvs, dvs_acl): + # Setup the policer + self.dvs_policer.create_policer("test_policer") + self.dvs_policer.verify_policer("test_policer") + + # Setup the mirror session + self.dvs_mirror.create_erspan_session("test_session", "1.1.1.1", "2.2.2.2", "0x6558", "8", "100", "0", policer="test_policer") + + state_db = dvs.get_state_db() + state_db.wait_for_field_match("MIRROR_SESSION_TABLE", "test_session", {"status": "active"}) + + # Create the mirror table + dvs_acl.create_acl_table("EVERFLOW_TEST", "MIRROR", ["Ethernet12"]) + + + # TODO: The standard methods for counting ACL tables and ACL rules break down after warm reboot b/c the OIDs + # of the default tables change. We also can't just re-initialize the default value b/c we've added another + # table and rule that aren't part of the base device config. We should come up with a way to handle warm reboot + # changes more gracefully to make it easier for future tests. + asic_db = dvs.get_asic_db() + asic_db.wait_for_n_keys("ASIC_STATE:SAI_OBJECT_TYPE_ACL_TABLE", 1 + len(asic_db.default_acl_tables)) + + # Create a mirror rule + dvs_acl.create_mirror_acl_rule("EVERFLOW_TEST", "TEST_RULE", {"SRC_IP": "20.0.0.2"}, "test_session") + asic_db.wait_for_n_keys("ASIC_STATE:SAI_OBJECT_TYPE_ACL_ENTRY", 1 + len(asic_db.default_acl_entries)) + + # Execute the warm reboot + dvs.runcmd("config warm_restart enable swss") + dvs.stop_swss() + dvs.start_swss() + + # Make sure the system is stable + dvs.check_swss_ready() + + # Verify that the ASIC DB is intact + self.dvs_policer.verify_policer("test_policer") + state_db.wait_for_field_match("MIRROR_SESSION_TABLE", "test_session", {"status": "active"}) + + asic_db = dvs.get_asic_db() + asic_db.wait_for_n_keys("ASIC_STATE:SAI_OBJECT_TYPE_ACL_TABLE", 1 + len(asic_db.default_acl_tables)) + asic_db.wait_for_n_keys("ASIC_STATE:SAI_OBJECT_TYPE_ACL_ENTRY", 1 + len(asic_db.default_acl_entries)) + + # Clean up + dvs_acl.remove_acl_rule("EVERFLOW_TEST", "TEST_RULE") + asic_db.wait_for_n_keys("ASIC_STATE:SAI_OBJECT_TYPE_ACL_ENTRY", len(asic_db.default_acl_entries)) + + dvs_acl.remove_acl_table("EVERFLOW_TEST") + asic_db.wait_for_n_keys("ASIC_STATE:SAI_OBJECT_TYPE_ACL_TABLE", len(asic_db.default_acl_tables)) + + self.dvs_mirror.remove_mirror_session("test_session") + self.dvs_mirror.verify_no_mirror() + + self.dvs_policer.remove_policer("test_policer") + self.dvs_policer.verify_no_policer() + + # Reset for test cases after this one + dvs.stop_swss() + dvs.start_swss() + dvs.check_swss_ready() + # Add Dummy always-pass test at end as workaroud # for issue when Flaky fail on final test it invokes module tear-down before retrying