From bc4062b1215b92df56c7405a9693b4514dbc8627 Mon Sep 17 00:00:00 2001 From: Lawrence Lee Date: Tue, 2 May 2023 13:56:08 -0700 Subject: [PATCH] [mux]: Implement rollback for failed mux switchovers (#2714) - Make all SAI API operations needed for switchover idempotent - Implement rollback when a switchover fails Signed-off-by: Lawrence Lee --- .gitignore | 4 + orchagent/aclorch.cpp | 11 + orchagent/muxorch.cpp | 118 ++++-- orchagent/muxorch.h | 4 +- orchagent/neighorch.cpp | 35 +- tests/mock_tests/Makefile.am | 1 + tests/mock_tests/mock_orchagent_main.h | 3 + tests/mock_tests/mock_sai_api.h | 149 ++++++++ tests/mock_tests/mux_rollback_ut.cpp | 489 +++++++++++++++++++++++++ tests/mock_tests/ut_saihelper.cpp | 1 + 10 files changed, 764 insertions(+), 51 deletions(-) create mode 100644 tests/mock_tests/mock_sai_api.h create mode 100644 tests/mock_tests/mux_rollback_ut.cpp diff --git a/.gitignore b/.gitignore index 04c3b514c769..a0c8c5ac8214 100644 --- a/.gitignore +++ b/.gitignore @@ -76,6 +76,8 @@ teamsyncd/teamsyncd tests/tests tests/mock_tests/tests_response_publisher tests/mock_tests/tests_fpmsyncd +tests/mock_tests/tests_intfmgrd +tests/mock_tests/tests_portsyncd # Test Files # @@ -87,5 +89,7 @@ tests/mock_tests/tests.trs tests/test-suite.log tests/tests.log tests/tests.trs +tests/mock_tests/**/*log +tests/mock_tests/**/*trs orchagent/p4orch/tests/**/*gcda orchagent/p4orch/tests/**/*gcno diff --git a/orchagent/aclorch.cpp b/orchagent/aclorch.cpp index b6eb06c058bf..de64757fe8ac 100644 --- a/orchagent/aclorch.cpp +++ b/orchagent/aclorch.cpp @@ -1140,6 +1140,11 @@ bool AclRule::createRule() status = sai_acl_api->create_acl_entry(&m_ruleOid, gSwitchId, (uint32_t)rule_attrs.size(), rule_attrs.data()); if (status != SAI_STATUS_SUCCESS) { + if (status == SAI_STATUS_ITEM_ALREADY_EXISTS) + { + SWSS_LOG_NOTICE("ACL rule %s already exists", m_id.c_str()); + return true; + } SWSS_LOG_ERROR("Failed to create ACL rule %s, rv:%d", m_id.c_str(), status); AclRange::remove(range_objects, range_object_list.count); @@ -1201,6 +1206,12 @@ bool AclRule::removeRule() auto status = sai_acl_api->remove_acl_entry(m_ruleOid); if (status != SAI_STATUS_SUCCESS) { + if (status == SAI_STATUS_ITEM_NOT_FOUND) + { + SWSS_LOG_NOTICE("ACL rule already deleted"); + m_ruleOid = SAI_NULL_OBJECT_ID; + return true; + } SWSS_LOG_ERROR("Failed to delete ACL rule, status %s", sai_serialize_status(status).c_str()); return false; } diff --git a/orchagent/muxorch.cpp b/orchagent/muxorch.cpp index 1386196b2b1b..fc39e8aa2876 100644 --- a/orchagent/muxorch.cpp +++ b/orchagent/muxorch.cpp @@ -116,6 +116,10 @@ static sai_status_t create_route(IpPrefix &pfx, sai_object_id_t nh) sai_status_t status = sai_route_api->create_route_entry(&route_entry, (uint32_t)attrs.size(), attrs.data()); if (status != SAI_STATUS_SUCCESS) { + if (status == SAI_STATUS_ITEM_ALREADY_EXISTS) { + SWSS_LOG_NOTICE("Tunnel route to %s already exists", pfx.to_string().c_str()); + return SAI_STATUS_SUCCESS; + } SWSS_LOG_ERROR("Failed to create tunnel route %s,nh %" PRIx64 " rv:%d", pfx.getIp().to_string().c_str(), nh, status); return status; @@ -145,6 +149,10 @@ static sai_status_t remove_route(IpPrefix &pfx) sai_status_t status = sai_route_api->remove_route_entry(&route_entry); if (status != SAI_STATUS_SUCCESS) { + if (status == SAI_STATUS_ITEM_NOT_FOUND) { + SWSS_LOG_NOTICE("Tunnel route to %s already removed", pfx.to_string().c_str()); + return SAI_STATUS_SUCCESS; + } SWSS_LOG_ERROR("Failed to remove tunnel route %s, rv:%d", pfx.getIp().to_string().c_str(), status); return status; @@ -497,7 +505,7 @@ void MuxCable::setState(string new_state) mux_cb_orch_->updateMuxMetricState(mux_name_, new_state, true); - MuxState state = state_; + prev_state_ = state_; state_ = ns; st_chg_in_progress_ = true; @@ -505,7 +513,7 @@ void MuxCable::setState(string new_state) if (!(this->*(state_machine_handlers_[it->second]))()) { //Reset back to original state - state_ = state; + state_ = prev_state_; st_chg_in_progress_ = false; st_chg_failed_ = true; throw std::runtime_error("Failed to handle state transition"); @@ -521,6 +529,51 @@ void MuxCable::setState(string new_state) return; } +void MuxCable::rollbackStateChange() +{ + if (prev_state_ == MuxState::MUX_STATE_FAILED || prev_state_ == MuxState::MUX_STATE_PENDING) + { + SWSS_LOG_ERROR("[%s] Rollback to %s not supported", mux_name_.c_str(), + muxStateValToString.at(prev_state_).c_str()); + return; + } + SWSS_LOG_WARN("[%s] Rolling back state change to %s", mux_name_.c_str(), + muxStateValToString.at(prev_state_).c_str()); + mux_cb_orch_->updateMuxMetricState(mux_name_, muxStateValToString.at(prev_state_), true); + st_chg_in_progress_ = true; + state_ = prev_state_; + bool success = false; + switch (prev_state_) + { + case MuxState::MUX_STATE_ACTIVE: + success = stateActive(); + break; + case MuxState::MUX_STATE_INIT: + case MuxState::MUX_STATE_STANDBY: + success = stateStandby(); + break; + case MuxState::MUX_STATE_FAILED: + case MuxState::MUX_STATE_PENDING: + // Check at the start of the function means we will never reach here + SWSS_LOG_ERROR("[%s] Rollback to %s not supported", mux_name_.c_str(), + muxStateValToString.at(prev_state_).c_str()); + return; + } + st_chg_in_progress_ = false; + if (success) + { + st_chg_failed_ = false; + } + else + { + st_chg_failed_ = true; + SWSS_LOG_ERROR("[%s] Rollback to %s failed", + mux_name_.c_str(), muxStateValToString.at(prev_state_).c_str()); + } + mux_cb_orch_->updateMuxMetricState(mux_name_, muxStateValToString.at(state_), false); + mux_cb_orch_->updateMuxState(mux_name_, muxStateValToString.at(state_)); +} + string MuxCable::getState() { SWSS_LOG_INFO("Get state request for %s, state %s", @@ -838,8 +891,6 @@ void MuxNbrHandler::updateTunnelRoute(NextHopKey nh, bool add) } } -std::map MuxAclHandler::acl_table_; - MuxAclHandler::MuxAclHandler(sai_object_id_t port, string alias) { SWSS_LOG_ENTER(); @@ -857,32 +908,21 @@ MuxAclHandler::MuxAclHandler(sai_object_id_t port, string alias) port_ = port; alias_ = alias; - auto found = acl_table_.find(table_name); - if (found == acl_table_.end()) - { - SWSS_LOG_NOTICE("First time create for port %" PRIx64 "", port); + // Always try to create the table first. If it already exists, function will return early. + createMuxAclTable(port, table_name); - // First time handling of Mux Table, create ACL table, and bind - createMuxAclTable(port, table_name); + SWSS_LOG_NOTICE("Binding port %" PRIx64 "", port); + + AclRule* rule = gAclOrch->getAclRule(table_name, rule_name); + if (rule == nullptr) + { shared_ptr newRule = make_shared(gAclOrch, rule_name, table_name, false /*no counters*/); createMuxAclRule(newRule, table_name); } else { - SWSS_LOG_NOTICE("Binding port %" PRIx64 "", port); - - AclRule* rule = gAclOrch->getAclRule(table_name, rule_name); - if (rule == nullptr) - { - shared_ptr newRule = - make_shared(gAclOrch, rule_name, table_name, false /*no counters*/); - createMuxAclRule(newRule, table_name); - } - else - { - gAclOrch->updateAclRule(table_name, rule_name, MATCH_IN_PORTS, &port, RULE_OPER_ADD); - } + gAclOrch->updateAclRule(table_name, rule_name, MATCH_IN_PORTS, &port, RULE_OPER_ADD); } } @@ -915,23 +955,16 @@ void MuxAclHandler::createMuxAclTable(sai_object_id_t port, string strTable) { SWSS_LOG_ENTER(); - auto inserted = acl_table_.emplace(piecewise_construct, - std::forward_as_tuple(strTable), - std::forward_as_tuple(gAclOrch, strTable)); - - assert(inserted.second); - - AclTable& acl_table = inserted.first->second; - sai_object_id_t table_oid = gAclOrch->getTableById(strTable); if (table_oid != SAI_NULL_OBJECT_ID) { // DROP ACL table is already created - SWSS_LOG_NOTICE("ACL table %s exists, reuse the same", strTable.c_str()); - acl_table = *(gAclOrch->getTableByOid(table_oid)); + SWSS_LOG_INFO("ACL table %s exists, reuse the same", strTable.c_str()); return; } + SWSS_LOG_NOTICE("First time create for port %" PRIx64 "", port); + AclTable acl_table(gAclOrch, strTable); auto dropType = gAclOrch->getAclTableType(TABLE_TYPE_DROP); assert(dropType); acl_table.validateAddType(*dropType); @@ -1776,10 +1809,25 @@ bool MuxCableOrch::addOperation(const Request& request) { mux_obj->setState(state); } - catch(const std::runtime_error& error) + catch(const std::runtime_error& e) { SWSS_LOG_ERROR("Mux Error setting state %s for port %s. Error: %s", - state.c_str(), port_name.c_str(), error.what()); + state.c_str(), port_name.c_str(), e.what()); + mux_obj->rollbackStateChange(); + return true; + } + catch (const std::logic_error& e) + { + SWSS_LOG_ERROR("Logic error while setting state %s for port %s. Error: %s", + state.c_str(), port_name.c_str(), e.what()); + mux_obj->rollbackStateChange(); + return true; + } + catch (const std::exception& e) + { + SWSS_LOG_ERROR("Exception caught while setting state %s for port %s. Error: %s", + state.c_str(), port_name.c_str(), e.what()); + mux_obj->rollbackStateChange(); return true; } diff --git a/orchagent/muxorch.h b/orchagent/muxorch.h index e0e1022eccbe..ce6a4d9b3f6e 100644 --- a/orchagent/muxorch.h +++ b/orchagent/muxorch.h @@ -52,8 +52,6 @@ class MuxAclHandler void createMuxAclRule(shared_ptr rule, string strTable); void bindAllPorts(AclTable &acl_table); - // class shared dict: ACL table name -> ACL table - static std::map acl_table_; sai_object_id_t port_ = SAI_NULL_OBJECT_ID; bool is_ingress_acl_ = true; string alias_; @@ -99,6 +97,7 @@ class MuxCable using state_machine_handlers = map; void setState(string state); + void rollbackStateChange(); string getState(); bool isStateChangeInProgress() { return st_chg_in_progress_; } bool isStateChangeFailed() { return st_chg_failed_; } @@ -123,6 +122,7 @@ class MuxCable MuxCableType cable_type_; MuxState state_ = MuxState::MUX_STATE_INIT; + MuxState prev_state_; bool st_chg_in_progress_ = false; bool st_chg_failed_ = false; diff --git a/orchagent/neighorch.cpp b/orchagent/neighorch.cpp index 8e4c668a57e5..9ba8b9b28f3a 100644 --- a/orchagent/neighorch.cpp +++ b/orchagent/neighorch.cpp @@ -255,6 +255,12 @@ bool NeighOrch::addNextHop(const NextHopKey &nh) sai_status_t status = sai_next_hop_api->create_next_hop(&next_hop_id, gSwitchId, (uint32_t)next_hop_attrs.size(), next_hop_attrs.data()); if (status != SAI_STATUS_SUCCESS) { + if (status == SAI_STATUS_ITEM_ALREADY_EXISTS) + { + SWSS_LOG_NOTICE("Next hop %s on %s already exists", + nexthop.ip_address.to_string().c_str(), nexthop.alias.c_str()); + return true; + } SWSS_LOG_ERROR("Failed to create next hop %s on %s, rv:%d", nexthop.ip_address.to_string().c_str(), nexthop.alias.c_str(), status); task_process_status handle_status = handleSaiCreateStatus(SAI_API_NEXT_HOP, status); @@ -1014,7 +1020,7 @@ bool NeighOrch::removeNeighbor(const NeighborEntry &neighborEntry, bool disable) /* When next hop is not found, we continue to remove neighbor entry. */ if (status == SAI_STATUS_ITEM_NOT_FOUND) { - SWSS_LOG_ERROR("Failed to locate next hop %s on %s, rv:%d", + SWSS_LOG_NOTICE("Next hop %s on %s doesn't exist, rv:%d", ip_address.to_string().c_str(), alias.c_str(), status); } else @@ -1049,9 +1055,8 @@ bool NeighOrch::removeNeighbor(const NeighborEntry &neighborEntry, bool disable) { if (status == SAI_STATUS_ITEM_NOT_FOUND) { - SWSS_LOG_ERROR("Failed to locate neighbor %s on %s, rv:%d", + SWSS_LOG_NOTICE("Neighbor %s on %s already removed, rv:%d", m_syncdNeighbors[neighborEntry].mac.to_string().c_str(), alias.c_str(), status); - return true; } else { @@ -1064,22 +1069,24 @@ bool NeighOrch::removeNeighbor(const NeighborEntry &neighborEntry, bool disable) } } } - - if (neighbor_entry.ip_address.addr_family == SAI_IP_ADDR_FAMILY_IPV4) - { - gCrmOrch->decCrmResUsedCounter(CrmResourceType::CRM_IPV4_NEIGHBOR); - } else { - gCrmOrch->decCrmResUsedCounter(CrmResourceType::CRM_IPV6_NEIGHBOR); - } + if (neighbor_entry.ip_address.addr_family == SAI_IP_ADDR_FAMILY_IPV4) + { + gCrmOrch->decCrmResUsedCounter(CrmResourceType::CRM_IPV4_NEIGHBOR); + } + else + { + gCrmOrch->decCrmResUsedCounter(CrmResourceType::CRM_IPV6_NEIGHBOR); + } - removeNextHop(ip_address, alias); - m_intfsOrch->decreaseRouterIntfsRefCount(alias); + removeNextHop(ip_address, alias); + m_intfsOrch->decreaseRouterIntfsRefCount(alias); + SWSS_LOG_NOTICE("Removed neighbor %s on %s", + m_syncdNeighbors[neighborEntry].mac.to_string().c_str(), alias.c_str()); + } } - SWSS_LOG_NOTICE("Removed neighbor %s on %s", - m_syncdNeighbors[neighborEntry].mac.to_string().c_str(), alias.c_str()); /* Do not delete entry from cache if its disable request */ if (disable) diff --git a/tests/mock_tests/Makefile.am b/tests/mock_tests/Makefile.am index ab0c1431d40b..3741e7da27cf 100644 --- a/tests/mock_tests/Makefile.am +++ b/tests/mock_tests/Makefile.am @@ -50,6 +50,7 @@ tests_SOURCES = aclorch_ut.cpp \ flowcounterrouteorch_ut.cpp \ orchdaemon_ut.cpp \ intfsorch_ut.cpp \ + mux_rollback_ut.cpp \ warmrestartassist_ut.cpp \ test_failure_handling.cpp \ $(top_srcdir)/lib/gearboxutils.cpp \ diff --git a/tests/mock_tests/mock_orchagent_main.h b/tests/mock_tests/mock_orchagent_main.h index 0d4865ac3dfd..bf8456829d45 100644 --- a/tests/mock_tests/mock_orchagent_main.h +++ b/tests/mock_tests/mock_orchagent_main.h @@ -56,6 +56,8 @@ extern VRFOrch *gVrfOrch; extern NhgOrch *gNhgOrch; extern Srv6Orch *gSrv6Orch; extern BfdOrch *gBfdOrch; +extern AclOrch *gAclOrch; +extern PolicerOrch *gPolicerOrch; extern Directory gDirectory; extern sai_acl_api_t *sai_acl_api; @@ -70,6 +72,7 @@ extern sai_route_api_t *sai_route_api; extern sai_neighbor_api_t *sai_neighbor_api; extern sai_tunnel_api_t *sai_tunnel_api; extern sai_next_hop_api_t *sai_next_hop_api; +extern sai_next_hop_group_api_t *sai_next_hop_group_api; extern sai_hostif_api_t *sai_hostif_api; extern sai_policer_api_t *sai_policer_api; extern sai_buffer_api_t *sai_buffer_api; diff --git a/tests/mock_tests/mock_sai_api.h b/tests/mock_tests/mock_sai_api.h new file mode 100644 index 000000000000..63d8921bf1a2 --- /dev/null +++ b/tests/mock_tests/mock_sai_api.h @@ -0,0 +1,149 @@ +#include "mock_orchagent_main.h" +#include + +using ::testing::Return; +using ::testing::NiceMock; + +std::set apply_mock_fns; +std::set remove_mock_fns; + +#define CREATE_PARAMS(sai_object_type) _In_ const sai_##sai_object_type##_entry_t *sai_object_type##_entry, _In_ uint32_t attr_count, _In_ const sai_attribute_t *attr_list +#define REMOVE_PARAMS(sai_object_type) _In_ const sai_##sai_object_type##_entry_t *sai_object_type##_entry +#define CREATE_ARGS(sai_object_type) sai_object_type##_entry, attr_count, attr_list +#define REMOVE_ARGS(sai_object_type) sai_object_type##_entry +#define GENERIC_CREATE_PARAMS(sai_object_type) _Out_ sai_object_id_t *sai_object_type##_id, _In_ sai_object_id_t switch_id, _In_ uint32_t attr_count, _In_ const sai_attribute_t *attr_list +#define GENERIC_REMOVE_PARAMS(sai_object_type) _In_ sai_object_id_t sai_object_type##_id +#define GENERIC_CREATE_ARGS(sai_object_type) sai_object_type##_id, switch_id, attr_count, attr_list +#define GENERIC_REMOVE_ARGS(sai_object_type) sai_object_type##_id + +/* +The macro DEFINE_SAI_API_MOCK will perform the steps to mock the SAI API for the sai_object_type it is called on: +1. Create a pointer to store the original API +2. Create a new SAI_API where we can safely mock without affecting the original API +3. Define a class with mocked methods to create and remove the object type (to be used with gMock) +4. Create a pointer of the above class +5. Define two wrapper functions to create and remove the object type that has the same signature as the original SAI API function +6. Define a method to apply the mock +7. Define a method to remove the mock +*/ +#define DEFINE_SAI_API_MOCK(sai_object_type) \ + sai_##sai_object_type##_api_t *old_sai_##sai_object_type##_api; \ + sai_##sai_object_type##_api_t ut_sai_##sai_object_type##_api; \ + class mock_sai_##sai_object_type##_api_t \ + { \ + public: \ + mock_sai_##sai_object_type##_api_t() \ + { \ + ON_CALL(*this, create_##sai_object_type##_entry) \ + .WillByDefault( \ + [this](CREATE_PARAMS(sai_object_type)) { \ + return old_sai_##sai_object_type##_api->create_##sai_object_type##_entry(CREATE_ARGS(sai_object_type)); \ + }); \ + ON_CALL(*this, remove_##sai_object_type##_entry) \ + .WillByDefault( \ + [this](REMOVE_PARAMS(sai_object_type)) { \ + return old_sai_##sai_object_type##_api->remove_##sai_object_type##_entry(REMOVE_ARGS(sai_object_type)); \ + }); \ + } \ + MOCK_METHOD3(create_##sai_object_type##_entry, sai_status_t(CREATE_PARAMS(sai_object_type))); \ + MOCK_METHOD1(remove_##sai_object_type##_entry, sai_status_t(REMOVE_PARAMS(sai_object_type))); \ + }; \ + mock_sai_##sai_object_type##_api_t *mock_sai_##sai_object_type##_api; \ + sai_status_t mock_create_##sai_object_type##_entry(CREATE_PARAMS(sai_object_type)) \ + { \ + return mock_sai_##sai_object_type##_api->create_##sai_object_type##_entry(CREATE_ARGS(sai_object_type)); \ + } \ + sai_status_t mock_remove_##sai_object_type##_entry(REMOVE_PARAMS(sai_object_type)) \ + { \ + return mock_sai_##sai_object_type##_api->remove_##sai_object_type##_entry(REMOVE_ARGS(sai_object_type)); \ + } \ + void apply_sai_##sai_object_type##_api_mock() \ + { \ + mock_sai_##sai_object_type##_api = new NiceMock(); \ + \ + old_sai_##sai_object_type##_api = sai_##sai_object_type##_api; \ + ut_sai_##sai_object_type##_api = *sai_##sai_object_type##_api; \ + sai_##sai_object_type##_api = &ut_sai_##sai_object_type##_api; \ + \ + sai_##sai_object_type##_api->create_##sai_object_type##_entry = mock_create_##sai_object_type##_entry; \ + sai_##sai_object_type##_api->remove_##sai_object_type##_entry = mock_remove_##sai_object_type##_entry; \ + } \ + void remove_sai_##sai_object_type##_api_mock() \ + { \ + sai_##sai_object_type##_api = old_sai_##sai_object_type##_api; \ + delete mock_sai_##sai_object_type##_api; \ + } + +#define DEFINE_SAI_GENERIC_API_MOCK(sai_api_name, sai_object_type) \ + sai_##sai_api_name##_api_t *old_sai_##sai_api_name##_api; \ + sai_##sai_api_name##_api_t ut_sai_##sai_api_name##_api; \ + class mock_sai_##sai_api_name##_api_t \ + { \ + public: \ + mock_sai_##sai_api_name##_api_t() \ + { \ + ON_CALL(*this, create_##sai_object_type) \ + .WillByDefault( \ + [this](GENERIC_CREATE_PARAMS(sai_object_type)) { \ + return old_sai_##sai_api_name##_api->create_##sai_object_type(GENERIC_CREATE_ARGS(sai_object_type)); \ + }); \ + ON_CALL(*this, remove_##sai_object_type) \ + .WillByDefault( \ + [this](GENERIC_REMOVE_PARAMS(sai_object_type)) { \ + return old_sai_##sai_api_name##_api->remove_##sai_object_type(GENERIC_REMOVE_ARGS(sai_object_type)); \ + }); \ + } \ + MOCK_METHOD4(create_##sai_object_type, sai_status_t(GENERIC_CREATE_PARAMS(sai_object_type))); \ + MOCK_METHOD1(remove_##sai_object_type, sai_status_t(GENERIC_REMOVE_PARAMS(sai_object_type))); \ + }; \ + mock_sai_##sai_api_name##_api_t *mock_sai_##sai_api_name##_api; \ + sai_status_t mock_create_##sai_object_type(GENERIC_CREATE_PARAMS(sai_object_type)) \ + { \ + return mock_sai_##sai_api_name##_api->create_##sai_object_type(GENERIC_CREATE_ARGS(sai_object_type)); \ + } \ + sai_status_t mock_remove_##sai_object_type(GENERIC_REMOVE_PARAMS(sai_object_type)) \ + { \ + return mock_sai_##sai_api_name##_api->remove_##sai_object_type(GENERIC_REMOVE_ARGS(sai_object_type)); \ + } \ + void apply_sai_##sai_api_name##_api_mock() \ + { \ + mock_sai_##sai_api_name##_api = new NiceMock(); \ + \ + old_sai_##sai_api_name##_api = sai_##sai_api_name##_api; \ + ut_sai_##sai_api_name##_api = *sai_##sai_api_name##_api; \ + sai_##sai_api_name##_api = &ut_sai_##sai_api_name##_api; \ + \ + sai_##sai_api_name##_api->create_##sai_object_type = mock_create_##sai_object_type; \ + sai_##sai_api_name##_api->remove_##sai_object_type = mock_remove_##sai_object_type; \ + } \ + void remove_sai_##sai_api_name##_api_mock() \ + { \ + sai_##sai_api_name##_api = old_sai_##sai_api_name##_api; \ + delete mock_sai_##sai_api_name##_api; \ + } + +// Stores pointers to mock apply/remove functions to avoid needing to manually call each function +#define INIT_SAI_API_MOCK(sai_object_type) \ + apply_mock_fns.insert(&apply_sai_##sai_object_type##_api_mock); \ + remove_mock_fns.insert(&remove_sai_##sai_object_type##_api_mock); + +void MockSaiApis() +{ + if (apply_mock_fns.empty()) + { + EXPECT_TRUE(false) << "No mock application functions found. Did you call DEFINE_SAI_API_MOCK and INIT_SAI_API_MOCK for the necessary SAI object type?"; + } + + for (auto apply_fn : apply_mock_fns) + { + (*apply_fn)(); + } +} + +void RestoreSaiApis() +{ + for (auto remove_fn : remove_mock_fns) + { + (*remove_fn)(); + } +} diff --git a/tests/mock_tests/mux_rollback_ut.cpp b/tests/mock_tests/mux_rollback_ut.cpp new file mode 100644 index 000000000000..2653dcd6b9fb --- /dev/null +++ b/tests/mock_tests/mux_rollback_ut.cpp @@ -0,0 +1,489 @@ +#define private public +#include "directory.h" +#undef private +#define protected public +#include "orch.h" +#undef protected +#include "ut_helper.h" +#include "mock_orchagent_main.h" +#include "mock_sai_api.h" +#include "gtest/gtest.h" +#include + +DEFINE_SAI_API_MOCK(neighbor); +DEFINE_SAI_API_MOCK(route); +DEFINE_SAI_GENERIC_API_MOCK(acl, acl_entry); +DEFINE_SAI_GENERIC_API_MOCK(next_hop, next_hop); + +namespace mux_rollback_test +{ + using namespace std; + using ::testing::Return; + using ::testing::Throw; + + static const string PEER_SWITCH_HOSTNAME = "peer_hostname"; + static const string PEER_IPV4_ADDRESS = "1.1.1.1"; + static const string TEST_INTERFACE = "Ethernet4"; + static const string ACTIVE = "active"; + static const string STANDBY = "standby"; + static const string STATE = "state"; + static const string VLAN_NAME = "Vlan1000"; + static const string SERVER_IP = "192.168.0.2"; + + class MuxRollbackTest : public ::testing::Test + { + protected: + std::vector ut_orch_list; + shared_ptr m_app_db; + shared_ptr m_config_db; + shared_ptr m_state_db; + shared_ptr m_chassis_app_db; + MuxOrch *m_MuxOrch; + MuxCableOrch *m_MuxCableOrch; + MuxCable *m_MuxCable; + TunnelDecapOrch *m_TunnelDecapOrch; + MuxStateOrch *m_MuxStateOrch; + FlexCounterOrch *m_FlexCounterOrch; + mock_sai_neighbor_api_t mock_sai_neighbor_api_; + + void SetMuxStateFromAppDb(std::string state) + { + Table mux_cable_table = Table(m_app_db.get(), APP_MUX_CABLE_TABLE_NAME); + mux_cable_table.set(TEST_INTERFACE, { { STATE, state } }); + m_MuxCableOrch->addExistingData(&mux_cable_table); + static_cast(m_MuxCableOrch)->doTask(); + } + + void SetAndAssertMuxState(std::string state) + { + m_MuxCable->setState(state); + EXPECT_EQ(state, m_MuxCable->getState()); + } + + void ApplyDualTorConfigs() + { + Table peer_switch_table = Table(m_config_db.get(), CFG_PEER_SWITCH_TABLE_NAME); + Table tunnel_table = Table(m_app_db.get(), APP_TUNNEL_DECAP_TABLE_NAME); + Table mux_cable_table = Table(m_config_db.get(), CFG_MUX_CABLE_TABLE_NAME); + Table port_table = Table(m_app_db.get(), APP_PORT_TABLE_NAME); + Table vlan_table = Table(m_app_db.get(), APP_VLAN_TABLE_NAME); + Table vlan_member_table = Table(m_app_db.get(), APP_VLAN_MEMBER_TABLE_NAME); + Table neigh_table = Table(m_app_db.get(), APP_NEIGH_TABLE_NAME); + Table intf_table = Table(m_app_db.get(), APP_INTF_TABLE_NAME); + + auto ports = ut_helper::getInitialSaiPorts(); + port_table.set(TEST_INTERFACE, ports[TEST_INTERFACE]); + port_table.set("PortConfigDone", { { "count", to_string(1) } }); + port_table.set("PortInitDone", { {} }); + + neigh_table.set( + VLAN_NAME + neigh_table.getTableNameSeparator() + SERVER_IP, { { "neigh", "62:f9:65:10:2f:04" }, + { "family", "IPv4" } }); + + vlan_table.set(VLAN_NAME, { { "admin_status", "up" }, + { "mtu", "9100" }, + { "mac", "00:aa:bb:cc:dd:ee" } }); + vlan_member_table.set( + VLAN_NAME + vlan_member_table.getTableNameSeparator() + TEST_INTERFACE, + { { "tagging_mode", "untagged" } }); + + intf_table.set(VLAN_NAME, { { "grat_arp", "enabled" }, + { "proxy_arp", "enabled" }, + { "mac_addr", "00:00:00:00:00:00" } }); + intf_table.set( + VLAN_NAME + neigh_table.getTableNameSeparator() + "192.168.0.1/21", { + { "scope", "global" }, + { "family", "IPv4" }, + }); + + tunnel_table.set(MUX_TUNNEL, { { "dscp_mode", "uniform" }, + { "dst_ip", "2.2.2.2" }, + { "ecn_mode", "copy_from_outer" }, + { "encap_ecn_mode", "standard" }, + { "ttl_mode", "pipe" }, + { "tunnel_type", "IPINIP" } }); + + peer_switch_table.set(PEER_SWITCH_HOSTNAME, { { "address_ipv4", PEER_IPV4_ADDRESS } }); + + mux_cable_table.set(TEST_INTERFACE, { { "server_ipv4", SERVER_IP + "/32" }, + { "server_ipv6", "a::a/128" }, + { "state", "auto" } }); + + gPortsOrch->addExistingData(&port_table); + gPortsOrch->addExistingData(&vlan_table); + gPortsOrch->addExistingData(&vlan_member_table); + static_cast(gPortsOrch)->doTask(); + + gIntfsOrch->addExistingData(&intf_table); + static_cast(gIntfsOrch)->doTask(); + + m_TunnelDecapOrch->addExistingData(&tunnel_table); + static_cast(m_TunnelDecapOrch)->doTask(); + + m_MuxOrch->addExistingData(&peer_switch_table); + static_cast(m_MuxOrch)->doTask(); + + m_MuxOrch->addExistingData(&mux_cable_table); + static_cast(m_MuxOrch)->doTask(); + + gNeighOrch->addExistingData(&neigh_table); + static_cast(gNeighOrch)->doTask(); + + m_MuxCable = m_MuxOrch->getMuxCable(TEST_INTERFACE); + + // We always expect the mux to be initialized to standby + EXPECT_EQ(STANDBY, m_MuxCable->getState()); + } + + void PrepareSai() + { + sai_attribute_t attr; + + attr.id = SAI_SWITCH_ATTR_INIT_SWITCH; + attr.value.booldata = true; + + sai_status_t status = sai_switch_api->create_switch(&gSwitchId, 1, &attr); + ASSERT_EQ(status, SAI_STATUS_SUCCESS); + + // Get switch source MAC address + attr.id = SAI_SWITCH_ATTR_SRC_MAC_ADDRESS; + status = sai_switch_api->get_switch_attribute(gSwitchId, 1, &attr); + + ASSERT_EQ(status, SAI_STATUS_SUCCESS); + + gMacAddress = attr.value.mac; + + attr.id = SAI_SWITCH_ATTR_DEFAULT_VIRTUAL_ROUTER_ID; + status = sai_switch_api->get_switch_attribute(gSwitchId, 1, &attr); + + ASSERT_EQ(status, SAI_STATUS_SUCCESS); + + gVirtualRouterId = attr.value.oid; + + /* Create a loopback underlay router interface */ + vector underlay_intf_attrs; + + sai_attribute_t underlay_intf_attr; + underlay_intf_attr.id = SAI_ROUTER_INTERFACE_ATTR_VIRTUAL_ROUTER_ID; + underlay_intf_attr.value.oid = gVirtualRouterId; + underlay_intf_attrs.push_back(underlay_intf_attr); + + underlay_intf_attr.id = SAI_ROUTER_INTERFACE_ATTR_TYPE; + underlay_intf_attr.value.s32 = SAI_ROUTER_INTERFACE_TYPE_LOOPBACK; + underlay_intf_attrs.push_back(underlay_intf_attr); + + underlay_intf_attr.id = SAI_ROUTER_INTERFACE_ATTR_MTU; + underlay_intf_attr.value.u32 = 9100; + underlay_intf_attrs.push_back(underlay_intf_attr); + + status = sai_router_intfs_api->create_router_interface(&gUnderlayIfId, gSwitchId, (uint32_t)underlay_intf_attrs.size(), underlay_intf_attrs.data()); + ASSERT_EQ(status, SAI_STATUS_SUCCESS); + } + + void SetUp() override + { + map profile = { + { "SAI_VS_SWITCH_TYPE", "SAI_VS_SWITCH_TYPE_BCM56850" }, + { "KV_DEVICE_MAC_ADDRESS", "20:03:04:05:06:00" } + }; + + ut_helper::initSaiApi(profile); + m_app_db = make_shared("APPL_DB", 0); + m_config_db = make_shared("CONFIG_DB", 0); + m_state_db = make_shared("STATE_DB", 0); + m_chassis_app_db = make_shared("CHASSIS_APP_DB", 0); + + PrepareSai(); + + const int portsorch_base_pri = 40; + vector ports_tables = { + { APP_PORT_TABLE_NAME, portsorch_base_pri + 5 }, + { APP_VLAN_TABLE_NAME, portsorch_base_pri + 2 }, + { APP_VLAN_MEMBER_TABLE_NAME, portsorch_base_pri }, + { APP_LAG_TABLE_NAME, portsorch_base_pri + 4 }, + { APP_LAG_MEMBER_TABLE_NAME, portsorch_base_pri } + }; + + vector flex_counter_tables = { + CFG_FLEX_COUNTER_TABLE_NAME + }; + + m_FlexCounterOrch = new FlexCounterOrch(m_config_db.get(), flex_counter_tables); + gDirectory.set(m_FlexCounterOrch); + ut_orch_list.push_back((Orch **)&m_FlexCounterOrch); + + static const vector route_pattern_tables = { + CFG_FLOW_COUNTER_ROUTE_PATTERN_TABLE_NAME, + }; + gFlowCounterRouteOrch = new FlowCounterRouteOrch(m_config_db.get(), route_pattern_tables); + gDirectory.set(gFlowCounterRouteOrch); + ut_orch_list.push_back((Orch **)&gFlowCounterRouteOrch); + + gVrfOrch = new VRFOrch(m_app_db.get(), APP_VRF_TABLE_NAME, m_state_db.get(), STATE_VRF_OBJECT_TABLE_NAME); + gDirectory.set(gVrfOrch); + ut_orch_list.push_back((Orch **)&gVrfOrch); + + gIntfsOrch = new IntfsOrch(m_app_db.get(), APP_INTF_TABLE_NAME, gVrfOrch, m_chassis_app_db.get()); + gDirectory.set(gIntfsOrch); + ut_orch_list.push_back((Orch **)&gIntfsOrch); + + gPortsOrch = new PortsOrch(m_app_db.get(), m_state_db.get(), ports_tables, m_chassis_app_db.get()); + gDirectory.set(gPortsOrch); + ut_orch_list.push_back((Orch **)&gPortsOrch); + + const int fgnhgorch_pri = 15; + + vector fgnhg_tables = { + { CFG_FG_NHG, fgnhgorch_pri }, + { CFG_FG_NHG_PREFIX, fgnhgorch_pri }, + { CFG_FG_NHG_MEMBER, fgnhgorch_pri } + }; + + gFgNhgOrch = new FgNhgOrch(m_config_db.get(), m_app_db.get(), m_state_db.get(), fgnhg_tables, gNeighOrch, gIntfsOrch, gVrfOrch); + gDirectory.set(gFgNhgOrch); + ut_orch_list.push_back((Orch **)&gFgNhgOrch); + + const int fdborch_pri = 20; + + vector app_fdb_tables = { + { APP_FDB_TABLE_NAME, FdbOrch::fdborch_pri }, + { APP_VXLAN_FDB_TABLE_NAME, FdbOrch::fdborch_pri }, + { APP_MCLAG_FDB_TABLE_NAME, fdborch_pri } + }; + + TableConnector stateDbFdb(m_state_db.get(), STATE_FDB_TABLE_NAME); + TableConnector stateMclagDbFdb(m_state_db.get(), STATE_MCLAG_REMOTE_FDB_TABLE_NAME); + gFdbOrch = new FdbOrch(m_app_db.get(), app_fdb_tables, stateDbFdb, stateMclagDbFdb, gPortsOrch); + gDirectory.set(gFdbOrch); + ut_orch_list.push_back((Orch **)&gFdbOrch); + + gNeighOrch = new NeighOrch(m_app_db.get(), APP_NEIGH_TABLE_NAME, gIntfsOrch, gFdbOrch, gPortsOrch, m_chassis_app_db.get()); + gDirectory.set(gNeighOrch); + ut_orch_list.push_back((Orch **)&gNeighOrch); + + m_TunnelDecapOrch = new TunnelDecapOrch(m_app_db.get(), APP_TUNNEL_DECAP_TABLE_NAME); + gDirectory.set(m_TunnelDecapOrch); + ut_orch_list.push_back((Orch **)&m_TunnelDecapOrch); + vector mux_tables = { + CFG_MUX_CABLE_TABLE_NAME, + CFG_PEER_SWITCH_TABLE_NAME + }; + + TableConnector stateDbSwitchTable(m_state_db.get(), STATE_SWITCH_CAPABILITY_TABLE_NAME); + TableConnector app_switch_table(m_app_db.get(), APP_SWITCH_TABLE_NAME); + TableConnector conf_asic_sensors(m_config_db.get(), CFG_ASIC_SENSORS_TABLE_NAME); + + vector switch_tables = { + conf_asic_sensors, + app_switch_table + }; + vector policer_tables = { + TableConnector(m_config_db.get(), CFG_POLICER_TABLE_NAME), + TableConnector(m_config_db.get(), CFG_PORT_STORM_CONTROL_TABLE_NAME) + }; + + TableConnector stateDbStorm(m_state_db.get(), STATE_BUM_STORM_CAPABILITY_TABLE_NAME); + gPolicerOrch = new PolicerOrch(policer_tables, gPortsOrch); + gDirectory.set(gPolicerOrch); + ut_orch_list.push_back((Orch **)&gPolicerOrch); + + gSwitchOrch = new SwitchOrch(m_app_db.get(), switch_tables, stateDbSwitchTable); + gDirectory.set(gSwitchOrch); + ut_orch_list.push_back((Orch **)&gSwitchOrch); + + gNhgOrch = new NhgOrch(m_app_db.get(), APP_NEXTHOP_GROUP_TABLE_NAME); + gDirectory.set(gNhgOrch); + ut_orch_list.push_back((Orch **)&gNhgOrch); + + vector srv6_tables = { + APP_SRV6_SID_LIST_TABLE_NAME, + APP_SRV6_MY_SID_TABLE_NAME + }; + gSrv6Orch = new Srv6Orch(m_app_db.get(), srv6_tables, gSwitchOrch, gVrfOrch, gNeighOrch); + gDirectory.set(gSrv6Orch); + ut_orch_list.push_back((Orch **)&gSrv6Orch); + gCrmOrch = new CrmOrch(m_config_db.get(), CFG_CRM_TABLE_NAME); + gDirectory.set(gCrmOrch); + ut_orch_list.push_back((Orch **)&gCrmOrch); + + const int routeorch_pri = 5; + vector route_tables = { + { APP_ROUTE_TABLE_NAME, routeorch_pri }, + { APP_LABEL_ROUTE_TABLE_NAME, routeorch_pri } + }; + gRouteOrch = new RouteOrch(m_app_db.get(), route_tables, gSwitchOrch, gNeighOrch, gIntfsOrch, gVrfOrch, gFgNhgOrch, gSrv6Orch); + gDirectory.set(gRouteOrch); + ut_orch_list.push_back((Orch **)&gRouteOrch); + TableConnector stateDbMirrorSession(m_state_db.get(), STATE_MIRROR_SESSION_TABLE_NAME); + TableConnector confDbMirrorSession(m_config_db.get(), CFG_MIRROR_SESSION_TABLE_NAME); + gMirrorOrch = new MirrorOrch(stateDbMirrorSession, confDbMirrorSession, gPortsOrch, gRouteOrch, gNeighOrch, gFdbOrch, gPolicerOrch); + gDirectory.set(gMirrorOrch); + ut_orch_list.push_back((Orch **)&gMirrorOrch); + + TableConnector confDbAclTable(m_config_db.get(), CFG_ACL_TABLE_TABLE_NAME); + TableConnector confDbAclTableType(m_config_db.get(), CFG_ACL_TABLE_TYPE_TABLE_NAME); + TableConnector confDbAclRuleTable(m_config_db.get(), CFG_ACL_RULE_TABLE_NAME); + TableConnector appDbAclTable(m_app_db.get(), APP_ACL_TABLE_TABLE_NAME); + TableConnector appDbAclTableType(m_app_db.get(), APP_ACL_TABLE_TYPE_TABLE_NAME); + TableConnector appDbAclRuleTable(m_app_db.get(), APP_ACL_RULE_TABLE_NAME); + + vector acl_table_connectors = { + confDbAclTableType, + confDbAclTable, + confDbAclRuleTable, + appDbAclTable, + appDbAclRuleTable, + appDbAclTableType, + }; + gAclOrch = new AclOrch(acl_table_connectors, m_state_db.get(), + gSwitchOrch, gPortsOrch, gMirrorOrch, gNeighOrch, gRouteOrch, NULL); + gDirectory.set(gAclOrch); + ut_orch_list.push_back((Orch **)&gAclOrch); + + m_MuxOrch = new MuxOrch(m_config_db.get(), mux_tables, m_TunnelDecapOrch, gNeighOrch, gFdbOrch); + gDirectory.set(m_MuxOrch); + ut_orch_list.push_back((Orch **)&m_MuxOrch); + + m_MuxCableOrch = new MuxCableOrch(m_app_db.get(), m_state_db.get(), APP_MUX_CABLE_TABLE_NAME); + gDirectory.set(m_MuxCableOrch); + ut_orch_list.push_back((Orch **)&m_MuxCableOrch); + + m_MuxStateOrch = new MuxStateOrch(m_state_db.get(), STATE_HW_MUX_CABLE_TABLE_NAME); + gDirectory.set(m_MuxStateOrch); + ut_orch_list.push_back((Orch **)&m_MuxStateOrch); + + ApplyDualTorConfigs(); + INIT_SAI_API_MOCK(neighbor); + INIT_SAI_API_MOCK(route); + INIT_SAI_API_MOCK(acl); + INIT_SAI_API_MOCK(next_hop); + MockSaiApis(); + } + + void TearDown() override + { + for (std::vector::reverse_iterator rit = ut_orch_list.rbegin(); rit != ut_orch_list.rend(); ++rit) + { + Orch **orch = *rit; + delete *orch; + *orch = nullptr; + } + + gDirectory.m_values.clear(); + + RestoreSaiApis(); + ut_helper::uninitSaiApi(); + } + }; + + TEST_F(MuxRollbackTest, StandbyToActiveNeighborAlreadyExists) + { + EXPECT_CALL(*mock_sai_neighbor_api, create_neighbor_entry) + .WillOnce(Return(SAI_STATUS_ITEM_ALREADY_EXISTS)); + SetAndAssertMuxState(ACTIVE); + } + + TEST_F(MuxRollbackTest, ActiveToStandbyNeighborNotFound) + { + SetAndAssertMuxState(ACTIVE); + EXPECT_CALL(*mock_sai_neighbor_api, remove_neighbor_entry) + .WillOnce(Return(SAI_STATUS_ITEM_NOT_FOUND)); + SetAndAssertMuxState(STANDBY); + } + + TEST_F(MuxRollbackTest, StandbyToActiveRouteNotFound) + { + EXPECT_CALL(*mock_sai_route_api, remove_route_entry) + .WillOnce(Return(SAI_STATUS_ITEM_NOT_FOUND)); + SetAndAssertMuxState(ACTIVE); + } + + TEST_F(MuxRollbackTest, ActiveToStandbyRouteAlreadyExists) + { + SetAndAssertMuxState(ACTIVE); + EXPECT_CALL(*mock_sai_route_api, create_route_entry) + .WillOnce(Return(SAI_STATUS_ITEM_ALREADY_EXISTS)); + SetAndAssertMuxState(STANDBY); + } + + TEST_F(MuxRollbackTest, StandbyToActiveAclNotFound) + { + EXPECT_CALL(*mock_sai_acl_api, remove_acl_entry) + .WillOnce(Return(SAI_STATUS_ITEM_NOT_FOUND)); + SetAndAssertMuxState(ACTIVE); + } + + TEST_F(MuxRollbackTest, ActiveToStandbyAclAlreadyExists) + { + SetAndAssertMuxState(ACTIVE); + EXPECT_CALL(*mock_sai_acl_api, create_acl_entry) + .WillOnce(Return(SAI_STATUS_ITEM_ALREADY_EXISTS)); + SetAndAssertMuxState(STANDBY); + } + + TEST_F(MuxRollbackTest, StandbyToActiveNextHopAlreadyExists) + { + EXPECT_CALL(*mock_sai_next_hop_api, create_next_hop) + .WillOnce(Return(SAI_STATUS_ITEM_ALREADY_EXISTS)); + SetAndAssertMuxState(ACTIVE); + } + + TEST_F(MuxRollbackTest, ActiveToStandbyNextHopNotFound) + { + SetAndAssertMuxState(ACTIVE); + EXPECT_CALL(*mock_sai_next_hop_api, remove_next_hop) + .WillOnce(Return(SAI_STATUS_ITEM_NOT_FOUND)); + SetAndAssertMuxState(STANDBY); + } + + TEST_F(MuxRollbackTest, StandbyToActiveRuntimeErrorRollbackToStandby) + { + EXPECT_CALL(*mock_sai_route_api, remove_route_entry) + .WillOnce(Throw(runtime_error("Mock runtime error"))); + SetMuxStateFromAppDb(ACTIVE); + EXPECT_EQ(STANDBY, m_MuxCable->getState()); + } + + TEST_F(MuxRollbackTest, ActiveToStandbyRuntimeErrorRollbackToActive) + { + SetAndAssertMuxState(ACTIVE); + EXPECT_CALL(*mock_sai_route_api, create_route_entry) + .WillOnce(Throw(runtime_error("Mock runtime error"))); + SetMuxStateFromAppDb(STANDBY); + EXPECT_EQ(ACTIVE, m_MuxCable->getState()); + } + + TEST_F(MuxRollbackTest, StandbyToActiveLogicErrorRollbackToStandby) + { + EXPECT_CALL(*mock_sai_neighbor_api, create_neighbor_entry) + .WillOnce(Throw(logic_error("Mock logic error"))); + SetMuxStateFromAppDb(ACTIVE); + EXPECT_EQ(STANDBY, m_MuxCable->getState()); + } + + TEST_F(MuxRollbackTest, ActiveToStandbyLogicErrorRollbackToActive) + { + SetAndAssertMuxState(ACTIVE); + EXPECT_CALL(*mock_sai_neighbor_api, remove_neighbor_entry) + .WillOnce(Throw(logic_error("Mock logic error"))); + SetMuxStateFromAppDb(STANDBY); + EXPECT_EQ(ACTIVE, m_MuxCable->getState()); + } + + TEST_F(MuxRollbackTest, StandbyToActiveExceptionRollbackToStandby) + { + EXPECT_CALL(*mock_sai_next_hop_api, create_next_hop) + .WillOnce(Throw(exception())); + SetMuxStateFromAppDb(ACTIVE); + EXPECT_EQ(STANDBY, m_MuxCable->getState()); + } + + TEST_F(MuxRollbackTest, ActiveToStandbyExceptionRollbackToActive) + { + SetAndAssertMuxState(ACTIVE); + EXPECT_CALL(*mock_sai_next_hop_api, remove_next_hop) + .WillOnce(Throw(exception())); + SetMuxStateFromAppDb(STANDBY); + EXPECT_EQ(ACTIVE, m_MuxCable->getState()); + } +} diff --git a/tests/mock_tests/ut_saihelper.cpp b/tests/mock_tests/ut_saihelper.cpp index b871a2c137c3..f7a1f2fb2dbd 100644 --- a/tests/mock_tests/ut_saihelper.cpp +++ b/tests/mock_tests/ut_saihelper.cpp @@ -75,6 +75,7 @@ namespace ut_helper sai_api_query(SAI_API_NEIGHBOR, (void **)&sai_neighbor_api); sai_api_query(SAI_API_TUNNEL, (void **)&sai_tunnel_api); sai_api_query(SAI_API_NEXT_HOP, (void **)&sai_next_hop_api); + sai_api_query(SAI_API_NEXT_HOP_GROUP, (void **)&sai_next_hop_group_api); sai_api_query(SAI_API_ACL, (void **)&sai_acl_api); sai_api_query(SAI_API_HOSTIF, (void **)&sai_hostif_api); sai_api_query(SAI_API_POLICER, (void **)&sai_policer_api);