From 5a371bf26b0d38e0adbbd00e508d94f42b090546 Mon Sep 17 00:00:00 2001 From: Rajesh Sankaran Date: Tue, 21 Apr 2020 04:34:12 -0700 Subject: [PATCH] VxlanMgr changes for EVPN VXLAN 1. Add support for EVPN NVO config table. 2. Changes to VxlanTunnel create handler to support KLISH and openconfig. The VXLAN_TUNNEL table in the config db gets populated as follows. NULL,NULL - interface vxlan create NULL,NULL,src_ip, - when SIP is set. NULL,NULL - when SIP is unset. src_ip, - This format is when click is used. 3. VxlanTunnel and TunnelMap handlers are enhanced to also cache the data so that this can be used during the deletion. 4. ARP suppression changes Create state db for vxlanmgr and vlanmgr to sync the tunnel association to vlan 5. Changes to support warm reboot. HLD Location : https://github.com/Azure/SONiC/pull/437/commits Signed-off-by: Rajesh Sankaran --- cfgmgr/vxlanmgr.cpp | 637 +++++++++++++++++++++++++++++++++++++++++-- cfgmgr/vxlanmgr.h | 47 +++- cfgmgr/vxlanmgrd.cpp | 66 ++++- 3 files changed, 719 insertions(+), 31 deletions(-) diff --git a/cfgmgr/vxlanmgr.cpp b/cfgmgr/vxlanmgr.cpp index cdb3c2ab80..f21a2188d8 100644 --- a/cfgmgr/vxlanmgr.cpp +++ b/cfgmgr/vxlanmgr.cpp @@ -165,17 +165,28 @@ static int cmdDetachVxlanIfFromVnet(const swss::VxlanMgr::VxlanInfo & info, std: // Vxlanmgr VxlanMgr::VxlanMgr(DBConnector *cfgDb, DBConnector *appDb, DBConnector *stateDb, const vector &tables) : + m_app_db(appDb), Orch(cfgDb, tables), m_appVxlanTunnelTable(appDb, APP_VXLAN_TUNNEL_TABLE_NAME), m_appVxlanTunnelMapTable(appDb, APP_VXLAN_TUNNEL_MAP_TABLE_NAME), m_appSwitchTable(appDb, APP_SWITCH_TABLE_NAME), + m_appEvpnNvoTable(appDb, APP_EVPN_NVO_TABLE_NAME), m_cfgVxlanTunnelTable(cfgDb, CFG_VXLAN_TUNNEL_TABLE_NAME), m_cfgVnetTable(cfgDb, CFG_VNET_TABLE_NAME), m_stateVrfTable(stateDb, STATE_VRF_TABLE_NAME), - m_stateVxlanTable(stateDb, STATE_VXLAN_TABLE_NAME) + m_stateVxlanTable(stateDb, STATE_VXLAN_TABLE_NAME), + m_stateVlanTable(stateDb, STATE_VLAN_TABLE_NAME), + m_stateTunnelVlanMapTable(stateDb, STATE_TUNNEL_VLAN_MAP_TABLE_NAME), + m_stateVxlanTunnelTable(stateDb, STATE_VXLAN_TUNNEL_TABLE_NAME) { - // Clear old vxlan devices that were created at last time. - clearAllVxlanDevices(); + getAllVxlanNetDevices(); + + if (!WarmStart::isWarmStart()) + { + // Clear old vxlan devices that were created at last time. + clearAllVxlanDevices(); + } + } VxlanMgr::~VxlanMgr() @@ -209,6 +220,10 @@ void VxlanMgr::doTask(Consumer &consumer) { task_result = doVxlanTunnelMapCreateTask(t); } + else if(table_name == CFG_VXLAN_EVPN_NVO_TABLE_NAME) + { + task_result = doVxlanEvpnNvoCreateTask(t); + } else { SWSS_LOG_ERROR("Unknown table : %s", table_name.c_str()); @@ -228,6 +243,10 @@ void VxlanMgr::doTask(Consumer &consumer) { task_result = doVxlanTunnelMapDeleteTask(t); } + else if(table_name == CFG_VXLAN_EVPN_NVO_TABLE_NAME) + { + task_result = doVxlanEvpnNvoDeleteTask(t); + } else { SWSS_LOG_ERROR("Unknown table : %s", table_name.c_str()); @@ -377,10 +396,54 @@ bool VxlanMgr::doVxlanTunnelCreateTask(const KeyOpFieldsValuesTuple & t) SWSS_LOG_ENTER(); const std::string & vxlanTunnelName = kfvKey(t); - m_appVxlanTunnelTable.set(vxlanTunnelName, kfvFieldsValues(t)); // Update vxlan tunnel cache - m_vxlanTunnelCache[vxlanTunnelName] = kfvFieldsValues(t); + TunCacheT tuncache; + + tuncache.fvt = kfvFieldsValues(t); + tuncache.vlan_vni_refcnt = 0; + tuncache.m_sourceIp = fvValue(tuncache.fvt.back()); + + if(tuncache.fvt.size() == 2) + { + // NULL,NULL src_ip, + // This is for the openconfig set src_ip case + std::vector fvt; + + fvt = tuncache.fvt; + fvt.erase(fvt.begin()); + SWSS_LOG_NOTICE("openconfig set SIP case. fvt size = %lu field =%s value=%s", + fvt.size(), fvField(tuncache.fvt.back()).c_str(), + fvValue(tuncache.fvt.back()).c_str()); + + m_appVxlanTunnelTable.set(vxlanTunnelName, fvt); + } + else + { + if(fvField(tuncache.fvt.back()) == "NULL") + { + if(m_vxlanTunnelCache.find(vxlanTunnelName) == m_vxlanTunnelCache.end()) + { + // NULL,NULL openconfig interface create + SWSS_LOG_NOTICE("openconfig interface create"); + } + else + { + // NULL,NULL openconfig src_ip delete + m_appVxlanTunnelTable.del(vxlanTunnelName); + SWSS_LOG_NOTICE("openconfig sip unset"); + } + } + else + { + // src_ip, click create + // openconfig combined interface+sip set + m_appVxlanTunnelTable.set(vxlanTunnelName, kfvFieldsValues(t)); + SWSS_LOG_NOTICE("openconfig combined or click set"); + } + } + + m_vxlanTunnelCache[vxlanTunnelName] = tuncache;; SWSS_LOG_INFO("Create vxlan tunnel %s", vxlanTunnelName.c_str()); return true; @@ -391,12 +454,33 @@ bool VxlanMgr::doVxlanTunnelDeleteTask(const KeyOpFieldsValuesTuple & t) SWSS_LOG_ENTER(); const std::string & vxlanTunnelName = kfvKey(t); - m_appVxlanTunnelTable.del(vxlanTunnelName); - auto it = m_vxlanTunnelCache.find(vxlanTunnelName); - if (it != m_vxlanTunnelCache.end()) + // If there is an NVO referring to this tunnel then hold on. + std::map::iterator it = m_EvpnNvoCache.begin(); + + if((it != m_EvpnNvoCache.end()) && (it->second == vxlanTunnelName)) + { + SWSS_LOG_INFO("Tunnel %s deletion failed. Need to delete NVO", vxlanTunnelName.c_str()); + return false; + } + + // If there are mappings still against this tunnel then hold on. + if(m_vxlanTunnelCache[vxlanTunnelName].vlan_vni_refcnt) + { + SWSS_LOG_INFO("Tunnel %s deletion failed. Need to delete mapping entries", + vxlanTunnelName.c_str()); + return false; + } + + if(isTunnelActive(vxlanTunnelName)) + { + m_appVxlanTunnelTable.del(vxlanTunnelName); + } + + auto it1 = m_vxlanTunnelCache.find(vxlanTunnelName); + if (it1 != m_vxlanTunnelCache.end()) { - m_vxlanTunnelCache.erase(it); + m_vxlanTunnelCache.erase(it1); } SWSS_LOG_INFO("Delete vxlan tunnel %s", vxlanTunnelName.c_str()); @@ -409,9 +493,134 @@ bool VxlanMgr::doVxlanTunnelMapCreateTask(const KeyOpFieldsValuesTuple & t) std::string vxlanTunnelMapName = kfvKey(t); std::replace(vxlanTunnelMapName.begin(), vxlanTunnelMapName.end(), config_db_key_delimiter, delimiter); - m_appVxlanTunnelMapTable.set(vxlanTunnelMapName, kfvFieldsValues(t)); - SWSS_LOG_NOTICE("Create vxlan tunnel map %s", vxlanTunnelMapName.c_str()); + if(m_vxlanTunnelMapCache.find(vxlanTunnelMapName) != m_vxlanTunnelMapCache.end()) + { + SWSS_LOG_ERROR("Map already present : %s", vxlanTunnelMapName.c_str()); + return true; + } + + SWSS_LOG_INFO("Create vxlan tunnel map %s", vxlanTunnelMapName.c_str()); + /*Create vxlan tunnel in Linux kernel, the foramt is vxlanTunnelName-vni, such as VTTNL0001-1000*/ + std::string vlan, vlan_id, vni_id, src_ip, dst_ip(""); + for (auto i : kfvFieldsValues(t)) + { + const std::string & field = fvField(i); + const std::string & value = fvValue(i); + if (field == VLAN) + { + vlan = value; + } + else if (field == VNI) + { + vni_id = value; + } + } + + // Check for VLAN or VNI if they are already mapped + if(m_vlanMapCache.find(vlan) != m_vlanMapCache.end()) + { + SWSS_LOG_ERROR("Vlan %s already mapped. Map Create failed for : %s", + vlan.c_str(), vxlanTunnelMapName.c_str()); + return true; + } + + if(m_vniMapCache.find(vni_id) != m_vniMapCache.end()) + { + SWSS_LOG_ERROR("VNI %s already mapped. Map Create failed for : %s", + vni_id.c_str(), vxlanTunnelMapName.c_str()); + return true; + } + + const auto vlan_prefix = std::string("Vlan"); + const auto prefix_len = vlan_prefix.length(); + vlan_id = vlan.substr(prefix_len); + + size_t found = vxlanTunnelMapName.find(delimiter); + const auto vxlanTunnelName = vxlanTunnelMapName.substr(0, found); + + // If the vxlan tunnel has been created + auto it = m_vxlanTunnelCache.find(vxlanTunnelName); + //if (it == m_vxlanTunnelCache.end() ) + if(!isTunnelActive(vxlanTunnelName)) + { + SWSS_LOG_INFO("Vxlan tunnel %s has not been created", vxlanTunnelName.c_str()); + // Suspend this message util the vxlan tunnel is created + return false; + } + + if(!isVlanStateOk(vlan)) + { + SWSS_LOG_INFO("VLAN id is not yet created : %s",vxlanTunnelMapName.c_str()); + return false; + } + + /* Check the below condition only after the vxlanmgrd has reached reconcile state */ + WarmStart::WarmStartState state; + WarmStart::getWarmStartState("vxlanmgrd",state); + if (state == WarmStart::RECONCILED) + { + if(m_vxlanTunnelMapCache.empty()) + { + std::vector keys; + m_stateVxlanTunnelTable.getKeys(keys); + if(!keys.empty()) + { + SWSS_LOG_WARN("State VXLAN tunnel table not yet empty."); + return false; + } + } + } + + auto sourceIp = std::find_if( + it->second.fvt.begin(), + it->second.fvt.end(), + [](const FieldValueTuple & fvt){ return fvt.first == SOURCE_IP; }); + if (sourceIp == it->second.fvt.end()) + { + SWSS_LOG_DEBUG("Vxlan tunnel %s has no field src_ip", vxlanTunnelName.c_str()); + return true; + } + else + { + src_ip = sourceIp->second; + } + auto dstIp = std::find_if( + it->second.fvt.begin(), + it->second.fvt.end(), + [](const FieldValueTuple & fvt){ return fvt.first == DST_IP; }); + if (dstIp != it->second.fvt.end()) + { + dst_ip = dstIp->second; + } + else + { + dst_ip = ""; + } + + createAppDBTunnelMapTable(t); + createVxlanNetdevice(vxlanTunnelName, vni_id, src_ip, dst_ip, vlan_id); + + std::string vxlan_dev_name; + vxlan_dev_name = std::string("") + std::string(vxlanTunnelName) + "-" + std::string(vlan_id); + + MapCacheT map_entry; + map_entry.vxlan_dev_name = vxlan_dev_name; + map_entry.vlan = vlan; + map_entry.vni_id = vni_id; + + m_vxlanTunnelMapCache[vxlanTunnelMapName] = map_entry; + m_vlanMapCache[vlan] = vni_id; + m_vniMapCache[vni_id] = vlan; + m_vxlanTunnelCache[vxlanTunnelName].vlan_vni_refcnt++; + + //Inform the Vlan Mgr to update the tunnel flags if Arp/Nd Suppression is set. + std::string key = "Vlan" + std::string(vlan_id); + vector fvVector; + FieldValueTuple s("netdev", vxlan_dev_name); + fvVector.push_back(s); + m_stateTunnelVlanMapTable.set(key,fvVector); + return true; } @@ -421,9 +630,109 @@ bool VxlanMgr::doVxlanTunnelMapDeleteTask(const KeyOpFieldsValuesTuple & t) std::string vxlanTunnelMapName = kfvKey(t); std::replace(vxlanTunnelMapName.begin(), vxlanTunnelMapName.end(), config_db_key_delimiter, delimiter); - m_appVxlanTunnelMapTable.del(vxlanTunnelMapName); - SWSS_LOG_NOTICE("Delete vxlan tunnel map %s", vxlanTunnelMapName.c_str()); + delAppDBTunnelMapTable(vxlanTunnelMapName); + + SWSS_LOG_INFO("Delete vxlan tunnel map %s", vxlanTunnelMapName.c_str()); + + // ip link del dev {{VXLAN}} + size_t found = vxlanTunnelMapName.find(delimiter); + const auto vxlanTunnelName = vxlanTunnelMapName.substr(0, found); + + std::string vxlan_dev_name,vlan,vni_id; + MapCacheT map_entry; + + try + { + map_entry = m_vxlanTunnelMapCache.at(vxlanTunnelMapName); + } + catch (const std::out_of_range& oor) + { + SWSS_LOG_ERROR("Error deleting tunnmap : %s exception : %s", + vxlanTunnelMapName.c_str(), oor.what()); + return true; + } + + vxlan_dev_name = map_entry.vxlan_dev_name; + vlan = map_entry.vlan; + vni_id = map_entry.vni_id; + deleteVxlanNetdevice(vxlan_dev_name); + + m_vxlanTunnelMapCache.erase(vxlanTunnelMapName); + m_vlanMapCache.erase(vlan); + m_vniMapCache.erase(vni_id); + m_vxlanTunnelCache[vxlanTunnelName].vlan_vni_refcnt--; + + //Delete the state table map of vlan to tunnel name. + std::string vlan_delimiter = "-"; + found = vxlan_dev_name.find(vlan_delimiter); + std::string key = "Vlan" + vxlan_dev_name.substr(found+1,vxlan_dev_name.length()); + SWSS_LOG_INFO("Delete Tunnel Map for %s -> %s ", key.c_str(), vxlan_dev_name.c_str()); + m_stateTunnelVlanMapTable.del(key); + return true; +} + +bool VxlanMgr::doVxlanEvpnNvoCreateTask(const KeyOpFieldsValuesTuple & t) +{ + SWSS_LOG_ENTER(); + + std::string EvpnNvoName = kfvKey(t); + + if(m_EvpnNvoCache.find(EvpnNvoName) != m_EvpnNvoCache.end()) + { + SWSS_LOG_ERROR("Only Single NVO object allowed"); + return true; + } + + for (auto i : kfvFieldsValues(t)) + { + const std::string & field = fvField(i); + const std::string & value = fvValue(i); + //if(m_vxlanTunnelCache.find(value) == m_vxlanTunnelCache.end()) + if(!isTunnelActive(value)) + { + SWSS_LOG_ERROR("NVO %s creation failed. VTEP not present",EvpnNvoName.c_str()); + return false; + } + if (field == SOURCE_VTEP) + { + m_EvpnNvoCache[EvpnNvoName] = value; + } + } + + std::replace(EvpnNvoName.begin(), EvpnNvoName.end(), config_db_key_delimiter, delimiter); + m_appEvpnNvoTable.set(EvpnNvoName, kfvFieldsValues(t)); + + SWSS_LOG_INFO("Create evpn nvo %s", EvpnNvoName.c_str()); + return true; +} + +bool VxlanMgr::doVxlanEvpnNvoDeleteTask(const KeyOpFieldsValuesTuple & t) +{ + SWSS_LOG_ENTER(); + + std::string EvpnNvoName = kfvKey(t); + std::string vtep_name; + try + { + vtep_name = m_EvpnNvoCache.at(EvpnNvoName); + } + catch (const std::out_of_range& oor) + { + SWSS_LOG_ERROR("NVOdeletion NVO : %s not found exception : %s", EvpnNvoName.c_str(), oor.what()); + return true; + } + + // If there are mappings still then the NVO cannot be deleted. + if(m_vxlanTunnelCache[vtep_name].vlan_vni_refcnt) + return false; + + m_EvpnNvoCache.erase(EvpnNvoName); + + std::replace(EvpnNvoName.begin(), EvpnNvoName.end(), config_db_key_delimiter, delimiter); + m_appEvpnNvoTable.del(EvpnNvoName); + + SWSS_LOG_INFO("Delete evpn nvo %s", EvpnNvoName.c_str()); return true; } @@ -456,6 +765,23 @@ bool VxlanMgr::isVxlanStateOk(const std::string & vxlanName) return false; } +bool VxlanMgr::isVlanStateOk(const std::string &vlanName) +{ + SWSS_LOG_ENTER(); + std::vector temp; + + if (!vlanName.compare(0, strlen(VLAN_PREFIX), VLAN_PREFIX)) + { + if (m_stateVlanTable.get(vlanName, temp)) + { + SWSS_LOG_DEBUG("%s is ready", vlanName.c_str()); + return true; + } + } + SWSS_LOG_INFO("%s is not ready", vlanName.c_str()); + return false; +} + std::pair VxlanMgr::getVxlanRouterMacAddress() { std::vector temp; @@ -585,10 +911,125 @@ bool VxlanMgr::deleteVxlan(const VxlanInfo & info) return true; } -void VxlanMgr::clearAllVxlanDevices() +void VxlanMgr::createAppDBTunnelMapTable(const KeyOpFieldsValuesTuple & t) +{ + std::string vxlanTunnelMapName = kfvKey(t); + + std::replace(vxlanTunnelMapName.begin(), vxlanTunnelMapName.end(), config_db_key_delimiter, delimiter); + + /* Case 1: Entry exist - Erase from cache & return + * Case 2: Enry does not exist - Write to AppDB + * Case 3: Entry exist but modified - Not taken care. Will address later + */ + if (true == m_in_reconcile) + { + auto it = find(m_appVxlanTunnelMapKeysRecon.begin(), m_appVxlanTunnelMapKeysRecon.end(), vxlanTunnelMapName); + if (it != m_appVxlanTunnelMapKeysRecon.end()) + { + m_appVxlanTunnelMapKeysRecon.erase(it); + SWSS_LOG_NOTICE("Reconcile App Tunnel Map Table create %s reconciled. Pending %lu", + vxlanTunnelMapName.c_str(), m_appVxlanTunnelMapKeysRecon.size()); + return; + } + else + { + SWSS_LOG_NOTICE("Reconcile App Tunnel Map Table create %s doesnt not exist. Pending %lu", + vxlanTunnelMapName.c_str(), m_appVxlanTunnelMapKeysRecon.size()); + } + } + else + { + SWSS_LOG_NOTICE("App Tunnel Map Table create %s", vxlanTunnelMapName.c_str()); + } + m_appVxlanTunnelMapTable.set(vxlanTunnelMapName, kfvFieldsValues(t)); + + return; +} + +void VxlanMgr::delAppDBTunnelMapTable(std::string vxlanTunnelMapName) +{ + m_appVxlanTunnelMapTable.del(vxlanTunnelMapName); +} + +int VxlanMgr::createVxlanNetdevice(std::string vxlanTunnelName, std::string vni_id, + std::string src_ip, std::string dst_ip, std::string vlan_id) +{ + int ret = 0; + + //ip link add type vxlan id local remote dstport 4789 + //ip link set master DOT1Q_BRIDGE_NAME + //bridge vlan add vid dev + //ip link set up + std::string vxlan_dev_name; + vxlan_dev_name = std::string("") + std::string(vxlanTunnelName) + "-" + std::string(vlan_id); + + SWSS_LOG_NOTICE("Kernel tnl_name: %s vni_id: %s src_ip: %s dst_ip:%s vlan_id: %s", + vxlanTunnelName.c_str(), vni_id.c_str(), src_ip.c_str(), dst_ip.c_str(), vlan_id.c_str()); + /* Case 1: Entry exist - Erase from cache & return + * Case 2: Enry does not exist - Create netDevice in Kernel + * Case 3: Entry exist but modified - Not taken care. Will address later + */ + if (true == m_in_reconcile) + { + auto it = m_vxlanNetDevices.find(vxlan_dev_name); + if (it != m_vxlanNetDevices.end()) + { + m_vxlanNetDevices.erase(it); + SWSS_LOG_NOTICE("Reconcile VxlanNetDevice %s reconciled. Pending %lu", + vxlan_dev_name.c_str(), m_vxlanNetDevices.size()); + return 0; + } + else + { + SWSS_LOG_NOTICE("Reconcile VxlanNetDevice %s doesn't not exist. Pending %lu", + vxlan_dev_name.c_str(), m_vxlanNetDevices.size()); + } + } + else + { + SWSS_LOG_NOTICE("Creating VxlanNetDevice %s", vxlan_dev_name.c_str()); + } + + std::string res, chkcmd, cmds; + chkcmd = "ip link show " + vxlan_dev_name; + ret = swss::exec(chkcmd,res); + if(ret == 0) + { + cmds = "ip link del dev " + vxlan_dev_name ; + EXEC_WITH_ERROR_THROW(cmds, res); + } + cmds = std::string("") + + BASH_CMD + " -c \"" + + IP_CMD + " link add " + vxlan_dev_name + " type vxlan id " + std::string(vni_id) + " address " + gMacAddress.to_string() + + " local " + src_ip + ((dst_ip == "")? "":(" remote " + dst_ip)) + " nolearning " +" dstport 4789 " + " && " + + IP_CMD + " link set " + vxlan_dev_name + " master Bridge " + + " && " + + BRIDGE_CMD + " vlan add vid " + std::string(vlan_id) + " dev " + vxlan_dev_name + + " && " + + BRIDGE_CMD + " vlan add vid " + std::string(vlan_id) + " untagged pvid dev " + vxlan_dev_name + + " && "; + if( vlan_id != "1") + { + cmds = cmds + BRIDGE_CMD + " vlan del vid 1 dev " + vxlan_dev_name + " && "; + } + cmds = cmds + IP_CMD + " link set " + vxlan_dev_name + " up " + "\""; + EXEC_WITH_ERROR_THROW(cmds, res); + return ret; +} + +int VxlanMgr::deleteVxlanNetdevice(std::string vxlan_dev_name) +{ + int ret = 0; + std::string res; + const std::string cmd = std::string("") + IP_CMD " link del dev " + vxlan_dev_name; + EXECUTE(cmd, res); + return ret; +} + +void VxlanMgr::getAllVxlanNetDevices() { std::string stdout; - const std::string cmd = std::string("") + IP_CMD + " link"; + const std::string cmd = std::string("") + IP_CMD + " link show type vxlan"; int ret = swss::exec(cmd, stdout); if (ret != 0) { @@ -600,22 +1041,170 @@ void VxlanMgr::clearAllVxlanDevices() auto lines = tokenize(stdout, '\n'); for (const std::string & line : lines) { + SWSS_LOG_NOTICE("line : %s\n",line.c_str()); if (!std::regex_search(line, match_result, device_name_pattern)) { continue; } - std::string res; - std::string device_name = match_result[1]; - VxlanInfo info; - if (device_name.find(VXLAN_NAME_PREFIX) == 0) + std::string vxlan_dev_name = match_result[1]; + m_vxlanNetDevices[vxlan_dev_name] = vxlan_dev_name; + } + return; +} + +void VxlanMgr::restoreVxlanNetDevices() +{ + /* Fetch the Src_ip from the vxlanAppTunnelTable */ + /* Currently, there is only 1 src vtep tunnel, hence picking the src_ip from that entry */ + Table vxlanAppTunnelTable = Table(this->m_app_db, APP_VXLAN_TUNNEL_TABLE_NAME); + vector appVxlanTunnelTableKeys; + vxlanAppTunnelTable.getKeys(appVxlanTunnelTableKeys); + std::string src_ip; + std::string dst_ip(""); + dst_ip = ""; + for (auto &k : appVxlanTunnelTableKeys) + { + std::vector temp; + if (vxlanAppTunnelTable.get(k, temp)) { - info.m_vxlan = device_name; - cmdDeleteVxlan(info, res); + for (auto fv: temp) + { + std::string field = fvField(fv); + std::string value = fvValue(fv); + SWSS_LOG_NOTICE("RESTORE Vxlan Tunnel Table key: %s field: %s value: %s", + k.c_str(), field.c_str(), value.c_str()); + if (field == "src_ip") + { + src_ip = value; + SWSS_LOG_NOTICE("RESTORE Vxlan Tunnel Table src_ip: %s", src_ip.c_str()); + } + } } - else if (device_name.find(VXLAN_IF_NAME_PREFIX) == 0) + else { - info.m_vxlanIf = device_name; - cmdDeleteVxlanIf(info, res); + SWSS_LOG_NOTICE("RESTORE VxLAN Tunnel Table Key(%s)", k.c_str()); } } + + Table vxlanAppTunnelMapTable = Table(this->m_app_db, APP_VXLAN_TUNNEL_MAP_TABLE_NAME); + std::vector::iterator it; + for (it = m_appVxlanTunnelMapKeysRecon.begin(); + it != m_appVxlanTunnelMapKeysRecon.end(); + it++) + { + std::string vlan, vlan_id, vni_id; + std::string vxlanTunnelMapName = *it; + std::vector temp; + if (vxlanAppTunnelMapTable.get(vxlanTunnelMapName, temp)) + { + for (auto fv: temp) + { + std::string field = fvField(fv); + std::string value = fvValue(fv); + SWSS_LOG_NOTICE("RESTORE Vxlan Tunnel MAP Table key: %s field: %s value: %s", + vxlanTunnelMapName.c_str(), field.c_str(), value.c_str()); + if (field == VLAN) + { + vlan = value; + } + else if (field == VNI) + { + vni_id = value; + } + } + } + else + { + SWSS_LOG_NOTICE("RESTORE VxLAN Tunnel Map Table Key(%s)", vxlanTunnelMapName.c_str()); + } + const auto vlan_prefix = std::string("Vlan"); + const auto prefix_len = vlan_prefix.length(); + vlan_id = vlan.substr(prefix_len); + + size_t found = vxlanTunnelMapName.find(delimiter); + const auto vxlanTunnelName = vxlanTunnelMapName.substr(0, found); + + createVxlanNetdevice(vxlanTunnelName, vni_id, src_ip, dst_ip, vlan_id); + SWSS_LOG_NOTICE("RESTORE Created Kernel Net Device (%s-%s)", vxlanTunnelName.c_str(), vlan_id.c_str()); + } + + SWSS_LOG_NOTICE("RESTORE Delete Stale Kernel Net Devices"); + clearAllVxlanDevices(); + SWSS_LOG_NOTICE("RESTORE Recreate Kernel Cache"); + getAllVxlanNetDevices(); +} + +void VxlanMgr::clearAllVxlanDevices() +{ + for (auto it = m_vxlanNetDevices.begin(); it != m_vxlanNetDevices.end();) + { + SWSS_LOG_NOTICE("Deleting Stale NetDevice vxlandevname %s\n", (it->first).c_str()); + deleteVxlanNetdevice(it->first); + it = m_vxlanNetDevices.erase(it); + } +} + +void VxlanMgr::waitTillReadyToReconcile() +{ + for(;;) + { + WarmStart::WarmStartState state; + WarmStart::getWarmStartState("vlanmgrd", state); + + if ((WarmStart::REPLAYED == state) || + (WarmStart::RECONCILED == state)) + { + SWSS_LOG_NOTICE("Vlanmgrd Reconciled %d", (int) state); + return; + } + SWSS_LOG_NOTICE("Vlanmgrd NOT Reconciled %d", (int) state); + sleep(1); + } + return; +} + +void VxlanMgr::beginReconcile(bool warm) +{ + m_in_reconcile = true; + Table vxlanAppTunnelMapTable = Table(this->m_app_db, APP_VXLAN_TUNNEL_MAP_TABLE_NAME); + vxlanAppTunnelMapTable.getKeys(m_appVxlanTunnelMapKeysRecon); + for (auto &k : m_appVxlanTunnelMapKeysRecon) + { + SWSS_LOG_NOTICE("App Tunnel Map Key: %s", k.c_str()); + } + SWSS_LOG_NOTICE("Pending %lu entries for the Tunnel Map Table", m_appVxlanTunnelMapKeysRecon.size()); + return; +} + +void VxlanMgr::endReconcile(bool warm) +{ + /* Delete all stale entries from appDb */ + while (m_appVxlanTunnelMapKeysRecon.size()) + { + std::vector::iterator it = m_appVxlanTunnelMapKeysRecon.begin(); + if (it != m_appVxlanTunnelMapKeysRecon.end()) + { + SWSS_LOG_NOTICE("Reconcile Deleting Stale Entry vxlandevname %s\n", m_appVxlanTunnelMapKeysRecon[0].c_str()); + delAppDBTunnelMapTable(m_appVxlanTunnelMapKeysRecon[0]); + m_appVxlanTunnelMapKeysRecon.erase(it); + } + } + SWSS_LOG_NOTICE("End App Tunnel Map Table Reconcile"); + + /* Delete all the stale netDevices from the Kernel */ + clearAllVxlanDevices(); + + m_in_reconcile = false; +} + +bool VxlanMgr::isTunnelActive(std::string vxlanTunnelName) +{ + auto it = m_vxlanTunnelCache.find(vxlanTunnelName); + if (it == m_vxlanTunnelCache.end()) + return false; + + if(m_vxlanTunnelCache[vxlanTunnelName].m_sourceIp == "NULL") + return false; + + return true; } diff --git a/cfgmgr/vxlanmgr.h b/cfgmgr/vxlanmgr.h index d4a1fb0dc2..e90738e18d 100644 --- a/cfgmgr/vxlanmgr.h +++ b/cfgmgr/vxlanmgr.h @@ -28,6 +28,27 @@ class VxlanMgr : public Orch std::string m_vxlanIf; std::string m_macAddress; } VxlanInfo; + + typedef struct TunCacheT + { + std::vector fvt; + std::string m_sourceIp; + uint32_t vlan_vni_refcnt; + } TunCacheT; + + typedef struct MapCacheT + { + std::string vxlan_dev_name; + std::string vlan; + std::string vni_id; + } MapCacheT; + + void waitTillReadyToReconcile(); + void beginReconcile(bool warm); + void endReconcile(bool warm); + void restoreVxlanNetDevices(); + bool isTunnelActive(std::string vxlanTunnelName); + ~VxlanMgr(); private: void doTask(Consumer &consumer); @@ -41,6 +62,16 @@ class VxlanMgr : public Orch bool doVxlanTunnelMapCreateTask(const KeyOpFieldsValuesTuple & t); bool doVxlanTunnelMapDeleteTask(const KeyOpFieldsValuesTuple & t); + bool doVxlanEvpnNvoCreateTask(const KeyOpFieldsValuesTuple & t); + bool doVxlanEvpnNvoDeleteTask(const KeyOpFieldsValuesTuple & t); + + void createAppDBTunnelMapTable(const KeyOpFieldsValuesTuple & t); + void delAppDBTunnelMapTable(std::string vxlanTunnelMapName); + int createVxlanNetdevice(std::string vxlanTunnelName, std::string vni_id, + std::string src_ip, std::string dst_ip, std::string vlan_id); + int deleteVxlanNetdevice(std::string vxlan_dev_name); + void getAllVxlanNetDevices(); + /* * Query the state of vrf by STATE_VRF_TABLE * Return @@ -49,6 +80,7 @@ class VxlanMgr : public Orch */ bool isVrfStateOk(const std::string & vrfName); bool isVxlanStateOk(const std::string & vxlanName); + bool isVlanStateOk(const string &vlanName); std::pair getVxlanRouterMacAddress(); bool createVxlan(const VxlanInfo & info); @@ -56,21 +88,32 @@ class VxlanMgr : public Orch void clearAllVxlanDevices(); - ProducerStateTable m_appVxlanTunnelTable,m_appVxlanTunnelMapTable; + ProducerStateTable m_appVxlanTunnelTable,m_appVxlanTunnelMapTable,m_appEvpnNvoTable; Table m_cfgVxlanTunnelTable,m_cfgVnetTable,m_stateVrfTable,m_stateVxlanTable, m_appSwitchTable; + Table m_stateVlanTable, m_stateTunnelVlanMapTable, m_stateVxlanTunnelTable; /* * Vxlan Tunnel Cache * Key: tunnel name * Value: Field Value pairs of vxlan tunnel */ - std::map > m_vxlanTunnelCache; + std::map m_vxlanTunnelCache; + std::map m_vxlanTunnelMapCache; + std::map m_vlanMapCache; + std::map m_vniMapCache; + std::map m_EvpnNvoCache; + /* * Vnet Cache * Key: Vnet name * Value: Vxlan information of this vnet */ std::map m_vnetCache; + + DBConnector *m_app_db; + bool m_in_reconcile; + vector m_appVxlanTunnelMapKeysRecon; + std::map m_vxlanNetDevices; }; } diff --git a/cfgmgr/vxlanmgrd.cpp b/cfgmgr/vxlanmgrd.cpp index 8e86cfbe49..094447718f 100644 --- a/cfgmgr/vxlanmgrd.cpp +++ b/cfgmgr/vxlanmgrd.cpp @@ -13,6 +13,7 @@ #include "producerstatetable.h" #include "vxlanmgr.h" #include "shellcmd.h" +#include "warm_restart.h" using namespace std; using namespace swss; @@ -35,6 +36,7 @@ ofstream gRecordOfs; string gRecordFile; /* Global database mutex */ mutex gDbMutex; +MacAddress gMacAddress; int main(int argc, char **argv) { @@ -45,26 +47,71 @@ int main(int argc, char **argv) try { - DBConnector cfgDb("CONFIG_DB", 0); - DBConnector appDb("APPL_DB", 0); - DBConnector stateDb("STATE_DB", 0); + DBConnector cfgDb(CONFIG_DB, DBConnector::DEFAULT_UNIXSOCKET, 0); + DBConnector appDb(APPL_DB, DBConnector::DEFAULT_UNIXSOCKET, 0); + DBConnector stateDb(STATE_DB, DBConnector::DEFAULT_UNIXSOCKET, 0); + + WarmStart::initialize("vxlanmgrd", "swss"); + WarmStart::checkWarmStart("vxlanmgrd", "swss"); + if (WarmStart::isWarmStart()) + { + WarmStart::setWarmStartState("vxlanmgrd", WarmStart::INITIALIZED); + } vector cfg_vnet_tables = { CFG_VNET_TABLE_NAME, CFG_VXLAN_TUNNEL_TABLE_NAME, CFG_VXLAN_TUNNEL_MAP_TABLE_NAME, + CFG_VXLAN_EVPN_NVO_TABLE_NAME, }; VxlanMgr vxlanmgr(&cfgDb, &appDb, &stateDb, cfg_vnet_tables); - std::vector cfgOrchList = {&vxlanmgr}; - swss::Select s; for (Orch *o : cfgOrchList) { s.addSelectables(o->getSelectables()); } + /* + * swss service starts after interfaces-config.service which will have + * switch_mac set. + * Dynamic switch_mac update is not supported for now. + */ + Table table(&cfgDb, "DEVICE_METADATA"); + std::vector ovalues; + table.get("localhost", ovalues); + auto it = std::find_if( ovalues.begin(), ovalues.end(), [](const FieldValueTuple& t){ return t.first == "mac";} ); + if ( it == ovalues.end() ) { + throw runtime_error("couldn't find MAC address of the device from config DB"); + } + gMacAddress = MacAddress(it->second); + + auto in_recon = true; + vxlanmgr.beginReconcile(true); + + if (WarmStart::isWarmStart()) + { + vxlanmgr.waitTillReadyToReconcile(); + vxlanmgr.restoreVxlanNetDevices(); + WarmStart::setWarmStartState("vxlanmgrd", WarmStart::REPLAYED); + uint16_t wait_secs = 0; + string val = ""; + Table wb_tbl = Table(&stateDb, STATE_WARM_RESTART_TABLE_NAME); + wb_tbl.hget("orchagent", "restore_count", val); + if ((val != "") or (val != "0")) + { + wb_tbl.hget("orchagent", "state", val); + while (val != "reconciled") + { + SWSS_LOG_NOTICE("Waiting Until Orchagent is reconciled. Current %s. Waited %u secs", val.c_str(), wait_secs); + sleep(1); + wait_secs++; + wb_tbl.hget("orchagent", "state", val); + } + } + } + SWSS_LOG_NOTICE("starting main loop"); while (true) { @@ -79,6 +126,15 @@ int main(int argc, char **argv) } if (ret == Select::TIMEOUT) { + if (true == in_recon) + { + in_recon = false; + vxlanmgr.endReconcile(false); + if (WarmStart::isWarmStart()) + { + WarmStart::setWarmStartState("vxlanmgrd", WarmStart::RECONCILED); + } + } vxlanmgr.doTask(); continue; }