[tunnelmgrd]: Warm boot support (#2166)

* [tunnelmgrd]: Warm boot support
Signed-off-by: Lawrence Lee <lawlee@microsoft.com>

cfgmgr/tunnelmgr.cpp

@@ -9,6 +9,7 @@
 #include "tokenize.h"
 #include "shellcmd.h"
 #include "exec.h"
+#include "warm_restart.h"
 
 using namespace std;
 using namespace swss;
@@ -107,7 +108,8 @@ static int cmdIpTunnelRouteDel(const std::string& pfx, std::string & res)
 TunnelMgr::TunnelMgr(DBConnector *cfgDb, DBConnector *appDb, const std::vector<std::string> &tableNames) :
  Orch(cfgDb, tableNames),
  m_appIpInIpTunnelTable(appDb, APP_TUNNEL_DECAP_TABLE_NAME),
- m_cfgPeerTable(cfgDb, CFG_PEER_SWITCH_TABLE_NAME)
+ m_cfgPeerTable(cfgDb, CFG_PEER_SWITCH_TABLE_NAME),
+ m_cfgTunnelTable(cfgDb, CFG_TUNNEL_TABLE_NAME)
 {
  std::vector<string> peer_keys;
  m_cfgPeerTable.getKeys(peer_keys);
@@ -126,6 +128,23 @@ TunnelMgr::TunnelMgr(DBConnector *cfgDb, DBConnector *appDb, const std::vector<s
  }
  }
  }
+
+ if (WarmStart::isWarmStart())
+ {
+ std::vector<string> tunnel_keys;
+ m_cfgTunnelTable.getKeys(tunnel_keys);
+
+ for (auto tunnel: tunnel_keys)
+ {
+ m_tunnelReplay.insert(tunnel);
+ }
+ if (m_tunnelReplay.empty())
+ {
+ finalizeWarmReboot();
+ }
+
+ }
+
 
  auto consumerStateTable = new swss::ConsumerStateTable(appDb, APP_TUNNEL_ROUTE_TABLE_NAME,
  TableConsumable::DEFAULT_POP_BATCH_SIZE, default_orch_pri);
@@ -191,6 +210,11 @@ void TunnelMgr::doTask(Consumer &consumer)
  ++it;
  }
  }
+
+ if (!replayDone && m_tunnelReplay.empty() && WarmStart::isWarmStart())
+ {
+ finalizeWarmReboot();
+ }
 }
 
 bool TunnelMgr::doTunnelTask(const KeyOpFieldsValuesTuple & t)
@@ -230,8 +254,16 @@ bool TunnelMgr::doTunnelTask(const KeyOpFieldsValuesTuple & t)
  SWSS_LOG_NOTICE("Peer/Remote IP not configured");
  }
 
- m_appIpInIpTunnelTable.set(tunnelName, kfvFieldsValues(t));
+ /* If the tunnel is already in hardware (i.e. present in the replay),
+ * don't try to create it again since it will cause an OA crash
+ * (warmboot case)
+ */
+ if (m_tunnelReplay.find(tunnelName) == m_tunnelReplay.end())
+ {
+ m_appIpInIpTunnelTable.set(tunnelName, kfvFieldsValues(t));
+ }
  }
+ m_tunnelReplay.erase(tunnelName);
  m_tunnelCache[tunnelName] = tunInfo;
  }
  else
@@ -356,3 +388,13 @@ bool TunnelMgr::configIpTunnel(const TunnelInfo& tunInfo)
 
  return true;
 }
+
+
+void TunnelMgr::finalizeWarmReboot()
+{
+ replayDone = true;
+ WarmStart::setWarmStartState("tunnelmgrd", WarmStart::REPLAYED);
+ SWSS_LOG_NOTICE("tunnelmgrd warmstart state set to REPLAYED");
+ WarmStart::setWarmStartState("tunnelmgrd", WarmStart::RECONCILED);
+ SWSS_LOG_NOTICE("tunnelmgrd warmstart state set to RECONCILED");
+}

cfgmgr/tunnelmgr.h

@@ -4,6 +4,8 @@
 #include "producerstatetable.h"
 #include "orch.h"
 
+#include <set>
+
 namespace swss {
 
 struct TunnelInfo
@@ -28,12 +30,18 @@ class TunnelMgr : public Orch
 
  bool configIpTunnel(const TunnelInfo& info);
 
+ void finalizeWarmReboot();
+
  ProducerStateTable m_appIpInIpTunnelTable;
  Table m_cfgPeerTable;
+ Table m_cfgTunnelTable;
 
  std::map<std::string, TunnelInfo > m_tunnelCache;
  std::map<std::string, IpPrefix> m_intfCache;
  std::string m_peerIp;
+
+ std::set<std::string> m_tunnelReplay;
+ bool replayDone = false;
 };
 
 }

cfgmgr/tunnelmgrd.cpp

@@ -11,6 +11,7 @@
 #include "exec.h"
 #include "schema.h"
 #include "tunnelmgr.h"
+#include "warm_restart.h"
 
 using namespace std;
 using namespace swss;
@@ -54,6 +55,9 @@ int main(int argc, char **argv)
  DBConnector cfgDb("CONFIG_DB", 0);
  DBConnector appDb("APPL_DB", 0);
 
+ WarmStart::initialize("tunnelmgrd", "swss");
+ WarmStart::checkWarmStart("tunnelmgrd", "swss");
+
  TunnelMgr tunnelmgr(&cfgDb, &appDb, cfgTunTables);
 
  std::vector<Orch *> cfgOrchList = {&tunnelmgr};

tests/test_warm_reboot.py

@@ -2365,6 +2365,35 @@ def test_EverflowWarmReboot(self, dvs, dvs_acl):
  dvs.start_swss()
  dvs.check_swss_ready()
 
+ def test_TunnelMgrdWarmRestart(self, dvs):
+ tunnel_name = "MuxTunnel0"
+ tunnel_table = "TUNNEL_DECAP_TABLE"
+ tunnel_params = {
+ "tunnel_type": "IPINIP",
+ "dst_ip": "10.1.0.32",
+ "dscp_mode": "uniform",
+ "ecn_mode": "standard",
+ "ttl_mode": "pipe"
+ }
+
+ pubsub = dvs.SubscribeAppDbObject(tunnel_table)
+
+ dvs.runcmd("config warm_restart enable swss")
+ config_db = dvs.get_config_db()
+ config_db.create_entry("TUNNEL", tunnel_name, tunnel_params)
+
+ app_db = dvs.get_app_db()
+ app_db.wait_for_matching_keys(tunnel_table, [tunnel_name])
+
+ nadd, ndel = dvs.CountSubscribedObjects(pubsub)
+ assert nadd == len(tunnel_params)
+ assert ndel == 1 # Expect 1 deletion as part of table creation
+
+ dvs.runcmd("supervisorctl restart tunnelmgrd")
+ dvs.check_services_ready()
+ nadd, ndel = dvs.CountSubscribedObjects(pubsub)
+ assert nadd == 0
+ assert ndel == 0
 
 # Add Dummy always-pass test at end as workaroud
 # for issue when Flaky fail on final test it invokes module tear-down before retrying