Add warm boot support with removed/created port (#515)

* Add warm boot support with removed/created port

* Fix aspell

* Update sai_warmboot.bin location to local folder

syncd/syncd.cpp

@@ -1257,6 +1257,7 @@ void get_port_related_objects(
 
 void post_port_remove(
  _In_ std::shared_ptr<SaiSwitch> sw,
+ _In_ sai_object_id_t port_rid,
  _In_ const std::vector<sai_object_id_t>& relatedRids)
 {
  SWSS_LOG_ENTER();
@@ -1307,16 +1308,16 @@ void post_port_remove(
 
  sai_object_type_t ot = redis_sai_object_type_query(vid);
 
- std::string key = sai_serialize_object_type(ot) + ":" + str_vid;
+ std::string key = ASIC_STATE_TABLE + std::string(":") + sai_serialize_object_type(ot) + ":" + str_vid;
 
  SWSS_LOG_INFO("removing ASIC DB key: %s", key.c_str());
 
  g_redisClient->del(key);
  }
 
- SWSS_LOG_NOTICE("post port remove actions succeeded");
+ sw->onPostPortRemove(port_rid);
 
- // TODO lane map must be updated (for warm boot)
+ SWSS_LOG_NOTICE("post port remove actions succeeded");
 }
 
 void post_port_create(
@@ -1328,7 +1329,7 @@ void post_port_create(
 
  sw->onPostPortCreate(port_rid, port_vid);
 
- // TODO lane map must be updated (for warm boot)
+ SWSS_LOG_NOTICE("post port create actions succeeded");
 }
 
 sai_status_t handle_generic(
@@ -1560,7 +1561,7 @@ sai_status_t handle_generic(
 
  if (object_type == SAI_OBJECT_TYPE_PORT)
  {
- post_port_remove(switches.at(switch_vid), related);
+ post_port_remove(switches.at(switch_vid), rid, related);
  }
  }
  }
@@ -2418,6 +2419,16 @@ sai_status_t processEventInInitViewMode(
  {
  case SAI_COMMON_API_CREATE:
 
+ if (object_type == SAI_OBJECT_TYPE_PORT)
+ {
+ // reason for this is that if user will create port,
+ // new port is not actually created so when for example
+ // querying new queues for new created port, there are
+ // not there, since no actual port create was issued on
+ // the ASIC
+ SWSS_LOG_THROW("port object can't be created in init view mode");
+ }
+
  if (info->isnonobjectid)
  {
  /*
@@ -2450,6 +2461,17 @@ sai_status_t processEventInInitViewMode(
 
  case SAI_COMMON_API_REMOVE:
 
+ if (object_type == SAI_OBJECT_TYPE_PORT)
+ {
+ // reason for this is that if user will remove port, actual
+ // resources for it wont be release, lanes would be still
+ // occupied and there is extra logic required in post port
+ // remove which clears OIDs (ipgs,queues,SGs) from redis db
+ // that are automatically removed by vendor SAI, and comparison
+ // logic don't support that
+ SWSS_LOG_THROW("port object can't be removed in init view mode");
+ }
+
  if (object_type == SAI_OBJECT_TYPE_SWITCH)
  {
  /*

syncd/syncd_applyview.cpp

@@ -6985,6 +6985,7 @@ void populateExistingObjects(
  auto sw = switches.begin()->second;
 
  auto coldBootDiscoveredVids = sw->getColdBootDiscoveredVids();
+ auto warmBootDiscoveredVids = sw->getWarmBootDiscoveredVids();
 
  /*
  * If some objects that are existing objects on switch are not present in
@@ -7058,9 +7059,47 @@ void populateExistingObjects(
  * NOTE: If we are here, then this RID exists only in current view, and
  * if this object contains any OID attributes, discovery logic queried
  * them so they are also existing in current view.
+ *
+ * Also in warm boot, when user removed port, and then created some new
+ * ports, new QUEUEs, IPGs and SGs will be created automatically by
+ * SAI. Those new created objects mot likely will have different RID
+ * values then previous instances for given port. Those values should
+ * also be copied to temporary view, since they will not exist on cold
+ * boot discovered VIDs. If not, then comparison logic will try to remove
+ * them which is not what we want.
+ *
+ * This is tricky scenario, and there could be some issues also when
+ * other object types would be created by user.
  */
 
- if (coldBootDiscoveredVids.find(vid) == coldBootDiscoveredVids.end())
+ bool performColdCheck = true;
+
+ if (warmBootDiscoveredVids.find(vid) != warmBootDiscoveredVids.end())
+ {
+ sai_object_type_t ot = redis_sai_object_type_query(vid);
+
+ switch (ot)
+ {
+ case SAI_OBJECT_TYPE_QUEUE:
+ case SAI_OBJECT_TYPE_INGRESS_PRIORITY_GROUP:
+ case SAI_OBJECT_TYPE_SCHEDULER_GROUP:
+
+ // TODO this case may require adjustment, if user will do a
+ // warm boot then remove/add some ports and make another
+ // warm boot, it may happen that current logic will be
+ // confused which of those objects are from previous warm
+ // boot or second one, need better way to mark changes to
+ // those objects in redis DB between warm boots
+
+ performColdCheck = false;
+
+ break;
+ default:
+ break;
+ }
+ }
+
+ if (performColdCheck && coldBootDiscoveredVids.find(vid) == coldBootDiscoveredVids.end())
  {
  SWSS_LOG_INFO("object is not on default existing list: %s RID %s VID %s",
  sai_serialize_object_type(sai_object_type_query(rid)).c_str(),

syncd/syncd_hard_reinit.cpp

@@ -1349,7 +1349,7 @@ void performWarmRestart()
  * Perform all get operations on existing switch.
  */
 
- auto sw = switches[switch_vid] = std::make_shared<SaiSwitch>(switch_vid, switch_rid);
+ auto sw = switches[switch_vid] = std::make_shared<SaiSwitch>(switch_vid, switch_rid, true);
 
  g_switch_rid = switch_rid;
  g_switch_vid = switch_vid;

syncd/syncd_saiswitch.cpp

@@ -7,6 +7,8 @@
 #include <unordered_map>
 #include <set>
 
+const int maxLanesPerPort = 8;
+
 /*
  * NOTE: all those methods could be implemented inside SaiSwitch class so then
  * we could skip using switch_id in params and even they could be public then.
@@ -148,18 +150,16 @@ std::unordered_map<sai_uint32_t, sai_object_id_t> SaiSwitch::saiGetHardwareLaneM
  * addressed in future.
  */
 
- const int lanesPerPort = 8;
-
  for (const auto &port_rid : portList)
  {
- sai_uint32_t lanes[lanesPerPort];
+ sai_uint32_t lanes[maxLanesPerPort];
 
  memset(lanes, 0, sizeof(lanes));
 
  sai_attribute_t attr;
 
  attr.id = SAI_PORT_ATTR_HW_LANE_LIST;
- attr.value.u32list.count = lanesPerPort;
+ attr.value.u32list.count = maxLanesPerPort;
  attr.value.u32list.list = lanes;
 
  sai_status_t status = sai_metadata_sai_port_api->get_port_attribute(port_rid, 1, &attr);
@@ -1059,6 +1059,13 @@ std::set<sai_object_id_t> SaiSwitch::getColdBootDiscoveredVids() const
  return discoveredVids;
 }
 
+std::set<sai_object_id_t> SaiSwitch::getWarmBootDiscoveredVids() const
+{
+ SWSS_LOG_ENTER();
+
+ return m_warmBootDiscoveredVids;
+}
+
 void SaiSwitch::redisSaveColdBootDiscoveredVids() const
 {
  SWSS_LOG_ENTER();
@@ -1224,6 +1231,21 @@ sai_object_id_t SaiSwitch::getDefaultValueForOidAttr(
  return ita->second;
 }
 
+void SaiSwitch::helperPopulateWarmBootVids()
+{
+ SWSS_LOG_ENTER();
+
+ if (!m_warmBoot)
+ return;
+
+ for (sai_object_id_t rid: m_discovered_rids)
+ {
+ sai_object_id_t vid = translate_rid_to_vid(rid, m_switch_vid);
+
+ m_warmBootDiscoveredVids.insert(vid);
+ }
+}
+
 /*
  * NOTE: If real ID will change during hard restarts, then we need to remap all
  * VID/RID, but we can only do that if we will save entire tree with all
@@ -1232,7 +1254,9 @@ sai_object_id_t SaiSwitch::getDefaultValueForOidAttr(
 
 SaiSwitch::SaiSwitch(
  _In_ sai_object_id_t switch_vid,
- _In_ sai_object_id_t switch_rid)
+ _In_ sai_object_id_t switch_rid,
+ _In_ bool warmBoot):
+ m_warmBoot(warmBoot)
 {
  SWSS_LOG_ENTER();
 
@@ -1263,9 +1287,68 @@ SaiSwitch::SaiSwitch(
 
  helperLoadColdVids();
 
+ helperPopulateWarmBootVids();
+
  saiGetMacAddress(m_default_mac_address);
 }
 
+std::vector<uint32_t> SaiSwitch::saiGetPortLanes(
+ _In_ sai_object_id_t port_rid)
+{
+ SWSS_LOG_ENTER();
+
+ std::vector<uint32_t> lanes;
+
+ lanes.resize(maxLanesPerPort);
+
+ sai_attribute_t attr;
+
+ attr.id = SAI_PORT_ATTR_HW_LANE_LIST;
+ attr.value.u32list.count = maxLanesPerPort;
+ attr.value.u32list.list = lanes.data();
+
+ sai_status_t status = sai_metadata_sai_port_api->get_port_attribute(port_rid, 1, &attr);
+
+ if (status != SAI_STATUS_SUCCESS)
+ {
+ SWSS_LOG_THROW("failed to get hardware lane list port RID %s: %s",
+ sai_serialize_object_id(port_rid).c_str(),
+ sai_serialize_status(status).c_str());
+ }
+
+ if (attr.value.u32list.count == 0)
+ {
+ SWSS_LOG_THROW("switch returned lane count ZERO for port RID %s",
+ sai_serialize_object_id(port_rid).c_str());
+ }
+
+ lanes.resize(attr.value.u32list.count);
+
+ return lanes;
+}
+
+void SaiSwitch::redisUpdatePortLaneMap(
+ _In_ sai_object_id_t port_rid)
+{
+ SWSS_LOG_ENTER();
+
+ auto lanes = saiGetPortLanes(port_rid);
+
+ for (uint32_t lane: lanes)
+ {
+ std::string strLane = sai_serialize_number(lane);
+ std::string strPortId = sai_serialize_object_id(port_rid);
+
+ auto key = getRedisLanesKey();
+
+ g_redisClient->hset(key, strLane, strPortId);
+ }
+
+ SWSS_LOG_NOTICE("added %zu lanes to redis lane map for port RID %s",
+ lanes.size(),
+ sai_serialize_object_id(port_rid).c_str());
+}
+
 void SaiSwitch::onPostPortCreate(
  _In_ sai_object_id_t port_rid,
  _In_ sai_object_id_t port_vid)
@@ -1282,7 +1365,8 @@ void SaiSwitch::onPostPortCreate(
 
  m_discovered_rids.insert(discovered.begin(), discovered.end());
 
- SWSS_LOG_NOTICE("putting ALL new discovered objects to redis");
+ SWSS_LOG_NOTICE("putting ALL new discovered objects to redis for port %s",
+ sai_serialize_object_id(port_vid).c_str());
 
  for (sai_object_id_t rid: discovered)
  {
@@ -1296,5 +1380,48 @@ void SaiSwitch::onPostPortCreate(
 
  redisSetDummyAsicStateForRealObjectId(rid);
  }
+
+ redisUpdatePortLaneMap(port_rid);
 }
 
+void SaiSwitch::onPostPortRemove(
+ _In_ sai_object_id_t port_rid)
+{
+ SWSS_LOG_ENTER();
+
+ int removed = 0;
+
+ // key - lane number, value - port RID
+ auto map = redisGetLaneMap();
+
+ for (auto& kv: map)
+ {
+ if (kv.second == port_rid)
+ {
+ auto key = getRedisLanesKey();
+
+ std::string strLane = sai_serialize_number(kv.first);
+
+ g_redisClient->hdel(key, strLane);
+
+ removed++;
+ }
+ }
+
+ SWSS_LOG_NOTICE("removed %u lanes from redis lane map for port RID %s",
+ removed,
+ sai_serialize_object_id(port_rid).c_str());
+
+ if (removed == 0)
+ {
+ SWSS_LOG_THROW("NO LANES found in redis lane map for given port RID %s",
+ sai_serialize_object_id(port_rid).c_str());
+ }
+}
+
+bool SaiSwitch::isWarmBoot() const
+{
+ SWSS_LOG_ENTER();
+
+ return m_warmBoot;
+}