From 6cb1b312847ac2e0a9825c97d85d935a1c58d4c2 Mon Sep 17 00:00:00 2001 From: Kamil Cudnik Date: Tue, 17 Sep 2019 15:37:06 +0200 Subject: [PATCH] Add support for port remove and port create (no warm boot) (#500) * Add support for port remove and port create * Fix aspell dictionary * Remove invalid comments * fix merge issues --- meta/sai_meta.cpp | 284 ++++++++++++++++++ syncd/syncd.cpp | 180 ++++++++++- syncd/syncd_saiswitch.cpp | 33 +++ syncd/syncd_saiswitch.h | 11 + tests/aspell.en.pws | 8 +- tests/brcm.pl | 17 ++ tests/brcm/remove_create_port.rec | 11 + tests/brcm/remove_port.rec | 26 ++ vslib/src/sai_vs_port.cpp | 428 ++++++++++++++++++++++++++- vslib/src/sai_vs_switch_BCM56850.cpp | 210 ++++++++----- 10 files changed, 1132 insertions(+), 76 deletions(-) create mode 100644 tests/brcm/remove_create_port.rec create mode 100644 tests/brcm/remove_port.rec diff --git a/meta/sai_meta.cpp b/meta/sai_meta.cpp index f52548365cae..e9a074b88d13 100644 --- a/meta/sai_meta.cpp +++ b/meta/sai_meta.cpp @@ -44,6 +44,17 @@ bool meta_unittests_enabled() return unittests_enabled; } +/** + * @brief Port map to related objects set. + * + * Key in map is port OID, and value is set of related objects ids + * like queues, ipgs and scheduler groups. + * + * This map will help to identify objects to be automatically removed + * when port will be removed. + */ +std::map> map_port_to_related_set; + sai_status_t meta_unittests_allow_readonly_set_once( _In_ sai_object_type_t object_type, _In_ int32_t attr_id) @@ -206,6 +217,13 @@ class SaiAttrWrapper return &m_attr; } + const sai_attr_metadata_t* getMeta() const + { + SWSS_LOG_ENTER(); + + return m_meta; + } + private: SaiAttrWrapper(const SaiAttrWrapper&); @@ -453,6 +471,8 @@ sai_status_t meta_init_db() ObjectAttrHash.clear(); AttributeKeys.clear(); + map_port_to_related_set.clear(); + return SAI_STATUS_SUCCESS; } @@ -1705,6 +1725,147 @@ sai_status_t meta_generic_validation_create( return SAI_STATUS_SUCCESS; } +bool meta_is_object_in_default_state( + _In_ sai_object_id_t oid) +{ + SWSS_LOG_ENTER(); + + if (oid == SAI_NULL_OBJECT_ID) + SWSS_LOG_THROW("not expected NULL object id"); + + if (!object_reference_exists(oid)) + { + SWSS_LOG_WARN("object %s refrence not exists, bug!", + sai_serialize_object_id(oid).c_str()); + return false; + } + + sai_object_meta_key_t meta_key; + + meta_key.objecttype = sai_object_type_query(oid); + meta_key.objectkey.key.object_id = oid; + + std::string key = sai_serialize_object_meta_key(meta_key); + + if (!object_exists(key)) + { + SWSS_LOG_WARN("object %s don't exists in local database, bug!", + sai_serialize_object_id(oid).c_str()); + return false; + } + + auto& attrs = ObjectAttrHash[key]; + + for (const auto& attr: attrs) + { + auto &md = *attr.second->getMeta(); + + auto *a = attr.second->getattr(); + + if (md.isreadonly) + continue; + + if (!md.isoidattribute) + continue; + + if (md.attrvaluetype == SAI_ATTR_VALUE_TYPE_OBJECT_ID) + { + if (a->value.oid != SAI_NULL_OBJECT_ID) + { + SWSS_LOG_ERROR("object %s has non default state on %s: %s, expected NULL", + sai_serialize_object_id(oid).c_str(), + md.attridname, + sai_serialize_object_id(a->value.oid).c_str()); + + return false; + } + } + else if (md.attrvaluetype == SAI_ATTR_VALUE_TYPE_OBJECT_LIST) + { + for (uint32_t i = 0; i < a->value.objlist.count; i++) + { + if (a->value.objlist.list[i] != SAI_NULL_OBJECT_ID) + { + SWSS_LOG_ERROR("object %s has non default state on %s[%u]: %s, expected NULL", + sai_serialize_object_id(oid).c_str(), + md.attridname, + i, + sai_serialize_object_id(a->value.objlist.list[i]).c_str()); + + return false; + } + } + } + else + { + // unable to check whether object is in default state, need fix + + SWSS_LOG_ERROR("unsupported oid attribute: %s, FIX ME!", md.attridname); + return false; + } + } + + return true; +} + +sai_status_t meta_port_remove_validation( + _In_ const sai_object_meta_key_t& meta_key) +{ + SWSS_LOG_ENTER(); + + sai_object_id_t port_id = meta_key.objectkey.key.object_id; + + auto it = map_port_to_related_set.find(port_id); + + if (it == map_port_to_related_set.end()) + { + // user didn't query any queues, ipgs or scheduler groups + // for this port, then we can just skip this + return SAI_STATUS_SUCCESS; + } + + if (object_reference_count(port_id) != 0) + { + SWSS_LOG_ERROR("port %s reference count is not zero, can't remove", + sai_serialize_object_id(port_id).c_str()); + + return SAI_STATUS_FAILURE; + } + + if (!meta_is_object_in_default_state(port_id)) + { + SWSS_LOG_ERROR("port %s is not in default state, can't remove", + sai_serialize_object_id(port_id).c_str()); + + return SAI_STATUS_FAILURE; + } + + for (auto oid: it->second) + { + if (object_reference_count(oid) != 0) + { + SWSS_LOG_ERROR("port %s related object %s reference count is not zero, can't remove", + sai_serialize_object_id(port_id).c_str(), + sai_serialize_object_id(oid).c_str()); + + return SAI_STATUS_FAILURE; + } + + if (!meta_is_object_in_default_state(oid)) + { + SWSS_LOG_ERROR("port related object %s is not in default state, can't remove", + sai_serialize_object_id(oid).c_str()); + + return SAI_STATUS_FAILURE; + } + } + + SWSS_LOG_NOTICE("all objects related to port %s are in default state, can be remove", + sai_serialize_object_id(port_id).c_str()); + + return SAI_STATUS_SUCCESS; +} + sai_status_t meta_generic_validation_remove( _In_ const sai_object_meta_key_t& meta_key) { @@ -1785,6 +1946,11 @@ sai_status_t meta_generic_validation_remove( return SAI_STATUS_INVALID_PARAMETER; } + if (meta_key.objecttype == SAI_OBJECT_TYPE_PORT) + { + return meta_port_remove_validation(meta_key); + } + // should be safe to remove return SAI_STATUS_SUCCESS; @@ -2816,6 +2982,51 @@ void meta_generic_validation_post_create( } } +void meta_generic_validation_post_remove( + _In_ const sai_object_meta_key_t& meta_key); + +void post_port_remove( + _In_ const sai_object_meta_key_t& meta_key) +{ + SWSS_LOG_ENTER(); + + sai_object_id_t port_id = meta_key.objectkey.key.object_id; + + auto it = map_port_to_related_set.find(port_id); + + if (it == map_port_to_related_set.end()) + { + // user didn't query any queues, ipgs or scheduler groups + // for this port, then we can just skip this + + return; + } + + for (auto oid: it->second) + { + // to remove existing objects, let's just call post remove for them + // and metadata will take the rest + + sai_object_meta_key_t meta; + + meta.objecttype = sai_object_type_query(oid); + meta.objectkey.key.object_id = oid; + + SWSS_LOG_INFO("attempt to remove port related object: %s: %s", + sai_serialize_object_type(meta.objecttype).c_str(), + sai_serialize_object_id(oid).c_str()); + + meta_generic_validation_post_remove(meta); + } + + // all related objects were removed, we need to clear current state + + it->second.clear(); + + SWSS_LOG_NOTICE("success executing post port remove actions: %s", + sai_serialize_object_id(port_id).c_str()); +} + void meta_generic_validation_post_remove( _In_ const sai_object_meta_key_t& meta_key) { @@ -2992,6 +3203,11 @@ void meta_generic_validation_post_remove( AttributeKeys.erase(ok); } + + if (meta_key.objecttype == SAI_OBJECT_TYPE_PORT) + { + post_port_remove(meta_key); + } } void meta_generic_validation_post_set( @@ -3337,6 +3553,69 @@ void meta_generic_validation_post_get_objlist( }\ } +void meta_add_port_to_related_map( + _In_ sai_object_id_t port_id, + _In_ const sai_object_list_t& list) +{ + SWSS_LOG_ENTER(); + + for (uint32_t i = 0; i < list.count; i++) + { + sai_object_id_t rel = list.list[i]; + + if (rel == SAI_NULL_OBJECT_ID) + SWSS_LOG_THROW("not expected NULL oid on the list"); + + map_port_to_related_set[port_id].insert(rel); + } +} + +void meta_post_port_get( + _In_ const sai_object_meta_key_t& meta_key, + _In_ sai_object_id_t switch_id, + _In_ const uint32_t attr_count, + _In_ const sai_attribute_t *attr_list) +{ + SWSS_LOG_ENTER(); + + /* + * User may or may not query one of below attributes to get some port + * objects, and those objects are special since when user decide to remove + * port, then those object will be removed automatically by vendor SAI, and + * this action needs to be reflected here too, so if user will remove port, + * those objects would need to be remove from local database too. + * + * TODO: There will be issue here, since we need to know which of those + * objects are user created, for example if user will create some extra + * queues with specific port, and then query queues list, those extra + * queues would need to be explicitly removed first by user, otherwise this + * logic here will also consider those user created queues as switch + * default, and it will remove them when port will be removed. Such action + * should be prevented. + */ + + const sai_object_id_t port_id = meta_key.objectkey.key.object_id; + + for (uint32_t idx = 0; idx < attr_count; ++idx) + { + const sai_attribute_t& attr = attr_list[idx]; + + auto& md = *sai_metadata_get_attr_metadata(meta_key.objecttype, attr.id); + + switch (md.attrid) + { + case SAI_PORT_ATTR_QOS_QUEUE_LIST: + case SAI_PORT_ATTR_QOS_SCHEDULER_GROUP_LIST: + case SAI_PORT_ATTR_INGRESS_PRIORITY_GROUP_LIST: + meta_add_port_to_related_map(port_id, attr.value.objlist); + break; + + default: + break; + } + } +} + void meta_generic_validation_post_get( _In_ const sai_object_meta_key_t& meta_key, _In_ sai_object_id_t switch_id, @@ -3573,6 +3852,11 @@ void meta_generic_validation_post_get( } } } + + if (meta_key.objecttype == SAI_OBJECT_TYPE_PORT) + { + meta_post_port_get(meta_key, switch_id, attr_count, attr_list); + } } // FDB ENTRY diff --git a/syncd/syncd.cpp b/syncd/syncd.cpp index ca3137389be0..db5fcf0fbfa1 100644 --- a/syncd/syncd.cpp +++ b/syncd/syncd.cpp @@ -20,6 +20,7 @@ extern "C" { #include #define DEF_SAI_WARM_BOOT_DATA_FILE "/var/warmboot/sai-warmboot.bin" +#define MAX_OBJLIST_LEN 128 /** * @brief Global mutex for thread synchronization @@ -1200,6 +1201,136 @@ bool is_set_attribute_workaround( return false; } +void get_port_related_objects( + _In_ sai_object_id_t port_rid, + _Out_ std::vector& related) +{ + SWSS_LOG_ENTER(); + + related.clear(); + + sai_object_meta_key_t meta_key; + + meta_key.objecttype = SAI_OBJECT_TYPE_PORT; + meta_key.objectkey.key.object_id = port_rid; + + sai_attr_id_t attrs[3] = { + SAI_PORT_ATTR_QOS_QUEUE_LIST, + SAI_PORT_ATTR_QOS_SCHEDULER_GROUP_LIST, + SAI_PORT_ATTR_INGRESS_PRIORITY_GROUP_LIST + }; + + auto info = sai_metadata_get_object_type_info(SAI_OBJECT_TYPE_PORT); + + for (size_t i = 0; i < sizeof(attrs)/sizeof(sai_attr_id_t); i++) + { + std::vector objlist; + + objlist.resize(MAX_OBJLIST_LEN); + + sai_attribute_t attr; + + attr.id = attrs[i]; + + attr.value.objlist.count = MAX_OBJLIST_LEN; + attr.value.objlist.list = objlist.data(); + + auto status = info->get(&meta_key, 1, &attr); + + if (status != SAI_STATUS_SUCCESS) + { + SWSS_LOG_THROW("failed to obtain related obejcts for port rid %s: %s, attr id: %d", + sai_serialize_object_id(port_rid).c_str(), + sai_serialize_status(status).c_str(), + attr.id); + } + + objlist.resize(attr.value.objlist.count); + + related.insert(related.end(), objlist.begin(), objlist.end()); + } + + SWSS_LOG_NOTICE("obtained %zu port %s related RIDs", + related.size(), + sai_serialize_object_id(port_rid).c_str()); +} + +void post_port_remove( + _In_ std::shared_ptr sw, + _In_ const std::vector& relatedRids) +{ + SWSS_LOG_ENTER(); + + /* + * Port was successfully removed from vendor SAI, + * we need to remove queues, ipgs and sg from: + * + * - redis ASIC DB + * - discovered existing objects in saiswitch class + * - local vid2rid map + * - redis RIDTOVID map + */ + + for (auto rid: relatedRids) + { + // remove from existing objects + + if (sw->isDiscoveredRid(rid)) + { + sw->removeExistingObjectReference(rid); + } + + // remove from RID2VID and VID2RID map in redis + + std::string str_rid = sai_serialize_object_id(rid); + + auto pvid = g_redisClient->hget(RIDTOVID, str_rid); + + if (pvid == nullptr) + { + SWSS_LOG_THROW("expected rid %s to be present in RIDTOVID", str_rid.c_str()); + } + + std::string str_vid = *pvid; + + sai_object_id_t vid; + sai_deserialize_object_id(str_vid, vid); + + g_redisClient->hdel(VIDTORID, str_vid); + g_redisClient->hdel(RIDTOVID, str_rid); + + // remove from local vid2rid and rid2vid map + + remove_rid_and_vid_from_local(rid, vid); + + // remove from ASIC DB + + sai_object_type_t ot = redis_sai_object_type_query(vid); + + std::string key = sai_serialize_object_type(ot) + ":" + str_vid; + + SWSS_LOG_INFO("removing ASIC DB key: %s", key.c_str()); + + g_redisClient->del(key); + } + + SWSS_LOG_NOTICE("post port remove actions succeeded"); + + // TODO lane map must be updated (for warm boot) +} + +void post_port_create( + _In_ std::shared_ptr sw, + _In_ sai_object_id_t port_rid, + _In_ sai_object_id_t port_vid) +{ + SWSS_LOG_ENTER(); + + sw->onPostPortCreate(port_rid, port_vid); + + // TODO lane map must be updated (for warm boot) +} + sai_status_t handle_generic( _In_ sai_object_type_t object_type, _In_ const std::string &str_object_id, @@ -1283,6 +1414,19 @@ sai_status_t handle_generic( } } + if (object_type == SAI_OBJECT_TYPE_PORT) + { + if (isInitViewMode()) + { + // reason for this is that if user will create port, + // new port is not actually created so when for example + // querying new queues for new created port, there are + // not there, since no actual port create was issued on + // the ASIC + SWSS_LOG_THROW("port object can't be created in init view mode"); + } + } + sai_status_t status = info->create(&meta_key, switch_id, attr_count, attr_list); if (status == SAI_STATUS_SUCCESS) @@ -1304,7 +1448,6 @@ sai_status_t handle_generic( */ { - g_redisClient->hset(VIDTORID, str_vid, str_rid); g_redisClient->hset(RIDTOVID, str_rid, str_vid); @@ -1319,6 +1462,13 @@ sai_status_t handle_generic( gSwitchId = real_object_id; SWSS_LOG_NOTICE("Initialize gSwitchId with ID = 0x%" PRIx64, gSwitchId); } + + if (object_type == SAI_OBJECT_TYPE_PORT) + { + sai_object_id_t switch_vid = redis_sai_switch_id_query(object_id); + + post_port_create(switches.at(switch_vid), real_object_id, object_id); + } } return status; @@ -1331,6 +1481,25 @@ sai_status_t handle_generic( meta_key.objectkey.key.object_id = rid; + std::vector related; + + if (object_type == SAI_OBJECT_TYPE_PORT) + { + if (isInitViewMode()) + { + // reason for this is that if user will remove port, + // and the create new one in init view mode, then this + // new port is not actually created so when for example + // querying new queues for new created port, there are + // not there, since no actual port create was issued on + // the ASIC + SWSS_LOG_THROW("port object can't be removed in init view mode"); + } + + // collect queues, ipgs, sg that belong to port + get_port_related_objects(rid, related); + } + sai_status_t status = info->remove(&meta_key); if (status == SAI_STATUS_SUCCESS) @@ -1350,6 +1519,10 @@ sai_status_t handle_generic( remove_rid_and_vid_from_local(rid, object_id); } + // TODO remove all related objects from REDIS DB and also + // from existing object references since at this point + // they are no longer valid + if (object_type == SAI_OBJECT_TYPE_SWITCH) { on_switch_remove(object_id); @@ -1384,6 +1557,11 @@ sai_status_t handle_generic( { switches.at(switch_vid)->removeExistingObjectReference(rid); } + + if (object_type == SAI_OBJECT_TYPE_PORT) + { + post_port_remove(switches.at(switch_vid), related); + } } } diff --git a/syncd/syncd_saiswitch.cpp b/syncd/syncd_saiswitch.cpp index 51259425d438..ad585b6508a3 100644 --- a/syncd/syncd_saiswitch.cpp +++ b/syncd/syncd_saiswitch.cpp @@ -1265,3 +1265,36 @@ SaiSwitch::SaiSwitch( saiGetMacAddress(m_default_mac_address); } + +void SaiSwitch::onPostPortCreate( + _In_ sai_object_id_t port_rid, + _In_ sai_object_id_t port_vid) +{ + SWSS_LOG_ENTER(); + + std::set discovered; + + saiDiscover(port_rid, discovered); + + SWSS_LOG_NOTICE("discovered %zu new objects (including port) after creating port VID: %s", + discovered.size(), + sai_serialize_object_id(port_vid).c_str()); + + m_discovered_rids.insert(discovered.begin(), discovered.end()); + + SWSS_LOG_NOTICE("putting ALL new discovered objects to redis"); + + for (sai_object_id_t rid: discovered) + { + /* + * We also could thing of optimizing this since it's one call to redis + * per rid, and probably this should be ATOMIC. + * + * NOTE: We are also storing read only object's here, like default + * virtual router, CPU, default trap group, etc. + */ + + redisSetDummyAsicStateForRealObjectId(rid); + } +} + diff --git a/syncd/syncd_saiswitch.h b/syncd/syncd_saiswitch.h index 506a79dc5c25..271fec8dbfb3 100644 --- a/syncd/syncd_saiswitch.h +++ b/syncd/syncd_saiswitch.h @@ -183,6 +183,17 @@ class SaiSwitch */ std::set getColdBootDiscoveredVids() const; + /** + * @brief On post port create. + * + * Performs actions needed after port creation. Will discover new + * queues, ipgs and scheduler groups that belong to new created port, + * and updated ASIC DB accordingly. + */ + void onPostPortCreate( + _In_ sai_object_id_t port_rid, + _In_ sai_object_id_t port_vid); + private: /* diff --git a/tests/aspell.en.pws b/tests/aspell.en.pws index 21b18b89ed76..c78e22cdaf47 100644 --- a/tests/aspell.en.pws +++ b/tests/aspell.en.pws @@ -25,6 +25,7 @@ attrs attrvalue bool Bool +booldata brcm broadcom candidateObjects @@ -89,6 +90,7 @@ INSEG ip IP IPG +ipgs ipmc IPv isobjectid @@ -107,9 +109,9 @@ Mellanox metadata mlnx mpls +MTU mutex mutexes -MTU namespace namespaces NHG @@ -175,9 +177,11 @@ sanitycheck sdk SDK selectable +setBuffered setMinPrio setPortCounterList setQueueCounterList +sg SGs sleeptime soAll @@ -244,5 +248,3 @@ VXLAN workaroung xoff xon -booldata -setBuffered diff --git a/tests/brcm.pl b/tests/brcm.pl index 2a9484e2c77b..9a7c5db194d6 100755 --- a/tests/brcm.pl +++ b/tests/brcm.pl @@ -464,6 +464,20 @@ sub test_acl_counter play "acl_counter.rec", 0; } +sub test_remove_port +{ + fresh_start; + + play "remove_port.rec"; +} + +sub test_remove_create_port +{ + fresh_start; + + play "remove_create_port.rec"; +} + sub test_acl_counter2 { fresh_start; @@ -477,6 +491,9 @@ sub test_acl_counter2 } # RUN TESTS + +test_remove_port; +test_remove_create_port; test_acl_counter2; test_acl_counter; test_tunnel_map; diff --git a/tests/brcm/remove_create_port.rec b/tests/brcm/remove_create_port.rec new file mode 100644 index 000000000000..2fddb6089513 --- /dev/null +++ b/tests/brcm/remove_create_port.rec @@ -0,0 +1,11 @@ +2017-06-14.01:55:46.543987|a|INIT_VIEW +2017-06-14.01:55:46.551164|A|SAI_STATUS_SUCCESS +2017-06-14.01:55:46.555975|c|SAI_OBJECT_TYPE_SWITCH:oid:0x21000000000000|SAI_SWITCH_ATTR_INIT_SWITCH=true +2017-06-14.01:56:05.520538|g|SAI_OBJECT_TYPE_SWITCH:oid:0x21000000000000|SAI_SWITCH_ATTR_PORT_LIST=32:oid:0x0,oid:0x0,oid:0x0,oid:0x0,oid:0x0,oid:0x0,oid:0x0,oid:0x0,oid:0x0,oid:0x0,oid:0x0,oid:0x0,oid:0x0,oid:0x0,oid:0x0,oid:0x0,oid:0x0,oid:0x0,oid:0x0,oid:0x0,oid:0x0,oid:0x0,oid:0x0,oid:0x0,oid:0x0,oid:0x0,oid:0x0,oid:0x0,oid:0x0,oid:0x0,oid:0x0,oid:0x0 +2017-06-14.01:56:05.525938|G|SAI_STATUS_SUCCESS|SAI_SWITCH_ATTR_PORT_LIST=32:oid:0x1000000000002,oid:0x1000000000003,oid:0x1000000000004,oid:0x1000000000005,oid:0x1000000000006,oid:0x1000000000007,oid:0x1000000000008,oid:0x1000000000009,oid:0x100000000000a,oid:0x100000000000b,oid:0x100000000000c,oid:0x100000000000d,oid:0x100000000000e,oid:0x100000000000f,oid:0x1000000000010,oid:0x1000000000011,oid:0x1000000000012,oid:0x1000000000013,oid:0x1000000000014,oid:0x1000000000015,oid:0x1000000000016,oid:0x1000000000017,oid:0x1000000000018,oid:0x1000000000019,oid:0x100000000001a,oid:0x100000000001b,oid:0x100000000001c,oid:0x100000000001d,oid:0x100000000001e,oid:0x100000000001f,oid:0x1000000000020,oid:0x1000000000021 +2017-06-14.01:56:11.535437|r|SAI_OBJECT_TYPE_PORT:oid:0x1000000000002 +2017-06-14.01:56:05.520538|g|SAI_OBJECT_TYPE_SWITCH:oid:0x21000000000000|SAI_SWITCH_ATTR_PORT_LIST=31:oid:0x0,oid:0x0,oid:0x0,oid:0x0,oid:0x0,oid:0x0,oid:0x0,oid:0x0,oid:0x0,oid:0x0,oid:0x0,oid:0x0,oid:0x0,oid:0x0,oid:0x0,oid:0x0,oid:0x0,oid:0x0,oid:0x0,oid:0x0,oid:0x0,oid:0x0,oid:0x0,oid:0x0,oid:0x0,oid:0x0,oid:0x0,oid:0x0,oid:0x0,oid:0x0,oid:0x0 +2017-06-14.01:56:05.525938|G|SAI_STATUS_SUCCESS|SAI_SWITCH_ATTR_PORT_LIST=31:oid:0x1000000000003,oid:0x1000000000004,oid:0x1000000000005,oid:0x1000000000006,oid:0x1000000000007,oid:0x1000000000008,oid:0x1000000000009,oid:0x100000000000a,oid:0x100000000000b,oid:0x100000000000c,oid:0x100000000000d,oid:0x100000000000e,oid:0x100000000000f,oid:0x1000000000010,oid:0x1000000000011,oid:0x1000000000012,oid:0x1000000000013,oid:0x1000000000014,oid:0x1000000000015,oid:0x1000000000016,oid:0x1000000000017,oid:0x1000000000018,oid:0x1000000000019,oid:0x100000000001a,oid:0x100000000001b,oid:0x100000000001c,oid:0x100000000001d,oid:0x100000000001e,oid:0x100000000001f,oid:0x1000000000020,oid:0x1000000000021 +2017-06-14.01:56:11.535437|r|SAI_OBJECT_TYPE_PORT:oid:0x1000000000003 +2017-06-14.01:56:06.151337|a|APPLY_VIEW +2017-06-14.01:56:06.156740|A|SAI_STATUS_SUCCESS diff --git a/tests/brcm/remove_port.rec b/tests/brcm/remove_port.rec new file mode 100644 index 000000000000..572a55b797e2 --- /dev/null +++ b/tests/brcm/remove_port.rec @@ -0,0 +1,26 @@ +2017-06-14.01:55:46.543987|a|INIT_VIEW +2017-06-14.01:55:46.551164|A|SAI_STATUS_SUCCESS +2017-06-14.01:55:46.555975|c|SAI_OBJECT_TYPE_SWITCH:oid:0x21000000000000|SAI_SWITCH_ATTR_INIT_SWITCH=true +2017-06-14.01:56:05.520538|g|SAI_OBJECT_TYPE_SWITCH:oid:0x21000000000000|SAI_SWITCH_ATTR_PORT_LIST=32:oid:0x0,oid:0x0,oid:0x0,oid:0x0,oid:0x0,oid:0x0,oid:0x0,oid:0x0,oid:0x0,oid:0x0,oid:0x0,oid:0x0,oid:0x0,oid:0x0,oid:0x0,oid:0x0,oid:0x0,oid:0x0,oid:0x0,oid:0x0,oid:0x0,oid:0x0,oid:0x0,oid:0x0,oid:0x0,oid:0x0,oid:0x0,oid:0x0,oid:0x0,oid:0x0,oid:0x0,oid:0x0 +2017-06-14.01:56:05.525938|G|SAI_STATUS_SUCCESS|SAI_SWITCH_ATTR_PORT_LIST=32:oid:0x1000000000002,oid:0x1000000000003,oid:0x1000000000004,oid:0x1000000000005,oid:0x1000000000006,oid:0x1000000000007,oid:0x1000000000008,oid:0x1000000000009,oid:0x100000000000a,oid:0x100000000000b,oid:0x100000000000c,oid:0x100000000000d,oid:0x100000000000e,oid:0x100000000000f,oid:0x1000000000010,oid:0x1000000000011,oid:0x1000000000012,oid:0x1000000000013,oid:0x1000000000014,oid:0x1000000000015,oid:0x1000000000016,oid:0x1000000000017,oid:0x1000000000018,oid:0x1000000000019,oid:0x100000000001a,oid:0x100000000001b,oid:0x100000000001c,oid:0x100000000001d,oid:0x100000000001e,oid:0x100000000001f,oid:0x1000000000020,oid:0x1000000000021 +2017-06-14.01:56:06.534160|g|SAI_OBJECT_TYPE_PORT:oid:0x1000000000002|SAI_PORT_ATTR_QOS_QUEUE_LIST=20:oid:0x0,oid:0x0,oid:0x0,oid:0x0,oid:0x0,oid:0x0,oid:0x0,oid:0x0,oid:0x0,oid:0x0,oid:0x0,oid:0x0,oid:0x0,oid:0x0,oid:0x0,oid:0x0,oid:0x0,oid:0x0,oid:0x0,oid:0x0 +2017-06-14.01:56:06.536525|G|SAI_STATUS_SUCCESS|SAI_PORT_ATTR_QOS_QUEUE_LIST=20:oid:0x1500000000006c,oid:0x1500000000006d,oid:0x1500000000006e,oid:0x1500000000006f,oid:0x15000000000070,oid:0x15000000000071,oid:0x15000000000072,oid:0x15000000000073,oid:0x15000000000074,oid:0x15000000000075,oid:0x15000000000078,oid:0x15000000000079,oid:0x1500000000007a,oid:0x1500000000007b,oid:0x1500000000007c,oid:0x1500000000007d,oid:0x1500000000007e,oid:0x1500000000007f,oid:0x15000000000080,oid:0x15000000000081 +2017-06-14.01:56:27.782501|g|SAI_OBJECT_TYPE_PORT:oid:0x1000000000002|SAI_PORT_ATTR_QOS_SCHEDULER_GROUP_LIST=13:oid:0x0,oid:0x0,oid:0x0,oid:0x0,oid:0x0,oid:0x0,oid:0x0,oid:0x0,oid:0x0,oid:0x0,oid:0x0,oid:0x0,oid:0x0 +2017-06-14.01:56:27.785556|G|SAI_STATUS_SUCCESS|SAI_PORT_ATTR_QOS_SCHEDULER_GROUP_LIST=13:oid:0x17000000000043,oid:0x17000000000076,oid:0x17000000000083,oid:0x17000000000082,oid:0x17000000000084,oid:0x17000000000085,oid:0x17000000000086,oid:0x17000000000087,oid:0x17000000000088,oid:0x17000000000089,oid:0x1700000000008a,oid:0x1700000000008b,oid:0x1700000000008c +2017-06-14.01:56:06.529091|g|SAI_OBJECT_TYPE_PORT:oid:0x1000000000002|SAI_PORT_ATTR_INGRESS_PRIORITY_GROUP_LIST=8:oid:0x0,oid:0x0,oid:0x0,oid:0x0,oid:0x0,oid:0x0,oid:0x0,oid:0x0 +2017-06-14.01:56:06.531439|G|SAI_STATUS_SUCCESS|SAI_PORT_ATTR_INGRESS_PRIORITY_GROUP_LIST=8:oid:0x1a000000000044,oid:0x1a000000000045,oid:0x1a000000000046,oid:0x1a000000000047,oid:0x1a000000000048,oid:0x1a000000000049,oid:0x1a00000000004a,oid:0x1a00000000004b +2017-06-14.01:56:34.445579|c|SAI_OBJECT_TYPE_BUFFER_POOL:oid:0x1800000000061f|SAI_BUFFER_POOL_ATTR_THRESHOLD_MODE=SAI_BUFFER_POOL_THRESHOLD_MODE_DYNAMIC|SAI_BUFFER_POOL_ATTR_SIZE=5491712|SAI_BUFFER_POOL_ATTR_TYPE=SAI_BUFFER_POOL_TYPE_INGRESS +2017-06-14.01:56:34.471663|c|SAI_OBJECT_TYPE_BUFFER_PROFILE:oid:0x19000000000624|SAI_BUFFER_PROFILE_ATTR_POOL_ID=oid:0x1800000000061f|SAI_BUFFER_PROFILE_ATTR_RESERVED_BUFFER_SIZE=1518|SAI_BUFFER_PROFILE_ATTR_THRESHOLD_MODE=SAI_BUFFER_PROFILE_THRESHOLD_MODE_DYNAMIC|SAI_BUFFER_PROFILE_ATTR_SHARED_DYNAMIC_TH=1 +2017-06-14.01:56:34.473539|s|SAI_OBJECT_TYPE_INGRESS_PRIORITY_GROUP:oid:0x1a000000000044|SAI_INGRESS_PRIORITY_GROUP_ATTR_BUFFER_PROFILE=oid:0x19000000000624 +2017-06-14.01:56:34.473539|s|SAI_OBJECT_TYPE_INGRESS_PRIORITY_GROUP:oid:0x1a000000000044|SAI_INGRESS_PRIORITY_GROUP_ATTR_BUFFER_PROFILE=oid:0x0 +2017-06-14.01:56:11.535437|r|SAI_OBJECT_TYPE_PORT:oid:0x1000000000002 +2017-06-14.01:56:11.535437|c|SAI_OBJECT_TYPE_PORT:oid:0x1000000000022|SAI_PORT_ATTR_SPEED=10000|SAI_PORT_ATTR_HW_LANE_LIST=1:202 +2017-06-14.01:56:11.535437|c|SAI_OBJECT_TYPE_PORT:oid:0x1000000000023|SAI_PORT_ATTR_SPEED=10000|SAI_PORT_ATTR_HW_LANE_LIST=1:203 +2017-06-14.01:56:11.535437|c|SAI_OBJECT_TYPE_PORT:oid:0x1000000000024|SAI_PORT_ATTR_SPEED=10000|SAI_PORT_ATTR_HW_LANE_LIST=1:204 +2017-06-14.01:56:11.535437|c|SAI_OBJECT_TYPE_PORT:oid:0x1000000000025|SAI_PORT_ATTR_SPEED=10000|SAI_PORT_ATTR_HW_LANE_LIST=1:205 +2017-06-14.01:56:11.535437|r|SAI_OBJECT_TYPE_PORT:oid:0x1000000000022 +2017-06-14.01:56:11.535437|r|SAI_OBJECT_TYPE_PORT:oid:0x1000000000023 +2017-06-14.01:56:11.535437|r|SAI_OBJECT_TYPE_PORT:oid:0x1000000000024 +2017-06-14.01:56:11.535437|r|SAI_OBJECT_TYPE_PORT:oid:0x1000000000025 +2017-06-14.01:56:06.151337|a|APPLY_VIEW +2017-06-14.01:56:06.156740|A|SAI_STATUS_SUCCESS diff --git a/vslib/src/sai_vs_port.cpp b/vslib/src/sai_vs_port.cpp index 3a31a2294f41..c41861bd3124 100644 --- a/vslib/src/sai_vs_port.cpp +++ b/vslib/src/sai_vs_port.cpp @@ -4,6 +4,10 @@ #include "sai_vs_switch_BCM56850.h" #include "sai_vs_switch_MLNX2700.h" +#include + +#define MAX_OBJLIST_LEN 128 + sai_status_t vs_clear_port_all_stats( _In_ sai_object_id_t port_id) { @@ -24,8 +28,15 @@ sai_status_t vs_create_port( SWSS_LOG_ENTER(); /* create port */ - CHECK_STATUS(meta_sai_create_oid((sai_object_type_t)SAI_OBJECT_TYPE_PORT, - port_id,switch_id,attr_count,attr_list,&vs_generic_create)); + CHECK_STATUS(meta_sai_create_oid( + (sai_object_type_t)SAI_OBJECT_TYPE_PORT, + port_id, + switch_id, + attr_count, + attr_list, + &vs_generic_create)); + + // TODO needs to be revisited if (g_vs_switch_type == SAI_VS_SWITCH_TYPE_BCM56850) { @@ -35,6 +46,418 @@ sai_status_t vs_create_port( { vs_create_port_MLNX2700(*port_id, switch_id); } + else + { + SWSS_LOG_ERROR("unknown switch type: %d", g_vs_switch_type); + return SAI_STATUS_FAILURE; + } + + return SAI_STATUS_SUCCESS; +} + +static bool vs_get_object_list( + _In_ sai_object_id_t object_id, + _In_ sai_attr_id_t attr_id, + _Out_ std::vector& objlist) +{ + SWSS_LOG_ENTER(); + + objlist.clear(); + + sai_object_type_t object_type = sai_object_type_query(object_id); + + auto* meta = sai_metadata_get_attr_metadata(object_type, attr_id); + + if (meta == nullptr) + { + SWSS_LOG_THROW("failed to get metadata for OID %s and attrid: %d", + sai_serialize_object_id(object_id).c_str(), + attr_id); + } + + if (meta->attrvaluetype != SAI_ATTR_VALUE_TYPE_OBJECT_LIST) + { + SWSS_LOG_THROW("attr %s is not objlist attribute", meta->attridname); + } + + sai_status_t status; + + sai_attribute_t attr; + + objlist.resize(MAX_OBJLIST_LEN); + + attr.id = attr_id; + + attr.value.objlist.count = MAX_OBJLIST_LEN; + attr.value.objlist.list = objlist.data(); + + status = vs_generic_get(object_type, object_id, 1, &attr); + + if (status != SAI_STATUS_SUCCESS) + { + SWSS_LOG_ERROR("failed to obtain %s for %s queues: %s", + meta->attridname, + sai_serialize_object_id(object_id).c_str(), + sai_serialize_status(status).c_str()); + + objlist.clear(); + return false; + } + + objlist.resize(attr.value.objlist.count); + + SWSS_LOG_NOTICE("%s returned %zu objects for %s", + meta->attridname, + objlist.size(), + sai_serialize_object_id(object_id).c_str()); + + return true; +} + +static bool vs_get_port_queues( + _In_ sai_object_id_t port_id, + _Out_ std::vector& queues) +{ + SWSS_LOG_ENTER(); + + return vs_get_object_list(port_id, SAI_PORT_ATTR_QOS_QUEUE_LIST, queues); +} + +static bool vs_get_port_ipgs( + _In_ sai_object_id_t port_id, + _Out_ std::vector& ipgs) +{ + SWSS_LOG_ENTER(); + + return vs_get_object_list(port_id, SAI_PORT_ATTR_INGRESS_PRIORITY_GROUP_LIST, ipgs); +} + +static bool vs_get_port_sg( + _In_ sai_object_id_t port_id, + _Out_ std::vector& sg) +{ + SWSS_LOG_ENTER(); + + // scheduler groups are organized in tree, but + // SAI_PORT_ATTR_INGRESS_PRIORITY_GROUP_LIST should return all scheduler groups in that tree + + return vs_get_object_list(port_id, SAI_PORT_ATTR_QOS_SCHEDULER_GROUP_LIST, sg); +} + +bool vs_check_object_default_state( + _In_ sai_object_id_t object_id) +{ + SWSS_LOG_ENTER(); + + sai_object_type_t object_type = sai_object_type_query(object_id); + + if (object_type == SAI_OBJECT_TYPE_NULL) + { + SWSS_LOG_ERROR("failed to get object type for oid: %s", + sai_serialize_object_id(object_id).c_str()); + + return false; + } + + auto* oti = sai_metadata_get_object_type_info(object_type); + + if (oti == nullptr) + { + SWSS_LOG_THROW("failed to get object type info for object type: %s", + sai_serialize_object_type(object_type).c_str()); + } + + // iterate over all attributes + + for (size_t i = 0; i < oti->attrmetadatalength; i++) + { + auto* meta = oti->attrmetadata[i]; + + // skip readonly, mandatory on create and non oid attributes + + if (meta->isreadonly) + continue; + + if (!meta->isoidattribute) + continue; + + // those attributes must be skipped since those dependencies will be automatically broken + if (meta->objecttype == SAI_OBJECT_TYPE_SCHEDULER_GROUP && meta->attrid == SAI_SCHEDULER_GROUP_ATTR_PORT_ID) + continue; + + if (meta->objecttype == SAI_OBJECT_TYPE_SCHEDULER_GROUP && meta->attrid == SAI_SCHEDULER_GROUP_ATTR_PARENT_NODE) + continue; + + if (meta->objecttype == SAI_OBJECT_TYPE_QUEUE && meta->attrid == SAI_QUEUE_ATTR_PORT) + continue; + + if (meta->objecttype == SAI_OBJECT_TYPE_QUEUE && meta->attrid == SAI_QUEUE_ATTR_PARENT_SCHEDULER_NODE) + continue; + + if (meta->objecttype == SAI_OBJECT_TYPE_INGRESS_PRIORITY_GROUP && meta->attrid == SAI_INGRESS_PRIORITY_GROUP_ATTR_PORT) + continue; + + // here we have only oid/object list attrs and we expect each of this + // attribute will be in default state which for oid is usually NULL, + // and for object list is empty + + sai_attribute_t attr; + + attr.id = meta->attrid; + + sai_status_t status; + + std::vector objlist; + + if (meta->attrvaluetype == SAI_ATTR_VALUE_TYPE_OBJECT_ID) + { + // ok + } + else if (meta->attrvaluetype == SAI_ATTR_VALUE_TYPE_OBJECT_LIST) + { + objlist.resize(MAX_OBJLIST_LEN); + + attr.value.objlist.count = MAX_OBJLIST_LEN; + attr.value.objlist.list = objlist.data(); + } + else + { + // unable to check whether object is in default state, need fix + + SWSS_LOG_ERROR("unsupported oid attribute: %s, FIX ME!", meta->attridname); + return false; + } + + status = vs_generic_get(object_type, object_id, 1, &attr); + + switch (status) + { + case SAI_STATUS_NOT_IMPLEMENTED: + case SAI_STATUS_NOT_SUPPORTED: + continue; + + case SAI_STATUS_SUCCESS: + break; + + default: + + SWSS_LOG_ERROR("unexpected status %s on %s obj %s", + sai_serialize_status(status).c_str(), + meta->attridname, + sai_serialize_object_id(object_id).c_str()); + return false; + + } + + + if (meta->attrvaluetype == SAI_ATTR_VALUE_TYPE_OBJECT_ID) + { + if (attr.value.oid != SAI_NULL_OBJECT_ID) + { + SWSS_LOG_ERROR("expected null object id on %s on %s, but got: %s", + meta->attridname, + sai_serialize_object_id(object_id).c_str(), + sai_serialize_object_id(attr.value.oid).c_str()); + + return false; + } + + } + else if (meta->attrvaluetype == SAI_ATTR_VALUE_TYPE_OBJECT_LIST) + { + if (objlist.size()) + { + SWSS_LOG_ERROR("expected empty list on %s on %s, contents:", + meta->attridname, + sai_serialize_object_id(object_id).c_str()); + + for (auto oid: objlist) + { + SWSS_LOG_ERROR(" - oid: %s", sai_serialize_object_id(oid).c_str()); + } + + return false; + } + } + else + { + // unable to check whether object is in default state, need fix + + SWSS_LOG_ERROR("unsupported oid attribute: %s, FIX ME!", meta->attridname); + return false; + } + } + + // TODO later there can be issue when we for example add extra queues to + // the port those new queues should be removed by user first before + // removing port, and currently we don't have a way to differentiate those + + // object is in default state + return true; +} + +bool vs_check_object_list_default_state( + _Out_ const std::vector& objlist) +{ + SWSS_LOG_ENTER(); + + return std::all_of(objlist.begin(), objlist.end(), + [](sai_object_id_t oid) { return vs_check_object_default_state(oid); }); +} + +sai_status_t vs_check_port_dependencies( + _In_ sai_object_id_t port_id, + _Out_ std::vector& dep) +{ + SWSS_LOG_ENTER(); + + // check if port exists's + + sai_object_id_t switch_id = sai_switch_id_query(port_id); + + if (switch_id == SAI_NULL_OBJECT_ID) + { + SWSS_LOG_ERROR("failed to obtain switch_id from object %s", + sai_serialize_object_id(port_id).c_str()); + + return SAI_STATUS_FAILURE; + } + + sai_object_type_t ot = sai_object_type_query(port_id); + + if (ot != SAI_OBJECT_TYPE_PORT) + { + SWSS_LOG_ERROR("expected object type PORT but object %s has type %s", + sai_serialize_object_id(port_id).c_str(), + sai_serialize_object_type(ot).c_str()); + + return SAI_STATUS_FAILURE; + } + + std::string str_port_id = sai_serialize_object_id(port_id); + + auto &objectHash = g_switch_state_map.at(switch_id)->objectHash.at(ot); + + auto it = objectHash.find(str_port_id); + + if (it == objectHash.end()) + { + SWSS_LOG_ERROR("port not found %s:%s", + sai_serialize_object_type(ot).c_str(), + str_port_id.c_str()); + + return SAI_STATUS_ITEM_NOT_FOUND; + } + + // port was found + SWSS_LOG_NOTICE("port %s found, for removal", + sai_serialize_object_id(port_id).c_str()); + + // obtain objects to examine + + std::vector queues; + std::vector ipgs; + std::vector sg; + + bool result = true; + + result &= vs_get_port_queues(port_id, queues); + result &= vs_get_port_ipgs(port_id, ipgs); + result &= vs_get_port_sg(port_id, sg); + + if (!result) + { + SWSS_LOG_ERROR("failed to obtain required objects on port %s", + sai_serialize_object_id(port_id).c_str()); + + return SAI_STATUS_FAILURE; + } + + // check if all objects are in default state + + result &= vs_check_object_default_state(port_id); + result &= vs_check_object_list_default_state(queues); + result &= vs_check_object_list_default_state(ipgs); + result &= vs_check_object_list_default_state(sg); + + if (!result) + { + SWSS_LOG_ERROR("one of objects is not in default state, can't remove port %s", + sai_serialize_object_id(port_id).c_str()); + + return SAI_STATUS_FAILURE; + } + + SWSS_LOG_NOTICE("all depending objects on port %s are in default state", + sai_serialize_object_id(port_id).c_str()); + + dep.insert(dep.end(), queues.begin(), queues.end()); + dep.insert(dep.end(), ipgs.begin(), ipgs.end()); + dep.insert(dep.end(), sg.begin(), sg.end()); + + // TODO there may be issues with bridge ports created on that port if they + // are not removed before port remove, this needs to be addressed for warm + // boot support + + return SAI_STATUS_SUCCESS; +} + +sai_status_t vs_remove_port( + _In_ sai_object_id_t port_id) +{ + MUTEX(); + SWSS_LOG_ENTER(); + + std::vector dep; + + sai_status_t status = vs_check_port_dependencies(port_id, dep); + + if (status != SAI_STATUS_SUCCESS) + { + return status; + } + + // NOTE: we should check references on depending objects to see if it's safe + // to remove every object but we count on metadata references count to do + // that for us + + status = meta_sai_remove_oid( + (sai_object_type_t)SAI_OBJECT_TYPE_PORT, + port_id, + &vs_generic_remove); + + if (status != SAI_STATUS_SUCCESS) + { + SWSS_LOG_ERROR("failed to remove port: %s", + sai_serialize_object_id(port_id).c_str()); + + return status; + } + + SWSS_LOG_NOTICE("port %s was successfully removed, removing depending objects now", + sai_serialize_object_id(port_id).c_str()); + + for (auto oid: dep) + { + // meta_sai_remove_oid automatically removed related oids internally + // so we just need to execute remove for virtual switch db + + status = vs_generic_remove( + sai_object_type_query(oid), + oid); + + if (status != SAI_STATUS_SUCCESS) + { + // we can't continue, there is a bug somewhere if we can't remove + // port related objects: queues, ipgs, sg + + SWSS_LOG_THROW("FATAL: failed to removed port related oid: %s: %s, bug!", + sai_serialize_object_type(sai_object_type_query(oid)).c_str(), + sai_serialize_object_id(oid).c_str()); + } + } + + SWSS_LOG_NOTICE("successfully removed all %zu port related objects", dep.size()); return SAI_STATUS_SUCCESS; } @@ -170,7 +593,6 @@ sai_status_t vs_set_port_attribute( return meta_sai_set_oid((sai_object_type_t)SAI_OBJECT_TYPE_PORT, port_id, attr, &vs_generic_set); } -VS_REMOVE(PORT,port); VS_GET(PORT,port); VS_GENERIC_QUAD(PORT_POOL,port_pool); VS_GENERIC_STATS(PORT,port); diff --git a/vslib/src/sai_vs_switch_BCM56850.cpp b/vslib/src/sai_vs_switch_BCM56850.cpp index 606a5c897f98..922a9936089a 100644 --- a/vslib/src/sai_vs_switch_BCM56850.cpp +++ b/vslib/src/sai_vs_switch_BCM56850.cpp @@ -1,6 +1,7 @@ #include "sai_vs.h" #include "sai_vs_state.h" #include +#include // TODO extra work may be needed on GET api if N on list will be > then actual @@ -216,33 +217,6 @@ static sai_status_t create_ports() return SAI_STATUS_SUCCESS; } -static sai_status_t create_port_list() -{ - SWSS_LOG_ENTER(); - - SWSS_LOG_INFO("create port list"); - - // TODO this is static, when we start to "create/remove" ports - // we need to update this list since it's dynamic - - sai_attribute_t attr; - - sai_object_id_t switch_object_id = ss->getSwitchId(); - - uint32_t port_count = (uint32_t)port_list.size(); - - attr.id = SAI_SWITCH_ATTR_PORT_LIST; - attr.value.objlist.count = port_count; - attr.value.objlist.list = port_list.data(); - - CHECK_STATUS(vs_generic_set(SAI_OBJECT_TYPE_SWITCH, switch_object_id, &attr)); - - attr.id = SAI_SWITCH_ATTR_PORT_NUMBER; - attr.value.u32 = port_count; - - return vs_generic_set(SAI_OBJECT_TYPE_SWITCH, switch_object_id, &attr); -} - static sai_status_t create_bridge_ports() { SWSS_LOG_ENTER(); @@ -479,7 +453,7 @@ static sai_status_t create_qos_queues() for (auto &port_id : port_list) { - create_qos_queues_per_port(switch_object_id, port_id); + CHECK_STATUS(create_qos_queues_per_port(switch_object_id, port_id)); } return SAI_STATUS_SUCCESS; @@ -530,10 +504,9 @@ static sai_status_t create_ingress_priority_groups() sai_object_id_t switch_object_id = ss->getSwitchId(); - // for (auto &port_id : port_list) { - create_ingress_priority_groups_per_port(switch_object_id, port_id); + CHECK_STATUS(create_ingress_priority_groups_per_port(switch_object_id, port_id)); } return SAI_STATUS_SUCCESS; @@ -577,6 +550,11 @@ static sai_status_t create_scheduler_group_tree( sai_attribute_t attr; + attr.id = SAI_SCHEDULER_GROUP_ATTR_PORT_ID; + attr.value.oid = port_id; + + CHECK_STATUS(vs_generic_set(SAI_OBJECT_TYPE_SCHEDULER_GROUP, sg_0, &attr)); + attr.id = SAI_SCHEDULER_GROUP_ATTR_CHILD_COUNT; attr.value.u32 = 2; @@ -603,6 +581,11 @@ static sai_status_t create_scheduler_group_tree( sai_attribute_t attr; + attr.id = SAI_SCHEDULER_GROUP_ATTR_PORT_ID; + attr.value.oid = port_id; + + CHECK_STATUS(vs_generic_set(SAI_OBJECT_TYPE_SCHEDULER_GROUP, sg_1, &attr)); + attr.id = SAI_SCHEDULER_GROUP_ATTR_CHILD_COUNT; attr.value.u32 = 8; @@ -657,6 +640,11 @@ static sai_status_t create_scheduler_group_tree( sai_attribute_t attr; + attr.id = SAI_SCHEDULER_GROUP_ATTR_PORT_ID; + attr.value.oid = port_id; + + CHECK_STATUS(vs_generic_set(SAI_OBJECT_TYPE_SCHEDULER_GROUP, sg_2, &attr)); + attr.id = SAI_SCHEDULER_GROUP_ATTR_CHILD_COUNT; attr.value.u32 = 2; @@ -700,12 +688,12 @@ static sai_status_t create_scheduler_group_tree( return SAI_STATUS_SUCCESS; } -static sai_status_t create_scheduler_groups() +static sai_status_t create_scheduler_groups_per_port( + _In_ sai_object_id_t switch_id, + _In_ sai_object_id_t port_id) { SWSS_LOG_ENTER(); - SWSS_LOG_INFO("create scheduler groups"); - uint32_t port_sgs_count = 13; // brcm default // NOTE: this is only static data, to keep track of this @@ -714,40 +702,57 @@ static sai_status_t create_scheduler_groups() // solution when we will start using different "profiles" // currently this is good enough - for (const auto &port_id : port_list) - { - sai_attribute_t attr; + sai_attribute_t attr; - attr.id = SAI_PORT_ATTR_QOS_NUMBER_OF_SCHEDULER_GROUPS; - attr.value.u32 = port_sgs_count; + attr.id = SAI_PORT_ATTR_QOS_NUMBER_OF_SCHEDULER_GROUPS; + attr.value.u32 = port_sgs_count; - CHECK_STATUS(vs_generic_set(SAI_OBJECT_TYPE_PORT, port_id, &attr)); + CHECK_STATUS(vs_generic_set(SAI_OBJECT_TYPE_PORT, port_id, &attr)); - // scheduler groups per port + // scheduler groups per port - std::vector sgs; + std::vector sgs; - for (uint32_t i = 0; i < port_sgs_count; ++i) - { - sai_object_id_t sg_id; + for (uint32_t i = 0; i < port_sgs_count; ++i) + { + sai_object_id_t sg_id; - CHECK_STATUS(vs_generic_create(SAI_OBJECT_TYPE_SCHEDULER_GROUP, &sg_id, ss->getSwitchId(), 0, NULL)); + CHECK_STATUS(vs_generic_create(SAI_OBJECT_TYPE_SCHEDULER_GROUP, &sg_id, ss->getSwitchId(), 0, NULL)); - sgs.push_back(sg_id); - } + sgs.push_back(sg_id); + } - attr.id = SAI_PORT_ATTR_QOS_SCHEDULER_GROUP_LIST; - attr.value.objlist.count = port_sgs_count; - attr.value.objlist.list = sgs.data(); + attr.id = SAI_PORT_ATTR_QOS_SCHEDULER_GROUP_LIST; + attr.value.objlist.count = port_sgs_count; + attr.value.objlist.list = sgs.data(); - CHECK_STATUS(vs_generic_set(SAI_OBJECT_TYPE_PORT, port_id, &attr)); + CHECK_STATUS(vs_generic_set(SAI_OBJECT_TYPE_PORT, port_id, &attr)); - CHECK_STATUS(create_scheduler_group_tree(sgs, port_id)); - } + CHECK_STATUS(create_scheduler_group_tree(sgs, port_id)); + // TODO // SAI_SCHEDULER_GROUP_ATTR_CHILD_COUNT // sched_groups + count // scheduler group are organized in tree and on the bottom there are queues // order matters in returning api + + return SAI_STATUS_SUCCESS; +} + +static sai_status_t create_scheduler_groups() +{ + SWSS_LOG_ENTER(); + + SWSS_LOG_INFO("create scheduler groups"); + + // TODO scheduler groups size may change when we will modify sg or ports + + sai_object_id_t switch_object_id = ss->getSwitchId(); + + for (auto &port_id : port_list) + { + CHECK_STATUS(create_scheduler_groups_per_port(switch_object_id, port_id)); + } + return SAI_STATUS_SUCCESS; } @@ -876,7 +881,6 @@ static sai_status_t initialize_default_objects() CHECK_STATUS(create_default_1q_bridge()); CHECK_STATUS(create_default_trap_group()); CHECK_STATUS(create_ports()); - CHECK_STATUS(create_port_list()); CHECK_STATUS(create_bridge_ports()); CHECK_STATUS(create_vlan_members()); CHECK_STATUS(create_acl_entry_min_prio()); @@ -1218,6 +1222,76 @@ static sai_status_t refresh_scheduler_groups( return SAI_STATUS_SUCCESS; } +static sai_status_t refresh_port_list( + _In_ const sai_attr_metadata_t *meta, + _In_ sai_object_id_t switch_id) +{ + SWSS_LOG_ENTER(); + + // since now port can be added or removed, we need to update port list + // dynamically + + sai_attribute_t attr; + + attr.id = SAI_SWITCH_ATTR_CPU_PORT; + + CHECK_STATUS(vs_generic_get(SAI_OBJECT_TYPE_SWITCH, switch_id, 1, &attr)); + + const sai_object_id_t cpu_port_id = attr.value.oid; + + port_list.clear(); + + // iterate via ASIC state to find all the ports + + auto &objectHash = g_switch_state_map.at(switch_id)->objectHash.at(SAI_OBJECT_TYPE_PORT); + + for (const auto& it: objectHash) + { + sai_object_id_t port_id; + sai_deserialize_object_id(it.first, port_id); + + // don't put CPU port id on the list + + if (port_id == cpu_port_id) + continue; + + port_list.push_back(port_id); + } + + /* + * TODO: + * + * Currently we don't know what's happen on brcm SAI implementation when + * port is removed and then added, will new port could get the same vendor + * OID or always different, and what is order of those new oids on the + * PORT_LIST attribute. + * + * This needs to be investigated, and to reflect exact behaviour here. + * Currently we just sort all the port oids. + */ + + std::sort(port_list.begin(), port_list.end()); + + sai_object_id_t switch_object_id = ss->getSwitchId(); + + uint32_t port_count = (uint32_t)port_list.size(); + + attr.id = SAI_SWITCH_ATTR_PORT_LIST; + attr.value.objlist.count = port_count; + attr.value.objlist.list = port_list.data(); + + CHECK_STATUS(vs_generic_set(SAI_OBJECT_TYPE_SWITCH, switch_object_id, &attr)); + + attr.id = SAI_SWITCH_ATTR_PORT_NUMBER; + attr.value.u32 = port_count; + + CHECK_STATUS(vs_generic_set(SAI_OBJECT_TYPE_SWITCH, switch_object_id, &attr)); + + SWSS_LOG_NOTICE("refreshed port list, current port number: %zu, not counting cpu port", port_list.size()); + + return SAI_STATUS_SUCCESS; +} + /* * NOTE For recalculation we can add flag on create/remove specific object type * so we can deduce whether actually need to perform recalculation, as @@ -1235,9 +1309,6 @@ sai_status_t refresh_read_only_BCM56850( { switch (meta->attrid) { - case SAI_SWITCH_ATTR_PORT_NUMBER: - return SAI_STATUS_SUCCESS; - case SAI_SWITCH_ATTR_CPU_PORT: case SAI_SWITCH_ATTR_DEFAULT_VIRTUAL_ROUTER_ID: case SAI_SWITCH_ATTR_DEFAULT_TRAP_GROUP: @@ -1258,13 +1329,9 @@ sai_status_t refresh_read_only_BCM56850( case SAI_SWITCH_ATTR_NUMBER_OF_ECMP_GROUPS: return SAI_STATUS_SUCCESS; - /* - * We don't need to recalculate port list, since now we assume - * that port list will not change. - */ - + case SAI_SWITCH_ATTR_PORT_NUMBER: case SAI_SWITCH_ATTR_PORT_LIST: - return SAI_STATUS_SUCCESS; + return refresh_port_list(meta, switch_id); case SAI_SWITCH_ATTR_QOS_MAX_NUMBER_OF_CHILDS_PER_SCHEDULER_GROUP: return SAI_STATUS_SUCCESS; @@ -1334,6 +1401,8 @@ sai_status_t vs_create_port_BCM56850( { SWSS_LOG_ENTER(); + // this method is post create action on generic create object + sai_attribute_t attr; attr.id = SAI_PORT_ATTR_ADMIN_STATE; @@ -1341,13 +1410,16 @@ sai_status_t vs_create_port_BCM56850( CHECK_STATUS(vs_generic_set(SAI_OBJECT_TYPE_PORT, port_id, &attr)); - /* create priority groups */ - create_ingress_priority_groups_per_port(switch_id, port_id); + // attributes are not required since they will be set outside this function - /* create qos queues */ - create_qos_queues_per_port(switch_id, port_id); + CHECK_STATUS(create_ingress_priority_groups_per_port(switch_id, port_id)); - return SAI_STATUS_SUCCESS; -} + CHECK_STATUS(create_qos_queues_per_port(switch_id, port_id)); + + CHECK_STATUS(create_scheduler_groups_per_port(switch_id, port_id)); + // TODO should bridge ports should also be created when new port is created? + // this needs to be checked on real ASIC and updated here + return SAI_STATUS_SUCCESS; +}