From b5f0ab220934a8044c3eb09d3b4ab6ad719da2f4 Mon Sep 17 00:00:00 2001 From: minkang_tsai Date: Fri, 13 Dec 2019 14:05:08 +0800 Subject: [PATCH] [routeorch/vrforch] set the default route for non default VRF What I did Add IPv4/IPv6 default route after creating non default VRF. Why I did it There is a restriction on broadcom platform, that is, first route in a VRF has to be a default route. So if there is no default route in non defaulte route and configure ip for L3 interface which be bound to non default VRF, sycnd will get error and stop swss. Test procedure: config vrf add Vrf-test config interface vrf bind Ethernet0 Vrf-test config interface ip add Ethernet0 192.168.5.5/24 Error message: Nov 10 14:14:17.492668 sonic NOTICE swss#orchagent: :- addRouterIntfs: Create router interface Ethernet0 MTU 9100 Nov 10 14:14:18.158264 sonic NOTICE syncd#syncd: :- setRifCounterList: Router interface oid:0x32000600000001 does not have supported counters Nov 10 14:14:22.286894 sonic NOTICE swss#orchagent: :- addIp2MeRoute: Create IP2me route ip:192.168.5.5 Nov 10 14:14:22.288595 sonic ERR syncd#syncd: _brcm_sai_l3_route_config:1564 L3 route add failed with error Invalid parameter (0xfffffffc). Nov 10 14:14:22.288595 sonic ERR syncd#syncd: brcm_sai_create_route_entry:332 L3 route add failed with error -5. Nov 10 14:14:22.288595 sonic ERR syncd#syncd: :- processEvent: attr: SAI_ROUTE_ENTRY_ATTR_PACKET_ACTION: SAI_PACKET_ACTION_FORWARD Nov 10 14:14:22.288595 sonic ERR syncd#syncd: :- processEvent: attr: SAI_ROUTE_ENTRY_ATTR_NEXT_HOP_ID: oid:0x1000000000001 Nov 10 14:14:22.288595 sonic ERR syncd#syncd: :- processEvent: failed to execute api: create, key: SAI_OBJECT_TYPE_ROUTE_ENTRY:{"dest":"192.168.5.5/32","switch_id":"oid:0x21000000000000","vr":"oid:0x30000000005ca"}, status: SAI_STATUS_INVALID_PARAMETER Nov 10 14:14:22.288595 sonic ERR syncd#syncd: :- syncd_main: Runtime error: :- processEvent: failed to execute api: create, key: SAI_OBJECT_TYPE_ROUTE_ENTRY:{"dest":"192.168.5.5/32","switch_id":"oid:0x21000000000000","vr":"oid:0x30000000005ca"}, status: SAI_STATUS_INVALID_PARAMETER Nov 10 14:14:22.288595 sonic NOTICE syncd#syncd: :- notify_OA_about_syncd_exception: sending switch_shutdown_request notification to OA Nov 10 14:14:22.288595 sonic INFO syncd#supervisord: syncd 0:soc_th_alpm_insert: VRF 2: First route in a VRF has to be a default route in this mode#015 Nov 10 14:14:22.288595 sonic NOTICE syncd#syncd: :- notify_OA_about_syncd_exception: notification send successfull Nov 10 14:14:22.288892 sonic NOTICE swss#orchagent: :- handle_switch_shutdown_request: switch shutdown request Nov 10 14:14:22.291922 sonic INFO swss#supervisord: orchagent terminate called after throwing an instance of 'std::invalid_argument' Nov 10 14:14:22.291922 sonic INFO swss#supervisord: orchagent what(): parse error - unexpected end of input Nov 10 14:14:23.501280 sonic INFO swss#supervisor-proc-exit-listener: Process orchagent exited unxepectedly. Terminating supervisor... Nov 10 14:14:27.494174 sonic INFO swss#supervisord 2016-11-10 14:14:22,498 INFO exited: orchagent (terminated by SIGABRT (core dumped); not expected) Nov 10 14:14:27.494174 sonic INFO swss#supervisord 2016-11-10 14:14:23,502 WARN received SIGTERM indicating exit request Nov 10 14:14:27.494174 sonic INFO swss#supervisord 2016-11-10 14:14:23,502 INFO waiting for vrfmgrd, neighsyncd, intfmgrd, portmgrd, vxlanmgrd, buffermgrd, portsyncd, nbrmgrd, vlanmgrd, supervisor-proc-exit-listener, rsyslogd to die Nov 10 14:14:27.494174 sonic INFO swss#supervisord 2016-11-10 14:14:23,503 INFO stopped: vxlanmgrd (terminated by SIGTERM) Nov 10 14:14:27.494174 sonic INFO swss#supervisord 2016-11-10 14:14:23,505 INFO stopped: nbrmgrd (terminated by SIGTERM) Nov 10 14:14:27.494242 sonic INFO swss#supervisord 2016-11-10 14:14:24,510 INFO stopped: vrfmgrd (terminated by SIGTERM) Nov 10 14:14:27.494242 sonic INFO swss#supervisord 2016-11-10 14:14:25,515 INFO stopped: buffermgrd (terminated by SIGTERM) Nov 10 14:14:27.494300 sonic INFO swss#supervisord 2016-11-10 14:14:25,517 INFO stopped: portmgrd (terminated by SIGTERM) Nov 10 14:14:27.494300 sonic INFO swss#supervisord 2016-11-10 14:14:25,519 INFO stopped: intfmgrd (terminated by SIGTERM) Nov 10 14:14:27.494300 sonic INFO swss#supervisord 2016-11-10 14:14:25,522 INFO stopped: vlanmgrd (terminated by SIGTERM) Nov 10 14:14:27.494300 sonic INFO swss#supervisord 2016-11-10 14:14:26,524 INFO waiting for neighsyncd, portsyncd, supervisor-proc-exit-listener, rsyslogd to die Nov 10 14:14:29.731715 sonic INFO containerd[452]: time="2016-11-10T14:14:29.731179759Z" level=info msg="shim reaped" id=1dfc4256171f90a97e84fcbd6cf13e7286de26c25c44aec33e4db7adb895dd10 Nov 10 14:14:29.741860 sonic INFO dockerd[481]: time="2016-11-10T14:14:29.741651634Z" level=info msg="ignoring event" module=libcontainerd namespace=moby topic=/tasks/delete type="*events.TaskDelete" Nov 10 14:14:29.887573 sonic INFO swss.sh[2763]: No longer waiting on container 'swss' Nov 10 14:14:29.922005 sonic NOTICE root: Stopping swss service... How I verified it 1. Repeat the test procedure. 2. Check swss and related container status. 3. Use the following command to check whether default route in non default VRF or not. drivshell>l3 defip show l3 defip show Unit 0, Total Number of DEFIP entries: 98304 16404 2 192.168.5.5/32 00:00:00:00:00:00 100003 0 0 0 1 n 32788 2 192.168.5.0/24 00:00:00:00:00:00 100003 0 0 0 1 n 0 2 0.0.0.0/0 00:00:00:00:00:00 100002 0 0 0 0 n 0 0 0.0.0.0/0 00:00:00:00:00:00 100002 0 0 0 0 n drivshell>l3 ip6route show l3 ip6route show Unit 0, Total Number of IPv6 entries: 49152 Max number of ECMP paths 64 Free IPv6 entries available: 49146 16392 0 fe80:0000:0000:0000:ce37:abff:fe60:7f9d/128 00:00:00:00:00:00 10003 0 0 0 1 n 9 0 fe80:0000:0000:0000:0000:0000:0000:0000/10 00:00:00:00:00:00 10003 0 0 0 1 n 0 2 0000:0000:0000:0000:0000:0000:0000:0000/0 00:00:00:00:00:00 10002 0 0 0 0 n 0 0 0000:0000:0000:0000:0000:0000:0000:0000/0 00:00:00:00:00:00 10002 0 0 0 0 n --- orchagent/routeorch.cpp | 129 ++++++++++++++++++++++++++-------------- orchagent/routeorch.h | 2 + orchagent/vrforch.cpp | 4 ++ 3 files changed, 92 insertions(+), 43 deletions(-) diff --git a/orchagent/routeorch.cpp b/orchagent/routeorch.cpp index 8aabef4a9e..50b8175321 100644 --- a/orchagent/routeorch.cpp +++ b/orchagent/routeorch.cpp @@ -62,49 +62,7 @@ RouteOrch::RouteOrch(DBConnector *db, string tableName, NeighOrch *neighOrch, In } SWSS_LOG_NOTICE("Maximum number of ECMP groups supported is %d", m_maxNextHopGroupCount); - IpPrefix default_ip_prefix("0.0.0.0/0"); - - sai_route_entry_t unicast_route_entry; - unicast_route_entry.vr_id = gVirtualRouterId; - unicast_route_entry.switch_id = gSwitchId; - copy(unicast_route_entry.destination, default_ip_prefix); - subnet(unicast_route_entry.destination, unicast_route_entry.destination); - - attr.id = SAI_ROUTE_ENTRY_ATTR_PACKET_ACTION; - attr.value.s32 = SAI_PACKET_ACTION_DROP; - - status = sai_route_api->create_route_entry(&unicast_route_entry, 1, &attr); - if (status != SAI_STATUS_SUCCESS) - { - SWSS_LOG_ERROR("Failed to create IPv4 default route with packet action drop"); - throw runtime_error("Failed to create IPv4 default route with packet action drop"); - } - - gCrmOrch->incCrmResUsedCounter(CrmResourceType::CRM_IPV4_ROUTE); - - /* Add default IPv4 route into the m_syncdRoutes */ - m_syncdRoutes[gVirtualRouterId][default_ip_prefix] = NextHopGroupKey(); - - SWSS_LOG_NOTICE("Create IPv4 default route with packet action drop"); - - IpPrefix v6_default_ip_prefix("::/0"); - - copy(unicast_route_entry.destination, v6_default_ip_prefix); - subnet(unicast_route_entry.destination, unicast_route_entry.destination); - - status = sai_route_api->create_route_entry(&unicast_route_entry, 1, &attr); - if (status != SAI_STATUS_SUCCESS) - { - SWSS_LOG_ERROR("Failed to create IPv6 default route with packet action drop"); - throw runtime_error("Failed to create IPv6 default route with packet action drop"); - } - - gCrmOrch->incCrmResUsedCounter(CrmResourceType::CRM_IPV6_ROUTE); - - /* Add default IPv6 route into the m_syncdRoutes */ - m_syncdRoutes[gVirtualRouterId][v6_default_ip_prefix] = NextHopGroupKey(); - - SWSS_LOG_NOTICE("Create IPv6 default route with packet action drop"); + vrfDefaultRoute(gVirtualRouterId, SET_COMMAND); /* All the interfaces have the same MAC address and hence the same * auto-generated link-local ipv6 address with eui64 interface-id. @@ -1187,3 +1145,88 @@ bool RouteOrch::removeRoute(sai_object_id_t vrf_id, const IpPrefix &ipPrefix) return true; } + +void RouteOrch::vrfDefaultRoute(sai_object_id_t vrf_id, string op) +{ + IpPrefix default_ip_prefix("0.0.0.0/0"); + + sai_route_entry_t unicast_route_entry; + unicast_route_entry.vr_id = vrf_id; + unicast_route_entry.switch_id = gSwitchId; + copy(unicast_route_entry.destination, default_ip_prefix); + subnet(unicast_route_entry.destination, unicast_route_entry.destination); + + sai_status_t status = SAI_STATUS_FAILURE; + sai_attribute_t attr; + attr.id = SAI_ROUTE_ENTRY_ATTR_PACKET_ACTION; + attr.value.s32 = SAI_PACKET_ACTION_DROP; + + if (op == SET_COMMAND) + { + sai_status_t status = sai_route_api->create_route_entry(&unicast_route_entry, 1, &attr); + if (status != SAI_STATUS_SUCCESS) + { + SWSS_LOG_ERROR("Failed to create IPv4 default route with packet action drop"); + throw runtime_error("Failed to create IPv4 default route with packet action drop"); + } + + gCrmOrch->incCrmResUsedCounter(CrmResourceType::CRM_IPV4_ROUTE); + + /* Add default IPv4 route into the m_syncdRoutes */ + m_syncdRoutes[vrf_id][default_ip_prefix] = NextHopGroupKey(); + + SWSS_LOG_NOTICE("Create IPv4 default route with packet action drop"); + } + else + { + if (vrf_id != gVirtualRouterId) + { + sai_status_t status = sai_route_api->remove_route_entry(&unicast_route_entry); + if (status != SAI_STATUS_SUCCESS) + { + SWSS_LOG_ERROR("Failed to remove IPv4 default route for non default VRF!!!"); + throw runtime_error("Failed to remove IPv4 default route for non default VRF!!!"); + } + + gCrmOrch->decCrmResUsedCounter(CrmResourceType::CRM_IPV4_ROUTE); + } + } + + IpPrefix v6_default_ip_prefix("::/0"); + + copy(unicast_route_entry.destination, v6_default_ip_prefix); + subnet(unicast_route_entry.destination, unicast_route_entry.destination); + + if (op == SET_COMMAND) + { + status = sai_route_api->create_route_entry(&unicast_route_entry, 1, &attr); + if (status != SAI_STATUS_SUCCESS) + { + SWSS_LOG_ERROR("Failed to create IPv6 default route with packet action drop"); + throw runtime_error("Failed to create IPv6 default route with packet action drop"); + } + + gCrmOrch->incCrmResUsedCounter(CrmResourceType::CRM_IPV6_ROUTE); + + /* Add default IPv6 route into the m_syncdRoutes */ + m_syncdRoutes[vrf_id][v6_default_ip_prefix] = NextHopGroupKey(); + + SWSS_LOG_NOTICE("Create IPv6 default route with packet action drop"); + } + else + { + if (vrf_id != gVirtualRouterId) + { + status = sai_route_api->remove_route_entry(&unicast_route_entry); + if (status != SAI_STATUS_SUCCESS) + { + SWSS_LOG_ERROR("Failed to create IPv6 default route for non default VRF!!!"); + throw runtime_error("Failed to create IPv6 default route for non default VRF!!!"); + } + + gCrmOrch->decCrmResUsedCounter(CrmResourceType::CRM_IPV6_ROUTE); + + m_syncdRoutes.erase(vrf_id); + } + } +} diff --git a/orchagent/routeorch.h b/orchagent/routeorch.h index 84550d7589..76460f1e15 100644 --- a/orchagent/routeorch.h +++ b/orchagent/routeorch.h @@ -76,6 +76,8 @@ class RouteOrch : public Orch, public Subject bool invalidnexthopinNextHopGroup(const NextHopKey&); void notifyNextHopChangeObservers(sai_object_id_t, const IpPrefix&, const NextHopGroupKey&, bool); + + void vrfDefaultRoute(sai_object_id_t vrf_id, string op); private: NeighOrch *m_neighOrch; IntfsOrch *m_intfsOrch; diff --git a/orchagent/vrforch.cpp b/orchagent/vrforch.cpp index 62b56e1e0e..d769930a32 100644 --- a/orchagent/vrforch.cpp +++ b/orchagent/vrforch.cpp @@ -10,12 +10,14 @@ #include "orch.h" #include "request_parser.h" #include "vrforch.h" +#include "routeorch.h" using namespace std; using namespace swss; extern sai_virtual_router_api_t* sai_virtual_router_api; extern sai_object_id_t gSwitchId; +extern RouteOrch *gRouteOrch; bool VRFOrch::addOperation(const Request& request) { @@ -86,6 +88,7 @@ bool VRFOrch::addOperation(const Request& request) vrf_id_table_[router_id] = vrf_name; m_stateVrfObjectTable.hset(vrf_name, "state", "ok"); SWSS_LOG_NOTICE("VRF '%s' was added", vrf_name.c_str()); + gRouteOrch->vrfDefaultRoute(router_id, SET_COMMAND); } else { @@ -124,6 +127,7 @@ bool VRFOrch::delOperation(const Request& request) return false; sai_object_id_t router_id = vrf_table_[vrf_name].vrf_id; + gRouteOrch->vrfDefaultRoute(router_id, DEL_COMMAND); sai_status_t status = sai_virtual_router_api->remove_virtual_router(router_id); if (status != SAI_STATUS_SUCCESS) {