[sai_failure_dump]Invoking dump during SAI failure (sonic-net#1198)

* [sai_failure_dump]Invoking dump during SAI failure
sumanbrcm · Feb 2, 2023 · ad3d4d6 · ad3d4d6
1 parent 2f6cbd3
commit ad3d4d6
Show file tree

Hide file tree

Showing 8 changed files with 105 additions and 7 deletions.
diff --git a/lib/RedisRemoteSaiInterface.cpp b/lib/RedisRemoteSaiInterface.cpp
@@ -1975,6 +1975,7 @@ sai_status_t RedisRemoteSaiInterface::sai_redis_notify_syncd(
         case SAI_REDIS_NOTIFY_SYNCD_INIT_VIEW:
         case SAI_REDIS_NOTIFY_SYNCD_APPLY_VIEW:
         case SAI_REDIS_NOTIFY_SYNCD_INSPECT_ASIC:
+        case SAI_REDIS_NOTIFY_SYNCD_INVOKE_DUMP:
             break;
 
         default:
@@ -2016,6 +2017,12 @@ sai_status_t RedisRemoteSaiInterface::sai_redis_notify_syncd(
 
                 break;
 
+            case SAI_REDIS_NOTIFY_SYNCD_INVOKE_DUMP:
+
+                SWSS_LOG_NOTICE("invoked DUMP succeeded");
+
+                break;
+
             default:
                 break;
         }

diff --git a/lib/sairedis.h b/lib/sairedis.h
@@ -51,8 +51,9 @@ typedef enum _sai_redis_notify_syncd_t
 
     SAI_REDIS_NOTIFY_SYNCD_APPLY_VIEW,
 
-    SAI_REDIS_NOTIFY_SYNCD_INSPECT_ASIC
+    SAI_REDIS_NOTIFY_SYNCD_INSPECT_ASIC,
 
+    SAI_REDIS_NOTIFY_SYNCD_INVOKE_DUMP
 } sai_redis_notify_syncd_t;
 
 typedef enum _sai_redis_communication_mode_t
@@ -248,5 +249,4 @@ typedef enum _sai_redis_switch_attr_t
      * @default 60000
      */
     SAI_REDIS_SWITCH_ATTR_SYNC_OPERATION_RESPONSE_TIMEOUT,
-
 } sai_redis_switch_attr_t;
diff --git a/lib/sairediscommon.h b/lib/sairediscommon.h
@@ -8,6 +8,7 @@
 #define SYNCD_INIT_VIEW     "INIT_VIEW"
 #define SYNCD_APPLY_VIEW    "APPLY_VIEW"
 #define SYNCD_INSPECT_ASIC  "SYNCD_INSPECT_ASIC"
+#define SYNCD_INVOKE_DUMP   "SYNCD_INVOKE_DUMP"
 
 #define ASIC_STATE_TABLE    "ASIC_STATE"
 #define TEMP_PREFIX         "TEMP_"

diff --git a/meta/SaiSerialize.cpp b/meta/SaiSerialize.cpp
@@ -2226,6 +2226,9 @@ std::string sai_serialize(
         case SAI_REDIS_NOTIFY_SYNCD_INSPECT_ASIC:
             return SYNCD_INSPECT_ASIC;
 
+        case SAI_REDIS_NOTIFY_SYNCD_INVOKE_DUMP:
+            return SYNCD_INVOKE_DUMP;
+
         default:
 
             SWSS_LOG_THROW("unknown value on sai_redis_notify_syncd_t: %d", value);
@@ -4253,6 +4256,10 @@ void sai_deserialize(
     {
         value = SAI_REDIS_NOTIFY_SYNCD_INSPECT_ASIC;
     }
+    else if (s == SYNCD_INVOKE_DUMP)
+    {
+        value = SAI_REDIS_NOTIFY_SYNCD_INVOKE_DUMP;
+    }
     else
     {
         SWSS_LOG_THROW("enum %s not found in sai_redis_notify_syncd_t", s.c_str());

diff --git a/syncd/Syncd.cpp b/syncd/Syncd.cpp
@@ -19,6 +19,7 @@
 #include "swss/select.h"
 #include "swss/tokenize.h"
 #include "swss/notificationproducer.h"
+#include "swss/exec.h"
 
 #include "meta/sai_serialize.h"
 #include "meta/ZeroMQSelectableChannel.h"
@@ -34,6 +35,7 @@
 #include <algorithm>
 
 #define DEF_SAI_WARM_BOOT_DATA_FILE "/var/warmboot/sai-warmboot.bin"
+#define SAI_FAILURE_DUMP_SCRIPT "/usr/bin/sai_failure_dump.sh"
 
 using namespace syncd;
 using namespace saimeta;
@@ -3242,6 +3244,7 @@ sai_status_t Syncd::processNotifySyncd(
     SWSS_LOG_ENTER();
 
     auto& key = kfvKey(kco);
+    sai_status_t status = SAI_STATUS_SUCCESS;
 
     if (!m_commandLineOptions->m_enableTempView)
     {
@@ -3254,6 +3257,20 @@ sai_status_t Syncd::processNotifySyncd(
 
     auto redisNotifySyncd = sai_deserialize_redis_notify_syncd(key);
 
+    if (redisNotifySyncd == SAI_REDIS_NOTIFY_SYNCD_INVOKE_DUMP)
+    {
+        SWSS_LOG_NOTICE("Invoking SAI failure dump");
+        std::string ret_str;
+        int ret = swss::exec(SAI_FAILURE_DUMP_SCRIPT, ret_str);
+        if (ret != 0)
+        {
+            SWSS_LOG_ERROR("Error in executing SAI failure dump %s", ret_str.c_str());
+            status = SAI_STATUS_FAILURE;
+        }
+        sendNotifyResponse(status);
+        return status;
+    }
+
     if (m_veryFirstRun && m_firstInitWasPerformed && redisNotifySyncd == SAI_REDIS_NOTIFY_SYNCD_INIT_VIEW)
     {
         /*
@@ -3268,7 +3285,6 @@ sai_status_t Syncd::processNotifySyncd(
     {
         SWSS_LOG_NOTICE("very first run is TRUE, op = %s", key.c_str());
 
-        sai_status_t status = SAI_STATUS_SUCCESS;
 
         /*
          * On the very first start of syncd, "compile" view is directly applied
@@ -3343,7 +3359,6 @@ sai_status_t Syncd::processNotifySyncd(
 
         SWSS_LOG_WARN("syncd received APPLY VIEW, will translate");
 
-        sai_status_t status;
 
         try
         {

diff --git a/syncd/scripts/sai_failure_dump.sh b/syncd/scripts/sai_failure_dump.sh
@@ -0,0 +1,22 @@
+#!/usr/bin/env bash
+#
+# Script for sai failure dump
+#
+
+# Source the platform specific dump file
+
+SAI_MAX_FAILURE_DUMPS=10
+
+DUMPDIR=/var/log/sai_failure_dump
+
+if [ -f /usr/bin/platform_syncd_dump.sh ]; then
+    . ./usr/bin/platform_syncd_dump.sh
+fi
+
+if [ -z "$(ls -A $DUMPDIR/)" ]; then
+    exit 0
+fi
+
+# Perform rotation
+
+ls -1td $DUMPDIR/* | tail -n +$(($SAI_MAX_FAILURE_DUMPS+1)) | xargs rm -rf
diff --git a/syncd/scripts/syncd_init_common.sh b/syncd/scripts/syncd_init_common.sh
@@ -31,6 +31,9 @@ if [[ "$(cat /proc/cmdline)" != *"SONIC_BOOT_TYPE=fast-reboot"* ]]; then
     CMD_ARGS+=" -u"
 fi
 
+# Create a folder for SAI failure dump files
+mkdir -p /var/log/sai_failure_dump/
+
 # Use bulk APIs in SAI
 # currently disabled since most vendors don't support that yet
 # CMD_ARGS+=" -l"

diff --git a/syncd/tests.cpp b/syncd/tests.cpp
@@ -42,7 +42,26 @@ using namespace syncd;
     if ((status)!=SAI_STATUS_SUCCESS) \
         SWSS_LOG_THROW(format ": %s", ##__VA_ARGS__, sai_serialize_status(status).c_str());
 
+#define SAI_FAILURE_DUMP_SCRIPT "/usr/bin/sai_failure_dump.sh"
+
+#define CHECK_STATUS(x)  \
+    if (status != SAI_STATUS_SUCCESS) { exit(1); }
+
+
 using namespace saimeta;
+
+std::string mockCallArg;
+
+namespace swss {
+    int exec(const std::string &cmd, std::string &stdout)
+    {
+        SWSS_LOG_ENTER();
+
+        mockCallArg=cmd;
+        return 0;
+    }
+}
+
 static std::shared_ptr<swss::DBConnector> g_db1;
 
 static sai_next_hop_group_api_t test_next_hop_group_api;
@@ -652,9 +671,6 @@ void test_bulk_route_set()
     ASSERT_SUCCESS("Failed to bulk remove route entry");
 }
 
-#define CHECK_STATUS(x)  \
-    if (status != SAI_STATUS_SUCCESS) { exit(1); }
-
 void syncdThread()
 {
     SWSS_LOG_ENTER();
@@ -678,6 +694,31 @@ void syncdThread()
     syncd->run();
 }
 
+void test_invoke_dump()
+{
+    SWSS_LOG_ENTER();
+    clearDB();
+
+    auto syncd = std::make_shared<std::thread>(syncdThread);
+    syncd->detach();
+
+    sai_attribute_t attr;
+    attr.id = SAI_REDIS_SWITCH_ATTR_NOTIFY_SYNCD;
+    attr.value.s32 = SAI_REDIS_NOTIFY_SYNCD_INVOKE_DUMP;
+
+    auto sairedis = std::make_shared<sairedis::Sai>();
+
+    sai_status_t status = sairedis->initialize(0, &test_services);
+
+    CHECK_STATUS(status);
+
+    status = sairedis->set(SAI_OBJECT_TYPE_SWITCH, SAI_NULL_OBJECT_ID, &attr);
+
+    ASSERT_SUCCESS("Failed to invoke dump");
+    assert(mockCallArg == SAI_FAILURE_DUMP_SCRIPT);
+}
+
+
 void test_bulk_route_create()
 {
     SWSS_LOG_ENTER();
@@ -875,6 +916,8 @@ int main()
         printf("\n[ %s ]\n\n", sai_serialize_status(SAI_STATUS_SUCCESS).c_str());
 
         test_watchdog_timer_clock_rollback();
+
+        test_invoke_dump();
     }
     catch (const std::exception &e)
     {