Add self timeout and crash if exceeded. (#1502)

Log callstack on timeout.

What I did
Add self timeout and crash on timeout.
Before crash log the error and call stack.

How I did it
Add a signal based alarm and the handler to print error & call stack.

How to verify it
Artificially introduce a sleep (> TIMEOUT, which is 2mins) in the script in any function that is in the call path.
Invoke the script. Watch it crash and note the error & stack logged in syslog.

scripts/route_check.py

@@ -43,6 +43,8 @@
 import sys
 import syslog
 import time
+import signal
+import traceback
 
 from swsscommon import swsscommon
 
@@ -53,6 +55,9 @@
 
 SUBSCRIBE_WAIT_SECS = 1
 
+# Max of 2 minutes
+TIMEOUT_SECONDS = 120
+
 UNIT_TESTING = 0
 
 os.environ['PYTHONUNBUFFERED']='True'
@@ -75,6 +80,14 @@ def __str__(self):
 report_level = syslog.LOG_ERR
 write_to_syslog = False
 
+def handler(signum, frame):
+ print_message(syslog.LOG_ERR,
+ "Aborting routeCheck.py upon timeout signal after {} seconds".
+ format(TIMEOUT_SECONDS))
+ print_message(syslog.LOG_ERR, str(traceback.extract_stack()))
+ raise Exception("timeout occurred")
+
+
 def set_level(lvl, log_to_syslog):
  """
  Sets the log level
@@ -429,7 +442,7 @@ def main():
  parser=argparse.ArgumentParser(description="Verify routes between APPL-DB & ASIC-DB are in sync")
  parser.add_argument('-m', "--mode", type=Level, choices=list(Level), default='ERR')
  parser.add_argument("-i", "--interval", type=int, default=0, help="Scan interval in seconds")
- parser.add_argument("-s", "--log_to_syslog", action="store_true", default=False, help="Write message to syslog")
+ parser.add_argument("-s", "--log_to_syslog", action="store_true", default=True, help="Write message to syslog")
  args = parser.parse_args()
 
  set_level(args.mode, args.log_to_syslog)
@@ -444,8 +457,12 @@ def main():
  if UNIT_TESTING:
  interval = 1
 
+ signal.signal(signal.SIGALRM, handler)
+
  while True:
+ signal.alarm(TIMEOUT_SECONDS)
  ret, res= check_routes()
+ signal.alarm(0)
 
  if interval:
  time.sleep(interval)

tests/route_check_test.py

@@ -2,6 +2,7 @@
 import json
 import os
 import sys
+import time
 from unittest.mock import MagicMock, patch
 
 import pytest
@@ -276,6 +277,7 @@ def table_side_effect(db, tbl):
 
 class mock_selector:
  TIMEOUT = 1
+ EMULATE_HANG = False
 
  def __init__(self):
  self.select_state = 0
@@ -295,6 +297,9 @@ def select(self, timeout):
  state = self.select_state
  self.subs.update()
 
+ if mock_selector.EMULATE_HANG:
+ time.sleep(60)
+
  if self.select_state == 0:
  self.select_state = self.TIMEOUT
  else:
@@ -423,6 +428,21 @@ def test_server(self, mock_subs, mock_sel, mock_table, mock_conn):
  assert res == expect_res
 
 
+ # Test timeout
+ route_check.TIMEOUT_SECONDS = 5
+ mock_selector.EMULATE_HANG = True
+ ex_raised = False
+
+ try:
+ ret, res = route_check.main()
+ except Exception as err:
+ ex_raised = True
+ expect = "timeout occurred"
+ ex_str = str(err)
+ assert ex_str == expect, "{} != {}".format(ex_str, expect)
+ assert ex_raised, "Exception expected"
+
+