Improve SONiC disk checker to handle disk full case and mount overlay fs to allow remote user login.

scripts/disk_check.py

@@ -3,33 +3,40 @@
 
 """
 What:
-    There have been cases, where disk turns Read-only due to kernel bug.
-    In Read-only state, system blocks new remote user login via TACACS.
-    This utility is to check & make transient recovery as needed.
+    This utility is designed to address two specific issues:
+    1. Disk becoming read-only due to kernel bugs.
+    2. Disk running out of space.
+    When either of these issues occurs, the system prevents new remote user logins via TACACS.
 
 How:
-    check for Read-Write permission. If Read-only, create writable overlay using tmpfs.
+    Checks for read-write permissions and available disk space.
+    If an issue is detected, create writable overlay using tmpfs.
 
-    By default "/etc" & "/home" are checked and if in Read-only state, make them Read-Write
+    By default "/etc" & "/home" are checked and if issue detected, make them Read-Write
     using overlay on top of tmpfs.
 
     Making /etc & /home as writable lets successful new remote user login.
 
-    If in Read-only state or in Read-Write state with the help of tmpfs overlay,
-    syslog ERR messages are written, to help raise alerts.
+    Write syslog ERR messages to help raise alerts in the following cases:
+    1. Disk in read-only state.
+    2. Disk out of space.
+    3. Mounted tmpfs overlay.
 
     Monit may be used to invoke it periodically, to help scan & fix and
     report via syslog.
 
 Tidbit:
-    If you would like to test this script, you could simulate a RO disk
-    with the following command. Reboot will revert the effect.
+    To test this script:
+    1. Simulate a RO disk with the following command. Reboot will revert the effect.
         sudo bash -c "echo u > /proc/sysrq-trigger"
+    2. Use up all disk space by create big file in /var/dump/:
+         dd if=/dev/zero of=/var/dump/sonic_dump_devicename_20241126_204132.tar bs=1G count=50
 
 """
 
 import argparse
 import os
+import shutil
 import sys
 import syslog
 import subprocess
@@ -40,10 +47,16 @@
 WORK_DIR = "/run/mount/work"
 MOUNTS_FILE = "/proc/mounts"
 
+# Threshold of free block counts: On most file systems, the block size is 4096 bytes.
+FREE_SPACE_THRESHOLD = 1024
+
 EVENTS_PUBLISHER_SOURCE = "sonic-events-host"
 EVENTS_PUBLISHER_TAG = "event-disk"
 events_handle = None
 
+DISK_RO_EVENT = "read_only"
+DISK_FULL_EVENT = "disk_full"
+
 chk_log_level = syslog.LOG_ERR
 
 def _log_msg(lvl, pfx, msg):
@@ -64,21 +77,35 @@ def log_debug(m):
     _log_msg(syslog.LOG_DEBUG, "Debug", m)
 
 
-def event_pub():
+def event_pub(event):
     param_dict = FieldValueMap()
-    param_dict["fail_type"] = "read_only"
+    param_dict["fail_type"] = event
     event_publish(events_handle, EVENTS_PUBLISHER_TAG, param_dict)
 
 
+def test_disk_full(dirs):
+    for d in dirs:
+        space = os.statvfs(d)
+        if space.f_bavail < FREE_SPACE_THRESHOLD:
+            log_err("{} has no free disk space".format(d))
+            event_pub(DISK_FULL_EVENT)
+            return True
+        else:
+            log_debug("{} has enough disk space".format(d))
+
+    return False
+
+
 def test_writable(dirs): 
     for d in dirs:
         rw = os.access(d, os.W_OK)
         if not rw:
             log_err("{} is not read-write".format(d))
-            event_pub()
+            event_pub(DISK_RO_EVENT)
             return False
         else:
             log_debug("{} is Read-Write".format(d))
+
     return True
 
 
@@ -101,7 +128,7 @@ def get_dname(path_name):
     return os.path.basename(os.path.normpath(path_name))
 
 
-def do_mnt(dirs):
+def do_mnt(dirs, overlay_prefix):
     if os.path.exists(UPPER_DIR):
         log_err("Already mounted")
         return 1
@@ -110,7 +137,7 @@ def do_mnt(dirs):
         try:
             os.mkdir(i)
         except OSError as error:
-            log_err("Failed to create {}".format(i))
+            log_err("Failed to create {}, error: {}".format(i, error))
             return 1
 
     for d in dirs:
@@ -120,7 +147,7 @@ def do_mnt(dirs):
         os.mkdir(d_upper)
         os.mkdir(d_work)
 
-        ret = run_cmd(["mount", "-t", "overlay", "overlay_{}".format(d_name),\
+        ret = run_cmd(["mount", "-t", "overlay", "{}_{}".format(overlay_prefix, d_name),
                 "-o", "lowerdir={},upperdir={},workdir={}".format(d, d_upper, d_work), d])
         if ret:
             break
@@ -132,13 +159,36 @@ def do_mnt(dirs):
     return ret
 
 
-def is_mounted(dirs):
+def do_unmnt(dirs, overlay_prefix):
+    for d in dirs:
+        d_name = get_dname(d)
+
+        ret = run_cmd(["umount", "-l", "{}_{}".format(overlay_prefix, d_name)])
+        if ret:
+            break
+
+    if ret:
+        log_err("Failed to umount {}".format(dirs))
+    else:
+        log_info("{} are unmounted".format(dirs))
+
+    for i in (UPPER_DIR, WORK_DIR):
+        try:
+            shutil.rmtree(i)
+        except OSError as error:
+            log_err("Failed to delete {}, error: {}".format(i, error))
+            return 1
+
+    return ret
+
+
+def is_mounted(dirs, overlay_prefix):
     if not os.path.exists(UPPER_DIR):
         return False
 
     onames = set()
     for d in dirs:
-        onames.add("overlay_{}".format(get_dname(d)))
+        onames.add("{}_{}".format(overlay_prefix, get_dname(d)))
 
     with open(MOUNTS_FILE, "r") as s:
         for ln in s.readlines():
@@ -153,12 +203,31 @@ def do_check(skip_mount, dirs):
     ret = 0
     if not test_writable(dirs):
         if not skip_mount:
-            ret = do_mnt(dirs)
+            ret = do_mnt(dirs, "overlay")
 
     # Check if mounted
-    if (not ret) and is_mounted(dirs):
+    if (not ret) and is_mounted(dirs, "overlay"):
         log_err("READ-ONLY: Mounted {} to make Read-Write".format(dirs))
-        event_pub()
+        event_pub(DISK_RO_EVENT)
+
+    if ret:
+        # When disk mounted, disk no free space issue also been fixed.
+        return ret
+
+    # Handle disk no free space case
+    if test_disk_full(dirs):
+        if not skip_mount:
+            ret = do_mnt(dirs, "overlay_disk_full")
+
+    # Check if mounted
+    if (not ret) and is_mounted(dirs, "overlay_disk_full"):
+        log_err("DISK-FULL: Mounted {} to make Read-Write".format(dirs))
+        event_pub(DISK_FULL_EVENT)
+
+    # Unmount when disk space issue fixed
+    if is_mounted(dirs, "overlay_disk_full") and not test_disk_full(["/"]):
+        log_debug("umount for disk space issue fixed")
+        do_unmnt(dirs, "overlay_disk_full")
 
     return ret
 

tests/disk_check_test.py

@@ -1,3 +1,4 @@
+import os
 import sys
 import syslog
 from unittest.mock import patch
@@ -131,7 +132,9 @@ def setup(self):
 
     @patch("disk_check.syslog.syslog")
     @patch("disk_check.subprocess.run")
-    def test_readonly(self, mock_proc, mock_log):
+    @patch('os.statvfs', return_value=os.statvfs_result((4096, 4096, 1909350, 1491513, 4096,
+                                                         971520, 883302, 883302, 4096, 255)))
+    def test_readonly(self, mock_os_statvfs, mock_proc, mock_log):
         global err_data, cmds, max_log_lvl
 
         mock_proc.side_effect = mock_subproc_run
@@ -177,6 +180,72 @@ def test_readonly(self, mock_proc, mock_log):
 
         assert max_log_lvl == syslog.LOG_ERR
 
+    @patch("disk_check.syslog.syslog")
+    @patch("disk_check.subprocess.run")
+    @patch('os.access', return_value=True)
+    @patch('os.statvfs', return_value=os.statvfs_result((4096, 4096, 1909350, 1491513, 0,
+                                                         971520, 883302, 883302, 4096, 255)))
+    def test_mount_disk_full(self, mock_os_statvfs, mock_os_access, mock_proc, mock_log):
+        global max_log_lvl
+        max_log_lvl = -1
+        mock_proc.side_effect = mock_subproc_run
+        mock_log.side_effect = report_err_msg
+
+        tc = {
+            "upperdir": "/tmp",
+        }
+        swap_upper(tc)
+
+        with patch('sys.argv', ["", "-d", "/tmpx"]):
+            disk_check.main()
+
+    @patch("disk_check.syslog.syslog")
+    @patch("disk_check.subprocess.run")
+    @patch('shutil.rmtree')
+    @patch('os.access', return_value=True)
+    @patch('os.statvfs', return_value=os.statvfs_result((4096, 4096, 1909350, 1491513, 4096,
+                                                         971520, 883302, 883302, 4096, 255)))
+    def test_unmount_disk_full(self, mock_os_statvfs, mock_os_access, mock_rmtree, mock_proc, mock_log):
+        global max_log_lvl
+        max_log_lvl = -1
+        mock_proc.side_effect = mock_subproc_run
+        mock_log.side_effect = report_err_msg
+
+        tc = {
+            "upperdir": "/tmp/tmpx",
+            "workdir": "/tmp/tmpy"
+        }
+        swap_upper(tc)
+        swap_work(tc)
+
+        with patch('sys.argv', ["", "-d", "/tmpx"]):
+            disk_check.main()
+
+    @patch("disk_check.syslog.syslog")
+    @patch("disk_check.subprocess.run")
+    @patch('os.access', return_value=True)
+    @patch('os.statvfs', return_value=os.statvfs_result((4096, 4096, 1909350, 1491513, 0,
+                                                         971520, 883302, 883302, 4096, 255)))
+    def test_diskfull(self, mock_os_statvfs, mock_os_access, mock_proc, mock_log):
+        global max_log_lvl
+        max_log_lvl = -1
+        mock_proc.side_effect = mock_subproc_run
+        mock_log.side_effect = report_err_msg
+
+        result = disk_check.test_disk_full(["/etc"])
+        assert result is True
+
+    @patch("disk_check.syslog.syslog")
+    @patch("disk_check.subprocess.run")
+    def test_do_unmnt(self, mock_proc, mock_log):
+        global max_log_lvl
+        max_log_lvl = -1
+        mock_proc.side_effect = mock_subproc_run
+        mock_log.side_effect = report_err_msg
+
+        disk_check.do_unmnt(["/etc"], "overlay_prefix")
+
+
     @classmethod
     def teardown_class(cls):
         subprocess.run("rm -rf /tmp/tmp*", shell=True)  # cleanup the temporary dirs