From c8a77dfb8d16d325f87a71a8f23f23170783ab41 Mon Sep 17 00:00:00 2001
From: Hua Liu <58683130+liuh-80@users.noreply.github.com>
Date: Fri, 27 Dec 2024 14:05:51 +0800
Subject: [PATCH 01/22] Update disk_check.py

---
 scripts/disk_check.py | 10 +++++++++-
 1 file changed, 9 insertions(+), 1 deletion(-)

diff --git a/scripts/disk_check.py b/scripts/disk_check.py
index de1557ca2a..f2431f4bd5 100644
--- a/scripts/disk_check.py
+++ b/scripts/disk_check.py
@@ -40,6 +40,8 @@
 WORK_DIR = "/run/mount/work"
 MOUNTS_FILE = "/proc/mounts"
 
+FREE_SPACE_THRESHOLD = 1024
+
 EVENTS_PUBLISHER_SOURCE = "sonic-events-host"
 EVENTS_PUBLISHER_TAG = "event-disk"
 events_handle = None
@@ -78,7 +80,13 @@ def test_writable(dirs):
             event_pub()
             return False
         else:
-            log_debug("{} is Read-Write".format(d))
+            print("{} is Read-Write".format(d))
+            space = os.statvfs(d)
+            if space.f_bavail < FREE_SPACE_THRESHOLD:
+                log_err("{} is no free space".format(d))
+                event_pub()
+                return False
+
     return True
 
 

From f47ef4bba8b21d899b143e7d59daaed98ae0cbe9 Mon Sep 17 00:00:00 2001
From: Hua Liu <58683130+liuh-80@users.noreply.github.com>
Date: Fri, 27 Dec 2024 14:10:34 +0800
Subject: [PATCH 02/22] Update disk_check.py

---
 scripts/disk_check.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/scripts/disk_check.py b/scripts/disk_check.py
index f2431f4bd5..e57118345b 100644
--- a/scripts/disk_check.py
+++ b/scripts/disk_check.py
@@ -80,7 +80,6 @@ def test_writable(dirs):
             event_pub()
             return False
         else:
-            print("{} is Read-Write".format(d))
             space = os.statvfs(d)
             if space.f_bavail < FREE_SPACE_THRESHOLD:
                 log_err("{} is no free space".format(d))

From b6dc4c10adfc7777a9fe599566d4041cfe19dc00 Mon Sep 17 00:00:00 2001
From: Hua Liu <58683130+liuh-80@users.noreply.github.com>
Date: Fri, 27 Dec 2024 14:49:48 +0800
Subject: [PATCH 03/22] Fix UT

---
 scripts/disk_check.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/scripts/disk_check.py b/scripts/disk_check.py
index e57118345b..acf0acb00d 100644
--- a/scripts/disk_check.py
+++ b/scripts/disk_check.py
@@ -80,11 +80,14 @@ def test_writable(dirs):
             event_pub()
             return False
         else:
+            log_debug("{} is Read-Write".format(d))
             space = os.statvfs(d)
             if space.f_bavail < FREE_SPACE_THRESHOLD:
                 log_err("{} is no free space".format(d))
                 event_pub()
                 return False
+            else:
+                log_debug("{} has enough disk space".format(d))
 
     return True
 

From d375bf274be241f979e91c8c0fae33a3794718d1 Mon Sep 17 00:00:00 2001
From: Hua Liu <58683130+liuh-80@users.noreply.github.com>
Date: Fri, 27 Dec 2024 16:47:02 +0800
Subject: [PATCH 04/22] Update disk_check_test.py

---
 tests/disk_check_test.py | 12 ++++++++++++
 1 file changed, 12 insertions(+)

diff --git a/tests/disk_check_test.py b/tests/disk_check_test.py
index ac541b05b9..3e7667d8a9 100644
--- a/tests/disk_check_test.py
+++ b/tests/disk_check_test.py
@@ -177,6 +177,18 @@ def test_readonly(self, mock_proc, mock_log):
             
         assert max_log_lvl == syslog.LOG_ERR
 
+
+    @patch("disk_check.syslog.syslog")
+    @patch("disk_check.subprocess.run")
+    @patch('os.statvfs', return_value=os.statvfs_result((4096, 4096, 1909350, 1491513, 1394521, 971520, 883302, 883302, 0, 255)))
+    def test_diskfull(self, mock_proc, mock_log, os_statvfs):
+        mock_proc.side_effect = mock_subproc_run
+        mock_log.side_effect = report_err_msg
+
+        result = disk_check.test_writable(["/etc"])
+        assert result == False
+
+
     @classmethod
     def teardown_class(cls):
         subprocess.run("rm -rf /tmp/tmp*", shell=True)  # cleanup the temporary dirs

From 08c514b5a5a249d99807271b2869f17964291dbb Mon Sep 17 00:00:00 2001
From: Hua Liu <58683130+liuh-80@users.noreply.github.com>
Date: Fri, 27 Dec 2024 17:44:18 +0800
Subject: [PATCH 05/22] Update disk_check_test.py

---
 tests/disk_check_test.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/tests/disk_check_test.py b/tests/disk_check_test.py
index 3e7667d8a9..a07a0b364a 100644
--- a/tests/disk_check_test.py
+++ b/tests/disk_check_test.py
@@ -1,3 +1,4 @@
+import os
 import sys
 import syslog
 from unittest.mock import patch

From 14db2bd3138039f515f3b105e11454c87401e771 Mon Sep 17 00:00:00 2001
From: Hua Liu <58683130+liuh-80@users.noreply.github.com>
Date: Fri, 27 Dec 2024 17:53:47 +0800
Subject: [PATCH 06/22] Update disk_check_test.py

---
 tests/disk_check_test.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/tests/disk_check_test.py b/tests/disk_check_test.py
index a07a0b364a..28438cf827 100644
--- a/tests/disk_check_test.py
+++ b/tests/disk_check_test.py
@@ -178,16 +178,16 @@ def test_readonly(self, mock_proc, mock_log):
             
         assert max_log_lvl == syslog.LOG_ERR
 
-
     @patch("disk_check.syslog.syslog")
     @patch("disk_check.subprocess.run")
-    @patch('os.statvfs', return_value=os.statvfs_result((4096, 4096, 1909350, 1491513, 1394521, 971520, 883302, 883302, 0, 255)))
+    @patch('os.statvfs', return_value=os.statvfs_result((4096, 4096, 1909350, 1491513, 1394521,
+                                                         971520, 883302, 883302, 0, 255)))
     def test_diskfull(self, mock_proc, mock_log, os_statvfs):
         mock_proc.side_effect = mock_subproc_run
         mock_log.side_effect = report_err_msg
 
         result = disk_check.test_writable(["/etc"])
-        assert result == False
+        assert result is False
 
 
     @classmethod

From cd9ef1bb45ef311a277c388e651b567f5368fbf7 Mon Sep 17 00:00:00 2001
From: Hua Liu <58683130+liuh-80@users.noreply.github.com>
Date: Fri, 27 Dec 2024 18:39:47 +0800
Subject: [PATCH 07/22] Update disk_check_test.py

---
 tests/disk_check_test.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tests/disk_check_test.py b/tests/disk_check_test.py
index 28438cf827..adebd85d35 100644
--- a/tests/disk_check_test.py
+++ b/tests/disk_check_test.py
@@ -180,8 +180,8 @@ def test_readonly(self, mock_proc, mock_log):
 
     @patch("disk_check.syslog.syslog")
     @patch("disk_check.subprocess.run")
-    @patch('os.statvfs', return_value=os.statvfs_result((4096, 4096, 1909350, 1491513, 1394521,
-                                                         971520, 883302, 883302, 0, 255)))
+    @patch('os.statvfs', return_value=os.statvfs_result((4096, 4096, 1909350, 1491513, 0,
+                                                         971520, 883302, 883302, 4096, 255)))
     def test_diskfull(self, mock_proc, mock_log, os_statvfs):
         mock_proc.side_effect = mock_subproc_run
         mock_log.side_effect = report_err_msg

From 0c2f047d05fa4a9cc08a503e0d3e006e07f07120 Mon Sep 17 00:00:00 2001
From: Hua Liu <58683130+liuh-80@users.noreply.github.com>
Date: Fri, 27 Dec 2024 18:44:03 +0800
Subject: [PATCH 08/22] Update disk_check_test.py

---
 tests/disk_check_test.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/tests/disk_check_test.py b/tests/disk_check_test.py
index adebd85d35..ee3fcabee2 100644
--- a/tests/disk_check_test.py
+++ b/tests/disk_check_test.py
@@ -180,6 +180,7 @@ def test_readonly(self, mock_proc, mock_log):
 
     @patch("disk_check.syslog.syslog")
     @patch("disk_check.subprocess.run")
+    @patch('os.access', return_value=True)
     @patch('os.statvfs', return_value=os.statvfs_result((4096, 4096, 1909350, 1491513, 0,
                                                          971520, 883302, 883302, 4096, 255)))
     def test_diskfull(self, mock_proc, mock_log, os_statvfs):

From 074fb4a37ae5c662d791209ae7688e8cb5f262b1 Mon Sep 17 00:00:00 2001
From: Hua Liu <58683130+liuh-80@users.noreply.github.com>
Date: Fri, 27 Dec 2024 19:21:02 +0800
Subject: [PATCH 09/22] Update disk_check_test.py

---
 tests/disk_check_test.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/disk_check_test.py b/tests/disk_check_test.py
index ee3fcabee2..91c8736c4d 100644
--- a/tests/disk_check_test.py
+++ b/tests/disk_check_test.py
@@ -183,7 +183,7 @@ def test_readonly(self, mock_proc, mock_log):
     @patch('os.access', return_value=True)
     @patch('os.statvfs', return_value=os.statvfs_result((4096, 4096, 1909350, 1491513, 0,
                                                          971520, 883302, 883302, 4096, 255)))
-    def test_diskfull(self, mock_proc, mock_log, os_statvfs):
+    def test_diskfull(self, mock_proc, mock_log, os_access, os_statvfs):
         mock_proc.side_effect = mock_subproc_run
         mock_log.side_effect = report_err_msg
 

From 7a555cdddb1e10f4c05ac41ad2057d9cc0737f72 Mon Sep 17 00:00:00 2001
From: Hua Liu <58683130+liuh-80@users.noreply.github.com>
Date: Fri, 27 Dec 2024 22:12:09 +0800
Subject: [PATCH 10/22] Update disk_check_test.py

---
 tests/disk_check_test.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/tests/disk_check_test.py b/tests/disk_check_test.py
index 91c8736c4d..1ca4b5aa00 100644
--- a/tests/disk_check_test.py
+++ b/tests/disk_check_test.py
@@ -184,6 +184,8 @@ def test_readonly(self, mock_proc, mock_log):
     @patch('os.statvfs', return_value=os.statvfs_result((4096, 4096, 1909350, 1491513, 0,
                                                          971520, 883302, 883302, 4096, 255)))
     def test_diskfull(self, mock_proc, mock_log, os_access, os_statvfs):
+        global max_log_lvl
+        max_log_lvl = -1
         mock_proc.side_effect = mock_subproc_run
         mock_log.side_effect = report_err_msg
 

From 2cd6323f8f3895624ffcb7ac93259c34a3585374 Mon Sep 17 00:00:00 2001
From: Hua Liu <58683130+liuh-80@users.noreply.github.com>
Date: Mon, 6 Jan 2025 11:25:03 +0800
Subject: [PATCH 11/22] Reorder test_diskfull function parameters

---
 tests/disk_check_test.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/disk_check_test.py b/tests/disk_check_test.py
index 1ca4b5aa00..4c61790d41 100644
--- a/tests/disk_check_test.py
+++ b/tests/disk_check_test.py
@@ -183,7 +183,7 @@ def test_readonly(self, mock_proc, mock_log):
     @patch('os.access', return_value=True)
     @patch('os.statvfs', return_value=os.statvfs_result((4096, 4096, 1909350, 1491513, 0,
                                                          971520, 883302, 883302, 4096, 255)))
-    def test_diskfull(self, mock_proc, mock_log, os_access, os_statvfs):
+    def test_diskfull(self, mock_os_statvfs, mock_os_access, mock_proc, mock_log):
         global max_log_lvl
         max_log_lvl = -1
         mock_proc.side_effect = mock_subproc_run

From ff540d539114ccca80b4ada7329c0b9abbb89cf7 Mon Sep 17 00:00:00 2001
From: Hua Liu <58683130+liuh-80@users.noreply.github.com>
Date: Wed, 8 Jan 2025 13:53:59 +0800
Subject: [PATCH 12/22] Add event parameter to event_pub function

---
 scripts/disk_check.py | 11 ++++++-----
 1 file changed, 6 insertions(+), 5 deletions(-)

diff --git a/scripts/disk_check.py b/scripts/disk_check.py
index acf0acb00d..d1be19c21a 100644
--- a/scripts/disk_check.py
+++ b/scripts/disk_check.py
@@ -40,6 +40,7 @@
 WORK_DIR = "/run/mount/work"
 MOUNTS_FILE = "/proc/mounts"
 
+# Threshold of free block counts: On most file systems, the block size is 4096 bytes.
 FREE_SPACE_THRESHOLD = 1024
 
 EVENTS_PUBLISHER_SOURCE = "sonic-events-host"
@@ -66,9 +67,9 @@ def log_debug(m):
     _log_msg(syslog.LOG_DEBUG, "Debug", m)
 
 
-def event_pub():
+def event_pub(event):
     param_dict = FieldValueMap()
-    param_dict["fail_type"] = "read_only"
+    param_dict["fail_type"] = event
     event_publish(events_handle, EVENTS_PUBLISHER_TAG, param_dict)
 
 
@@ -77,14 +78,14 @@ def test_writable(dirs):
         rw = os.access(d, os.W_OK)
         if not rw:
             log_err("{} is not read-write".format(d))
-            event_pub()
+            event_pub("read_only")
             return False
         else:
             log_debug("{} is Read-Write".format(d))
             space = os.statvfs(d)
             if space.f_bavail < FREE_SPACE_THRESHOLD:
-                log_err("{} is no free space".format(d))
-                event_pub()
+                log_err("{} has no free disk space".format(d))
+                event_pub("disk_full")
                 return False
             else:
                 log_debug("{} has enough disk space".format(d))

From f067be5340e392cff873f61ebfc111b3003b2528 Mon Sep 17 00:00:00 2001
From: Hua Liu <58683130+liuh-80@users.noreply.github.com>
Date: Wed, 8 Jan 2025 14:39:43 +0800
Subject: [PATCH 13/22] "Add argument to event_pub call"

---
 scripts/disk_check.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scripts/disk_check.py b/scripts/disk_check.py
index d1be19c21a..033b6bc0ef 100644
--- a/scripts/disk_check.py
+++ b/scripts/disk_check.py
@@ -169,7 +169,7 @@ def do_check(skip_mount, dirs):
     # Check if mounted
     if (not ret) and is_mounted(dirs):
         log_err("READ-ONLY: Mounted {} to make Read-Write".format(dirs))
-        event_pub()
+        event_pub("read_only")
 
     return ret
 

From 118f051ebfe1f9d0de4ef401f8127426d23f56d8 Mon Sep 17 00:00:00 2001
From: Hua Liu <58683130+liuh-80@users.noreply.github.com>
Date: Tue, 14 Jan 2025 16:23:47 +0800
Subject: [PATCH 14/22] Add disk full check and overlay prefix

---
 scripts/disk_check.py | 82 ++++++++++++++++++++++++++++++++++---------
 1 file changed, 66 insertions(+), 16 deletions(-)

diff --git a/scripts/disk_check.py b/scripts/disk_check.py
index 033b6bc0ef..6e99e0969a 100644
--- a/scripts/disk_check.py
+++ b/scripts/disk_check.py
@@ -30,6 +30,7 @@
 
 import argparse
 import os
+import shutil
 import sys
 import syslog
 import subprocess
@@ -47,6 +48,9 @@
 EVENTS_PUBLISHER_TAG = "event-disk"
 events_handle = None
 
+DISK_RO_EVENT = "read_only"
+DISK_FULL_EVENT = "disk_full"
+
 chk_log_level = syslog.LOG_ERR
 
 def _log_msg(lvl, pfx, msg):
@@ -73,22 +77,28 @@ def event_pub(event):
     event_publish(events_handle, EVENTS_PUBLISHER_TAG, param_dict)
 
 
+def test_disk_full(dirs): 
+    for d in dirs:
+        space = os.statvfs(d)
+        if space.f_bavail < FREE_SPACE_THRESHOLD:
+            log_err("{} has no free disk space".format(d))
+            event_pub(DISK_FULL_EVENT)
+            return True
+        else:
+            log_debug("{} has enough disk space".format(d))
+
+    return False
+
+
 def test_writable(dirs): 
     for d in dirs:
         rw = os.access(d, os.W_OK)
         if not rw:
             log_err("{} is not read-write".format(d))
-            event_pub("read_only")
+            event_pub(DISK_RO_EVENT)
             return False
         else:
             log_debug("{} is Read-Write".format(d))
-            space = os.statvfs(d)
-            if space.f_bavail < FREE_SPACE_THRESHOLD:
-                log_err("{} has no free disk space".format(d))
-                event_pub("disk_full")
-                return False
-            else:
-                log_debug("{} has enough disk space".format(d))
 
     return True
 
@@ -112,7 +122,7 @@ def get_dname(path_name):
     return os.path.basename(os.path.normpath(path_name))
 
 
-def do_mnt(dirs):
+def do_mnt(dirs, overlay_prefix):
     if os.path.exists(UPPER_DIR):
         log_err("Already mounted")
         return 1
@@ -121,7 +131,7 @@ def do_mnt(dirs):
         try:
             os.mkdir(i)
         except OSError as error:
-            log_err("Failed to create {}".format(i))
+            log_err("Failed to create {}, error: {}".format(i, error))
             return 1
 
     for d in dirs:
@@ -131,7 +141,7 @@ def do_mnt(dirs):
         os.mkdir(d_upper)
         os.mkdir(d_work)
 
-        ret = run_cmd(["mount", "-t", "overlay", "overlay_{}".format(d_name),\
+        ret = run_cmd(["mount", "-t", "overlay", "{}_{}".format(overlay_prefix, d_name),\
                 "-o", "lowerdir={},upperdir={},workdir={}".format(d, d_upper, d_work), d])
         if ret:
             break
@@ -142,14 +152,35 @@ def do_mnt(dirs):
         log_info("{} are mounted as Read-Write".format(dirs))
     return ret
 
+def do_unmnt(dirs, overlay_prefix):
+    for d in dirs:
+        d_name = get_dname(d)
+        ret = run_cmd(["umount", "-l", "{}_{}".format(overlay_prefix, d_name)])
+        if ret:
+            break
+
+    if ret:
+        log_err("Failed to umount {}".format(dirs))
+    else:
+        log_info("{} are unmounted".format(dirs))
+
+    for i in (UPPER_DIR, WORK_DIR):
+        try:
+            shutil.rmtree(i)
+        except OSError as error:
+            log_err("Failed to delete {}, error: {}".format(i, error))
+            return 1
+
+    return ret
+
 
-def is_mounted(dirs):
+def is_mounted(dirs, overlay_prefix):
     if not os.path.exists(UPPER_DIR):
         return False
 
     onames = set()
     for d in dirs:
-        onames.add("overlay_{}".format(get_dname(d)))
+        onames.add("{}_{}".format(overlay_prefix, get_dname(d)))
 
     with open(MOUNTS_FILE, "r") as s:
         for ln in s.readlines():
@@ -164,12 +195,31 @@ def do_check(skip_mount, dirs):
     ret = 0
     if not test_writable(dirs):
         if not skip_mount:
-            ret = do_mnt(dirs)
+            ret = do_mnt(dirs, "overlay")
 
     # Check if mounted
-    if (not ret) and is_mounted(dirs):
+    if (not ret) and is_mounted(dirs, "overlay"):
         log_err("READ-ONLY: Mounted {} to make Read-Write".format(dirs))
-        event_pub("read_only")
+        event_pub(DISK_RO_EVENT)
+
+    if ret:
+        # When disk mounted, disk no free space issue also been fixed.
+        return ret
+
+    # Handle disk no free space case
+    if test_disk_full(dirs):
+        if not skip_mount:
+            ret = do_mnt(dirs, "overlay_disk_full")
+
+    # Check if mounted
+    if (not ret) and is_mounted(dirs, "overlay_disk_full"):
+        log_err("DISK-FULL: Mounted {} to make Read-Write".format(dirs))
+        event_pub(DISK_FULL_EVENT)
+
+    # Unmount when disk space issue fixed
+    if is_mounted(dirs, "overlay_disk_full") and not test_disk_full(["/"]):
+        log_debug("umount for disk space issue fixed")
+        do_unmnt(dirs, "overlay_disk_full")
 
     return ret
 

From c6416c7da970fa4f2d77623243bb6cd8afe24846 Mon Sep 17 00:00:00 2001
From: Hua Liu <58683130+liuh-80@users.noreply.github.com>
Date: Tue, 14 Jan 2025 16:29:22 +0800
Subject: [PATCH 15/22] Fix formatting and add missing newline

---
 scripts/disk_check.py | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/scripts/disk_check.py b/scripts/disk_check.py
index 6e99e0969a..ae128c82f3 100644
--- a/scripts/disk_check.py
+++ b/scripts/disk_check.py
@@ -77,7 +77,7 @@ def event_pub(event):
     event_publish(events_handle, EVENTS_PUBLISHER_TAG, param_dict)
 
 
-def test_disk_full(dirs): 
+def test_disk_full(dirs):
     for d in dirs:
         space = os.statvfs(d)
         if space.f_bavail < FREE_SPACE_THRESHOLD:
@@ -141,7 +141,7 @@ def do_mnt(dirs, overlay_prefix):
         os.mkdir(d_upper)
         os.mkdir(d_work)
 
-        ret = run_cmd(["mount", "-t", "overlay", "{}_{}".format(overlay_prefix, d_name),\
+        ret = run_cmd(["mount", "-t", "overlay", "{}_{}".format(overlay_prefix, d_name),
                 "-o", "lowerdir={},upperdir={},workdir={}".format(d, d_upper, d_work), d])
         if ret:
             break
@@ -152,9 +152,11 @@ def do_mnt(dirs, overlay_prefix):
         log_info("{} are mounted as Read-Write".format(dirs))
     return ret
 
+
 def do_unmnt(dirs, overlay_prefix):
     for d in dirs:
         d_name = get_dname(d)
+
         ret = run_cmd(["umount", "-l", "{}_{}".format(overlay_prefix, d_name)])
         if ret:
             break

From b5eedc5c4ebe0536c08cdf41087e00a603ad069a Mon Sep 17 00:00:00 2001
From: Hua Liu <58683130+liuh-80@users.noreply.github.com>
Date: Tue, 14 Jan 2025 17:12:31 +0800
Subject: [PATCH 16/22] Add mocks for os.access and os.statvfs

---
 tests/disk_check_test.py | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/tests/disk_check_test.py b/tests/disk_check_test.py
index 4c61790d41..57856bac62 100644
--- a/tests/disk_check_test.py
+++ b/tests/disk_check_test.py
@@ -132,7 +132,10 @@ def setup(self):
 
     @patch("disk_check.syslog.syslog")
     @patch("disk_check.subprocess.run")
-    def test_readonly(self, mock_proc, mock_log):
+    @patch('os.access', return_value=True)
+    @patch('os.statvfs', return_value=os.statvfs_result((4096, 4096, 1909350, 1491513, 4096,
+                                                         971520, 883302, 883302, 4096, 255)))
+    def test_readonly(self, mock_os_statvfs, mock_os_access, mock_proc, mock_log):
         global err_data, cmds, max_log_lvl
 
         mock_proc.side_effect = mock_subproc_run
@@ -189,8 +192,8 @@ def test_diskfull(self, mock_os_statvfs, mock_os_access, mock_proc, mock_log):
         mock_proc.side_effect = mock_subproc_run
         mock_log.side_effect = report_err_msg
 
-        result = disk_check.test_writable(["/etc"])
-        assert result is False
+        result = disk_check.test_disk_full(["/etc"])
+        assert result is True
 
 
     @classmethod

From 1b2c7816a159a313638d00ba8816b982fe23a6e6 Mon Sep 17 00:00:00 2001
From: Hua Liu <58683130+liuh-80@users.noreply.github.com>
Date: Tue, 14 Jan 2025 19:10:51 +0800
Subject: [PATCH 17/22] Remove unused mock in disk check test

---
 tests/disk_check_test.py | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/tests/disk_check_test.py b/tests/disk_check_test.py
index 57856bac62..6a1f5f509c 100644
--- a/tests/disk_check_test.py
+++ b/tests/disk_check_test.py
@@ -132,10 +132,9 @@ def setup(self):
 
     @patch("disk_check.syslog.syslog")
     @patch("disk_check.subprocess.run")
-    @patch('os.access', return_value=True)
     @patch('os.statvfs', return_value=os.statvfs_result((4096, 4096, 1909350, 1491513, 4096,
                                                          971520, 883302, 883302, 4096, 255)))
-    def test_readonly(self, mock_os_statvfs, mock_os_access, mock_proc, mock_log):
+    def test_readonly(self, mock_os_statvfs, mock_proc, mock_log):
         global err_data, cmds, max_log_lvl
 
         mock_proc.side_effect = mock_subproc_run

From 4ce667663ca5020634fdf0776a72ac7179d4f0e7 Mon Sep 17 00:00:00 2001
From: Hua Liu <58683130+liuh-80@users.noreply.github.com>
Date: Tue, 14 Jan 2025 20:21:12 +0800
Subject: [PATCH 18/22] Add new disk full and unmount tests

---
 tests/disk_check_test.py | 59 ++++++++++++++++++++++++++++++++++++++++
 1 file changed, 59 insertions(+)

diff --git a/tests/disk_check_test.py b/tests/disk_check_test.py
index 6a1f5f509c..77298e8f2a 100644
--- a/tests/disk_check_test.py
+++ b/tests/disk_check_test.py
@@ -1,4 +1,5 @@
 import os
+import shutil
 import sys
 import syslog
 from unittest.mock import patch
@@ -180,6 +181,44 @@ def test_readonly(self, mock_os_statvfs, mock_proc, mock_log):
             
         assert max_log_lvl == syslog.LOG_ERR
 
+    @patch("disk_check.syslog.syslog")
+    @patch("disk_check.subprocess.run")
+    @patch('os.access', return_value=True)
+    @patch('os.statvfs', return_value=os.statvfs_result((4096, 4096, 1909350, 1491513, 0,
+                                                         971520, 883302, 883302, 4096, 255)))
+    def test_mount_disk_full(self, mock_os_statvfs, mock_os_access, mock_proc, mock_log):
+        global max_log_lvl
+        max_log_lvl = -1
+        mock_proc.side_effect = mock_subproc_run
+        mock_log.side_effect = report_err_msg
+
+        tc = {
+            "upperdir": "/tmp",
+        }
+        swap_upper(tc)
+
+        with patch('sys.argv', ["", "-d", "/tmpx"]):
+            disk_check.main()
+
+    @patch("disk_check.syslog.syslog")
+    @patch("disk_check.subprocess.run")
+    @patch('os.access', return_value=True)
+    @patch('os.statvfs', return_value=os.statvfs_result((4096, 4096, 1909350, 1491513, 4096,
+                                                         971520, 883302, 883302, 4096, 255)))
+    def test_unmount_disk_full(self, mock_os_statvfs, mock_os_access, mock_proc, mock_log):
+        global max_log_lvl
+        max_log_lvl = -1
+        mock_proc.side_effect = mock_subproc_run
+        mock_log.side_effect = report_err_msg
+
+        tc = {
+            "upperdir": "/tmp",
+        }
+        swap_upper(tc)
+
+        with patch('sys.argv', ["", "-d", "/tmpx"]):
+            disk_check.main()
+
     @patch("disk_check.syslog.syslog")
     @patch("disk_check.subprocess.run")
     @patch('os.access', return_value=True)
@@ -194,6 +233,26 @@ def test_diskfull(self, mock_os_statvfs, mock_os_access, mock_proc, mock_log):
         result = disk_check.test_disk_full(["/etc"])
         assert result is True
 
+    @patch("disk_check.syslog.syslog")
+    @patch("disk_check.subprocess.run")
+    def test_do_unmnt(self, mock_proc, mock_log):
+        global max_log_lvl
+        max_log_lvl = -1
+        mock_proc.side_effect = mock_subproc_run
+        mock_log.side_effect = report_err_msg
+
+        disk_check.do_unmnt(["/etc"])
+
+    @patch("disk_check.syslog.syslog")
+    @patch("disk_check.subprocess.run")
+    def test_do_unmnt(self, mock_proc, mock_log):
+        global max_log_lvl
+        max_log_lvl = -1
+        mock_proc.side_effect = mock_subproc_run
+        mock_log.side_effect = report_err_msg
+
+        disk_check.do_unmnt(["/etc"])
+
 
     @classmethod
     def teardown_class(cls):

From e119b7be7159277dbccd98e66e3c8d75e699b20e Mon Sep 17 00:00:00 2001
From: Hua Liu <58683130+liuh-80@users.noreply.github.com>
Date: Tue, 14 Jan 2025 20:32:05 +0800
Subject: [PATCH 19/22] Add shutil.rmtree mock and update test

---
 tests/disk_check_test.py | 17 +++++------------
 1 file changed, 5 insertions(+), 12 deletions(-)

diff --git a/tests/disk_check_test.py b/tests/disk_check_test.py
index 77298e8f2a..8377008e95 100644
--- a/tests/disk_check_test.py
+++ b/tests/disk_check_test.py
@@ -202,19 +202,22 @@ def test_mount_disk_full(self, mock_os_statvfs, mock_os_access, mock_proc, mock_
 
     @patch("disk_check.syslog.syslog")
     @patch("disk_check.subprocess.run")
+    @patch('shutil.rmtree')
     @patch('os.access', return_value=True)
     @patch('os.statvfs', return_value=os.statvfs_result((4096, 4096, 1909350, 1491513, 4096,
                                                          971520, 883302, 883302, 4096, 255)))
-    def test_unmount_disk_full(self, mock_os_statvfs, mock_os_access, mock_proc, mock_log):
+    def test_unmount_disk_full(self, mock_os_statvfs, mock_os_access, mock_rmtree, mock_proc, mock_log):
         global max_log_lvl
         max_log_lvl = -1
         mock_proc.side_effect = mock_subproc_run
         mock_log.side_effect = report_err_msg
 
         tc = {
-            "upperdir": "/tmp",
+            "upperdir": "/tmp/tmpx",
+            "workdir": "/tmp/tmpy"
         }
         swap_upper(tc)
+        swap_work(tc)
 
         with patch('sys.argv', ["", "-d", "/tmpx"]):
             disk_check.main()
@@ -243,16 +246,6 @@ def test_do_unmnt(self, mock_proc, mock_log):
 
         disk_check.do_unmnt(["/etc"])
 
-    @patch("disk_check.syslog.syslog")
-    @patch("disk_check.subprocess.run")
-    def test_do_unmnt(self, mock_proc, mock_log):
-        global max_log_lvl
-        max_log_lvl = -1
-        mock_proc.side_effect = mock_subproc_run
-        mock_log.side_effect = report_err_msg
-
-        disk_check.do_unmnt(["/etc"])
-
 
     @classmethod
     def teardown_class(cls):

From 52a715e5b71c2608abdccee9b375af436f6a102a Mon Sep 17 00:00:00 2001
From: Hua Liu <58683130+liuh-80@users.noreply.github.com>
Date: Tue, 14 Jan 2025 21:48:38 +0800
Subject: [PATCH 20/22] Update do_unmnt test with overlay_prefix

---
 tests/disk_check_test.py | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/tests/disk_check_test.py b/tests/disk_check_test.py
index 8377008e95..5a6c60dd5a 100644
--- a/tests/disk_check_test.py
+++ b/tests/disk_check_test.py
@@ -1,5 +1,4 @@
 import os
-import shutil
 import sys
 import syslog
 from unittest.mock import patch
@@ -244,7 +243,7 @@ def test_do_unmnt(self, mock_proc, mock_log):
         mock_proc.side_effect = mock_subproc_run
         mock_log.side_effect = report_err_msg
 
-        disk_check.do_unmnt(["/etc"])
+        disk_check.do_unmnt(["/etc"], "overlay_prefix")
 
 
     @classmethod

From 0b77af007ce4d6196780d754c1ebf1600ac1c016 Mon Sep 17 00:00:00 2001
From: Hua Liu <58683130+liuh-80@users.noreply.github.com>
Date: Fri, 7 Feb 2025 14:41:37 +0800
Subject: [PATCH 21/22] Update disk_check.py to handle disk space issues

---
 scripts/disk_check.py | 24 +++++++++++++++---------
 1 file changed, 15 insertions(+), 9 deletions(-)

diff --git a/scripts/disk_check.py b/scripts/disk_check.py
index ae128c82f3..763a08a1d6 100644
--- a/scripts/disk_check.py
+++ b/scripts/disk_check.py
@@ -3,28 +3,34 @@
 
 """
 What:
-    There have been cases, where disk turns Read-only due to kernel bug.
-    In Read-only state, system blocks new remote user login via TACACS.
-    This utility is to check & make transient recovery as needed.
+    This utility is designed to address two specific issues:
+    1. Disk becoming read-only due to kernel bugs.
+    2. Disk running out of space.
+    When either of these issues occurs, the system prevents new remote user logins via TACACS. 
 
 How:
-    check for Read-Write permission. If Read-only, create writable overlay using tmpfs.
+    Checks for read-write permissions and available disk space.
+    If an issue is detected, create writable overlay using tmpfs.
 
-    By default "/etc" & "/home" are checked and if in Read-only state, make them Read-Write
+    By default "/etc" & "/home" are checked and if issue detected, make them Read-Write
     using overlay on top of tmpfs.
 
     Making /etc & /home as writable lets successful new remote user login.
 
-    If in Read-only state or in Read-Write state with the help of tmpfs overlay,
-    syslog ERR messages are written, to help raise alerts.
+    Write syslog ERR messages to help raise alerts in the following cases:
+    1. Disk in read-only state.
+    2. Disk out of space.
+    3. Mounted tmpfs overlay.
 
     Monit may be used to invoke it periodically, to help scan & fix and
     report via syslog.
 
 Tidbit:
-    If you would like to test this script, you could simulate a RO disk
-    with the following command. Reboot will revert the effect.
+    To test this script:
+    1. Simulate a RO disk with the following command. Reboot will revert the effect.
         sudo bash -c "echo u > /proc/sysrq-trigger"
+    2. Use up all disk space by create big file in /var/dump/:
+         dd if=/dev/zero of=/var/dump/sonic_dump_devicename_20241126_204132.tar bs=1G count=50
 
 """
 

From 05a4a6b56e8c70aac3f7f6f997670c03303bfec7 Mon Sep 17 00:00:00 2001
From: Hua Liu <58683130+liuh-80@users.noreply.github.com>
Date: Fri, 7 Feb 2025 14:44:11 +0800
Subject: [PATCH 22/22] Remove trailing whitespace in disk_check.py

---
 scripts/disk_check.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scripts/disk_check.py b/scripts/disk_check.py
index 763a08a1d6..459f5d4b10 100644
--- a/scripts/disk_check.py
+++ b/scripts/disk_check.py
@@ -6,7 +6,7 @@
     This utility is designed to address two specific issues:
     1. Disk becoming read-only due to kernel bugs.
     2. Disk running out of space.
-    When either of these issues occurs, the system prevents new remote user logins via TACACS. 
+    When either of these issues occurs, the system prevents new remote user logins via TACACS.
 
 How:
     Checks for read-write permissions and available disk space.