feat: add disk measurements

changelog/disks.feature

@@ -0,0 +1 @@
+Adding support for pulling disk information, usage both atomic and process wise. All measure are implemented for Linux. For QNX only disk info is implemneted.

docs/source/conf.py

@@ -19,7 +19,7 @@
 sys.path.insert(0, os.path.abspath("."))
 sys.path.insert(0, os.path.abspath(os.path.join("..", "..")))
 
-from src import __version__  # noqa: E402
+from remoteperf import __version__  # noqa: E402
 
 # -- Project information -----------------------------------------------------
 

docs/source/tutorial.rst

@@ -22,8 +22,8 @@ To communicate with a remote host, a handler is needed to specify which operatin
 
 .. code-block:: python
 
-    from src.clients import SSHClient
-    from src.handlers import LinuxHandler
+    from remoteperf.clients import SSHClient
+    from remoteperf.handlers import LinuxHandler
 
     with SSHClient("127.0.0.1", port=22, username="root", password="root") as instance:
         handler = LinuxHandler(instance)
@@ -110,8 +110,8 @@ The SSH client specifically also supports jump-posting:
 
 .. code-block:: python
 
-    from src.clients import SSHClient
-    from src.handlers import LinuxHandler
+    from remoteperf.clients import SSHClient
+    from remoteperf.handlers import LinuxHandler
 
     with SSHClient("127.0.0.1", port=22, username="root", password="root") as jump_client1:
         with SSHClient("host2", port=22, username="root", password="root", jump_client=jump_client1) as jump_client2:
@@ -125,8 +125,8 @@ Example 2: Basic Android Usage:
 
 .. code-block:: python
 
-    from src.clients import ADBClient
-    from src.handlers import AndroidHandler
+    from remoteperf.clients import ADBClient
+    from remoteperf.handlers import AndroidHandler
 
     with ADBClient(device_id=...) as instance:
         handler = AndroidHandler(instance)
@@ -147,8 +147,8 @@ Example 3: Basic QNX Usage:
 
 .. code-block:: python
 
-    from src.clients import SSHClient
-    from src.handlers import QNXHandler
+    from remoteperf.clients import SSHClient
+    from remoteperf.handlers import QNXHandler
 
     with SSHClient("127.0.0.1", port=22, username="root", password="root")  as instance:
         handler = QNXHandler(instance)
@@ -174,8 +174,8 @@ Example 4: Continuous background measurement:
 
 .. code-block:: python
 
-    from src.clients import SSHClient
-    from src.handlers import LinuxHandler
+    from remoteperf.clients import SSHClient
+    from remoteperf.handlers import LinuxHandler
 
     with SSHClient("127.0.0.1", port=22, username="root", password="root") as instance:
         handler = LinuxHandler(instance)
@@ -211,8 +211,8 @@ Example 5: Processwise Resource Measurement:
 
 .. code-block:: python
 
-    from src.clients import SSHClient
-    from src.handlers import LinuxHandler
+    from remoteperf.clients import SSHClient
+    from remoteperf.handlers import LinuxHandler
 
     with SSHClient("127.0.0.1", port=22, username="root", password="root") as client:
         handler = LinuxHandler(client)
@@ -240,8 +240,8 @@ Example 6: Continuous Processwise Resource Measurement:
 
 .. code-block:: python
 
-    from src.clients import SSHClient
-    from src.handlers import LinuxHandler
+    from remoteperf.clients import SSHClient
+    from remoteperf.handlers import LinuxHandler
 
     with SSHClient("127.0.0.1", port=22, username="root", password="root") as client:
         handler = LinuxHandler(client)
@@ -261,3 +261,35 @@ Example result (for a docker container running only sshd):
         ProcessInfo(pid=2769 name='sshd' command='sshd: root@notty' start_time='2507319' samples=[LinuxResourceSample(timestamp=datetime.datetime(2024, 8, 29, 15, 55, 57, 465394), mem_usage=8116.0, cpu_load=0.0), LinuxResourceSample(timestamp=datetime.datetime(2024, 8, 29, 15, 55, 58, 760324), mem_usage=8116.0, cpu_load=0.08), LinuxResourceSample(timestamp=datetime.datetime(2024, 8, 29, 15, 55, 59, 459172), mem_usage=8116.0, cpu_load=0.0)])
     ]
 
+
+
+Example 7: Disk Info and IO:
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+.. code-block:: python
+
+    from remoteperf.clients import SSHClient
+    from remoteperf.handlers import LinuxHandler
+
+    with SSHClient("127.0.0.1", port=22, username="root", password="root") as client:
+        handler = LinuxHandler(client)
+        handler.get_diskinfo()
+        handler.get_diskio()
+        handler.get_diskio_proc_wise()
+
+The result is then a list of handler-specific disk information models. Info pulls the current available storage, usage pulls the current io on the disks and usage_proc_wise pulls pulls read and write access by each process.
+All these measures can also be pulled continuously.
+
+.. code-block:: python
+
+    handler.start_disk_info_measurement(0.1)
+    time.sleep(1)
+    result = handler.stop_disk_info_measurement()
+
+    handler.start_diskio_measurement(0.1)
+    time.sleep(1)
+    result = handler.stop_diskio_measurement()
+
+    handler.start_diskio_measurement_proc_wise(0.1)
+    time.sleep(1)
+    result = handler.stop_diskio_measurement_proc_wise()

docs/source/tutorial_math.rst

@@ -8,15 +8,15 @@ Remoteperf is not a mathematical library nor does it intend to be, so for more c
 large scale computations we advice you to export data first and and do them in a more suitable
 environment. However, it does support a few basic operations on all objects returned by
 all continuous measurement functions, and if you feel some might be missing you're more than
-welcome to contribute by extending the existing list wrappers in  ``src/models/super.py``.
+welcome to contribute by extending the existing list wrappers in  ``remoteperf/models/super.py``.
 
 Example 1: Highest average load/usage
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
 .. code-block:: python
 
-    from src.clients import ADBClient
-    from src.handlers import AndroidHandler
+    from remoteperf.clients import ADBClient
+    from remoteperf.handlers import AndroidHandler
 
     with ADBClient(device_id="emulator-5554") as instance:
         handler = AndroidHandler(instance)
@@ -75,8 +75,8 @@ Example 2: Highest average priocess-wise load/usage
 
 .. code-block:: python
 
-    from src.clients import ADBClient
-    from src.handlers import AndroidHandler
+    from remoteperf.clients import ADBClient
+    from remoteperf.handlers import AndroidHandler
 
     with ADBClient(device_id="emulator-5554") as instance:
         handler = AndroidHandler(instance)
@@ -240,8 +240,8 @@ Sorting, filtering, and querying can be combined in an advanced manner as such:
 
 .. code-block:: python
 
-    from src.clients import ADBClient
-    from src.handlers import AndroidHandler
+    from remoteperf.clients import ADBClient
+    from remoteperf.handlers import AndroidHandler
 
     with ADBClient(device_id="emulator-5554") as instance:
         handler = AndroidHandler(instance)

docs/source/tutorial_remotefs.rst

@@ -14,8 +14,8 @@ To check wheteher a file exists, the following functions can be used:
 
 .. code-block:: python
 
-    from src.clients import SSHClient
-    from src.handlers import QNXHandler
+    from remoteperf.clients import SSHClient
+    from remoteperf.handlers import QNXHandler
 
     with SSHClient("127.0.0.1", port=22, username="root", password="root") as instance:
         handler = QNXHandler(instance, log_path="/tmp/core")
@@ -30,8 +30,8 @@ In addition, it also allows for removal of files and directory:
 
 .. code-block:: python
 
-    from src.clients import SSHClient
-    from src.utils.fs_utils import RemoteFS
+    from remoteperf.clients import SSHClient
+    from remoteperf.utils.fs_utils import RemoteFS
 
     with SSHClient("127.0.0.1", port=22, username="root", password="root") as instance:
         fs = RemoteFS(instance)
@@ -47,8 +47,8 @@ There is also support for remote temporary directories:
 
 .. code-block:: python
 
-    from src.clients import SSHClient
-    from src.handlers import LinuxHandler
+    from remoteperf.clients import SSHClient
+    from remoteperf.handlers import LinuxHandler
 
     with SSHClient("127.0.0.1", port=22, username="root", password="root") as instance:
         handler = LinuxHandler(instance)

integration_tests/image/mocks/nicinfo

@@ -0,0 +1,30 @@
+
+#!/bin/bash
+nicinfo_output='eq0:\n
+   Ethernet Controller\n
+\n
+  Physical Node ID ........................... 000000 000000\n
+  Current Physical Node ID ................... 000001 000001\n
+  Current Operation Rate ..................... 2.50 Mb/s full-duplex\n
+  Active Interface Type ...................... SMI\n
+    Active PHY address ....................... 0\n
+  Maximum Transmittable data Unit ............ 0\n
+  Maximum Receivable data Unit ............... 0\n
+  Promiscuous Mode ........................... Off\n
+  Multicast Support .......................... Enabled\n
+\n
+  Packets Transmitted OK ..................... 9225590\n
+  Bytes Transmitted OK ....................... 1366702382\n
+  Broadcast Packets Transmitted OK ........... 19\n
+  Multicast Packets Transmitted OK ........... 8671\n
+\n
+  Packets Received OK ........................ 10786584\n
+  Bytes Received OK .......................... 1198846300\n
+  Broadcast Packets Received OK .............. 7634\n
+  Multicast Packets Received OK .............. 80894\n
+\n
+  Received packets with CRC errors ........... 0\n
+  Short packets .............................. 0\n
+\n
+vlan0:'
+echo -e $nicinfo_output

integration_tests/tests/conftest.py

@@ -7,8 +7,8 @@
 
 import pytest
 
-from src.clients.adb_client import ADBClient
-from src.clients.ssh_client import SSHClient
+from remoteperf.clients.adb_client import ADBClient
+from remoteperf.clients.ssh_client import SSHClient
 
 
 def singleton(cls):

integration_tests/tests/test_adb_client.py

@@ -5,13 +5,13 @@
 
 import pytest
 
-from src.clients.adb_client import ADBClientException
-from src.clients.base_client import BaseClientException
-from src.handlers.android_handler import AndroidHandler
-from src.handlers.qnx_handler import MissingQnxCapabilityException, QNXHandler
-from src.models.base import BaseCpuSample, BaseMemorySample, SystemMemory
-from src.models.linux import LinuxCpuUsageInfo, LinuxResourceSample
-from src.models.super import ProcessInfo
+from remoteperf.clients.adb_client import ADBClientException
+from remoteperf.clients.base_client import BaseClientException
+from remoteperf.handlers.android_handler import AndroidHandler
+from remoteperf.handlers.qnx_handler import MissingQnxCapabilityException, QNXHandler
+from remoteperf.models.base import BaseCpuSample, BaseMemorySample, SystemMemory
+from remoteperf.models.linux import LinuxCpuUsageInfo, LinuxResourceSample
+from remoteperf.models.super import ProcessInfo
 
 
 def kill_adb_server():

integration_tests/tests/test_fs_utils.py

@@ -1,4 +1,4 @@
-from src.handlers.linux_handler import LinuxHandler
+from remoteperf.handlers.linux_handler import LinuxHandler
 
 
 def test_remove_file(ssh_client, random_name):

integration_tests/tests/test_ssh_client.py

@@ -5,20 +5,19 @@
 import pytest
 from conftest import override_client_config
 
-from src.clients.base_client import BaseClientException
-from src.clients.ssh_client import SSHClient, SSHClientException
-from src.handlers.linux_handler import LinuxHandler
-from src.models.base import (
+from remoteperf.clients.base_client import BaseClientException
+from remoteperf.clients.ssh_client import SSHClient, SSHClientException
+from remoteperf.handlers.linux_handler import LinuxHandler
+from remoteperf.models.base import (
     BaseCpuSample,
     BaseCpuUsageInfo,
     BaseMemorySample,
     BootTimeInfo,
-    Command,
     SystemMemory,
     SystemUptimeInfo,
 )
-from src.models.linux import LinuxCpuUsageInfo, LinuxResourceSample
-from src.models.super import ProcessInfo
+from remoteperf.models.linux import LinuxCpuUsageInfo, LinuxResourceSample
+from remoteperf.models.super import DiskInfoList, DiskIOList, ProcessDiskIOList, ProcessInfo
 
 
 def close_transport(client: SSHClient):
@@ -217,13 +216,13 @@ def test_ssh_pull_directory(ssh_client):
 
 
 def test_ssh_pull_dir(ssh_client):
-    path = f"/tmp"
+    path = "/tmp"
     with pytest.raises(BaseClientException):
         ssh_client.pull_file("/usr/bin", path)
 
 
 def test_ssh_pull_imaginary_dir(ssh_client):
-    path = f"/sdfgsdf/sfgdh"
+    path = "/sdfgsdf/sfgdh"
     with pytest.raises(BaseClientException):
         ssh_client.pull_file("/usr/bin/xz", path)
 
@@ -334,3 +333,25 @@ def test_ssh_client_retry(ssh_client):
         with pytest.raises(SSHClientException):
             client.run_command("sleep 1.5")
         assert time.time() - t0 > 3 and time.time() - t0 < 7
+
+
+def test_diskinfo(ssh_client):
+    handler = LinuxHandler(ssh_client)
+    output = handler.get_diskinfo()
+    assert output
+    assert isinstance(output, DiskInfoList)
+
+
+def test_diskio(ssh_client):
+    handler = LinuxHandler(ssh_client)
+    output = handler.get_diskio()
+    assert output
+    assert isinstance(output, DiskIOList)
+
+
+def test_diskio_proc_wise(ssh_client):
+    handler = LinuxHandler(ssh_client)
+    output = handler.get_diskio_proc_wise()
+    assert output
+    assert isinstance(output, ProcessDiskIOList)
+    assert all((isinstance(model, ProcessInfo) for model in output))