diff --git a/changelog/pressure.feature b/changelog/pressure.feature new file mode 100644 index 0000000..457a914 --- /dev/null +++ b/changelog/pressure.feature @@ -0,0 +1 @@ +add pressure data for cpu, io and memory for linux and android OS. diff --git a/docs/source/tutorial.rst b/docs/source/tutorial.rst index 8b019cc..0803884 100644 --- a/docs/source/tutorial.rst +++ b/docs/source/tutorial.rst @@ -326,3 +326,27 @@ The following is showing a continuous measurement: result = handler.stop_net_interface_measurement() The result is then a list of handler-specific network information models. It is the same as the single measurement, except that it is a list of measurement. + + +Example97: Pulling pressure data (cpu, io, memory): +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. code-block:: python + from src.clients import SSHClient + from src.handlers import LinuxHandler + + with SSHClient("127.0.0.1", port=22, username="root", password="root") as client: + handler = LinuxHandler(client) + result = handler.get_pressure() + +The result is a model of pressure information for cpu, io and memory respectively. This is also possible to do continuously: + +.. code-block:: python + + with SSHClient("127.0.0.1", port=22, username="root", password="root") as client: + handler = LinuxHandler(client) + handler.start_pressure_measurement(0.1) + time.sleep(1) + result = handler.stop_pressure_measurement() + +And will return a list of the objects obtained by get_pressure, together with some helpful properties. diff --git a/integration_tests/tests/test_ssh_client.py b/integration_tests/tests/test_ssh_client.py index 8e9f761..68bcb1c 100644 --- a/integration_tests/tests/test_ssh_client.py +++ b/integration_tests/tests/test_ssh_client.py @@ -16,7 +16,7 @@ SystemMemory, SystemUptimeInfo, ) -from remoteperf.models.linux import LinuxCpuUsageInfo, LinuxResourceSample +from remoteperf.models.linux import LinuxCpuUsageInfo, LinuxPressureInfo, LinuxResourceSample from remoteperf.models.super import DiskInfoList, DiskIOList, ProcessDiskIOList, ProcessInfo @@ -355,3 +355,20 @@ def test_diskio_proc_wise(ssh_client): assert output assert isinstance(output, ProcessDiskIOList) assert all((isinstance(model, ProcessInfo) for model in output)) + + +def test_ssh_get_pressure(ssh_client): + handler = LinuxHandler(ssh_client) + output = handler.get_pressure() + assert output + assert isinstance(output, LinuxPressureInfo) + + +def test_ssh_continuous_get_pressure(ssh_client): + handler = LinuxHandler(ssh_client) + handler.start_pressure_measurement(interval=0.1) + time.sleep(1) + output = handler.stop_pressure_measurement() + assert output + assert output.cpu.full[0] + assert all((isinstance(model, LinuxPressureInfo) for model in output)) diff --git a/remoteperf/_parsers/linux.py b/remoteperf/_parsers/linux.py index d29b723..06cb8c6 100644 --- a/remoteperf/_parsers/linux.py +++ b/remoteperf/_parsers/linux.py @@ -352,3 +352,27 @@ def parse_net_deltadata(net_sample: dict) -> List[LinuxNetworkInterfaceDeltaSamp ) return list_of_interfaces + + +def parse_pressure(raw_data: str, types: Tuple[str], separator: str, timestamp=None) -> dict: + if timestamp is None: + timestamp = datetime.now() + result = {} + for metric, split_data in zip((types), raw_data.split(separator)): + presssure_pattern = r"(\w+)\s+avg10=(\d+.\d+)\s+avg60=(\d+.\d+)\s+avg300=(\d+.\d+)\s+total=(\d+)" + parameters = ["name", "avg10", "avg60", "avg300", "total"] + + matches = re.findall(presssure_pattern, split_data) + if not len(matches) == 2: + raise ParsingError(f"Could not parse {metric} pressure data from: {raw_data}") + result[metric] = {} + for match in matches: + named_result = dict(zip(parameters, match)) + name = named_result.pop("name", None) + result[metric][name] = named_result + result[metric][name]["timestamp"] = timestamp + + if not result: + raise ParsingError(f"Could not parse pressure data from: {raw_data}") + + return result diff --git a/remoteperf/handlers/base_linux_handler.py b/remoteperf/handlers/base_linux_handler.py index 7e5d50a..38856e3 100644 --- a/remoteperf/handlers/base_linux_handler.py +++ b/remoteperf/handlers/base_linux_handler.py @@ -19,7 +19,7 @@ SystemMemory, SystemUptimeInfo, ) -from remoteperf.models.linux import LinuxCpuUsageInfo, LinuxResourceSample +from remoteperf.models.linux import LinuxCpuUsageInfo, LinuxPressureInfo, LinuxPressureInfoList, LinuxResourceSample from remoteperf.models.super import ( CpuList, DiskInfoList, @@ -109,6 +109,13 @@ def get_system_uptime(self) -> SystemUptimeInfo: output = self._client.run_command(command) return SystemUptimeInfo(total=float(output)) + def get_pressure(self) -> LinuxPressureInfo: + result = linux_parsers.parse_pressure( + self._pressure_measurement(), ("cpu", "io", "memory"), self._nonexistant_separator_file + ) + print(result) + return LinuxPressureInfo(**result) + def get_diskinfo(self) -> DiskInfoList: output = linux_parsers.parse_df(self._get_df()) return DiskInfoList([DiskInfo(**kpis) for _, kpis in output.items()]) @@ -155,6 +162,20 @@ def stop_mem_measurement(self) -> MemoryList: ) return MemoryList(parsed_results) + def start_pressure_measurement(self, interval: float) -> None: + self._start_measurement(self._pressure_measurement, interval) + + def stop_pressure_measurement(self) -> LinuxPressureInfoList: + result, _ = self._stop_measurement(self._pressure_measurement) + parsed_result = [ + LinuxPressureInfo( + **linux_parsers.parse_pressure(sample.data, ("cpu", "io", "memory"), self._nonexistant_separator_file) + ) + for sample in result + ] + + return LinuxPressureInfoList(parsed_result) + def start_diskinfo_measurement(self, interval: float) -> None: self._start_measurement(self._get_df, interval) @@ -318,3 +339,11 @@ def _get_df(self, **_): def _net_measurement(self, **_) -> dict: command = "cat /proc/net/dev && date --iso-8601=ns" return linux_parsers.parse_proc_net_dev(self._client.run_command(command)) + + def _pressure_measurement(self, **_): + command = ( + f"cat /proc/pressure/cpu {self._nonexistant_separator_file} " + f"/proc/pressure/io {self._nonexistant_separator_file} " + f"/proc/pressure/memory 2>&1" + ) + return self._client.run_command(command) diff --git a/remoteperf/models/base.py b/remoteperf/models/base.py index a161c6a..5c8ddf0 100644 --- a/remoteperf/models/base.py +++ b/remoteperf/models/base.py @@ -254,8 +254,8 @@ class BaseNetworkTranceiveDeltaSample(BasePacketData): @attrs_init_replacement @attr.s(auto_attribs=True, kw_only=True) class BaseNetworkInterfaceDeltaSampleList(BaseNetworkInterfaceSample): - receive: List[BasePacketData] - transmit: List[BasePacketData] + receive: List[BaseNetworkInterfaceDeltaSample] + transmit: List[BaseNetworkInterfaceDeltaSample] @property def transceive(self) -> List[BaseNetworkTranceiveDeltaSample]: diff --git a/remoteperf/models/linux.py b/remoteperf/models/linux.py index 49be1da..a7087bc 100644 --- a/remoteperf/models/linux.py +++ b/remoteperf/models/linux.py @@ -6,10 +6,12 @@ import attr from remoteperf.models.base import ( + ArithmeticBaseInfoModel, BaseCpuSample, BaseCpuUsageInfo, BaseRemoteperfModel, BootTimeInfo, + ModelList, ) from remoteperf.utils.attrs_util import attrs_init_replacement @@ -47,3 +49,56 @@ class LinuxResourceSample(BaseCpuSample): @attr.s(auto_attribs=True, kw_only=True) class LinuxBootTimeInfo(BootTimeInfo): extra: Dict[str, float] + + +@attrs_init_replacement +@attr.s(auto_attribs=True, kw_only=True) +class LinuxPressureModel(ArithmeticBaseInfoModel): + total: int + avg10: float + avg60: float + avg300: float + + +class LinuxPressureModelList(ModelList[LinuxPressureModel]): + def highest_pressure(self, n: int = 5) -> "LinuxPressureModelList": + return LinuxPressureModelList(sorted(self, key=lambda m: m.avg10, reverse=True)[:n]) + + +@attrs_init_replacement +@attr.s(auto_attribs=True, kw_only=True) +class PressureMetricInfo(BaseRemoteperfModel): + some: LinuxPressureModel + full: LinuxPressureModel + + +class LinuxPressureMetricInfoList(ModelList[PressureMetricInfo]): + @property + def some(self) -> LinuxPressureModelList: + return LinuxPressureModelList((model.some for model in self)) + + @property + def full(self) -> LinuxPressureModelList: + return LinuxPressureModelList((model.full for model in self)) + + +@attrs_init_replacement +@attr.s(auto_attribs=True, kw_only=True) +class LinuxPressureInfo(BaseRemoteperfModel): + cpu: PressureMetricInfo + io: PressureMetricInfo + memory: PressureMetricInfo + + +class LinuxPressureInfoList(ModelList[LinuxPressureInfo]): + @property + def cpu(self) -> LinuxPressureMetricInfoList: + return LinuxPressureMetricInfoList((model.cpu for model in self)) + + @property + def io(self) -> LinuxPressureMetricInfoList: + return LinuxPressureMetricInfoList((model.io for model in self)) + + @property + def memory(self) -> LinuxPressureMetricInfoList: + return LinuxPressureMetricInfoList((model.memory for model in self)) diff --git a/remoteperf/utils/attrs_util.py b/remoteperf/utils/attrs_util.py index 85459eb..702a6b8 100644 --- a/remoteperf/utils/attrs_util.py +++ b/remoteperf/utils/attrs_util.py @@ -10,7 +10,6 @@ # Could not figure out how to make typing work, so we do this workaround def attrs_init_replacement(cls=None): def init(self, **kwargs): - for attr in fields(cls): if not hasattr(self, attr.name): value = kwargs.get(attr.name) if attr.name in kwargs else attr.default diff --git a/tests/data/valid_handler_data.yaml b/tests/data/valid_handler_data.yaml index 0ced070..73a13fc 100644 --- a/tests/data/valid_handler_data.yaml +++ b/tests/data/valid_handler_data.yaml @@ -444,3 +444,33 @@ systemd-analyze: | Short packets .............................. 0 vlan1: + + +"cat /proc/pressure*": + - | + some avg10=0.13 avg60=0.13 avg300=0.13 total=205171290 + full avg10=0.00 avg60=0.00 avg300=0.00 total=0 + /usr/bin/cat: e39f7761903b: No such file or directory + some avg10=0.00 avg60=0.00 avg300=0.00 total=22426415 + full avg10=0.00 avg60=0.00 avg300=0.00 total=18933747 + /usr/bin/cat: e39f7761903b: No such file or directory + some avg10=0.00 avg60=0.00 avg300=0.00 total=923912 + full avg10=0.00 avg60=0.00 avg300=0.00 total=898585 + - | + some avg10=0.00 avg60=0.04 avg300=0.10 total=205817451 + full avg10=0.00 avg60=0.00 avg300=0.00 total=0 + /usr/bin/cat: e39f7761903b: No such file or directory + some avg10=0.00 avg60=0.00 avg300=0.00 total=22484643 + full avg10=0.00 avg60=0.00 avg300=0.00 total=18986463 + /usr/bin/cat: e39f7761903b: No such file or directory + some avg10=0.00 avg60=0.00 avg300=0.00 total=923912 + full avg10=0.00 avg60=0.00 avg300=0.00 total=898585 + - | + some avg10=0.32 avg60=0.10 avg300=0.11 total=205946054 + full avg10=1.00 avg60=0.00 avg300=0.00 total=0 + /usr/bin/cat: e39f7761903b: No such file or directory + some avg10=0.00 avg60=0.00 avg300=0.00 total=22492086 + full avg10=1.00 avg60=0.00 avg300=0.00 total=18992689 + /usr/bin/cat: e39f7761903b: No such file or directory + some avg10=0.00 avg60=0.00 avg300=0.00 total=923912 + full avg10=1.00 avg60=0.00 avg300=0.00 total=898585 diff --git a/tests/test_linux_handler.py b/tests/test_linux_handler.py index e5c3efa..7ad4f38 100644 --- a/tests/test_linux_handler.py +++ b/tests/test_linux_handler.py @@ -565,3 +565,78 @@ def test_network_calc_avg_transmit_rate(linux_handler): desired_avg_transmit_output = {"lo": 2.0, "eth0": 5.0} for interface in output: assert interface.avg_transmit_rate == desired_avg_transmit_output[interface.name] + + +def test_get_pressure(linux_handler): + result = linux_handler.get_pressure() + desired_cpu = { + "full": {"avg10": 0.0, "avg300": 0.0, "avg60": 0.0, "total": 0}, + "some": {"avg10": 0.13, "avg300": 0.13, "avg60": 0.13, "total": 205171290}, + } + + desired_io = { + "full": {"avg10": 0.0, "avg300": 0.0, "avg60": 0.0, "total": 18933747}, + "some": {"avg10": 0.0, "avg300": 0.0, "avg60": 0.0, "total": 22426415}, + } + desired_memory = { + "full": {"avg10": 0.0, "avg300": 0.0, "avg60": 0.0, "total": 898585}, + "some": {"avg10": 0.0, "avg300": 0.0, "avg60": 0.0, "total": 923912}, + } + assert result.cpu.model_dump(exclude="timestamp") == desired_cpu + assert result.io.model_dump(exclude="timestamp") == desired_io + assert result.memory.model_dump(exclude="timestamp") == desired_memory + + +def test_get_continuous_pressure(linux_handler): + linux_handler.start_pressure_measurement(0.1) + time.sleep(0.3) + result = linux_handler.stop_pressure_measurement() + desired_cpu = [ + { + "some": {"total": 205171290, "avg10": 0.13, "avg60": 0.13, "avg300": 0.13}, + "full": {"total": 0, "avg10": 0.0, "avg60": 0.0, "avg300": 0.0}, + }, + { + "some": {"total": 205817451, "avg10": 0.0, "avg60": 0.04, "avg300": 0.1}, + "full": {"total": 0, "avg10": 0.0, "avg60": 0.0, "avg300": 0.0}, + }, + ] + + desired_io = [ + { + "some": {"total": 22426415, "avg10": 0.0, "avg60": 0.0, "avg300": 0.0}, + "full": {"total": 18933747, "avg10": 0.0, "avg60": 0.0, "avg300": 0.0}, + }, + { + "some": {"total": 22484643, "avg10": 0.0, "avg60": 0.0, "avg300": 0.0}, + "full": {"total": 18986463, "avg10": 0.0, "avg60": 0.0, "avg300": 0.0}, + }, + ] + desired_memory = [ + { + "some": {"total": 923912, "avg10": 0.0, "avg60": 0.0, "avg300": 0.0}, + "full": {"total": 898585, "avg10": 0.0, "avg60": 0.0, "avg300": 0.0}, + }, + { + "some": {"total": 923912, "avg10": 0.0, "avg60": 0.0, "avg300": 0.0}, + "full": {"total": 898585, "avg10": 0.0, "avg60": 0.0, "avg300": 0.0}, + }, + ] + + assert result.cpu.model_dump(exclude="timestamp")[:2] == desired_cpu + assert result.io.model_dump(exclude="timestamp")[:2] == desired_io + assert result.memory.model_dump(exclude="timestamp")[:2] == desired_memory + + +def test_get_highest_continuous_pressure(linux_handler): + linux_handler.start_pressure_measurement(0.1) + time.sleep(0.3) + result = linux_handler.stop_pressure_measurement() + + desired_cpu = {"total": 0, "avg10": 1.0, "avg60": 0.0, "avg300": 0.0} + desired_io = {"total": 18992689, "avg10": 1.0, "avg60": 0.0, "avg300": 0.0} + desired_memory = {"total": 898585, "avg10": 1.0, "avg60": 0.0, "avg300": 0.0} + + assert result.cpu.full.highest_pressure(1)[0].model_dump(exclude="timestamp") == desired_cpu + assert result.io.full.highest_pressure(1)[0].model_dump(exclude="timestamp") == desired_io + assert result.memory.full.highest_pressure(1)[0].model_dump(exclude="timestamp") == desired_memory diff --git a/tests/test_qnx_handler.py b/tests/test_qnx_handler.py index 0ce16d9..2f10a01 100644 --- a/tests/test_qnx_handler.py +++ b/tests/test_qnx_handler.py @@ -369,7 +369,8 @@ def test_network_calc_avg_transceive_rate(qnx_handler): output = qnx_handler.stop_net_interface_measurement() desired_avg_transceive_output = {"eq0": 30.0, "vlan0": 90.0} for interface in output: - assert interface.avg_transceive_rate == desired_avg_transceive_output[interface.name] + assert interface.avg_transceive_rate > desired_avg_transceive_output[interface.name] / 1.05 + assert interface.avg_transceive_rate < desired_avg_transceive_output[interface.name] * 1.05 def test_network_calc_avg_receive_rate(qnx_handler):