From 29e7c3aa3c9dcaab398750016cf985f7fadeb048 Mon Sep 17 00:00:00 2001 From: James Souter Date: Tue, 27 Aug 2024 08:21:16 +0100 Subject: [PATCH] Add FrameReceiver controllers and tests Move FrameProccesor controllers to odin_data.py --- dev/one_node_fp/odin_server.cfg | 2 +- dev/two_node_fp/odin_server.cfg | 2 +- src/odin_fastcs/__main__.py | 4 +- src/odin_fastcs/frame_receiver.py | 5 - src/odin_fastcs/odin_adapter_controller.py | 4 +- src/odin_fastcs/odin_controller.py | 9 +- .../{frame_processor.py => odin_data.py} | 110 +++++++++---- tests/input/two_node_fr_response.json | 154 ++++++++++++++++++ tests/test_controllers.py | 51 +++++- tests/test_introspection.py | 40 ++++- 10 files changed, 331 insertions(+), 50 deletions(-) delete mode 100644 src/odin_fastcs/frame_receiver.py rename src/odin_fastcs/{frame_processor.py => odin_data.py} (68%) create mode 100644 tests/input/two_node_fr_response.json diff --git a/dev/one_node_fp/odin_server.cfg b/dev/one_node_fp/odin_server.cfg index 94fb771..ae645b5 100644 --- a/dev/one_node_fp/odin_server.cfg +++ b/dev/one_node_fp/odin_server.cfg @@ -8,7 +8,7 @@ adapters = fp, fr, mw logging = error [adapter.fr] -module = odin_data.control.frame_receiver_adapter.FrameReceiverAdapter +module = odin_data.control.odin_data_adapter.OdinDataAdapter endpoints = 127.0.0.1:10000 update_interval = 0.2 diff --git a/dev/two_node_fp/odin_server.cfg b/dev/two_node_fp/odin_server.cfg index e9f87d1..5017347 100644 --- a/dev/two_node_fp/odin_server.cfg +++ b/dev/two_node_fp/odin_server.cfg @@ -8,7 +8,7 @@ adapters = fp, fr, mw logging = error [adapter.fr] -module = odin_data.control.frame_receiver_adapter.FrameReceiverAdapter +module = odin_data.control.odin_data_adapter.OdinDataAdapter endpoints = 127.0.0.1:10000, 127.0.0.1:10010 update_interval = 0.2 diff --git a/src/odin_fastcs/__main__.py b/src/odin_fastcs/__main__.py index 5e5b194..60c2329 100644 --- a/src/odin_fastcs/__main__.py +++ b/src/odin_fastcs/__main__.py @@ -6,9 +6,7 @@ from fastcs.backends.epics.gui import EpicsGUIOptions from fastcs.connections.ip_connection import IPConnectionSettings -from odin_fastcs.odin_controller import ( - OdinController, -) +from odin_fastcs.odin_controller import OdinController from . import __version__ diff --git a/src/odin_fastcs/frame_receiver.py b/src/odin_fastcs/frame_receiver.py deleted file mode 100644 index 2b190ca..0000000 --- a/src/odin_fastcs/frame_receiver.py +++ /dev/null @@ -1,5 +0,0 @@ -from odin_fastcs.odin_controller import OdinAdapterController - - -class FrameRecieverController(OdinAdapterController): - pass diff --git a/src/odin_fastcs/odin_adapter_controller.py b/src/odin_fastcs/odin_adapter_controller.py index d4e6d06..81013ee 100644 --- a/src/odin_fastcs/odin_adapter_controller.py +++ b/src/odin_fastcs/odin_adapter_controller.py @@ -10,9 +10,7 @@ from fastcs.util import snake_to_pascal from odin_fastcs.http_connection import HTTPConnection -from odin_fastcs.util import ( - OdinParameter, -) +from odin_fastcs.util import OdinParameter types = {"float": Float(), "int": Int(), "bool": Bool(), "str": String()} diff --git a/src/odin_fastcs/odin_controller.py b/src/odin_fastcs/odin_controller.py index 7070c27..d1c88fa 100644 --- a/src/odin_fastcs/odin_controller.py +++ b/src/odin_fastcs/odin_controller.py @@ -3,10 +3,13 @@ from fastcs.datatypes import Bool, Float, Int, String from odin_fastcs.eiger_fan import EigerFanAdapterController -from odin_fastcs.frame_processor import FrameProcessorAdapterController from odin_fastcs.http_connection import HTTPConnection from odin_fastcs.meta_writer import MetaWriterAdapterController from odin_fastcs.odin_adapter_controller import OdinAdapterController +from odin_fastcs.odin_data import ( + FrameProcessorAdapterController, + FrameReceiverAdapterController, +) from odin_fastcs.util import OdinParameter, create_odin_parameters types = {"float": Float(), "int": Int(), "bool": Bool(), "str": String()} @@ -70,6 +73,10 @@ def _create_adapter_controller( return FrameProcessorAdapterController( connection, parameters, f"{self.API_PREFIX}/fp" ) + case "fr": + return FrameReceiverAdapterController( + connection, parameters, f"{self.API_PREFIX}/fr" + ) case "mw": return MetaWriterAdapterController( connection, parameters, f"{self.API_PREFIX}/mw" diff --git a/src/odin_fastcs/frame_processor.py b/src/odin_fastcs/odin_data.py similarity index 68% rename from src/odin_fastcs/frame_processor.py rename to src/odin_fastcs/odin_data.py index 448f2a3..837e5db 100644 --- a/src/odin_fastcs/frame_processor.py +++ b/src/odin_fastcs/odin_data.py @@ -1,3 +1,4 @@ +import logging import re from collections.abc import Iterable, Sequence @@ -12,27 +13,32 @@ ) from odin_fastcs.util import OdinParameter, partition -UNIQUE_FP_CONFIG = [ - "rank", - "number", - "ctrl_endpoint", - "meta_endpoint", - "fr_ready_cnxn", - "fr_release_cnxn", -] +class OdinDataController(OdinAdapterController): + def _remove_metadata_fields_paths(self): + # paths ending in name or description are invalid in Odin's BaseParameterTree + self._parameters, invalid = partition( + self._parameters, lambda p: p.uri[-1] not in ["name", "description"] + ) + if invalid: + invalid_names = ["/".join(param.uri) for param in invalid] + logging.warning(f"Removing parameters with invalid names: {invalid_names}") -class FrameProcessorAdapterController(OdinAdapterController): + def _process_parameters(self): + self._remove_metadata_fields_paths() + for parameter in self._parameters: + # Remove duplicate index from uri + parameter.uri = parameter.uri[1:] + # Remove redundant status/config from parameter path + parameter.set_path(parameter.uri[1:]) + + +class OdinDataAdapterController(OdinAdapterController): """Sub controller for the frame processor adapter in an odin control server.""" - frames_written: AttrR = AttrR( - Int(), - handler=StatusSummaryUpdater([re.compile("FP*"), "HDF"], "frames_written", sum), - ) - writing: AttrR = AttrR( - Bool(), - handler=StatusSummaryUpdater([re.compile("FP*"), "HDF"], "writing", any), - ) + _unique_config: list[str] = [] + _subcontroller_label: str = "OD" + _subcontroller_cls: type[OdinDataController] = OdinDataController async def initialise(self): idx_parameters, self._parameters = partition( @@ -45,12 +51,14 @@ async def initialise(self): idx_parameters, lambda p, idx=idx: p.uri[0] == idx ) - adapter_controller = FrameProcessorController( + adapter_controller = self._subcontroller_cls( self._connection, fp_parameters, f"{self._api_prefix}/{idx}", ) - self.register_sub_controller(f"FP{idx}", adapter_controller) + self.register_sub_controller( + f"{self._subcontroller_label}{idx}", adapter_controller + ) await adapter_controller.initialise() self._create_attributes() @@ -62,11 +70,11 @@ def _create_config_fan_attributes(self): for sub_controller in get_all_sub_controllers(self): for parameter in sub_controller._parameters: mode, key = parameter.uri[0], parameter.uri[-1] - if mode == "config" and key not in UNIQUE_FP_CONFIG: + if mode == "config" and key not in self._unique_config: try: attr = getattr(sub_controller, parameter.name) except AttributeError: - print( + logging.warning( f"Controller has parameter {parameter}, " f"but no corresponding attribute {parameter.name}" ) @@ -88,7 +96,37 @@ def _create_config_fan_attributes(self): ) -class FrameProcessorController(OdinAdapterController): +class FrameReceiverController(OdinDataController): + async def initialise(self): + self._process_parameters() + + def __decoder_parameter(parameter: OdinParameter): + return "decoder" in parameter.path[:-1] + + decoder_parameters, self._parameters = partition( + self._parameters, __decoder_parameter + ) + decoder_controller = FrameReceiverDecoderController( + self._connection, decoder_parameters, f"{self._api_prefix}" + ) + self.register_sub_controller("DECODER", decoder_controller) + await decoder_controller.initialise() + self._create_attributes() + + +class FrameReceiverAdapterController(OdinDataAdapterController): + _subcontroller_label = "FR" + _subcontroller_cls = FrameReceiverController + + +class FrameReceiverDecoderController(OdinAdapterController): + def _process_parameters(self): + for parameter in self._parameters: + # remove redundant status/decoder part from path + parameter.set_path(parameter.uri[2:]) + + +class FrameProcessorController(OdinDataController): """Sub controller for a frame processor application.""" async def initialise(self): @@ -109,13 +147,6 @@ async def initialise(self): await self._create_plugin_sub_controllers(plugins) self._create_attributes() - def _process_parameters(self): - for parameter in self._parameters: - # Remove duplicate index from uri - parameter.uri = parameter.uri[1:] - # Remove redundant status/config from parameter path - parameter.set_path(parameter.uri[1:]) - async def _create_plugin_sub_controllers(self, plugins: Sequence[str]): for plugin in plugins: @@ -136,6 +167,27 @@ def __parameter_in_plugin( await plugin_controller.initialise() +class FrameProcessorAdapterController(OdinDataAdapterController): + frames_written: AttrR = AttrR( + Int(), + handler=StatusSummaryUpdater([re.compile("FP*"), "HDF"], "frames_written", sum), + ) + writing: AttrR = AttrR( + Bool(), + handler=StatusSummaryUpdater([re.compile("FP*"), "HDF"], "writing", any), + ) + _unique_config = [ + "rank", + "number", + "ctrl_endpoint", + "meta_endpoint", + "fr_ready_cnxn", + "fr_release_cnxn", + ] + _subcontroller_label = "FP" + _subcontroller_cls = FrameProcessorController + + class FrameProcessorPluginController(OdinAdapterController): """SubController for a plugin in a frameProcessor application.""" diff --git a/tests/input/two_node_fr_response.json b/tests/input/two_node_fr_response.json new file mode 100644 index 0000000..a1743f3 --- /dev/null +++ b/tests/input/two_node_fr_response.json @@ -0,0 +1,154 @@ +{ + "api": { + "value": 0.1, + "writeable": false, + "type": "float" + }, + "module": { + "value": "OdinDataAdapter", + "writeable": false, + "type": "str" + }, + "endpoints": [ + { + "ip_address": { + "value": "127.0.0.1", + "writeable": false, + "type": "str" + }, + "port": { + "value": 10000, + "writeable": false, + "type": "int" + } + }, + { + "ip_address": { + "value": "127.0.0.1", + "writeable": false, + "type": "str" + }, + "port": { + "value": 10010, + "writeable": false, + "type": "int" + } + } + ], + "count": { + "value": 2, + "writeable": false, + "type": "int" + }, + "update_interval": { + "value": 0.2, + "writeable": false, + "type": "float" + }, + "0": { + "status": { + "status": { + "ipc_configured": true, + "decoder_configured": true, + "buffer_manager_configured": true, + "rx_thread_configured": true, + "configuration_complete": true + }, + "decoder": { + "name": "DummyUDPFrameDecoder", + "status_get_count": 7, + "packets_received": 0, + "packets_lost": 0, + "packets_dropped": 0 + }, + "buffers": { + "total": 292, + "empty": 292, + "mapped": 0 + }, + "frames": { + "timedout": 0, + "received": 0, + "released": 0, + "dropped": 0 + }, + "timestamp": "2024-08-29T13:56:17.796578", + "error": [], + "connected": true + }, + "config": { + "ctrl_endpoint": "tcp://0.0.0.0:10000", + "rx_endpoint": "inproc://rx_channel", + "frame_ready_endpoint": "tcp://127.0.0.1:10001", + "frame_release_endpoint": "tcp://127.0.0.1:10002", + "decoder_path": "/scratch/hqv85942/scratchprojects/fast2/odin-data/prefix//lib", + "decoder_type": "DummyUDP", + "decoder_config": { + "enable_packet_logging": false, + "frame_timeout_ms": 1000, + "udp_packets_per_frame": 359, + "udp_packet_size": 8000 + }, + "shared_buffer_name": "OdinDataBuffer", + "max_buffer_mem": 840000000, + "rx_type": "udp", + "rx_address": "0.0.0.0", + "rx_ports": "61649", + "rx_recv_buffer_size": 30000000, + "frame_count": 1953718630 + } + }, + "1": { + "status": { + "status": { + "ipc_configured": true, + "decoder_configured": true, + "buffer_manager_configured": true, + "rx_thread_configured": true, + "configuration_complete": true + }, + "decoder": { + "name": "DummyUDPFrameDecoder", + "status_get_count": 7, + "packets_received": 0, + "packets_lost": 0, + "packets_dropped": 0 + }, + "buffers": { + "total": 292, + "empty": 292, + "mapped": 0 + }, + "frames": { + "timedout": 0, + "received": 0, + "released": 0, + "dropped": 0 + }, + "timestamp": "2024-08-29T13:56:17.817041", + "error": [], + "connected": true + }, + "config": { + "ctrl_endpoint": "tcp://0.0.0.0:10010", + "rx_endpoint": "inproc://rx_channel", + "frame_ready_endpoint": "tcp://127.0.0.1:10011", + "frame_release_endpoint": "tcp://127.0.0.1:10012", + "decoder_path": "/scratch/hqv85942/scratchprojects/fast2/odin-data/prefix//lib", + "decoder_type": "DummyUDP", + "decoder_config": { + "enable_packet_logging": false, + "frame_timeout_ms": 1000, + "udp_packets_per_frame": 359, + "udp_packet_size": 8000 + }, + "shared_buffer_name": "OdinDataBuffer", + "max_buffer_mem": 840000000, + "rx_type": "udp", + "rx_address": "0.0.0.0", + "rx_ports": "61650", + "rx_recv_buffer_size": 30000000, + "frame_count": 1953718630 + } + } +} \ No newline at end of file diff --git a/tests/test_controllers.py b/tests/test_controllers.py index 882557f..f47e767 100644 --- a/tests/test_controllers.py +++ b/tests/test_controllers.py @@ -6,10 +6,6 @@ from fastcs.datatypes import Bool, Float, Int from pytest_mock import MockerFixture -from odin_fastcs.frame_processor import ( - FrameProcessorController, - FrameProcessorPluginController, -) from odin_fastcs.http_connection import HTTPConnection from odin_fastcs.odin_adapter_controller import ( ConfigFanSender, @@ -17,6 +13,12 @@ StatusSummaryUpdater, ) from odin_fastcs.odin_controller import OdinAdapterController +from odin_fastcs.odin_data import ( + FrameProcessorController, + FrameProcessorPluginController, + FrameReceiverController, + FrameReceiverDecoderController, +) from odin_fastcs.util import OdinParameter HERE = Path(__file__).parent @@ -213,3 +215,44 @@ async def test_config_fan_sender(mocker: MockerFixture): attr1.process.assert_called_once_with(10) attr2.process.assert_called_once_with(10) attr.set.assert_called_once_with(10) + + +@pytest.mark.asyncio +async def test_frame_reciever_controllers(): + valid_non_decoder_parameter = OdinParameter( + uri=["0", "status", "buffers", "total"], + metadata={"value": 292, "type": "int", "writeable": False}, + ) + valid_decoder_parameter = OdinParameter( + uri=["0", "status", "decoder", "packets_dropped"], + metadata={"value": 0, "type": "int", "writeable": False}, + ) + + invalid_decoder_parameter = OdinParameter( + uri=["0", "status", "decoder", "name"], + metadata={ + "value": "DummyUDPFrameDecoder", + "type": "str", + "writeable": False, + }, + ) + parameters = [ + valid_non_decoder_parameter, + valid_decoder_parameter, + invalid_decoder_parameter, + ] + fr_controller = FrameReceiverController( + HTTPConnection("", 0), parameters, "api/0.1" + ) + await fr_controller.initialise() + assert isinstance(fr_controller, FrameReceiverController) + assert valid_non_decoder_parameter in fr_controller._parameters + assert len(fr_controller._parameters) == 1 + assert "DECODER" in fr_controller.get_sub_controllers() + + decoder_controller = fr_controller.get_sub_controllers()["DECODER"] + assert isinstance(decoder_controller, FrameReceiverDecoderController) + assert valid_decoder_parameter in decoder_controller._parameters + assert invalid_decoder_parameter not in decoder_controller._parameters + # index, status, decoder parts removed from path + assert decoder_controller._parameters[0]._path == ["packets_dropped"] diff --git a/tests/test_introspection.py b/tests/test_introspection.py index c9b4502..18695f4 100644 --- a/tests/test_introspection.py +++ b/tests/test_introspection.py @@ -4,13 +4,18 @@ import pytest from pytest_mock import MockerFixture -from odin_fastcs.frame_processor import FrameProcessorAdapterController +from odin_fastcs.odin_data import ( + FrameProcessorAdapterController, + FrameProcessorController, + FrameReceiverAdapterController, + FrameReceiverController, +) from odin_fastcs.util import create_odin_parameters HERE = Path(__file__).parent -def test_one_node(): +def test_one_node_fp(): with (HERE / "input/one_node_fp_response.json").open() as f: response = json.loads(f.read()) @@ -18,7 +23,7 @@ def test_one_node(): assert len(parameters) == 97 -def test_two_node(): +def test_two_node_fp(): with (HERE / "input/two_node_fp_response.json").open() as f: response = json.loads(f.read()) @@ -41,6 +46,35 @@ async def get_plugins(idx: int): controller = FrameProcessorAdapterController(mock_connection, parameters, "prefix") await controller.initialise() assert all(fpx in controller.get_sub_controllers() for fpx in ("FP0", "FP1")) + assert all( + isinstance(fpx, FrameProcessorController) + for fpx in controller.get_sub_controllers().values() + ) + + +def test_two_node_fr(): + with (HERE / "input/two_node_fr_response.json").open() as f: + response = json.loads(f.read()) + + parameters = create_odin_parameters(response) + assert len(parameters) == 82 + + +@pytest.mark.asyncio +async def test_fr_initialise(mocker: MockerFixture): + with (HERE / "input/two_node_fr_response.json").open() as f: + response = json.loads(f.read()) + + mock_connection = mocker.MagicMock() + + parameters = create_odin_parameters(response) + controller = FrameReceiverAdapterController(mock_connection, parameters, "prefix") + await controller.initialise() + assert all(frx in controller.get_sub_controllers() for frx in ("FR0", "FR1")) + assert all( + isinstance(frx, FrameReceiverController) + for frx in controller.get_sub_controllers().values() + ) def test_node_with_empty_list_is_correctly_counted():