From f267dd27d0ba064fd21df10363a4380d63c552eb Mon Sep 17 00:00:00 2001 From: Gary Yendell Date: Fri, 17 May 2024 12:30:08 +0000 Subject: [PATCH 01/10] Set title of UI --- pyproject.toml | 2 +- src/odin_fastcs/__main__.py | 4 +++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index b3a19c2..fca2916 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -12,7 +12,7 @@ classifiers = [ description = "FastCS support for the Odin detector software framework" dependencies = [ "aiohttp", - "fastcs>=0.3.0", + "fastcs~=0.4.0", ] dynamic = ["version"] license.file = "LICENSE" diff --git a/src/odin_fastcs/__main__.py b/src/odin_fastcs/__main__.py index a2fb896..3a2fe20 100644 --- a/src/odin_fastcs/__main__.py +++ b/src/odin_fastcs/__main__.py @@ -47,7 +47,9 @@ def ioc(pv_prefix: str = typer.Argument()): mapping = get_controller_mapping() backend = EpicsBackend(mapping, pv_prefix) - backend.create_gui(options=EpicsGUIOptions(output_path=Path.cwd() / "odin.bob")) + backend.create_gui( + options=EpicsGUIOptions(output_path=Path.cwd() / "odin.bob", title="Odin") + ) backend.get_ioc().run() From 8b13105bc145a9fe2498e7030e9b11bc394d3729 Mon Sep 17 00:00:00 2001 From: Gary Yendell Date: Fri, 17 May 2024 13:59:11 +0100 Subject: [PATCH 02/10] Remove unnecessary type narrowing --- src/odin_fastcs/odin_controller.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/odin_fastcs/odin_controller.py b/src/odin_fastcs/odin_controller.py index 179559f..3b77943 100644 --- a/src/odin_fastcs/odin_controller.py +++ b/src/odin_fastcs/odin_controller.py @@ -152,7 +152,6 @@ async def initialise(self) -> None: response = await self._connection.get( f"{self.API_PREFIX}/{adapter}", headers=REQUEST_METADATA_HEADER ) - assert isinstance(response, Mapping) root_tree = {k: v for k, v in response.items() if not k.isdigit()} indexed_trees = { k: v From cf37a2397edce8540c05132e12756450b595598a Mon Sep 17 00:00:00 2001 From: Gary Yendell Date: Fri, 17 May 2024 16:25:26 +0100 Subject: [PATCH 03/10] Rename Controller classes Add docstrings --- src/odin_fastcs/__main__.py | 4 ++-- src/odin_fastcs/odin_controller.py | 35 ++++++++++++++++++------------ 2 files changed, 23 insertions(+), 16 deletions(-) diff --git a/src/odin_fastcs/__main__.py b/src/odin_fastcs/__main__.py index 3a2fe20..e20cada 100644 --- a/src/odin_fastcs/__main__.py +++ b/src/odin_fastcs/__main__.py @@ -8,7 +8,7 @@ from fastcs.mapping import Mapping from odin_fastcs.odin_controller import ( - OdinTopController, + OdinController, ) from . import __version__ @@ -62,7 +62,7 @@ def asyncio(): def get_controller_mapping() -> Mapping: - controller = OdinTopController(IPConnectionSettings("127.0.0.1", 8888)) + controller = OdinController(IPConnectionSettings("127.0.0.1", 8888)) return Mapping(controller) diff --git a/src/odin_fastcs/odin_controller.py b/src/odin_fastcs/odin_controller.py index 3b77943..74b09dc 100644 --- a/src/odin_fastcs/odin_controller.py +++ b/src/odin_fastcs/odin_controller.py @@ -32,7 +32,7 @@ class ParamTreeHandler(Handler): async def put( self, - controller: "OdinController", + controller: "OdinSubController", attr: AttrW[Any], value: Any, ) -> None: @@ -46,7 +46,7 @@ async def put( async def update( self, - controller: "OdinController", + controller: "OdinSubController", attr: AttrR[Any], ) -> None: try: @@ -63,15 +63,24 @@ async def update( logging.error("Update loop failed for %s:\n%s", self.path, e) -class OdinController(SubController): +class OdinSubController(SubController): def __init__( self, connection: HTTPConnection, param_tree: Mapping[str, Any], api_prefix: str, - process_prefix: str, + path_prefix: str, ): - super().__init__(process_prefix) + """A ``SubController`` for a subsystem in an Odin control server. + + Args: + connection: HTTP connection to communicate with Odin server + param_tree: The parameter tree from the Odin server for this subsytem + api_prefix: The base URL of this subsystem in the Odin server API + path_prefix: The path of this ``Controller`` within a parent ``Controller`` + + """ + super().__init__(path_prefix) self._connection = connection self._param_tree = param_tree @@ -115,10 +124,8 @@ async def _create_parameter_tree(self): setattr(self, parameter.name.replace(".", ""), attr) -class OdinTopController(Controller): - """ - Connects all sub controllers on connect - """ +class OdinController(Controller): + """A root ``Controller`` for an Odin control server.""" API_PREFIX = "api/0.1" @@ -159,7 +166,7 @@ async def initialise(self) -> None: if k.isdigit() and isinstance(v, Mapping) } - odin_controller = OdinController( + odin_controller = OdinSubController( self._connection, root_tree, f"{self.API_PREFIX}/{adapter}", @@ -169,7 +176,7 @@ async def initialise(self) -> None: self.register_sub_controller(odin_controller) for idx, tree in indexed_trees.items(): - odin_controller = OdinController( + odin_controller = OdinSubController( self._connection, tree, f"{self.API_PREFIX}/{adapter}/{idx}", @@ -184,7 +191,7 @@ async def connect(self) -> None: self._connection.open() -class FPOdinController(OdinController): +class OdinFPController(OdinSubController): def __init__( self, connection: HTTPConnection, @@ -199,7 +206,7 @@ def __init__( ) -class FROdinController(OdinController): +class FROdinController(OdinSubController): def __init__( self, connection: HTTPConnection, @@ -214,7 +221,7 @@ def __init__( ) -class MLOdinController(OdinController): +class MLOdinController(OdinSubController): def __init__( self, connection: HTTPConnection, From a7737931988d5ae1add3aa04f6fdee05a9f61351 Mon Sep 17 00:00:00 2001 From: Gary Yendell Date: Fri, 17 May 2024 16:27:20 +0100 Subject: [PATCH 04/10] Use only first path element for Group name --- src/odin_fastcs/odin_controller.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/src/odin_fastcs/odin_controller.py b/src/odin_fastcs/odin_controller.py index 74b09dc..544f2c8 100644 --- a/src/odin_fastcs/odin_controller.py +++ b/src/odin_fastcs/odin_controller.py @@ -106,10 +106,8 @@ async def _create_parameter_tree(self): else None ) - if len(parameter.uri) >= 3: - group = snake_to_pascal( - f"{parameter.uri[0].capitalize()}_{parameter.uri[1].capitalize()}" - ) + if len(parameter.path) >= 2: + group = snake_to_pascal(f"{parameter.path[0].capitalize()}") else: group = None From a86b26d0e9c6fba7d4413c82e5546974054d19b1 Mon Sep 17 00:00:00 2001 From: Gary Yendell Date: Fri, 17 May 2024 16:32:46 +0100 Subject: [PATCH 05/10] Implement OdinFPController This is used for adapters called "fp" and removes "status" and "config" from the parameter path. --- src/odin_fastcs/odin_controller.py | 60 ++++++++++++++++++++++++------ src/odin_fastcs/util.py | 17 +++++++-- 2 files changed, 62 insertions(+), 15 deletions(-) diff --git a/src/odin_fastcs/odin_controller.py b/src/odin_fastcs/odin_controller.py index 544f2c8..a2a3c9e 100644 --- a/src/odin_fastcs/odin_controller.py +++ b/src/odin_fastcs/odin_controller.py @@ -12,6 +12,7 @@ from odin_fastcs.http_connection import HTTPConnection from odin_fastcs.util import ( + OdinParameter, create_odin_parameters, ) @@ -86,8 +87,10 @@ def __init__( self._param_tree = param_tree self._api_prefix = api_prefix - async def _create_parameter_tree(self): + def create_attributes(self): + """Create ``Attributes`` from Odin server parameter tree.""" parameters = create_odin_parameters(self._param_tree) + parameters = self.process_odin_parameters(parameters) for parameter in parameters: if "writeable" in parameter.metadata and parameter.metadata["writeable"]: @@ -121,6 +124,10 @@ async def _create_parameter_tree(self): setattr(self, parameter.name.replace(".", ""), attr) + def process_odin_parameters(self, parameters: list[OdinParameter]): + """Hook for child classes to modify the generated ``OdinParameter``s.""" + return parameters + class OdinController(Controller): """A root ``Controller`` for an Odin control server.""" @@ -164,27 +171,51 @@ async def initialise(self) -> None: if k.isdigit() and isinstance(v, Mapping) } - odin_controller = OdinSubController( + adapter_root_controller = OdinSubController( self._connection, root_tree, f"{self.API_PREFIX}/{adapter}", f"{adapter.upper()}", ) - await odin_controller._create_parameter_tree() - self.register_sub_controller(odin_controller) + adapter_root_controller.create_attributes() + self.register_sub_controller(adapter_root_controller) for idx, tree in indexed_trees.items(): - odin_controller = OdinSubController( + adapter_controller = self._create_adapter_controller( self._connection, tree, f"{self.API_PREFIX}/{adapter}/{idx}", f"{adapter.upper()}{idx}", ) - await odin_controller._create_parameter_tree() - self.register_sub_controller(odin_controller) + adapter_controller = self._create_adapter_controller( + self._connection, tree, adapter, idx + ) + adapter_controller.create_attributes() + self.register_sub_controller(adapter_controller) await self._connection.close() + def _create_adapter_controller( + self, + connection: HTTPConnection, + param_tree: Mapping[str, Any], + adapter: str, + index: int, + ) -> OdinSubController: + """Create an ``OdinSubController`` for an adapter in an Odin control server.""" + + match adapter: + # TODO: May not be called "fp", it is configurable in the server + case "fp": + return OdinFPController(connection, param_tree, self.API_PREFIX, index) + case _: + return OdinSubController( + connection, + param_tree, + f"{self.API_PREFIX}/{adapter}/{index}", + f"{adapter.upper()}{index}", + ) + async def connect(self) -> None: self._connection.open() @@ -194,15 +225,20 @@ def __init__( self, connection: HTTPConnection, param_tree: Mapping[str, Any], - api: str = "0.1", + api_prefix: str, + index: int, ): super().__init__( - connection, - param_tree, - f"api/{api}/fp", - "FP", + connection, param_tree, f"{api_prefix}/fp/{index}", f"FP{index}" ) + def process_odin_parameters(self, parameters: list[OdinParameter]): + for parameter in parameters: + # First uri element is redundant status/config + parameter.set_path(parameter.uri[1:]) + + return parameters + class FROdinController(OdinSubController): def __init__( diff --git a/src/odin_fastcs/util.py b/src/odin_fastcs/util.py index 2a4fff2..bba517c 100644 --- a/src/odin_fastcs/util.py +++ b/src/odin_fastcs/util.py @@ -1,5 +1,5 @@ -from collections.abc import Iterator, Mapping -from dataclasses import dataclass +from collections.abc import Callable, Iterator, Mapping +from dataclasses import dataclass, field from typing import Any @@ -14,10 +14,21 @@ class OdinParameter: metadata: dict[str, Any] """JSON response from GET of parameter.""" + _path: list[str] = field(default_factory=list) + + @property + def path(self) -> str: + """Reduced path of parameter to override uri when constructing name.""" + return self._path or self.uri + @property def name(self) -> str: """Unique name of parameter.""" - return "_".join(self.uri) + return "_".join(self.path) + + def set_path(self, path: list[str]): + """Set reduced path of parameter to override uri when constructing name.""" + self._path = path def create_odin_parameters(metadata: Mapping[str, Any]) -> list[OdinParameter]: From 59c025e56d3b63413e3afa159ed13cb5e4202f5f Mon Sep 17 00:00:00 2001 From: Gary Yendell Date: Mon, 20 May 2024 09:52:45 +0000 Subject: [PATCH 06/10] Typing --- src/odin_fastcs/util.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/src/odin_fastcs/util.py b/src/odin_fastcs/util.py index bba517c..45db8ff 100644 --- a/src/odin_fastcs/util.py +++ b/src/odin_fastcs/util.py @@ -17,7 +17,7 @@ class OdinParameter: _path: list[str] = field(default_factory=list) @property - def path(self) -> str: + def path(self) -> list[str]: """Reduced path of parameter to override uri when constructing name.""" return self._path or self.uri @@ -61,8 +61,7 @@ def _walk_odin_metadata( """ for node_name, node_value in tree.items(): - if node_name: - node_path = path + [node_name] + node_path = path + [node_name] if isinstance(node_value, dict) and not is_metadata_object(node_value): yield from _walk_odin_metadata(node_value, node_path) @@ -74,7 +73,7 @@ def _walk_odin_metadata( yield from _walk_odin_metadata(sub_node, sub_node_path) else: # Leaves - if is_metadata_object(node_value): + if isinstance(node_value, dict) and is_metadata_object(node_value): yield (node_path, node_value) elif isinstance(node_value, list): if "config" in node_path: @@ -93,7 +92,7 @@ def _walk_odin_metadata( yield (node_path, infer_metadata(node_value, node_path)) -def infer_metadata(parameter: int | float | bool | str, uri: list[str]): +def infer_metadata(parameter: Any, uri: list[str]): """Create metadata for a parameter from its type and URI. Args: From 57d6005718c3cba4a8e1b15585b0ed662b078a9a Mon Sep 17 00:00:00 2001 From: Gary Yendell Date: Mon, 20 May 2024 16:16:51 +0000 Subject: [PATCH 07/10] Add generic creation of SubController from fp plugins --- src/odin_fastcs/odin_controller.py | 121 ++++++++++++++++++++--------- src/odin_fastcs/util.py | 32 +++++++- 2 files changed, 116 insertions(+), 37 deletions(-) diff --git a/src/odin_fastcs/odin_controller.py b/src/odin_fastcs/odin_controller.py index a2a3c9e..a755643 100644 --- a/src/odin_fastcs/odin_controller.py +++ b/src/odin_fastcs/odin_controller.py @@ -1,6 +1,5 @@ import asyncio import logging -from collections.abc import Mapping from dataclasses import dataclass from typing import Any @@ -14,6 +13,7 @@ from odin_fastcs.util import ( OdinParameter, create_odin_parameters, + partition, ) types = {"float": Float(), "int": Int(), "bool": Bool(), "str": String()} @@ -68,29 +68,28 @@ class OdinSubController(SubController): def __init__( self, connection: HTTPConnection, - param_tree: Mapping[str, Any], + parameters: list[OdinParameter], api_prefix: str, - path_prefix: str, + path: list[str], ): """A ``SubController`` for a subsystem in an Odin control server. Args: connection: HTTP connection to communicate with Odin server - param_tree: The parameter tree from the Odin server for this subsytem + parameter_tree: The parameter tree from the Odin server for this subsytem api_prefix: The base URL of this subsystem in the Odin server API - path_prefix: The path of this ``Controller`` within a parent ``Controller`` + path: ``SubController`` path """ - super().__init__(path_prefix) + super().__init__(path) self._connection = connection - self._param_tree = param_tree + self._parameters = parameters self._api_prefix = api_prefix def create_attributes(self): """Create ``Attributes`` from Odin server parameter tree.""" - parameters = create_odin_parameters(self._param_tree) - parameters = self.process_odin_parameters(parameters) + parameters = self.process_odin_parameters(self._parameters) for parameter in parameters: if "writeable" in parameter.metadata and parameter.metadata["writeable"]: @@ -110,7 +109,7 @@ def create_attributes(self): ) if len(parameter.path) >= 2: - group = snake_to_pascal(f"{parameter.path[0].capitalize()}") + group = snake_to_pascal(f"{parameter.path[0]}") else: group = None @@ -124,10 +123,18 @@ def create_attributes(self): setattr(self, parameter.name.replace(".", ""), attr) - def process_odin_parameters(self, parameters: list[OdinParameter]): - """Hook for child classes to modify the generated ``OdinParameter``s.""" + async def initialise(self): + pass + + def process_odin_parameters( + self, parameters: list[OdinParameter] + ) -> list[OdinParameter]: + """Hook for child classes to process parameters before creating attributes.""" return parameters + def create_odin_parameters(self): + return create_odin_parameters(self._parameter_tree) + class OdinController(Controller): """A root ``Controller`` for an Odin control server.""" @@ -173,23 +180,19 @@ async def initialise(self) -> None: adapter_root_controller = OdinSubController( self._connection, - root_tree, + create_odin_parameters(root_tree), f"{self.API_PREFIX}/{adapter}", - f"{adapter.upper()}", + [f"{adapter.upper()}"], ) + await adapter_root_controller.initialise() adapter_root_controller.create_attributes() self.register_sub_controller(adapter_root_controller) for idx, tree in indexed_trees.items(): adapter_controller = self._create_adapter_controller( - self._connection, - tree, - f"{self.API_PREFIX}/{adapter}/{idx}", - f"{adapter.upper()}{idx}", - ) - adapter_controller = self._create_adapter_controller( - self._connection, tree, adapter, idx + self._connection, create_odin_parameters(tree), adapter, int(idx) ) + await adapter_controller.initialise() adapter_controller.create_attributes() self.register_sub_controller(adapter_controller) @@ -198,7 +201,7 @@ async def initialise(self) -> None: def _create_adapter_controller( self, connection: HTTPConnection, - param_tree: Mapping[str, Any], + parameters: list[OdinParameter], adapter: str, index: int, ) -> OdinSubController: @@ -207,13 +210,13 @@ def _create_adapter_controller( match adapter: # TODO: May not be called "fp", it is configurable in the server case "fp": - return OdinFPController(connection, param_tree, self.API_PREFIX, index) + return OdinFPController(connection, parameters, self.API_PREFIX, index) case _: return OdinSubController( connection, - param_tree, + parameters, f"{self.API_PREFIX}/{adapter}/{index}", - f"{adapter.upper()}{index}", + [f"{adapter.upper()}{index}"], ) async def connect(self) -> None: @@ -221,22 +224,68 @@ async def connect(self) -> None: class OdinFPController(OdinSubController): + def __init__( self, connection: HTTPConnection, - param_tree: Mapping[str, Any], + parameters: list[OdinParameter], api_prefix: str, index: int, ): super().__init__( - connection, param_tree, f"{api_prefix}/fp/{index}", f"FP{index}" + connection, parameters, f"{api_prefix}/fp/{index}", [f"FP{index}"] ) - def process_odin_parameters(self, parameters: list[OdinParameter]): - for parameter in parameters: - # First uri element is redundant status/config + async def initialise(self): + plugins_response = await self._connection.get( + f"{self._api_prefix}/status/plugins/names" + ) + match plugins_response: + case {"names": [*plugin_list]}: + plugins = tuple(a for a in plugin_list if isinstance(a, str)) + if len(plugins) != len(plugin_list): + raise ValueError(f"Received invalid plugins list:\n{plugin_list}") + case _: + raise ValueError( + f"Did not find valid plugins in response:\n{plugins_response}" + ) + + # Remove redundant status/config from parameter path + for parameter in self._parameters: parameter.set_path(parameter.uri[1:]) + for plugin in plugins: + plugin_parameters, self._parameters = partition( + self._parameters, lambda p, plugin=plugin: p.path[0] == plugin + ) + plugin_controller = OdinFPPluginController( + self._connection, + plugin_parameters, + f"{self._api_prefix}", + self.path + [plugin], + ) + plugin_controller.create_attributes() + self.register_sub_controller(plugin_controller) + + +class OdinFPPluginController(OdinSubController): + def __init__( + self, + connection: HTTPConnection, + parameters: list[OdinParameter], + api_prefix: str, + path: list[str], + ): + super().__init__(connection, parameters, api_prefix, path) + + def process_odin_parameters( + self, parameters: list[OdinParameter] + ) -> list[OdinParameter]: + for parameter in parameters: + # Remove plugin name included in controller base path + parameter.set_path(parameter.path[1:]) + + # TODO: Make a copy? return parameters @@ -244,14 +293,14 @@ class FROdinController(OdinSubController): def __init__( self, connection: HTTPConnection, - param_tree: Mapping[str, Any], + parameters: list[OdinParameter], api: str = "0.1", ): super().__init__( connection, - param_tree, + parameters, f"api/{api}/fr", - "FR", + ["FR"], ) @@ -259,12 +308,12 @@ class MLOdinController(OdinSubController): def __init__( self, connection: HTTPConnection, - param_tree: Mapping[str, Any], + parameters: list[OdinParameter], api: str = "0.1", ): super().__init__( connection, - param_tree, + parameters, f"api/{api}/meta_listener", - "ML", + ["ML"], ) diff --git a/src/odin_fastcs/util.py b/src/odin_fastcs/util.py index 45db8ff..c004726 100644 --- a/src/odin_fastcs/util.py +++ b/src/odin_fastcs/util.py @@ -1,6 +1,6 @@ from collections.abc import Callable, Iterator, Mapping from dataclasses import dataclass, field -from typing import Any +from typing import Any, TypeVar def is_metadata_object(v: Any) -> bool: @@ -105,3 +105,33 @@ def infer_metadata(parameter: Any, uri: list[str]): "type": type(parameter).__name__, "writeable": "config" in uri, } + + +T = TypeVar("T") + + +def partition( + elements: list[T], predicate: Callable[[T], bool] +) -> tuple[list[T], list[T]]: + """Split a list of elements in two based on predicate. + + If the predicate returns ``True``, the element will be placed in the truthy list, + if it does not, it will be placed in the falsy list. + + Args: + elements: List of T + predicate: Predicate to filter the list with + + Returns: + (truthy, falsy) + + """ + truthy: list[T] = [] + falsy: list[T] = [] + for parameter in elements: + if predicate(parameter): + truthy.append(parameter) + else: + falsy.append(parameter) + + return truthy, falsy From 17344c699bba51fc6a7fc8d86ef27b250eb6c2af Mon Sep 17 00:00:00 2001 From: Gary Yendell Date: Thu, 23 May 2024 09:35:14 +0000 Subject: [PATCH 08/10] Extract fp specific logic from OdinController --- src/odin_fastcs/odin_controller.py | 85 +++++++++++++----------------- 1 file changed, 37 insertions(+), 48 deletions(-) diff --git a/src/odin_fastcs/odin_controller.py b/src/odin_fastcs/odin_controller.py index a755643..17059ed 100644 --- a/src/odin_fastcs/odin_controller.py +++ b/src/odin_fastcs/odin_controller.py @@ -171,30 +171,13 @@ async def initialise(self) -> None: response = await self._connection.get( f"{self.API_PREFIX}/{adapter}", headers=REQUEST_METADATA_HEADER ) - root_tree = {k: v for k, v in response.items() if not k.isdigit()} - indexed_trees = { - k: v - for k, v in response.items() - if k.isdigit() and isinstance(v, Mapping) - } - - adapter_root_controller = OdinSubController( - self._connection, - create_odin_parameters(root_tree), - f"{self.API_PREFIX}/{adapter}", - [f"{adapter.upper()}"], - ) - await adapter_root_controller.initialise() - adapter_root_controller.create_attributes() - self.register_sub_controller(adapter_root_controller) - for idx, tree in indexed_trees.items(): - adapter_controller = self._create_adapter_controller( - self._connection, create_odin_parameters(tree), adapter, int(idx) - ) - await adapter_controller.initialise() - adapter_controller.create_attributes() - self.register_sub_controller(adapter_controller) + adapter_controller = self._create_adapter_controller( + self._connection, create_odin_parameters(response), adapter + ) + await adapter_controller.initialise() + adapter_controller.create_attributes() + self.register_sub_controller(adapter_controller) await self._connection.close() @@ -203,39 +186,51 @@ def _create_adapter_controller( connection: HTTPConnection, parameters: list[OdinParameter], adapter: str, - index: int, ) -> OdinSubController: """Create an ``OdinSubController`` for an adapter in an Odin control server.""" match adapter: # TODO: May not be called "fp", it is configurable in the server case "fp": - return OdinFPController(connection, parameters, self.API_PREFIX, index) + return OdinFPAdapterController( + connection, parameters, f"{self.API_PREFIX}/fp", ["FP"] + ) case _: return OdinSubController( connection, parameters, - f"{self.API_PREFIX}/{adapter}/{index}", - [f"{adapter.upper()}{index}"], + f"{self.API_PREFIX}/{adapter}", + [f"{adapter.upper()}"], ) async def connect(self) -> None: self._connection.open() -class OdinFPController(OdinSubController): - - def __init__( - self, - connection: HTTPConnection, - parameters: list[OdinParameter], - api_prefix: str, - index: int, - ): - super().__init__( - connection, parameters, f"{api_prefix}/fp/{index}", [f"FP{index}"] +class OdinFPAdapterController(OdinSubController): + async def initialise(self): + idx_parameters, self._parameters = partition( + self._parameters, lambda p: p.uri[0].isdigit() ) + while idx_parameters: + idx = idx_parameters[0].uri[0] + fp_parameters, idx_parameters = partition( + idx_parameters, lambda p, idx=idx: p.uri[0] == idx + ) + + adapter_controller = OdinFPController( + self._connection, + fp_parameters, + f"{self._api_prefix}/{idx}", + [f"FP{idx}"], + ) + await adapter_controller.initialise() + adapter_controller.create_attributes() + self.register_sub_controller(adapter_controller) + + +class OdinFPController(OdinSubController): async def initialise(self): plugins_response = await self._connection.get( f"{self._api_prefix}/status/plugins/names" @@ -250,8 +245,10 @@ async def initialise(self): f"Did not find valid plugins in response:\n{plugins_response}" ) - # Remove redundant status/config from parameter path for parameter in self._parameters: + # Remove duplicate index from uri + parameter.uri = parameter.uri[1:] + # Remove redundant status/config from parameter path parameter.set_path(parameter.uri[1:]) for plugin in plugins: @@ -269,15 +266,7 @@ async def initialise(self): class OdinFPPluginController(OdinSubController): - def __init__( - self, - connection: HTTPConnection, - parameters: list[OdinParameter], - api_prefix: str, - path: list[str], - ): - super().__init__(connection, parameters, api_prefix, path) - + # TODO: Just use initialise? def process_odin_parameters( self, parameters: list[OdinParameter] ) -> list[OdinParameter]: From 1f525d69dd88ed7c626bb4d2177b0179e899e475 Mon Sep 17 00:00:00 2001 From: Gary Yendell Date: Fri, 24 May 2024 13:30:32 +0100 Subject: [PATCH 09/10] Improve dev environment Add support for multiple deployments in dev environment Remove workspaceFolder from devcontainer config Update README to use odin-data devcontainer Create odin extra for local development Print prompt to delete existing local deployment --- .devcontainer/devcontainer.json | 3 +- .vscode/launch.json | 2 +- README.md | 59 +++++--- dev/configure.sh | 37 +++-- dev/{templates => one_node_fp}/fp1.json | 0 dev/{templates => one_node_fp}/fr1.json | 2 +- dev/{templates => one_node_fp}/layout.kdl | 2 +- dev/{templates => one_node_fp}/log4cxx.xml | 0 .../odin_server.cfg | 0 .../stFrameProcessor1.sh | 0 .../stFrameReceiver1.sh | 0 .../stMetaWriter.sh | 0 .../stOdinServer.sh | 3 + dev/two_node_fp/fp1.json | 140 ++++++++++++++++++ dev/two_node_fp/fp2.json | 140 ++++++++++++++++++ dev/two_node_fp/fr1.json | 16 ++ dev/two_node_fp/fr2.json | 16 ++ dev/two_node_fp/layout.kdl | 22 +++ dev/two_node_fp/log4cxx.xml | 46 ++++++ dev/two_node_fp/odin_server.cfg | 32 ++++ dev/two_node_fp/stFrameProcessor1.sh | 7 + dev/two_node_fp/stFrameProcessor2.sh | 7 + dev/two_node_fp/stFrameReceiver1.sh | 5 + dev/two_node_fp/stFrameReceiver2.sh | 5 + dev/two_node_fp/stMetaWriter.sh | 3 + dev/two_node_fp/stOdinServer.sh | 11 ++ pyproject.toml | 4 + 27 files changed, 520 insertions(+), 42 deletions(-) rename dev/{templates => one_node_fp}/fp1.json (100%) rename dev/{templates => one_node_fp}/fr1.json (99%) rename dev/{templates => one_node_fp}/layout.kdl (87%) rename dev/{templates => one_node_fp}/log4cxx.xml (100%) rename dev/{templates => one_node_fp}/odin_server.cfg (100%) rename dev/{templates => one_node_fp}/stFrameProcessor1.sh (100%) rename dev/{templates => one_node_fp}/stFrameReceiver1.sh (100%) rename dev/{templates => one_node_fp}/stMetaWriter.sh (100%) rename dev/{templates => one_node_fp}/stOdinServer.sh (83%) create mode 100644 dev/two_node_fp/fp1.json create mode 100644 dev/two_node_fp/fp2.json create mode 100644 dev/two_node_fp/fr1.json create mode 100644 dev/two_node_fp/fr2.json create mode 100644 dev/two_node_fp/layout.kdl create mode 100644 dev/two_node_fp/log4cxx.xml create mode 100644 dev/two_node_fp/odin_server.cfg create mode 100755 dev/two_node_fp/stFrameProcessor1.sh create mode 100755 dev/two_node_fp/stFrameProcessor2.sh create mode 100755 dev/two_node_fp/stFrameReceiver1.sh create mode 100755 dev/two_node_fp/stFrameReceiver2.sh create mode 100755 dev/two_node_fp/stMetaWriter.sh create mode 100755 dev/two_node_fp/stOdinServer.sh diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index 52c2222..d3d639a 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -41,7 +41,6 @@ ], // Mount the parent as /workspaces so we can pip install peers as editable "workspaceMount": "source=${localWorkspaceFolder}/..,target=/workspaces,type=bind", - "workspaceFolder": "/workspaces", // After the container is created, install the python project in editable form "postCreateCommand": "pip install $([ -f dev-requirements.txt ] && echo '-c dev-requirements.txt') -e '.[dev]' && pre-commit install" -} \ No newline at end of file +} diff --git a/.vscode/launch.json b/.vscode/launch.json index d26abd1..3ac5f4a 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -29,7 +29,7 @@ "module": "odin.main", "justMyCode": false, "console": "integratedTerminal", - "args": ["--config", "${workspaceFolder}/dev/odin_server.cfg", "--logging", "debug"] + "args": ["--config", "${workspaceFolder}/dev/local/odin_server.cfg", "--logging", "debug"] }, { "name": "Dump Server Response", diff --git a/README.md b/README.md index d754aae..dbf5cbe 100644 --- a/README.md +++ b/README.md @@ -4,7 +4,6 @@ [![License](https://img.shields.io/badge/License-Apache%202.0-blue.svg)](https://opensource.org/licenses/Apache-2.0) # Odin FastCS - FastCS support for the Odin detector software framework Source | @@ -16,38 +15,52 @@ Releases | ## Development -Odin FastCS does not do much unless it has an Odin control server to talk to. It is -possible to test some functionality in isolation by dumping server responses and creating -tests that parse those responses. Responses can be dumped from various Odin systems and -tests written against them that can run in CI to ensure support for those systems is not -broken (or that the adapters need to be updated). The `tests/dump_server_response.py` -helper script will generate json files for each adapter in an Odin server to write tests -against. +1. Clone odin-data and open in its devcontainer (for now, checkout fastcs-dev branch) +2. Build odin-data into `vscode_prefix` + + i. `CMake: Delete Cache and Reconfigure` (select a compiler, /usr/bin/gcc is + probably best) + + ii. `CMake: Install` + +3. `Workspaces: Add Folder to Workspace...` to add odin-fastcs to the workspace +4. Install odin-fastcs and its odin dev environment + + i. `pip install -e .[dev,odin]` + + ii. `Python: Select Interpreter` and set it to `/venv/bin/python` for the workspace -Testing against static files is quite restrictive, so a dummy development environment is -provided to give developers as consistent live deployment as possible to work against -while developing the code. To set this up, run `dev/configure.sh` with the path to an -odin-data install prefix and the path to a venv with odin-control and odin-data -installed. This will populate the dev config with your environment - these changes -should not be checked in. The dev deployment can then be run with `dev/start.sh`. +5. Prepare the dev environment -Currently Odin FastCS depends on branches of both odin-control and odin-data, so these -branches are provided in `dev/requirements.txt` for convenience. Make a venv and then -`pip install -r dev/requirements.txt` will give an environment that the control server -and meta writer can run in. For the frameProcessor and frameReceiver, check out the -fastcs-dev branch of odin-data and build. It is recommended to use the vscode CMake -configuration to do this. + i. `dev/configure.sh one_node_fp /workspaces/odin-data/vscode_prefix /venv` + +6. Run the dev environment + + i. `dev/start.sh` (it may print some garbage to the terminal while it installs) + + ii. Click in the right panel and hit enter to run the odin server once all processes + running + + iii. To close zelijj and the processes, `Ctrl+Q` + +7. Run the `Odin IOC` launch config to run the odin-fastcs IOC If you need to run a dev version of any of the applications, stop that process in the deployment and run/debug it manually. There is a vscode launch config for an odin server -using the same config as the dev deployment for this purpose. For the python processes -it is convenient to `pip install -r dev/requirements.txt` inside the container as well -to make use of the launch configs. +using the same config as the dev deployment for this purpose. At boot time, FastCS will generate UIs that can be opened in Phoebus. This is the clearest way to see the PVs that have been generated for the Odin server. It is also possible to run `dbl()` in the EPICS shell to print a flat list of PVs. +Odin FastCS does not do much unless it has an Odin control server to talk to. It is +possible to test some functionality in isolation by dumping server responses and creating +tests that parse those responses. Responses can be dumped from various Odin systems and +tests written against them that can run in CI to ensure support for those systems is not +broken (or that the adapters need to be updated). The `tests/dump_server_response.py` +helper script will generate json files for each adapter in an Odin server to write tests +against. + See https://diamondlightsource.github.io/odin-fastcs for more detailed documentation. diff --git a/dev/configure.sh b/dev/configure.sh index 0bf00f9..26eb019 100755 --- a/dev/configure.sh +++ b/dev/configure.sh @@ -2,27 +2,36 @@ # Script to populate deployment with absolute paths of environment -if [[ "$1" == "-h" || "$1" == "--help" || "$#" -ne 2 ]]; then - echo "Usage: $0 " +if [[ "$1" == "-h" || "$1" == "--help" || "$#" -ne 3 ]]; then + echo "Usage: $0 " exit 0 fi -ODIN_DATA=$1 -VENV=$2 +DEPLOYMENT=$1 +ODIN_DATA=$2 +VENV=$3 SCRIPT_DIR=$(cd $(dirname "${BASH_SOURCE[0]}") && pwd) -mkdir ${SCRIPT_DIR}/local -cp ${SCRIPT_DIR}/templates/* ${SCRIPT_DIR}/local +LOCAL=${SCRIPT_DIR}/local -SERVER="${SCRIPT_DIR}/local/stOdinServer.sh" -FR="${SCRIPT_DIR}/local/stFrameReceiver1.sh" -FR_CONFIG="${SCRIPT_DIR}/local/fr1.json" -FP="${SCRIPT_DIR}/local/stFrameProcessor1.sh" -FP_CONFIG="${SCRIPT_DIR}/local/fp1.json" -META="${SCRIPT_DIR}/local/stMetaWriter.sh" -LAYOUT="${SCRIPT_DIR}/local/layout.kdl" +# Check if local already exists +if [ -d ${LOCAL} ]; then + echo "Local deployment ${LOCAL} already exists. Please remove it if you want to replace it." + exit 1 +fi + +mkdir ${LOCAL} +cp ${SCRIPT_DIR}/${DEPLOYMENT}/* ${LOCAL} + +SERVER="${LOCAL}/stOdinServer.sh" +FR="${LOCAL}/stFrameReceiver*.sh" +FR_CONFIG="${LOCAL}/fr*.json" +FP="${LOCAL}/stFrameProcessor*.sh" +FP_CONFIG="${LOCAL}/fp*.json" +META="${LOCAL}/stMetaWriter.sh" +LAYOUT="${LOCAL}/layout.kdl" sed -i "s++${ODIN_DATA}+g" ${FR} ${FR_CONFIG} ${FP} ${FP_CONFIG} sed -i "s++${VENV}+g" ${SERVER} ${META} -sed -i "s++${SCRIPT_DIR}/local+g" ${LAYOUT} +sed -i "s++${LOCAL}+g" ${LAYOUT} diff --git a/dev/templates/fp1.json b/dev/one_node_fp/fp1.json similarity index 100% rename from dev/templates/fp1.json rename to dev/one_node_fp/fp1.json diff --git a/dev/templates/fr1.json b/dev/one_node_fp/fr1.json similarity index 99% rename from dev/templates/fr1.json rename to dev/one_node_fp/fr1.json index 3314c61..4e47ab6 100644 --- a/dev/templates/fr1.json +++ b/dev/one_node_fp/fr1.json @@ -13,4 +13,4 @@ "udp_packet_size": 8000 } } -] \ No newline at end of file +] diff --git a/dev/templates/layout.kdl b/dev/one_node_fp/layout.kdl similarity index 87% rename from dev/templates/layout.kdl rename to dev/one_node_fp/layout.kdl index 76b17bc..fec4a10 100644 --- a/dev/templates/layout.kdl +++ b/dev/one_node_fp/layout.kdl @@ -9,7 +9,7 @@ layout { pane command="/stMetaWriter.sh" } pane split_direction="horizontal" { - pane command="/stOdinServer.sh" start_suspended=true + pane command="/stOdinServer.sh" } } pane size=2 borderless=true { diff --git a/dev/templates/log4cxx.xml b/dev/one_node_fp/log4cxx.xml similarity index 100% rename from dev/templates/log4cxx.xml rename to dev/one_node_fp/log4cxx.xml diff --git a/dev/templates/odin_server.cfg b/dev/one_node_fp/odin_server.cfg similarity index 100% rename from dev/templates/odin_server.cfg rename to dev/one_node_fp/odin_server.cfg diff --git a/dev/templates/stFrameProcessor1.sh b/dev/one_node_fp/stFrameProcessor1.sh similarity index 100% rename from dev/templates/stFrameProcessor1.sh rename to dev/one_node_fp/stFrameProcessor1.sh diff --git a/dev/templates/stFrameReceiver1.sh b/dev/one_node_fp/stFrameReceiver1.sh similarity index 100% rename from dev/templates/stFrameReceiver1.sh rename to dev/one_node_fp/stFrameReceiver1.sh diff --git a/dev/templates/stMetaWriter.sh b/dev/one_node_fp/stMetaWriter.sh similarity index 100% rename from dev/templates/stMetaWriter.sh rename to dev/one_node_fp/stMetaWriter.sh diff --git a/dev/templates/stOdinServer.sh b/dev/one_node_fp/stOdinServer.sh similarity index 83% rename from dev/templates/stOdinServer.sh rename to dev/one_node_fp/stOdinServer.sh index 3b4acf1..c13e8ca 100755 --- a/dev/templates/stOdinServer.sh +++ b/dev/one_node_fp/stOdinServer.sh @@ -5,4 +5,7 @@ SCRIPT_DIR="$( cd "$( dirname "$0" )" && pwd )" # Increase maximum fds available for ZeroMQ sockets ulimit -n 2048 +# Wait for other processes to start +sleep 3 + /bin/odin_control --config=$SCRIPT_DIR/odin_server.cfg --logging=info --access_logging=ERROR diff --git a/dev/two_node_fp/fp1.json b/dev/two_node_fp/fp1.json new file mode 100644 index 0000000..e6f563d --- /dev/null +++ b/dev/two_node_fp/fp1.json @@ -0,0 +1,140 @@ +[ + { + "fr_setup": { + "fr_ready_cnxn": "tcp://127.0.0.1:10001", + "fr_release_cnxn": "tcp://127.0.0.1:10002" + }, + "meta_endpoint": "tcp://*:10008" + }, + { + "plugin": { + "load": { + "index": "dummy", + "name": "DummyUDPProcessPlugin", + "library": "/lib/libDummyUDPProcessPlugin.so" + } + } + }, + { + "plugin": { + "load": { + "index": "offset", + "name": "OffsetAdjustmentPlugin", + "library": "/lib/libOffsetAdjustmentPlugin.so" + } + } + }, + { + "plugin": { + "load": { + "index": "param", + "name": "ParameterAdjustmentPlugin", + "library": "/lib/libParameterAdjustmentPlugin.so" + } + } + }, + { + "plugin": { + "load": { + "index": "hdf", + "name": "FileWriterPlugin", + "library": "/lib/libHdf5Plugin.so" + } + } + }, + { + "plugin": { + "connect": { + "index": "dummy", + "connection": "frame_receiver" + } + } + }, + { + "plugin": { + "connect": { + "index": "offset", + "connection": "dummy" + } + } + }, + { + "plugin": { + "connect": { + "index": "param", + "connection": "offset" + } + } + }, + { + "plugin": { + "connect": { + "index": "hdf", + "connection": "param" + } + } + }, + { + "hdf": { + "dataset": { + "compressed_size": { + "datatype": "uint32", + "chunks": [1000] + } + } + } + }, + { + "hdf": { + "dataset": { + "uid": { + "datatype": "uint64", + "chunks": [1000] + } + } + } + }, + { + "param": { + "parameter": { + "uid": { + "adjustment": 1 + } + } + } + }, + { + "hdf": { + "process": { + "number": 4, + "rank": 0 + } + } + }, + { + "hdf": { + "file": { + "flush_error_duration": 10000, + "write_error_duration": 10000, + "close_error_duration": 10000, + "create_error_duration": 10000 + } + } + }, + { + "hdf": { + "file": { + "first_number": 1 + } + } + }, + { + "hdf": { + "dataset": { + "data": { + "chunks": [1, 512, 256] + } + } + } + } +] diff --git a/dev/two_node_fp/fp2.json b/dev/two_node_fp/fp2.json new file mode 100644 index 0000000..7b698ef --- /dev/null +++ b/dev/two_node_fp/fp2.json @@ -0,0 +1,140 @@ +[ + { + "fr_setup": { + "fr_ready_cnxn": "tcp://127.0.0.1:10011", + "fr_release_cnxn": "tcp://127.0.0.1:10012" + }, + "meta_endpoint": "tcp://*:10018" + }, + { + "plugin": { + "load": { + "index": "dummy", + "name": "DummyUDPProcessPlugin", + "library": "/lib/libDummyUDPProcessPlugin.so" + } + } + }, + { + "plugin": { + "load": { + "index": "offset", + "name": "OffsetAdjustmentPlugin", + "library": "/lib/libOffsetAdjustmentPlugin.so" + } + } + }, + { + "plugin": { + "load": { + "index": "param", + "name": "ParameterAdjustmentPlugin", + "library": "/lib/libParameterAdjustmentPlugin.so" + } + } + }, + { + "plugin": { + "load": { + "index": "hdf", + "name": "FileWriterPlugin", + "library": "/lib/libHdf5Plugin.so" + } + } + }, + { + "plugin": { + "connect": { + "index": "dummy", + "connection": "frame_receiver" + } + } + }, + { + "plugin": { + "connect": { + "index": "offset", + "connection": "dummy" + } + } + }, + { + "plugin": { + "connect": { + "index": "param", + "connection": "offset" + } + } + }, + { + "plugin": { + "connect": { + "index": "hdf", + "connection": "param" + } + } + }, + { + "hdf": { + "dataset": { + "compressed_size": { + "datatype": "uint32", + "chunks": [1000] + } + } + } + }, + { + "hdf": { + "dataset": { + "uid": { + "datatype": "uint64", + "chunks": [1000] + } + } + } + }, + { + "param": { + "parameter": { + "uid": { + "adjustment": 1 + } + } + } + }, + { + "hdf": { + "process": { + "number": 4, + "rank": 0 + } + } + }, + { + "hdf": { + "file": { + "flush_error_duration": 10000, + "write_error_duration": 10000, + "close_error_duration": 10000, + "create_error_duration": 10000 + } + } + }, + { + "hdf": { + "file": { + "first_number": 1 + } + } + }, + { + "hdf": { + "dataset": { + "data": { + "chunks": [1, 512, 256] + } + } + } + } +] diff --git a/dev/two_node_fp/fr1.json b/dev/two_node_fp/fr1.json new file mode 100644 index 0000000..4e47ab6 --- /dev/null +++ b/dev/two_node_fp/fr1.json @@ -0,0 +1,16 @@ +[ + { + "frame_ready_endpoint": "tcp://127.0.0.1:10001", + "frame_release_endpoint": "tcp://127.0.0.1:10002", + "decoder_type": "DummyUDP", + "decoder_path": "/lib", + "rx_ports": "61649", + "max_buffer_mem": 840000000, + "decoder_config": { + "enable_packet_logging": false, + "frame_timeout_ms": 1000, + "udp_packets_per_frame": 359, + "udp_packet_size": 8000 + } + } +] diff --git a/dev/two_node_fp/fr2.json b/dev/two_node_fp/fr2.json new file mode 100644 index 0000000..22be903 --- /dev/null +++ b/dev/two_node_fp/fr2.json @@ -0,0 +1,16 @@ +[ + { + "frame_ready_endpoint": "tcp://127.0.0.1:10011", + "frame_release_endpoint": "tcp://127.0.0.1:10012", + "decoder_type": "DummyUDP", + "decoder_path": "/lib", + "rx_ports": "61650", + "max_buffer_mem": 840000000, + "decoder_config": { + "enable_packet_logging": false, + "frame_timeout_ms": 1000, + "udp_packets_per_frame": 359, + "udp_packet_size": 8000 + } + } +] diff --git a/dev/two_node_fp/layout.kdl b/dev/two_node_fp/layout.kdl new file mode 100644 index 0000000..8a16ac4 --- /dev/null +++ b/dev/two_node_fp/layout.kdl @@ -0,0 +1,22 @@ +layout { + pane size=1 borderless=true { + plugin location="zellij:tab-bar" + } + pane split_direction="vertical" { + pane split_direction="horizontal" { + pane command="/stFrameReceiver1.sh" + pane command="/stFrameProcessor1.sh" + pane command="/stFrameReceiver2.sh" + pane command="/stFrameProcessor2.sh" + pane command="/stMetaWriter.sh" + } + pane split_direction="horizontal" { + pane command="/stOdinServer.sh" + } + } + pane size=2 borderless=true { + plugin location="zellij:status-bar" + } +} +seasion_name "ODIN-FASTCS" +attach_to_session true diff --git a/dev/two_node_fp/log4cxx.xml b/dev/two_node_fp/log4cxx.xml new file mode 100644 index 0000000..42e726b --- /dev/null +++ b/dev/two_node_fp/log4cxx.xml @@ -0,0 +1,46 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/dev/two_node_fp/odin_server.cfg b/dev/two_node_fp/odin_server.cfg new file mode 100644 index 0000000..d720400 --- /dev/null +++ b/dev/two_node_fp/odin_server.cfg @@ -0,0 +1,32 @@ +[server] +debug_mode = 0 +http_port = 8888 +http_addr = 127.0.0.1 +adapters = od_fps, fp, od_frs, fr, od_mls, ml + +[tornado] +logging = error + +[adapter.od_frs] +module = odin_data.control.odin_data_adapter.OdinDataAdapter +endpoints = 127.0.0.1:10000, 127.0.0.1:10010 +update_interval = 0.2 + +[adapter.od_fps] +module = odin_data.control.odin_data_adapter.OdinDataAdapter +endpoints = 127.0.0.1:10004, 127.0.0.1:10014 +update_interval = 0.2 + +[adapter.od_mls] +module = odin_data.control.odin_data_adapter.OdinDataAdapter +endpoints = 127.0.0.1:10008 +update_interval = 0.2 + +[adapter.fr] +module = odin_data.control.frame_receiver_adapter.FrameReceiverAdapter + +[adapter.fp] +module = odin_data.control.frame_processor_adapter.FrameProcessorAdapter + +[adapter.ml] +module = odin_data.control.meta_listener_adapter.MetaListenerAdapter diff --git a/dev/two_node_fp/stFrameProcessor1.sh b/dev/two_node_fp/stFrameProcessor1.sh new file mode 100755 index 0000000..3d4149e --- /dev/null +++ b/dev/two_node_fp/stFrameProcessor1.sh @@ -0,0 +1,7 @@ +#!/bin/bash + +SCRIPT_DIR="$( cd "$( dirname "$0" )" && pwd )" + +export HDF5_PLUGIN_PATH=/dls_sw/prod/tools/RHEL7-x86_64/hdf5filters/0-7-0/prefix/hdf5_1.10/h5plugin + +/bin/frameProcessor --ctrl=tcp://0.0.0.0:10004 --config=$SCRIPT_DIR/fp1.json --log-config $SCRIPT_DIR/log4cxx.xml diff --git a/dev/two_node_fp/stFrameProcessor2.sh b/dev/two_node_fp/stFrameProcessor2.sh new file mode 100755 index 0000000..55aec0f --- /dev/null +++ b/dev/two_node_fp/stFrameProcessor2.sh @@ -0,0 +1,7 @@ +#!/bin/bash + +SCRIPT_DIR="$( cd "$( dirname "$0" )" && pwd )" + +export HDF5_PLUGIN_PATH=/dls_sw/prod/tools/RHEL7-x86_64/hdf5filters/0-7-0/prefix/hdf5_1.10/h5plugin + +/bin/frameProcessor --ctrl=tcp://0.0.0.0:10014 --config=$SCRIPT_DIR/fp2.json --log-config $SCRIPT_DIR/log4cxx.xml diff --git a/dev/two_node_fp/stFrameReceiver1.sh b/dev/two_node_fp/stFrameReceiver1.sh new file mode 100755 index 0000000..559d1fc --- /dev/null +++ b/dev/two_node_fp/stFrameReceiver1.sh @@ -0,0 +1,5 @@ +#!/bin/bash + +SCRIPT_DIR="$( cd "$( dirname "$0" )" && pwd )" + +/bin/frameReceiver --io-threads 2 --ctrl=tcp://0.0.0.0:10000 --config=$SCRIPT_DIR/fr1.json --log-config $SCRIPT_DIR/log4cxx.xml diff --git a/dev/two_node_fp/stFrameReceiver2.sh b/dev/two_node_fp/stFrameReceiver2.sh new file mode 100755 index 0000000..4e34a3e --- /dev/null +++ b/dev/two_node_fp/stFrameReceiver2.sh @@ -0,0 +1,5 @@ +#!/bin/bash + +SCRIPT_DIR="$( cd "$( dirname "$0" )" && pwd )" + +/bin/frameReceiver --io-threads 2 --ctrl=tcp://0.0.0.0:10010 --config=$SCRIPT_DIR/fr2.json --log-config $SCRIPT_DIR/log4cxx.xml diff --git a/dev/two_node_fp/stMetaWriter.sh b/dev/two_node_fp/stMetaWriter.sh new file mode 100755 index 0000000..e878ec7 --- /dev/null +++ b/dev/two_node_fp/stMetaWriter.sh @@ -0,0 +1,3 @@ +#!/bin/bash + +/bin/meta_writer --data-endpoints tcp://127.0.0.1:10008 diff --git a/dev/two_node_fp/stOdinServer.sh b/dev/two_node_fp/stOdinServer.sh new file mode 100755 index 0000000..c13e8ca --- /dev/null +++ b/dev/two_node_fp/stOdinServer.sh @@ -0,0 +1,11 @@ +#!/bin/bash + +SCRIPT_DIR="$( cd "$( dirname "$0" )" && pwd )" + +# Increase maximum fds available for ZeroMQ sockets +ulimit -n 2048 + +# Wait for other processes to start +sleep 3 + +/bin/odin_control --config=$SCRIPT_DIR/odin_server.cfg --logging=info --access_logging=ERROR diff --git a/pyproject.toml b/pyproject.toml index fca2916..46c27ba 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -42,6 +42,10 @@ dev = [ "types-mock", "types-requests", ] +odin = [ + "odin-control @ git+https://github.com/odin-detector/odin-control@param-tree-replace", + "odin-data[meta_writer] @ git+https://github.com/odin-detector/odin-data@fastcs-dev#subdirectory=python", +] [project.scripts] odin-fastcs = "odin_fastcs.__main__:app" From 91ba168bdddb550d5f6b66396a669e07d78542ce Mon Sep 17 00:00:00 2001 From: Gary Yendell Date: Mon, 17 Jun 2024 16:08:57 +0000 Subject: [PATCH 10/10] Refactor to improve tests --- pyproject.toml | 1 + src/odin_fastcs/odin_controller.py | 59 +-- ...esponse.json => one_node_fp_response.json} | 2 +- tests/input/two_node_fp_response.json | 356 ++++++++++++++++++ tests/test_controllers.py | 100 +++++ tests/test_introspection.py | 12 +- 6 files changed, 499 insertions(+), 31 deletions(-) rename tests/input/{dummy_fp_response.json => one_node_fp_response.json} (99%) create mode 100644 tests/input/two_node_fp_response.json create mode 100644 tests/test_controllers.py diff --git a/pyproject.toml b/pyproject.toml index 46c27ba..2fee4fe 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -33,6 +33,7 @@ dev = [ "pre-commit", "pydata-sphinx-theme>=0.12", "pytest", + "pytest-asyncio", "pytest-cov", "ruff", "sphinx-autobuild", diff --git a/src/odin_fastcs/odin_controller.py b/src/odin_fastcs/odin_controller.py index 17059ed..646a642 100644 --- a/src/odin_fastcs/odin_controller.py +++ b/src/odin_fastcs/odin_controller.py @@ -1,5 +1,6 @@ import asyncio import logging +from collections.abc import Sequence from dataclasses import dataclass from typing import Any @@ -87,11 +88,21 @@ def __init__( self._parameters = parameters self._api_prefix = api_prefix - def create_attributes(self): - """Create ``Attributes`` from Odin server parameter tree.""" - parameters = self.process_odin_parameters(self._parameters) + async def initialise(self): + self._process_parameters() + self._create_attributes() + + def _process_parameters(self): + """Hook to process ``OdinParameters`` before creating ``Attributes``. + + For example, renaming or removing a section of the parameter path. - for parameter in parameters: + """ + pass + + def _create_attributes(self): + """Create controller ``Attributes`` from ``OdinParameters``.""" + for parameter in self._parameters: if "writeable" in parameter.metadata and parameter.metadata["writeable"]: attr_class = AttrRW else: @@ -123,18 +134,6 @@ def create_attributes(self): setattr(self, parameter.name.replace(".", ""), attr) - async def initialise(self): - pass - - def process_odin_parameters( - self, parameters: list[OdinParameter] - ) -> list[OdinParameter]: - """Hook for child classes to process parameters before creating attributes.""" - return parameters - - def create_odin_parameters(self): - return create_odin_parameters(self._parameter_tree) - class OdinController(Controller): """A root ``Controller`` for an Odin control server.""" @@ -176,7 +175,6 @@ async def initialise(self) -> None: self._connection, create_odin_parameters(response), adapter ) await adapter_controller.initialise() - adapter_controller.create_attributes() self.register_sub_controller(adapter_controller) await self._connection.close() @@ -226,7 +224,6 @@ async def initialise(self): [f"FP{idx}"], ) await adapter_controller.initialise() - adapter_controller.create_attributes() self.register_sub_controller(adapter_controller) @@ -245,15 +242,27 @@ async def initialise(self): f"Did not find valid plugins in response:\n{plugins_response}" ) + self._process_parameters() + await self._create_plugin_sub_controllers(plugins) + self._create_attributes() + + def _process_parameters(self): for parameter in self._parameters: # Remove duplicate index from uri parameter.uri = parameter.uri[1:] # Remove redundant status/config from parameter path parameter.set_path(parameter.uri[1:]) + async def _create_plugin_sub_controllers(self, plugins: Sequence[str]): for plugin in plugins: + + def __parameter_in_plugin( + parameter: OdinParameter, plugin: str = plugin + ) -> bool: + return parameter.path[0] == plugin + plugin_parameters, self._parameters = partition( - self._parameters, lambda p, plugin=plugin: p.path[0] == plugin + self._parameters, __parameter_in_plugin ) plugin_controller = OdinFPPluginController( self._connection, @@ -261,22 +270,16 @@ async def initialise(self): f"{self._api_prefix}", self.path + [plugin], ) - plugin_controller.create_attributes() + await plugin_controller.initialise() self.register_sub_controller(plugin_controller) class OdinFPPluginController(OdinSubController): - # TODO: Just use initialise? - def process_odin_parameters( - self, parameters: list[OdinParameter] - ) -> list[OdinParameter]: - for parameter in parameters: + def _process_parameters(self): + for parameter in self._parameters: # Remove plugin name included in controller base path parameter.set_path(parameter.path[1:]) - # TODO: Make a copy? - return parameters - class FROdinController(OdinSubController): def __init__( diff --git a/tests/input/dummy_fp_response.json b/tests/input/one_node_fp_response.json similarity index 99% rename from tests/input/dummy_fp_response.json rename to tests/input/one_node_fp_response.json index 83c4c0f..53f61ce 100644 --- a/tests/input/dummy_fp_response.json +++ b/tests/input/one_node_fp_response.json @@ -187,4 +187,4 @@ } } } -} \ No newline at end of file +} diff --git a/tests/input/two_node_fp_response.json b/tests/input/two_node_fp_response.json new file mode 100644 index 0000000..d706890 --- /dev/null +++ b/tests/input/two_node_fp_response.json @@ -0,0 +1,356 @@ +{ + "api": { + "value": 0.1, + "writeable": false, + "type": "float" + }, + "module": { + "value": "OdinDataAdapter", + "writeable": false, + "type": "str" + }, + "endpoints": [ + { + "ip_address": { + "value": "127.0.0.1", + "writeable": false, + "type": "str" + }, + "port": { + "value": 10004, + "writeable": false, + "type": "int" + } + }, + { + "ip_address": { + "value": "127.0.0.1", + "writeable": false, + "type": "str" + }, + "port": { + "value": 10014, + "writeable": false, + "type": "int" + } + } + ], + "count": { + "value": 2, + "writeable": false, + "type": "int" + }, + "update_interval": { + "value": 0.2, + "writeable": false, + "type": "float" + }, + "0": { + "status": { + "shared_memory": { + "configured": true + }, + "plugins": { + "names": [ + "dummy", + "hdf", + "offset", + "param" + ] + }, + "dummy": { + "packets_lost": 0, + "timing": { + "last_process": 0, + "max_process": 0, + "mean_process": 0 + } + }, + "hdf": { + "writing": false, + "frames_max": 0, + "frames_written": 0, + "frames_processed": 0, + "file_path": "", + "file_name": "", + "acquisition_id": "", + "processes": 2, + "rank": 0, + "timeout_active": false, + "timing": { + "last_create": 0, + "max_create": 0, + "mean_create": 0, + "last_write": 0, + "max_write": 0, + "mean_write": 0, + "last_flush": 0, + "max_flush": 0, + "mean_flush": 0, + "last_close": 0, + "max_close": 0, + "mean_close": 0, + "last_process": 0, + "max_process": 0, + "mean_process": 0 + } + }, + "offset": { + "timing": { + "last_process": 0, + "max_process": 0, + "mean_process": 0 + } + }, + "param": { + "timing": { + "last_process": 0, + "max_process": 0, + "mean_process": 0 + } + }, + "timestamp": "2024-06-14T14:53:35.343227", + "error": [], + "connected": true + }, + "config": { + "ctrl_endpoint": "tcp://0.0.0.0:10004", + "meta_endpoint": "tcp://*:10008", + "fr_setup": { + "fr_ready_cnxn": "tcp://127.0.0.1:10001", + "fr_release_cnxn": "tcp://127.0.0.1:10002" + }, + "dummy": { + "width": 1400, + "height": 1024, + "copy_frame": true + }, + "hdf": { + "process": { + "number": 2, + "rank": 0, + "frames_per_block": 1, + "blocks_per_file": 0, + "earliest_version": false, + "alignment_threshold": 1, + "alignment_value": 1 + }, + "file": { + "path": "", + "prefix": "", + "use_numbers": true, + "first_number": 1, + "postfix": "", + "extension": "h5", + "create_error_duration": 10000, + "write_error_duration": 10000, + "flush_error_duration": 10000, + "close_error_duration": 10000 + }, + "frames": 0, + "master": "", + "acquisition_id": "", + "timeout_timer_period": 0, + "dataset": { + "compressed_size": { + "datatype": "uint32", + "compression": "none", + "blosc_compressor": 0, + "blosc_level": 0, + "blosc_shuffle": 0, + "chunks": [ + 1000 + ] + }, + "data": { + "datatype": "uint8", + "compression": "none", + "blosc_compressor": 0, + "blosc_level": 0, + "blosc_shuffle": 0, + "chunks": [ + 1, + 512, + 256 + ] + }, + "uid": { + "datatype": "uint64", + "compression": "none", + "blosc_compressor": 0, + "blosc_level": 0, + "blosc_shuffle": 0, + "chunks": [ + 1000 + ] + } + } + }, + "offset": { + "offset_adjustment": 0 + }, + "param": { + "parameter": { + "uid": { + "adjustment": 1, + "input": "" + } + } + } + } + }, + "1": { + "status": { + "shared_memory": { + "configured": true + }, + "plugins": { + "names": [ + "dummy", + "hdf", + "offset", + "param" + ] + }, + "dummy": { + "packets_lost": 0, + "timing": { + "last_process": 0, + "max_process": 0, + "mean_process": 0 + } + }, + "hdf": { + "writing": false, + "frames_max": 0, + "frames_written": 0, + "frames_processed": 0, + "file_path": "", + "file_name": "", + "acquisition_id": "", + "processes": 2, + "rank": 1, + "timeout_active": false, + "timing": { + "last_create": 0, + "max_create": 0, + "mean_create": 0, + "last_write": 0, + "max_write": 0, + "mean_write": 0, + "last_flush": 0, + "max_flush": 0, + "mean_flush": 0, + "last_close": 0, + "max_close": 0, + "mean_close": 0, + "last_process": 0, + "max_process": 0, + "mean_process": 0 + } + }, + "offset": { + "timing": { + "last_process": 0, + "max_process": 0, + "mean_process": 0 + } + }, + "param": { + "timing": { + "last_process": 0, + "max_process": 0, + "mean_process": 0 + } + }, + "timestamp": "2024-06-14T14:53:35.363616", + "error": [], + "connected": true + }, + "config": { + "ctrl_endpoint": "tcp://0.0.0.0:10014", + "meta_endpoint": "tcp://*:10018", + "fr_setup": { + "fr_ready_cnxn": "tcp://127.0.0.1:10011", + "fr_release_cnxn": "tcp://127.0.0.1:10012" + }, + "dummy": { + "width": 1400, + "height": 1024, + "copy_frame": true + }, + "hdf": { + "process": { + "number": 2, + "rank": 1, + "frames_per_block": 1, + "blocks_per_file": 0, + "earliest_version": false, + "alignment_threshold": 1, + "alignment_value": 1 + }, + "file": { + "path": "", + "prefix": "", + "use_numbers": true, + "first_number": 1, + "postfix": "", + "extension": "h5", + "create_error_duration": 10000, + "write_error_duration": 10000, + "flush_error_duration": 10000, + "close_error_duration": 10000 + }, + "frames": 0, + "master": "", + "acquisition_id": "", + "timeout_timer_period": 0, + "dataset": { + "compressed_size": { + "datatype": "uint32", + "compression": "none", + "blosc_compressor": 0, + "blosc_level": 0, + "blosc_shuffle": 0, + "chunks": [ + 1000 + ] + }, + "data": { + "datatype": "uint8", + "compression": "none", + "blosc_compressor": 0, + "blosc_level": 0, + "blosc_shuffle": 0, + "chunks": [ + 1, + 512, + 256 + ] + }, + "uid": { + "datatype": "uint64", + "compression": "none", + "blosc_compressor": 0, + "blosc_level": 0, + "blosc_shuffle": 0, + "chunks": [ + 1000 + ] + } + } + }, + "offset": { + "offset_adjustment": 0 + }, + "param": { + "parameter": { + "uid": { + "adjustment": 1, + "input": "" + } + } + } + } + } +} diff --git a/tests/test_controllers.py b/tests/test_controllers.py new file mode 100644 index 0000000..9a0e9f7 --- /dev/null +++ b/tests/test_controllers.py @@ -0,0 +1,100 @@ +from pathlib import Path + +import pytest +from fastcs.attributes import AttrR, AttrRW +from fastcs.datatypes import Bool, Float, Int + +from odin_fastcs.http_connection import HTTPConnection +from odin_fastcs.odin_controller import ( + OdinFPController, + OdinFPPluginController, + OdinSubController, +) +from odin_fastcs.util import OdinParameter + +HERE = Path(__file__).parent + + +def test_create_attributes(): + parameters = [ + OdinParameter(uri=["read_int"], metadata={"type": "int"}), + OdinParameter(uri=["write_bool"], metadata={"type": "bool", "writeable": True}), + OdinParameter(uri=["group", "float"], metadata={"type": "float"}), + ] + controller = OdinSubController(HTTPConnection("", 0), parameters, "api/0.1", []) + + controller._create_attributes() + + match controller: + case OdinSubController( + read_int=AttrR(datatype=Int()), + write_bool=AttrRW(datatype=Bool()), + group_float=AttrR(datatype=Float(), group="Group"), + ): + pass + case _: + pytest.fail("Controller Attributes not as expected") + + +def test_fp_process_parameters(): + parameters = [ + OdinParameter(["0", "status", "hdf", "frames_written"], metadata={}), + OdinParameter(["0", "config", "hdf", "frames"], metadata={}), + ] + + fpc = OdinFPController(HTTPConnection("", 0), parameters, "api/0.1", ["FP"]) + + fpc._process_parameters() + assert fpc._parameters == [ + OdinParameter( + uri=["status", "hdf", "frames_written"], + _path=["hdf", "frames_written"], + metadata={}, + ), + OdinParameter( + uri=["config", "hdf", "frames"], _path=["hdf", "frames"], metadata={} + ), + ] + + +@pytest.mark.asyncio +async def test_fp_create_plugin_sub_controllers(): + parameters = [ + OdinParameter( + uri=["config", "ctrl_endpoint"], + _path=["ctrl_endpoint"], + metadata={"type": "str"}, + ), + OdinParameter( + uri=["status", "hdf", "frames_written"], + _path=["hdf", "frames_written"], + metadata={"type": "int"}, + ), + ] + + fpc = OdinFPController(HTTPConnection("", 0), parameters, "api/0.1", ["FP"]) + + await fpc._create_plugin_sub_controllers(["hdf"]) + + # Check that hdf parameter has been split into a sub controller + assert fpc._parameters == [ + OdinParameter( + uri=["config", "ctrl_endpoint"], + _path=["ctrl_endpoint"], + metadata={"type": "str"}, + ) + ] + match fpc.get_sub_controllers(): + case [ + OdinFPPluginController( + _parameters=[ + OdinParameter( + uri=["status", "hdf", "frames_written"], + _path=["frames_written"], + ) + ] + ) + ]: + pass + case _: + pytest.fail("Sub controllers not as expected") diff --git a/tests/test_introspection.py b/tests/test_introspection.py index eed8056..feff864 100644 --- a/tests/test_introspection.py +++ b/tests/test_introspection.py @@ -6,9 +6,17 @@ HERE = Path(__file__).parent -def test_create_odin_parameters(): - with (HERE / "input/dummy_fp_response.json").open() as f: +def test_one_node(): + with (HERE / "input/one_node_fp_response.json").open() as f: response = json.loads(f.read()) parameters = create_odin_parameters(response) assert len(parameters) == 96 + + +def test_two_node(): + with (HERE / "input/two_node_fp_response.json").open() as f: + response = json.loads(f.read()) + + parameters = create_odin_parameters(response) + assert len(parameters) == 188