From d1870e7ad44d2de01d554ba2d92c3e08f8aba860 Mon Sep 17 00:00:00 2001 From: Giles Knap Date: Wed, 10 May 2023 10:16:17 +0000 Subject: [PATCH 01/16] add values support (broken) --- ibek-defs | 2 +- src/ibek/gen_scripts.py | 22 ++++++++++++++-- src/ibek/support.py | 13 ++++++++++ tests/samples/example-srrfioc08/st.cmd | 2 +- tests/samples/schemas/ibek.defs.schema.json | 28 +++++++++++++++++++++ 5 files changed, 63 insertions(+), 4 deletions(-) diff --git a/ibek-defs b/ibek-defs index e330e5458..4e73b4162 160000 --- a/ibek-defs +++ b/ibek-defs @@ -1 +1 @@ -Subproject commit e330e54581acf8ef1c2cd992556c97ff8436cfb2 +Subproject commit 4e73b4162584a735bec6cdea041b263c75a9ebe4 diff --git a/src/ibek/gen_scripts.py b/src/ibek/gen_scripts.py index a945a7b0c..088a10da2 100644 --- a/src/ibek/gen_scripts.py +++ b/src/ibek/gen_scripts.py @@ -4,7 +4,7 @@ import logging import re from pathlib import Path -from typing import List +from typing import Dict, List from jinja2 import Template from ruamel.yaml.main import YAML @@ -28,13 +28,31 @@ def ioc_deserialize(ioc_instance_yaml: Path, definition_yaml: List[Path]) -> IOC Returns an in memory object graph of the resulting ioc instance """ + all_values: Dict[str, Dict[str, str]] = {} + # Read and load the support module definitions for yaml in definition_yaml: support = Support.deserialize(YAML(typ="safe").load(yaml)) make_entity_classes(support) + # collect all definition 'values' for copying into entity instances + for definition in support.defs: + entity_def_name = f"{support.module}.{definition.name}" + all_values[entity_def_name] = {} + for value in definition.values: + all_values[entity_def_name][value.name] = value.name + # Create an IOC instance from it - return IOC.deserialize(YAML(typ="safe").load(ioc_instance_yaml)) + ioc_instance = IOC.deserialize(YAML(typ="safe").load(ioc_instance_yaml)) + + # copy over the values from the support module definitions to entity instances + for entity in ioc_instance.entities: + values_dict = all_values.get(entity.type) + if values_dict: + for name, value in values_dict.items(): + setattr(entity, name, value) + + return ioc_instance def create_db_script(ioc_instance: IOC, utility: Utils) -> str: diff --git a/src/ibek/support.py b/src/ibek/support.py index 5a72ba2a9..662eed093 100644 --- a/src/ibek/support.py +++ b/src/ibek/support.py @@ -139,6 +139,18 @@ class Once: value: A[str, desc("Startup script snippets defined as Jinja template")] = "" +@dataclass +class Value: + """A calculated string value for a definition""" + + name: A[str, desc("Name of the value that the IOC instance will expose")] + description: A[str, desc("Description of what the value will be used for")] + value: A[str, desc("The contents of the value")] + + def __str__(self): + return self.value + + @dataclass class Definition: """ @@ -148,6 +160,7 @@ class Definition: name: A[str, desc("Publish Definition as type . for IOC instances")] description: A[str, desc("Describes the purpose of the definition")] args: A[Sequence[Arg], desc("The arguments IOC instance should supply")] = () + values: A[Sequence[Value], desc("The values IOC instance should supply")] = () databases: A[Sequence[Database], desc("Databases to instantiate")] = () script: A[ Sequence[Union[str, Function, Once]], diff --git a/tests/samples/example-srrfioc08/st.cmd b/tests/samples/example-srrfioc08/st.cmd index 17b641a7f..646447263 100644 --- a/tests/samples/example-srrfioc08/st.cmd +++ b/tests/samples/example-srrfioc08/st.cmd @@ -15,7 +15,7 @@ ioc_registerRecordDeviceDriver pdbbase # Create a new Hy8002 carrier. # The resulting carrier handle is saved in an env variable. ipacAddHy8002 "4, 2" -epicsEnvSet IPAC4 0 +epicsEnvSet IPAC4 # Hy8401ipConfigure CardId IPACid IpSiteNumber InterruptVector InterruptEnable AiType ExternalClock ClockRate Inhibit SampleCount SampleSpacing SampleSize # IpSlot 0=A 1=B 2=C 3=D diff --git a/tests/samples/schemas/ibek.defs.schema.json b/tests/samples/schemas/ibek.defs.schema.json index 94e98270d..3163e7620 100644 --- a/tests/samples/schemas/ibek.defs.schema.json +++ b/tests/samples/schemas/ibek.defs.schema.json @@ -207,6 +207,34 @@ "description": "The arguments IOC instance should supply", "default": [] }, + "values": { + "type": "array", + "items": { + "type": "object", + "properties": { + "name": { + "type": "string", + "description": "Name of the value that the IOC instance will expose" + }, + "description": { + "type": "string", + "description": "Description of what the value will be used for" + }, + "value": { + "type": "string", + "description": "The contents of the value" + } + }, + "required": [ + "name", + "description", + "value" + ], + "additionalProperties": false + }, + "description": "The values IOC instance should supply", + "default": [] + }, "databases": { "type": "array", "items": { From d9c2e546d46778909b0e6444743074991c29b80e Mon Sep 17 00:00:00 2001 From: Giles Knap Date: Wed, 10 May 2023 11:36:08 +0000 Subject: [PATCH 02/16] use Callable in render_elements instead of str --- src/ibek/render.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/src/ibek/render.py b/src/ibek/render.py index cb34c52e7..38ee29ccd 100644 --- a/src/ibek/render.py +++ b/src/ibek/render.py @@ -3,7 +3,7 @@ """ from dataclasses import asdict -from typing import Any, Dict, List, Optional +from typing import Any, Callable, Dict, List, Optional from jinja2 import Template @@ -29,6 +29,7 @@ def _to_dict(self, instance: Any) -> Dict[str, Any]: jinja templates """ result = asdict(instance) + result.update(instance.__class__.__dict__) result["__utils__"] = self.utils return result @@ -170,11 +171,10 @@ def render_post_ioc_init(self, instance: Entity) -> Optional[str]: return script - def render_elements(self, ioc: IOC, element: str) -> str: + def render_elements(self, ioc: IOC, method: Callable) -> str: """ Render elements of a given IOC instance based on calling the correct method """ - method = getattr(self, element) elements = "" for instance in ioc.entities: if instance.entity_enabled: @@ -187,22 +187,22 @@ def render_script_elements(self, ioc: IOC) -> str: """ Render all of the startup script entries for a given IOC instance """ - return self.render_elements(ioc, "render_script") + return self.render_elements(ioc, self.render_script) def render_database_elements(self, ioc: IOC) -> str: """ Render all of the DBLoadRecords entries for a given IOC instance """ - return self.render_elements(ioc, "render_database") + return self.render_elements(ioc, self.render_database) def render_environment_variable_elements(self, ioc: IOC) -> str: """ Render all of the environment variable entries for a given IOC instance """ - return self.render_elements(ioc, "render_environment_variables") + return self.render_elements(ioc, self.render_environment_variables) def render_post_ioc_init_elements(self, ioc: IOC) -> str: """ Render all of the post-iocInit elements for a given IOC instance """ - return self.render_elements(ioc, "render_post_ioc_init") + return self.render_elements(ioc, self.render_post_ioc_init) From 7ad2ce2cea25f47630c308391e6d9bdc837f1b3c Mon Sep 17 00:00:00 2001 From: Giles Knap Date: Wed, 10 May 2023 13:06:26 +0000 Subject: [PATCH 03/16] creating jinja context with Entity.__post_init__ --- src/ibek/gen_scripts.py | 34 +++++++++++-------- src/ibek/ioc.py | 12 ++++++- src/ibek/render.py | 6 ++-- .../SR-RF-IOC-08.ibek.ioc.yaml | 16 +++++++-- tests/samples/example-srrfioc08/st.cmd | 6 +++- .../schemas/all.ibek.support.schema.json | 6 +++- 6 files changed, 56 insertions(+), 24 deletions(-) diff --git a/src/ibek/gen_scripts.py b/src/ibek/gen_scripts.py index 088a10da2..e88a38076 100644 --- a/src/ibek/gen_scripts.py +++ b/src/ibek/gen_scripts.py @@ -3,6 +3,7 @@ """ import logging import re +from dataclasses import asdict from pathlib import Path from typing import Dict, List @@ -11,7 +12,7 @@ from .ioc import IOC, make_entity_classes from .render import Render -from .support import Support +from .support import Definition, Support from .utils import Utils log = logging.getLogger(__name__) @@ -28,31 +29,34 @@ def ioc_deserialize(ioc_instance_yaml: Path, definition_yaml: List[Path]) -> IOC Returns an in memory object graph of the resulting ioc instance """ - all_values: Dict[str, Dict[str, str]] = {} + all_values: Dict[str, str] = {} # Read and load the support module definitions for yaml in definition_yaml: support = Support.deserialize(YAML(typ="safe").load(yaml)) - make_entity_classes(support) - - # collect all definition 'values' for copying into entity instances for definition in support.defs: - entity_def_name = f"{support.module}.{definition.name}" - all_values[entity_def_name] = {} for value in definition.values: - all_values[entity_def_name][value.name] = value.name + all_values[value.name] = value.value + make_entity_classes(support) + for definition in support.defs: + make_entity_context(definition) # Create an IOC instance from it ioc_instance = IOC.deserialize(YAML(typ="safe").load(ioc_instance_yaml)) + return ioc_instance - # copy over the values from the support module definitions to entity instances - for entity in ioc_instance.entities: - values_dict = all_values.get(entity.type) - if values_dict: - for name, value in values_dict.items(): - setattr(entity, name, value) - return ioc_instance +def make_entity_context(definition: Definition): + """ + Create a context dictionary for the given `Entity` instance + This is for use in Jinja expansion of instances of this Entity + """ + context = asdict(definition) + + for value in definition.values: + context[value.name] = value.value + + setattr(definition, "__context__", context) def create_db_script(ioc_instance: IOC, utility: Utils) -> str: diff --git a/src/ibek/ioc.py b/src/ibek/ioc.py index 022e86686..81a7729bf 100644 --- a/src/ibek/ioc.py +++ b/src/ibek/ioc.py @@ -6,7 +6,7 @@ import builtins import types -from dataclasses import Field, dataclass, field, make_dataclass +from dataclasses import Field, asdict, dataclass, field, make_dataclass from typing import Any, Dict, List, Mapping, Sequence, Tuple, Type, cast from apischema import ( @@ -50,6 +50,12 @@ def __post_init__(self): assert inst_id not in id_to_entity, f"Already got an instance {inst_id}" id_to_entity[inst_id] = self + # create a context dictionary for use in Jinja expansion of this Entity + context = asdict(self) + # todo jinja expand all values with context + self.__context__ = context + # then pass __context__ to jinja template in render.py + id_to_entity: Dict[str, Entity] = {} @@ -112,6 +118,10 @@ def lookup_instance(id): # it fields.append(("entity_enabled", bool, field(default=cast(Any, True)))) + # add in the values fields as class attributes + for value in definition.values: + fields.append((value.name, str, field(default=cast(Any, value.value)))) + namespace = dict(__definition__=definition) # make the Entity derived dataclass for this EntityClass, with a reference diff --git a/src/ibek/render.py b/src/ibek/render.py index 38ee29ccd..37ec14aac 100644 --- a/src/ibek/render.py +++ b/src/ibek/render.py @@ -2,7 +2,6 @@ Functions for rendering lines in the boot script using Jinja2 """ -from dataclasses import asdict from typing import Any, Callable, Dict, List, Optional from jinja2 import Template @@ -23,13 +22,12 @@ def __init__(self, utils: Utils): self.utils = utils self.once_done: List[str] = [] - def _to_dict(self, instance: Any) -> Dict[str, Any]: + def _to_dict(self, instance: Entity) -> Dict[str, Any]: """ add the global utils object to the instance so we can use them in the jinja templates """ - result = asdict(instance) - result.update(instance.__class__.__dict__) + result = instance.__context__ result["__utils__"] = self.utils return result diff --git a/tests/samples/example-srrfioc08/SR-RF-IOC-08.ibek.ioc.yaml b/tests/samples/example-srrfioc08/SR-RF-IOC-08.ibek.ioc.yaml index ceadc554d..db3d5a4cb 100644 --- a/tests/samples/example-srrfioc08/SR-RF-IOC-08.ibek.ioc.yaml +++ b/tests/samples/example-srrfioc08/SR-RF-IOC-08.ibek.ioc.yaml @@ -16,19 +16,31 @@ entities: name: Vec0 - type: epics.InterruptVectorVME name: Vec1 + - type: epics.InterruptVectorVME + name: Vec2 - type: ipac.Hy8002 name: IPAC4 slot: 4 + - type: ipac.Hy8002 + name: IPAC5 + slot: 5 + - type: Hy8401ip.Hy8401ip - name: SlotA + name: SlotA_Card4 carrier: IPAC4 ip_site_number: 0 vector: Vec0 - type: Hy8401ip.Hy8401ip - name: SlotC + name: SlotC_Card4 carrier: IPAC4 ip_site_number: 2 vector: Vec1 + + - type: Hy8401ip.Hy8401ip + name: SlotA_Card5 + carrier: IPAC5 + ip_site_number: 0 + vector: Vec2 diff --git a/tests/samples/example-srrfioc08/st.cmd b/tests/samples/example-srrfioc08/st.cmd index 646447263..6b97d3e33 100644 --- a/tests/samples/example-srrfioc08/st.cmd +++ b/tests/samples/example-srrfioc08/st.cmd @@ -6,6 +6,7 @@ epicsEnvSet EPICS_TS_NTP_INET 172.23.194.5 epicsEnvSet EPICS_TS_MIN_WEST 0 epicsEnvSet Vec0 192 epicsEnvSet Vec1 193 +epicsEnvSet Vec2 194 dbLoadDatabase dbd/ioc.dbd ioc_registerRecordDeviceDriver pdbbase @@ -15,13 +16,16 @@ ioc_registerRecordDeviceDriver pdbbase # Create a new Hy8002 carrier. # The resulting carrier handle is saved in an env variable. ipacAddHy8002 "4, 2" -epicsEnvSet IPAC4 +epicsEnvSet IPAC4 0 +ipacAddHy8002 "5, 2" +epicsEnvSet IPAC5 1 # Hy8401ipConfigure CardId IPACid IpSiteNumber InterruptVector InterruptEnable AiType ExternalClock ClockRate Inhibit SampleCount SampleSpacing SampleSize # IpSlot 0=A 1=B 2=C 3=D # ClockRate 0=1Hz 1=2Hz 2=5Hz 3=10Hz 4=20Hz 5=50Hz 6=100Hz7=200Hz 8=500Hz 9=1kHz 10=2kHz11=5kHz 12=10kHz 13=20kHz 14=50kHz 15=100kHz Hy8401ipConfigure 40 $(IPAC4) 0 $(Vec0) 0 0 0 15 0 1 1 0 Hy8401ipConfigure 42 $(IPAC4) 2 $(Vec1) 0 0 0 15 0 1 1 0 +Hy8401ipConfigure 50 $(IPAC5) 0 $(Vec2) 0 0 0 15 0 1 1 0 dbLoadRecords /tmp/ioc.db iocInit diff --git a/tests/samples/schemas/all.ibek.support.schema.json b/tests/samples/schemas/all.ibek.support.schema.json index 558b9c852..b54085b73 100644 --- a/tests/samples/schemas/all.ibek.support.schema.json +++ b/tests/samples/schemas/all.ibek.support.schema.json @@ -423,7 +423,7 @@ }, "name": { "type": "string", - "description": "IPAC identifier (suggested: IPAC{{ slot }})", + "description": "IPAC identifier (suggested: IPAC)", "vscode_ibek_plugin_type": "type_id" }, "slot": { @@ -439,6 +439,10 @@ "type": "string", "const": "ipac.Hy8002", "default": "ipac.Hy8002" + }, + "card_id": { + "type": "string", + "default": "{{ __utils__.counter(\"Carriers\", start=0) }}" } }, "required": [ From 6b1d3c79bf1bab0d9a00bdf18fa40e6e7a98d340 Mon Sep 17 00:00:00 2001 From: Giles Knap Date: Wed, 10 May 2023 13:18:28 +0000 Subject: [PATCH 04/16] Insert Utils object at context creation time Thus saving passing it around everywhere. --- src/ibek/__main__.py | 7 ++----- src/ibek/gen_scripts.py | 11 ++++------- src/ibek/ioc.py | 4 ++++ src/ibek/render.py | 5 +---- src/ibek/utils.py | 3 +-- tests/samples/example-srrfioc08/st.cmd | 2 +- 6 files changed, 13 insertions(+), 19 deletions(-) diff --git a/src/ibek/__main__.py b/src/ibek/__main__.py index beb55fcda..62bf56324 100644 --- a/src/ibek/__main__.py +++ b/src/ibek/__main__.py @@ -12,7 +12,6 @@ from .gen_scripts import create_boot_script, create_db_script, ioc_deserialize from .ioc import IOC, make_entity_classes from .support import Support -from .utils import Utils cli = typer.Typer() yaml = YAML() @@ -100,16 +99,14 @@ def build_startup( ioc_instance = ioc_deserialize(instance, definitions) - utility = Utils(ioc_instance.ioc_name) - - script_txt = create_boot_script(ioc_instance, utility) + script_txt = create_boot_script(ioc_instance) out.parent.mkdir(parents=True, exist_ok=True) with out.open("w") as stream: stream.write(script_txt) - db_txt = create_db_script(ioc_instance, utility) + db_txt = create_db_script(ioc_instance) with db_out.open("w") as stream: stream.write(db_txt) diff --git a/src/ibek/gen_scripts.py b/src/ibek/gen_scripts.py index e88a38076..29623f313 100644 --- a/src/ibek/gen_scripts.py +++ b/src/ibek/gen_scripts.py @@ -13,7 +13,6 @@ from .ioc import IOC, make_entity_classes from .render import Render from .support import Definition, Support -from .utils import Utils log = logging.getLogger(__name__) @@ -59,32 +58,30 @@ def make_entity_context(definition: Definition): setattr(definition, "__context__", context) -def create_db_script(ioc_instance: IOC, utility: Utils) -> str: +def create_db_script(ioc_instance: IOC) -> str: """ Create make_db.sh script for expanding the database templates """ with open(TEMPLATES / "make_db.jinja", "r") as f: template = Template(f.read()) - renderer = Render(utility) + renderer = Render() return template.render( - __util__=utility, database_elements=renderer.render_database_elements(ioc_instance), ) -def create_boot_script(ioc_instance: IOC, utility: Utils) -> str: +def create_boot_script(ioc_instance: IOC) -> str: """ Create the boot script for an IOC """ with open(TEMPLATES / "st.cmd.jinja", "r") as f: template = Template(f.read()) - renderer = Render(utility) + renderer = Render() return template.render( - __util__=utility, env_var_elements=renderer.render_environment_variable_elements(ioc_instance), script_elements=renderer.render_script_elements(ioc_instance), post_ioc_init_elements=renderer.render_post_ioc_init_elements(ioc_instance), diff --git a/src/ibek/ioc.py b/src/ibek/ioc.py index 81a7729bf..45fb7a63a 100644 --- a/src/ibek/ioc.py +++ b/src/ibek/ioc.py @@ -26,6 +26,7 @@ from . import modules from .globals import T, desc from .support import Definition, IdArg, ObjectArg, Support +from .utils import Utils class Entity: @@ -36,6 +37,8 @@ class Entity: # a link back to the Definition Object that generated this Definition __definition__: Definition + # a singleton Utility object for sharing state across all Entity renders + __utils__: Utils = Utils() entity_enabled: bool @@ -52,6 +55,7 @@ def __post_init__(self): # create a context dictionary for use in Jinja expansion of this Entity context = asdict(self) + context["__utils__"] = self.__utils__ # todo jinja expand all values with context self.__context__ = context # then pass __context__ to jinja template in render.py diff --git a/src/ibek/render.py b/src/ibek/render.py index 37ec14aac..eb79a3d12 100644 --- a/src/ibek/render.py +++ b/src/ibek/render.py @@ -8,7 +8,6 @@ from .ioc import IOC, Entity from .support import Function, Once -from .utils import Utils class Render: @@ -18,8 +17,7 @@ class Render: definition yaml with substitution values supplied in ioc entity yaml """ - def __init__(self, utils: Utils): - self.utils = utils + def __init__(self: "Render"): self.once_done: List[str] = [] def _to_dict(self, instance: Entity) -> Dict[str, Any]: @@ -28,7 +26,6 @@ def _to_dict(self, instance: Entity) -> Dict[str, Any]: jinja templates """ result = instance.__context__ - result["__utils__"] = self.utils return result diff --git a/src/ibek/utils.py b/src/ibek/utils.py index 5af69d82d..e12b398f6 100644 --- a/src/ibek/utils.py +++ b/src/ibek/utils.py @@ -34,8 +34,7 @@ class Utils: A Utility class for adding functions to the Jinja context """ - def __init__(self, ioc_name: str): - self.ioc_name = ioc_name + def __init__(self: "Utils"): self.variables: Dict[str, Any] = {} self.counters: Dict[str, Counter] = {} diff --git a/tests/samples/example-srrfioc08/st.cmd b/tests/samples/example-srrfioc08/st.cmd index 6b97d3e33..d4fe86a9e 100644 --- a/tests/samples/example-srrfioc08/st.cmd +++ b/tests/samples/example-srrfioc08/st.cmd @@ -14,7 +14,7 @@ ioc_registerRecordDeviceDriver pdbbase # ipacAddHy8002 "slot, interrupt_level" # Create a new Hy8002 carrier. -# The resulting carrier handle is saved in an env variable. +# The resulting carrier handle (card id) is saved in an env variable. ipacAddHy8002 "4, 2" epicsEnvSet IPAC4 0 ipacAddHy8002 "5, 2" From de87323b5545dbc199f0ca68c70fc4ff09594175 Mon Sep 17 00:00:00 2001 From: Giles Knap Date: Wed, 10 May 2023 13:23:28 +0000 Subject: [PATCH 05/16] remove redundant _as_dict from Render --- ibek-defs | 2 +- src/ibek/render.py | 21 ++++++--------------- 2 files changed, 7 insertions(+), 16 deletions(-) diff --git a/ibek-defs b/ibek-defs index 4e73b4162..9643225a1 160000 --- a/ibek-defs +++ b/ibek-defs @@ -1 +1 @@ -Subproject commit 4e73b4162584a735bec6cdea041b263c75a9ebe4 +Subproject commit 9643225a104afdcc5d01a6e24ae668c60bbe3e37 diff --git a/src/ibek/render.py b/src/ibek/render.py index eb79a3d12..f88536093 100644 --- a/src/ibek/render.py +++ b/src/ibek/render.py @@ -2,7 +2,7 @@ Functions for rendering lines in the boot script using Jinja2 """ -from typing import Any, Callable, Dict, List, Optional +from typing import Callable, List, Optional from jinja2 import Template @@ -20,15 +20,6 @@ class Render: def __init__(self: "Render"): self.once_done: List[str] = [] - def _to_dict(self, instance: Entity) -> Dict[str, Any]: - """ - add the global utils object to the instance so we can use them in the - jinja templates - """ - result = instance.__context__ - - return result - def render_text(self, instance: Entity, text: str, once=False, suffix="") -> str: """ render a line of jinja template in ``text`` using the values supplied @@ -48,7 +39,7 @@ def render_text(self, instance: Entity, text: str, once=False, suffix="") -> str return "" jinja_template = Template(text) - result = jinja_template.render(self._to_dict(instance)) # type: ignore + result = jinja_template.render(instance.__context__) # type: ignore # run the result through jinja again so we can refer to args for arg defaults # e.g. @@ -59,7 +50,7 @@ def render_text(self, instance: Entity, text: str, once=False, suffix="") -> str # default: "IPAC{{ slot }}" jinja_template = Template(result) - result = jinja_template.render(self._to_dict(instance)) # type: ignore + result = jinja_template.render(instance.__context__) # type: ignore if result == "": return "" @@ -130,12 +121,12 @@ def render_database(self, instance: Entity) -> Optional[str]: ) jinja_template = Template(jinja_txt) - db_txt = jinja_template.render(self._to_dict(instance)) # type: ignore + db_txt = jinja_template.render(instance.__context__) # type: ignore # run the result through jinja again so we can refer to args for arg defaults db_template = Template(db_txt) - db_txt = db_template.render(self._to_dict(instance)) # type: ignore + db_txt = db_template.render(instance.__context__) # type: ignore return db_txt + "\n" @@ -152,7 +143,7 @@ def render_environment_variables(self, instance: Entity) -> Optional[str]: for variable in variables: # Substitute the name and value of the environment variable from args env_template = Template(f"epicsEnvSet {variable.name} {variable.value}") - env_var_txt += env_template.render(self._to_dict(instance)) + env_var_txt += env_template.render(instance.__context__) return env_var_txt + "\n" def render_post_ioc_init(self, instance: Entity) -> Optional[str]: From 573dc924243b5c555a0a59ebae72d4f643d8b462 Mon Sep 17 00:00:00 2001 From: Giles Knap Date: Wed, 10 May 2023 13:34:02 +0000 Subject: [PATCH 06/16] fix tests --- src/ibek/utils.py | 7 +++++++ tests/test_ioc.py | 4 +++- tests/test_render.py | 10 +++++----- 3 files changed, 15 insertions(+), 6 deletions(-) diff --git a/src/ibek/utils.py b/src/ibek/utils.py index e12b398f6..cdbb5e5c3 100644 --- a/src/ibek/utils.py +++ b/src/ibek/utils.py @@ -35,6 +35,13 @@ class Utils: """ def __init__(self: "Utils"): + self.__reset__() + + def __reset__(self: "Utils"): + """ + Reset all saved state. For use in testing where more than one + IOC is rendered in a single session + """ self.variables: Dict[str, Any] = {} self.counters: Dict[str, Counter] = {} diff --git a/tests/test_ioc.py b/tests/test_ioc.py index eaf3a63fe..dbbbe42e2 100644 --- a/tests/test_ioc.py +++ b/tests/test_ioc.py @@ -2,7 +2,7 @@ from typer.testing import CliRunner -from ibek.ioc import clear_entity_classes +from ibek.ioc import Entity, clear_entity_classes from .test_cli import run_cli @@ -18,6 +18,7 @@ def test_example_ioc(tmp_path: Path, samples: Path, ibek_defs: Path): verifies that it starts up correctly. """ clear_entity_classes() + Entity.__utils__.__reset__() tmp_path = Path("/tmp/ibek_test") tmp_path.mkdir(exist_ok=True) @@ -59,6 +60,7 @@ def test_example_sr_rf_08(tmp_path: Path, samples: Path, ibek_defs: Path): """ clear_entity_classes() + Entity.__utils__.__reset__() tmp_path = Path("/tmp/ibek_test2") tmp_path.mkdir(exist_ok=True) diff --git a/tests/test_render.py b/tests/test_render.py index 0f0c959f1..c1cc74add 100644 --- a/tests/test_render.py +++ b/tests/test_render.py @@ -12,7 +12,7 @@ def test_pmac_asyn_ip_port_script(pmac_classes): generated_class = pmac_classes.PmacAsynIPPort pmac_asyn_ip = generated_class(name="my_pmac_instance", IP="111.111.111.111") - render = Render("test_ioc") + render = Render() script_txt = render.render_script(pmac_asyn_ip) assert script_txt == "pmacAsynIPConfigure(my_pmac_instance, 111.111.111.111:1025)\n" @@ -30,7 +30,7 @@ def test_geobrick_script(pmac_classes): movingPoll=800, ) - render = Render("test_ioc") + render = Render() script_txt = render.render_script(pmac_geobrick_instance) assert ( @@ -51,7 +51,7 @@ def test_geobrick_database(pmac_classes): movingPoll=800, ) - render = Render("test_ioc") + render = Render() db_txt = render.render_database(pmac_geobrick_instance) assert ( @@ -68,7 +68,7 @@ def test_epics_environment_variables(epics_classes): generated_class = epics_classes.EpicsCaMaxArrayBytes max_array_bytes_instance = generated_class(max_bytes=10000000) - render = Render("test_ioc") + render = Render() env_text = render.render_environment_variables(max_array_bytes_instance) assert env_text == "epicsEnvSet EPICS_CA_MAX_ARRAY_BYTES 10000000\n" @@ -153,7 +153,7 @@ def test_entity_disabled_does_not_render_elements(pmac_classes, epics_classes): "pmacCreateController(geobrick_enabled, geobrick_one_port, 0, 8, 800, 200)\n" "pmacCreateAxes(geobrick_enabled, 8)\n" ) - render = Render("test_ioc") + render = Render() script = render.render_script_elements(ioc) assert script == expected_script From 1a01af2401b3c2519242ed189aa4ec68cecf3f55 Mon Sep 17 00:00:00 2001 From: Giles Knap Date: Wed, 10 May 2023 19:03:58 +0000 Subject: [PATCH 07/16] make values a ClassVar of Entity --- src/ibek/ioc.py | 32 +++++++++++++++++++++++++++----- 1 file changed, 27 insertions(+), 5 deletions(-) diff --git a/src/ibek/ioc.py b/src/ibek/ioc.py index 45fb7a63a..90e319761 100644 --- a/src/ibek/ioc.py +++ b/src/ibek/ioc.py @@ -6,8 +6,19 @@ import builtins import types -from dataclasses import Field, asdict, dataclass, field, make_dataclass -from typing import Any, Dict, List, Mapping, Sequence, Tuple, Type, cast +from dataclasses import Field, dataclass, field, make_dataclass +from typing import ( + Any, + ClassVar, + Dict, + List, + Mapping, + Sequence, + Tuple, + Type, + cast, + get_type_hints, +) from apischema import ( Undefined, @@ -42,7 +53,7 @@ class Entity: entity_enabled: bool - def __post_init__(self): + def __post_init__(self: "Entity"): # If there is an argument which is an id then allow deserialization by that args = self.__definition__.args ids = set(a.name for a in args if isinstance(a, IdArg)) @@ -54,8 +65,13 @@ def __post_init__(self): id_to_entity[inst_id] = self # create a context dictionary for use in Jinja expansion of this Entity - context = asdict(self) + context: Dict[str, Any] = {} + for attribute in get_type_hints(self).keys(): + context[attribute] = getattr(self, attribute) + + # add in the global __utils__ object for state sharing context["__utils__"] = self.__utils__ + # todo jinja expand all values with context self.__context__ = context # then pass __context__ to jinja template in render.py @@ -124,7 +140,13 @@ def lookup_instance(id): # add in the values fields as class attributes for value in definition.values: - fields.append((value.name, str, field(default=cast(Any, value.value)))) + fields.append( + ( + value.name, + ClassVar[str], # type: ignore + field(default=cast(Any, value.value)), + ) + ) namespace = dict(__definition__=definition) From 4c80f84cae7a3abcae0c14afb9a11dcd9c907e3a Mon Sep 17 00:00:00 2001 From: Giles Knap Date: Wed, 10 May 2023 19:11:33 +0000 Subject: [PATCH 08/16] remove redundant changes to ioc_deserialize --- src/ibek/gen_scripts.py | 27 +++------------------------ 1 file changed, 3 insertions(+), 24 deletions(-) diff --git a/src/ibek/gen_scripts.py b/src/ibek/gen_scripts.py index 29623f313..984896f9c 100644 --- a/src/ibek/gen_scripts.py +++ b/src/ibek/gen_scripts.py @@ -3,16 +3,15 @@ """ import logging import re -from dataclasses import asdict from pathlib import Path -from typing import Dict, List +from typing import List from jinja2 import Template from ruamel.yaml.main import YAML from .ioc import IOC, make_entity_classes from .render import Render -from .support import Definition, Support +from .support import Support log = logging.getLogger(__name__) @@ -28,34 +27,14 @@ def ioc_deserialize(ioc_instance_yaml: Path, definition_yaml: List[Path]) -> IOC Returns an in memory object graph of the resulting ioc instance """ - all_values: Dict[str, str] = {} # Read and load the support module definitions for yaml in definition_yaml: support = Support.deserialize(YAML(typ="safe").load(yaml)) - for definition in support.defs: - for value in definition.values: - all_values[value.name] = value.value make_entity_classes(support) - for definition in support.defs: - make_entity_context(definition) # Create an IOC instance from it - ioc_instance = IOC.deserialize(YAML(typ="safe").load(ioc_instance_yaml)) - return ioc_instance - - -def make_entity_context(definition: Definition): - """ - Create a context dictionary for the given `Entity` instance - This is for use in Jinja expansion of instances of this Entity - """ - context = asdict(definition) - - for value in definition.values: - context[value.name] = value.value - - setattr(definition, "__context__", context) + return IOC.deserialize(YAML(typ="safe").load(ioc_instance_yaml)) def create_db_script(ioc_instance: IOC) -> str: From 8da08fad130bfcff38b59fe975172aa403a1e18a Mon Sep 17 00:00:00 2001 From: Giles Knap Date: Thu, 11 May 2023 06:46:26 +0000 Subject: [PATCH 09/16] add a test for repeated use of counter based value --- .../values_test/epics.ibek.support.yaml | 18 +++++++++ .../values_test/ipac.ibek.support.yaml | 36 +++++++++++++++++ tests/samples/values_test/st.cmd | 19 +++++++++ .../samples/values_test/values.ibek.ioc.yaml | 13 ++++++ tests/test_ioc.py | 40 +++++++++++++++++++ 5 files changed, 126 insertions(+) create mode 100644 tests/samples/values_test/epics.ibek.support.yaml create mode 100644 tests/samples/values_test/ipac.ibek.support.yaml create mode 100644 tests/samples/values_test/st.cmd create mode 100644 tests/samples/values_test/values.ibek.ioc.yaml diff --git a/tests/samples/values_test/epics.ibek.support.yaml b/tests/samples/values_test/epics.ibek.support.yaml new file mode 100644 index 000000000..e02b17df3 --- /dev/null +++ b/tests/samples/values_test/epics.ibek.support.yaml @@ -0,0 +1,18 @@ +# yaml-language-server: $schema=../../../ibek-defs/_global/ibek.defs.schema.json +module: epics +defs: + - name: InterruptVectorVME + description: Reserve a VME interrupt vector + args: + - type: id + name: name + description: A name for an interrupt vector variable + + - type: int + name: count + description: The number of interrupt vectors to reserve + default: 1 + + env_vars: + - name: "{{ name }}" + value: '{{ __utils__.counter("InterruptVector", start=192, stop=255, inc=count) }}' diff --git a/tests/samples/values_test/ipac.ibek.support.yaml b/tests/samples/values_test/ipac.ibek.support.yaml new file mode 100644 index 000000000..da21eb2f4 --- /dev/null +++ b/tests/samples/values_test/ipac.ibek.support.yaml @@ -0,0 +1,36 @@ +# yaml-language-server: $schema=../../../ibek-defs/_global/ibek.defs.schema.json + +module: ipac + +defs: + - name: Hy8002 + description: adds a Hy8002 carrier card to the IOC + args: + - type: id + name: name + description: "IPAC identifier (suggested: IPAC)" + + - type: int + name: slot + description: Crate Slot number + + - type: int + name: int_level + description: Interrupt level + default: 2 + + values: + - name: card_id + value: '{{ __utils__.counter("Carriers", start=0) }}' + description: Carrier Card Identifier + + script: + - type: function + name: ipacAddHy8002 + header: | + # Create a new Hy8002 carrier. + # The resulting carrier handle (card id) is saved in an env variable. + args: + # args are combined into a single string - hence the quoting below + '"slot, interrupt_level"': '"{{ slot }}, {{ int_level }}"' + - epicsEnvSet {{ name }} {{ card_id }} {{ card_id }} {{ card_id }} diff --git a/tests/samples/values_test/st.cmd b/tests/samples/values_test/st.cmd new file mode 100644 index 000000000..e00ff9126 --- /dev/null +++ b/tests/samples/values_test/st.cmd @@ -0,0 +1,19 @@ +# EPICS IOC Startup Script generated by https://github.com/epics-containers/ibek + +cd "/repos/epics/ioc" + +epicsEnvSet Vec0 192 + +dbLoadDatabase dbd/ioc.dbd +ioc_registerRecordDeviceDriver pdbbase + + +# ipacAddHy8002 "slot, interrupt_level" +# Create a new Hy8002 carrier. +# The resulting carrier handle (card id) is saved in an env variable. +ipacAddHy8002 "4, 2" +epicsEnvSet IPAC4 0 1 2 + +dbLoadRecords /tmp/ioc.db +iocInit + diff --git a/tests/samples/values_test/values.ibek.ioc.yaml b/tests/samples/values_test/values.ibek.ioc.yaml new file mode 100644 index 000000000..06c7dc9a1 --- /dev/null +++ b/tests/samples/values_test/values.ibek.ioc.yaml @@ -0,0 +1,13 @@ +# yaml-language-server: $schema=../schemas/all.ibek.support.schema.json + +ioc_name: sr-rf-ioc-08 +generic_ioc_image: none +description: test power supply IOC + +entities: + - type: epics.InterruptVectorVME + name: Vec0 + + - type: ipac.Hy8002 + name: IPAC4 + slot: 4 diff --git a/tests/test_ioc.py b/tests/test_ioc.py index dbbbe42e2..4ef33533b 100644 --- a/tests/test_ioc.py +++ b/tests/test_ioc.py @@ -90,3 +90,43 @@ def test_example_sr_rf_08(tmp_path: Path, samples: Path, ibek_defs: Path): assert example_boot == actual_boot assert example_db == actual_db + + +def test_values_ioc(tmp_path: Path, samples: Path, ibek_defs: Path): + """ + build values ioc from yaml and verify the result + + This IOC verifies that repeated reference to a 'values' field + with counter gets the same value every time. + + TODO: IMPORTANT: this test currently proves that the values are NOT the same + TODO: make sure samples/values_test/st.cmd is updated when this is fixed. + """ + + clear_entity_classes() + Entity.__utils__.__reset__() + + tmp_path = Path("/tmp/ibek_test2") + tmp_path.mkdir(exist_ok=True) + test_path = samples / "values_test" + + entity_file = test_path / "values.ibek.ioc.yaml" + definition_files = test_path.glob("*.support.yaml") + out_file = tmp_path / "new_dir" / "st.cmd" + out_db = tmp_path / "new_dir" / "make_db.sh" + + params = [ + "build-startup", + entity_file, + "--out", + out_file, + "--db-out", + out_db, + ] + params += definition_files + + run_cli(*params) + + example_boot = (test_path / "st.cmd").read_text() + actual_boot = out_file.read_text() + assert example_boot == actual_boot From 0c8d5a45bfdf364433c932edfbe28053e1935108 Mon Sep 17 00:00:00 2001 From: Giles Knap Date: Thu, 11 May 2023 07:16:39 +0000 Subject: [PATCH 10/16] remove trailing spaces in function renders --- src/ibek/render.py | 4 +-- tests/samples/example-srrfioc08/st.cmd | 16 +++++----- tests/samples/generate_samples.sh | 5 ++++ .../schemas/all.ibek.support.schema.json | 4 --- tests/samples/test_values | 30 +++++++++++++++++++ tests/samples/values_test/st.cmd | 5 ++-- 6 files changed, 46 insertions(+), 18 deletions(-) create mode 100644 tests/samples/test_values diff --git a/src/ibek/render.py b/src/ibek/render.py index f88536093..e482d09cb 100644 --- a/src/ibek/render.py +++ b/src/ibek/render.py @@ -71,9 +71,9 @@ def render_function(self, instance: Entity, function: Function) -> str: call += f"{value} " text = ( - self.render_text(instance, comment, once=True, suffix="func") + self.render_text(instance, comment.strip(), once=True, suffix="func") + self.render_text(instance, function.header, once=True, suffix="func_hdr") - + self.render_text(instance, call) + + self.render_text(instance, call.strip()) ) return text diff --git a/tests/samples/example-srrfioc08/st.cmd b/tests/samples/example-srrfioc08/st.cmd index d4fe86a9e..abd55fbc7 100644 --- a/tests/samples/example-srrfioc08/st.cmd +++ b/tests/samples/example-srrfioc08/st.cmd @@ -11,21 +11,19 @@ epicsEnvSet Vec2 194 dbLoadDatabase dbd/ioc.dbd ioc_registerRecordDeviceDriver pdbbase - -# ipacAddHy8002 "slot, interrupt_level" +# ipacAddHy8002 "slot, interrupt_level" # Create a new Hy8002 carrier. # The resulting carrier handle (card id) is saved in an env variable. -ipacAddHy8002 "4, 2" +ipacAddHy8002 "4, 2" epicsEnvSet IPAC4 0 -ipacAddHy8002 "5, 2" +ipacAddHy8002 "5, 2" epicsEnvSet IPAC5 1 - -# Hy8401ipConfigure CardId IPACid IpSiteNumber InterruptVector InterruptEnable AiType ExternalClock ClockRate Inhibit SampleCount SampleSpacing SampleSize +# Hy8401ipConfigure CardId IPACid IpSiteNumber InterruptVector InterruptEnable AiType ExternalClock ClockRate Inhibit SampleCount SampleSpacing SampleSize # IpSlot 0=A 1=B 2=C 3=D # ClockRate 0=1Hz 1=2Hz 2=5Hz 3=10Hz 4=20Hz 5=50Hz 6=100Hz7=200Hz 8=500Hz 9=1kHz 10=2kHz11=5kHz 12=10kHz 13=20kHz 14=50kHz 15=100kHz -Hy8401ipConfigure 40 $(IPAC4) 0 $(Vec0) 0 0 0 15 0 1 1 0 -Hy8401ipConfigure 42 $(IPAC4) 2 $(Vec1) 0 0 0 15 0 1 1 0 -Hy8401ipConfigure 50 $(IPAC5) 0 $(Vec2) 0 0 0 15 0 1 1 0 +Hy8401ipConfigure 40 $(IPAC4) 0 $(Vec0) 0 0 0 15 0 1 1 0 +Hy8401ipConfigure 42 $(IPAC4) 2 $(Vec1) 0 0 0 15 0 1 1 0 +Hy8401ipConfigure 50 $(IPAC5) 0 $(Vec2) 0 0 0 15 0 1 1 0 dbLoadRecords /tmp/ioc.db iocInit diff --git a/tests/samples/generate_samples.sh b/tests/samples/generate_samples.sh index 130c31643..3126163e8 100755 --- a/tests/samples/generate_samples.sh +++ b/tests/samples/generate_samples.sh @@ -49,5 +49,10 @@ ibek build-startup ${SAMPLES_DIR}/example-srrfioc08/SR-RF-IOC-08.ibek.ioc.yaml $ cp /tmp/ioc/st.cmd ${SAMPLES_DIR}/example-srrfioc08 cp /tmp/ioc/make_db.sh ${SAMPLES_DIR}/example-srrfioc08 +echo makgin values_test IOC +ibek build-startup ${SAMPLES_DIR}/values_test/values.ibek.ioc.yaml ${SAMPLES_DIR}/values_test/*.support.yaml --out /tmp/ioc/st.cmd --db-out /tmp/ioc/make_db.sh +cp /tmp/ioc/st.cmd ${SAMPLES_DIR}/values_test + + diff --git a/tests/samples/schemas/all.ibek.support.schema.json b/tests/samples/schemas/all.ibek.support.schema.json index b54085b73..ea7a1cdc7 100644 --- a/tests/samples/schemas/all.ibek.support.schema.json +++ b/tests/samples/schemas/all.ibek.support.schema.json @@ -439,10 +439,6 @@ "type": "string", "const": "ipac.Hy8002", "default": "ipac.Hy8002" - }, - "card_id": { - "type": "string", - "default": "{{ __utils__.counter(\"Carriers\", start=0) }}" } }, "required": [ diff --git a/tests/samples/test_values b/tests/samples/test_values new file mode 100644 index 000000000..abd55fbc7 --- /dev/null +++ b/tests/samples/test_values @@ -0,0 +1,30 @@ +# EPICS IOC Startup Script generated by https://github.com/epics-containers/ibek + +cd "/repos/epics/ioc" + +epicsEnvSet EPICS_TS_NTP_INET 172.23.194.5 +epicsEnvSet EPICS_TS_MIN_WEST 0 +epicsEnvSet Vec0 192 +epicsEnvSet Vec1 193 +epicsEnvSet Vec2 194 + +dbLoadDatabase dbd/ioc.dbd +ioc_registerRecordDeviceDriver pdbbase + +# ipacAddHy8002 "slot, interrupt_level" +# Create a new Hy8002 carrier. +# The resulting carrier handle (card id) is saved in an env variable. +ipacAddHy8002 "4, 2" +epicsEnvSet IPAC4 0 +ipacAddHy8002 "5, 2" +epicsEnvSet IPAC5 1 +# Hy8401ipConfigure CardId IPACid IpSiteNumber InterruptVector InterruptEnable AiType ExternalClock ClockRate Inhibit SampleCount SampleSpacing SampleSize +# IpSlot 0=A 1=B 2=C 3=D +# ClockRate 0=1Hz 1=2Hz 2=5Hz 3=10Hz 4=20Hz 5=50Hz 6=100Hz7=200Hz 8=500Hz 9=1kHz 10=2kHz11=5kHz 12=10kHz 13=20kHz 14=50kHz 15=100kHz +Hy8401ipConfigure 40 $(IPAC4) 0 $(Vec0) 0 0 0 15 0 1 1 0 +Hy8401ipConfigure 42 $(IPAC4) 2 $(Vec1) 0 0 0 15 0 1 1 0 +Hy8401ipConfigure 50 $(IPAC5) 0 $(Vec2) 0 0 0 15 0 1 1 0 + +dbLoadRecords /tmp/ioc.db +iocInit + diff --git a/tests/samples/values_test/st.cmd b/tests/samples/values_test/st.cmd index e00ff9126..698713775 100644 --- a/tests/samples/values_test/st.cmd +++ b/tests/samples/values_test/st.cmd @@ -7,11 +7,10 @@ epicsEnvSet Vec0 192 dbLoadDatabase dbd/ioc.dbd ioc_registerRecordDeviceDriver pdbbase - -# ipacAddHy8002 "slot, interrupt_level" +# ipacAddHy8002 "slot, interrupt_level" # Create a new Hy8002 carrier. # The resulting carrier handle (card id) is saved in an env variable. -ipacAddHy8002 "4, 2" +ipacAddHy8002 "4, 2" epicsEnvSet IPAC4 0 1 2 dbLoadRecords /tmp/ioc.db From 6a12de678e791a35e10fbb950dccb384ec06312e Mon Sep 17 00:00:00 2001 From: Giles Knap Date: Thu, 11 May 2023 07:45:14 +0000 Subject: [PATCH 11/16] move arg/value jinja renders into Entity.__post_init --- src/ibek/ioc.py | 10 ++++++++-- src/ibek/render.py | 14 +++++++------- tests/samples/values_test/st.cmd | 4 +++- tests/samples/values_test/values.ibek.ioc.yaml | 4 ++++ 4 files changed, 22 insertions(+), 10 deletions(-) diff --git a/src/ibek/ioc.py b/src/ibek/ioc.py index 90e319761..ff9334d93 100644 --- a/src/ibek/ioc.py +++ b/src/ibek/ioc.py @@ -31,6 +31,7 @@ ) from apischema.conversions import Conversion, reset_deserializers from apischema.metadata import conversion +from jinja2 import Template from typing_extensions import Annotated as A from typing_extensions import Literal @@ -72,9 +73,14 @@ def __post_init__(self: "Entity"): # add in the global __utils__ object for state sharing context["__utils__"] = self.__utils__ - # todo jinja expand all values with context + # Do Jinja expansion of any string args/values in the context + for arg, value in context.items(): + if isinstance(value, str): + jinja_template = Template(value) + rendered = jinja_template.render(context) + context[arg] = rendered + self.__context__ = context - # then pass __context__ to jinja template in render.py id_to_entity: Dict[str, Entity] = {} diff --git a/src/ibek/render.py b/src/ibek/render.py index e482d09cb..c5d8de51f 100644 --- a/src/ibek/render.py +++ b/src/ibek/render.py @@ -22,9 +22,12 @@ def __init__(self: "Render"): def render_text(self, instance: Entity, text: str, once=False, suffix="") -> str: """ - render a line of jinja template in ``text`` using the values supplied - in the ``instance`` object. Supports the ``once`` flag to only render - the line once per definitions file. + Add in the next line of text, honouring the ``once`` flag which will + only add the line once per IOC. + + Jinja rendering of values/args has already been done in Entity.__post_init__ + but we pass all strings though jinja again to render any other jinja + in the IOC (e.g. database and function entries) ``once`` uses the name of the definition + suffix to track which lines have been rendered already. The suffix can be used where a given @@ -38,9 +41,6 @@ def render_text(self, instance: Entity, text: str, once=False, suffix="") -> str else: return "" - jinja_template = Template(text) - result = jinja_template.render(instance.__context__) # type: ignore - # run the result through jinja again so we can refer to args for arg defaults # e.g. # @@ -49,7 +49,7 @@ def render_text(self, instance: Entity, text: str, once=False, suffix="") -> str # description: IPAC identifier # default: "IPAC{{ slot }}" - jinja_template = Template(result) + jinja_template = Template(text) result = jinja_template.render(instance.__context__) # type: ignore if result == "": diff --git a/tests/samples/values_test/st.cmd b/tests/samples/values_test/st.cmd index 698713775..94b22a860 100644 --- a/tests/samples/values_test/st.cmd +++ b/tests/samples/values_test/st.cmd @@ -11,7 +11,9 @@ ioc_registerRecordDeviceDriver pdbbase # Create a new Hy8002 carrier. # The resulting carrier handle (card id) is saved in an env variable. ipacAddHy8002 "4, 2" -epicsEnvSet IPAC4 0 1 2 +epicsEnvSet IPAC4 0 0 0 +ipacAddHy8002 "5, 2" +epicsEnvSet IPAC5 1 1 1 dbLoadRecords /tmp/ioc.db iocInit diff --git a/tests/samples/values_test/values.ibek.ioc.yaml b/tests/samples/values_test/values.ibek.ioc.yaml index 06c7dc9a1..397694e97 100644 --- a/tests/samples/values_test/values.ibek.ioc.yaml +++ b/tests/samples/values_test/values.ibek.ioc.yaml @@ -11,3 +11,7 @@ entities: - type: ipac.Hy8002 name: IPAC4 slot: 4 + + - type: ipac.Hy8002 + name: IPAC5 + slot: 5 From af36d3c2dc197d97f7ecc0aa1545a4c4974043c0 Mon Sep 17 00:00:00 2001 From: Giles Knap Date: Thu, 11 May 2023 07:52:47 +0000 Subject: [PATCH 12/16] remove unecessary add of values to Entity --- src/ibek/ioc.py | 31 +++++-------------------------- 1 file changed, 5 insertions(+), 26 deletions(-) diff --git a/src/ibek/ioc.py b/src/ibek/ioc.py index ff9334d93..03cb10897 100644 --- a/src/ibek/ioc.py +++ b/src/ibek/ioc.py @@ -6,19 +6,8 @@ import builtins import types -from dataclasses import Field, dataclass, field, make_dataclass -from typing import ( - Any, - ClassVar, - Dict, - List, - Mapping, - Sequence, - Tuple, - Type, - cast, - get_type_hints, -) +from dataclasses import Field, asdict, dataclass, field, make_dataclass +from typing import Any, Dict, List, Mapping, Sequence, Tuple, Type, cast from apischema import ( Undefined, @@ -66,9 +55,9 @@ def __post_init__(self: "Entity"): id_to_entity[inst_id] = self # create a context dictionary for use in Jinja expansion of this Entity - context: Dict[str, Any] = {} - for attribute in get_type_hints(self).keys(): - context[attribute] = getattr(self, attribute) + context: Dict[str, Any] = asdict(self) # type: ignore + for value in self.__definition__.values: + context[value.name] = value.value # add in the global __utils__ object for state sharing context["__utils__"] = self.__utils__ @@ -144,16 +133,6 @@ def lookup_instance(id): # it fields.append(("entity_enabled", bool, field(default=cast(Any, True)))) - # add in the values fields as class attributes - for value in definition.values: - fields.append( - ( - value.name, - ClassVar[str], # type: ignore - field(default=cast(Any, value.value)), - ) - ) - namespace = dict(__definition__=definition) # make the Entity derived dataclass for this EntityClass, with a reference From 2f9afdcd338e430bcd0804d020fb1903d6b41042 Mon Sep 17 00:00:00 2001 From: Giles Knap Date: Thu, 11 May 2023 08:19:00 +0000 Subject: [PATCH 13/16] restore blank lines before function headers --- src/ibek/render.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/ibek/render.py b/src/ibek/render.py index c5d8de51f..24f539630 100644 --- a/src/ibek/render.py +++ b/src/ibek/render.py @@ -71,9 +71,9 @@ def render_function(self, instance: Entity, function: Function) -> str: call += f"{value} " text = ( - self.render_text(instance, comment.strip(), once=True, suffix="func") + self.render_text(instance, comment.strip(" "), once=True, suffix="func") + self.render_text(instance, function.header, once=True, suffix="func_hdr") - + self.render_text(instance, call.strip()) + + self.render_text(instance, call.strip(" ")) ) return text From 6505db9c1a274c6f74599bb6b8d374c44985a004 Mon Sep 17 00:00:00 2001 From: Giles Knap Date: Thu, 11 May 2023 11:14:51 +0000 Subject: [PATCH 14/16] verify that all include_args exist --- ibek-defs | 2 +- src/ibek/render.py | 14 +- tests/samples/example-srrfioc08/st.cmd | 2 + .../schemas/all.ibek.support.schema.json | 147 +++++++++++++++++- .../bl45p-mo-ioc-04.ibek.entities.schema.json | 7 +- .../container.ibek.entities.schema.json | 7 +- .../schemas/pmac.ibek.entities.schema.json | 7 +- tests/samples/values_test/st.cmd | 1 + 8 files changed, 179 insertions(+), 8 deletions(-) diff --git a/ibek-defs b/ibek-defs index 9643225a1..938ddc610 160000 --- a/ibek-defs +++ b/ibek-defs @@ -1 +1 @@ -Subproject commit 9643225a104afdcc5d01a6e24ae668c60bbe3e37 +Subproject commit 938ddc61014c7c993d4ac3003d20e5cc8ce24c10 diff --git a/src/ibek/render.py b/src/ibek/render.py index 24f539630..64af50b19 100644 --- a/src/ibek/render.py +++ b/src/ibek/render.py @@ -111,9 +111,17 @@ def render_database(self, instance: Entity) -> Optional[str]: for template in templates: db_file = template.file.strip("\n") db_args = template.define_args.splitlines() - include_list = [ - f"{arg}={{{{ {arg} }}}}" for arg in template.include_args or [] - ] + + include_list = [] + for arg in template.include_args: + if arg in instance.__context__: + include_list.append(f"{arg}={{{{ {arg} }}}}") + else: + raise ValueError( + f"include arg '{arg}' in database template " + f"'{template.file}' not found in context" + ) + db_arg_string = ", ".join(db_args + include_list) jinja_txt += ( diff --git a/tests/samples/example-srrfioc08/st.cmd b/tests/samples/example-srrfioc08/st.cmd index abd55fbc7..b372e63c8 100644 --- a/tests/samples/example-srrfioc08/st.cmd +++ b/tests/samples/example-srrfioc08/st.cmd @@ -11,6 +11,7 @@ epicsEnvSet Vec2 194 dbLoadDatabase dbd/ioc.dbd ioc_registerRecordDeviceDriver pdbbase + # ipacAddHy8002 "slot, interrupt_level" # Create a new Hy8002 carrier. # The resulting carrier handle (card id) is saved in an env variable. @@ -18,6 +19,7 @@ ipacAddHy8002 "4, 2" epicsEnvSet IPAC4 0 ipacAddHy8002 "5, 2" epicsEnvSet IPAC5 1 + # Hy8401ipConfigure CardId IPACid IpSiteNumber InterruptVector InterruptEnable AiType ExternalClock ClockRate Inhibit SampleCount SampleSpacing SampleSize # IpSlot 0=A 1=B 2=C 3=D # ClockRate 0=1Hz 1=2Hz 2=5Hz 3=10Hz 4=20Hz 5=50Hz 6=100Hz7=200Hz 8=500Hz 9=1kHz 10=2kHz11=5kHz 12=10kHz 13=20kHz 14=50kHz 15=100kHz diff --git a/tests/samples/schemas/all.ibek.support.schema.json b/tests/samples/schemas/all.ibek.support.schema.json index ea7a1cdc7..5005588c2 100644 --- a/tests/samples/schemas/all.ibek.support.schema.json +++ b/tests/samples/schemas/all.ibek.support.schema.json @@ -858,7 +858,7 @@ "description": "New Target Monitor, only set to 0 for soft motors", "default": 1 }, - "FEHEIGH": { + "FEHIGH": { "type": "number", "description": "HIGH limit for following error", "default": 0.0 @@ -906,6 +906,11 @@ "description": "Set to a blank to allow this axis to have its homed", "default": "#" }, + "RLINK": { + "type": "string", + "description": "not sure what this is", + "default": "" + }, "type": { "type": "string", "const": "pmac.DlsPmacAsynMotor", @@ -1337,6 +1342,146 @@ ], "additionalProperties": false }, + { + "type": "object", + "properties": { + "entity_enabled": { + "type": "boolean", + "default": true + }, + "device": { + "type": "string", + "description": "Device Name, channel name until the last colon, up to 16 chars", + "vscode_ibek_plugin_type": "type_id" + }, + "carrier": { + "type": "string", + "description": "IPAC carrier name", + "vscode_ibek_plugin_type": "type_object" + }, + "ip_site_number": { + "type": "integer", + "description": "Site on the carrier for this IP Module (0=A, 1=B, 2=C, 3=D)" + }, + "interrupt_vector": { + "type": "string", + "description": "Interrupt Vector reserved with epics.InterruptVectorVME, count=3", + "vscode_ibek_plugin_type": "type_object" + }, + "link": { + "type": "integer", + "description": "Link number on this IP module (0 or 1)", + "default": 0 + }, + "type": { + "type": "string", + "const": "psc.PscIpModule", + "default": "psc.PscIpModule" + } + }, + "required": [ + "device", + "carrier", + "ip_site_number", + "interrupt_vector" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "entity_enabled": { + "type": "boolean", + "default": true + }, + "i_abserr": { + "type": "integer", + "description": "todo", + "default": 0 + }, + "abserr": { + "type": "integer", + "description": "todo", + "default": 0 + }, + "relerr": { + "type": "integer", + "description": "todo", + "default": 0 + }, + "controlerr": { + "type": "integer", + "description": "todo", + "default": 0 + }, + "startsw_evnt": { + "type": "integer", + "description": "todo", + "default": 0 + }, + "syncsw_evnt": { + "type": "integer", + "description": "todo", + "default": 0 + }, + "hyscycle_dly": { + "type": "integer", + "description": "todo", + "default": 0 + }, + "hyscydir": { + "type": "integer", + "description": "todo", + "default": 0 + }, + "hyscmask": { + "type": "integer", + "description": "todo", + "default": 0 + }, + "hyslock": { + "type": "integer", + "description": "todo", + "default": 0 + }, + "hysimin": { + "type": "integer", + "description": "todo", + "default": 0 + }, + "hysimax": { + "type": "integer", + "description": "todo", + "default": 0 + }, + "oninitcycle": { + "type": "integer", + "description": "todo", + "default": 0 + }, + "vdclink_adel": { + "type": "integer", + "description": "todo", + "default": 0 + }, + "vload_adel": { + "type": "integer", + "description": "todo", + "default": 0 + }, + "icharge_adel": { + "type": "integer", + "description": "todo", + "default": 0 + }, + "type": { + "type": "string", + "const": "psc.PscTemplate", + "default": "psc.PscTemplate" + } + }, + "additionalProperties": false + }, { "type": "object", "properties": { diff --git a/tests/samples/schemas/bl45p-mo-ioc-04.ibek.entities.schema.json b/tests/samples/schemas/bl45p-mo-ioc-04.ibek.entities.schema.json index ae8cd9ce7..84c0f9d96 100644 --- a/tests/samples/schemas/bl45p-mo-ioc-04.ibek.entities.schema.json +++ b/tests/samples/schemas/bl45p-mo-ioc-04.ibek.entities.schema.json @@ -548,7 +548,7 @@ "description": "New Target Monitor, only set to 0 for soft motors", "default": 1 }, - "FEHEIGH": { + "FEHIGH": { "type": "number", "description": "HIGH limit for following error", "default": 0.0 @@ -596,6 +596,11 @@ "description": "Set to a blank to allow this axis to have its homed", "default": "#" }, + "RLINK": { + "type": "string", + "description": "not sure what this is", + "default": "" + }, "type": { "type": "string", "const": "pmac.DlsPmacAsynMotor", diff --git a/tests/samples/schemas/container.ibek.entities.schema.json b/tests/samples/schemas/container.ibek.entities.schema.json index de4e023ac..e2065eea0 100644 --- a/tests/samples/schemas/container.ibek.entities.schema.json +++ b/tests/samples/schemas/container.ibek.entities.schema.json @@ -505,7 +505,7 @@ "description": "New Target Monitor, only set to 0 for soft motors", "default": 1 }, - "FEHEIGH": { + "FEHIGH": { "type": "number", "description": "HIGH limit for following error", "default": 0.0 @@ -553,6 +553,11 @@ "description": "Set to a blank to allow this axis to have its homed", "default": "#" }, + "RLINK": { + "type": "string", + "description": "not sure what this is", + "default": "" + }, "type": { "type": "string", "const": "pmac.DlsPmacAsynMotor", diff --git a/tests/samples/schemas/pmac.ibek.entities.schema.json b/tests/samples/schemas/pmac.ibek.entities.schema.json index fe34544cc..a0f3d71c0 100644 --- a/tests/samples/schemas/pmac.ibek.entities.schema.json +++ b/tests/samples/schemas/pmac.ibek.entities.schema.json @@ -386,7 +386,7 @@ "description": "New Target Monitor, only set to 0 for soft motors", "default": 1 }, - "FEHEIGH": { + "FEHIGH": { "type": "number", "description": "HIGH limit for following error", "default": 0.0 @@ -434,6 +434,11 @@ "description": "Set to a blank to allow this axis to have its homed", "default": "#" }, + "RLINK": { + "type": "string", + "description": "not sure what this is", + "default": "" + }, "type": { "type": "string", "const": "pmac.DlsPmacAsynMotor", diff --git a/tests/samples/values_test/st.cmd b/tests/samples/values_test/st.cmd index 94b22a860..989e88d58 100644 --- a/tests/samples/values_test/st.cmd +++ b/tests/samples/values_test/st.cmd @@ -7,6 +7,7 @@ epicsEnvSet Vec0 192 dbLoadDatabase dbd/ioc.dbd ioc_registerRecordDeviceDriver pdbbase + # ipacAddHy8002 "slot, interrupt_level" # Create a new Hy8002 carrier. # The resulting carrier handle (card id) is saved in an env variable. From 0054358fb6c4b9779bcdf2341d107cd3fa5a8161 Mon Sep 17 00:00:00 2001 From: Giles Knap Date: Thu, 11 May 2023 13:19:57 +0000 Subject: [PATCH 15/16] add mypy hints for __context__ --- NOTES.txt | 32 ++++++++++++++++++++++++++++++++ src/ibek/ioc.py | 2 ++ 2 files changed, 34 insertions(+) create mode 100644 NOTES.txt diff --git a/NOTES.txt b/NOTES.txt new file mode 100644 index 000000000..fd610b739 --- /dev/null +++ b/NOTES.txt @@ -0,0 +1,32 @@ + +psc.substitutions file format: + +device Device Name: channel name until the last colon, up to 16 chars +c Card Number : card are numbered by the occurence of ipacAddCarrier in the startup.script: first card is "0", second is "1", ... +s Slot Number : IP Carrier slot number, module in that slot has to be configured by the pscAddIpModule function in the startup.script: pscAddIpModule(cardno, slot, intVec, links) +link Link Number : each module has two links: "0" and "1" +abserr Absolute Error : absolute error limit in [A] +relerr Relative Error : relative error limit [0..1] +controlerr: Control Error: +startsw-evnt Event to start Soft ramps: ignore +syncsw-evnt Event to synchronise Soft ramps: ignore +hyscycle-dly Seconds to wait in hysteresis cycle: time of PS between min and max value +hyscydir Direction of hysteresis cycle: approaching setpoint from above (-) or below (+) at the end of the cycle +hyscmask Defines the number of hysteresis cycles to run (for values >1 (3, 7, ...) the seq delay has to be specified in an extra template) +hyslock The value "locked" will force the PS to do a full cycle, whenever the value I-SET is changed in the wrong direction (against HYCDIR) +hysimin Minimum value for hysteresis cycle +hysimax Maximum value for hysteresis cycle +oninitcycle Flag to determine if power supply should do a hysteresis cycle when it is switched ON + +file $(PSC)/db/psc.template +{ +pattern { device, c, s, link, i_abserr, abserr, relerr, controlerr, startsw-evnt, syncsw-evnt, hyscycle-dly, hyscydir, hyscmask, hyslock, hysimin, hysimax, oninitcycle, vdclink_adel, vload_adel, icharge_adel } + { "SR25A-PC-TEST-01", "0", "0", "0", "0.0005", "0.0005", "0.0002", "0.0005", "53", "54", "3", "0", "3", "0", "-3", "3", "0", "0.5", "0.1", "0.004" } + { "SR25A-PC-TEST-02", "0", "0", "1", "0.0005", "0.0005", "0.0002", "0.0005", "53", "54", "3", "0", "3", "0", "-3", "3", "0", "0.5", "0.1", "0.004" } + { "SR25A-PC-TEST-03", "0", "1", "0", "0.0005", "0.0005", "0.0002", "0.0005", "53", "54", "3", "0", "3", "0", "-3", "3", "0", "0.5", "0.1", "0.004" } + { "SR25A-PC-TEST-04", "0", "1", "1", "0.0005", "0.0005", "0.0002", "0.0005", "53", "54", "3", "0", "3", "0", "-3", "3", "0", "0.5", "0.1", "0.004" } + { "SR25A-PC-TEST-05", "0", "2", "0", "0.0005", "0.0005", "0.0002", "0.0005", "53", "54", "3", "0", "3", "0", "-3", "3", "0", "0.5", "0.1", "0.004" } + { "SR25A-PC-TEST-06", "0", "2", "1", "0.0005", "0.0005", "0.0002", "0.0005", "53", "54", "3", "0", "3", "0", "-3", "3", "0", "0.5", "0.1", "0.004" } + { "SR25A-PC-TEST-07", "0", "3", "0", "0.0005", "0.0005", "0.0002", "0.0005", "53", "54", "3", "0", "3", "0", "-3", "3", "0", "0.5", "0.1", "0.004" } + { "SR25A-PC-TEST-08", "0", "3", "1", "0.0005", "0.0005", "0.0002", "0.0005", "53", "54", "3", "0", "3", "0", "-3", "3", "0", "0.5", "0.1", "0.004" } +} \ No newline at end of file diff --git a/src/ibek/ioc.py b/src/ibek/ioc.py index 03cb10897..4fb6c2933 100644 --- a/src/ibek/ioc.py +++ b/src/ibek/ioc.py @@ -40,6 +40,8 @@ class Entity: __definition__: Definition # a singleton Utility object for sharing state across all Entity renders __utils__: Utils = Utils() + # Context of expanded args and values to be passed to all Jinja expansion + __context__: Dict[str, Any] entity_enabled: bool From 67515128695397fa8336c3c08adbfab57a306ee6 Mon Sep 17 00:00:00 2001 From: Giles Knap Date: Thu, 11 May 2023 13:28:57 +0000 Subject: [PATCH 16/16] fix comment and render_elements type hint --- NOTES.txt | 32 -------------------------------- src/ibek/render.py | 15 +++++---------- 2 files changed, 5 insertions(+), 42 deletions(-) delete mode 100644 NOTES.txt diff --git a/NOTES.txt b/NOTES.txt deleted file mode 100644 index fd610b739..000000000 --- a/NOTES.txt +++ /dev/null @@ -1,32 +0,0 @@ - -psc.substitutions file format: - -device Device Name: channel name until the last colon, up to 16 chars -c Card Number : card are numbered by the occurence of ipacAddCarrier in the startup.script: first card is "0", second is "1", ... -s Slot Number : IP Carrier slot number, module in that slot has to be configured by the pscAddIpModule function in the startup.script: pscAddIpModule(cardno, slot, intVec, links) -link Link Number : each module has two links: "0" and "1" -abserr Absolute Error : absolute error limit in [A] -relerr Relative Error : relative error limit [0..1] -controlerr: Control Error: -startsw-evnt Event to start Soft ramps: ignore -syncsw-evnt Event to synchronise Soft ramps: ignore -hyscycle-dly Seconds to wait in hysteresis cycle: time of PS between min and max value -hyscydir Direction of hysteresis cycle: approaching setpoint from above (-) or below (+) at the end of the cycle -hyscmask Defines the number of hysteresis cycles to run (for values >1 (3, 7, ...) the seq delay has to be specified in an extra template) -hyslock The value "locked" will force the PS to do a full cycle, whenever the value I-SET is changed in the wrong direction (against HYCDIR) -hysimin Minimum value for hysteresis cycle -hysimax Maximum value for hysteresis cycle -oninitcycle Flag to determine if power supply should do a hysteresis cycle when it is switched ON - -file $(PSC)/db/psc.template -{ -pattern { device, c, s, link, i_abserr, abserr, relerr, controlerr, startsw-evnt, syncsw-evnt, hyscycle-dly, hyscydir, hyscmask, hyslock, hysimin, hysimax, oninitcycle, vdclink_adel, vload_adel, icharge_adel } - { "SR25A-PC-TEST-01", "0", "0", "0", "0.0005", "0.0005", "0.0002", "0.0005", "53", "54", "3", "0", "3", "0", "-3", "3", "0", "0.5", "0.1", "0.004" } - { "SR25A-PC-TEST-02", "0", "0", "1", "0.0005", "0.0005", "0.0002", "0.0005", "53", "54", "3", "0", "3", "0", "-3", "3", "0", "0.5", "0.1", "0.004" } - { "SR25A-PC-TEST-03", "0", "1", "0", "0.0005", "0.0005", "0.0002", "0.0005", "53", "54", "3", "0", "3", "0", "-3", "3", "0", "0.5", "0.1", "0.004" } - { "SR25A-PC-TEST-04", "0", "1", "1", "0.0005", "0.0005", "0.0002", "0.0005", "53", "54", "3", "0", "3", "0", "-3", "3", "0", "0.5", "0.1", "0.004" } - { "SR25A-PC-TEST-05", "0", "2", "0", "0.0005", "0.0005", "0.0002", "0.0005", "53", "54", "3", "0", "3", "0", "-3", "3", "0", "0.5", "0.1", "0.004" } - { "SR25A-PC-TEST-06", "0", "2", "1", "0.0005", "0.0005", "0.0002", "0.0005", "53", "54", "3", "0", "3", "0", "-3", "3", "0", "0.5", "0.1", "0.004" } - { "SR25A-PC-TEST-07", "0", "3", "0", "0.0005", "0.0005", "0.0002", "0.0005", "53", "54", "3", "0", "3", "0", "-3", "3", "0", "0.5", "0.1", "0.004" } - { "SR25A-PC-TEST-08", "0", "3", "1", "0.0005", "0.0005", "0.0002", "0.0005", "53", "54", "3", "0", "3", "0", "-3", "3", "0", "0.5", "0.1", "0.004" } -} \ No newline at end of file diff --git a/src/ibek/render.py b/src/ibek/render.py index 64af50b19..4b4a910e2 100644 --- a/src/ibek/render.py +++ b/src/ibek/render.py @@ -2,7 +2,7 @@ Functions for rendering lines in the boot script using Jinja2 """ -from typing import Callable, List, Optional +from typing import Callable, List, Optional, Union from jinja2 import Template @@ -41,14 +41,7 @@ def render_text(self, instance: Entity, text: str, once=False, suffix="") -> str else: return "" - # run the result through jinja again so we can refer to args for arg defaults - # e.g. - # - # - type: str - # name: IPACid - # description: IPAC identifier - # default: "IPAC{{ slot }}" - + # Render Jinja entries in the text jinja_template = Template(text) result = jinja_template.render(instance.__context__) # type: ignore @@ -165,7 +158,9 @@ def render_post_ioc_init(self, instance: Entity) -> Optional[str]: return script - def render_elements(self, ioc: IOC, method: Callable) -> str: + def render_elements( + self, ioc: IOC, method: Callable[[Entity], Union[str, None]] + ) -> str: """ Render elements of a given IOC instance based on calling the correct method """