Skip to content

Commit

Permalink
Merge pull request #115 from epics-containers/dev
Browse files Browse the repository at this point in the history
Tidy up where we get files from
  • Loading branch information
gilesknap authored Sep 29, 2023
2 parents 6e541ee + 9174f07 commit 9319e53
Show file tree
Hide file tree
Showing 50 changed files with 637 additions and 101 deletions.
4 changes: 1 addition & 3 deletions .vscode/launch.json
Original file line number Diff line number Diff line change
Expand Up @@ -14,9 +14,7 @@
"program": "builder2ibek.support.py",
"console": "integratedTerminal",
"args": [
"/dls_sw/prod/R3.14.12.7/support/ADAravis/2-2-1dls9",
"-o",
"336:34"
"/dls_sw/prod/R3.14.12.7/support/devIocStats/3-1-14dls3-3"
],
"justMyCode": false
},
Expand Down
51 changes: 37 additions & 14 deletions builder2ibek.support.py
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@

import argparse
import inspect
import os
import re
import sys

Expand Down Expand Up @@ -59,7 +60,7 @@ class ArgInfo:
"""

description_re = re.compile(r"(.*)\n<(?:type|class)")
name_re = re.compile(r"iocbuilder\.modules\.(.*)")
name_re = re.compile(r"iocbuilder\.modules\.(?:.*)\.(.*)")
arg_num = 1

def __init__(self, name, unique_name, description, overrides):
Expand All @@ -75,8 +76,6 @@ def __init__(self, name, unique_name, description, overrides):
self.yaml_args = []
# the root of the definition in yaml that holds above yaml_args
self.yaml_defs = ordereddict()
self.yaml_defs["name"] = self.name_re.findall(name)[0]
self.yaml_defs["args"] = self.yaml_args
# set of args and values to use for instantiating a builder object
self.builder_args = {}
# list of all the arg names only (across multiple add_arg_info)
Expand All @@ -89,7 +88,11 @@ def __init__(self, name, unique_name, description, overrides):
desc = description.strip()
else:
desc = "TODO:ADD DESCRIPTION"

print(name)
self.yaml_defs["name"] = self.name_re.findall(name)[0]
self.yaml_defs["description"] = PreservedScalarString(desc)
self.yaml_defs["args"] = self.yaml_args

def add_arg_info(self, arginfo):
"""
Expand Down Expand Up @@ -141,7 +144,9 @@ def _interpret(self):
if default:
new_yaml_arg["default"] = default
if typ == "enum":
new_yaml_arg["values"] = {label: None for label in details.labels}
new_yaml_arg["values"] = {
str(label): None for label in details.labels
}

self.yaml_args.append(new_yaml_arg)
self.all_args.append(arg_name)
Expand Down Expand Up @@ -180,7 +185,7 @@ def make_arg(self, name, details, default=None):
else:
typ = "UNKNOWN TODO TODO"

if hasattr(details, "labels"):
if hasattr(details, "labels") and typ != "bool":
value = default or details.labels[0]
typ = "enum"

Expand Down Expand Up @@ -275,12 +280,14 @@ def _make_builder_object(self, name, builder_class):
getattr(builder_class, "__doc__"),
self.arg_value_overrides,
)
arg_info.add_arg_info(builder_class.ArgInfo)

if hasattr(builder_class, "LibFileList"):
self.libs |= set(builder_class.LibFileList)
if hasattr(builder_class, "DbdFileList"):
self.dbds |= set(builder_class.DbdFileList)
for a_cls in (builder_class,) + builder_class.Dependencies:
if hasattr(a_cls, "ArgInfo"):
arg_info.add_arg_info(a_cls.ArgInfo)
if hasattr(a_cls, "LibFileList"):
self.libs |= set(a_cls.LibFileList)
if hasattr(a_cls, "DbdFileList"):
self.dbds |= set(a_cls.DbdFileList)

builder_object = builder_class(**arg_info.builder_args)

Expand Down Expand Up @@ -316,9 +323,16 @@ def _extract_substitutions(self, arginfo):

print("\nDB Template %s :" % template)

arginfo.add_arg_info(first_substitution.ArgInfo)
no_vals = {k: None for k in arginfo.builder_args}
database.insert(3, "args", no_vals)
if hasattr(first_substitution, "ArgInfo"):
arginfo.add_arg_info(first_substitution.ArgInfo)
# the DB Arg entries in the YAML are Dictionary entries with no value
no_values = {k: None for k in arginfo.builder_args}
elif hasattr(first_substitution, "Arguments"):
no_values = {k: None for k in first_substitution.Arguments}
else:
no_values = {"TODO": "No args for this template"}

database.insert(3, "args", no_values)

if len(databases) > 0:
arginfo.yaml_defs["databases"] = databases
Expand Down Expand Up @@ -416,6 +430,7 @@ def tidy_up(yaml):
" post_init:",
"module",
"defs",
" - type:",
]:
yaml = re.sub(r"(\n%s)" % field, "\n\\g<1>", yaml)

Expand Down Expand Up @@ -480,10 +495,18 @@ def parse_args():

if __name__ == "__main__":
support_module_path, filename, arg_value_overrides = parse_args()

etc_folder = support_module_path + "/etc"
if not os.path.exists(etc_folder):
raise ValueError("The support module path must contain an etc folder")

builder2support = Builder2Support(support_module_path, arg_value_overrides)
# builder2support.dump_subst_file()
builder2support.make_yaml_tree()
builder2support.write_yaml_tree(filename)
if len(builder2support.yaml_tree["defs"]) > 0:
builder2support.write_yaml_tree(filename)
else:
print("\nNo definitions - no YAML file needed for %s" % support_module_path)

print("\nYou will require the following to make the Generic IOC Dockerfile:\n")
print("DBD files: " + ", ".join(builder2support.dbds))
Expand Down
2 changes: 1 addition & 1 deletion config/st.cmd
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
# EPICS IOC Startup Script generated by https://github.com/epics-containers/ibek

cd "/repos/epics/ioc"
cd "/epics/ioc"
dbLoadDatabase dbd/ioc.dbd
ioc_registerRecordDeviceDriver pdbbase

Expand Down
7 changes: 6 additions & 1 deletion pyproject.toml
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,6 @@ dependencies = [
"ruamel.yaml",
"jinja2",
"GitPython",
"rich",
] # Add project dependencies here, e.g. ["click", "numpy"]
dynamic = ["version"]
license.file = "LICENSE"
Expand Down Expand Up @@ -120,3 +119,9 @@ select = [
]
[tool.ruff.per-file-ignores]
"builder2ibek.support.py" = ["E402"]

[tool.setuptools.packages.find]
where = ["src"]

[tool.setuptools.package-data]
"ibek.templates" = ["**/*"]
10 changes: 9 additions & 1 deletion src/ibek/__main__.py
Original file line number Diff line number Diff line change
Expand Up @@ -4,14 +4,22 @@
from ruamel.yaml import YAML

from ibek.ioc_cmds.commands import ioc_cli
from ibek.startup_cmds.commands import startup_cli
from ibek.runtime_cmds.commands import startup_cli
from ibek.support_cmds.commands import support_cli

from ._version import __version__
from .globals import NaturalOrderGroup

cli = typer.Typer(cls=NaturalOrderGroup)

# TODO too much trace output but this docmented way to suppress it doesn't work
try:
from rich.traceback import install

install(suppress=[typer])
install(show_locals=False)
except ImportError:
pass # we often uninstall rich because of the above not working

cli.add_typer(
support_cli,
Expand Down
2 changes: 1 addition & 1 deletion src/ibek/gen_scripts.py
Original file line number Diff line number Diff line change
Expand Up @@ -9,14 +9,14 @@
from jinja2 import Template
from ruamel.yaml.main import YAML

from .globals import TEMPLATES
from .ioc import IOC, clear_entity_model_ids, make_entity_models, make_ioc_model
from .render import Render
from .render_db import RenderDb
from .support import Support

log = logging.getLogger(__name__)

TEMPLATES = Path(__file__).parent / "templates"

schema_modeline = re.compile(r"# *yaml-language-server *: *\$schema=([^ ]*)")
url_f = r"file://"
Expand Down
21 changes: 10 additions & 11 deletions src/ibek/globals.py
Original file line number Diff line number Diff line change
Expand Up @@ -12,26 +12,25 @@

from .utils import UTILS

# note requirement for environment variable EPICS_BASE
EPICS_BASE = Path(str(os.getenv("EPICS_BASE")))
EPICS_ROOT = Path(str(os.getenv("EPICS_ROOT")))
# get the container paths from environment variables
EPICS_BASE = Path(os.getenv("EPICS_BASE", "/epics/epics-base"))
EPICS_ROOT = Path(os.getenv("EPICS_ROOT", "/epics/"))
IOC_FOLDER = Path(os.getenv("IOC", "/epics/ioc"))
SUPPORT = Path(os.getenv("SUPPORT", "/epics/support"))

# all support modules will reside under this directory
SUPPORT = Path(str(os.getenv("SUPPORT")))
# the global RELEASE file which lists all support modules
RELEASE = SUPPORT / "configure/RELEASE"
# a bash script to export the macros defined in RELEASE as environment vars
RELEASE_SH = SUPPORT / "configure/RELEASE.shell"
# global MODULES file used to determine order of build
MODULES = SUPPORT / "configure/MODULES"
# the root IOC folder
IOC_FOLDER = Path(str(os.getenv("IOC")))

# Folder containing Makefile.jinja
MAKE_FOLDER = IOC_FOLDER / "iocApp/src"
# Folder containing ibek support scripts
# WARNING: this will only work if PROJECT NAME has been set in devcontainer.json
PROJECT_NAME = os.getenv("PROJECT_NAME", "no-project-name")
PROJECT_ROOT_FOLDER = Path("/workspaces") / PROJECT_NAME
# Folder containing templates for IOC src etc.
TEMPLATES = Path(__file__).parent / "templates"
# Folder containing symlinks to useful files
SYMLINKS = EPICS_ROOT / "links"

IOC_DBDS = SUPPORT / "configure/dbd_list"
IOC_LIBS = SUPPORT / "configure/lib_list"
Expand Down
4 changes: 0 additions & 4 deletions src/ibek/ioc.py
Original file line number Diff line number Diff line change
Expand Up @@ -190,8 +190,4 @@ class IOC(BaseSettings):

ioc_name: str = Field(description="Name of IOC instance")
description: str = Field(description="Description of what the IOC does")
generic_ioc_image: str = Field(
description="The generic IOC container image registry URL"
)
# this will be replaced in derived classes made by make_ioc_model
entities: Sequence[Entity]
90 changes: 90 additions & 0 deletions src/ibek/ioc_cmds/assets.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,90 @@
import shutil
import subprocess
from pathlib import Path
from typing import List

import typer

from ibek.globals import IOC_FOLDER, SYMLINKS


def move_file(src: Path, dest: Path, binary: List[str]):
"""
Move a file / tree / symlink from src to dest, stripping symbols from
binaries if they are in the binary list.
"""
dest.parent.mkdir(exist_ok=True, parents=True)

if src.is_symlink():
# copy the symlink
shutil.rmtree(dest, ignore_errors=True)
dest = dest.parent
typer.echo(f"Symlink {src} to {dest}")
shutil.copy(src, dest, follow_symlinks=False)
else:
typer.echo(f"Moving {src} to {dest}")
if subprocess.call(["bash", "-c", f"mv {src} {dest}"]) > 0:
raise RuntimeError(f"Failed to move {src} to {dest}")
if dest.name in binary:
# strip the symbols from the binary
cmd = f"strip $(find {dest} -type f) &> /dev/null"
subprocess.call(["bash", "-c", cmd])


def extract_assets(destination: Path, source: Path, extras: List[Path], defaults: bool):
"""
Find all the runtime assets in an EPICS installation and copy them to a
new folder hierarchy for packaging into a container runtime stage.
This should be performed in a throw away container stage (runtime_prep)
as it is destructive of the source folder, because it uses move for speed.
"""
asset_matches = "bin|configure|db|dbd|include|lib|template|config|*.sh"

just_copy = (
[
source / "support" / "configure",
SYMLINKS,
IOC_FOLDER,
Path("/venv"),
]
if defaults
else []
)

# identify EPICS modules as folders with binary output folders
binary = ["bin", "lib"]

binaries: List[Path] = []
for find in binary:
# only look two levels deep
binaries.extend(source.glob(f"*/*/{find}"))
binaries.extend(source.glob(f"*/{find}"))

modules = [binary.parent for binary in binaries]

destination.mkdir(exist_ok=True, parents=True)
for module in modules:
# make sure dest folder exists
destination_module = destination / module.relative_to("/")

# use globs to make a list of the things we want to copy
asset_globs = [module.glob(match) for match in asset_matches.split("|")]
assets: List[Path] = [
asset for asset_glob in asset_globs for asset in asset_glob
]

for asset in assets:
src = module / asset
if src.exists():
dest_file = destination_module / asset.relative_to(module)
move_file(src, dest_file, binary)

extra_files = just_copy + extras
for asset in extra_files:
src = source / asset
if src.exists():
dest_file = destination / asset.relative_to("/")
move_file(src, dest_file, binary)
else:
raise RuntimeError(f"extra runtime asset {src} missing")
Loading

0 comments on commit 9319e53

Please sign in to comment.