add system test

.github/workflows/code.yml

@@ -29,6 +29,17 @@ jobs:
       - name: Lint
         run: tox -e pre-commit,mypy
 
+  system-test:
+    if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name != github.repository
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+
+      - name: Run System Test
+        run: ./tests/sys-test.sh
+
   test:
     if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name != github.repository
     strategy:
@@ -114,7 +125,7 @@ jobs:
         run: python -m $(ls src | head -1) --version
 
   container:
-    needs: [lint, dist, test]
+    needs: [lint, dist, test, system-test]
     runs-on: ubuntu-latest
 
     permissions:
@@ -174,7 +185,7 @@ jobs:
 
   release:
     # upload to PyPI and make a release on every tag
-    needs: [lint, dist, test]
+    needs: [lint, dist, test, system-test]
     if: ${{ github.event_name == 'push' && startsWith(github.ref, 'refs/tags') }}
     runs-on: ubuntu-latest
 

tests/samples/example-ibek-config/README.md

@@ -0,0 +1,5 @@
+This folder contains an example config to map into a generic IOC at
+runtime.
+
+The system test 'tests/sys-test.sh' uses this to launch the generic
+ioc 'ioc-template' and verify that it serves the correct PVs.

tests/samples/generate_samples.sh

@@ -36,7 +36,7 @@ echo making bl45p-mo-ioc-04
 ibek build-startup ${SAMPLES_DIR}/yaml/bl45p-mo-ioc-04.ibek.ioc.yaml ${SAMPLES_DIR}/yaml/{epics,pmac}.ibek.support.yaml --out /tmp/ioc/st.cmd --db-out /tmp/ioc/make_db.sh
 cp /tmp/ioc/st.cmd ${SAMPLES_DIR}/boot_scripts/stbl45p-mo-ioc-04
 echo making test-ioc
-ibek build-startup ${SAMPLES_DIR}/config/test.ibek.ioc.yaml ${SAMPLES_DIR}/yaml/{epics,deviocstats}.ibek.support.yaml --out /tmp/ioc/st.cmd --db-out /tmp/ioc/make_db.sh
+ibek build-startup ${SAMPLES_DIR}/example-ibek-config/ioc.yaml ${SAMPLES_DIR}/yaml/{epics,deviocstats}.ibek.support.yaml --out /tmp/ioc/st.cmd --db-out /tmp/ioc/make_db.sh
 cp /tmp/ioc/st.cmd ${SAMPLES_DIR}/boot_scripts/test.ioc.cmd
 cp /tmp/ioc/make_db.sh ${SAMPLES_DIR}/boot_scripts/test.ioc.make_db.sh
 

tests/sys-test.sh

@@ -0,0 +1,81 @@
+# A system test script for ibek
+#
+# STEPS:
+# 1. Launch the ioc-template container in the background with these mounts:
+#    example-ibek-config -> /repos/epics/ioc/config
+#    this repo root -> /repos/ibek
+# 2. Install this instance of ibek into the container's global venv
+#    using /repos/ibek
+# 3. Launch the IOC by running /repos/epics/ioc/start_ioc.sh
+#    (backgrounded)
+# 4. Use the epics-base container to check the IOC is
+#    working as expected by invoking caget and caput.
+#
+
+THIS_DIR=$(realpath $(dirname $0))
+ROOT=$(realpath ${THIS_DIR}/..)
+
+set -ex
+
+base_args="
+    --rm
+    --network podman
+    -e EPICS_CA_SERVER_PORT=7064
+    ghcr.io/epics-containers/epics-base-linux-runtime:23.3.1
+"
+
+check_pv () {
+    podman run ${base_args} caget ${1} > /tmp/pv_out.txt
+    if ! grep -q ${2} /tmp/pv_out.txt ; then
+        echo "ERROR: IOC unexpected result from ${1}:SUM"
+        cat /tmp/pv_out.txt
+        return 1
+    fi
+}
+
+check_ioc() {
+    podman run ${base_args} caput ${1}:A 1.4
+    podman run ${base_args} caput ${1}:B 1.5
+    podman run ${base_args} caput ${1}:SUM.PROC 0
+    check_pv ${1}:SUM 2.9
+}
+
+cont="ibek-test-container"
+config="/repos/epics/ioc/config"
+ioc_args="
+--security-opt label=disable
+--network podman
+--name ${cont}
+--entrypoint bash
+-dit
+ghcr.io/epics-containers/ioc-template-linux-developer:23.3.1
+"
+mounts="
+-v ${THIS_DIR}/samples/example-ibek-config:${config}
+-v ${ROOT}:/repos/ibek
+"
+
+cd ${ROOT}
+
+# remove any existing container
+if podman container exists ${cont}; then
+    podman rm -ft0 ${cont}
+fi
+# launch epics-base container in the background
+podman run  ${mounts} ${ioc_args}
+# install the this ibek into it's global venv
+podman exec ${cont} pip install /repos/ibek
+# launch the IOC
+podman exec -dit ${cont} bash -c "/repos/epics/ioc/start.sh >> /repos/ibek/start.log 2>&1"
+# wait for ibek to get the IOC up and running.
+for retry in {1..10} ; do
+    if check_pv 'test-ibek-ioc:EPICS_VERS' 'R7.0.7'; then break; fi
+    sleep 1
+done
+# output the log from the IOC
+cat start.log
+# verify expected PVs
+check_pv 'test-ibek-ioc:EPICS_VERS' 'R7.0.7'
+check_ioc "EXAMPLE:IBEK"
+
+

tests/test_ioc.py

@@ -14,13 +14,17 @@ def test_example_ioc(tmp_path: Path, samples: Path):
     build an ioc from yaml and verify the result
 
     includes use of the feature 'pre_ioc_init'
+
+    NOTE: the system test in tests/sys-test.sh uses the same example-ibek-config
+    but instead of verifying the output, it runs the ioc in a container and \
+    verifies that it starts up correctly.
     """
     clear_entity_classes()
 
     tmp_path = Path("/tmp/ibek_test")
     tmp_path.mkdir(exist_ok=True)
 
-    entity_file = samples / "config" / "test.ibek.ioc.yaml"
+    entity_file = samples / "example-ibek-config" / "ioc.yaml"
     definition_file = samples / "yaml" / "epics.ibek.support.yaml"
     definition_file2 = samples / "yaml" / "deviocstats.ibek.support.yaml"
     out_file = tmp_path / "new_dir" / "test.ioc.cmd"
@@ -45,12 +49,3 @@ def test_example_ioc(tmp_path: Path, samples: Path):
 
     assert example_boot == actual_boot
     assert example_db == actual_db
-
-
-def test_system_example_ioc(tmp_path: Path, samples: Path):
-    """
-    Install the this local version of ibek into an epic-base container
-    and verify that it can execute an IOC defined by YAML
-
-    Pre-requisites: podman needs to be installed
-    """