Update to work with a deployment of multiple frame processors

.devcontainer/devcontainer.json

@@ -41,7 +41,6 @@
     ],
     // Mount the parent as /workspaces so we can pip install peers as editable
     "workspaceMount": "source=${localWorkspaceFolder}/..,target=/workspaces,type=bind",
-    "workspaceFolder": "/workspaces",
     // After the container is created, install the python project in editable form
     "postCreateCommand": "pip install $([ -f dev-requirements.txt ] && echo '-c dev-requirements.txt') -e '.[dev]' && pre-commit install"
-}
+}

.vscode/launch.json

@@ -29,7 +29,7 @@
             "module": "odin.main",
             "justMyCode": false,
             "console": "integratedTerminal",
-            "args": ["--config", "${workspaceFolder}/dev/odin_server.cfg", "--logging", "debug"]
+            "args": ["--config", "${workspaceFolder}/dev/local/odin_server.cfg", "--logging", "debug"]
         },
         {
             "name": "Dump Server Response",

README.md

@@ -4,7 +4,6 @@
 [![License](https://img.shields.io/badge/License-Apache%202.0-blue.svg)](https://opensource.org/licenses/Apache-2.0)
 
 # Odin FastCS
-
 FastCS support for the Odin detector software framework
 
 Source          | <https://github.com/DiamondLightSource/odin-fastcs>
@@ -16,38 +15,52 @@ Releases        | <https://github.com/DiamondLightSource/odin-fastcs/releases>
 
 ## Development
 
-Odin FastCS does not do much unless it has an Odin control server to talk to. It is
-possible to test some functionality in isolation by dumping server responses and creating
-tests that parse those responses. Responses can be dumped from various Odin systems and
-tests written against them that can run in CI to ensure support for those systems is not
-broken (or that the adapters need to be updated). The `tests/dump_server_response.py`
-helper script will generate json files for each adapter in an Odin server to write tests
-against.
+1. Clone odin-data and open in its devcontainer (for now, checkout fastcs-dev branch)
+2. Build odin-data into `vscode_prefix`
+
+    i. `CMake: Delete Cache and Reconfigure` (select a compiler, /usr/bin/gcc is
+    probably best)
+
+    ii. `CMake: Install`
+
+3. `Workspaces: Add Folder to Workspace...` to add odin-fastcs to the workspace
+4. Install odin-fastcs and its odin dev environment
+
+    i. `pip install -e .[dev,odin]`
+
+    ii. `Python: Select Interpreter` and set it to `/venv/bin/python` for the workspace
 
-Testing against static files is quite restrictive, so a dummy development environment is
-provided to give developers as consistent live deployment as possible to work against
-while developing the code. To set this up, run `dev/configure.sh` with the path to an
-odin-data install prefix and the path to a venv with odin-control and odin-data
-installed. This will populate the dev config with your environment - these changes
-should not be checked in. The dev deployment can then be run with `dev/start.sh`.
+5. Prepare the dev environment
 
-Currently Odin FastCS depends on branches of both odin-control and odin-data, so these
-branches are provided in `dev/requirements.txt` for convenience. Make a venv and then
-`pip install -r dev/requirements.txt` will give an environment that the control server
-and meta writer can run in. For the frameProcessor and frameReceiver, check out the
-fastcs-dev branch of odin-data and build. It is recommended to use the vscode CMake
-configuration to do this.
+    i. `dev/configure.sh one_node_fp /workspaces/odin-data/vscode_prefix /venv`
+
+6. Run the dev environment
+
+    i. `dev/start.sh` (it may print some garbage to the terminal while it installs)
+
+    ii. Click in the right panel and hit enter to run the odin server once all processes
+    running
+
+    iii. To close zelijj and the processes, `Ctrl+Q`
+
+7. Run the `Odin IOC` launch config to run the odin-fastcs IOC
 
 If you need to run a dev version of any of the applications, stop that process in the
 deployment and run/debug it manually. There is a vscode launch config for an odin server
-using the same config as the dev deployment for this purpose. For the python processes
-it is convenient to `pip install -r dev/requirements.txt` inside the container as well
-to make use of the launch configs.
+using the same config as the dev deployment for this purpose.
 
 At boot time, FastCS will generate UIs that can be opened in Phoebus. This is the
 clearest way to see the PVs that have been generated for the Odin server. It is also
 possible to run `dbl()` in the EPICS shell to print a flat list of PVs.
 
+Odin FastCS does not do much unless it has an Odin control server to talk to. It is
+possible to test some functionality in isolation by dumping server responses and creating
+tests that parse those responses. Responses can be dumped from various Odin systems and
+tests written against them that can run in CI to ensure support for those systems is not
+broken (or that the adapters need to be updated). The `tests/dump_server_response.py`
+helper script will generate json files for each adapter in an Odin server to write tests
+against.
+
 <!-- README only content. Anything below this line won't be included in index.md -->
 
 See https://diamondlightsource.github.io/odin-fastcs for more detailed documentation.

dev/configure.sh

@@ -2,27 +2,36 @@
 
 # Script to populate deployment with absolute paths of environment
 
-if [[ "$1" == "-h" || "$1" == "--help" || "$#" -ne 2 ]]; then
-    echo "Usage: $0 <path-to-odin-data-prefix> <path-to-venv>"
+if [[ "$1" == "-h" || "$1" == "--help" || "$#" -ne 3 ]]; then
+    echo "Usage: $0 <deployment-name> <path-to-odin-data-prefix> <path-to-venv>"
     exit 0
 fi
 
-ODIN_DATA=$1
-VENV=$2
+DEPLOYMENT=$1
+ODIN_DATA=$2
+VENV=$3
 
 SCRIPT_DIR=$(cd $(dirname "${BASH_SOURCE[0]}") && pwd)
 
-mkdir ${SCRIPT_DIR}/local
-cp ${SCRIPT_DIR}/templates/* ${SCRIPT_DIR}/local
+LOCAL=${SCRIPT_DIR}/local
 
-SERVER="${SCRIPT_DIR}/local/stOdinServer.sh"
-FR="${SCRIPT_DIR}/local/stFrameReceiver1.sh"
-FR_CONFIG="${SCRIPT_DIR}/local/fr1.json"
-FP="${SCRIPT_DIR}/local/stFrameProcessor1.sh"
-FP_CONFIG="${SCRIPT_DIR}/local/fp1.json"
-META="${SCRIPT_DIR}/local/stMetaWriter.sh"
-LAYOUT="${SCRIPT_DIR}/local/layout.kdl"
+# Check if local already exists
+if [ -d ${LOCAL} ]; then
+    echo "Local deployment ${LOCAL} already exists. Please remove it if you want to replace it."
+    exit 1
+fi
+
+mkdir ${LOCAL}
+cp ${SCRIPT_DIR}/${DEPLOYMENT}/* ${LOCAL}
+
+SERVER="${LOCAL}/stOdinServer.sh"
+FR="${LOCAL}/stFrameReceiver*.sh"
+FR_CONFIG="${LOCAL}/fr*.json"
+FP="${LOCAL}/stFrameProcessor*.sh"
+FP_CONFIG="${LOCAL}/fp*.json"
+META="${LOCAL}/stMetaWriter.sh"
+LAYOUT="${LOCAL}/layout.kdl"
 
 sed -i "s+<ODIN_DATA>+${ODIN_DATA}+g" ${FR} ${FR_CONFIG} ${FP} ${FP_CONFIG}
 sed -i "s+<VENV>+${VENV}+g" ${SERVER} ${META}
-sed -i "s+<SCRIPT_DIR>+${SCRIPT_DIR}/local+g" ${LAYOUT}
+sed -i "s+<SCRIPT_DIR>+${LOCAL}+g" ${LAYOUT}

dev/templates/fr1.json → dev/one_node_fp/fr1.json

@@ -13,4 +13,4 @@
       "udp_packet_size": 8000
     }
   }
-]
+]

dev/templates/layout.kdl → dev/one_node_fp/layout.kdl

@@ -9,7 +9,7 @@ layout {
             pane command="<SCRIPT_DIR>/stMetaWriter.sh"
         }
         pane split_direction="horizontal" {
-            pane command="<SCRIPT_DIR>/stOdinServer.sh" start_suspended=true
+            pane command="<SCRIPT_DIR>/stOdinServer.sh"
         }
     }
     pane size=2 borderless=true {

dev/templates/stOdinServer.sh → dev/one_node_fp/stOdinServer.sh

@@ -5,4 +5,7 @@ SCRIPT_DIR="$( cd "$( dirname "$0" )" && pwd )"
 # Increase maximum fds available for ZeroMQ sockets
 ulimit -n 2048
 
+# Wait for other processes to start
+sleep 3
+
 <VENV>/bin/odin_control --config=$SCRIPT_DIR/odin_server.cfg --logging=info --access_logging=ERROR

dev/two_node_fp/fp1.json

@@ -0,0 +1,140 @@
+[
+  {
+    "fr_setup": {
+      "fr_ready_cnxn": "tcp://127.0.0.1:10001",
+      "fr_release_cnxn": "tcp://127.0.0.1:10002"
+    },
+    "meta_endpoint": "tcp://*:10008"
+  },
+  {
+    "plugin": {
+      "load": {
+        "index": "dummy",
+        "name": "DummyUDPProcessPlugin",
+        "library": "<ODIN_DATA>/lib/libDummyUDPProcessPlugin.so"
+      }
+    }
+  },
+  {
+    "plugin": {
+      "load": {
+        "index": "offset",
+        "name": "OffsetAdjustmentPlugin",
+        "library": "<ODIN_DATA>/lib/libOffsetAdjustmentPlugin.so"
+      }
+    }
+  },
+  {
+    "plugin": {
+      "load": {
+        "index": "param",
+        "name": "ParameterAdjustmentPlugin",
+        "library": "<ODIN_DATA>/lib/libParameterAdjustmentPlugin.so"
+      }
+    }
+  },
+  {
+    "plugin": {
+      "load": {
+        "index": "hdf",
+        "name": "FileWriterPlugin",
+        "library": "<ODIN_DATA>/lib/libHdf5Plugin.so"
+      }
+    }
+  },
+  {
+    "plugin": {
+      "connect": {
+        "index": "dummy",
+        "connection": "frame_receiver"
+      }
+    }
+  },
+  {
+    "plugin": {
+      "connect": {
+        "index": "offset",
+        "connection": "dummy"
+      }
+    }
+  },
+  {
+    "plugin": {
+      "connect": {
+        "index": "param",
+        "connection": "offset"
+      }
+    }
+  },
+  {
+    "plugin": {
+      "connect": {
+        "index": "hdf",
+        "connection": "param"
+      }
+    }
+  },
+  {
+    "hdf": {
+      "dataset": {
+        "compressed_size": {
+          "datatype": "uint32",
+          "chunks": [1000]
+        }
+      }
+    }
+  },
+  {
+    "hdf": {
+      "dataset": {
+        "uid": {
+          "datatype": "uint64",
+          "chunks": [1000]
+        }
+      }
+    }
+  },
+  {
+    "param": {
+      "parameter": {
+        "uid": {
+          "adjustment": 1
+        }
+      }
+    }
+  },
+  {
+    "hdf": {
+      "process": {
+        "number": 4,
+        "rank": 0
+      }
+    }
+  },
+  {
+    "hdf": {
+      "file": {
+        "flush_error_duration": 10000,
+        "write_error_duration": 10000,
+        "close_error_duration": 10000,
+        "create_error_duration": 10000
+      }
+    }
+  },
+  {
+    "hdf": {
+      "file": {
+        "first_number": 1
+      }
+    }
+  },
+  {
+    "hdf": {
+      "dataset": {
+        "data": {
+          "chunks": [1, 512, 256]
+        }
+      }
+    }
+  }
+]