Merge pull request #15 from epics-containers/fix-rsync

Add new features stress, trace. Improved error handling.

src/rtems_proxy/__main__.py

@@ -1,13 +1,19 @@
+from datetime import datetime
 from pathlib import Path
+from time import sleep
 
 import typer
 from jinja2 import Template
 from ruamel.yaml import YAML
 
+from rtems_proxy.trace import parse_stack_trace
+from rtems_proxy.utils import run_command
+
 from . import __version__
+from .configure import Configure
 from .copy import copy_rtems
 from .globals import GLOBALS
-from .telnet import ioc_connect, report
+from .telnet import ioc_connect, motboot_connect, report
 
 __all__ = ["main"]
 
@@ -46,6 +52,9 @@ def start(
     reboot: bool = typer.Option(
         True, "--reboot/--no-reboot", help="reboot the IOC first"
     ),
+    raise_errors: bool = typer.Option(
+        True, "--raise-errors/--no-raise-errors", help="raise errors instead of exiting"
+    ),
 ):
     """
     Starts an RTEMS IOC. Places the IOC binaries in the expected location,
@@ -70,7 +79,9 @@ def start(
     if copy:
         copy_rtems()
     if connect:
-        ioc_connect(GLOBALS.RTEMS_CONSOLE, reboot=reboot)
+        ioc_connect(
+            GLOBALS.RTEMS_CONSOLE, reboot=reboot, attach=True, raise_errors=raise_errors
+        )
     else:
         report("IOC console connection disabled. ")
 
@@ -154,8 +165,59 @@ def dev(
     typer.echo(f"\n\nPlease first source {script_file} to set up the dev environment.")
 
 
-# test with:
-#     pipenv run python -m ibek
+@cli.command()
+def configure(
+    debug: bool = typer.Option(False, help="use debug ioc binary"),
+    attach: bool = typer.Option(
+        False, help="attach to the IOC console after configuration"
+    ),
+):
+    """
+    Configure the RTEMS IOC boot parameters
+    """
+    telnet = motboot_connect(GLOBALS.RTEMS_CONSOLE)
+    config = Configure(telnet, debug)
+    config.apply_settings()
+    telnet.close()
+    if attach:
+        run_command(telnet.command)
+
+
+@cli.command()
+def stress():
+    """
+    Stress test the IOC by constantly rebooting and checking for failed boot
+
+    Aborts and prints the time when a failed boot is detected
+    """
+    try:
+        tries = 0
+        while True:
+            tries += 1
+            print(f">>>>>> REBOOT ATTEMPT {tries} <<<<<<<")
+            ioc_connect(
+                GLOBALS.RTEMS_CONSOLE, reboot=True, attach=False, raise_errors=True
+            )
+            sleep(5)
+    except Exception as e:
+        msg = f"\n\nIOC boot number {tries} failed at {datetime.now()}.\n\n"
+        raise RuntimeError(msg) from e
+
+
+@cli.command()
+def trace(
+    trace_file: Path = typer.Argument(
+        ...,
+        help="The path to the file containing the stack trace",
+        file_okay=True,
+        exists=True,
+    ),
+):
+    """
+    Parse a stack trace from a RTEMS failure
+    """
+    trace = trace_file.read_text()
+    parse_stack_trace(trace)
 
 
 if __name__ == "__main__":

src/rtems_proxy/configure.py

@@ -0,0 +1,48 @@
+"""
+Class to apply MOTBoot configuration to a VME crate.
+"""
+
+from .globals import GLOBALS
+from .telnet import TelnetRTEMS
+
+
+class Configure:
+    def __init__(self, telnet: TelnetRTEMS, debug: bool = False):
+        self.telnet = telnet
+        self.debug = debug
+
+    def apply_nvm(self, variable: str, value: str):
+        self.telnet.sendline(f"gevE {variable}")
+        self.telnet.expect(r"\(Blank line terminates input.\)")
+        self.telnet.sendline(value + "\r")
+        self.telnet.sendline("\r")
+        self.telnet.expect(r"\?")
+        self.telnet.sendline("Y\r")
+
+    def apply_settings(self):
+        nfs_mount = f"{GLOBALS.RTEMS_NFS_IP}:/iocs/{GLOBALS.IOC_NAME}:/epics"
+        ioc_bin = "ioc" if self.debug else "ioc.boot"
+        mot_boot = (
+            f"dla=malloc 0x4000000\r"
+            f"tftpGet -d/dev/enet1"
+            f" -f{GLOBALS.IOC_NAME.lower()}/ioc/bin/RTEMS-beatnik/{ioc_bin}"
+            f" -m{GLOBALS.RTEMS_IOC_NETMASK}"
+            f" -g{GLOBALS.RTEMS_IOC_GATEWAY}"
+            f" -s{GLOBALS.RTEMS_TFTP_IP}"
+            f" -c{GLOBALS.RTEMS_IOC_IP}"
+            f" -adla -r4\r"
+            f"go -a04000000\r"
+            f"reset"
+        )
+
+        self.apply_nvm("mot-/dev/enet0-snma", GLOBALS.RTEMS_IOC_NETMASK)
+        self.apply_nvm("mot-/dev/enet0-gipa", GLOBALS.RTEMS_IOC_GATEWAY)
+        self.apply_nvm("mot-/dev/enet0-sipa", GLOBALS.RTEMS_NFS_IP)
+        self.apply_nvm("mot-/dev/enet0-cipa", GLOBALS.RTEMS_IOC_IP)
+        self.apply_nvm("mot-boot-device", "/dev/em1")
+        self.apply_nvm("mot-script-boot", mot_boot)
+        self.apply_nvm("rtems-client-name", GLOBALS.IOC_NAME)
+        self.apply_nvm("epics-script", "/epics/runtime/st.cmd")
+        self.apply_nvm("epics-nfsmount", nfs_mount)
+        # self.apply_nvm_variable("epics-ntpserver", "EPICS_TS_NTP_INET")
+        self.apply_nvm("mot-/dev/enet0-snma", GLOBALS.RTEMS_IOC_NETMASK)

src/rtems_proxy/copy.py

@@ -2,16 +2,15 @@
 functions for moving IOC assets into position for a remote IOC to access
 """
 
-import re
 import shutil
-from pathlib import Path
 
 from .globals import GLOBALS
 
 
 def copy_rtems():
     """
-    Copy RTEMS binaries to a location where the RTEMS IOC can access them
+    Copy RTEMS IOC binary and startup assets to a location where the RTEMS IOC
+    can access them
 
     IMPORTANT: local_root and nfs_root are different perspectives on the same
                folder.
@@ -23,7 +22,6 @@ def copy_rtems():
                 will look for them using NFS.
     """
     local_root = GLOBALS.RTEMS_TFTP_PATH
-    nfs_root = Path("/iocs") / GLOBALS.IOC_NAME
 
     # where to copy the Generic IOC folder to. This will contain the IOC binary
     # and the files
@@ -32,10 +30,12 @@ def copy_rtems():
     # st.cmd and ioc.db
     dest_runtime = local_root / "runtime"
 
-    # TODO - perhaps do protocol files in this fashion for linux IOCs too,
-    # in which case this needs to go somewhere generic
     protocol_folder = GLOBALS.RUNTIME / "protocol"
     protocol_folder.mkdir(parents=True, exist_ok=True)
+
+    # TODO - perhaps do protocol files in this fashion for linux IOCs too,
+    # in which case this needs to go somewhere generic
+    dest_ioc.mkdir(parents=True, exist_ok=True)
     protocol_files = GLOBALS.SUPPORT.glob("**/*.proto*")
     for proto_file in protocol_files:
         dest = protocol_folder / proto_file.name
@@ -53,9 +53,3 @@ def copy_rtems():
             GLOBALS.IOC.readlink() / folder, dest_ioc / folder, dirs_exist_ok=True
         )
     shutil.copytree(GLOBALS.RUNTIME, dest_runtime, dirs_exist_ok=True)
-
-    # because we moved the ioc files we need to fix up startup script paths
-    startup = dest_runtime / "st.cmd"
-    cmd_txt = startup.read_text()
-    cmd_txt = re.sub("/epics/", f"{str(nfs_root)}/", cmd_txt)
-    startup.write_text(cmd_txt)

src/rtems_proxy/rsync.sh.jinja

@@ -48,7 +48,7 @@ while true; do
   for i in 1 2 3 ; do
     # repeat because inotify fires on the first change of several
     # don't copy the huge ioc binary file with symbols
-    rsync -rim --exclude bin/RTEMS-beatnik/ioc --delete /$RTEMS_TFTP_PATH/ \
+    rsync -rim --delete /$RTEMS_TFTP_PATH/ \
           "rsync://$RTEMS_TFTP_IP:12002/files/$IOC_NAME/"
     sleep 1
   done

src/rtems_proxy/telnet.py

@@ -36,8 +36,9 @@ class TelnetRTEMS:
     IOC_CHECK = "\ntaskwdShow"
     IOC_RESPONSE = "free nodes"
     NO_CONNECTION = "Connection closed by foreign host"
+    FAIL_STRINGS = ["Exception", "exception", "RTEMS_FATAL_SOURCE_EXCEPTION"]
 
-    def __init__(self, host_and_port: str, ioc_reboot: bool):
+    def __init__(self, host_and_port: str, ioc_reboot: bool = False):
         self._hostname, self._port = host_and_port.split(":")
         self._ioc_reboot = ioc_reboot
         self._child = None
@@ -91,6 +92,7 @@ def check_prompt(self, retries=5) -> RtemsState:
         while retries > 0:
             try:
                 # see if we are in the IOC shell
+                sleep(0.5)
                 self._child.sendline(self.IOC_CHECK)
                 self._child.expect(self.IOC_RESPONSE, timeout=1)
             except pexpect.exceptions.TIMEOUT:
@@ -136,6 +138,12 @@ def reboot(self, into: RtemsState):
             # send space to boot the IOC
             self._child.send(" ")
 
+    def wait_epics_prompt(self, timeout=50):
+        expects = self.FAIL_STRINGS + [self.IOC_STARTED]
+        index = self._child.expect(expects, timeout=timeout)
+        if index != len(self.FAIL_STRINGS):
+            raise RuntimeError(f"IOC boot failed - output included '{expects[index]}'")
+
     def get_epics_prompt(self):
         """
         Get to the IOC shell prompt, if the IOC is not already running, reboot
@@ -149,12 +157,12 @@ def get_epics_prompt(self):
             sleep(0.2)
             self.reboot(RtemsState.IOC)
             self.ioc_rebooted = True
-            self._child.expect(self.IOC_STARTED, timeout=50)
+            self.wait_epics_prompt()
         else:
             if self._ioc_reboot and not self.ioc_rebooted:
                 self.ioc_rebooted = True
                 self.reboot(RtemsState.IOC)
-                self._child.expect(self.IOC_STARTED, timeout=50)
+                self.wait_epics_prompt()
 
     def get_boot_prompt(self):
         """
@@ -171,6 +179,20 @@ def get_boot_prompt(self):
 
         report("press enter for bootloader prompt")
 
+    def sendline(self, command: str) -> None:
+        """
+        Send a command to the telnet session
+        """
+        assert self._child, "must call connect before send"
+        self._child.sendline(command)
+
+    def expect(self, pattern, timeout=10) -> None:
+        """
+        Expect a pattern in the telnet session
+        """
+        assert self._child, "must call connect before expect"
+        self._child.expect(pattern, timeout=timeout)
+
     def close(self):
         if self._child:
             self._child.close()
@@ -187,7 +209,12 @@ def report(message):
     print(f"\n>>>> {message} <<<<\n")
 
 
-def ioc_connect(host_and_port: str, reboot: bool = False):
+def ioc_connect(
+    host_and_port: str,
+    reboot: bool = False,
+    attach: bool = True,
+    raise_errors: bool = False,
+):
     """
     Entrypoint to make a connection to an RTEMS IOC over telnet.
     Once connected, enters an interactive user session with the IOC.
@@ -200,14 +227,48 @@ def ioc_connect(host_and_port: str, reboot: bool = False):
 
     try:
         telnet.connect()
+
+        # this will untangle a partially executed gevEdit command
+        for _ in range(3):
+            telnet.sendline("\r")
+
         if reboot:
             telnet.get_epics_prompt()
         else:
             report("Auto reboot disabled. Skipping reboot")
+
     except (CannotConnect, pexpect.exceptions.TIMEOUT):
-        report("Connection failed. Exiting")
+        report("Connection failed, Exiting.")
         telnet.close()
-    else:
+        raise
+
+    except Exception as e:
+        # still show the remaining output
+        telnet.expect("_main_")
+        report(f"An error occurred: {e}")
         telnet.close()
+        if raise_errors:
+            raise
+
+    telnet.close()
+    if attach:
         report("Connecting to IOC console, hit enter for a prompt")
         run_command(telnet.command)
+
+
+def motboot_connect(host_and_port: str) -> TelnetRTEMS:
+    """
+    Connect to the MOTBoot bootloader prompt, rebooting if needed.
+
+    Returns a TelnetRTEMS object that is connected to the MOTBoot bootloader
+    """
+    telnet = TelnetRTEMS(host_and_port)
+    telnet.connect()
+
+    # this will untangle a partially executed gevEdit command
+    for _ in range(3):
+        telnet.sendline("\r")
+
+    telnet.get_boot_prompt()
+
+    return telnet