From adc73fd7e76841932d8eec4433b883f1fe5dc2b5 Mon Sep 17 00:00:00 2001
From: Ruben Horn <>
Date: Tue, 13 Aug 2024 17:30:44 +0200
Subject: [PATCH] config-generator: non-interactive mode

---
 examples/config-generator/README.md           |   6 +-
 .../assets/configuration_template.json        |  13 +-
 examples/config-generator/run                 |   4 +-
 .../generators/{domain.py => domain_size.py}  |  11 +-
 .../src/generators/filtering.py               |   2 +-
 .../src/generators/simulation.py              |   5 +-
 .../src/generators/solver_md.py               |   4 +-
 .../src/generators/use_checkpoint.py          |   7 +-
 examples/config-generator/src/main.py         | 198 ++++++++++++++----
 examples/plotting/plot-couette-profiles.py    |  20 +-
 10 files changed, 195 insertions(+), 75 deletions(-)
 rename examples/config-generator/src/generators/{domain.py => domain_size.py} (85%)

diff --git a/examples/config-generator/README.md b/examples/config-generator/README.md
index 003122b6f..a6360c2eb 100644
--- a/examples/config-generator/README.md
+++ b/examples/config-generator/README.md
@@ -42,6 +42,8 @@ This file has the following structure:
 
 ### Usage
 1. Navigate to the target directory for the `couette.xml` file (most likely the MaMiCo build folder)
-2. Execute `$MAMICO_DIR/examples/config-generator/run`
+2. Execute `$MAMICO_DIR/examples/config-generator/run`  
+Specific values can be fixed by providing them as a command line argument, e.g. `--override=domain_size=large,use_checkpoint=False,multi_md=2`.  
+The corresponding entries will be removed from the interactive menu. If all configurations are overwritten, the script will run non-interactively. (May be used for automated testing.)
 3. Customize the generator settings and generate the `couette.xml` file
-3. Customize the generated configuration even further according to the [documentation](https://github.com/HSU-HPC/MaMiCo/wiki/couette.xml) (optional)
+4. Customize the generated configuration even further according to the [documentation](https://github.com/HSU-HPC/MaMiCo/wiki/couette.xml) (optional)
diff --git a/examples/config-generator/assets/configuration_template.json b/examples/config-generator/assets/configuration_template.json
index 0dd1b176c..6c818da1f 100644
--- a/examples/config-generator/assets/configuration_template.json
+++ b/examples/config-generator/assets/configuration_template.json
@@ -1,22 +1,19 @@
 [
   {
-    "key": "domain",
+    "key": "domain_size",
     "label": "domain size",
     "options": [
       {
         "selected": true,
-        "value": 1,
-        "label": "small",
+        "value": "small",
         "description": "MD/CFD domain size = 30\u00B3/50\u00B3"
       },
       {
-        "value": 2,
-        "label": "medium",
+        "value": "medium",
         "description": "MD/CFD domain size = 60\u00B3/100\u00B3"
       },
       {
-        "value": 3,
-        "label": "large",
+        "value": "large",
         "description": "MD/CFD domain size = 120\u00B3/200\u00B3"
       }
     ]
@@ -188,4 +185,4 @@
       }
     ]
   }
-]
\ No newline at end of file
+]
diff --git a/examples/config-generator/run b/examples/config-generator/run
index f2b100015..2d280c8dd 100755
--- a/examples/config-generator/run
+++ b/examples/config-generator/run
@@ -4,7 +4,7 @@
 # Any dependencies are automatically installed.
 # The output of the script is saved to the current working directory.
 
-OUTPUT="$(pwd)/couette.xml"
+OUTPUT="$(pwd)"
 
 cd "$(dirname "$0")" || exit
 
@@ -22,4 +22,4 @@ fi
 source .venv/bin/activate
 
 # Run the config builder
-python3 src/main.py "$OUTPUT"
+python3 src/main.py --output "$OUTPUT" "$@"
diff --git a/examples/config-generator/src/generators/domain.py b/examples/config-generator/src/generators/domain_size.py
similarity index 85%
rename from examples/config-generator/src/generators/domain.py
rename to examples/config-generator/src/generators/domain_size.py
index 1b7d8cfe4..c39ca44af 100644
--- a/examples/config-generator/src/generators/domain.py
+++ b/examples/config-generator/src/generators/domain_size.py
@@ -2,19 +2,22 @@
 from generators.mpi_ranks import get_ranks_xyz
 
 
-def _get_domain_size(get_config_value) -> int:
+def get_domain_size(get_config_value) -> int:
     key = __name__.split(".")[-1]
-    return get_config_value(key)
+    domain_size_name = get_config_value(key)
+    domain_size_names = ["small", "medium", "large"]
+    return domain_size_names.index(domain_size_name) + 1
 
 
 def get_domain_sizes(get_config_value) -> int:
-    size = _get_domain_size(get_config_value)
+    size = get_domain_size(get_config_value)
     md_base_size = 30
     cfd_base_size = 50
     md_domain_size = md_base_size * size
     cfd_domain_size = cfd_base_size * size
     if size == 3:
         md_domain_size = 120  # instead of 90
+        cfd_domain_size = 200  # instead of 150
     return md_domain_size, cfd_domain_size
 
 
@@ -35,7 +38,7 @@ def validate(get_config_value) -> str:
 def apply(partial_xml, get_config_value) -> None:
     equilibration_steps = 10000
     equilibration_steps_max = 20000
-    size = _get_domain_size(get_config_value)
+    size = get_domain_size(get_config_value)
     md_domain_size, cfd_domain_size = get_domain_sizes(get_config_value)
     domain_offset_xy = (cfd_domain_size - md_domain_size) / 2  # Centered
     partial_xml.substitute("md-size", md_domain_size)
diff --git a/examples/config-generator/src/generators/filtering.py b/examples/config-generator/src/generators/filtering.py
index 62bc56a3c..32f2cba37 100644
--- a/examples/config-generator/src/generators/filtering.py
+++ b/examples/config-generator/src/generators/filtering.py
@@ -25,7 +25,7 @@ def apply(partial_xml, get_config_value) -> None:
 """.strip()
 
     if filtering == False:
-        per_instance_filtering_xml = "<per-instance output=\"md\"></per-instance>"
+        per_instance_filtering_xml = '<per-instance output="md"></per-instance>'
     elif filtering == "gauss":
         per_instance_filtering_xml = xml_2d_gaussian
     elif filtering == "nlm":
diff --git a/examples/config-generator/src/generators/simulation.py b/examples/config-generator/src/generators/simulation.py
index f6a9e788c..b42cdf1b3 100644
--- a/examples/config-generator/src/generators/simulation.py
+++ b/examples/config-generator/src/generators/simulation.py
@@ -1,7 +1,10 @@
+from generators.domain_size import get_domain_size
+
+
 def apply(partial_xml, get_config_value) -> None:
     key = __name__.split(".")[-1]
     simulation_type = get_config_value(key)
-    domain_size = get_config_value("domain")
+    domain_size = get_domain_size(get_config_value)
     match simulation_type:
         case "test":
             partial_xml.substitute("coupling-cycles", 50)
diff --git a/examples/config-generator/src/generators/solver_md.py b/examples/config-generator/src/generators/solver_md.py
index 048d3f84d..6c1416409 100644
--- a/examples/config-generator/src/generators/solver_md.py
+++ b/examples/config-generator/src/generators/solver_md.py
@@ -1,7 +1,7 @@
 from pathlib import Path
 
 from generators.cell_size import get_molecules_per_direction
-from generators.domain import get_domain_sizes
+from generators.domain_size import get_domain_sizes
 from utils import get_asset_text
 from xml_templating import PartialXml
 
@@ -15,7 +15,7 @@ def _create_ls1_config(get_config_value) -> None:
     xml.substitute("grid-filler-lattice", grid_filler_lattice)
     boundary_condition = get_config_value("boundary")
     xml.substitute("boundary-condition", boundary_condition)
-    filename = Path(get_config_value("output_filename")).parent / "ls1config.xml"
+    filename = Path(get_config_value("output_dir")) / "ls1config.xml"
     with open(filename, "w") as file:
         file.write(xml.get())
 
diff --git a/examples/config-generator/src/generators/use_checkpoint.py b/examples/config-generator/src/generators/use_checkpoint.py
index 0e51c0bc0..95f0c490a 100644
--- a/examples/config-generator/src/generators/use_checkpoint.py
+++ b/examples/config-generator/src/generators/use_checkpoint.py
@@ -1,6 +1,8 @@
 import shutil
 from pathlib import Path
 
+from generators.domain_size import get_domain_size
+
 
 def validate(get_config_value) -> str:
     """MaMiCo config validation:
@@ -8,7 +10,7 @@ def validate(get_config_value) -> str:
     """
     key = __name__.split(".")[-1]
     use_checkpoint = get_config_value(key)
-    domain_size = get_config_value("domain")
+    domain_size = get_domain_size(get_config_value)
     solver_md = get_config_value("solver_md")
     if use_checkpoint and (domain_size > 1 or solver_md != "md"):
         return f"Example checkpoint file only provided for small Simple MD simulation."
@@ -29,8 +31,7 @@ def apply(partial_xml, get_config_value) -> None:
             / f"CheckpointSimpleMD_10000_{boundary_condition}_0.checkpoint"
         )
         checkpoint_dst_path = (
-            Path(get_config_value("output_filename")).parent
-            / "CheckpointSimpleMD.checkpoint"
+            Path(get_config_value("output_dir")) / "CheckpointSimpleMD.checkpoint"
         )
         # Avoid reading and writing the contents of the file, because it is rather large
         shutil.copyfile(checkpoint_src_path, checkpoint_dst_path)
diff --git a/examples/config-generator/src/main.py b/examples/config-generator/src/main.py
index 59f6f8b12..d77f38000 100644
--- a/examples/config-generator/src/main.py
+++ b/examples/config-generator/src/main.py
@@ -1,10 +1,11 @@
 #! /usr/bin/env python3
 
 """
-Utility script to generate basic couette.xml configuration for MaMiCo.
+Utility script to generate basic couette.xml configurations for MaMiCo.
 Do NOT run this script directly! Use ../run instead.
 """
 
+import argparse
 import importlib
 import json
 import os
@@ -16,16 +17,47 @@
 from xml_templating import PartialXml
 
 
-def select_option(config: dict) -> None:
+def select_option(configs: list, key: str, value: str) -> None:
+    """Select an option based on a serialized key value pair without user input.
+
+    Keyword arguments:
+    configs -- All configurations in which to update the configuration with the specified key
+    key -- The key by which to select the configuration
+    value -- The serialized representation of the value which should be selected
+    """
+    for config in configs:
+        if config["key"] == key:
+            get_selected(config)["selected"] = False
+            for option in config["options"]:
+                if str(option["value"]) == value:
+                    option["selected"] = True
+                    return
+            raise ValueError(
+                f'No option with the value "{value}" exists for the configuration with the key "{key}"'
+            )
+    raise ValueError(f'No configuration with the key "{key}" exists')
+
+
+def select_option_interactive(configs: list, label: str) -> None:
     """Show a terminal menu for a single option of the configuration and apply the selection of the user to the current configurations.
 
     Keyword arguments:
-    config -- The configuration from all configurations (by reference) which should be updated
+    configs -- All configurations in which to update the configuration with the specified label
+    label -- The menu label by which to select the configuration
     """
-    print(config)
+    selected_config = None
+    for config in configs:
+        if config["label"] == label:
+            if selected_config is not None:
+                raise RuntimeError(
+                    f'Ambiguous selection. Multiple configs match the label "{label}"'
+                )
+            else:
+                selected_config = config
+    del config  # Avoid bugs by re-using variable
     pre_selected = None
     option_labels = []
-    for i, option in enumerate(config["options"]):
+    for i, option in enumerate(selected_config["options"]):
         if "selected" in option and option["selected"]:
             pre_selected = i
             option["selected"] = False
@@ -34,9 +66,9 @@ def select_option(config: dict) -> None:
             label += "\t" + option["description"]
         option_labels.append(label)
     selected = term.select(
-        option_labels, title=config["label"], pre_selected=pre_selected
+        option_labels, title=selected_config["label"], pre_selected=pre_selected
     )
-    config["options"][selected]["selected"] = True
+    selected_config["options"][selected]["selected"] = True
 
 
 def get_selected(config: dict) -> object:
@@ -77,27 +109,32 @@ def load_generator(generator: str) -> object:
         exit(1)
 
 
-def generate(configs: list, filename: str) -> None:
+def generate(configs: list, output_dir: str) -> None:
     """Create the configuration by loading the template, applying the generators, and saving the resulting XML file.
+    This function terminates the process.
 
     Keyword arguments:
     configs -- The list of configurations which to apply using the generators corresponding to the keys
-    filename -- The output XML filename
+    output_dir -- The directory where couette.xml and other files will be created
     """
     xml = PartialXml(get_asset_text("couette.xml.template"))
     print("Loaded configuration template")
     for config in configs:
         generator = load_generator(config["key"])
         # Make output path accessible to generators
-        config_output_filename = [
-            dict(key="output_filename", options=[dict(selected=True, value=filename)])
+        config_output_dir = [
+            dict(key="output_dir", options=[dict(selected=True, value=output_dir)])
         ]
-        generator.apply(
-            xml, lambda k: get_config_value(configs + config_output_filename, k)
-        )
-    with open(filename, "w") as file:
+        generator.apply(xml, lambda k: get_config_value(configs + config_output_dir, k))
+    path = output_dir / "couette.xml"
+    with open(path, "w") as file:
         file.write(xml.get())
-    print(f"\nWrote configuration to {filename}")
+    print(f"\nWrote configuration to {path}")
+    ranks = get_config_value(configs, "mpi_ranks") * get_config_value(
+        configs, "multi_md"
+    )
+    print(f'\nRun simulation using "mpirun -n {ranks} $MAMICO_DIR/build/couette"')
+    exit(0)
 
 
 def validate(configs: list) -> str:
@@ -115,7 +152,8 @@ def validate(configs: list) -> str:
             )
         except AttributeError:
             print(
-                f"Could not invoke validation for generator \"{config['key']}\"", file=sys.stderr,
+                f"Could not invoke validation for generator \"{config['key']}\"",
+                file=sys.stderr,
             )
             continue
         if validation_error is not None:
@@ -123,10 +161,76 @@ def validate(configs: list) -> str:
     return validation_errors.strip()
 
 
+def parse_args(argv: dict = sys.argv[1:], configs: list = []) -> object:
+    """Parse the arguments (from the command line) and return them as a dictionary.
+    If the usage should be printed, the program is terminated instead.
+
+    Keyword arguments:
+    argv -- The command line arguments excluding the name of the script itself or the interpreter
+    configs -- All configurations which may be overwritten using command line arguments
+    """
+    # Extract all possible key=value overrides for help text and validation
+    all_override_kvs = set()
+    all_overrides = ""
+    for config in configs:
+        key = config["key"]
+        all_overrides += f"\n\t{key}\t=\t<"
+        for i, option in enumerate(config["options"]):
+            value = str(option["value"])
+            all_override_kvs.add(f"{key}={value}")
+            if i > 0:
+                all_overrides += "|"
+            all_overrides += value
+        all_overrides += ">"
+    # Parse arguments
+    arg_parser = argparse.ArgumentParser(
+        description="A simple utility to generate basic couette.xml configurations for MaMiCo.",
+        epilog="Available overrides:" + all_overrides,
+        formatter_class=argparse.RawTextHelpFormatter,
+    )
+    arg_parser.add_argument(
+        "-o",
+        "--output",
+        type=Path,
+        required=True,
+        help="The directory where couette.xml and other files will be created",
+    )
+    arg_parser.add_argument(
+        "--override",
+        type=str,
+        default="",
+        help="Comma separated list of key-value (key=value) config overrides. (Will be removed from interactive menu)",
+    )
+    # Parse key=value overrides
+    args = arg_parser.parse_args(argv)
+    override = {}
+    for kv in args.override.split(","):
+        if len(kv.strip()) == 0:
+            continue
+        parts = kv.split("=")
+        if len(parts) != 2:
+            arg_parser.error(
+                "The value of --override must be a comma separated list of key-value pairs (key=value)."
+            )
+        k, v = (p.strip() for p in parts)
+        override[k] = v
+    args.override = override
+    # Check if every key=value override is valid
+    for key, value in args.override.items():
+        if f"{key}={value}" not in all_override_kvs:
+            arg_parser.error(f'Invalid override "{key}={value}"')
+    if not Path(args.output).is_dir():
+        arg_parser.error(
+            "The value of --output must be a path to a directory, not a file."
+        )
+    return args
+
+
 def main() -> None:
     os.chdir(Path(__file__).parent)
     # 1. Load configuration options
     configs = json.loads(get_asset_text("configuration_template.json"))
+    args = parse_args(configs=configs)
     # Fill missing labels
     for config in configs:
         if "label" not in config:
@@ -140,36 +244,46 @@ def main() -> None:
         title = "MaMiCo couette.xml Generator\n"
         main_menu = []
         for config in configs:
-            main_menu.append(config["label"] + "\t" + get_selected(config)["label"])
+            # Apply overrides from command line
+            key = config["key"]
+            if key in args.override:
+                select_option(configs, key, args.override[key])
+            else:
+                main_menu.append(config["label"] + "\t" + get_selected(config)["label"])
         # 3. Validate
         validation_errors = validate(configs)
         is_valid = len(validation_errors.strip()) == 0
-        if is_valid:
-            main_menu[-1] += "\n"
-            main_menu.append("Generate")
+        is_interactive = (
+            len(main_menu) > 0
+        )  # Skip interactive mode if there are no options
+        if not is_interactive:
+            if is_valid:
+                generate(configs, args.output)
+            else:
+                print(validation_errors, file=sys.stderr)
+                exit(1)
         else:
-            title += "\n" + validation_errors + "\n"
-        # 4. Show and the menu
-        selected = term.select(
-            main_menu,
-            title=title,
-            pre_selected=-1 if is_valid else None,
-        )
-        if selected == -1:
-            exit(1)
-        elif is_valid and selected == len(main_menu) - 1:
-            # Generate
-            generate(configs, sys.argv[1])
-            ranks = get_config_value(configs, "mpi_ranks") * get_config_value(
-                configs, "multi_md"
+            if is_valid:
+                main_menu[-1] += "\n"
+                main_menu.append("Generate")
+            else:
+                title += "\n" + validation_errors + "\n"
+            # 4. Show and the menu
+            selected = term.select(
+                main_menu,
+                title=title,
+                pre_selected=-1 if is_valid else None,
             )
-            print(
-                f'\nRun simulation using "mpirun -n {ranks} $MAMICO_DIR/build/couette"'
-            )
-            exit(0)
-        else:
-            # Update configuration
-            select_option(configs[selected])
+            if selected == -1:
+                exit(1)
+            elif is_valid and selected == len(main_menu) - 1:
+                # Generate
+                generate(configs, args.output)
+            else:
+                # Update configuration
+                # Remove explanation behind label
+                selected_label = main_menu[selected].split("\t")[0]
+                select_option_interactive(configs, selected_label)
 
 
 if __name__ == "__main__":
diff --git a/examples/plotting/plot-couette-profiles.py b/examples/plotting/plot-couette-profiles.py
index a6267bdb5..ae3a6a210 100644
--- a/examples/plotting/plot-couette-profiles.py
+++ b/examples/plotting/plot-couette-profiles.py
@@ -1,7 +1,7 @@
 #! /usr/bin/env python3
 
 """
-Simple but versitile script to plot couette flow profiles
+Simple but versatile script to plot couette flow profiles
 
 (Replaces https://github.com/HSU-HPC/MaMiCo/commit/42ad244c75640a692ae1b70d56c7060431fdab0d)
 """
@@ -95,18 +95,18 @@ def parse_args(argv=sys.argv[1:]):
     global args
     arg_parser = argparse.ArgumentParser()
     # Scenario parameters
-    arg_parser.add_argument("--offset", default=2.5)
-    arg_parser.add_argument("--wall-velocity", default=0.5)
-    arg_parser.add_argument("--channel-height", default=50.0)
-    arg_parser.add_argument("--density", default=0.813037037)
-    arg_parser.add_argument("--viscosity", default=2.4)
-    arg_parser.add_argument("--coupling-cells", default=6)
+    arg_parser.add_argument("--offset", default=2.5, type=float)
+    arg_parser.add_argument("--wall-velocity", default=0.5, type=float)
+    arg_parser.add_argument("--channel-height", default=50.0, type=float)
+    arg_parser.add_argument("--density", default=0.813037037, type=float)
+    arg_parser.add_argument("--viscosity", default=2.4, type=float)
+    arg_parser.add_argument("--coupling-cells", default=6, type=int)
     arg_parser.add_argument(
-        "--overlap-size", default=3, help="In number of coupling cells"
+        "--overlap-size", default=3, help="In number of coupling cells", type=int
     )
-    arg_parser.add_argument("--coupling-cell-size", default=2.5)
+    arg_parser.add_argument("--coupling-cell-size", default=2.5, type=float)
     # Script parameters
-    arg_parser.add_argument("--workdir", default=Path(__file__).parent.parent)
+    arg_parser.add_argument("--workdir", default=Path(), type=Path)
     arg_parser.add_argument(
         "--coupling-cycles",
         default="",