feat: playground test update (#1488)

test/playground/commands.py

@@ -1,28 +1,26 @@
-import subprocess
-import time
-from pathlib import Path
-
 import psutil
+import platform
 import pytest
+import subprocess
+import time
 import yaml
+from pathlib import Path
 from rich.text import Text
 
 from .rules import get_rule
 from .shared import results
 from .utils import (
-    create_compound_input_csv,
     get_command_names,
     get_commands,
     handle_error_logging,
-    save_as_json,
 )
 
 config = yaml.safe_load(Path("config.yml").read_text())
 delete_model = config.get("delete_model", False)
 activate_docker = config.get("activate_docker", False)
 runner = config.get("runner", "single")
 cli_type = config.get("cli_type", "all")
-output_file = config.get("output_file")
+output_file = config.get("output_files")
 input_file = config.get("input_file")
 redirect = config.get("output_redirection", False)
 
@@ -37,19 +35,45 @@
 from_dockerhub = "--from_dockerhub" in config.get("fetch_flags", "")
 
 
-def execute_command(command, description="", dest_path=None, repo_path=None):
-    # generating input eg.
-    create_compound_input_csv(config.get("input_file"))
-    # docker sys control
+def manage_docker(config):
     docker_activated = False
+
     if config and config.get("activate_docker"):
-        docker_status = subprocess.run(["systemctl", "is-active", "--quiet", "docker"])
-        if docker_status.returncode != 0:
-            subprocess.run(["systemctl", "start", "docker"], check=True)
+        system_platform = platform.system()
+
+        if system_platform == "Linux":
+            docker_status = subprocess.run(
+                ["systemctl", "is-active", "--quiet", "docker"]
+            )
+            if docker_status.returncode != 0:
+                subprocess.run(["systemctl", "start", "docker"], check=True)
+        elif system_platform == "Darwin":  # macOS
+            docker_status = subprocess.run(
+                ["docker", "info"], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL
+            )
+            if docker_status.returncode != 0:
+                print(
+                    "Docker is not running. Please start the Docker Desktop application."
+                )
+                raise RuntimeError(
+                    "Docker is not running on macOS. Start Docker Desktop."
+                )
+        else:
+            raise OSError(f"Unsupported platform: {system_platform}")
+
         docker_activated = True
     else:
-        subprocess.run(["systemctl", "stop", "docker"], check=True)
+        if platform.system() == "Linux":
+            subprocess.run(["systemctl", "stop", "docker"], check=True)
+        elif platform.system() == "Darwin":
+            print("Stopping Docker programmatically is not supported on macOS.")
+            raise RuntimeError("Cannot stop Docker programmatically on macOS.")
+
+    return docker_activated
+
 
+def execute_command(command, description="", dest_path=None, repo_path=None):
+    docker_activated = manage_docker(config)
     (
         start_time,
         max_memory,
@@ -88,9 +112,6 @@ def execute_command(command, description="", dest_path=None, repo_path=None):
 
         pytest.fail(f"{description} '{' '.join(command)}' failed with error: {result}")
 
-    if description == "run" and success and config.get("output_redirection"):
-        save_as_json(result, output_file, remove_list=[input_file])
-
     checkups = apply_rules(command, description, dest_path, repo_path, config)
 
     rules_success = all(check["status"] for check in checkups)
@@ -149,19 +170,32 @@ def apply_rules(command, description, dest_path, repo_path, config):
                         expected_status=True,
                     )
                 )
-        elif description == "run":
+        elif "run" in description:
+            file = next(
+                (
+                    t
+                    for t in config["output_files"]
+                    if Path(t).suffix.replace(".", "") in description
+                ),
+                None,
+            )
             checkups.append(
                 get_rule(
                     "file_exists",
-                    file_path=output_file,
+                    file_path=file,
                     expected_status=True,
                 )
             )
+
+            inp_type = next(
+                (t for t in config["input_types"] if t in description), None
+            )
+
             checkups.append(
                 get_rule(
                     "file_content_check",
-                    file_path=output_file,
-                    expected_status="not null",
+                    file_path=file,
+                    expected_input=inp_type,
                 )
             )
         elif description == "serve":
@@ -182,7 +216,6 @@ def apply_rules(command, description, dest_path, repo_path, config):
 
 @pytest.fixture(scope="module", autouse=True)
 def delete_model_command():
-    """Deletes models if delete_model is True."""
     if delete_model:
         for model_id in model_ids:
             dest_path = base_path / "dest" / model_id
@@ -221,7 +254,7 @@ def test_command(model_id, command_name):
 
     assert success, f"Command '{command_name}' failed for model ID {model_id}"
 
-    if command_name == "run" and max_runtime_minutes is not None:
+    if "run" in command_name and max_runtime_minutes is not None:
         assert (
             time_taken <= max_runtime_minutes
         ), f"Command '{command_name}' for model ID {model_id} exceeded max runtime of {max_runtime_minutes} minutes"

test/playground/config.yml

@@ -3,6 +3,10 @@ cli_type: all
 delete_model: true
 fetch_flags: --from_github
 input_file: files/input.csv
+input_types:
+- str
+- csv
+- list
 log_error: true
 log_path: files/error_log
 max_runtime_minutes: 10
@@ -11,11 +15,15 @@ model_ids:
 - eos3b5e
 - eos4e40
 - eos9gg2
-output_file: files/result.csv
+number_of_input_samples: 1
+output_files:
+- files/result.json
+- files/result.csv
+- files/result.h5
 output_redirection: false
 overwrite_ersilia_repo: false
 python_version: 3.10.10
 run_flags: ''
-runner: single
+runner: signle
 serve_flags: ''
 use_existing_env: true

test/playground/noxfile.py

@@ -54,7 +54,29 @@ def setup_ersilia(session):
     session.chdir(ORIGINAL_DIR)
 
 
-@nox.session(venv_backend="conda", python=get_python_version())
+def parse_and_update_config(session):
+    passed_args = {
+        arg.split("=")[0]: arg.split("=")[1] for arg in session.posargs if "=" in arg
+    }
+
+    for key, value in passed_args.items():
+        if value.lower() in ("true", "false"):
+            passed_args[key] = value.lower() == "true"
+        elif "," in value:
+            passed_args[key] = value.split(",")
+
+    new_config = {key: passed_args.get(key, config.get(key)) for key in config.keys()}
+
+    logger.info("Parsed arguments and updated configuration:")
+    for key in passed_args.keys():
+        logger.info(f"  {key}: {passed_args[key]}")
+
+    update_yaml_values(new_config)
+
+    return new_config
+
+
+@nox.session(venv_backend="conda", python=get_python_version(), reuse_venv=True)
 def setup(session):
     install_dependencies(session)
     setup_ersilia(session)
@@ -63,35 +85,41 @@ def setup(session):
 @nox.session(venv_backend="conda", python=get_python_version())
 def test_from_github(session):
     install_dependencies(session)
+    setup_ersilia(session)
+    update_yaml_values({"fetch_flags": "--from_github"})
     logger.info(
         f'CLI test for model: {config.get("model_id")} and {config.get("fetch_flags")}'
     )
+    parse_and_update_config(session)
     session.run("pytest", "commands.py", "-v", silent=False)
 
 
-@nox.session(venv_backend="conda", python=get_python_version())
+@nox.session(venv_backend="conda", python=get_python_version(), reuse_venv=True)
 def test_from_dockerhub(session):
     install_dependencies(session)
+    setup_ersilia(session)
     update_yaml_values({"fetch_flags": "--from_dockerhub"})
     logger.info(
         f'CLI test for model: {config.get("model_id")} and {config.get("fetch_flags")}'
     )
+    parse_and_update_config(session)
     session.run("pytest", "commands.py", "-v", silent=False)
 
 
 @nox.session(venv_backend="conda", python=get_python_version())
 def test_auto_fetcher_decider(session):
     install_dependencies(session)
+    setup_ersilia(session)
     update_yaml_values({"fetch_flags": ""})
     logger.info(
         f'CLI test for model: {config.get("model_id")} and auto fetcher decider'
     )
+    parse_and_update_config(session)
     session.run("pytest", "commands.py", "-v", silent=False)
 
 
 @nox.session(venv_backend="conda", python=get_python_version())
 def test_fetch_multiple_models(session):
-    install_dependencies(session)
     update_yaml_values(
         {
             "runner": "multiple",
@@ -100,32 +128,34 @@ def test_fetch_multiple_models(session):
         }
     )
     logger.info("Fetching and Serving Multiple Models: Fetching")
+    parse_and_update_config(session)
     session.run("pytest", "commands.py", "-v", silent=False)
 
 
 @nox.session(venv_backend="conda", python=get_python_version())
 def test_serve_multiple_models(session):
-    install_dependencies(session)
     update_yaml_values(
         {"runner": "multiple", "cli_type": "serve", "delete_model": False}
     )
     logger.info("Fetching and Serving Multiple Models: Serving")
+    parse_and_update_config(session)
     session.run("pytest", "commands.py", "-v", silent=False)
 
 
 @nox.session(venv_backend="conda", python=get_python_version())
 def test_conventional_run(session):
-    """Run pytest for standard and conventional run."""
-    install_dependencies(session)
     update_yaml_values(
         {
             "runner": "single",
             "cli_type": "all",
             "fetch_flags": "--from_dockerhub",
-            "output_file": "files/output_eos9gg2_0.json",
-            "output_redirection": "true",
+            "output_files": [
+                "files/output_eos9gg2_0.json",
+                "files/output_eos9gg2_1.h5",
+            ],
             "delete_model": True,
         }
     )
     logger.info("Standard and Conventional Run: Conventional")
+    parse_and_update_config(session)
     session.run("pytest", "commands.py", "-v", silent=False)