DRAFT: Config loading

pyproject.toml

@@ -13,6 +13,7 @@ dependencies = [
     # Keep sorted
     "click>=8.1.7",
     "enlighten>=1.12.4",
+    "gitpython>=3.1.44",
     "hjson>=3.1.0",
     "logzero>=1.7.0",
     "mistletoe>=1.4.0",

src/dvsim/cli.py

@@ -24,13 +24,11 @@
 import datetime
 import os
 import random
-import shlex
-import subprocess
 import sys
 import textwrap
 from pathlib import Path
 
-from dvsim.flow.factory import make_cfg
+from dvsim.flow.factory import make_flow
 from dvsim.job.deploy import RunTest
 from dvsim.launcher.base import Launcher
 from dvsim.launcher.factory import set_launcher_type
@@ -40,7 +38,8 @@
 from dvsim.launcher.sge import SgeLauncher
 from dvsim.launcher.slurm import SlurmLauncher
 from dvsim.logging import configure_logging, log
-from dvsim.utils import TS_FORMAT, TS_FORMAT_LONG, Timer, rm_path, run_cmd_with_timeout
+from dvsim.project import Project
+from dvsim.utils import TS_FORMAT, TS_FORMAT_LONG, Timer
 
 # TODO: add dvsim_cfg.hjson to retrieve this info
 version = 0.1
@@ -49,50 +48,6 @@
 _LIST_CATEGORIES = ["build_modes", "run_modes", "tests", "regressions"]
 
 
-# Function to resolve the scratch root directory among the available options:
-# If set on the command line, then use that as a preference.
-# Else, check if $SCRATCH_ROOT env variable exists and is a directory.
-# Else use the default (<proj_root>/scratch)
-# Try to create the directory if it does not already exist.
-def resolve_scratch_root(arg_scratch_root, proj_root):
-    default_scratch_root = proj_root + "/scratch"
-    scratch_root = os.environ.get("SCRATCH_ROOT")
-    if not arg_scratch_root:
-        if scratch_root is None:
-            arg_scratch_root = default_scratch_root
-        else:
-            # Scratch space could be mounted in a filesystem (such as NFS) on a network drive.
-            # If the network is down, it could cause the access access check to hang. So run a
-            # simple ls command with a timeout to prevent the hang.
-            (out, status) = run_cmd_with_timeout(
-                cmd="ls -d " + scratch_root,
-                timeout=1,
-                exit_on_failure=0,
-            )
-            if status == 0 and out != "":
-                arg_scratch_root = scratch_root
-            else:
-                arg_scratch_root = default_scratch_root
-                log.warning(
-                    f'Env variable $SCRATCH_ROOT="{scratch_root}" is not accessible.\n'
-                    f'Using "{arg_scratch_root}" instead.',
-                )
-    else:
-        arg_scratch_root = os.path.realpath(arg_scratch_root)
-
-    try:
-        os.makedirs(arg_scratch_root, exist_ok=True)
-    except PermissionError as e:
-        log.fatal(f"Failed to create scratch root {arg_scratch_root}:\n{e}.")
-        sys.exit(1)
-
-    if not os.access(arg_scratch_root, os.W_OK):
-        log.fatal(f"Scratch root {arg_scratch_root} is not writable!")
-        sys.exit(1)
-
-    return arg_scratch_root
-
-
 def read_max_parallel(arg):
     """Take value for --max-parallel as an integer."""
     try:
@@ -129,124 +84,6 @@ def resolve_max_parallel(arg):
     return 16
 
 
-def resolve_branch(branch):
-    """Choose a branch name for output files.
-
-    If the --branch argument was passed on the command line, the branch
-    argument is the branch name to use. Otherwise it is None and we use git to
-    find the name of the current branch in the working directory.
-
-    Note, as this name will be used to generate output files any forward slashes
-    are replaced with single dashes to avoid being interpreted as directory hierarchy.
-    """
-    if branch is not None:
-        return branch.replace("/", "-")
-
-    result = subprocess.run(
-        ["git", "rev-parse", "--abbrev-ref", "HEAD"],
-        stdout=subprocess.PIPE,
-        check=False,
-    )
-    branch = result.stdout.decode("utf-8").strip().replace("/", "-")
-    if not branch:
-        log.warning('Failed to find current git branch. Setting it to "default"')
-        branch = "default"
-
-    return branch
-
-
-# Get the project root directory path - this is used to construct the full paths
-def get_proj_root():
-    cmd = ["git", "rev-parse", "--show-toplevel"]
-    result = subprocess.run(cmd, capture_output=True, check=False)
-    proj_root = result.stdout.decode("utf-8").strip()
-    if not proj_root:
-        log.error(
-            "Attempted to find the root of this GitHub repository by running:\n"
-            "{}\n"
-            "But this command has failed:\n"
-            "{}".format(" ".join(cmd), result.stderr.decode("utf-8")),
-        )
-        sys.exit(1)
-    return proj_root
-
-
-def resolve_proj_root(args):
-    """Update proj_root based on how DVSim is invoked.
-
-    If --remote switch is set, a location in the scratch area is chosen as the
-    new proj_root. The entire repo is copied over to this location. Else, the
-    proj_root is discovered using get_proj_root() method, unless the user
-    overrides it on the command line.
-
-    This function returns the updated proj_root src and destination path. If
-    --remote switch is not set, the destination path is identical to the src
-    path. Likewise, if --dry-run is set.
-    """
-    proj_root_src = args.proj_root or get_proj_root()
-
-    # Check if jobs are dispatched to external compute machines. If yes,
-    # then the repo needs to be copied over to the scratch area
-    # accessible to those machines.
-    # If --purge arg is set, then purge the repo_top that was copied before.
-    if args.remote and not args.dry_run:
-        proj_root_dest = os.path.join(args.scratch_root, args.branch, "repo_top")
-        if args.purge:
-            rm_path(proj_root_dest)
-        copy_repo(proj_root_src, proj_root_dest)
-    else:
-        proj_root_dest = proj_root_src
-
-    return proj_root_src, proj_root_dest
-
-
-def copy_repo(src, dest) -> None:
-    """Copy over the repo to a new location.
-
-    The repo is copied over from src to dest area. It tentatively uses the
-    rsync utility which provides the ability to specify a file containing some
-    exclude patterns to skip certain things from being copied over. With GitHub
-    repos, an existing `.gitignore` serves this purpose pretty well.
-    """
-    rsync_cmd = [
-        "rsync",
-        "--recursive",
-        "--links",
-        "--checksum",
-        "--update",
-        "--inplace",
-        "--no-group",
-    ]
-
-    # Supply `.gitignore` from the src area to skip temp files.
-    ignore_patterns_file = Path(src) / ".gitignore"
-    if ignore_patterns_file.exists():
-        # TODO: hack - include hw/foundry since it is excluded in .gitignore.
-        rsync_cmd += [
-            "--include=hw/foundry",
-            f"--exclude-from={ignore_patterns_file}",
-            "--exclude=.*",
-        ]
-
-    rsync_cmd += [src + "/.", dest]
-    rsync_str = " ".join([shlex.quote(w) for w in rsync_cmd])
-
-    cmd = ["flock", "--timeout", "600", dest, "--command", rsync_str]
-
-    log.info("[copy_repo] [dest]: %s", dest)
-    log.verbose("[copy_repo] [cmd]: \n%s", " ".join(cmd))
-
-    # Make sure the dest exists first.
-    os.makedirs(dest, exist_ok=True)
-    try:
-        subprocess.run(cmd, check=True, capture_output=True)
-    except subprocess.CalledProcessError as e:
-        log.exception(
-            "Failed to copy over %s to %s: %s", src, dest, e.stderr.decode("utf-8").strip()
-        )
-    log.info("Done.")
-
-
 def wrapped_docstring():
     """Return a text-wrapped version of the module docstring."""
     paras = []
@@ -823,27 +660,24 @@ def main() -> None:
         debug=args.verbose == "debug",
     )
 
-    if not Path(args.cfg).exists():
-        log.fatal("Path to config file %s appears to be invalid.", args.cfg)
-        sys.exit(1)
-
-    args.branch = resolve_branch(args.branch)
-    proj_root_src, proj_root = resolve_proj_root(args)
-    args.scratch_root = resolve_scratch_root(args.scratch_root, proj_root)
-    log.info("[proj_root]: %s", proj_root)
-
-    # Create an empty FUSESOC_IGNORE file in scratch_root. This ensures that
-    # any fusesoc invocation from a job won't search within scratch_root for
-    # core files.
-    (Path(args.scratch_root) / "FUSESOC_IGNORE").touch()
+    project_cfg = Project.init(
+        cfg_path=Path(args.cfg),
+        proj_root=Path(args.proj_root) if args.proj_root else None,
+        scratch_root=Path(args.scratch_root) if args.scratch_root else None,
+        branch=args.branch,
+        job_prefix=args.job_prefix,
+        purge=args.purge,
+        dry_run=args.dry_run,
+        remote=args.remote,
+        select_cfgs=args.select_cfgs,
+        args=args,
+    )
+    project_cfg.save()
 
-    args.cfg = Path(args.cfg).resolve()
-    if args.remote:
-        cfg_path = args.cfg.replace(proj_root_src + "/", "")
-        args.cfg = os.path.join(proj_root, cfg_path)
+    log.set_logfile(path=project_cfg.logfile)
 
     # Add timestamp to args that all downstream objects can use.
-    curr_ts = datetime.datetime.utcnow()
+    curr_ts = datetime.datetime.now(datetime.UTC)
     args.timestamp_long = curr_ts.strftime(TS_FORMAT_LONG)
     args.timestamp = curr_ts.strftime(TS_FORMAT)
 
@@ -867,50 +701,47 @@ def main() -> None:
 
     # Build infrastructure from hjson file and create the list of items to
     # be deployed.
-    global cfg
-    cfg = make_cfg(args.cfg, args, proj_root)
+    flow = make_flow(
+        project_cfg=project_cfg,
+        args=args,
+    )
 
     # List items available for run if --list switch is passed, and exit.
     if args.list is not None:
-        cfg.print_list()
+        flow.print_list()
         sys.exit(0)
 
     # Purge the scratch path if --purge option is set.
     if args.purge:
-        cfg.purge()
+        flow.purge()
 
     # If --cov-unr is passed, run UNR to generate report for unreachable
     # exclusion file.
     if args.cov_unr:
-        cfg.cov_unr()
-        cfg.deploy_objects()
+        flow.cov_unr()
+        flow.deploy_objects()
         sys.exit(0)
 
     # In simulation mode: if --cov-analyze switch is passed, then run the GUI
     # tool.
     if args.cov_analyze:
-        cfg.cov_analyze()
-        cfg.deploy_objects()
+        flow.cov_analyze()
+        flow.deploy_objects()
         sys.exit(0)
 
-    # Deploy the builds and runs
-    if args.items:
-        # Create deploy objects.
-        cfg.create_deploy_objects()
-        results = cfg.deploy_objects()
-
-        # Generate results.
-        cfg.gen_results(results)
-
-    else:
+    if not args.items:
         log.error("Nothing to run!")
         sys.exit(1)
 
+    # Deploy the builds and runs
+    # Create deploy objects.
+    flow.create_deploy_objects()
+    results = flow.deploy_objects()
+
+    # Generate results.
+    flow.gen_results(results)
+
     # Exit with non-zero status if there were errors or failures.
-    if cfg.has_errors():
+    if flow.has_errors():
         log.error("Errors were encountered in this run.")
         sys.exit(1)
-
-
-if __name__ == "__main__":
-    main()

src/dvsim/config/__init__.py

@@ -0,0 +1,5 @@
+# Copyright lowRISC contributors (OpenTitan project).
+# Licensed under the Apache License, Version 2.0, see LICENSE for details.
+# SPDX-License-Identifier: Apache-2.0
+
+"""Configuration for DVSim."""

src/dvsim/config/errors.py

@@ -0,0 +1,26 @@
+# Copyright lowRISC contributors (OpenTitan project).
+# Licensed under the Apache License, Version 2.0, see LICENSE for details.
+# SPDX-License-Identifier: Apache-2.0
+
+"""Custom Exception classes for config handling."""
+
+
+class ConflictingConfigValueError(Exception):
+    """Config values can not be merged as their values conflict."""
+
+    def __init__(self, *, key: str, value: object, new_value: object) -> None:
+        """Initialise the Error object.
+
+        Args:
+            key: the mapping key
+            value: initial value
+            new_value: the value in the other config
+
+        """
+        self.key = key
+        self.value = value
+        self.new_value = new_value
+
+        super().__init__(
+            f"Values {value!r} and {new_value!r} are in conflict as they cannot be merged",
+        )