adds slurm runner (#799)

- adds slurm runner, with an initial implementation for docker
- refactors runner_host.DockerRunner so children classes can reuse most of the code
- introduces a PopperConfig class that deals with configuration of Popper
- adds tests for slurm runner on docker

ci/test/engine-conf

@@ -17,62 +17,93 @@ action "run" {
 }
 EOF
 
-# config file called settings.py in the project root.
-cat <<EOF > settings.py
-ENGINE = {
-    "name": "dont use this name",
-    "image": "abc/xyz",
-    "hostname": "xYz.local"
-}
+# config file called settings.yml in the project root.
+cat <<EOF > settings.yml
+engine:
+  name: docker
+  options:
+    image: abc/xyz
+    hostname: xYz.local
 EOF
 
-popper run --wfile main.workflow --conf settings.py > output
+popper run --wfile main.workflow --conf settings.yml > output
 grep -Fxq "xYz.local" output
 
 popper run --wfile main.workflow > output
 grep -vFxq "xYz.local" output
 
 # config file with different name in the project root.
-cat <<EOF > myconf.py
-ENGINE = {
-    "name": "dont use this name",
-    "image": "abc/xyz",
-    "hostname": "xYz.local"
-}
+cat <<EOF > myconf.yml
+engine:
+  name: docker
+  options:
+    image: abc/xyz
+    hostname: xYz.local
 EOF
 
-popper run --wfile main.workflow --conf myconf.py > output
+popper run --wfile main.workflow --conf myconf.yml > output
 grep -Fxq "xYz.local" output
 
 popper run --wfile main.workflow > output
 grep -vFxq "xYz.local" output
 
 # config file in different directory than project root.
 mkdir -p /tmp/myengineconf/
-cat <<EOF > /tmp/myengineconf/mysettings.py
-ENGINE = {
-    "name": "dont use this name",
-    "image": "abc/xyz",
-    "hostname": "xYz.local"
-}
+cat <<EOF > /tmp/myengineconf/mysettings.yml
+engine:
+  name: docker
+  options:
+    image: abc/xyz
+    hostname: xYz.local
 EOF
 
-popper run --wfile main.workflow --conf /tmp/myengineconf/mysettings.py > output
+popper run --wfile main.workflow --conf /tmp/myengineconf/mysettings.yml > output
 grep -Fxq "xYz.local" output
 
 popper run --wfile main.workflow > output
 grep -vFxq "xYz.local" output
 
 cat <<EOF > settings
-ENGINE = {
-    "name": "dont use this name",
-    "image": "abc/xyz",
-    "hostname": "xYz.local"
-}
+engine:
+  name: docker
+  options:
+    image: abc/xyz
+    hostname: xYz.local
 EOF
 
 popper run --wfile main.workflow --conf settings && exit 1
 
 popper run --wfile main.workflow --conf conf && exit 1
 
+# fail since engine config has no name attribute
+cat <<EOF > settings.yml
+engine:
+  options:
+    image: abc/xyz
+    hostname: xYz.local
+EOF
+
+popper run --wfile main.workflow --conf settings.yml && exit 1
+
+# fail since resource manager config has no name attribute
+cat <<EOF > settings.yml
+engine:
+  name: docker
+  options:
+    image: abc/xyz
+    hostname: xYz.local
+
+resource_manager:
+  options:
+    foo: bar
+EOF
+
+popper run --wfile main.workflow --conf settings.yml && exit 1
+
+# fail since config file is empty.
+cat <<EOF > settings.yml
+EOF
+
+popper run --wfile main.workflow --conf settings.yml && exit 1
+
 echo "Test ENGINE-CONF passed."

cli/popper/commands/cmd_run.py

@@ -50,11 +50,16 @@
     is_flag=True,
 )
 @click.option(
+    '-e',
     '--engine',
-    help='Specify runtime for executing the workflow [default: docker].',
-    type=click.Choice(['docker', 'singularity', 'vagrant']),
-    required=False,
-    default='docker'
+    help='Specify runtime for executing the workflow.',
+    type=click.Choice(['docker', 'singularity', 'vagrant'])
+)
+@click.option(
+    '-r',
+    '--resource-manager',
+    help='Specify resource manager for executing the workflow.',
+    type=click.Choice(['host', 'slurm'])
 )
 @click.option(
     '--skip',
@@ -111,8 +116,8 @@
 )
 @pass_context
 def cli(ctx, step, wfile, debug, dry_run, log_file, quiet, reuse,
-        engine, skip, skip_pull, skip_clone, substitution, allow_loose,
-        with_dependencies, workspace, conf):
+        engine, resource_manager, skip, skip_pull, skip_clone,
+        substitution, allow_loose, with_dependencies, workspace, conf):
     """Runs a Popper workflow. Only execute STEP if given."""
     # set the logging levels.
     level = 'STEP_INFO'
@@ -142,7 +147,13 @@ def cli(ctx, step, wfile, debug, dry_run, log_file, quiet, reuse,
                       include_step_dependencies=with_dependencies)
 
     # instantiate the runner
-    runner = WorkflowRunner(config_file=conf, dry_run=dry_run, reuse=reuse,
-                            skip_pull=skip_pull, skip_clone=skip_clone,
-                            workspace_dir=workspace)
+    runner = WorkflowRunner(
+        engine,
+        resource_manager,
+        config_file=conf,
+        dry_run=dry_run,
+        reuse=reuse,
+        skip_pull=skip_pull,
+        skip_clone=skip_clone,
+        workspace_dir=workspace)
     runner.run(wf)

cli/popper/config.py

@@ -0,0 +1,59 @@
+import os
+
+from hashlib import shake_256
+
+from popper.cli import log as log
+
+import popper.scm as scm
+import popper.utils as pu
+
+
+class PopperConfig(object):
+    def __init__(self, **kwargs):
+        self.repo = scm.new_repo()
+        self.workspace_dir = os.path.realpath(kwargs['workspace_dir'])
+        self.wid = shake_256(self.workspace_dir.encode('utf-8')).hexdigest(4)
+        self.workspace_sha = scm.get_sha(self.repo)
+        self.config_file = kwargs['config_file']
+        self.dry_run = kwargs['dry_run']
+        self.skip_clone = kwargs['skip_clone']
+        self.skip_pull = kwargs['skip_pull']
+        self.quiet = kwargs['quiet']
+        self.reuse = kwargs['reuse']
+        self.engine_name = kwargs.get('engine', None)
+        self.resman_name = kwargs.get('resource_manager', None)
+        self.engine_options = kwargs['engine_options']
+        self.resman_options = kwargs['resman_options']
+        self.config_from_file = pu.load_config_file(self.config_file)
+
+    def parse(self):
+        self.validate()
+        self.normalize()
+
+    def validate(self):
+        if self.config_from_file.get('engine', None):
+            if not self.config_from_file['engine'].get('name', None):
+                log.fail(
+                    'engine config must have the name property.')
+
+        if self.config_from_file.get('resource_manager', None):
+            if not self.config_from_file['resource_manager'].get('name', None):
+                log.fail(
+                    'resource_manager config must have the name property.')
+
+    def normalize(self):
+        if not self.engine_name:
+            if self.config_from_file.get('engine', None):
+                self.engine_name = self.config_from_file['engine']['name']
+                self.engine_options = self.config_from_file['engine'].get(
+                    'options', dict())
+            else:
+                self.engine_name = 'docker'
+
+        if not self.resman_name:
+            if self.config_from_file.get('resource_manager', None):
+                self.resman_name = self.config_from_file['resource_manager']['name']
+                self.resman_options = self.config_from_file['resource_manager'].get(
+                    'options', dict())
+            else:
+                self.resman_name = 'host'

cli/popper/log.py

@@ -34,12 +34,12 @@ class PopperFormatter(logging.Formatter):
     BOLD_RED = ''
 
     log_format = {
-        'DEBUG':       f'{BOLD_CYAN}%(levelname)s: %(msg)s {RESET}',
+        'DEBUG': f'{BOLD_CYAN}%(levelname)s: %(msg)s {RESET}',
         'STEP_INFO': '%(msg)s',
-        'INFO':        '%(msg)s',
-        'WARNING':     f'{BOLD_YELLOW}%(levelname)s: %(msg)s{RESET}',
-        'ERROR':       f'{BOLD_RED}%(levelname)s: %(msg)s{RESET}',
-        'CRITICAL':    f'{BOLD_RED}%(levelname)s: %(msg)s{RESET}'
+        'INFO': '%(msg)s',
+        'WARNING': f'{BOLD_YELLOW}%(levelname)s: %(msg)s{RESET}',
+        'ERROR': f'{BOLD_RED}%(levelname)s: %(msg)s{RESET}',
+        'CRITICAL': f'{BOLD_RED}%(levelname)s: %(msg)s{RESET}'
     }
 
     log_format_no_colors = {
@@ -168,6 +168,7 @@ def warning(self, msg='', *args, **kwargs):
 
 class LevelFilter(logging.Filter):
     """Filters the level that are to be accepted and rejected."""
+
     def __init__(self, passlevels, reject):
         self.passlevels = passlevels
         self.reject = reject

cli/popper/parser.py

@@ -9,10 +9,19 @@
 from builtins import str, dict
 from popper.cli import log as log
 
+import popper.scm as scm
 import popper.utils as pu
 
 
-VALID_STEP_ATTRS = ["uses", "args", "needs", "runs", "secrets", "env", "name", "next"]
+VALID_STEP_ATTRS = [
+    "uses",
+    "args",
+    "needs",
+    "runs",
+    "secrets",
+    "env",
+    "name",
+    "next"]
 
 
 class Workflow(object):
@@ -623,7 +632,7 @@ def complete_graph(self):
                         if not self.visited.get(tuple(next_set), None):
                             step['next'] = next_set
                             for nsa in next_set:
-                                self.steps[nsa]['needs'] = id
+                                self.steps[nsa]['needs'] = [id]
                             self.visited[tuple(curr_set)] = True
 
         # Finally, generate the root.

cli/popper/runner.py

@@ -3,12 +3,10 @@
 import os
 import sys
 
-from dotmap import DotMap
-from hashlib import shake_256
-
 import popper.scm as scm
 import popper.utils as pu
 
+from popper.config import PopperConfig
 from popper.cli import log
 
 
@@ -18,48 +16,33 @@ class WorkflowRunner(object):
     # class variable that holds references to runner singletons
     runners = {}
 
-    def __init__(self, config_file=None, workspace_dir=os.getcwd(),
-                 reuse=False, engine_name='docker', dry_run=False, quiet=False,
-                 skip_pull=False, skip_clone=False):
-
-        # save all args in a member dictionary
-        self.config = DotMap(locals())
-        self.config.pop('self')
-        self.config.workspace_dir = os.path.realpath(workspace_dir)
-        self.config.wid = shake_256(workspace_dir.encode('utf-8')).hexdigest(4)
-
-        # cretate a repo object for the project
-        self.repo = scm.new_repo()
-        self.config.workspace_sha = scm.get_sha(self.repo)
-
-        if config_file:
-            # read options from config file
-            config_from_file = pu.load_config_file(config_file)
+    def __init__(self, engine, resource_manager, config_file=None,
+                 workspace_dir=os.getcwd(), reuse=False, dry_run=False,
+                 quiet=False, skip_pull=False, skip_clone=False,
+                 engine_options=dict(), resman_options=dict()):
 
-            if hasattr(config_from_file, 'ENGINE'):
-                self.config.engine_options = config_from_file.ENGINE
-            if hasattr(config_from_file, 'RESOURCE_MANAGER'):
-                self.config.resman_options = config_from_file.RESOURCE_MANAGER
-
-        if not self.config.resman:
-            self.config.resman = 'host'
+        # create a config object from the given arguments
+        kwargs = locals()
+        kwargs.pop('self')
+        self.config = PopperConfig(**kwargs)
+        self.config.parse()
 
         # dynamically load resource manager
-        resman_mod_name = f'popper.runner_{self.config.resman}'
+        resman_mod_name = f'popper.runner_{self.config.resman_name}'
         resman_spec = importlib.util.find_spec(resman_mod_name)
         if not resman_spec:
-            raise ValueError(f'Invalid resource manager: {self.config.resman}')
+            raise ValueError(
+                f'Invalid resource manager: {self.config.resman_name}')
         self.resman_mod = importlib.import_module(resman_mod_name)
 
         log.debug(f'WorkflowRunner config:\n{pu.prettystr(self.config)}')
 
-
     def __enter__(self):
         return self
 
     def __exit__(self, exc_type, exc, traceback):
         """calls __exit__ on all instantiated step runners"""
-        self.repo.close()
+        self.config.repo.close()
         for _, r in WorkflowRunner.runners.items():
             r.__exit__(exc_type, exc, traceback)
         WorkflowRunner.runners = {}
@@ -82,7 +65,7 @@ def signal_handler(sig, frame):
         sys.exit(0)
 
     @staticmethod
-    def process_secrets(wf, config=DotMap({})):
+    def process_secrets(wf, config):
         """Checks whether the secrets defined for a step are available in the
         execution environment. When the environment variable `CI` is set to
         `true` and no environment variable is defined for a secret, the
@@ -202,6 +185,7 @@ def step_runner(self, engine_name, step):
 
 class StepRunner(object):
     """Base class for step runners, assumed to be singletons."""
+
     def __init__(self, config):
         self.config = config