bugfix/worker-all-steps

CHANGELOG.md

@@ -4,6 +4,13 @@ All notable changes to Merlin will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## [Unreleased]
+### Fixed
+- A bug where assigning a worker all steps also assigned steps to the default worker
+
+### Added
+- Tests to make sure the default worker is being assigned properly
+
 ## [1.10.0]
 ### Fixed
 - Pip wheel wasn't including .sh files for merlin examples

merlin/spec/specification.py

@@ -385,8 +385,11 @@ def process_spec_defaults(self):
                 MerlinSpec.fill_missing_defaults(worker_settings, defaults.WORKER)
                 worker_steps.extend(worker_settings["steps"])
 
-            # Figure out which steps still need workers
-            steps_that_need_workers = list(set(all_workflow_steps) - set(worker_steps))
+            if "all" in worker_steps:
+                steps_that_need_workers = []
+            else:
+                # Figure out which steps still need workers
+                steps_that_need_workers = list(set(all_workflow_steps) - set(worker_steps))
 
             # If there are still steps remaining that haven't been assigned a worker yet,
             # assign the remaining steps to the default worker. If all the steps still need workers

tests/integration/test_definitions.py

@@ -283,6 +283,16 @@ def define_tests():  # pylint: disable=R0914,R0915
             "conditions": [HasReturnCode(), HasRegex("custom_verify_queue")],
             "run type": "local",
         },
+        "default_worker assigned": {
+            "cmds": f"{workers} {test_specs}/default_worker_test.yaml --echo",
+            "conditions": [HasReturnCode(), HasRegex(r"default_worker.*-Q '\[merlin\]_step_4_queue'")],
+            "run type": "local",
+        },
+        "no default_worker assigned": {
+            "cmds": f"{workers} {test_specs}/no_default_worker_test.yaml --echo",
+            "conditions": [HasReturnCode(), HasRegex(r"default_worker", negate=True)],
+            "run type": "local",
+        },
     }
     wf_format_tests = {
         "local minimum_format": {

tests/integration/test_specs/default_worker_test.yaml

@@ -0,0 +1,56 @@
+description:
+    name: multiple_workers
+    description: a very simple merlin workflow with multiple workers
+
+global.parameters:
+    GREET:
+        values : ["hello","hola"]
+        label  : GREET.%%
+    WORLD:
+        values : ["world","mundo"]
+        label  : WORLD.%%
+
+study:
+    - name: step_1
+      description: say hello
+      run:
+          cmd: |
+              echo "$(GREET), $(WORLD)!"
+          task_queue: hello_queue
+
+    - name: step_2
+      description: step 2
+      run:
+        cmd: |
+            echo "step_2"
+        depends: [step_1_*]
+        task_queue: echo_queue
+
+    - name: step_3
+      description: stop workers
+      run:
+        cmd: |
+            echo "stop workers"
+        depends: [step_2]
+        task_queue: other_queue
+
+    - name: step_4
+      description: another step
+      run:
+        cmd: |
+            echo "another step"
+        depends: [step_3]
+        task_queue: step_4_queue
+
+merlin:
+    resources:
+        workers:
+            step_1_merlin_test_worker:
+                args: -l INFO
+                steps: [step_1]
+            step_2_merlin_test_worker:
+                args: -l INFO
+                steps: [step_2]
+            other_merlin_test_worker:
+                args: -l INFO
+                steps: [step_3]

tests/integration/test_specs/no_default_worker_test.yaml

@@ -0,0 +1,56 @@
+description:
+    name: multiple_workers
+    description: a very simple merlin workflow with multiple workers
+
+global.parameters:
+    GREET:
+        values : ["hello","hola"]
+        label  : GREET.%%
+    WORLD:
+        values : ["world","mundo"]
+        label  : WORLD.%%
+
+study:
+    - name: step_1
+      description: say hello
+      run:
+          cmd: |
+              echo "$(GREET), $(WORLD)!"
+          task_queue: hello_queue
+
+    - name: step_2
+      description: step 2
+      run:
+        cmd: |
+            echo "step_2"
+        depends: [step_1_*]
+        task_queue: echo_queue
+
+    - name: step_3
+      description: stop workers
+      run:
+        cmd: |
+            echo "stop workers"
+        depends: [step_2]
+        task_queue: other_queue
+
+    - name: step_4
+      description: another step
+      run:
+        cmd: |
+            echo "another step"
+        depends: [step_3]
+        task_queue: step_4_queue
+
+merlin:
+    resources:
+        workers:
+            step_1_merlin_test_worker:
+                args: -l INFO
+                steps: [step_1]
+            step_2_merlin_test_worker:
+                args: -l INFO
+                steps: [step_2]
+            other_merlin_test_worker:
+                args: -l INFO
+                steps: [all]