diff --git a/CHANGELOG.md b/CHANGELOG.md index 42811b601..e5ba817ef 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,13 @@ All notable changes to Merlin will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [Unreleased] +### Fixed +- A bug where assigning a worker all steps also assigned steps to the default worker + +### Added +- Tests to make sure the default worker is being assigned properly + ## [1.10.0] ### Fixed - Pip wheel wasn't including .sh files for merlin examples diff --git a/merlin/spec/specification.py b/merlin/spec/specification.py index 6fb524f5b..9f3a6c16f 100644 --- a/merlin/spec/specification.py +++ b/merlin/spec/specification.py @@ -385,8 +385,11 @@ def process_spec_defaults(self): MerlinSpec.fill_missing_defaults(worker_settings, defaults.WORKER) worker_steps.extend(worker_settings["steps"]) - # Figure out which steps still need workers - steps_that_need_workers = list(set(all_workflow_steps) - set(worker_steps)) + if "all" in worker_steps: + steps_that_need_workers = [] + else: + # Figure out which steps still need workers + steps_that_need_workers = list(set(all_workflow_steps) - set(worker_steps)) # If there are still steps remaining that haven't been assigned a worker yet, # assign the remaining steps to the default worker. If all the steps still need workers diff --git a/tests/integration/test_definitions.py b/tests/integration/test_definitions.py index 6fd8e3ce9..2b990ea40 100644 --- a/tests/integration/test_definitions.py +++ b/tests/integration/test_definitions.py @@ -283,6 +283,16 @@ def define_tests(): # pylint: disable=R0914,R0915 "conditions": [HasReturnCode(), HasRegex("custom_verify_queue")], "run type": "local", }, + "default_worker assigned": { + "cmds": f"{workers} {test_specs}/default_worker_test.yaml --echo", + "conditions": [HasReturnCode(), HasRegex(r"default_worker.*-Q '\[merlin\]_step_4_queue'")], + "run type": "local", + }, + "no default_worker assigned": { + "cmds": f"{workers} {test_specs}/no_default_worker_test.yaml --echo", + "conditions": [HasReturnCode(), HasRegex(r"default_worker", negate=True)], + "run type": "local", + }, } wf_format_tests = { "local minimum_format": { diff --git a/tests/integration/test_specs/default_worker_test.yaml b/tests/integration/test_specs/default_worker_test.yaml new file mode 100644 index 000000000..974921593 --- /dev/null +++ b/tests/integration/test_specs/default_worker_test.yaml @@ -0,0 +1,56 @@ +description: + name: multiple_workers + description: a very simple merlin workflow with multiple workers + +global.parameters: + GREET: + values : ["hello","hola"] + label : GREET.%% + WORLD: + values : ["world","mundo"] + label : WORLD.%% + +study: + - name: step_1 + description: say hello + run: + cmd: | + echo "$(GREET), $(WORLD)!" + task_queue: hello_queue + + - name: step_2 + description: step 2 + run: + cmd: | + echo "step_2" + depends: [step_1_*] + task_queue: echo_queue + + - name: step_3 + description: stop workers + run: + cmd: | + echo "stop workers" + depends: [step_2] + task_queue: other_queue + + - name: step_4 + description: another step + run: + cmd: | + echo "another step" + depends: [step_3] + task_queue: step_4_queue + +merlin: + resources: + workers: + step_1_merlin_test_worker: + args: -l INFO + steps: [step_1] + step_2_merlin_test_worker: + args: -l INFO + steps: [step_2] + other_merlin_test_worker: + args: -l INFO + steps: [step_3] diff --git a/tests/integration/test_specs/no_default_worker_test.yaml b/tests/integration/test_specs/no_default_worker_test.yaml new file mode 100644 index 000000000..b2f15d2ee --- /dev/null +++ b/tests/integration/test_specs/no_default_worker_test.yaml @@ -0,0 +1,56 @@ +description: + name: multiple_workers + description: a very simple merlin workflow with multiple workers + +global.parameters: + GREET: + values : ["hello","hola"] + label : GREET.%% + WORLD: + values : ["world","mundo"] + label : WORLD.%% + +study: + - name: step_1 + description: say hello + run: + cmd: | + echo "$(GREET), $(WORLD)!" + task_queue: hello_queue + + - name: step_2 + description: step 2 + run: + cmd: | + echo "step_2" + depends: [step_1_*] + task_queue: echo_queue + + - name: step_3 + description: stop workers + run: + cmd: | + echo "stop workers" + depends: [step_2] + task_queue: other_queue + + - name: step_4 + description: another step + run: + cmd: | + echo "another step" + depends: [step_3] + task_queue: step_4_queue + +merlin: + resources: + workers: + step_1_merlin_test_worker: + args: -l INFO + steps: [step_1] + step_2_merlin_test_worker: + args: -l INFO + steps: [step_2] + other_merlin_test_worker: + args: -l INFO + steps: [all]