Skip to content

Commit

Permalink
Updated outputMethod values
Browse files Browse the repository at this point in the history
  • Loading branch information
Iacopo Colonnelli authored and GlassOfWhiskey committed Sep 18, 2024
1 parent 4bb5329 commit 5566827
Show file tree
Hide file tree
Showing 27 changed files with 57 additions and 47 deletions.
4 changes: 2 additions & 2 deletions cwltool/checker.py
Original file line number Diff line number Diff line change
Expand Up @@ -517,10 +517,10 @@ def is_conditional_step(param_to_step: Dict[str, CWLObjectType], parm_id: str) -


def is_all_output_method_loop_step(param_to_step: Dict[str, CWLObjectType], parm_id: str) -> bool:
"""Check if a step contains a `loop` directive with `all` outputMethod."""
"""Check if a step contains a `loop` directive with `all_iterations` outputMethod."""
source_step: Optional[MutableMapping[str, Any]] = param_to_step.get(parm_id)
if source_step is not None:
if source_step.get("loop") is not None and source_step.get("outputMethod") == "all":
if source_step.get("loop") is not None and source_step.get("outputMethod") == "all_iterations":
return True
return False

Expand Down
25 changes: 13 additions & 12 deletions cwltool/schemas/v1.3.0-dev1/Workflow.yml
Original file line number Diff line number Diff line change
Expand Up @@ -497,8 +497,8 @@ $graph:
docParent: "#LoopWorkflowStep"
doc: The loop output method, as described in [workflow step loop](#LoopWorkflowStep).
symbols:
- last
- all
- last_iteration
- all_iterations


- name: AbstractWorkflowStep
Expand Down Expand Up @@ -705,14 +705,14 @@ $graph:
The `outputMethod` field describes how to deal with loop outputs after
termination:
* **last** specifies that only the last computed element for each output
parameter should be propagated to the subsequent steps. This is the
default value.
* **last_iteration** specifies that only the last computed element for
each output parameter should be propagated to the subsequent steps.
This is the default value.
* **all** specifies that an array with all output values computed at the
end of each loop iteration should be propagated to the subsequent steps.
Elements in the array must be ordered according to the loop iterations
that produced them.
* **all_iterations** specifies that an array with all output values
computed at the end of each loop iteration should be propagated to
the subsequent steps. Elements in the array must be ordered according
to the loop iterations that produced them.
Iterative execution in CWL is an optional feature and is not required
to be implemented by all consumers of CWL documents. An implementation that
Expand All @@ -734,9 +734,9 @@ $graph:
mapPredicate: outputSource
- name: outputMethod
doc: |
If not specified, the default method is "last".
If not specified, the default method is "last_iteration".
type: LoopOutputMethod?
default: last
default: last_iteration
jsonldPredicate:
"_id": "cwl:outputMethod"
"_type": "@vocab"
Expand All @@ -748,7 +748,8 @@ $graph:
Only run the next iteration when the expression evaluates to `true`.
If the first iteration evaluates to `false` the step is skipped.
A skipped step produces a `null` on each output if the `outputMethod`
is set to `last`, and an empty array if the `outputMethod` is set to `all`.
is set to `last_iteration`, and an empty array if the `outputMethod`
is set to `all_iterations`.
- name: Workflow
Expand Down
11 changes: 10 additions & 1 deletion cwltool/update.py
Original file line number Diff line number Diff line change
Expand Up @@ -51,7 +51,16 @@ def rewrite_loop_requirements(t: CWLObjectType) -> None:
el["outputSource"] = source
s["loop"] = r["loop"]
if "outputMethod" in r:
s["outputMethod"] = r["outputMethod"]
if r["outputMethod"] == "all":
s["outputMethod"] = "all_iterations"
elif r["outputMethod"] == "last":
s["outputMethod"] = "last_iterations"
else:
raise SourceLine(
r, raise_type=ValidationException
).makeError( # pragma: no cover
f"Invalid value {r["outputMethod"]} for `outputMethod`."
)
cast(
MutableSequence[CWLObjectType],
s["requirements"],
Expand Down
10 changes: 5 additions & 5 deletions cwltool/workflow_job.py
Original file line number Diff line number Diff line change
Expand Up @@ -744,7 +744,7 @@ def valueFromFunc(k: str, v: Optional[CWLOutputType]) -> Optional[CWLOutputType]
_logger.info("[%s] will be skipped", step.name)
if (
step.tool.get("loop") is not None
and step.tool.get("outputMethod", "last") == "all"
and step.tool.get("outputMethod", "last_iteration") == "all_iterations"
):
callback({k["id"]: [] for k in outputparms}, "skipped")
else:
Expand Down Expand Up @@ -874,7 +874,7 @@ def _set_empty_output(self, outputMethod: str) -> None:
for i in self.step.tool["outputs"]:
if "id" in i:
iid = cast(str, i["id"])
if outputMethod == "all":
if outputMethod == "all_iterations":
self.output_buffer[iid] = cast(MutableSequence[Optional[CWLOutputType]], [])
else:
self.output_buffer[iid] = None
Expand All @@ -887,7 +887,7 @@ def job(
) -> JobsGeneratorType:
"""Generate a WorkflowJobStep job until the `when` condition evaluates to False."""
self.joborder = joborder
outputMethod = self.step.tool.get("outputMethod", "last")
outputMethod = self.step.tool.get("outputMethod", "last_iteration")

callback = functools.partial(
self.loop_callback,
Expand Down Expand Up @@ -953,14 +953,14 @@ def loop_callback(
self.iteration += 1
try:
loop = cast(MutableSequence[CWLObjectType], self.step.tool.get("loop", []))
outputMethod = self.step.tool.get("outputMethod", "last")
outputMethod = self.step.tool.get("outputMethod", "last_iteration")
state: Dict[str, Optional[WorkflowStateItem]] = {}
for i in self.step.tool["outputs"]:
if "id" in i:
iid = cast(str, i["id"])
if iid in jobout:
state[iid] = WorkflowStateItem(i, jobout[iid], processStatus)
if outputMethod == "all":
if outputMethod == "all_iterations":
if iid not in self.output_buffer:
self.output_buffer[iid] = cast(
MutableSequence[Optional[CWLOutputType]], []
Expand Down
2 changes: 1 addition & 1 deletion tests/loop/all-output-loop-no-iteration.cwl
Original file line number Diff line number Diff line change
Expand Up @@ -25,4 +25,4 @@ steps:
when: $(inputs.i1 < 1)
loop:
i1: o1
outputMethod: all
outputMethod: all_iterations
2 changes: 1 addition & 1 deletion tests/loop/all-output-loop.cwl
Original file line number Diff line number Diff line change
Expand Up @@ -25,4 +25,4 @@ steps:
when: $(inputs.i1 < 10)
loop:
i1: o1
outputMethod: all
outputMethod: all_iterations
2 changes: 1 addition & 1 deletion tests/loop/default-value-loop.cwl
Original file line number Diff line number Diff line change
Expand Up @@ -44,4 +44,4 @@ steps:
i1:
outputSource: o1
default: 5
outputMethod: all
outputMethod: all_iterations
2 changes: 1 addition & 1 deletion tests/loop/invalid-loop-scatter.cwl
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,7 @@ steps:
when: $(inputs.i1 < 10)
loop:
i1: o1
outputMethod: last
outputMethod: last_iteration
in:
i1: i1
i2: i2
Expand Down
2 changes: 1 addition & 1 deletion tests/loop/invalid-loop-when-exception.cwl
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,7 @@ steps:
}
loop:
i1: o1
outputMethod: last
outputMethod: last_iteration
in:
i1: i1
i2: i2
Expand Down
2 changes: 1 addition & 1 deletion tests/loop/invalid-loop-when-exception2.cwl
Original file line number Diff line number Diff line change
Expand Up @@ -33,7 +33,7 @@ steps:
}
loop:
i1: o1
outputMethod: last
outputMethod: last_iteration
in:
i1: i1
i2: i2
Expand Down
2 changes: 1 addition & 1 deletion tests/loop/invalid-multi-source-loop-no-requirement.cwl
Original file line number Diff line number Diff line change
Expand Up @@ -61,4 +61,4 @@ steps:
i1:
loopSource: [osmall, obig]
pickValue: the_only_non_null
outputMethod: all
outputMethod: all_iterations
2 changes: 1 addition & 1 deletion tests/loop/invalid-no-loopWhen.cwl
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,7 @@ steps:
${return {'o1': inputs.i1 + inputs.i2};}
loop:
i1: o1
outputMethod: last
outputMethod: last_iteration
in:
i1: i1
i2: i2
Expand Down
2 changes: 1 addition & 1 deletion tests/loop/invalid-non-boolean-loopWhen.cwl
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,7 @@ steps:
when: $(inputs.i1)
loop:
i1: o1
outputMethod: last
outputMethod: last_iteration
in:
i1: i1
i2: i2
Expand Down
2 changes: 1 addition & 1 deletion tests/loop/invalid-non-boolean-loopWhen2.cwl
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,7 @@ steps:
when: '$(inputs.i1 == 1 ? true : "I am a string")'
loop:
i1: o1
outputMethod: last
outputMethod: last_iteration
in:
i1: i1
i2: i2
Expand Down
2 changes: 1 addition & 1 deletion tests/loop/invalid-value-from-loop-no-requirement.cwl
Original file line number Diff line number Diff line change
Expand Up @@ -29,4 +29,4 @@ steps:
loop:
i1:
valueFrom: $(inputs.i1 + 1)
outputMethod: last
outputMethod: last_iteration
4 changes: 2 additions & 2 deletions tests/loop/loop-inside-loop-all.cwl
Original file line number Diff line number Diff line change
Expand Up @@ -46,7 +46,7 @@ steps:
when: $(inputs.i1 <= inputs.i2)
loop:
i1: o1
outputMethod: all
outputMethod: all_iterations
in:
i1: i1
i2: i2
Expand All @@ -55,4 +55,4 @@ steps:
loop:
i2:
valueFrom: $(inputs.i2 + 1)
outputMethod: all
outputMethod: all_iterations
4 changes: 2 additions & 2 deletions tests/loop/loop-inside-loop.cwl
Original file line number Diff line number Diff line change
Expand Up @@ -42,7 +42,7 @@ steps:
when: $(inputs.i1 <= inputs.i2)
loop:
i1: o1
outputMethod: last
outputMethod: last_iteration
in:
i1: i1
i2: i2
Expand All @@ -51,4 +51,4 @@ steps:
loop:
i2:
valueFrom: $(inputs.i2 + 1)
outputMethod: all
outputMethod: all_iterations
2 changes: 1 addition & 1 deletion tests/loop/loop-inside-scatter.cwl
Original file line number Diff line number Diff line change
Expand Up @@ -41,7 +41,7 @@ steps:
when: $(inputs.i1 < 10)
loop:
i1: o1
outputMethod: last
outputMethod: last_iteration
in:
i1: i1
i2: i2
Expand Down
2 changes: 1 addition & 1 deletion tests/loop/multi-source-loop.cwl
Original file line number Diff line number Diff line change
Expand Up @@ -62,4 +62,4 @@ steps:
i1:
outputSource: [osmall, obig]
pickValue: the_only_non_null
outputMethod: all
outputMethod: all_iterations
2 changes: 1 addition & 1 deletion tests/loop/opt-var-loop.cwl
Original file line number Diff line number Diff line change
Expand Up @@ -27,4 +27,4 @@ steps:
when: $(inputs.i1 < 10)
loop:
i1: o1
outputMethod: last
outputMethod: last_iteration
2 changes: 1 addition & 1 deletion tests/loop/scatter-inside-loop.cwl
Original file line number Diff line number Diff line change
Expand Up @@ -46,4 +46,4 @@ steps:
when: $(inputs.i1[0] < 10)
loop:
i1: o1
outputMethod: last
outputMethod: last_iteration
2 changes: 1 addition & 1 deletion tests/loop/single-var-loop-no-iteration.cwl
Original file line number Diff line number Diff line change
Expand Up @@ -26,4 +26,4 @@ steps:
when: $(inputs.i1 < 1)
loop:
i1: o1
outputMethod: last
outputMethod: last_iteration
2 changes: 1 addition & 1 deletion tests/loop/single-var-loop.cwl
Original file line number Diff line number Diff line change
Expand Up @@ -26,4 +26,4 @@ steps:
when: $(inputs.i1 < 10)
loop:
i1: o1
outputMethod: last
outputMethod: last_iteration
2 changes: 1 addition & 1 deletion tests/loop/two-vars-loop-2.cwl
Original file line number Diff line number Diff line change
Expand Up @@ -28,4 +28,4 @@ steps:
when: $(inputs.i1 < 10)
loop:
i1: o1
outputMethod: last
outputMethod: last_iteration
2 changes: 1 addition & 1 deletion tests/loop/two-vars-loop.cwl
Original file line number Diff line number Diff line change
Expand Up @@ -30,4 +30,4 @@ steps:
loop:
i1: o1
i2: o2
outputMethod: last
outputMethod: last_iteration
2 changes: 1 addition & 1 deletion tests/loop/value-from-loop.cwl
Original file line number Diff line number Diff line change
Expand Up @@ -30,4 +30,4 @@ steps:
loop:
i1:
valueFrom: $(inputs.i1 + 1)
outputMethod: last
outputMethod: last_iteration
6 changes: 3 additions & 3 deletions tests/test_loop.py
Original file line number Diff line number Diff line change
Expand Up @@ -143,7 +143,7 @@ def test_loop_two_variables_single_backpropagation() -> None:


def test_loop_with_all_output_method() -> None:
"""Test a loop case with outputMethod set to all."""
"""Test a loop case with outputMethod set to all_iterations."""
stream = StringIO()
params = [
"--enable-dev",
Expand All @@ -156,7 +156,7 @@ def test_loop_with_all_output_method() -> None:


def test_loop_with_all_output_method_no_iteration() -> None:
"""Test a loop case with outputMethod set to all and a false 'when' condition."""
"""Test a loop case with outputMethod set to all_iterations and a false 'when' condition."""
stream = StringIO()
params = [
"--enable-dev",
Expand Down Expand Up @@ -244,7 +244,7 @@ def test_nested_loops() -> None:


def test_nested_loops_all() -> None:
"""Test a workflow with two nested loops, both with outputMethod set to all."""
"""Test a workflow with two nested loops, both with outputMethod set to all_iterations."""
stream = StringIO()
params = [
"--enable-dev",
Expand Down

0 comments on commit 5566827

Please sign in to comment.