diff --git a/pydra/design/base.py b/pydra/design/base.py index 22ab9ebed..0b237af33 100644 --- a/pydra/design/base.py +++ b/pydra/design/base.py @@ -62,6 +62,13 @@ def convert_default_value(value: ty.Any, self_: "Field") -> ty.Any: return TypeParser[self_.type](self_.type, label=self_.name)(value) +def allowed_values_converter(value: ty.Iterable[str] | None) -> list[str] | None: + """Ensure the allowed_values field is a list of strings or None""" + if value is None: + return None + return list(value) + + @attrs.define class Requirement: """Define a requirement for a task input field @@ -76,14 +83,19 @@ class Requirement: """ name: str - allowed_values: list[str] = attrs.field(factory=list, converter=list) + allowed_values: list[str] | None = attrs.field( + default=None, converter=allowed_values_converter + ) def satisfied(self, inputs: "TaskDef") -> bool: """Check if the requirement is satisfied by the inputs""" value = getattr(inputs, self.name) - if value is attrs.NOTHING: + field = {f.name: f for f in list_fields(inputs)}[self.name] + if value is attrs.NOTHING or field.type is bool and value is False: return False - return not self.allowed_values or value in self.allowed_values + if self.allowed_values is None: + return True + return value in self.allowed_values @classmethod def parse(cls, value: ty.Any) -> Self: @@ -350,8 +362,8 @@ def get_fields(klass, field_type, auto_attribs, helps) -> dict[str, Field]: if not issubclass(klass, spec_type): raise ValueError( - f"The canonical form of {spec_type.__module__.split('.')[-1]} task definitions, " - f"{klass}, must inherit from {spec_type}" + f"When using the canonical form for {spec_type.__module__.split('.')[-1]} " + f"tasks, {klass} must inherit from {spec_type}" ) inputs = get_fields(klass, arg_type, auto_attribs, input_helps) @@ -364,8 +376,8 @@ def get_fields(klass, field_type, auto_attribs, helps) -> dict[str, Field]: ) from None if not issubclass(outputs_klass, outputs_type): raise ValueError( - f"The canonical form of {spec_type.__module__.split('.')[-1]} task definitions, " - f"{klass}, must inherit from {spec_type}" + f"When using the canonical form for {outputs_type.__module__.split('.')[-1]} " + f"task outputs {outputs_klass}, you must inherit from {outputs_type}" ) output_helps, _ = parse_doc_string(outputs_klass.__doc__) @@ -416,10 +428,12 @@ def make_task_def( spec_type._check_arg_refs(inputs, outputs) + # Check that the field attributes are valid after all fields have been set + # (especially the type) for inpt in inputs.values(): - set_none_default_if_optional(inpt) - for outpt in inputs.values(): - set_none_default_if_optional(outpt) + attrs.validate(inpt) + for outpt in outputs.values(): + attrs.validate(outpt) if name is None and klass is not None: name = klass.__name__ @@ -459,10 +473,10 @@ def make_task_def( if getattr(arg, "path_template", False): if is_optional(arg.type): field_type = Path | bool | None - # Will default to None and not be inserted into the command + attrs_kwargs = {"default": None} else: field_type = Path | bool - attrs_kwargs = {"default": True} + attrs_kwargs = {"default": True} # use the template by default elif is_optional(arg.type): field_type = Path | None else: @@ -988,12 +1002,10 @@ def check_explicit_fields_are_none(klass, inputs, outputs): def _get_attrs_kwargs(field: Field) -> dict[str, ty.Any]: kwargs = {} - if not hasattr(field, "default"): - kwargs["factory"] = nothing_factory - elif field.default is not NO_DEFAULT: + if field.default is not NO_DEFAULT: kwargs["default"] = field.default - elif is_optional(field.type): - kwargs["default"] = None + # elif is_optional(field.type): + # kwargs["default"] = None else: kwargs["factory"] = nothing_factory if field.hash_eq: @@ -1005,9 +1017,9 @@ def nothing_factory(): return attrs.NOTHING -def set_none_default_if_optional(field: Field) -> None: - if is_optional(field.type) and field.default is NO_DEFAULT: - field.default = None +# def set_none_default_if_optional(field: Field) -> None: +# if is_optional(field.type) and field.default is NO_DEFAULT: +# field.default = None white_space_re = re.compile(r"\s+") diff --git a/pydra/design/shell.py b/pydra/design/shell.py index 1f0e75543..ebdee524c 100644 --- a/pydra/design/shell.py +++ b/pydra/design/shell.py @@ -110,14 +110,16 @@ def _sep_default(self): @sep.validator def _validate_sep(self, _, sep): - if self.type is ty.Any: - return - if ty.get_origin(self.type) is MultiInputObj: + if self.type is MultiInputObj: + tp = ty.Any + elif ty.get_origin(self.type) is MultiInputObj: tp = ty.get_args(self.type)[0] else: tp = self.type if is_optional(tp): tp = optional_type(tp) + if tp is ty.Any: + return origin = ty.get_origin(tp) or tp if ( @@ -238,16 +240,21 @@ class outarg(arg, Out): @path_template.validator def _validate_path_template(self, attribute, value): - if value and self.default not in (NO_DEFAULT, True, None): - raise ValueError( - f"path_template ({value!r}) can only be provided when no default " - f"({self.default!r}) is provided" - ) - if value and not (is_fileset_or_union(self.type) or self.type is ty.Any): - raise ValueError( - f"path_template ({value!r}) can only be provided when type is a FileSet, " - f"or union thereof, not {self.type!r}" - ) + if value: + if self.default not in (NO_DEFAULT, True, None): + raise ValueError( + f"path_template ({value!r}) can only be provided when no default " + f"({self.default!r}) is provided" + ) + if not (is_fileset_or_union(self.type) or self.type is ty.Any): + raise ValueError( + f"path_template ({value!r}) can only be provided when type is a FileSet, " + f"or union thereof, not {self.type!r}" + ) + if self.argstr is None: + raise ValueError( + f"path_template ({value!r}) can only be provided when argstr is not None" + ) @keep_extension.validator def _validate_keep_extension(self, attribute, value): @@ -386,6 +393,7 @@ def make( input_helps=input_helps, output_helps=output_helps, ) + if name: class_name = name else: @@ -679,6 +687,10 @@ def from_type_str(type_str) -> type: if ext_type.ext is not None: path_template = name + ext_type.ext kwds["path_template"] = path_template + # Set the default value to None if the field is optional and no default is + # provided + if is_optional(type_) and "default" not in kwds: + kwds["default"] = None if option is None: add_arg(name, field_type, kwds) else: diff --git a/pydra/design/tests/test_shell.py b/pydra/design/tests/test_shell.py index 184c8e05e..4c2c2f91b 100644 --- a/pydra/design/tests/test_shell.py +++ b/pydra/design/tests/test_shell.py @@ -434,7 +434,6 @@ def test_interface_template_with_type_overrides(): name="int_arg", argstr="--int-arg", type=int | None, - default=None, position=5, ), shell.arg( diff --git a/pydra/engine/core.py b/pydra/engine/core.py index 9b0ac65d7..e68f4959f 100644 --- a/pydra/engine/core.py +++ b/pydra/engine/core.py @@ -371,7 +371,7 @@ def run(self, rerun: bool = False): self.audit.audit_task(task=self) try: self.audit.monitor() - self.definition._run(self) + self.definition._run(self, rerun) result.outputs = self.definition.Outputs._from_task(self) except Exception: etype, eval, etr = sys.exc_info() @@ -425,7 +425,7 @@ async def run_async(self, rerun: bool = False) -> Result: self.audit.start_audit(odir=self.output_dir) try: self.audit.monitor() - await self.definition._run_async(self) + await self.definition._run_async(self, rerun) result.outputs = self.definition.Outputs._from_task(self) except Exception: etype, eval, etr = sys.exc_info() @@ -628,8 +628,7 @@ def clear_cache( @classmethod def construct( - cls, - definition: WorkflowDef[WorkflowOutputsType], + cls, definition: WorkflowDef[WorkflowOutputsType], dont_cache: bool = False ) -> Self: """Construct a workflow from a definition, caching the constructed worklow""" @@ -710,7 +709,7 @@ def construct( f"{len(output_lazy_fields)} ({output_lazy_fields})" ) for outpt, outpt_lf in zip(output_fields, output_lazy_fields): - # Automatically combine any uncombined state arrays into lists + # Automatically combine any uncombined state arrays into a single lists if TypeParser.get_origin(outpt_lf._type) is StateArray: outpt_lf._type = list[TypeParser.strip_splits(outpt_lf._type)[0]] setattr(outputs, outpt.name, outpt_lf) @@ -722,8 +721,8 @@ def construct( f"Expected outputs {unset_outputs} to be set by the " f"constructor of {workflow!r}" ) - - cls._constructed_cache[defn_hash][non_lazy_keys][non_lazy_hash] = workflow + if not dont_cache: + cls._constructed_cache[defn_hash][non_lazy_keys][non_lazy_hash] = workflow return workflow @@ -735,8 +734,7 @@ def under_construction(cls) -> "Workflow[ty.Any]": # Find the frame where the construct method was called if ( frame.f_code.co_name == "construct" - and "cls" in frame.f_locals - and frame.f_locals["cls"] is cls + and frame.f_locals.get("cls") is cls and "workflow" in frame.f_locals ): return frame.f_locals["workflow"] # local var "workflow" in construct diff --git a/pydra/engine/helpers_file.py b/pydra/engine/helpers_file.py index 5be17047b..567fa3cc5 100644 --- a/pydra/engine/helpers_file.py +++ b/pydra/engine/helpers_file.py @@ -135,9 +135,7 @@ def template_update( if isinstance(field, shell.outarg) and field.path_template and getattr(definition, field.name) - and all( - getattr(definition, required_field) for required_field in field.requires - ) + and all(req.satisfied(definition) for req in field.requires) ] dict_mod = {} diff --git a/pydra/engine/node.py b/pydra/engine/node.py index 8fd3bf041..29cf3aedb 100644 --- a/pydra/engine/node.py +++ b/pydra/engine/node.py @@ -134,7 +134,7 @@ def lzout(self) -> OutputType: type_, _ = TypeParser.strip_splits(outpt._type) if self._state.combiner: type_ = list[type_] - for _ in range(self._state.depth - int(bool(self._state.combiner))): + for _ in range(self._state.depth()): type_ = StateArray[type_] outpt._type = type_ # Flag the output lazy fields as being not typed checked (i.e. assigned to @@ -272,7 +272,7 @@ def _get_upstream_states(self) -> dict[str, tuple["State", list[str]]]: if ( isinstance(val, lazy.LazyOutField) and val._node.state - and val._node.state.depth + and val._node.state.depth() ): node: Node = val._node # variables that are part of inner splitters should be treated as a containers @@ -305,26 +305,6 @@ def _extract_input_el(self, inputs, inp_nm, ind): else: return getattr(inputs, inp_nm)[ind] - def _split_definition(self) -> dict[StateIndex, "TaskDef[OutputType]"]: - """Split the definition into the different states it will be run over""" - # TODO: doesn't work properly for more cmplicated wf (check if still an issue) - if not self.state: - return {None: self._definition} - split_defs = {} - for input_ind in self.state.inputs_ind: - inputs_dict = {} - for inp in set(self.input_names): - if f"{self.name}.{inp}" in input_ind: - inputs_dict[inp] = self._extract_input_el( - inputs=self._definition, - inp_nm=inp, - ind=input_ind[f"{self.name}.{inp}"], - ) - split_defs[StateIndex(input_ind)] = attrs.evolve( - self._definition, **inputs_dict - ) - return split_defs - # else: # # todo it never gets here # breakpoint() diff --git a/pydra/engine/specs.py b/pydra/engine/specs.py index baea685f1..3f65e752e 100644 --- a/pydra/engine/specs.py +++ b/pydra/engine/specs.py @@ -32,7 +32,7 @@ from . import helpers_state as hlpst from . import lazy from pydra.utils.hash import hash_function, Cache -from pydra.utils.typing import StateArray, MultiInputObj +from pydra.utils.typing import StateArray, is_multi_input from pydra.design.base import Field, Arg, Out, RequirementSet, NO_DEFAULT from pydra.design import shell @@ -231,7 +231,6 @@ def __call__( cache_locations=cache_locations, messenger_args=messenger_args, messengers=messengers, - rerun=rerun, environment=environment, worker=worker, **kwargs, @@ -239,6 +238,7 @@ def __call__( result = sub( self, hooks=hooks, + rerun=rerun, ) except TypeError as e: # Catch any inadvertent passing of task definition parameters to the @@ -260,7 +260,7 @@ def __call__( else: errors = result.errors raise RuntimeError( - f"Task {self} failed @ {errors['time of crash']} with following errors:\n" + f"Task {self} failed @ {errors['time of crash']} with the following errors:\n" + "\n".join(errors["error message"]) ) return result.outputs @@ -338,7 +338,9 @@ def split( ): split_val = StateArray(value) else: - raise TypeError(f"Could not split {value} as it is not a sequence type") + raise TypeError( + f"Could not split {value!r} as it is not a sequence type" + ) split_inputs[name] = split_val split_def = attrs.evolve(self, **split_inputs) split_def._splitter = splitter @@ -507,7 +509,7 @@ def _check_rules(self): # Raise error if any required field is unset. if ( - value is not None + value and field.requires and not any(rs.satisfied(self) for rs in field.requires) ): @@ -686,7 +688,7 @@ class PythonDef(TaskDef[PythonOutputsType]): _task_type = "python" - def _run(self, task: "Task[PythonDef]") -> None: + def _run(self, task: "Task[PythonDef]", rerun: bool = True) -> None: # Prepare the inputs to the function inputs = attrs_values(self) del inputs["function"] @@ -773,13 +775,13 @@ class WorkflowDef(TaskDef[WorkflowOutputsType]): _constructed = attrs.field(default=None, init=False, repr=False, eq=False) - def _run(self, task: "Task[WorkflowDef]") -> None: + def _run(self, task: "Task[WorkflowDef]", rerun: bool) -> None: """Run the workflow.""" - task.submitter.expand_workflow(task) + task.submitter.expand_workflow(task, rerun) - async def _run_async(self, task: "Task[WorkflowDef]") -> None: + async def _run_async(self, task: "Task[WorkflowDef]", rerun: bool) -> None: """Run the workflow asynchronously.""" - await task.submitter.expand_workflow_async(task) + await task.submitter.expand_workflow_async(task, rerun) def construct(self) -> "Workflow": from pydra.engine.core import Workflow @@ -971,7 +973,7 @@ class ShellDef(TaskDef[ShellOutputsType]): RESERVED_FIELD_NAMES = TaskDef.RESERVED_FIELD_NAMES + ("cmdline",) - def _run(self, task: "Task[ShellDef]") -> None: + def _run(self, task: "Task[ShellDef]", rerun: bool = True) -> None: """Run the shell command.""" task.return_values = task.environment.execute(task) @@ -981,6 +983,7 @@ def cmdline(self) -> str: the current working directory.""" # checking the inputs fields before returning the command line self._check_resolved() + self._check_rules() # Skip the executable, which can be a multi-part command, e.g. 'docker run'. cmd_args = self._command_args() cmdline = cmd_args[0] @@ -1013,7 +1016,7 @@ def _command_args( for field in list_fields(self): name = field.name value = inputs[name] - if value is None: + if value is None or is_multi_input(field.type) and value == []: continue if name == "executable": pos_args.append(self._command_shelltask_executable(field, value)) @@ -1126,7 +1129,7 @@ def _command_pos_args( # if False, nothing is added to the command. if value is True: cmd_add.append(field.argstr) - elif ty.get_origin(tp) is MultiInputObj: + elif is_multi_input(tp): # if the field is MultiInputObj, it is used to create a list of arguments for val in value or []: cmd_add += self._format_arg(field, val) @@ -1147,7 +1150,9 @@ def _format_arg(self, field: shell.arg, value: ty.Any) -> list[str]: argstr_formatted_l = [] for val in value: argstr_f = argstr_formatting( - field.argstr, self, value_updates={field.name: val} + field.argstr, + self, + value_updates={field.name: val}, ) argstr_formatted_l.append(f" {argstr_f}") cmd_el_str = field.sep.join(argstr_formatted_l) @@ -1218,20 +1223,20 @@ def split_cmd(cmd: str | None): def argstr_formatting( - argstr: str, inputs: dict[str, ty.Any], value_updates: dict[str, ty.Any] = None + argstr: str, inputs: TaskDef[OutputsType], value_updates: dict[str, ty.Any] = None ): """formatting argstr that have form {field_name}, using values from inputs and updating with value_update if provided """ # if there is a value that has to be updated (e.g. single value from a list) # getting all fields that should be formatted, i.e. {field_name}, ... + inputs_dict = attrs_values(inputs) if value_updates: - inputs = copy(inputs) - inputs.update(value_updates) + inputs_dict.update(value_updates) inp_fields = parse_format_string(argstr) val_dict = {} for fld_name in inp_fields: - fld_value = inputs[fld_name] + fld_value = inputs_dict[fld_name] fld_attr = getattr(attrs.fields(type(inputs)), fld_name) if fld_value is None or ( fld_value is False diff --git a/pydra/engine/state.py b/pydra/engine/state.py index 11a4290bc..d078c1065 100644 --- a/pydra/engine/state.py +++ b/pydra/engine/state.py @@ -231,29 +231,38 @@ def names(self): names.append(token) return names - @property - def depth(self) -> int: + def depth(self, after_combine: bool = True) -> int: """Return the number of splits of the state, i.e. the number nested state arrays to wrap around the type of lazy out fields + Parameters + ---------- + after_combine : :obj:`bool` + if True, the depth is after combining the fields, otherwise it is before + any combinations + Returns ------- int - number of uncombined independent splits (i.e. linked splits only add 1) + number of splits in the state (i.e. linked splits only add 1) """ depth = 0 stack = [] + + def included(s): + return s not in self.combiner if after_combine else True + for spl in self.splitter_rpn: if spl in [".", "*"]: if spl == ".": - depth += int(all(s not in self.combiner for s in stack)) + depth += int(all(included(s) for s in stack)) else: assert spl == "*" - depth += len([s for s in stack if s not in self.combiner]) + depth += len([s for s in stack if included(s)]) stack = [] else: stack.append(spl) - remaining_stack = [s for s in stack if s not in self.combiner] + remaining_stack = [s for s in stack if included(s)] return depth + len(remaining_stack) @property diff --git a/pydra/engine/submitter.py b/pydra/engine/submitter.py index 3236306a5..afbdbc3bf 100644 --- a/pydra/engine/submitter.py +++ b/pydra/engine/submitter.py @@ -26,6 +26,7 @@ from pydra.utils.messenger import AuditFlag, Messenger from pydra.utils import default_run_cache_dir from pydra.design import workflow +from .state import State import logging logger = logging.getLogger("pydra.submitter") @@ -35,7 +36,7 @@ from .specs import WorkflowDef, TaskDef, TaskOutputs, TaskHooks, Result from .core import Workflow from .environments import Environment - from .state import State + DefType = ty.TypeVar("DefType", bound="TaskDef") OutputType = ty.TypeVar("OutputType", bound="TaskOutputs") @@ -58,9 +59,6 @@ class Submitter: The worker to use, by default "cf" environment: Environment, optional The execution environment to use, by default None - rerun : bool, optional - Whether to force the re-computation of the task results even if existing - results are found, by default False cache_locations : list[os.PathLike], optional Alternate cache locations to check for pre-computed results, by default None audit_flags : AuditFlag, optional @@ -81,7 +79,6 @@ class Submitter: cache_dir: os.PathLike worker: Worker environment: "Environment | None" - rerun: bool cache_locations: list[os.PathLike] audit_flags: AuditFlag messengers: ty.Iterable[Messenger] @@ -91,10 +88,10 @@ class Submitter: def __init__( self, + /, cache_dir: os.PathLike | None = None, worker: str | ty.Type[Worker] | Worker | None = "debug", environment: "Environment | None" = None, - rerun: bool = False, cache_locations: list[os.PathLike] | None = None, audit_flags: AuditFlag = AuditFlag.NONE, messengers: ty.Iterable[Messenger] | None = None, @@ -127,7 +124,6 @@ def __init__( self.cache_dir = cache_dir self.cache_locations = cache_locations self.environment = environment if environment is not None else Native() - self.rerun = rerun self.loop = get_open_loop() self._own_loop = not self.loop.is_running() if isinstance(worker, Worker): @@ -177,6 +173,7 @@ def __call__( task_def: "TaskDef[OutputType]", hooks: "TaskHooks | None" = None, raise_errors: bool | None = None, + rerun: bool = False, ) -> "Result[OutputType]": """Submitter run function. @@ -190,6 +187,9 @@ def __call__( raise_errors : bool, optional Whether to raise errors, by default True if the 'debug' worker is used, otherwise False + rerun : bool, optional + Whether to force the re-computation of the task results even if existing + results are found, by default False Returns ------- @@ -210,7 +210,22 @@ def __call__( if task_def._splitter: from pydra.engine.specs import TaskDef - output_types = {o.name: list[o.type] for o in list_fields(task_def.Outputs)} + state = State( + name="not-important", + definition=task_def, + splitter=task_def._splitter, + combiner=task_def._combiner, + ) + list_depth = 2 if state.depth(after_combine=False) != state.depth() else 1 + + def wrap_type(tp): + for _ in range(list_depth): + tp = list[tp] + return tp + + output_types = { + o.name: wrap_type(o.type) for o in list_fields(task_def.Outputs) + } @workflow.define(outputs=output_types) def Split( @@ -242,11 +257,9 @@ def Split( try: self.run_start_time = datetime.now() if self.worker.is_async: # Only workflow tasks can be async - self.loop.run_until_complete( - self.worker.run_async(task, rerun=self.rerun) - ) + self.loop.run_until_complete(self.worker.run_async(task, rerun=rerun)) else: - self.worker.run(task, rerun=self.rerun) + self.worker.run(task, rerun=rerun) except Exception as e: msg = ( f"Full crash report for {type(task_def).__name__!r} task is here: " @@ -256,7 +269,7 @@ def Split( e.add_note(msg) raise e else: - logger.error("\nTask execution failed\n" + msg) + logger.error("\nTask execution failed\n%s", msg) finally: self.run_start_time = None PersistentCache().clean_up() @@ -288,7 +301,7 @@ def __setstate__(self, state): self._worker = WORKERS[self.worker_name](**self.worker_kwargs) self.worker.loop = self.loop - def expand_workflow(self, workflow_task: "Task[WorkflowDef]") -> None: + def expand_workflow(self, workflow_task: "Task[WorkflowDef]", rerun: bool) -> None: """Expands and executes a workflow task synchronously. Typically only used during debugging and testing, as the asynchronous version is more efficient. @@ -305,11 +318,13 @@ def expand_workflow(self, workflow_task: "Task[WorkflowDef]") -> None: tasks = self.get_runnable_tasks(exec_graph) while tasks or any(not n.done for n in exec_graph.nodes): for task in tasks: - self.worker.run(task, rerun=self.rerun) + self.worker.run(task, rerun=rerun) tasks = self.get_runnable_tasks(exec_graph) workflow_task.return_values = {"workflow": wf, "exec_graph": exec_graph} - async def expand_workflow_async(self, workflow_task: "Task[WorkflowDef]") -> None: + async def expand_workflow_async( + self, workflow_task: "Task[WorkflowDef]", rerun: bool + ) -> None: """ Expand and execute a workflow task asynchronously. @@ -400,9 +415,9 @@ async def expand_workflow_async(self, workflow_task: "Task[WorkflowDef]") -> Non raise RuntimeError(msg) for task in tasks: if task.is_async: - await self.worker.run_async(task, rerun=self.rerun) + await self.worker.run_async(task, rerun=rerun) else: - task_futures.add(self.worker.run(task, rerun=self.rerun)) + task_futures.add(self.worker.run(task, rerun=rerun)) task_futures = await self.worker.fetch_finished(task_futures) tasks = self.get_runnable_tasks(exec_graph) workflow_task.return_values = {"workflow": wf, "exec_graph": exec_graph} @@ -630,12 +645,9 @@ def _generate_tasks(self) -> ty.Iterable["Task[DefType]"]: name=self.node.name, ) else: - for index, split_defn in self.node._split_definition().items(): + for index, split_defn in self._split_definition().items(): yield Task( - definition=self._resolve_lazy_inputs( - task_def=split_defn, - state_index=index, - ), + definition=split_defn, submitter=self.submitter, environment=self.node._environment, name=self.node.name, @@ -671,6 +683,34 @@ def _resolve_lazy_inputs( ) return attrs.evolve(task_def, **resolved) + def _split_definition(self) -> dict[StateIndex, "TaskDef[OutputType]"]: + """Split the definition into the different states it will be run over""" + # TODO: doesn't work properly for more cmplicated wf (check if still an issue) + if not self.node.state: + return {None: self.node._definition} + split_defs = {} + for input_ind in self.node.state.inputs_ind: + inputs_dict = {} + for inp in set(self.node.input_names): + if f"{self.node.name}.{inp}" in input_ind: + value = getattr(self.node._definition, inp) + if isinstance(value, LazyField): + inputs_dict[inp] = value._get_value( + workflow=self.workflow, + graph=self.graph, + state_index=StateIndex(input_ind), + ) + else: + inputs_dict[inp] = self.node._extract_input_el( + inputs=self.node._definition, + inp_nm=inp, + ind=input_ind[f"{self.node.name}.{inp}"], + ) + split_defs[StateIndex(input_ind)] = attrs.evolve( + self.node._definition, **inputs_dict + ) + return split_defs + def get_runnable_tasks(self, graph: DiGraph) -> list["Task[DefType]"]: """For a given node, check to see which tasks have been successfully run, are ready to run, can't be run due to upstream errors, or are blocked on other tasks to complete. diff --git a/pydra/engine/tests/test_node_task.py b/pydra/engine/tests/test_node_task.py index 51316324f..1abfc1d96 100644 --- a/pydra/engine/tests/test_node_task.py +++ b/pydra/engine/tests/test_node_task.py @@ -1,15 +1,13 @@ import os import shutil -import attr +import attrs import typing as ty import numpy as np import time -from unittest import mock from pathlib import Path import pytest -import time from fileformats.generic import File -from pydra.design import python +from pydra.design import python, workflow from .utils import ( FunAddTwo, @@ -24,9 +22,39 @@ Op4Var, ) -from ..core import Task +from pydra.engine.core import Task +from pydra.engine.specs import TaskDef +from pydra.engine.state import State from pydra.utils.typing import StateArray -from ..submitter import Submitter +from pydra.engine.submitter import Submitter +from pydra.engine.core import Workflow + + +@workflow.define +def IdentityWorkflow(a: int) -> int: + + @python.define + def Identity(a): + return a + + a = workflow.add(Identity(a=a)) + return a.out + + +def get_state(task: TaskDef, name="NA") -> State: + """helper function to get the state of the task once it has been added to workflow""" + identity_workflow = IdentityWorkflow(a=1) + wf = Workflow.construct(identity_workflow, dont_cache=True) + wf.add(task, name=name) + node = wf[name] + if node.state: + node.state.prepare_states() + node.state.prepare_inputs() + return node.state + + +def num_python_cache_dirs(cache_path: Path) -> int: + return len(list(cache_path.glob("python-*"))) @pytest.fixture(scope="module") @@ -42,15 +70,6 @@ def move2orig(): request.addfinalizer(move2orig) -# Tests for tasks initializations -def test_task_init_1(): - """task with mandatory arguments only""" - nn = FunAddTwo() - assert isinstance(nn, Task) - assert nn.name == "fun_addtwo" - assert hasattr(nn, "__call__") - - def test_task_init_1a(): with pytest.raises(TypeError): FunAddTwo("NA") @@ -58,10 +77,11 @@ def test_task_init_1a(): def test_task_init_2(): """task with a name and inputs""" - nn = FunAddTwo(name="NA", a=3) + nn = FunAddTwo(a=3) # adding NA to the name of the variable - assert getattr(nn.inputs, "a") == 3 - assert nn.state is None + assert nn.a == 3 + state = get_state(nn) + assert state is None @pytest.mark.parametrize( @@ -77,15 +97,15 @@ def test_task_init_3( if input_type == "array": a_in = np.array(a_in) - nn = FunAddTwo().split(splitter=splitter, a=a_in) + nn = FunAddTwo().split(splitter, a=a_in) - assert np.allclose(nn.inputs.a, [3, 5]) - assert nn.state.splitter == state_splitter - assert nn.state.splitter_rpn == state_rpn + assert np.allclose(nn.a, [3, 5]) + state = get_state(nn) + assert state.splitter == state_splitter + assert state.splitter_rpn == state_rpn - nn.state.prepare_states(nn.inputs) - assert nn.state.states_ind == states_ind - assert nn.state.states_val == states_val + assert state.states_ind == states_ind + assert state.states_val == states_val @pytest.mark.parametrize( @@ -127,168 +147,164 @@ def test_task_init_3a( a_in, b_in = np.array(a_in), np.array(b_in) elif input_type == "mixed": a_in = np.array(a_in) - nn = FunAddVar(name="NA").split(splitter=splitter, a=a_in, b=b_in) + nn = FunAddVar().split(splitter, a=a_in, b=b_in) + state = get_state(nn) - assert np.allclose(nn.inputs.a, [3, 5]) - assert np.allclose(nn.inputs.b, [10, 20]) - assert nn.state.splitter == state_splitter - assert nn.state.splitter_rpn == state_rpn + assert np.allclose(nn.a, [3, 5]) + assert np.allclose(nn.b, [10, 20]) + assert state.splitter == state_splitter + assert state.splitter_rpn == state_rpn - nn.state.prepare_states(nn.inputs) - assert nn.state.states_ind == states_ind - assert nn.state.states_val == states_val + assert state.states_ind == states_ind + assert state.states_val == states_val def test_task_init_4(): """task with interface splitter and inputs set in the split method""" - nn = FunAddTwo(name="NA") - nn.split(splitter="a", a=[3, 5]) - assert np.allclose(nn.inputs.a, [3, 5]) + nn = FunAddTwo() + nn = nn.split("a", a=[3, 5]) + state = get_state(nn) + assert np.allclose(nn.a, [3, 5]) - assert nn.state.splitter == "NA.a" - assert nn.state.splitter_rpn == ["NA.a"] + assert state.splitter == "NA.a" + assert state.splitter_rpn == ["NA.a"] - nn.state.prepare_states(nn.inputs) - assert nn.state.states_ind == [{"NA.a": 0}, {"NA.a": 1}] - assert nn.state.states_val == [{"NA.a": 3}, {"NA.a": 5}] + assert state.states_ind == [{"NA.a": 0}, {"NA.a": 1}] + assert state.states_val == [{"NA.a": 3}, {"NA.a": 5}] def test_task_init_4b(): """updating splitter using overwrite=True""" - nn = FunAddTwo(name="NA") - nn.split(splitter="a", a=[1, 2]) - nn.split(splitter="a", a=[3, 5], overwrite=True) - assert np.allclose(nn.inputs.a, [3, 5]) + nn = FunAddTwo() + nn = nn.split("a", a=[1, 2]) + nn = nn.split("a", a=[3, 5], overwrite=True) + state = get_state(nn) + assert np.allclose(nn.a, [3, 5]) - assert nn.state.splitter == "NA.a" - assert nn.state.splitter_rpn == ["NA.a"] + assert state.splitter == "NA.a" + assert state.splitter_rpn == ["NA.a"] - nn.state.prepare_states(nn.inputs) - assert nn.state.states_ind == [{"NA.a": 0}, {"NA.a": 1}] - assert nn.state.states_val == [{"NA.a": 3}, {"NA.a": 5}] + assert state.states_ind == [{"NA.a": 0}, {"NA.a": 1}] + assert state.states_val == [{"NA.a": 3}, {"NA.a": 5}] def test_task_init_4c(): """trying to set splitter twice without using overwrite""" - nn = FunAddVar(name="NA").split(splitter="b", b=[1, 2]) + nn = FunAddVar().split("b", b=[1, 2]) + state = get_state(nn) with pytest.raises(Exception) as excinfo: - nn.split(splitter="a", a=[3, 5]) - assert "splitter has been already set" in str(excinfo.value) + nn.split("a", a=[3, 5]) + assert "Cannot overwrite existing splitter" in str(excinfo.value) - assert nn.state.splitter == "NA.b" + assert state.splitter == "NA.b" def test_task_init_4d(): """trying to set the same splitter twice without using overwrite if the splitter is the same, the exception shouldn't be raised """ - nn = FunAddTwo(name="NA").split(splitter="a", a=[3, 5]) - nn.split(splitter="a", a=[3, 5]) - assert nn.state.splitter == "NA.a" + nn = FunAddTwo().split("a", a=[3, 5]) + nn = nn.split("a", a=[3, 5], overwrite=True) + state = get_state(nn) + assert state.splitter == "NA.a" def test_task_init_5(): """task with inputs, splitter and combiner""" - nn = ( - FunAddVar(name="NA").split(splitter=["a", "b"], a=[3, 5], b=[1, 2]).combine("b") - ) + nn = FunAddVar().split(["a", "b"], a=[3, 5], b=[1, 2]).combine("b") + state = get_state(nn) - assert nn.state.splitter == ["NA.a", "NA.b"] - assert nn.state.splitter_rpn == ["NA.a", "NA.b", "*"] - assert nn.state.combiner == ["NA.b"] + assert state.splitter == ["NA.a", "NA.b"] + assert state.splitter_rpn == ["NA.a", "NA.b", "*"] + assert state.combiner == ["NA.b"] - assert nn.state.splitter_final == "NA.a" - assert nn.state.splitter_rpn_final == ["NA.a"] + assert state.splitter_final == "NA.a" + assert state.splitter_rpn_final == ["NA.a"] - nn.state.prepare_states(nn.inputs) - assert nn.state.states_ind == [ + assert state.states_ind == [ {"NA.a": 0, "NA.b": 0}, {"NA.a": 0, "NA.b": 1}, {"NA.a": 1, "NA.b": 0}, {"NA.a": 1, "NA.b": 1}, ] - assert nn.state.states_val == [ + assert state.states_val == [ {"NA.a": 3, "NA.b": 1}, {"NA.a": 3, "NA.b": 2}, {"NA.a": 5, "NA.b": 1}, {"NA.a": 5, "NA.b": 2}, ] - assert nn.state.final_combined_ind_mapping == {0: [0, 1], 1: [2, 3]} + assert state.final_combined_ind_mapping == {0: [0, 1], 1: [2, 3]} def test_task_init_5a(): """updating combiner using overwrite=True""" - nn = ( - FunAddVar(name="NA").split(splitter=["a", "b"], a=[3, 5], b=[1, 2]).combine("b") - ) - nn.combine("a", overwrite=True) + nn = FunAddVar().split(["a", "b"], a=[3, 5], b=[1, 2]).combine("b") + nn = nn.combine("a", overwrite=True) + state = get_state(nn) - assert nn.state.splitter == ["NA.a", "NA.b"] - assert nn.state.splitter_rpn == ["NA.a", "NA.b", "*"] - assert nn.state.combiner == ["NA.a"] + assert state.splitter == ["NA.a", "NA.b"] + assert state.splitter_rpn == ["NA.a", "NA.b", "*"] + assert state.combiner == ["NA.a"] - assert nn.state.splitter_final == "NA.b" - assert nn.state.splitter_rpn_final == ["NA.b"] + assert state.splitter_final == "NA.b" + assert state.splitter_rpn_final == ["NA.b"] - nn.state.prepare_states(nn.inputs) - assert nn.state.states_ind == [ + assert state.states_ind == [ {"NA.a": 0, "NA.b": 0}, {"NA.a": 0, "NA.b": 1}, {"NA.a": 1, "NA.b": 0}, {"NA.a": 1, "NA.b": 1}, ] - assert nn.state.states_val == [ + assert state.states_val == [ {"NA.a": 3, "NA.b": 1}, {"NA.a": 3, "NA.b": 2}, {"NA.a": 5, "NA.b": 1}, {"NA.a": 5, "NA.b": 2}, ] - assert nn.state.final_combined_ind_mapping == {0: [0, 2], 1: [1, 3]} + assert state.final_combined_ind_mapping == {0: [0, 2], 1: [1, 3]} def test_task_init_5b(): """updating combiner without using overwrite""" - nn = ( - FunAddVar(name="NA").split(splitter=["a", "b"], a=[3, 5], b=[1, 2]).combine("b") - ) + nn = FunAddVar().split(["a", "b"], a=[3, 5], b=[1, 2]).combine("b") + state = get_state(nn) with pytest.raises(Exception) as excinfo: nn.combine("a") - assert "combiner has been already set" in str(excinfo.value) + assert "Attempting to overwrite existing combiner" in str(excinfo.value) - assert nn.state.combiner == ["NA.b"] + assert state.combiner == ["NA.b"] def test_task_init_5c(): """trying to set the same combiner twice without using overwrite if the combiner is the same, the exception shouldn't be raised """ - nn = ( - FunAddVar(name="NA").split(splitter=["a", "b"], a=[3, 5], b=[1, 2]).combine("b") - ) - nn.combine("b") + nn = FunAddVar().split(["a", "b"], a=[3, 5], b=[1, 2]).combine("b") + state = get_state(nn) + nn = nn.combine("b", overwrite=True) - assert nn.state.splitter == ["NA.a", "NA.b"] - assert nn.state.splitter_rpn == ["NA.a", "NA.b", "*"] - assert nn.state.combiner == ["NA.b"] + assert state.splitter == ["NA.a", "NA.b"] + assert state.splitter_rpn == ["NA.a", "NA.b", "*"] + assert state.combiner == ["NA.b"] - assert nn.state.splitter_final == "NA.a" - assert nn.state.splitter_rpn_final == ["NA.a"] + assert state.splitter_final == "NA.a" + assert state.splitter_rpn_final == ["NA.a"] def test_task_init_6(): """task with splitter, but the input is an empty list""" - nn = FunAddTwo(name="NA") - nn.split(splitter="a", a=[]) - assert nn.inputs.a == [] + nn = FunAddTwo() + nn = nn.split("a", a=[]) + state = get_state(nn) + assert nn.a == [] - assert nn.state.splitter == "NA.a" - assert nn.state.splitter_rpn == ["NA.a"] + assert state.splitter == "NA.a" + assert state.splitter_rpn == ["NA.a"] - nn.state.prepare_states(nn.inputs) - assert nn.state.states_ind == [] - assert nn.state.states_val == [] + assert state.states_ind == [] + assert state.states_val == [] def test_task_init_7(tmp_path): @@ -301,8 +317,8 @@ def test_task_init_7(tmp_path): with open(file2, "w") as f: f.write("from pydra\n") - nn1 = FunFileList(name="NA", filename_list=[file1, file2]) - output_dir1 = nn1.output_dir + nn1 = FunFileList(filename_list=[file1, file2]) + hash1 = nn1._hash # changing the content of the file time.sleep(2) # need the mtime to be different @@ -310,43 +326,35 @@ def test_task_init_7(tmp_path): with open(file2, "w") as f: f.write("from pydra") - nn2 = FunFileList(name="NA", filename_list=[file1, file2]) - output_dir2 = nn2.output_dir + nn2 = FunFileList(filename_list=[file1, file2]) + hash2 = nn2._hash # the checksum should be different - content of file2 is different - assert output_dir1.name != output_dir2.name + assert hash1 != hash2 def test_task_init_8(): - """task without setting the input, the value should be set to attr.NOTHING""" - nn = FunAddTwo(name="NA") - assert nn.inputs.a is attr.NOTHING + """task without setting the input, the value should be set to attrs.NOTHING""" + nn = FunAddTwo() + assert nn.a is attrs.NOTHING def test_task_init_9(): """task without setting the input, but using the default avlue from function""" - nn1 = FunAddVarDefault(name="NA", a=2) - assert nn1.inputs.b == 1 + nn1 = FunAddVarDefault(a=2) + assert nn1.b == 1 - nn2 = FunAddVarDefault(name="NA", a=2, b=1) - assert nn2.inputs.b == 1 + nn2 = FunAddVarDefault(a=2, b=1) + assert nn2.b == 1 # both tasks should have the same checksum - assert nn1.checksum == nn2.checksum + assert nn1._hash == nn2._hash -def test_task_error(): - func = FunDiv(name="div", a=1, b=0) +def test_task_error(tmp_path): + func = FunDiv(a=1, b=0) with pytest.raises(ZeroDivisionError): - func() - assert (func.output_dir / "_error.pklz").exists() - - -def test_odir_init(): - """checking if output_dir is available for a task without init - before running the task - """ - nn = FunAddTwo(name="NA", a=3) - assert nn.output_dir + func(cache_dir=tmp_path) + assert (next(tmp_path.iterdir()) / "_error.pklz").exists() # Tests for tasks without state (i.e. no splitter) @@ -355,124 +363,111 @@ def test_odir_init(): @pytest.mark.flaky(reruns=2) # when dask def test_task_nostate_1(plugin_dask_opt, tmp_path): """task without splitter""" - nn = FunAddTwo(name="NA", a=3) - nn.cache_dir = tmp_path - assert np.allclose(nn.inputs.a, [3]) - assert nn.state is None + nn = FunAddTwo(a=3) - with Submitter(worker=plugin_dask_opt) as sub: - sub(nn) + assert np.allclose(nn.a, [3]) + state = get_state(nn) + assert state is None + + with Submitter(worker=plugin_dask_opt, cache_dir=tmp_path) as sub: + results = sub(nn) + assert not results.errored, "\n".join(results.errors["error message"]) # checking the results - results = nn.result() - assert results.output.out == 5 - # checking the return_inputs option, either is return_inputs is True, or "val", - # it should give values of inputs that corresponds to the specific element - results_verb = nn.result(return_inputs=True) - results_verb_val = nn.result(return_inputs="val") - assert results_verb[0] == results_verb_val[0] == {"NA.a": 3} - assert results_verb[1].output.out == results_verb_val[1].output.out == 5 - # checking the return_inputs option return_inputs="ind" - # it should give indices of inputs (instead of values) for each element - results_verb_ind = nn.result(return_inputs="ind") - assert results_verb_ind[0] == {"NA.a": None} - assert results_verb_ind[1].output.out == 5 + assert results.outputs.out == 5 # checking the output_dir - assert nn.output_dir.exists() + assert results.output_dir.exists() -def test_task_nostate_1_call(): +def test_task_nostate_1_call(tmp_path): """task without splitter""" - nn = FunAddTwo(name="NA", a=3) - nn() + nn = FunAddTwo(a=3) + with Submitter(cache_dir=tmp_path) as sub: + results = sub(nn) + assert not results.errored, "\n".join(results.errors["error message"]) # checking the results - results = nn.result() - assert results.output.out == 5 + + assert results.outputs.out == 5 # checking the output_dir - assert nn.output_dir.exists() + assert results.output_dir.exists() @pytest.mark.flaky(reruns=2) # when dask def test_task_nostate_1_call_subm(plugin_dask_opt, tmp_path): """task without splitter""" - nn = FunAddTwo(name="NA", a=3) - nn.cache_dir = tmp_path - assert np.allclose(nn.inputs.a, [3]) - assert nn.state is None + nn = FunAddTwo(a=3) - with Submitter(worker=plugin_dask_opt) as sub: - nn(submitter=sub) + assert np.allclose(nn.a, [3]) + state = get_state(nn) + assert state is None + + with Submitter(worker=plugin_dask_opt, cache_dir=tmp_path) as sub: + results = sub(nn) + assert not results.errored, "\n".join(results.errors["error message"]) # checking the results - results = nn.result() - assert results.output.out == 5 + + assert results.outputs.out == 5 # checking the output_dir - assert nn.output_dir.exists() + assert results.output_dir.exists() @pytest.mark.flaky(reruns=2) # when dask def test_task_nostate_1_call_plug(plugin_dask_opt, tmp_path): """task without splitter""" - nn = FunAddTwo(name="NA", a=3) - nn.cache_dir = tmp_path - assert np.allclose(nn.inputs.a, [3]) - assert nn.state is None - - nn(plugin=plugin_dask_opt) + nn = FunAddTwo(a=3) - # checking the results - results = nn.result() - assert results.output.out == 5 - # checking the output_dir - assert nn.output_dir.exists() + assert np.allclose(nn.a, [3]) + state = get_state(nn) + assert state is None - -def test_task_nostate_1_call_updateinp(): - """task without splitter""" - nn = FunAddTwo(name="NA", a=30) - # updating input when calling the node - nn(a=3) + with Submitter(cache_dir=tmp_path, worker=plugin_dask_opt) as sub: + results = sub(nn) + assert not results.errored, "\n".join(results.errors["error message"]) # checking the results - results = nn.result() - assert results.output.out == 5 + + assert results.outputs.out == 5 # checking the output_dir - assert nn.output_dir.exists() + assert results.output_dir.exists() def test_task_nostate_2(plugin, tmp_path): """task with a list as an input, but no splitter""" - nn = Moment(name="NA", n=3, lst=[2, 3, 4]) - nn.cache_dir = tmp_path - assert np.allclose(nn.inputs.n, [3]) - assert np.allclose(nn.inputs.lst, [2, 3, 4]) - assert nn.state is None + nn = Moment(n=3, lst=[2, 3, 4]) - with Submitter(worker=plugin) as sub: - sub(nn) + assert np.allclose(nn.n, [3]) + assert np.allclose(nn.lst, [2, 3, 4]) + state = get_state(nn) + assert state is None + + with Submitter(worker=plugin, cache_dir=tmp_path) as sub: + results = sub(nn) + assert not results.errored, "\n".join(results.errors["error message"]) # checking the results - results = nn.result() - assert results.output.out == 33 + + assert results.outputs.out == 33 # checking the output_dir - assert nn.output_dir.exists() + assert results.output_dir.exists() def test_task_nostate_3(plugin, tmp_path): """task with a dictionary as an input""" - nn = FunDict(name="NA", d={"a": "ala", "b": "bala"}) - nn.cache_dir = tmp_path - assert nn.inputs.d == {"a": "ala", "b": "bala"} + nn = FunDict(d={"a": "ala", "b": "bala"}) - with Submitter(worker=plugin) as sub: - sub(nn) + assert nn.d == {"a": "ala", "b": "bala"} + + with Submitter(worker=plugin, cache_dir=tmp_path) as sub: + results = sub(nn) + assert not results.errored, "\n".join(results.errors["error message"]) # checking the results - results = nn.result() - assert results.output.out == "a:ala_b:bala" + + assert results.outputs.out == "a:ala_b:bala" # checking the output_dir - assert nn.output_dir.exists() + assert results.output_dir.exists() def test_task_nostate_4(plugin, tmp_path): @@ -481,17 +476,17 @@ def test_task_nostate_4(plugin, tmp_path): with open(file1, "w") as f: f.write("hello from pydra\n") - nn = FunFile(name="NA", filename=file1) - nn.cache_dir = tmp_path + nn = FunFile(filename=file1) - with Submitter(plugin) as sub: - sub(nn) + with Submitter(worker=plugin, cache_dir=tmp_path) as sub: + results = sub(nn) + assert not results.errored, "\n".join(results.errors["error message"]) # checking the results - results = nn.result() - assert results.output.out == "hello from pydra\n" + + assert results.outputs.out == "hello from pydra\n" # checking the output_dir - assert nn.output_dir.exists() + assert results.output_dir.exists() def test_task_nostate_5(tmp_path): @@ -504,40 +499,38 @@ def test_task_nostate_5(tmp_path): with open(file2, "w") as f: f.write("from pydra\n") - nn = FunFileList(name="NA", filename_list=[file1, file2]) + nn = FunFileList(filename_list=[file1, file2]) - nn() + outputs = nn() # checking the results - results = nn.result() - assert results.output.out == "hello from pydra\n" - # checking the output_dir - assert nn.output_dir.exists() + + assert outputs.out == "hello from pydra\n" def test_task_nostate_6(): """checking if the function gets the None value""" - nn = FunAddVarNone(name="NA", a=2, b=None) - assert nn.inputs.b is None - nn() - assert nn.result().output.out == 2 + nn = FunAddVarNone(a=2, b=None) + assert nn.b is None + outputs = nn() + assert outputs.out == 2 def test_task_nostate_6a_exception(): - """checking if the function gets the attr.Nothing value""" - nn = FunAddVarNone(name="NA", a=2) - assert nn.inputs.b is attr.NOTHING - with pytest.raises(TypeError) as excinfo: + """checking if the function gets the attrs.Nothing value""" + nn = FunAddVarNone(a=2) + assert nn.b is attrs.NOTHING + with pytest.raises(ValueError) as excinfo: nn() - assert "unsupported" in str(excinfo.value) + assert "Mandatory field 'b' is not set" in str(excinfo.value) def test_task_nostate_7(): """using the default value from the function for b input""" - nn = FunAddVarDefault(name="NA", a=2) - assert nn.inputs.b == 1 - nn() - assert nn.result().output.out == 3 + nn = FunAddVarDefault(a=2) + assert nn.b == 1 + outputs = nn() + assert outputs.out == 3 # Testing caching for tasks without states @@ -548,16 +541,18 @@ def test_task_nostate_cachedir(plugin_dask_opt, tmp_path): """task with provided cache_dir using pytest tmp_path""" cache_dir = tmp_path / "test_task_nostate" cache_dir.mkdir() - nn = FunAddTwo(name="NA", a=3, cache_dir=cache_dir) - assert np.allclose(nn.inputs.a, [3]) - assert nn.state is None + nn = FunAddTwo(a=3) + state = get_state(nn) + assert np.allclose(nn.a, [3]) + assert state is None - with Submitter(worker=plugin_dask_opt) as sub: - sub(nn) + with Submitter(worker=plugin_dask_opt, cache_dir=cache_dir) as sub: + results = sub(nn) + assert not results.errored, "\n".join(results.errors["error message"]) # checking the results - results = nn.result() - assert results.output.out == 5 + + assert results.outputs.out == 5 @pytest.mark.flaky(reruns=2) # when dask @@ -567,16 +562,18 @@ def test_task_nostate_cachedir_relativepath(tmp_path, plugin_dask_opt): cache_dir = "test_task_nostate" (tmp_path / cache_dir).mkdir() - nn = FunAddTwo(name="NA", a=3, cache_dir=cache_dir) - assert np.allclose(nn.inputs.a, [3]) - assert nn.state is None + nn = FunAddTwo(a=3) + assert np.allclose(nn.a, [3]) + state = get_state(nn) + assert state is None - with Submitter(worker=plugin_dask_opt) as sub: - sub(nn) + with Submitter(worker=plugin_dask_opt, cache_dir=cache_dir) as sub: + results = sub(nn) + assert not results.errored, "\n".join(results.errors["error message"]) # checking the results - results = nn.result() - assert results.output.out == 5 + + assert results.outputs.out == 5 shutil.rmtree(cache_dir) @@ -592,21 +589,24 @@ def test_task_nostate_cachelocations(plugin_dask_opt, tmp_path): cache_dir2 = tmp_path / "test_task_nostate2" cache_dir2.mkdir() - nn = FunAddTwo(name="NA", a=3, cache_dir=cache_dir) - with Submitter(worker=plugin_dask_opt) as sub: - sub(nn) + nn = FunAddTwo(a=3) + with Submitter(worker=plugin_dask_opt, cache_dir=cache_dir) as sub: + results = sub(nn) + assert not results.errored, "\n".join(results.errors["error message"]) - nn2 = FunAddTwo(name="NA", a=3, cache_dir=cache_dir2, cache_locations=cache_dir) - with Submitter(worker=plugin_dask_opt) as sub: - sub(nn2) + nn2 = FunAddTwo(a=3) + with Submitter( + worker=plugin_dask_opt, cache_dir=cache_dir2, cache_locations=cache_dir + ) as sub: + results2 = sub(nn2) + assert not results2.errored, "\n".join(results.errors["error message"]) # checking the results - results2 = nn2.result() - assert results2.output.out == 5 + + assert results2.outputs.out == 5 # checking if the second task didn't run the interface again - assert nn.output_dir.exists() - assert not nn2.output_dir.exists() + assert results.output_dir == results2.output_dir def test_task_nostate_cachelocations_forcererun(plugin, tmp_path): @@ -620,21 +620,24 @@ def test_task_nostate_cachelocations_forcererun(plugin, tmp_path): cache_dir2 = tmp_path / "test_task_nostate2" cache_dir2.mkdir() - nn = FunAddTwo(name="NA", a=3, cache_dir=cache_dir) - with Submitter(worker=plugin) as sub: - sub(nn) + nn = FunAddTwo(a=3) + with Submitter(worker=plugin, cache_dir=cache_dir) as sub: + results = sub(nn) + assert not results.errored, "\n".join(results.errors["error message"]) - nn2 = FunAddTwo(name="NA", a=3, cache_dir=cache_dir2, cache_locations=cache_dir) - with Submitter(worker=plugin) as sub: - sub(nn2, rerun=True) + nn2 = FunAddTwo(a=3) + with Submitter( + worker=plugin, cache_dir=cache_dir2, cache_locations=cache_dir + ) as sub: + results2 = sub(nn2, rerun=True) # checking the results - results2 = nn2.result() - assert results2.output.out == 5 + + assert results2.outputs.out == 5 # checking if the second task rerun the interface - assert nn.output_dir.exists() - assert nn2.output_dir.exists() + assert results.output_dir.exists() + assert results2.output_dir.exists() def test_task_nostate_cachelocations_nosubmitter(tmp_path): @@ -647,19 +650,19 @@ def test_task_nostate_cachelocations_nosubmitter(tmp_path): cache_dir2 = tmp_path / "test_task_nostate2" cache_dir2.mkdir() - nn = FunAddTwo(name="NA", a=3, cache_dir=cache_dir) - nn() + nn = FunAddTwo(a=3) + nn(cache_dir=cache_dir) - nn2 = FunAddTwo(name="NA", a=3, cache_dir=cache_dir2, cache_locations=cache_dir) - nn2() + nn2 = FunAddTwo(a=3) + outputs2 = nn2(cache_dir=cache_dir2, cache_locations=cache_dir) # checking the results - results2 = nn2.result() - assert results2.output.out == 5 + + assert outputs2.out == 5 # checking if the second task didn't run the interface again - assert nn.output_dir.exists() - assert not nn2.output_dir.exists() + assert num_python_cache_dirs(cache_dir) == 1 + assert not num_python_cache_dirs(cache_dir2) def test_task_nostate_cachelocations_nosubmitter_forcererun(tmp_path): @@ -673,19 +676,19 @@ def test_task_nostate_cachelocations_nosubmitter_forcererun(tmp_path): cache_dir2 = tmp_path / "test_task_nostate2" cache_dir2.mkdir() - nn = FunAddTwo(name="NA", a=3, cache_dir=cache_dir) - nn() + nn = FunAddTwo(a=3) + nn(cache_dir=cache_dir) - nn2 = FunAddTwo(name="NA", a=3, cache_dir=cache_dir2, cache_locations=cache_dir) - nn2(rerun=True) + nn2 = FunAddTwo(a=3) + outputs2 = nn2(rerun=True, cache_dir=cache_dir2, cache_locations=cache_dir) # checking the results - results2 = nn2.result() - assert results2.output.out == 5 + + assert outputs2.out == 5 # checking if the second task run the interface again - assert nn.output_dir.exists() - assert nn2.output_dir.exists() + assert num_python_cache_dirs(cache_dir) == 1 + assert num_python_cache_dirs(cache_dir2) def test_task_nostate_cachelocations_updated(plugin, tmp_path): @@ -702,22 +705,32 @@ def test_task_nostate_cachelocations_updated(plugin, tmp_path): cache_dir2 = tmp_path / "test_task_nostate2" cache_dir2.mkdir() - nn = FunAddTwo(name="NA", a=3, cache_dir=cache_dir) - with Submitter(worker=plugin) as sub: - sub(nn) + nn = FunAddTwo(a=3) + with Submitter(worker=plugin, cache_dir=cache_dir) as sub: + results = sub(nn) + assert not results.errored, "\n".join(results.errors["error message"]) + + nn2 = FunAddTwo(a=3) + with Submitter( + worker=plugin, cache_dir=cache_dir2, cache_locations=cache_dir + ) as sub: + results1 = sub(nn2) + assert not results1.errored, "\n".join(results.errors["error message"]) - nn2 = FunAddTwo(name="NA", a=3, cache_dir=cache_dir2, cache_locations=cache_dir) # updating cache location to non-existing dir - with Submitter(worker=plugin) as sub: - sub(nn2, cache_locations=cache_dir1) + with Submitter( + worker=plugin, cache_locations=cache_dir1, cache_dir=tmp_path + ) as sub: + results2 = sub(nn2) + assert not results2.errored, "\n".join(results.errors["error message"]) # checking the results - results2 = nn2.result() - assert results2.output.out == 5 + + assert results2.outputs.out == 5 # checking if both tasks run interface - assert nn.output_dir.exists() - assert nn2.output_dir.exists() + assert results.output_dir == results1.output_dir + assert results.output_dir != results2.output_dir # Tests for tasks with states (i.e. with splitter) @@ -731,90 +744,72 @@ def test_task_state_1(plugin_dask_opt, input_type, tmp_path): if input_type == "array": a_in = np.array(a_in) - nn = FunAddTwo(name="NA").split(splitter="a", a=a_in) - nn.cache_dir = tmp_path + nn = FunAddTwo().split("a", a=a_in) + state = get_state(nn) - assert nn.state.splitter == "NA.a" - assert nn.state.splitter_rpn == ["NA.a"] - assert (nn.inputs.a == np.array([3, 5])).all() + assert state.splitter == "NA.a" + assert state.splitter_rpn == ["NA.a"] + assert (nn.a == np.array([3, 5])).all() - with Submitter(worker=plugin_dask_opt) as sub: - sub(nn) + with Submitter(worker=plugin_dask_opt, cache_dir=tmp_path) as sub: + results = sub(nn) + assert not results.errored, "\n".join(results.errors["error message"]) # checking the results - results = nn.result() - expected = [({"NA.a": 3}, 5), ({"NA.a": 5}, 7)] - for i, res in enumerate(expected): - assert results[i].output.out == res[1] - # checking the return_inputs option, either return_inputs is True or "val", - # it should give values of inputs that corresponds to the specific element - results_verb = nn.result(return_inputs=True) - results_verb_val = nn.result(return_inputs="val") + expected = [({"NA.a": 3}, 5), ({"NA.a": 5}, 7)] for i, res in enumerate(expected): - assert (results_verb[i][0], results_verb[i][1].output.out) == res - assert (results_verb_val[i][0], results_verb_val[i][1].output.out) == res - - # checking the return_inputs option return_inputs="ind" - # it should give indices of inputs (instead of values) for each element - results_verb_ind = nn.result(return_inputs="ind") - expected_ind = [({"NA.a": 0}, 5), ({"NA.a": 1}, 7)] - for i, res in enumerate(expected_ind): - assert (results_verb_ind[i][0], results_verb_ind[i][1].output.out) == res - - # checking the output_dir - assert nn.output_dir - for odir in nn.output_dir: - assert odir.exists() + assert results.outputs.out[i] == res[1] def test_task_state_1a(plugin, tmp_path): """task with the simplest splitter (inputs set separately)""" - nn = FunAddTwo(name="NA") - nn.split(splitter="a", a=[1, 2]) - nn.inputs.a = StateArray([3, 5]) - nn.cache_dir = tmp_path + nn = FunAddTwo() + nn = nn.split("a", a=[1, 2]) + nn.a = StateArray([3, 5]) - assert nn.state.splitter == "NA.a" - assert nn.state.splitter_rpn == ["NA.a"] - assert (nn.inputs.a == np.array([3, 5])).all() + state = get_state(nn) - with Submitter(worker=plugin) as sub: - sub(nn) + assert state.splitter == "NA.a" + assert state.splitter_rpn == ["NA.a"] + assert (nn.a == np.array([3, 5])).all() + + with Submitter(worker=plugin, cache_dir=tmp_path) as sub: + results = sub(nn) + assert not results.errored, "\n".join(results.errors["error message"]) # checking the results - results = nn.result() + expected = [({"NA.a": 3}, 5), ({"NA.a": 5}, 7)] for i, res in enumerate(expected): - assert results[i].output.out == res[1] + assert results.outputs.out[i] == res[1] def test_task_state_singl_1(plugin, tmp_path): """Tasks with two inputs and a splitter (no combiner) one input is a single value, the other is in the splitter and combiner """ - nn = FunAddVar(name="NA").split(splitter="a", a=[3, 5], b=10) - nn.cache_dir = tmp_path + nn = FunAddVar(b=10).split("a", a=[3, 5]) + state = get_state(nn) - assert nn.inputs.a == [3, 5] - assert nn.inputs.b == 10 - assert nn.state.splitter == "NA.a" - assert nn.state.splitter_rpn == ["NA.a"] - assert nn.state.splitter_final == "NA.a" - assert nn.state.splitter_rpn_final == ["NA.a"] + assert nn.a == [3, 5] + assert nn.b == 10 + assert state.splitter == "NA.a" + assert state.splitter_rpn == ["NA.a"] + assert state.splitter_final == "NA.a" + assert state.splitter_rpn_final == ["NA.a"] - with Submitter(worker=plugin) as sub: - sub(nn) + with Submitter(worker=plugin, cache_dir=tmp_path) as sub: + results = sub(nn) + assert not results.errored, "\n".join(results.errors["error message"]) # checking the results expected = [({"NA.a": 3, "NA.b": 10}, 13), ({"NA.a": 5, "NA.b": 10}, 15)] - results = nn.result() + for i, res in enumerate(expected): - assert results[i].output.out == res[1] + assert results.outputs.out[i] == res[1] # checking the output_dir - assert nn.output_dir - for odir in nn.output_dir: - assert odir.exists() + assert results.output_dir.exists() @pytest.mark.parametrize( @@ -863,63 +858,44 @@ def test_task_state_2( a_in, b_in = np.array(a_in), np.array(b_in) elif input_type == "mixed": a_in = np.array(a_in) - nn = FunAddVar(name="NA").split(splitter=splitter, a=a_in, b=b_in) - nn.cache_dir = tmp_path + nn = FunAddVar().split(splitter, a=a_in, b=b_in) + state = get_state(nn) - assert (nn.inputs.a == np.array([3, 5])).all() - assert (nn.inputs.b == np.array([10, 20])).all() - assert nn.state.splitter == state_splitter - assert nn.state.splitter_rpn == state_rpn - assert nn.state.splitter_final == state_splitter - assert nn.state.splitter_rpn_final == state_rpn + assert (nn.a == np.array([3, 5])).all() + assert (nn.b == np.array([10, 20])).all() + assert state.splitter == state_splitter + assert state.splitter_rpn == state_rpn + assert state.splitter_final == state_splitter + assert state.splitter_rpn_final == state_rpn - with Submitter(worker=plugin) as sub: - sub(nn) + with Submitter(worker=plugin, cache_dir=tmp_path) as sub: + results = sub(nn) + assert not results.errored, "\n".join(results.errors["error message"]) # checking the results - results = nn.result() - for i, res in enumerate(expected): - assert results[i].output.out == res[1] - # checking the return_inputs option, either return_inputs is True or "val", - # it should give values of inputs that corresponds to the specific element - results_verb = nn.result(return_inputs=True) - results_verb_val = nn.result(return_inputs="val") for i, res in enumerate(expected): - assert (results_verb[i][0], results_verb[i][1].output.out) == res - assert (results_verb_val[i][0], results_verb_val[i][1].output.out) == res - - # checking the return_inputs option return_inputs="ind" - # it should give indices of inputs (instead of values) for each element - results_verb_ind = nn.result(return_inputs="ind") - for i, res in enumerate(expected_ind): - assert (results_verb_ind[i][0], results_verb_ind[i][1].output.out) == res - - # checking the output_dir - assert nn.output_dir - for odir in nn.output_dir: - assert odir.exists() + assert results.outputs.out[i] == res[1] def test_task_state_3(plugin, tmp_path): """task with the simplest splitter, the input is an empty list""" - nn = FunAddTwo(name="NA").split(splitter="a", a=[]) - nn.cache_dir = tmp_path + nn = FunAddTwo().split("a", a=[]) + state = get_state(nn) - assert nn.state.splitter == "NA.a" - assert nn.state.splitter_rpn == ["NA.a"] - assert nn.inputs.a == [] + assert state.splitter == "NA.a" + assert state.splitter_rpn == ["NA.a"] + assert nn.a == [] - with Submitter(worker=plugin) as sub: - sub(nn) + with Submitter(worker=plugin, cache_dir=tmp_path) as sub: + results = sub(nn) + assert not results.errored, "\n".join(results.errors["error message"]) # checking the results - results = nn.result() + expected = [] for i, res in enumerate(expected): - assert results[i].output.out == res[1] - # checking the output_dir - assert nn.output_dir == [] + assert results.outputs.out[i] == res[1] @pytest.mark.parametrize("input_type", ["list", "array"]) @@ -928,196 +904,151 @@ def test_task_state_4(plugin, input_type, tmp_path): lst_in = [[2, 3, 4], [1, 2, 3]] if input_type == "array": lst_in = np.array(lst_in, dtype=int) - nn = Moment(name="NA", n=3).split(splitter="lst", lst=lst_in) - nn.cache_dir = tmp_path + nn = Moment(n=3).split("lst", lst=lst_in) + state = get_state(nn) - assert np.allclose(nn.inputs.n, 3) - assert np.allclose(nn.inputs.lst, [[2, 3, 4], [1, 2, 3]]) - assert nn.state.splitter == "NA.lst" + assert np.allclose(nn.n, 3) + assert np.allclose(nn.lst, [[2, 3, 4], [1, 2, 3]]) + assert state.splitter == "NA.lst" - with Submitter(worker=plugin) as sub: - sub(nn) + with Submitter(worker=plugin, cache_dir=tmp_path) as sub: + results = sub(nn) + assert not results.errored, "\n".join(results.errors["error message"]) # checking that split is done across dim 0 - el_0 = nn.state.states_val[0]["NA.lst"] + el_0 = state.states_val[0]["NA.lst"] if input_type == "list": assert el_0 == [2, 3, 4] elif input_type == "array": assert el_0 == [2, 3, 4] # checking the results - results = nn.result() + for i, expected in enumerate([33, 12]): - assert results[i].output.out == expected - # checking the output_dir - assert nn.output_dir - for odir in nn.output_dir: - assert odir.exists() + assert results.outputs.out[i] == expected def test_task_state_4a(plugin, tmp_path): """task with a tuple as an input, and a simple splitter""" - nn = Moment(name="NA", n=3).split(splitter="lst", lst=[(2, 3, 4), (1, 2, 3)]) - nn.cache_dir = tmp_path + nn = Moment(n=3).split("lst", lst=[(2, 3, 4), (1, 2, 3)]) + state = get_state(nn) - assert np.allclose(nn.inputs.n, 3) - assert np.allclose(nn.inputs.lst, [[2, 3, 4], [1, 2, 3]]) - assert nn.state.splitter == "NA.lst" + assert np.allclose(nn.n, 3) + assert np.allclose(nn.lst, [[2, 3, 4], [1, 2, 3]]) + assert state.splitter == "NA.lst" - with Submitter(worker=plugin) as sub: - sub(nn) + with Submitter(worker=plugin, cache_dir=tmp_path) as sub: + results = sub(nn) + assert not results.errored, "\n".join(results.errors["error message"]) # checking the results - results = nn.result() + for i, expected in enumerate([33, 12]): - assert results[i].output.out == expected - # checking the output_dir - assert nn.output_dir - for odir in nn.output_dir: - assert odir.exists() + assert results.outputs.out[i] == expected def test_task_state_5(plugin, tmp_path): """task with a list as an input, and the variable is part of the scalar splitter""" - nn = Moment(name="NA").split( - splitter=("n", "lst"), n=[1, 3], lst=[[2, 3, 4], [1, 2, 3]] - ) - nn.cache_dir = tmp_path + nn = Moment().split(("n", "lst"), n=[1, 3], lst=[[2, 3, 4], [1, 2, 3]]) + state = get_state(nn) - assert np.allclose(nn.inputs.n, [1, 3]) - assert np.allclose(nn.inputs.lst, [[2, 3, 4], [1, 2, 3]]) - assert nn.state.splitter == ("NA.n", "NA.lst") + assert np.allclose(nn.n, [1, 3]) + assert np.allclose(nn.lst, [[2, 3, 4], [1, 2, 3]]) + assert state.splitter == ("NA.n", "NA.lst") - with Submitter(worker=plugin) as sub: - sub(nn) + with Submitter(worker=plugin, cache_dir=tmp_path) as sub: + results = sub(nn) + assert not results.errored, "\n".join(results.errors["error message"]) # checking the results - results = nn.result() + for i, expected in enumerate([3, 12]): - assert results[i].output.out == expected - # checking the output_dir - assert nn.output_dir - for odir in nn.output_dir: - assert odir.exists() + assert results.outputs.out[i] == expected def test_task_state_5_exception(plugin, tmp_path): """task with a list as an input, and the variable is part of the scalar splitter the shapes are not matching, so exception should be raised """ - nn = Moment(name="NA").split( - splitter=("n", "lst"), n=[1, 3, 3], lst=[[2, 3, 4], [1, 2, 3]] - ) - nn.cache_dir = tmp_path + nn = Moment().split(("n", "lst"), n=[1, 3, 3], lst=[[2, 3, 4], [1, 2, 3]]) - assert np.allclose(nn.inputs.n, [1, 3, 3]) - assert np.allclose(nn.inputs.lst, [[2, 3, 4], [1, 2, 3]]) - assert nn.state.splitter == ("NA.n", "NA.lst") + assert np.allclose(nn.n, [1, 3, 3]) + assert np.allclose(nn.lst, [[2, 3, 4], [1, 2, 3]]) with pytest.raises(Exception) as excinfo: - with Submitter(worker=plugin) as sub: - sub(nn) + get_state(nn) + assert "shape" in str(excinfo.value) def test_task_state_6(plugin, tmp_path): """ask with a list as an input, and the variable is part of the outer splitter""" - nn = Moment(name="NA").split( - splitter=["n", "lst"], n=[1, 3], lst=[[2, 3, 4], [1, 2, 3]] - ) - nn.cache_dir = tmp_path + nn = Moment().split(["n", "lst"], n=[1, 3], lst=[[2, 3, 4], [1, 2, 3]]) + state = get_state(nn) - assert np.allclose(nn.inputs.n, [1, 3]) - assert np.allclose(nn.inputs.lst, [[2, 3, 4], [1, 2, 3]]) - assert nn.state.splitter == ["NA.n", "NA.lst"] + assert np.allclose(nn.n, [1, 3]) + assert np.allclose(nn.lst, [[2, 3, 4], [1, 2, 3]]) + assert state.splitter == ["NA.n", "NA.lst"] - with Submitter(worker=plugin) as sub: - sub(nn) + with Submitter(worker=plugin, cache_dir=tmp_path) as sub: + results = sub(nn) + assert not results.errored, "\n".join(results.errors["error message"]) # checking the results - results = nn.result() + for i, expected in enumerate([3, 2, 33, 12]): - assert results[i].output.out == expected - # checking the output_dir - assert nn.output_dir - for odir in nn.output_dir: - assert odir.exists() + assert results.outputs.out[i] == expected def test_task_state_6a(plugin, tmp_path): """ask with a tuple as an input, and the variable is part of the outer splitter""" - nn = Moment(name="NA").split( - splitter=["n", "lst"], n=[1, 3], lst=[(2, 3, 4), (1, 2, 3)] - ) - nn.cache_dir = tmp_path + nn = Moment().split(["n", "lst"], n=[1, 3], lst=[(2, 3, 4), (1, 2, 3)]) + state = get_state(nn) - assert np.allclose(nn.inputs.n, [1, 3]) - assert np.allclose(nn.inputs.lst, [[2, 3, 4], [1, 2, 3]]) - assert nn.state.splitter == ["NA.n", "NA.lst"] + assert np.allclose(nn.n, [1, 3]) + assert np.allclose(nn.lst, [[2, 3, 4], [1, 2, 3]]) + assert state.splitter == ["NA.n", "NA.lst"] - with Submitter(worker=plugin) as sub: - sub(nn) + with Submitter(worker=plugin, cache_dir=tmp_path) as sub: + results = sub(nn) + assert not results.errored, "\n".join(results.errors["error message"]) # checking the results - results = nn.result() + for i, expected in enumerate([3, 2, 33, 12]): - assert results[i].output.out == expected - # checking the output_dir - assert nn.output_dir - for odir in nn.output_dir: - assert odir.exists() + assert results.outputs.out[i] == expected @pytest.mark.flaky(reruns=2) # when dask def test_task_state_comb_1(plugin_dask_opt, tmp_path): """task with the simplest splitter and combiner""" - nn = FunAddTwo(name="NA").split(a=[3, 5], splitter="a").combine(combiner="a") - nn.cache_dir = tmp_path + nn = FunAddTwo().split(a=[3, 5]).combine(combiner="a") + state = get_state(nn) - assert (nn.inputs.a == np.array([3, 5])).all() + assert (nn.a == np.array([3, 5])).all() - assert nn.state.splitter == "NA.a" - assert nn.state.splitter_rpn == ["NA.a"] - assert nn.state.combiner == ["NA.a"] - assert nn.state.splitter_final is None - assert nn.state.splitter_rpn_final == [] + assert state.splitter == ["NA.a"] + assert state.splitter_rpn == ["NA.a"] + assert state.combiner == ["NA.a"] + assert state.splitter_final is None + assert state.splitter_rpn_final == [] - with Submitter(worker=plugin_dask_opt) as sub: - sub(nn) + with Submitter(worker=plugin_dask_opt, cache_dir=tmp_path) as sub: + results = sub(nn) + assert not results.errored, "\n".join(results.errors["error message"]) - assert nn.state.states_ind == [{"NA.a": 0}, {"NA.a": 1}] - assert nn.state.states_val == [{"NA.a": 3}, {"NA.a": 5}] + assert state.states_ind == [{"NA.a": 0}, {"NA.a": 1}] + assert state.states_val == [{"NA.a": 3}, {"NA.a": 5}] # checking the results - results = nn.result() - # fully combined (no nested list) - combined_results = [res.output.out for res in results] - assert combined_results == [5, 7] - expected = [({"NA.a": 3}, 5), ({"NA.a": 5}, 7)] - expected_ind = [({"NA.a": 0}, 5), ({"NA.a": 1}, 7)] - # checking the return_inputs option, either return_inputs is True or "val", - # it should give values of inputs that corresponds to the specific element - results_verb = nn.result(return_inputs=True) - results_verb_val = nn.result(return_inputs="val") - for i, res in enumerate(expected): - assert (results_verb[i][0], results_verb[i][1].output.out) == res - assert (results_verb_val[i][0], results_verb_val[i][1].output.out) == res - # checking the return_inputs option return_inputs="ind" - # it should give indices of inputs (instead of values) for each element - results_verb_ind = nn.result(return_inputs="ind") - for i, res in enumerate(expected_ind): - assert (results_verb_ind[i][0], results_verb_ind[i][1].output.out) == res - - # checking the output_dir - assert nn.output_dir - for odir in nn.output_dir: - assert odir.exists() + # fully combined (no nested list) + assert results.outputs.out == [5, 7] @pytest.mark.parametrize( "splitter, combiner, state_splitter, state_rpn, state_combiner, state_combiner_all, " - "state_splitter_final, state_rpn_final, expected, expected_val", + "state_splitter_final, state_rpn_final, expected", # , expected_val", [ ( ("a", "b"), @@ -1129,7 +1060,7 @@ def test_task_state_comb_1(plugin_dask_opt, tmp_path): None, [], [13, 25], - [({"NA.a": 3, "NA.b": 10}, 13), ({"NA.a": 5, "NA.b": 20}, 25)], + # [({"NA.a": 3, "NA.b": 10}, 13), ({"NA.a": 5, "NA.b": 20}, 25)], ), ( ("a", "b"), @@ -1141,7 +1072,7 @@ def test_task_state_comb_1(plugin_dask_opt, tmp_path): None, [], [13, 25], - [({"NA.a": 3, "NA.b": 10}, 13), ({"NA.a": 5, "NA.b": 20}, 25)], + # [({"NA.a": 3, "NA.b": 10}, 13), ({"NA.a": 5, "NA.b": 20}, 25)], ), ( ["a", "b"], @@ -1153,10 +1084,10 @@ def test_task_state_comb_1(plugin_dask_opt, tmp_path): "NA.b", ["NA.b"], [[13, 15], [23, 25]], - [ - [({"NA.a": 3, "NA.b": 10}, 13), ({"NA.a": 5, "NA.b": 10}, 15)], - [({"NA.a": 3, "NA.b": 20}, 23), ({"NA.a": 5, "NA.b": 20}, 25)], - ], + # [ + # [({"NA.a": 3, "NA.b": 10}, 13), ({"NA.a": 5, "NA.b": 10}, 15)], + # [({"NA.a": 3, "NA.b": 20}, 23), ({"NA.a": 5, "NA.b": 20}, 25)], + # ], ), ( ["a", "b"], @@ -1168,10 +1099,10 @@ def test_task_state_comb_1(plugin_dask_opt, tmp_path): "NA.a", ["NA.a"], [[13, 23], [15, 25]], - [ - [({"NA.a": 3, "NA.b": 10}, 13), ({"NA.a": 3, "NA.b": 20}, 23)], - [({"NA.a": 5, "NA.b": 10}, 15), ({"NA.a": 5, "NA.b": 20}, 25)], - ], + # [ + # [({"NA.a": 3, "NA.b": 10}, 13), ({"NA.a": 3, "NA.b": 20}, 23)], + # [({"NA.a": 5, "NA.b": 10}, 15), ({"NA.a": 5, "NA.b": 20}, 25)], + # ], ), ( ["a", "b"], @@ -1183,12 +1114,12 @@ def test_task_state_comb_1(plugin_dask_opt, tmp_path): None, [], [13, 23, 15, 25], - [ - ({"NA.a": 3, "NA.b": 10}, 13), - ({"NA.a": 3, "NA.b": 20}, 23), - ({"NA.a": 5, "NA.b": 10}, 15), - ({"NA.a": 5, "NA.b": 20}, 25), - ], + # [ + # ({"NA.a": 3, "NA.b": 10}, 13), + # ({"NA.a": 3, "NA.b": 20}, 23), + # ({"NA.a": 5, "NA.b": 10}, 15), + # ({"NA.a": 5, "NA.b": 20}, 25), + # ], ), ], ) @@ -1203,160 +1134,132 @@ def test_task_state_comb_2( state_splitter_final, state_rpn_final, expected, - expected_val, + # expected_val, tmp_path, ): """Tasks with scalar and outer splitters and partial or full combiners""" - nn = ( - FunAddVar(name="NA") - .split(a=[3, 5], b=[10, 20], splitter=splitter) - .combine(combiner=combiner) - ) - nn.cache_dir = tmp_path + nn = FunAddVar().split(splitter, a=[3, 5], b=[10, 20]).combine(combiner=combiner) + state = get_state(nn) - assert (nn.inputs.a == np.array([3, 5])).all() + assert (nn.a == np.array([3, 5])).all() - assert nn.state.splitter == state_splitter - assert nn.state.splitter_rpn == state_rpn - assert nn.state.combiner == state_combiner + assert state.splitter == state_splitter + assert state.splitter_rpn == state_rpn + assert state.combiner == state_combiner - with Submitter(worker=plugin) as sub: - sub(nn) + with Submitter(worker=plugin, cache_dir=tmp_path) as sub: + results = sub(nn) + assert not results.errored, "\n".join(results.errors["error message"]) - assert nn.state.splitter_final == state_splitter_final - assert nn.state.splitter_rpn_final == state_rpn_final - assert set(nn.state.current_combiner_all) == set(state_combiner_all) + assert state.splitter_final == state_splitter_final + assert state.splitter_rpn_final == state_rpn_final + assert set(state.current_combiner_all) == set(state_combiner_all) # checking the results - results = nn.result() + # checking the return_inputs option, either return_inputs is True or "val", # it should give values of inputs that corresponds to the specific element - results_verb = nn.result(return_inputs=True) + # results_verb = nn.result(return_inputs=True) - if nn.state.splitter_rpn_final: + if state.splitter_rpn_final: for i, res in enumerate(expected): - assert [res.output.out for res in results[i]] == res + assert results.outputs.out == res # results_verb - for i, res_l in enumerate(expected_val): - for j, res in enumerate(res_l): - assert (results_verb[i][j][0], results_verb[i][j][1].output.out) == res + # for i, res_l in enumerate(expected_val): + # for j, res in enumerate(res_l): + # assert (results_verb[i][j][0], results_verb[i][j][1].output.out) == res # if the combiner is full expected is "a flat list" else: - assert [res.output.out for res in results] == expected - for i, res in enumerate(expected_val): - assert (results_verb[i][0], results_verb[i][1].output.out) == res - - # checking the output_dir - assert nn.output_dir - for odir in nn.output_dir: - assert odir.exists() + assert results.outputs.out == expected + # for i, res in enumerate(expected_val): + # assert (results_verb[i][0], results_verb[i][1].output.out) == res def test_task_state_comb_singl_1(plugin, tmp_path): """Tasks with two inputs; one input is a single value, the other is in the splitter and combiner """ - nn = FunAddVar(name="NA").split(splitter="a", a=[3, 5], b=10).combine(combiner="a") - nn.cache_dir = tmp_path - - assert nn.inputs.a == [3, 5] - assert nn.inputs.b == 10 - assert nn.state.splitter == "NA.a" - assert nn.state.splitter_rpn == ["NA.a"] - assert nn.state.combiner == ["NA.a"] - assert nn.state.splitter_final is None - assert nn.state.splitter_rpn_final == [] - - with Submitter(worker=plugin) as sub: - sub(nn) + nn = FunAddVar(b=10).split("a", a=[3, 5]).combine(combiner="a") + state = get_state(nn) - # checking the results - expected = ({}, [13, 15]) - results = nn.result() - # full combiner, no nested list - combined_results = [res.output.out for res in results] - assert combined_results == expected[1] - # checking the output_dir - assert nn.output_dir - for odir in nn.output_dir: - assert odir.exists() + assert nn.a == [3, 5] + assert nn.b == 10 + assert state.splitter == "NA.a" + assert state.splitter_rpn == ["NA.a"] + assert state.combiner == ["NA.a"] + assert state.splitter_final is None + assert state.splitter_rpn_final == [] + + with Submitter(worker=plugin, cache_dir=tmp_path) as sub: + results = sub(nn) + assert not results.errored, "\n".join(results.errors["error message"]) + + assert results.outputs.out == [13, 15] def test_task_state_comb_3(plugin, tmp_path): """task with the simplest splitter, the input is an empty list""" - nn = FunAddTwo(name="NA").split(splitter="a", a=[]).combine(combiner=["a"]) - nn.cache_dir = tmp_path + nn = FunAddTwo().split("a", a=[]).combine(combiner=["a"]) + state = get_state(nn) - assert nn.state.splitter == "NA.a" - assert nn.state.splitter_rpn == ["NA.a"] - assert nn.inputs.a == [] + assert state.splitter == "NA.a" + assert state.splitter_rpn == ["NA.a"] + assert nn.a == [] - with Submitter(worker=plugin) as sub: - sub(nn) + with Submitter(worker=plugin, cache_dir=tmp_path) as sub: + results = sub(nn) + assert not results.errored, "\n".join(results.errors["error message"]) # checking the results - results = nn.result() + expected = [] for i, res in enumerate(expected): - assert results[i].output.out == res[1] - # checking the output_dir - assert nn.output_dir == [] + assert results.outputs.out[i] == res[1] -def test_task_state_comb_order(): +def test_task_state_comb_order(tmp_path): """tasks with an outer splitter and various combiner; showing the order of results """ # single combiner "a" - will create two lists, first one for b=3, second for b=5 - nn_a = ( - FunAddVar(name="NA") - .split(splitter=["a", "b"], a=[10, 20], b=[3, 5]) - .combine(combiner="a") - ) - assert nn_a.state.combiner == ["NA.a"] + nn_a = FunAddVar().split(["a", "b"], a=[10, 20], b=[3, 5]).combine(combiner="a") + state_a = get_state(nn_a) + assert state_a.combiner == ["NA.a"] - results_a = nn_a() - combined_results_a = [[res.output.out for res in res_l] for res_l in results_a] - assert combined_results_a == [[13, 23], [15, 25]] + outputs = nn_a(cache_dir=tmp_path / "cache") + # combined_results_a = [[res.output.out for res in res_l] for res_l in results_a] + assert outputs.out == [[13, 23], [15, 25]] # single combiner "b" - will create two lists, first one for a=10, second for a=20 - nn_b = ( - FunAddVar(name="NA") - .split(splitter=["a", "b"], a=[10, 20], b=[3, 5]) - .combine(combiner="b") - ) - assert nn_b.state.combiner == ["NA.b"] + nn_b = FunAddVar().split(["a", "b"], a=[10, 20], b=[3, 5]).combine(combiner="b") + state_b = get_state(nn_b) + assert state_b.combiner == ["NA.b"] - results_b = nn_b() - combined_results_b = [[res.output.out for res in res_l] for res_l in results_b] - assert combined_results_b == [[13, 15], [23, 25]] + outputs_b = nn_b(cache_dir=tmp_path / "cache_b") + # combined_results_b = [[res.output.out for res in res_l] for res_l in results_b] + assert outputs_b.out == [[13, 15], [23, 25]] # combiner with both fields ["a", "b"] - will create one list nn_ab = ( - FunAddVar(name="NA") - .split(splitter=["a", "b"], a=[10, 20], b=[3, 5]) - .combine(combiner=["a", "b"]) + FunAddVar().split(["a", "b"], a=[10, 20], b=[3, 5]).combine(combiner=["a", "b"]) ) - assert nn_ab.state.combiner == ["NA.a", "NA.b"] + state_ab = get_state(nn_ab) + assert state_ab.combiner == ["NA.a", "NA.b"] - results_ab = nn_ab() - # full combiner, no nested list - combined_results_ab = [res.output.out for res in results_ab] - assert combined_results_ab == [13, 15, 23, 25] + outputs_ab = nn_ab(cache_dir=tmp_path / "cache_ab") + assert outputs_ab.out == [13, 15, 23, 25] # combiner with both fields ["b", "a"] - will create the same list as nn_ab # no difference in the order for setting combiner nn_ba = ( - FunAddVar(name="NA") - .split(splitter=["a", "b"], a=[10, 20], b=[3, 5]) - .combine(combiner=["b", "a"]) + FunAddVar().split(["a", "b"], a=[10, 20], b=[3, 5]).combine(combiner=["b", "a"]) ) - assert nn_ba.state.combiner == ["NA.b", "NA.a"] + state_ba = get_state(nn_ba) + assert state_ba.combiner == ["NA.b", "NA.a"] - results_ba = nn_ba() - combined_results_ba = [res.output.out for res in results_ba] - assert combined_results_ba == [13, 15, 23, 25] + outputs_ba = nn_ba(cache_dir=tmp_path / "cache_ba") + assert outputs_ba.out == [13, 15, 23, 25] # Testing with container dimensions for the input @@ -1365,30 +1268,22 @@ def test_task_state_comb_order(): def test_task_state_contdim_1(tmp_path): """task with a spliter and container dimension for one of the value""" task_4var = Op4Var( - name="op_4var", a="a1", - cache_dir=tmp_path, - ) - task_4var.split( + ).split( ("b", ["c", "d"]), b=[["b1", "b2"], ["b3", "b4"]], c=["c1", "c2"], d=["d1", "d2"], cont_dim={"b": 2}, ) - task_4var() - res = task_4var.result() - assert len(res) == 4 - assert res[3].output.out == "a1 b4 c2 d2" + outputs = task_4var(cache_dir=tmp_path) + assert len(outputs.out) == 4 + assert outputs.out[3] == "a1 b4 c2 d2" def test_task_state_contdim_2(tmp_path): """task with a splitter and container dimension for one of the value""" - task_4var = Op4Var( - name="op_4var", - cache_dir=tmp_path, - ) - task_4var.split( + task_4var = Op4Var().split( ["a", ("b", ["c", "d"])], cont_dim={"b": 2}, a=["a1", "a2"], @@ -1396,50 +1291,46 @@ def test_task_state_contdim_2(tmp_path): c=["c1", "c2"], d=["d1", "d2"], ) - task_4var() - res = task_4var.result() - assert len(res) == 8 - assert res[7].output.out == "a2 b4 c2 d2" + outputs = task_4var(cache_dir=tmp_path) + assert len(outputs.out) == 8 + assert outputs.out[7] == "a2 b4 c2 d2" def test_task_state_comb_contdim_1(tmp_path): """task with a splitter-combiner, and container dimension for one of the value""" - task_4var = Op4Var( - name="op_4var", - a="a1", - cache_dir=tmp_path, + task_4var = ( + Op4Var(a="a1") + .split( + ("b", ["c", "d"]), + cont_dim={"b": 2}, + b=[["b1", "b2"], ["b3", "b4"]], + c=["c1", "c2"], + d=["d1", "d2"], + ) + .combine("b") ) - task_4var.split( - ("b", ["c", "d"]), - cont_dim={"b": 2}, - b=[["b1", "b2"], ["b3", "b4"]], - c=["c1", "c2"], - d=["d1", "d2"], - ).combine("b") - task_4var() - res = task_4var.result() - assert len(res) == 4 - assert res[3].output.out == "a1 b4 c2 d2" + outputs = task_4var(cache_dir=tmp_path) + assert len(outputs.out) == 4 + assert outputs.out[3] == "a1 b4 c2 d2" def test_task_state_comb_contdim_2(tmp_path): """task with a splitter-combiner, and container dimension for one of the value""" - task_4var = Op4Var( - name="op_4var", - cache_dir=tmp_path, + task_4var = ( + Op4Var() + .split( + ["a", ("b", ["c", "d"])], + a=["a1", "a2"], + b=[["b1", "b2"], ["b3", "b4"]], + c=["c1", "c2"], + d=["d1", "d2"], + cont_dim={"b": 2}, + ) + .combine("a") ) - task_4var.split( - ["a", ("b", ["c", "d"])], - a=["a1", "a2"], - b=[["b1", "b2"], ["b3", "b4"]], - c=["c1", "c2"], - d=["d1", "d2"], - cont_dim={"b": 2}, - ).combine("a") - task_4var() - res = task_4var.result() - assert len(res) == 4 - assert res[3][1].output.out == "a2 b4 c2 d2" + outputs = task_4var(cache_dir=tmp_path) + assert len(outputs.out) == 4 + assert outputs.out[3][1] == "a2 b4 c2 d2" # Testing caching for tasks with states @@ -1450,19 +1341,21 @@ def test_task_state_cachedir(plugin_dask_opt, tmp_path): """task with a state and provided cache_dir using pytest tmp_path""" cache_dir = tmp_path / "test_task_nostate" cache_dir.mkdir() - nn = FunAddTwo(name="NA", cache_dir=cache_dir).split(splitter="a", a=[3, 5]) + nn = FunAddTwo().split("a", a=[3, 5]) + state = get_state(nn) - assert nn.state.splitter == "NA.a" - assert (nn.inputs.a == np.array([3, 5])).all() + assert state.splitter == "NA.a" + assert (nn.a == np.array([3, 5])).all() - with Submitter(worker=plugin_dask_opt) as sub: - sub(nn) + with Submitter(worker=plugin_dask_opt, cache_dir=cache_dir) as sub: + results = sub(nn) + assert not results.errored, "\n".join(results.errors["error message"]) # checking the results - results = nn.result() + expected = [({"NA.a": 3}, 5), ({"NA.a": 5}, 7)] for i, res in enumerate(expected): - assert results[i].output.out == res[1] + assert results.outputs.out[i] == res[1] def test_task_state_cachelocations(plugin, tmp_path): @@ -1475,24 +1368,25 @@ def test_task_state_cachelocations(plugin, tmp_path): cache_dir2 = tmp_path / "test_task_nostate2" cache_dir2.mkdir() - nn = FunAddTwo(name="NA", a=3, cache_dir=cache_dir).split(splitter="a", a=[3, 5]) - with Submitter(worker=plugin) as sub: + nn = FunAddTwo(a=3).split("a", a=[3, 5]) + with Submitter(worker=plugin, cache_dir=cache_dir) as sub: sub(nn) - nn2 = FunAddTwo( - name="NA", a=3, cache_dir=cache_dir2, cache_locations=cache_dir - ).split(splitter="a", a=[3, 5]) - with Submitter(worker=plugin) as sub: - sub(nn2) + nn2 = FunAddTwo(a=3).split("a", a=[3, 5]) + with Submitter( + worker=plugin, cache_dir=cache_dir2, cache_locations=cache_dir + ) as sub: + results2 = sub(nn2) + assert not results2.errored, "\n".join(results.errors["error message"]) # checking the results - results2 = nn2.result() expected = [({"NA.a": 3}, 5), ({"NA.a": 5}, 7)] for i, res in enumerate(expected): - assert results2[i].output.out == res[1] + assert results2.outputs.out[i] == res[1] - assert all([dir.exists() for dir in nn.output_dir]) - assert not any([dir.exists() for dir in nn2.output_dir]) + # Would ideally check for all nodes of the workflows + assert num_python_cache_dirs(cache_dir) == 2 + assert not num_python_cache_dirs(cache_dir2) def test_task_state_cachelocations_forcererun(plugin, tmp_path): @@ -1506,25 +1400,25 @@ def test_task_state_cachelocations_forcererun(plugin, tmp_path): cache_dir2 = tmp_path / "test_task_nostate2" cache_dir2.mkdir() - nn = FunAddTwo(name="NA", a=3, cache_dir=cache_dir).split(splitter="a", a=[3, 5]) - with Submitter(worker=plugin) as sub: + nn = FunAddTwo(a=3).split("a", a=[3, 5]) + with Submitter(worker=plugin, cache_dir=cache_dir) as sub: sub(nn) - nn2 = FunAddTwo( - name="NA", a=3, cache_dir=cache_dir2, cache_locations=cache_dir - ).split(splitter="a", a=[3, 5]) - with Submitter(worker=plugin) as sub: - sub(nn2, rerun=True) + nn2 = FunAddTwo(a=3).split("a", a=[3, 5]) + with Submitter( + worker=plugin, cache_dir=cache_dir2, cache_locations=cache_dir + ) as sub: + results2 = sub(nn2, rerun=True) # checking the results - results2 = nn2.result() + expected = [({"NA.a": 3}, 5), ({"NA.a": 5}, 7)] for i, res in enumerate(expected): - assert results2[i].output.out == res[1] + assert results2.outputs.out[i] == res[1] # both workflows should be run - assert all([dir.exists() for dir in nn.output_dir]) - assert all([dir.exists() for dir in nn2.output_dir]) + assert num_python_cache_dirs(cache_dir) == 2 + assert num_python_cache_dirs(cache_dir2) == 2 def test_task_state_cachelocations_updated(plugin, tmp_path): @@ -1541,25 +1435,26 @@ def test_task_state_cachelocations_updated(plugin, tmp_path): cache_dir2 = tmp_path / "test_task_nostate2" cache_dir2.mkdir() - nn = FunAddTwo(name="NA", cache_dir=cache_dir).split(splitter="a", a=[3, 5]) - with Submitter(worker=plugin) as sub: + nn = FunAddTwo().split("a", a=[3, 5]) + with Submitter(worker=plugin, cache_dir=cache_dir) as sub: sub(nn) - nn2 = FunAddTwo(name="NA", cache_dir=cache_dir2, cache_locations=cache_dir).split( - splitter="a", a=[3, 5] - ) - with Submitter(worker=plugin) as sub: - sub(nn2, cache_locations=cache_dir1) + nn2 = FunAddTwo().split("a", a=[3, 5]) + with Submitter( + worker=plugin, cache_dir=cache_dir2, cache_locations=cache_dir1 + ) as sub: + results2 = sub(nn2) + assert not results2.errored, "\n".join(results.errors["error message"]) # checking the results - results2 = nn2.result() + expected = [({"NA.a": 3}, 5), ({"NA.a": 5}, 7)] for i, res in enumerate(expected): - assert results2[i].output.out == res[1] + assert results2.outputs.out[i] == res[1] # both workflows should be run - assert all([dir.exists() for dir in nn.output_dir]) - assert all([dir.exists() for dir in nn2.output_dir]) + assert num_python_cache_dirs(cache_dir) == 2 + assert num_python_cache_dirs(cache_dir2) == 2 def test_task_files_cachelocations(plugin_dask_opt, tmp_path): @@ -1579,23 +1474,24 @@ def test_task_files_cachelocations(plugin_dask_opt, tmp_path): input2 = input_dir / "input2.txt" input2.write_text("test") - nn = FunFile(name="NA", filename=input1, cache_dir=cache_dir) - with Submitter(worker=plugin_dask_opt) as sub: - sub(nn) + nn = FunFile(filename=input1) + with Submitter(worker=plugin_dask_opt, cache_dir=cache_dir) as sub: + results = sub(nn) + assert not results.errored, "\n".join(results.errors["error message"]) - nn2 = FunFile( - name="NA", filename=input2, cache_dir=cache_dir2, cache_locations=cache_dir - ) - with Submitter(worker=plugin_dask_opt) as sub: - sub(nn2) + nn2 = FunFile(filename=input2) + with Submitter( + worker=plugin_dask_opt, cache_dir=cache_dir2, cache_locations=cache_dir + ) as sub: + results2 = sub(nn2) + assert not results2.errored, "\n".join(results.errors["error message"]) # checking the results - results2 = nn2.result() - assert results2.output.out == "test" + + assert results2.outputs.out == "test" # checking if the second task didn't run the interface again - assert nn.output_dir.exists() - assert not nn2.output_dir.exists() + assert results.output_dir == results2.output_dir class OverriddenContentsFile(File): @@ -1618,10 +1514,10 @@ def byte_chunks(self, **kwargs) -> ty.Generator[ty.Tuple[str, bytes], None, None yield from super().byte_chunks(**kwargs) @property - def contents(self): + def raw_contents(self): if self._contents is not None: return self._contents - return super().contents + return super().raw_contents def test_task_files_persistentcache(tmp_path): @@ -1637,21 +1533,14 @@ def test_task_files_persistentcache(tmp_path): @python.define def read_contents(x: OverriddenContentsFile) -> bytes: - return x.contents + return x.raw_contents - assert ( - read_contents(x=test_file, cache_dir=cache_dir)(plugin="serial").output.out - == b"foo" - ) + assert read_contents(x=test_file)(cache_dir=cache_dir).out == b"foo" test_file._contents = b"bar" # should return result from the first run using the persistent cache - assert ( - read_contents(x=test_file, cache_dir=cache_dir)(plugin="serial").output.out - == b"foo" - ) + assert read_contents(x=test_file)(cache_dir=cache_dir).out == b"foo" time.sleep(2) # Windows has a 2-second resolution for mtime test_file_path.touch() # update the mtime to invalidate the persistent cache value assert ( - read_contents(x=test_file, cache_dir=cache_dir)(plugin="serial").output.out - == b"bar" + read_contents(x=test_file)(cache_dir=cache_dir).out == b"bar" ) # returns the overridden value diff --git a/pydra/engine/tests/test_shelltask.py b/pydra/engine/tests/test_shelltask.py index 17dec45a0..3e362a9a7 100644 --- a/pydra/engine/tests/test_shelltask.py +++ b/pydra/engine/tests/test_shelltask.py @@ -6,10 +6,9 @@ from pathlib import Path import re import stat - -from ..task import ShellDef -from ..submitter import Submitter -from ..specs import ( +from pydra.engine.submitter import Submitter +from pydra.design import shell, workflow +from pydra.engine.specs import ( ShellOutputs, ShellDef, ) @@ -20,8 +19,9 @@ from pydra.utils.typing import ( MultiOutputFile, MultiInputObj, + StateArray, ) -from .utils import run_no_submitter, run_submitter, no_win +from .utils import run_no_submitter, run_submitter, no_win, get_output_names if sys.platform.startswith("win"): pytest.skip("SLURM not available in windows", allow_module_level=True) @@ -32,13 +32,13 @@ def test_shell_cmd_1(plugin_dask_opt, results_function, tmp_path): """simple command, no arguments""" cmd = ["pwd"] - shelly = ShellDef(name="shelly", executable=cmd, cache_dir=tmp_path) + shelly = shell.define(cmd)() assert shelly.cmdline == " ".join(cmd) - res = results_function(shelly, plugin=plugin_dask_opt) - assert Path(res.output.stdout.rstrip()) == shelly.output_dir - assert res.output.return_code == 0 - assert res.output.stderr == "" + outputs = results_function(shelly, plugin=plugin_dask_opt, cache_dir=tmp_path) + assert Path(outputs.stdout.rstrip()).parent == tmp_path + assert outputs.return_code == 0 + assert outputs.stderr == "" @pytest.mark.parametrize("results_function", [run_no_submitter, run_submitter]) @@ -47,28 +47,28 @@ def test_shell_cmd_1_strip(plugin, results_function, tmp_path): strip option to remove \n at the end os stdout """ cmd = ["pwd"] - shelly = ShellDef(name="shelly", executable=cmd, strip=True) - shelly.cache_dir = tmp_path + shelly = shell.define(cmd)() + assert shelly.cmdline == " ".join(cmd) - res = results_function(shelly, plugin) - assert Path(res.output.stdout) == Path(shelly.output_dir) - assert res.output.return_code == 0 - assert res.output.stderr == "" + outputs = results_function(shelly, plugin=plugin, cache_dir=tmp_path) + assert Path(outputs.stdout).parent == tmp_path + assert outputs.return_code == 0 + assert outputs.stderr == "" @pytest.mark.parametrize("results_function", [run_no_submitter, run_submitter]) def test_shell_cmd_2(plugin, results_function, tmp_path): """a command with arguments, cmd and args given as executable""" cmd = ["echo", "hail", "pydra"] - shelly = ShellDef(name="shelly", executable=cmd) - shelly.cache_dir = tmp_path + shelly = shell.define(cmd)() + assert shelly.cmdline == " ".join(cmd) - res = results_function(shelly, plugin) - assert res.output.stdout.strip() == " ".join(cmd[1:]) - assert res.output.return_code == 0 - assert res.output.stderr == "" + outputs = results_function(shelly, plugin=plugin, cache_dir=tmp_path) + assert outputs.stdout.strip() == " ".join(cmd[1:]) + assert outputs.return_code == 0 + assert outputs.stderr == "" @pytest.mark.parametrize("results_function", [run_no_submitter, run_submitter]) @@ -77,32 +77,32 @@ def test_shell_cmd_2a(plugin, results_function, tmp_path): cmd_exec = "echo" cmd_args = ["hail", "pydra"] # separate command into exec + args - shelly = ShellDef(name="shelly", executable=cmd_exec, args=cmd_args) - shelly.cache_dir = tmp_path - assert shelly.definition.executable == "echo" + shelly = shell.define(cmd_exec)(additional_args=cmd_args) + + assert shelly.executable == "echo" assert shelly.cmdline == "echo " + " ".join(cmd_args) - res = results_function(shelly, plugin) - assert res.output.stdout.strip() == " ".join(cmd_args) - assert res.output.return_code == 0 - assert res.output.stderr == "" + outputs = results_function(shelly, plugin=plugin, cache_dir=tmp_path) + assert outputs.stdout.strip() == " ".join(cmd_args) + assert outputs.return_code == 0 + assert outputs.stderr == "" @pytest.mark.parametrize("results_function", [run_no_submitter, run_submitter]) def test_shell_cmd_2b(plugin, results_function, tmp_path): """a command with arguments, using strings executable and args""" cmd_exec = "echo" - cmd_args = "pydra" + cmd_args = ["pydra"] # separate command into exec + args - shelly = ShellDef(name="shelly", executable=cmd_exec, args=cmd_args) - shelly.cache_dir = tmp_path - assert shelly.definition.executable == "echo" + shelly = shell.define(cmd_exec)(additional_args=cmd_args) + + assert shelly.executable == "echo" assert shelly.cmdline == "echo pydra" - res = results_function(shelly, plugin) - assert res.output.stdout == "pydra\n" - assert res.output.return_code == 0 - assert res.output.stderr == "" + outputs = results_function(shelly, plugin=plugin, cache_dir=tmp_path) + assert outputs.stdout == "pydra\n" + assert outputs.return_code == 0 + assert outputs.stderr == "" # tests with State @@ -116,19 +116,18 @@ def test_shell_cmd_3(plugin_dask_opt, tmp_path): cmd = ["pwd", "whoami"] # all args given as executable - shelly = ShellDef(name="shelly").split("executable", executable=cmd) - shelly.cache_dir = tmp_path + shelly = shell.define("shelly")().split(executable=cmd) # assert shelly.cmdline == ["pwd", "whoami"] - res = shelly(plugin=plugin_dask_opt) - assert Path(res[0].output.stdout.rstrip()) == shelly.output_dir[0] + outputs = shelly(plugin=plugin_dask_opt, cache_dir=tmp_path) + assert Path(outputs.stdout[0].rstrip()).parent == tmp_path if "USER" in os.environ: - assert res[1].output.stdout == f"{os.environ['USER']}\n" + assert outputs.stdout[1] == f"{os.environ['USER']}\n" else: - assert res[1].output.stdout - assert res[0].output.return_code == res[1].output.return_code == 0 - assert res[0].output.stderr == res[1].output.stderr == "" + assert outputs.stdout[1] + assert outputs.return_code[0] == outputs.return_code[1] == 0 + assert outputs.stderr[0] == outputs.stderr[1] == "" def test_shell_cmd_4(plugin, tmp_path): @@ -136,23 +135,20 @@ def test_shell_cmd_4(plugin, tmp_path): splitter=args """ cmd_exec = "echo" - cmd_args = ["nipype", "pydra"] + cmd_args = [["nipype"], ["pydra"]] # separate command into exec + args - shelly = ShellDef(name="shelly", executable=cmd_exec).split( - splitter="args", args=cmd_args - ) - shelly.cache_dir = tmp_path + shelly = shell.define(cmd_exec)().split(additional_args=cmd_args) - assert shelly.inputs.executable == "echo" - assert shelly.inputs.args == ["nipype", "pydra"] + assert shelly.executable == "echo" + assert shelly.additional_args == StateArray([["nipype"], ["pydra"]]) # assert shelly.cmdline == ["echo nipype", "echo pydra"] - res = shelly(plugin=plugin) + outputs = shelly(plugin=plugin) - assert res[0].output.stdout == "nipype\n" - assert res[1].output.stdout == "pydra\n" + assert outputs.stdout[0] == "nipype\n" + assert outputs.stdout[1] == "pydra\n" - assert res[0].output.return_code == res[1].output.return_code == 0 - assert res[0].output.stderr == res[1].output.stderr == "" + assert outputs.return_code[0] == outputs.return_code[1] == 0 + assert outputs.stderr[0] == outputs.stderr[1] == "" def test_shell_cmd_5(plugin, tmp_path): @@ -160,22 +156,21 @@ def test_shell_cmd_5(plugin, tmp_path): using splitter and combiner for args """ cmd_exec = "echo" - cmd_args = ["nipype", "pydra"] + cmd_args = [["nipype"], ["pydra"]] # separate command into exec + args shelly = ( - ShellDef(name="shelly", executable=cmd_exec) - .split(splitter="args", args=cmd_args) - .combine("args") + shell.define(cmd_exec)() + .split(additional_args=cmd_args) + .combine("additional_args") ) - shelly.cache_dir = tmp_path - assert shelly.inputs.executable == "echo" - assert shelly.inputs.args == ["nipype", "pydra"] + assert shelly.executable == "echo" + assert shelly.additional_args == StateArray([["nipype"], ["pydra"]]) # assert shelly.cmdline == ["echo nipype", "echo pydra"] - res = shelly(plugin=plugin) + outputs = shelly(plugin=plugin) - assert res[0].output.stdout == "nipype\n" - assert res[1].output.stdout == "pydra\n" + assert outputs.stdout[0] == "nipype\n" + assert outputs.stdout[1] == "pydra\n" def test_shell_cmd_6(plugin, tmp_path): @@ -183,40 +178,39 @@ def test_shell_cmd_6(plugin, tmp_path): outer splitter for executable and args """ cmd_exec = ["echo", ["echo", "-n"]] - cmd_args = ["nipype", "pydra"] + cmd_args = [["nipype"], ["pydra"]] # separate command into exec + args - shelly = ShellDef(name="shelly").split( - splitter=["executable", "args"], executable=cmd_exec, args=cmd_args + shelly = shell.define("shelly")().split( + executable=cmd_exec, additional_args=cmd_args ) - shelly.cache_dir = tmp_path - assert shelly.inputs.executable == ["echo", ["echo", "-n"]] - assert shelly.inputs.args == ["nipype", "pydra"] + assert shelly.executable == ["echo", ["echo", "-n"]] + assert shelly.additional_args == StateArray([["nipype"], ["pydra"]]) # assert shelly.cmdline == [ # "echo nipype", # "echo pydra", # "echo -n nipype", # "echo -n pydra", # ] - res = shelly(plugin=plugin) + outputs = shelly(plugin=plugin) - assert res[0].output.stdout == "nipype\n" - assert res[1].output.stdout == "pydra\n" - assert res[2].output.stdout == "nipype" - assert res[3].output.stdout == "pydra" + assert outputs.stdout[0] == "nipype\n" + assert outputs.stdout[1] == "pydra\n" + assert outputs.stdout[2] == "nipype" + assert outputs.stdout[3] == "pydra" assert ( - res[0].output.return_code - == res[1].output.return_code - == res[2].output.return_code - == res[3].output.return_code + outputs.return_code[0] + == outputs.return_code[1] + == outputs.return_code[2] + == outputs.return_code[3] == 0 ) assert ( - res[0].output.stderr - == res[1].output.stderr - == res[2].output.stderr - == res[3].output.stderr + outputs.stderr[0] + == outputs.stderr[1] + == outputs.stderr[2] + == outputs.stderr[3] == "" ) @@ -226,25 +220,23 @@ def test_shell_cmd_7(plugin, tmp_path): outer splitter for executable and args, and combiner=args """ cmd_exec = ["echo", ["echo", "-n"]] - cmd_args = ["nipype", "pydra"] + cmd_args = [["nipype"], ["pydra"]] # separate command into exec + args shelly = ( - ShellDef(name="shelly") - .split(splitter=["executable", "args"], executable=cmd_exec, args=cmd_args) - .combine("args") + shell.define("shelly")() + .split(executable=cmd_exec, additional_args=cmd_args) + .combine("additional_args") ) - shelly.cache_dir = tmp_path - assert shelly.inputs.executable == ["echo", ["echo", "-n"]] - assert shelly.inputs.args == ["nipype", "pydra"] + assert shelly.executable == ["echo", ["echo", "-n"]] + assert shelly.additional_args == StateArray([["nipype"], ["pydra"]]) - res = shelly(plugin=plugin) + outputs = shelly(plugin=plugin) - assert res[0][0].output.stdout == "nipype\n" - assert res[0][1].output.stdout == "pydra\n" - - assert res[1][0].output.stdout == "nipype" - assert res[1][1].output.stdout == "pydra" + assert outputs.stdout[0][0] == "nipype\n" + assert outputs.stdout[0][1] == "pydra" + assert outputs.stdout[1][0] == "nipype" + assert outputs.stdout[1][1] == "pydra" # tests with workflows @@ -252,25 +244,20 @@ def test_shell_cmd_7(plugin, tmp_path): def test_wf_shell_cmd_1(plugin, tmp_path): """a workflow with two connected commands""" - wf = Workflow(name="wf", input_spec=["cmd1", "cmd2"]) - wf.inputs.cmd1 = "pwd" - wf.inputs.cmd2 = "ls" - wf.add(ShellDef(name="shelly_pwd", executable=wf.lzin.cmd1, strip=True)) - wf.add( - ShellDef( - name="shelly_ls", executable=wf.lzin.cmd2, args=wf.shelly_pwd.lzout.stdout - ) - ) - wf.set_output([("out", wf.shelly_ls.lzout.stdout)]) - wf.cache_dir = tmp_path + @workflow.define + def Workflow(cmd1, cmd2): + shelly_pwd = workflow.add(shell.define(cmd1)) + shelly_ls = workflow.add(shell.define(cmd2, additional_args=shelly_pwd.stdout)) + return shelly_ls.stdout + + wf = Workflow(cmd1="pwd", cmd2="ls") - with Submitter(worker=plugin) as sub: - wf(submitter=sub) + with Submitter(plugin=plugin, cache_dir=tmp_path) as sub: + res = sub(wf) - res = wf.result() - assert "_result.pklz" in res.output.out - assert "_task.pklz" in res.output.out + assert "_result.pklz" in res.outputs.out + assert "_task.pklz" in res.outputs.out # customised input definition @@ -284,36 +271,28 @@ def test_shell_cmd_inputspec_1(plugin, results_function, tmp_path): """ cmd_exec = "echo" cmd_opt = True - cmd_args = "hello from pydra" - my_input_spec = SpecInfo( - name="Input", - fields=[ - ( - "opt_n", - attr.ib( - type=bool, - metadata={"position": 1, "argstr": "-n", "help": "option"}, - ), - ) - ], - bases=(ShellDef,), - ) + cmd_args = ["hello from pydra"] + + @shell.define + class Shelly(ShellDef["Shelly.Outputs"]): + executable = cmd_exec + opt_n: bool = shell.arg( + position=1, + argstr="-n", + help="option", + ) + + class Outputs(ShellOutputs): + pass # separate command into exec + args - shelly = ShellDef( - name="shelly", - executable=cmd_exec, - args=cmd_args, - opt_n=cmd_opt, - input_spec=my_input_spec, - cache_dir=tmp_path, - ) - assert shelly.definition.executable == cmd_exec - assert shelly.definition.args == cmd_args + shelly = Shelly(additional_args=cmd_args, opt_n=cmd_opt) + assert shelly.executable == cmd_exec + assert shelly.additional_args == cmd_args assert shelly.cmdline == "echo -n 'hello from pydra'" - res = results_function(shelly, plugin) - assert res.output.stdout == "hello from pydra" + outputs = results_function(shelly, plugin=plugin, cache_dir=tmp_path) + assert outputs.stdout == "hello from pydra" @pytest.mark.parametrize("results_function", [run_no_submitter, run_submitter]) @@ -326,42 +305,31 @@ def test_shell_cmd_inputspec_2(plugin, results_function, tmp_path): cmd_opt = True cmd_opt_hello = "HELLO" cmd_args = "from pydra" - my_input_spec = SpecInfo( - name="Input", - fields=[ - ( - "opt_hello", - attr.ib( - type=str, - metadata={"position": 3, "help": "todo", "argstr": ""}, - ), - ), - ( - "opt_n", - attr.ib( - type=bool, - metadata={"position": 1, "help": "todo", "argstr": "-n"}, - ), - ), - ], - bases=(ShellDef,), - ) + + @shell.define + class Shelly(ShellDef["Shelly.Outputs"]): + executable = cmd_exec + opt_hello: str = shell.arg( + position=3, + help="todo", + argstr="", + ) + opt_n: bool = shell.arg( + position=1, + help="todo", + argstr="-n", + ) + + class Outputs(ShellOutputs): + pass # separate command into exec + args - shelly = ShellDef( - name="shelly", - executable=cmd_exec, - args=cmd_args, - opt_n=cmd_opt, - opt_hello=cmd_opt_hello, - input_spec=my_input_spec, - cache_dir=tmp_path, - ) - assert shelly.definition.executable == cmd_exec - assert shelly.definition.args == cmd_args + shelly = Shelly(additional_args=cmd_args, opt_n=cmd_opt, opt_hello=cmd_opt_hello) + assert shelly.executable == cmd_exec + assert shelly.additional_args == cmd_args assert shelly.cmdline == "echo -n HELLO 'from pydra'" - res = results_function(shelly, plugin) - assert res.output.stdout == "HELLO from pydra" + outputs = results_function(shelly, plugin=plugin, cache_dir=tmp_path) + assert outputs.stdout == "HELLO from pydra" @pytest.mark.parametrize("results_function", [run_no_submitter, run_submitter]) @@ -369,70 +337,49 @@ def test_shell_cmd_inputspec_3(plugin, results_function, tmp_path): """mandatory field added to fields, value provided""" cmd_exec = "echo" hello = "HELLO" - my_input_spec = SpecInfo( - name="Input", - fields=[ - ( - "text", - attr.ib( - type=str, - metadata={ - "position": 1, - "help": "text", - "mandatory": True, - "argstr": "", - }, - ), - ) - ], - bases=(ShellDef,), - ) + + @shell.define + class Shelly(ShellDef["Shelly.Outputs"]): + executable = cmd_exec + text: str = shell.arg( + position=1, + help="text", + argstr="", + ) + + class Outputs(ShellOutputs): + pass # separate command into exec + args - shelly = ShellDef( - name="shelly", - executable=cmd_exec, - text=hello, - input_spec=my_input_spec, - cache_dir=tmp_path, - ) - assert shelly.definition.executable == cmd_exec + shelly = Shelly(text=hello) + assert shelly.executable == cmd_exec assert shelly.cmdline == "echo HELLO" - res = results_function(shelly, plugin) - assert res.output.stdout == "HELLO\n" + outputs = results_function(shelly, plugin=plugin, cache_dir=tmp_path) + assert outputs.stdout == "HELLO\n" @pytest.mark.parametrize("results_function", [run_no_submitter, run_submitter]) def test_shell_cmd_inputspec_3a(plugin, results_function, tmp_path): """mandatory field added to fields, value provided - using shorter syntax for input definition (no attr.ib) + using shorter syntax for input (no attr.ib) """ cmd_exec = "echo" hello = "HELLO" - my_input_spec = SpecInfo( - name="Input", - fields=[ - ( - "text", - str, - {"position": 1, "help": "text", "mandatory": True, "argstr": ""}, - ) - ], - bases=(ShellDef,), - ) + + @shell.define + class Shelly(ShellDef["Shelly.Outputs"]): + executable = cmd_exec + text: str = shell.arg(position=1, help="text", argstr="") + + class Outputs(ShellOutputs): + pass # separate command into exec + args - shelly = ShellDef( - name="shelly", - executable=cmd_exec, - text=hello, - input_spec=my_input_spec, - cache_dir=tmp_path, - ) - assert shelly.definition.executable == cmd_exec + shelly = Shelly(text=hello) + assert shelly.executable == cmd_exec assert shelly.cmdline == "echo HELLO" - res = results_function(shelly, plugin) - assert res.output.stdout == "HELLO\n" + outputs = results_function(shelly, plugin=plugin, cache_dir=tmp_path) + assert outputs.stdout == "HELLO\n" @pytest.mark.parametrize("results_function", [run_no_submitter, run_submitter]) @@ -440,62 +387,46 @@ def test_shell_cmd_inputspec_3b(plugin, results_function, tmp_path): """mandatory field added to fields, value provided after init""" cmd_exec = "echo" hello = "HELLO" - my_input_spec = SpecInfo( - name="Input", - fields=[ - ( - "text", - attr.ib( - type=str, - metadata={ - "position": 1, - "help": "text", - "mandatory": True, - "argstr": "", - }, - ), - ) - ], - bases=(ShellDef,), - ) + + @shell.define + class Shelly(ShellDef["Shelly.Outputs"]): + executable = cmd_exec + text: str = shell.arg( + position=1, + help="text", + argstr="", + ) + + class Outputs(ShellOutputs): + pass # separate command into exec + args - shelly = ShellDef( - name="shelly", executable=cmd_exec, input_spec=my_input_spec, cache_dir=tmp_path - ) - shelly.definition.text = hello + shelly = Shelly(executable=cmd_exec) + shelly.text = hello - assert shelly.definition.executable == cmd_exec + assert shelly.executable == cmd_exec assert shelly.cmdline == "echo HELLO" - res = results_function(shelly, plugin) - assert res.output.stdout == "HELLO\n" + outputs = results_function(shelly, plugin=plugin, cache_dir=tmp_path) + assert outputs.stdout == "HELLO\n" def test_shell_cmd_inputspec_3c_exception(plugin, tmp_path): """mandatory field added to fields, value is not provided, so exception is raised""" cmd_exec = "echo" - my_input_spec = SpecInfo( - name="Input", - fields=[ - ( - "text", - attr.ib( - type=str, - metadata={ - "position": 1, - "help": "text", - "mandatory": True, - "argstr": "", - }, - ), - ) - ], - bases=(ShellDef,), - ) - shelly = ShellDef( - name="shelly", executable=cmd_exec, input_spec=my_input_spec, cache_dir=tmp_path - ) + @shell.define + class Shelly(ShellDef["Shelly.Outputs"]): + executable = cmd_exec + text: str = shell.arg( + position=1, + help="text", + argstr="", + ) + + class Outputs(ShellOutputs): + pass + + shelly = Shelly(executable=cmd_exec) with pytest.raises(Exception) as excinfo: shelly() @@ -506,181 +437,155 @@ def test_shell_cmd_inputspec_3c_exception(plugin, tmp_path): def test_shell_cmd_inputspec_3c(plugin, results_function, tmp_path): """mandatory=False, so tasks runs fine even without the value""" cmd_exec = "echo" - my_input_spec = SpecInfo( - name="Input", - fields=[ - ( - "text", - attr.ib( - type=ty.Optional[str], - default=None, - metadata={ - "position": 1, - "help": "text", - "mandatory": False, - "argstr": "", - }, - ), - ) - ], - bases=(ShellDef,), - ) + + @shell.define + class Shelly(ShellDef["Shelly.Outputs"]): + executable = cmd_exec + text: ty.Optional[str] = shell.arg( + default=None, + position=1, + help="text", + argstr="", + ) + + class Outputs(ShellOutputs): + pass # separate command into exec + args - shelly = ShellDef( - name="shelly", executable=cmd_exec, input_spec=my_input_spec, cache_dir=tmp_path - ) + shelly = Shelly(executable=cmd_exec) - assert shelly.definition.executable == cmd_exec + assert shelly.executable == cmd_exec assert shelly.cmdline == "echo" - res = results_function(shelly, plugin) - assert res.output.stdout == "\n" + outputs = results_function(shelly, plugin=plugin, cache_dir=tmp_path) + assert outputs.stdout == "\n" @pytest.mark.parametrize("results_function", [run_no_submitter, run_submitter]) def test_shell_cmd_inputspec_4(plugin, results_function, tmp_path): """mandatory field added to fields, value provided""" cmd_exec = "echo" - my_input_spec = SpecInfo( - name="Input", - fields=[ - ( - "text", - attr.ib( - type=str, - default="Hello", - metadata={"position": 1, "help": "text", "argstr": ""}, - ), - ) - ], - bases=(ShellDef,), - ) + + @shell.define + class Shelly(ShellDef["Shelly.Outputs"]): + executable = cmd_exec + text: str = shell.arg( + default="Hello", + position=1, + help="text", + argstr="", + ) + + class Outputs(ShellOutputs): + pass # separate command into exec + args - shelly = ShellDef( - name="shelly", executable=cmd_exec, input_spec=my_input_spec, cache_dir=tmp_path - ) + shelly = Shelly(executable=cmd_exec) - assert shelly.definition.executable == cmd_exec + assert shelly.executable == cmd_exec assert shelly.cmdline == "echo Hello" - res = results_function(shelly, plugin) - assert res.output.stdout == "Hello\n" + outputs = results_function(shelly, plugin=plugin, cache_dir=tmp_path) + assert outputs.stdout == "Hello\n" @pytest.mark.parametrize("results_function", [run_no_submitter, run_submitter]) def test_shell_cmd_inputspec_4a(plugin, results_function, tmp_path): """mandatory field added to fields, value provided - using shorter syntax for input definition (no attr.ib) + using shorter syntax for input (no attr.ib) """ cmd_exec = "echo" - my_input_spec = SpecInfo( - name="Input", - fields=[("text", str, "Hello", {"position": 1, "help": "text", "argstr": ""})], - bases=(ShellDef,), - ) + + @shell.define + class Shelly(ShellDef["Shelly.Outputs"]): + executable = cmd_exec + text: str = shell.arg(default="Hello", position=1, help="text", argstr="") + + class Outputs(ShellOutputs): + pass # separate command into exec + args - shelly = ShellDef( - name="shelly", executable=cmd_exec, input_spec=my_input_spec, cache_dir=tmp_path - ) + shelly = Shelly(executable=cmd_exec) - assert shelly.definition.executable == cmd_exec + assert shelly.executable == cmd_exec assert shelly.cmdline == "echo Hello" - res = results_function(shelly, plugin) - assert res.output.stdout == "Hello\n" + outputs = results_function(shelly, plugin=plugin, cache_dir=tmp_path) + assert outputs.stdout == "Hello\n" @pytest.mark.parametrize("results_function", [run_no_submitter, run_submitter]) def test_shell_cmd_inputspec_4b(plugin, results_function, tmp_path): """mandatory field added to fields, value provided""" cmd_exec = "echo" - my_input_spec = SpecInfo( - name="Input", - fields=[ - ( - "text", - attr.ib( - type=str, - default="Hi", - metadata={"position": 1, "help": "text", "argstr": ""}, - ), - ) - ], - bases=(ShellDef,), - ) + + @shell.define + class Shelly(ShellDef["Shelly.Outputs"]): + executable = cmd_exec + text: str = shell.arg( + default="Hi", + position=1, + help="text", + argstr="", + ) + + class Outputs(ShellOutputs): + pass # separate command into exec + args - shelly = ShellDef( - name="shelly", executable=cmd_exec, input_spec=my_input_spec, cache_dir=tmp_path - ) + shelly = Shelly(executable=cmd_exec) - assert shelly.definition.executable == cmd_exec + assert shelly.executable == cmd_exec assert shelly.cmdline == "echo Hi" - res = results_function(shelly, plugin) - assert res.output.stdout == "Hi\n" + outputs = results_function(shelly, plugin=plugin, cache_dir=tmp_path) + assert outputs.stdout == "Hi\n" def test_shell_cmd_inputspec_4c_exception(plugin): """mandatory field added to fields, value provided""" cmd_exec = "echo" - my_input_spec = SpecInfo( - name="Input", - fields=[ - ( - "text", - attr.ib( - type=str, - default="Hello", - metadata={ - "position": 1, - "help": "text", - "mandatory": True, - "argstr": "", - }, - ), - ) - ], - bases=(ShellDef,), - ) # separate command into exec + args with pytest.raises( Exception, match=r"default value \('Hello'\) should not be set when the field" ): - ShellDef(name="shelly", executable=cmd_exec, input_spec=my_input_spec) + + @shell.define + class Shelly(ShellDef["Shelly.Outputs"]): + executable = cmd_exec + text: str = shell.arg( + default="Hello", + position=1, + help="text", + argstr="", + ) + + class Outputs(ShellOutputs): + pass def test_shell_cmd_inputspec_4d_exception(plugin): """mandatory field added to fields, value provided""" cmd_exec = "echo" - my_input_spec = SpecInfo( - name="Input", - fields=[ - ( - "text", - attr.ib( - type=str, - default="Hello", - metadata={ - "position": 1, - "help": "text", - "output_file_template": "exception", - "argstr": "", - }, - ), - ) - ], - bases=(ShellDef,), - ) # separate command into exec + args with pytest.raises( Exception, match=r"default value \('Hello'\) should not be set together" - ) as excinfo: - ShellDef(name="shelly", executable=cmd_exec, input_spec=my_input_spec) + ): + + @shell.define + class Shelly(ShellDef["Shelly.Outputs"]): + executable = cmd_exec + text: File = shell.outarg( + default="Hello", + position=1, + help="text", + path_template="exception", + argstr="", + ) + + class Outputs(ShellOutputs): + pass @pytest.mark.parametrize("results_function", [run_no_submitter, run_submitter]) @@ -688,48 +593,31 @@ def test_shell_cmd_inputspec_5_nosubm(plugin, results_function, tmp_path): """checking xor in metadata: task should work fine, since only one option is True""" cmd_exec = "ls" cmd_t = True - my_input_spec = SpecInfo( - name="Input", - fields=[ - ( - "opt_t", - attr.ib( - type=bool, - metadata={ - "position": 1, - "help": "opt t", - "argstr": "-t", - "xor": ["opt_S"], - }, - ), - ), - ( - "opt_S", - attr.ib( - type=bool, - metadata={ - "position": 2, - "help": "opt S", - "argstr": "-S", - "xor": ["opt_t"], - }, - ), - ), - ], - bases=(ShellDef,), - ) + + @shell.define + class Shelly(ShellDef["Shelly.Outputs"]): + executable = cmd_exec + opt_t: bool = shell.arg( + position=1, + help="opt t", + argstr="-t", + xor=["opt_S"], + ) + opt_S: bool = shell.arg( + position=2, + help="opt S", + argstr="-S", + xor=["opt_t"], + ) + + class Outputs(ShellOutputs): + pass # separate command into exec + args - shelly = ShellDef( - name="shelly", - executable=cmd_exec, - opt_t=cmd_t, - input_spec=my_input_spec, - cache_dir=tmp_path, - ) - assert shelly.definition.executable == cmd_exec + shelly = Shelly(opt_t=cmd_t) + assert shelly.executable == cmd_exec assert shelly.cmdline == "ls -t" - results_function(shelly, plugin) + results_function(shelly, plugin=plugin, cache_dir=tmp_path) def test_shell_cmd_inputspec_5a_exception(plugin, tmp_path): @@ -737,45 +625,27 @@ def test_shell_cmd_inputspec_5a_exception(plugin, tmp_path): cmd_exec = "ls" cmd_t = True cmd_S = True - my_input_spec = SpecInfo( - name="Input", - fields=[ - ( - "opt_t", - attr.ib( - type=bool, - metadata={ - "position": 1, - "help": "opt t", - "argstr": "-t", - "xor": ["opt_S"], - }, - ), - ), - ( - "opt_S", - attr.ib( - type=bool, - metadata={ - "position": 2, - "help": "opt S", - "argstr": "-S", - "xor": ["opt_t"], - }, - ), - ), - ], - bases=(ShellDef,), - ) - shelly = ShellDef( - name="shelly", - executable=cmd_exec, - opt_t=cmd_t, - opt_S=cmd_S, - input_spec=my_input_spec, - cache_dir=tmp_path, - ) + @shell.define + class Shelly(ShellDef["Shelly.Outputs"]): + executable = cmd_exec + opt_t: bool = shell.arg( + position=1, + help="opt t", + argstr="-t", + xor=["opt_S"], + ) + opt_S: bool = shell.arg( + position=2, + help="opt S", + argstr="-S", + xor=["opt_t"], + ) + + class Outputs(ShellOutputs): + pass + + shelly = Shelly(opt_t=cmd_t, opt_S=cmd_S) with pytest.raises(Exception) as excinfo: shelly() assert "is mutually exclusive" in str(excinfo.value) @@ -789,44 +659,30 @@ def test_shell_cmd_inputspec_6(plugin, results_function, tmp_path): cmd_exec = "ls" cmd_l = True cmd_t = True - my_input_spec = SpecInfo( - name="Input", - fields=[ - ( - "opt_t", - attr.ib( - type=bool, - metadata={ - "position": 2, - "help": "opt t", - "argstr": "-t", - "requires": ["opt_l"], - }, - ), - ), - ( - "opt_l", - attr.ib( - type=bool, - metadata={"position": 1, "help": "opt l", "argstr": "-l"}, - ), - ), - ], - bases=(ShellDef,), - ) + + @shell.define + class Shelly(ShellDef["Shelly.Outputs"]): + executable = cmd_exec + opt_t: bool = shell.arg( + position=2, + help="opt t", + argstr="-t", + requires=["opt_l"], + ) + opt_l: bool = shell.arg( + position=1, + help="opt l", + argstr="-l", + ) + + class Outputs(ShellOutputs): + pass # separate command into exec + args - shelly = ShellDef( - name="shelly", - executable=cmd_exec, - opt_t=cmd_t, - opt_l=cmd_l, - input_spec=my_input_spec, - cache_dir=tmp_path, - ) - assert shelly.definition.executable == cmd_exec + shelly = Shelly(opt_t=cmd_t, opt_l=cmd_l) + assert shelly.executable == cmd_exec assert shelly.cmdline == "ls -l -t" - results_function(shelly, plugin) + results_function(shelly, plugin=plugin, cache_dir=tmp_path) def test_shell_cmd_inputspec_6a_exception(plugin): @@ -835,35 +691,26 @@ def test_shell_cmd_inputspec_6a_exception(plugin): """ cmd_exec = "ls" cmd_t = True - my_input_spec = SpecInfo( - name="Input", - fields=[ - ( - "opt_t", - attr.ib( - type=bool, - metadata={ - "position": 2, - "help": "opt t", - "argstr": "-t", - "requires": ["opt_l"], - }, - ), - ), - ( - "opt_l", - attr.ib( - type=bool, - metadata={"position": 1, "help": "opt l", "argstr": "-l"}, - ), - ), - ], - bases=(ShellDef,), - ) - shelly = ShellDef( - name="shelly", executable=cmd_exec, opt_t=cmd_t, input_spec=my_input_spec - ) + @shell.define + class Shelly(ShellDef["Shelly.Outputs"]): + executable = cmd_exec + opt_t: bool = shell.arg( + position=2, + help="opt t", + argstr="-t", + requires=["opt_l"], + ) + opt_l: bool = shell.arg( + position=1, + help="opt l", + argstr="-l", + ) + + class Outputs(ShellOutputs): + pass + + shelly = Shelly(executable=cmd_exec, opt_t=cmd_t) with pytest.raises(Exception) as excinfo: shelly() assert "requires" in str(excinfo.value) @@ -877,45 +724,34 @@ def test_shell_cmd_inputspec_6b(plugin, results_function, tmp_path): cmd_exec = "ls" cmd_l = True cmd_t = True - my_input_spec = SpecInfo( - name="Input", - fields=[ - ( - "opt_t", - attr.ib( - type=bool, - metadata={ - "position": 2, - "help": "opt t", - "argstr": "-t", - "requires": ["opt_l"], - }, - ), - ), - ( - "opt_l", - attr.ib( - type=bool, - metadata={"position": 1, "help": "opt l", "argstr": "-l"}, - ), - ), - ], - bases=(ShellDef,), - ) + + @shell.define + class Shelly(ShellDef["Shelly.Outputs"]): + executable = cmd_exec + opt_t: bool = shell.arg( + position=2, + help="opt t", + argstr="-t", + requires=["opt_l"], + ) + opt_l: bool = shell.arg( + position=1, + help="opt l", + argstr="-l", + ) + + class Outputs(ShellOutputs): + pass # separate command into exec + args - shelly = ShellDef( - name="shelly", - executable=cmd_exec, - opt_t=cmd_t, + shelly = Shelly( + opt_t=cmd_t # opt_l=cmd_l, - input_spec=my_input_spec, - cache_dir=tmp_path, ) - shelly.definition.opt_l = cmd_l - assert shelly.definition.executable == cmd_exec + shelly.opt_l = cmd_l + assert shelly.executable == cmd_exec assert shelly.cmdline == "ls -l -t" - results_function(shelly, plugin) + results_function(shelly, plugin=plugin, cache_dir=tmp_path) @pytest.mark.parametrize("results_function", [run_no_submitter, run_submitter]) @@ -925,39 +761,26 @@ def test_shell_cmd_inputspec_7(plugin, results_function, tmp_path): using name_tamplate in metadata """ cmd = "touch" - args = "newfile_tmp.txt" + args = ["newfile_tmp.txt"] + + @shell.define + class Shelly(ShellDef["Shelly.Outputs"]): + executable = cmd - my_input_spec = SpecInfo( - name="Input", - fields=[ - ( - "out1", - attr.ib( - type=str, - metadata={ - "output_file_template": "{args}", - "help": "output file", - }, - ), + class Outputs(ShellOutputs): + out1: File = shell.outarg( + path_template="{args}", + help="output file", ) - ], - bases=(ShellDef,), - ) - shelly = ShellDef( - name="shelly", - executable=cmd, - args=args, - input_spec=my_input_spec, - cache_dir=tmp_path, - ) + shelly = Shelly(executable=cmd, additional_args=args) - res = results_function(shelly, plugin) - assert res.output.stdout == "" - out1 = res.output.out1.fspath + outputs = results_function(shelly, plugin=plugin, cache_dir=tmp_path) + assert outputs.stdout == "" + out1 = outputs.out1.fspath assert out1.exists() # checking if the file is created in a good place - assert shelly.output_dir == out1.parent + assert out1.parent.parent == tmp_path assert out1.name == "newfile_tmp.txt" @@ -969,39 +792,29 @@ def test_shell_cmd_inputspec_7a(plugin, results_function, tmp_path): and changing the output name for output_spec using output_field_name """ cmd = "touch" - args = "newfile_tmp.txt" + args = File.mock("newfile_tmp.txt") + + @shell.define + class Shelly(ShellDef["Shelly.Outputs"]): + executable = cmd - my_input_spec = SpecInfo( - name="Input", - fields=[ - ( - "out1", - attr.ib( - type=str, - metadata={ - "output_file_template": "{args}", - "output_field_name": "out1_changed", - "help": "output file", - }, - ), + class Outputs(ShellOutputs): + out1: File = shell.outarg( + path_template="{args}", + output_field_name="out1_changed", + help="output file", ) - ], - bases=(ShellDef,), - ) - shelly = ShellDef( - name="shelly", + shelly = Shelly( executable=cmd, - args=args, - input_spec=my_input_spec, - cache_dir=tmp_path, + additional_args=args, ) - res = results_function(shelly, plugin) - assert res.output.stdout == "" + outputs = results_function(shelly, plugin=plugin, cache_dir=tmp_path) + assert outputs.stdout == "" # checking if the file is created in a good place - assert shelly.output_dir == res.output.out1_changed.fspath.parent - assert res.output.out1_changed.fspath.name == "newfile_tmp.txt" + assert outputs.out1_changed.fspath.parent.parent == tmp_path + assert outputs.out1_changed.fspath.name == "newfile_tmp.txt" @pytest.mark.parametrize("results_function", [run_no_submitter, run_submitter]) @@ -1012,41 +825,26 @@ def test_shell_cmd_inputspec_7b(plugin, results_function, tmp_path): """ cmd = "touch" - my_input_spec = SpecInfo( - name="Input", - fields=[ - ( - "newfile", - attr.ib( - type=str, - metadata={"position": 1, "help": "new file", "argstr": ""}, - ), - ), - ( - "out1", - attr.ib( - type=str, - metadata={ - "output_file_template": "{newfile}", - "help": "output file", - }, - ), - ), - ], - bases=(ShellDef,), - ) + @shell.define + class Shelly(ShellDef["Shelly.Outputs"]): + executable = cmd + newfile: str = shell.arg( + position=1, + help="new file", + argstr="", + ) - shelly = ShellDef( - name="shelly", - executable=cmd, - newfile="newfile_tmp.txt", - input_spec=my_input_spec, - cache_dir=tmp_path, - ) + class Outputs(ShellOutputs): + out1: File = shell.outarg( + path_template="{newfile}", + help="output file", + ) - res = results_function(shelly, plugin) - assert res.output.stdout == "" - assert res.output.out1.fspath.exists() + shelly = Shelly(executable=cmd, newfile=File.mock("newfile_tmp.txt")) + + outputs = results_function(shelly, plugin=plugin, cache_dir=tmp_path) + assert outputs.stdout == "" + assert outputs.out1.fspath.exists() @pytest.mark.parametrize("results_function", [run_no_submitter, run_submitter]) @@ -1056,38 +854,25 @@ def test_shell_cmd_inputspec_7c(plugin, results_function, tmp_path): using name_tamplate with txt extension (extension from args should be removed """ cmd = "touch" - args = "newfile_tmp.txt" + args = File.mock("newfile_tmp.txt") - my_input_spec = SpecInfo( - name="Input", - fields=[ - ( - "out1", - attr.ib( - type=str, - metadata={ - "output_file_template": "{args}.txt", - "help": "output file", - }, - ), + @shell.define + class Shelly(ShellDef["Shelly.Outputs"]): + executable = cmd + + class Outputs(ShellOutputs): + out1: File = shell.outarg( + path_template="{args}.txt", + help="output file", ) - ], - bases=(ShellDef,), - ) - shelly = ShellDef( - name="shelly", - executable=cmd, - args=args, - input_spec=my_input_spec, - cache_dir=tmp_path, - ) + shelly = Shelly(executable=cmd, additional_args=args) - res = results_function(shelly, plugin) - assert res.output.stdout == "" + outputs = results_function(shelly, plugin=plugin, cache_dir=tmp_path) + assert outputs.stdout == "" # checking if the file is created in a good place - assert shelly.output_dir == res.output.out1.fspath.parent - assert res.output.out1.fspath.name == "newfile_tmp.txt" + assert outputs.out1.fspath.parent.parent == tmp_path + assert outputs.out1.fspath.name == "newfile_tmp.txt" @pytest.mark.parametrize("results_function", [run_no_submitter, run_submitter]) @@ -1098,53 +883,35 @@ def test_shell_cmd_inputspec_8(plugin, results_function, tmp_path): """ cmd = "touch" - my_input_spec = SpecInfo( - name="Input", - fields=[ - ( - "newfile", - attr.ib( - type=str, - metadata={"position": 2, "help": "new file", "argstr": ""}, - ), - ), - ( - "time", - attr.ib( - type=str, - metadata={ - "position": 1, - "argstr": "-t", - "help": "time of modif.", - }, - ), - ), - ( - "out1", - attr.ib( - type=str, - metadata={ - "output_file_template": "{newfile}", - "help": "output file", - }, - ), - ), - ], - bases=(ShellDef,), - ) + @shell.define + class Shelly(ShellDef["Shelly.Outputs"]): + executable = cmd + newfile: str = shell.arg( + position=2, + help="new file", + argstr="", + ) + time: str = shell.arg( + position=1, + argstr="-t", + help="time of modif.", + ) + + class Outputs(ShellOutputs): + out1: File = shell.outarg( + path_template="{newfile}", + help="output file", + ) - shelly = ShellDef( - name="shelly", + shelly = Shelly( executable=cmd, - newfile="newfile_tmp.txt", + newfile=File.mock("newfile_tmp.txt"), time="02121010", - input_spec=my_input_spec, - cache_dir=tmp_path, ) - res = results_function(shelly, plugin) - assert res.output.stdout == "" - assert res.output.out1.fspath.exists() + outputs = results_function(shelly, plugin=plugin, cache_dir=tmp_path) + assert outputs.stdout == "" + assert outputs.out1.fspath.exists() @pytest.mark.parametrize("results_function", [run_no_submitter, run_submitter]) @@ -1155,59 +922,41 @@ def test_shell_cmd_inputspec_8a(plugin, results_function, tmp_path): """ cmd = "touch" - my_input_spec = SpecInfo( - name="Input", - fields=[ - ( - "newfile", - attr.ib( - type=str, - metadata={"position": 2, "help": "new file", "argstr": ""}, - ), - ), - ( - "time", - attr.ib( - type=str, - metadata={ - "position": 1, - "argstr": "-t {time}", - "help": "time of modif.", - }, - ), - ), - ( - "out1", - attr.ib( - type=str, - metadata={ - "output_file_template": "{newfile}", - "help": "output file", - }, - ), - ), - ], - bases=(ShellDef,), - ) + @shell.define + class Shelly(ShellDef["Shelly.Outputs"]): + executable = cmd + newfile: str = shell.arg( + position=2, + help="new file", + argstr="", + ) + time: str = shell.arg( + position=1, + argstr="-t {time}", + help="time of modif.", + ) + + class Outputs(ShellOutputs): + out1: File = shell.outarg( + path_template="{newfile}", + help="output file", + ) - shelly = ShellDef( - name="shelly", + shelly = Shelly( executable=cmd, - newfile="newfile_tmp.txt", + newfile=File.mock("newfile_tmp.txt"), time="02121010", - input_spec=my_input_spec, - cache_dir=tmp_path, ) - res = results_function(shelly, plugin) - assert res.output.stdout == "" - assert res.output.out1.fspath.exists() + outputs = results_function(shelly, plugin=plugin, cache_dir=tmp_path) + assert outputs.stdout == "" + assert outputs.out1.fspath.exists() @pytest.mark.parametrize("results_function", [run_no_submitter, run_submitter]) def test_shell_cmd_inputspec_9(tmp_path, plugin, results_function): """ - providing output name using input_spec (output_file_template in metadata), + providing output name using input_spec (path_template in metadata), the template has a suffix, the extension of the file will be moved to the end """ cmd = "cp" @@ -1216,51 +965,39 @@ def test_shell_cmd_inputspec_9(tmp_path, plugin, results_function): file = ddir / ("file.txt") file.write_text("content\n") - my_input_spec = SpecInfo( - name="Input", - fields=[ - ( - "file_orig", - attr.ib( - type=File, - metadata={"position": 2, "help": "new file", "argstr": ""}, - ), - ), - ( - "file_copy", - attr.ib( - type=str, - metadata={ - "output_file_template": "{file_orig}_copy", - "help": "output file", - "argstr": "", - }, - ), - ), - ], - bases=(ShellDef,), - ) + @shell.define + class Shelly(ShellDef["Shelly.Outputs"]): + executable = cmd + file_orig: File = shell.arg( + position=2, + help="new file", + argstr="", + ) - shelly = ShellDef( - name="shelly", + class Outputs(ShellOutputs): + file_copy: File = shell.outarg( + path_template="{file_orig}_copy", + help="output file", + argstr="", + ) + + shelly = Shelly( executable=cmd, - input_spec=my_input_spec, file_orig=file, - cache_dir=tmp_path, ) - res = results_function(shelly, plugin) - assert res.output.stdout == "" - assert res.output.file_copy.fspath.exists() - assert res.output.file_copy.fspath.name == "file_copy.txt" + outputs = results_function(shelly, plugin=plugin, cache_dir=tmp_path) + assert outputs.stdout == "" + assert outputs.file_copy.fspath.exists() + assert outputs.file_copy.fspath.name == "file_copy.txt" # checking if it's created in a good place - assert shelly.output_dir == res.output.file_copy.fspath.parent + assert outputs.file_copy.fspath.parent.parent == tmp_path @pytest.mark.parametrize("results_function", [run_no_submitter]) def test_shell_cmd_inputspec_9a(tmp_path, plugin, results_function): """ - providing output name using input_spec (output_file_template in metadata), + providing output name using input_spec (path_template in metadata), the template has a suffix, the extension of the file will be moved to the end the change: input file has directory with a dot """ @@ -1269,97 +1006,74 @@ def test_shell_cmd_inputspec_9a(tmp_path, plugin, results_function): file.parent.mkdir() file.write_text("content\n") - my_input_spec = SpecInfo( - name="Input", - fields=[ - ( - "file_orig", - attr.ib( - type=File, - metadata={"position": 2, "help": "new file", "argstr": ""}, - ), - ), - ( - "file_copy", - attr.ib( - type=str, - metadata={ - "output_file_template": "{file_orig}_copy", - "help": "output file", - "argstr": "", - }, - ), - ), - ], - bases=(ShellDef,), - ) + @shell.define + class Shelly(ShellDef["Shelly.Outputs"]): + executable = cmd + file_orig: File = shell.arg( + position=2, + help="new file", + argstr="", + ) - shelly = ShellDef( - name="shelly", executable=cmd, input_spec=my_input_spec, file_orig=file - ) + class Outputs(ShellOutputs): + file_copy: File = shell.outarg( + path_template="{file_orig}_copy", + help="output file", + argstr="", + ) + + shelly = Shelly(executable=cmd, file_orig=file) - res = results_function(shelly, plugin) - assert res.output.stdout == "" - assert res.output.file_copy.fspath.exists() - assert res.output.file_copy.fspath.name == "file_copy.txt" + outputs = results_function(shelly, plugin=plugin, cache_dir=tmp_path) + assert outputs.stdout == "" + assert outputs.file_copy.fspath.exists() + assert outputs.file_copy.fspath.name == "file_copy.txt" # checking if it's created in a good place - assert shelly.output_dir == res.output.file_copy.fspath.parent + assert outputs.file_copy.fspath.parent.parent == tmp_path @pytest.mark.parametrize("results_function", [run_no_submitter, run_submitter]) def test_shell_cmd_inputspec_9b(tmp_path, plugin, results_function): """ - providing output name using input_spec (output_file_template in metadata) + providing output name using input_spec (path_template in metadata) and the keep_extension is set to False, so the extension is removed completely. """ cmd = "cp" file = tmp_path / "file.txt" file.write_text("content\n") - my_input_spec = SpecInfo( - name="Input", - fields=[ - ( - "file_orig", - attr.ib( - type=File, - metadata={"position": 2, "help": "new file", "argstr": ""}, - ), - ), - ( - "file_copy", - attr.ib( - type=str, - metadata={ - "output_file_template": "{file_orig}_copy", - "keep_extension": False, - "help": "output file", - "argstr": "", - }, - ), - ), - ], - bases=(ShellDef,), - ) + @shell.define + class Shelly(ShellDef["Shelly.Outputs"]): + executable = cmd + file_orig: File = shell.arg( + position=2, + help="new file", + argstr="", + ) + + class Outputs(ShellOutputs): + file_copy: File = shell.outarg( + path_template="{file_orig}_copy", + keep_extension=False, + help="output file", + argstr="", + ) - shelly = ShellDef( - name="shelly", + shelly = Shelly( executable=cmd, - input_spec=my_input_spec, file_orig=file, - cache_dir=tmp_path, ) - res = results_function(shelly, plugin) - assert res.output.stdout == "" - assert res.output.file_copy.fspath.exists() - assert res.output.file_copy.fspath.name == "file_copy" + outputs = results_function(shelly, plugin=plugin, cache_dir=tmp_path) + assert outputs.stdout == "" + assert outputs.file_copy.fspath.exists() + assert outputs.file_copy.fspath.name == "file_copy" @pytest.mark.parametrize("results_function", [run_no_submitter, run_submitter]) def test_shell_cmd_inputspec_9c(tmp_path, plugin, results_function): """ - providing output name using input_spec (output_file_template in metadata) + providing output name using input_spec (path_template in metadata) and the keep_extension is set to False, so the extension is removed completely, no suffix in the template. """ @@ -1367,52 +1081,37 @@ def test_shell_cmd_inputspec_9c(tmp_path, plugin, results_function): file = tmp_path / "file.txt" file.write_text("content\n") - my_input_spec = SpecInfo( - name="Input", - fields=[ - ( - "file_orig", - attr.ib( - type=File, - metadata={"position": 2, "help": "new file", "argstr": ""}, - ), - ), - ( - "file_copy", - attr.ib( - type=str, - metadata={ - "output_file_template": "{file_orig}", - "keep_extension": False, - "help": "output file", - "argstr": "", - }, - ), - ), - ], - bases=(ShellDef,), - ) + @shell.define + class Shelly(ShellDef["Shelly.Outputs"]): + executable = cmd + file_orig: File = shell.arg( + position=2, + help="new file", + argstr="", + ) - shelly = ShellDef( - name="shelly", - executable=cmd, - input_spec=my_input_spec, - file_orig=file, - cache_dir=tmp_path, - ) + class Outputs(ShellOutputs): + file_copy: File = shell.outarg( + path_template="{file_orig}", + keep_extension=False, + help="output file", + argstr="", + ) - res = results_function(shelly, plugin) - assert res.output.stdout == "" - assert res.output.file_copy.fspath.exists() - assert res.output.file_copy.fspath.name == "file" - assert res.output.file_copy.fspath.parent == shelly.output_dir + shelly = Shelly(executable=cmd, file_orig=file) + + outputs = results_function(shelly, plugin=plugin, cache_dir=tmp_path) + assert outputs.stdout == "" + assert outputs.file_copy.fspath.exists() + assert outputs.file_copy.fspath.name == "file" + assert outputs.file_copy.fspath.parent.parent == tmp_path @pytest.mark.parametrize("results_function", [run_no_submitter, run_submitter]) def test_shell_cmd_inputspec_9d(tmp_path, plugin, results_function): """ providing output name explicitly by manually setting value in input_spec - (instead of using default provided byoutput_file_template in metadata) + (instead of using default provided bypath_template in metadata) """ cmd = "cp" ddir = tmp_path / "data_inp" @@ -1420,46 +1119,34 @@ def test_shell_cmd_inputspec_9d(tmp_path, plugin, results_function): file = ddir / ("file.txt") file.write_text("content\n") - my_input_spec = SpecInfo( - name="Input", - fields=[ - ( - "file_orig", - attr.ib( - type=File, - metadata={"position": 2, "help": "new file", "argstr": ""}, - ), - ), - ( - "file_copy", - attr.ib( - type=str, - metadata={ - "output_file_template": "{file_orig}_copy", - "help": "output file", - "argstr": "", - }, - ), - ), - ], - bases=(ShellDef,), - ) + @shell.define + class Shelly(ShellDef["Shelly.Outputs"]): + executable = cmd + file_orig: File = shell.arg( + position=2, + help="new file", + argstr="", + ) - shelly = ShellDef( - name="shelly", + class Outputs(ShellOutputs): + file_copy: File = shell.outarg( + path_template="{file_orig}_copy", + help="output file", + argstr="", + ) + + shelly = Shelly( executable=cmd, - input_spec=my_input_spec, file_orig=file, file_copy="my_file_copy.txt", - cache_dir=tmp_path, ) - res = results_function(shelly, plugin) - assert res.output.stdout == "" - assert res.output.file_copy.fspath.exists() - assert res.output.file_copy.fspath.name == "my_file_copy.txt" + outputs = results_function(shelly, plugin=plugin, cache_dir=tmp_path) + assert outputs.stdout == "" + assert outputs.file_copy.fspath.exists() + assert outputs.file_copy.fspath.name == "my_file_copy.txt" # checking if it's created in a good place - assert shelly.output_dir == res.output.file_copy.fspath.parent + assert outputs.file_copy.fspath.parent.parent == tmp_path @pytest.mark.parametrize("results_function", [run_no_submitter, run_submitter]) @@ -1476,37 +1163,23 @@ def test_shell_cmd_inputspec_10(plugin, results_function, tmp_path): cmd_exec = "cat" files_list = [file_1, file_2] - my_input_spec = SpecInfo( - name="Input", - fields=[ - ( - "files", - attr.ib( - type=ty.List[File], - metadata={ - "position": 1, - "argstr": "...", - "sep": " ", - "help": "list of files", - "mandatory": True, - }, - ), - ) - ], - bases=(ShellDef,), - ) + @shell.define + class Shelly(ShellDef["Shelly.Outputs"]): + executable = cmd_exec + files: ty.List[File] = shell.arg( + position=1, + argstr="...", + sep=" ", + help="list of files", + ) - shelly = ShellDef( - name="shelly", - executable=cmd_exec, + shelly = Shelly( files=files_list, - input_spec=my_input_spec, - cache_dir=tmp_path, ) - assert shelly.definition.executable == cmd_exec - res = results_function(shelly, plugin) - assert res.output.stdout == "hello from boston" + assert shelly.executable == cmd_exec + outputs = results_function(shelly, plugin=plugin, cache_dir=tmp_path) + assert outputs.stdout == "hello from boston" def test_shell_cmd_inputspec_10_err(tmp_path): @@ -1525,83 +1198,54 @@ def test_shell_cmd_inputspec_10_err(tmp_path): cmd_exec = "cat" - my_input_spec = SpecInfo( - name="Input", - fields=[ - ( - "files", - attr.ib( - type=File, - metadata={ - "position": 1, - "argstr": "", - "help": "a file", - "mandatory": True, - }, - ), - ) - ], - bases=(ShellDef,), - ) + @shell.define + class Shelly(ShellDef["Shelly.Outputs"]): + executable = cmd_exec + files: File = shell.arg( + position=1, + argstr="", + help="a file", + ) with pytest.raises(FileNotFoundError): - shelly = ShellDef( - name="shelly", executable=cmd_exec, files=file_2, input_spec=my_input_spec - ) + Shelly(executable=cmd_exec, files=file_2) def test_shell_cmd_inputspec_11(tmp_path): - input_fields = [ - ( - "inputFiles", - attr.ib( - type=MultiInputObj[str], - metadata={ - "argstr": "...", - "help": "The list of input image files to be segmented.", - }, - ), - ) - ] - output_fields = [ - ( - "outputFiles", - attr.ib( - type=MultiOutputFile, - metadata={ - "help": "Corrected Output Images: should specify the same number of images as inputVolume, if only one element is given, then it is used as a file pattern where %s is replaced by the imageVolumeType, and %d by the index list location.", - "output_file_template": "{inputFiles}", - }, - ), + @shell.define + class Shelly(ShellDef["Shelly.Outputs"]): + inputFiles: MultiInputObj[str] = shell.arg( + argstr="...", + help="The list of input image files to be segmented.", ) - ] - input_spec = SpecInfo(name="Input", fields=input_fields, bases=(ShellDef,)) - output_spec = SpecInfo(name="Output", fields=output_fields, bases=(ShellOutputs,)) + executable = "touch" - task = ShellDef( - name="echoMultiple", - executable="touch", - input_spec=input_spec, - output_spec=output_spec, - ) + class Outputs(ShellOutputs): + outputFiles: MultiOutputFile = shell.outarg( + help="""Corrected Output Images: should specify the same number of + images as inputVolume, if only one element is given, then it is used as + a file pattern where %s is replaced by the imageVolumeType, + and %d by the index list location.""", + path_template="{inputFiles}", + ) - wf = Workflow(name="wf", input_spec=["inputFiles"], inputFiles=["test1", "test2"]) + @workflow.define + def Workflow(inputFiles): - task.definition.inputFiles = wf.lzin.inputFiles + echoMultiple = workflow.add(Shelly(inputFiles=inputFiles)) + return echoMultiple.outputFiles - wf.add(task) - wf.set_output([("out", wf.echoMultiple.lzout.outputFiles)]) + wf = Workflow(inputFiles=[File.mock("test1"), File.mock("test2")]) # XXX: Figure out why this fails with "cf". Occurs in CI when using Ubuntu + Python >= 3.10 # (but not when using macOS + Python >= 3.10). Same error occurs in test_shell_cmd_outputspec_7a # see https://github.com/nipype/pydra/issues/671 - with Submitter(worker="serial") as sub: - sub(wf) - result = wf.result() + with Submitter(worker="debug") as sub: + result = sub(wf) - for out_file in outputs.out: + for out_file in result.outputs.out: assert out_file.fspath.name == "test1" or out_file.fspath.name == "test2" @@ -1609,7 +1253,7 @@ def test_shell_cmd_inputspec_11(tmp_path): def test_shell_cmd_inputspec_12(tmp_path: Path, plugin, results_function): """ providing output name using input_spec - output_file_template is provided as a function that returns + path_template is provided as a function that returns various templates depending on the values of inputs fields """ cmd = "cp" @@ -1624,87 +1268,60 @@ def template_function(inputs): else: return "{file_orig}_odd" - my_input_spec = SpecInfo( - name="Input", - fields=[ - ( - "file_orig", - attr.ib( - type=File, - metadata={"position": 2, "help": "new file", "argstr": ""}, - ), - ), - ( - "number", - attr.ib( - type=int, - metadata={"help": "a number", "mandatory": True}, - ), - ), - ( - "file_copy", - attr.ib( - type=str, - metadata={ - "output_file_template": template_function, - "help": "output file", - "argstr": "", - }, - ), - ), - ], - bases=(ShellDef,), - ) + @shell.define + class Shelly(ShellDef["Shelly.Outputs"]): + executable = cmd + file_orig: File = shell.arg( + position=2, + help="new file", + argstr="", + ) + number: int = shell.arg( + help="a number", + ) - shelly = ShellDef( - name="shelly", + class Outputs(ShellOutputs): + file_copy: File = shell.outarg( + path_template=template_function, + help="output file", + argstr="", + ) + + shelly = Shelly( executable=cmd, - input_spec=my_input_spec, file_orig=file, number=2, - cache_dir=tmp_path, ) - res = results_function(shelly, plugin) - assert res.output.stdout == "" - fspath = res.output.file_copy.fspath + outputs = results_function(shelly, plugin=plugin, cache_dir=tmp_path) + assert outputs.stdout == "" + fspath = outputs.file_copy.fspath assert fspath.exists() assert fspath.name == "file_even.txt" # checking if it's created in a good place - assert shelly.output_dir == fspath.parent + assert fspath.parent.parent == tmp_path def test_shell_cmd_inputspec_with_iterable(): """Test formatting of argstr with different iterable types.""" - input_spec = SpecInfo( - name="Input", - fields=[ - ( - "iterable_1", - ty.Iterable[int], - { - "help": "iterable input 1", - "argstr": "--in1", - }, - ), - ( - "iterable_2", - ty.Iterable[str], - { - "help": "iterable input 2", - "argstr": "--in2...", - }, - ), - ], - bases=(ShellDef,), - ) + @shell.define + class Shelly(ShellDef["Shelly.Outputs"]): + executable = "test" + iterable_1: ty.Iterable[int] = shell.arg( + help="iterable input 1", + argstr="--in1", + ) + iterable_2: ty.Iterable[str] = shell.arg( + help="iterable input 2", + argstr="--in2...", + ) - task = ShellDef(name="test", input_spec=input_spec, executable="test") + task = Shelly() for iterable_type in (list, tuple): - task.definition.iterable_1 = iterable_type(range(3)) - task.definition.iterable_2 = iterable_type(["bar", "foo"]) + task.iterable_1 = iterable_type(range(3)) + task.iterable_2 = iterable_type(["bar", "foo"]) assert task.cmdline == "test --in1 0 1 2 --in2 bar --in2 foo" @@ -1720,50 +1337,30 @@ def test_shell_cmd_inputspec_copyfile_1(plugin, results_function, tmp_path): cmd = ["sed", "-is", "s/hello/hi/"] - my_input_spec = SpecInfo( - name="Input", - fields=[ - ( - "orig_file", - attr.ib( - type=File, - metadata={ - "position": 1, - "argstr": "", - "help": "orig file", - "mandatory": True, - "copyfile": True, - }, - ), - ), - ( - "out_file", - attr.ib( - type=str, - metadata={ - "output_file_template": "{orig_file}", - "help": "output file", - }, - ), - ), - ], - bases=(ShellDef,), - ) + @shell.define + class Shelly(ShellDef["Shelly.Outputs"]): + executable = cmd + orig_file: File = shell.arg( + position=1, + argstr="", + help="orig file", + copyfile=True, + ) - shelly = ShellDef( - name="shelly", - executable=cmd, - input_spec=my_input_spec, - orig_file=str(file), - cache_dir=tmp_path, - ) + class Outputs(ShellOutputs): + out_file: File = shell.outarg( + path_template="{orig_file}", + help="output file", + ) + + shelly = Shelly(executable=cmd, orig_file=str(file)) - res = results_function(shelly, plugin) - assert res.output.stdout == "" - assert res.output.out_file.fspath.exists() + outputs = results_function(shelly, plugin=plugin, cache_dir=tmp_path) + assert outputs.stdout == "" + assert outputs.out_file.fspath.exists() # the file is copied, and than it is changed in place - assert res.output.out_file.fspath.parent == shelly.output_dir - with open(res.output.out_file) as f: + assert outputs.out_file.fspath.parent.parent == tmp_path + with open(outputs.out_file) as f: assert "hi from pydra\n" == f.read() # the original file is unchanged with open(file) as f: @@ -1782,59 +1379,39 @@ def test_shell_cmd_inputspec_copyfile_1a(plugin, results_function, tmp_path): cmd = ["sed", "-is", "s/hello/hi/"] - my_input_spec = SpecInfo( - name="Input", - fields=[ - ( - "orig_file", - attr.ib( - type=File, - metadata={ - "position": 1, - "argstr": "", - "help": "orig file", - "mandatory": True, - "copyfile": "hardlink", - }, - ), - ), - ( - "out_file", - attr.ib( - type=str, - metadata={ - "output_file_template": "{orig_file}", - "help": "output file", - }, - ), - ), - ], - bases=(ShellDef,), - ) + @shell.define + class Shelly(ShellDef["Shelly.Outputs"]): + executable = cmd + orig_file: File = shell.arg( + position=1, + argstr="", + help="orig file", + copy_mode="hardlink", + ) - shelly = ShellDef( - name="shelly", - executable=cmd, - input_spec=my_input_spec, - orig_file=str(file), - cache_dir=tmp_path, - ) + class Outputs(ShellOutputs): + out_file: File = shell.outarg( + path_template="{orig_file}", + help="output file", + ) + + shelly = Shelly(executable=cmd, orig_file=str(file)) - res = results_function(shelly, plugin) - assert res.output.stdout == "" - assert res.output.out_file.fspath.exists() + outputs = results_function(shelly, plugin=plugin, cache_dir=tmp_path) + assert outputs.stdout == "" + assert outputs.out_file.fspath.exists() # the file is uses a soft link, but it creates and an extra copy before modifying - assert res.output.out_file.fspath.parent == shelly.output_dir + assert outputs.out_file.fspath.parent.parent == tmp_path - assert res.output.out_file.fspath.parent.joinpath( - res.output.out_file.fspath.name + "s" + assert outputs.out_file.fspath.parent.joinpath( + outputs.out_file.fspath.name + "s" ).exists() - with open(res.output.out_file) as f: + with open(outputs.out_file) as f: assert "hi from pydra\n" == f.read() # the file is uses a soft link, but it creates and an extra copy # it might depend on the OS - linked_file_copy = res.output.out_file.fspath.parent.joinpath( - res.output.out_file.fspath.name + "s" + linked_file_copy = outputs.out_file.fspath.parent.joinpath( + outputs.out_file.fspath.name + "s" ) if linked_file_copy.exists(): with open(linked_file_copy) as f: @@ -1861,49 +1438,32 @@ def test_shell_cmd_inputspec_copyfile_1b(plugin, results_function, tmp_path): cmd = ["sed", "-is", "s/hello/hi/"] - my_input_spec = SpecInfo( - name="Input", - fields=[ - ( - "orig_file", - attr.ib( - type=File, - metadata={ - "position": 1, - "argstr": "", - "help": "orig file", - "mandatory": True, - }, - ), - ), - ( - "out_file", - attr.ib( - type=str, - metadata={ - "output_file_template": "{orig_file}", - "help": "output file", - }, - ), - ), - ], - bases=(ShellDef,), - ) + @shell.define + class Shelly(ShellDef["Shelly.Outputs"]): + executable = cmd + orig_file: File = shell.arg( + position=1, + argstr="", + help="orig file", + ) + + class Outputs(ShellOutputs): + out_file: File = shell.outarg( + path_template="{orig_file}", + help="output file", + ) - shelly = ShellDef( - name="shelly", + shelly = Shelly( executable=cmd, - input_spec=my_input_spec, orig_file=str(file), - cache_dir=tmp_path, ) - res = results_function(shelly, plugin) - assert res.output.stdout == "" - assert res.output.out_file.fspath.exists() + outputs = results_function(shelly, plugin=plugin, cache_dir=tmp_path) + assert outputs.stdout == "" + assert outputs.out_file.fspath.exists() # the file is not copied, it is changed in place - assert res.output.out_file == file - with open(res.output.out_file) as f: + assert outputs.out_file == file + with open(outputs.out_file) as f: assert "hi from pydra\n" == f.read() @@ -1912,38 +1472,24 @@ def test_shell_cmd_inputspec_state_1(plugin, results_function, tmp_path): """adding state to the input from input_spec""" cmd_exec = "echo" hello = ["HELLO", "hi"] - my_input_spec = SpecInfo( - name="Input", - fields=[ - ( - "text", - attr.ib( - type=str, - metadata={ - "position": 1, - "help": "text", - "mandatory": True, - "argstr": "", - }, - ), - ) - ], - bases=(ShellDef,), - ) + + @shell.define + class Shelly(ShellDef["Shelly.Outputs"]): + executable = cmd_exec + text: str = shell.arg( + position=1, + help="text", + argstr="", + ) # separate command into exec + args - shelly = ShellDef( - name="shelly", - executable=cmd_exec, - input_spec=my_input_spec, - cache_dir=tmp_path, - ).split("text", text=hello) - assert shelly.inputs.executable == cmd_exec + shelly = Shelly().split("text", text=hello) + assert shelly.executable == cmd_exec # todo: this doesn't work when state # assert shelly.cmdline == "echo HELLO" - res = results_function(shelly, plugin) - assert res[0].output.stdout == "HELLO\n" - assert res[1].output.stdout == "hi\n" + outputs = results_function(shelly, plugin=plugin, cache_dir=tmp_path) + assert outputs.stdout[0] == "HELLO\n" + assert outputs.stdout[1] == "hi\n" def test_shell_cmd_inputspec_typeval_1(): @@ -1952,22 +1498,17 @@ def test_shell_cmd_inputspec_typeval_1(): """ cmd_exec = "echo" - my_input_spec = SpecInfo( - name="Input", - fields=[ - ( - "text", - attr.ib( - type=int, - metadata={"position": 1, "argstr": "", "help": "text"}, - ), - ) - ], - bases=(ShellDef,), - ) + @shell.define + class Shelly(ShellDef["Shelly.Outputs"]): + executable = cmd_exec + text: int = shell.arg( + position=1, + argstr="", + help="text", + ) with pytest.raises(TypeError): - ShellDef(executable=cmd_exec, text="hello", input_spec=my_input_spec) + Shelly() def test_shell_cmd_inputspec_typeval_2(): @@ -1976,14 +1517,14 @@ def test_shell_cmd_inputspec_typeval_2(): """ cmd_exec = "echo" - my_input_spec = SpecInfo( - name="Input", - fields=[("text", int, {"position": 1, "argstr": "", "help": "text"})], - bases=(ShellDef,), - ) + @shell.define + class Shelly(ShellDef["Shelly.Outputs"]): + executable = cmd_exec + + text: int = shell.arg(position=1, argstr="", help="text") with pytest.raises(TypeError): - ShellDef(executable=cmd_exec, text="hello", input_spec=my_input_spec) + Shelly(text="hello") @pytest.mark.parametrize("results_function", [run_no_submitter, run_submitter]) @@ -1992,30 +1533,23 @@ def test_shell_cmd_inputspec_state_1a(plugin, results_function, tmp_path): using shorter syntax for input_spec (without default) """ cmd_exec = "echo" - my_input_spec = SpecInfo( - name="Input", - fields=[ - ( - "text", - str, - {"position": 1, "help": "text", "mandatory": True, "argstr": ""}, - ) - ], - bases=(ShellDef,), - ) + + @shell.define + class Shelly(ShellDef["Shelly.Outputs"]): + executable = cmd_exec + text: str = shell.arg( + position=1, + help="text", + argstr="", + ) # separate command into exec + args - shelly = ShellDef( - name="shelly", - executable=cmd_exec, - input_spec=my_input_spec, - cache_dir=tmp_path, - ).split(text=["HELLO", "hi"]) - assert shelly.inputs.executable == cmd_exec + shelly = Shelly().split(text=["HELLO", "hi"]) + assert shelly.executable == cmd_exec - res = results_function(shelly, plugin) - assert res[0].output.stdout == "HELLO\n" - assert res[1].output.stdout == "hi\n" + outputs = results_function(shelly, plugin=plugin, cache_dir=tmp_path) + assert outputs.stdout[0] == "HELLO\n" + assert outputs.stdout[1] == "hi\n" @pytest.mark.parametrize("results_function", [run_no_submitter, run_submitter]) @@ -2024,37 +1558,25 @@ def test_shell_cmd_inputspec_state_2(plugin, results_function, tmp_path): adding splitter to input that is used in the output_file_tamplate """ cmd = "touch" - args = ["newfile_1.txt", "newfile_2.txt"] + args = [File.mock("newfile_1.txt"), File.mock("newfile_2.txt")] - my_input_spec = SpecInfo( - name="Input", - fields=[ - ( - "out1", - attr.ib( - type=str, - metadata={ - "output_file_template": "{args}", - "help": "output file", - }, - ), + @shell.define + class Shelly(ShellDef["Shelly.Outputs"]): + executable = cmd + + class Outputs(ShellOutputs): + out1: File = shell.outarg( + path_template="{args}", + help="output file", ) - ], - bases=(ShellDef,), - ) - shelly = ShellDef( - name="shelly", - executable=cmd, - input_spec=my_input_spec, - cache_dir=tmp_path, - ).split(args=args) + shelly = Shelly(executable=cmd).split(args=args) - res = results_function(shelly, plugin) + outputs = results_function(shelly, plugin=plugin, cache_dir=tmp_path) for i in range(len(args)): - assert res[i].output.stdout == "" - assert res[i].output.out1.fspath.exists() - assert res[i].output.out1.fspath.parent == shelly.output_dir[i] + assert outputs.stdout[i] == "" + assert outputs.out1[i].fspath.exists() + assert outputs.out1[i].fspath.parent.parent == tmp_path[i] @pytest.mark.parametrize("results_function", [run_no_submitter, run_submitter]) @@ -2070,38 +1592,23 @@ def test_shell_cmd_inputspec_state_3(plugin, results_function, tmp_path): cmd_exec = "cat" - my_input_spec = SpecInfo( - name="Input", - fields=[ - ( - "file", - attr.ib( - type=File, - metadata={ - "position": 1, - "help": "files", - "mandatory": True, - "argstr": "", - }, - ), - ) - ], - bases=(ShellDef,), - ) + @shell.define + class Shelly(ShellDef["Shelly.Outputs"]): + executable = cmd_exec + file: File = shell.arg( + position=1, + help="files", + argstr="", + ) - shelly = ShellDef( - name="shelly", - executable=cmd_exec, - input_spec=my_input_spec, - cache_dir=tmp_path, - ).split(file=[file_1, file_2]) + shelly = Shelly().split(file=[file_1, file_2]) - assert shelly.inputs.executable == cmd_exec + assert shelly.executable == cmd_exec # todo: this doesn't work when state # assert shelly.cmdline == "echo HELLO" - res = results_function(shelly, plugin) - assert res[0].output.stdout == "hello from pydra" - assert res[1].output.stdout == "have a nice one" + outputs = results_function(shelly, plugin=plugin, cache_dir=tmp_path) + assert outputs.stdout[0] == "hello from pydra" + assert outputs.stdout[1] == "have a nice one" @pytest.mark.parametrize("results_function", [run_no_submitter, run_submitter]) @@ -2119,51 +1626,34 @@ def test_shell_cmd_inputspec_copyfile_state_1(plugin, results_function, tmp_path files = [str(file1), str(file2)] cmd = ["sed", "-is", "s/hello/hi/"] - my_input_spec = SpecInfo( - name="Input", - fields=[ - ( - "orig_file", - attr.ib( - type=File, - metadata={ - "position": 1, - "argstr": "", - "help": "orig file", - "mandatory": True, - "copyfile": "copy", - }, - ), - ), - ( - "out_file", - attr.ib( - type=str, - metadata={ - "output_file_template": "{orig_file}", - "help": "output file", - }, - ), - ), - ], - bases=(ShellDef,), - ) + @shell.define + class Shelly(ShellDef["Shelly.Outputs"]): + executable = cmd + orig_file: File = shell.arg( + position=1, + argstr="", + help="orig file", + copy_mode="copy", + ) - shelly = ShellDef( - name="shelly", + class Outputs(ShellOutputs): + out_file: File = shell.outarg( + path_template="{orig_file}", + help="output file", + ) + + shelly = Shelly( executable=cmd, - input_spec=my_input_spec, - cache_dir=tmp_path, ).split("orig_file", orig_file=files) txt_l = ["from pydra", "world"] - res_l = results_function(shelly, plugin) - for i, res in enumerate(res_l): - assert res.output.stdout == "" - assert res.output.out_file.fspath.exists() + outputs = results_function(shelly, plugin=plugin, cache_dir=tmp_path) + for i in range(len(files)): + assert outputs.stdout[i] == "" + assert outputs.out_file[i].fspath.exists() # the file is copied, and than it is changed in place - assert res.output.out_file.fspath.parent == shelly.output_dir[i] - with open(res.output.out_file) as f: + assert outputs.out_file[i].fspath.parent.parent == tmp_path[i] + with open(outputs.out_file[i]) as f: assert f"hi {txt_l[i]}\n" == f.read() # the original file is unchanged with open(files[i]) as f: @@ -2175,480 +1665,324 @@ def test_shell_cmd_inputspec_copyfile_state_1(plugin, results_function, tmp_path @pytest.mark.flaky(reruns=2) # when dask def test_wf_shell_cmd_2(plugin_dask_opt, tmp_path): - """a workflow with input with defined output_file_template (str) + """a workflow with input with defined path_template (str) that requires wf.lzin """ - wf = Workflow(name="wf", input_spec=["cmd", "args"]) - - wf.inputs.cmd = "touch" - wf.inputs.args = "newfile.txt" - wf.cache_dir = tmp_path - - my_input_spec = SpecInfo( - name="Input", - fields=[ - ( - "out1", - attr.ib( - type=str, - metadata={ - "output_file_template": "{args}", - "help": "output file", - }, - ), + + @shell.define + class Shelly(ShellDef["Shelly.Outputs"]): + executable = "touch" + + class Outputs(ShellOutputs): + out1: File = shell.outarg( + path_template="{args}", + help="output file", ) - ], - bases=(ShellDef,), - ) - wf.add( - ShellDef( - name="shelly", - input_spec=my_input_spec, - executable=wf.lzin.cmd, - args=wf.lzin.args, + @workflow.define + def Workflow(cmd, args): + + shelly = workflow.add( + Shelly( + executable=cmd, + additional_args=args, + ) ) - ) - wf.set_output([("out_f", wf.shelly.lzout.out1), ("out", wf.shelly.lzout.stdout)]) + return shelly.out1, shelly.stdout + + wf = Workflow(cmd="touch", args=File.mock("newfile.txt")) - with Submitter(worker=plugin_dask_opt) as sub: - wf(submitter=sub) + with Submitter(plugin=plugin_dask_opt) as sub: + res = sub(wf) - res = wf.result() - assert res.output.out == "" - assert res.output.out_f.fspath.exists() - assert res.output.out_f.fspath.parent == wf.output_dir + assert res.outputs.out == "" + assert res.outputs.out_f.fspath.exists() + assert res.outputs.out_f.fspath.parent.parent == tmp_path def test_wf_shell_cmd_2a(plugin, tmp_path): - """a workflow with input with defined output_file_template (tuple) + """a workflow with input with defined path_template (tuple) that requires wf.lzin """ - wf = Workflow(name="wf", input_spec=["cmd", "args"]) - - wf.inputs.cmd = "touch" - wf.inputs.args = "newfile.txt" - wf.cache_dir = tmp_path - - my_input_spec = SpecInfo( - name="Input", - fields=[ - ( - "out1", - attr.ib( - type=str, - metadata={ - "output_file_template": "{args}", - "help": "output file", - }, - ), + + @shell.define + class Shelly(ShellDef["Shelly.Outputs"]): + executable = "shelly" + + class Outputs(ShellOutputs): + out1: File = shell.outarg( + path_template="{args}", + help="output file", ) - ], - bases=(ShellDef,), - ) - wf.add( - ShellDef( - name="shelly", - input_spec=my_input_spec, - executable=wf.lzin.cmd, - args=wf.lzin.args, + @workflow.define + def Workflow(cmd, args): + + shelly = workflow.add( + Shelly( + executable=cmd, + additional_args=args, + ) ) - ) - wf.set_output([("out_f", wf.shelly.lzout.out1), ("out", wf.shelly.lzout.stdout)]) + return shelly.out1, shelly.stdout + + wf = Workflow(cmd="touch", args=(File.mock("newfile.txt"),)) - with Submitter(worker=plugin) as sub: - wf(submitter=sub) + with Submitter(plugin=plugin) as sub: + res = sub(wf) - res = wf.result() - assert res.output.out == "" - assert res.output.out_f.fspath.exists() + assert res.outputs.out == "" + assert res.outputs.out_f.fspath.exists() def test_wf_shell_cmd_3(plugin, tmp_path): """a workflow with 2 tasks, - first one has input with output_file_template (str, uses wf.lzin), + first one has input with path_template (str, uses wf.lzin), that is passed to the second task """ - wf = Workflow(name="wf", input_spec=["cmd1", "cmd2", "args"]) - - wf.inputs.cmd1 = "touch" - wf.inputs.cmd2 = "cp" - wf.inputs.args = "newfile.txt" - wf.cache_dir = tmp_path - - my_input_spec1 = SpecInfo( - name="Input", - fields=[ - ( - "file", - attr.ib( - type=str, - metadata={ - "output_file_template": "{args}", - "help": "output file", - }, - ), + + @shell.define + class Shelly1(ShellDef["Shelly1.Outputs"]): + executable = "shelly" + + class Outputs(ShellOutputs): + file: File = shell.outarg( + path_template="{args}", + help="output file", ) - ], - bases=(ShellDef,), - ) - my_input_spec2 = SpecInfo( - name="Input", - fields=[ - ( - "orig_file", - attr.ib( - type=File, - metadata={ - "position": 1, - "help": "output file", - "argstr": "", - }, - ), - ), - ( - "out_file", - attr.ib( - type=str, - metadata={ - "position": 2, - "argstr": "", - "output_file_template": "{orig_file}_copy", - "help": "output file", - }, - ), - ), - ], - bases=(ShellDef,), - ) + @shell.define + class Shelly2(ShellDef["Shelly2.Outputs"]): + orig_file: File = shell.arg( + position=1, + help="output file", + argstr="", + ) - wf.add( - ShellDef( - name="shelly1", - input_spec=my_input_spec1, - executable=wf.lzin.cmd1, - args=wf.lzin.args, + class Outputs(ShellOutputs): + out_file: File = shell.outarg( + position=2, + argstr="", + path_template="{orig_file}_copy", + help="output file", + ) + + @workflow.define(outputs=["touch_file", "out1", "cp_file", "out2"]) + def Workflow(cmd1, cmd2, args): + + shelly1 = workflow.add( + Shelly1( + executable=cmd1, + additional_args=args, + ) ) - ) - wf.add( - ShellDef( - name="shelly2", - input_spec=my_input_spec2, - executable=wf.lzin.cmd2, - orig_file=wf.shelly1.lzout.file, + shelly2 = workflow.add( + Shelly2( + executable=cmd2, + orig_file=shelly1.file, + ) ) - ) - wf.set_output( - [ - ("touch_file", wf.shelly1.lzout.file), - ("out1", wf.shelly1.lzout.stdout), - ("cp_file", wf.shelly2.lzout.out_file), - ("out2", wf.shelly2.lzout.stdout), - ] - ) + return shelly1.file, shelly1.stdout, shelly2.out_file, shelly2.stdout + + wf = Workflow(cmd1="touch", cmd2="cp", args=File.mock("newfile.txt")) - with Submitter(worker=plugin) as sub: - wf(submitter=sub) + with Submitter(plugin=plugin) as sub: + res = sub(wf) - res = wf.result() - assert res.output.out1 == "" - assert res.output.touch_file.fspath.exists() - assert res.output.touch_file.fspath.parent == wf.output_dir - assert res.output.out2 == "" - assert res.output.cp_file.fspath.exists() - assert res.output.cp_file.fspath.parent == wf.output_dir + assert res.outputs.out1 == "" + assert res.outputs.touch_file.fspath.exists() + assert res.outputs.touch_file.fspath.parent.parent == tmp_path + assert res.outputs.out2 == "" + assert res.outputs.cp_file.fspath.exists() + assert res.outputs.cp_file.fspath.parent.parent == tmp_path def test_wf_shell_cmd_3a(plugin, tmp_path): """a workflow with 2 tasks, - first one has input with output_file_template (str, uses wf.lzin), + first one has input with path_template (str, uses wf.lzin), that is passed to the second task """ - wf = Workflow(name="wf", input_spec=["cmd1", "cmd2", "args"]) - - wf.inputs.cmd1 = "touch" - wf.inputs.cmd2 = "cp" - wf.inputs.args = "newfile.txt" - wf.cache_dir = tmp_path - - my_input_spec1 = SpecInfo( - name="Input", - fields=[ - ( - "file", - attr.ib( - type=str, - metadata={ - "output_file_template": "{args}", - "help": "output file", - }, - ), + + @shell.define + class Shelly1(ShellDef["Shelly1.Outputs"]): + class Outputs(ShellOutputs): + file: File = shell.outarg( + path_template="{args}", + help="output file", ) - ], - bases=(ShellDef,), - ) - my_input_spec2 = SpecInfo( - name="Input", - fields=[ - ( - "orig_file", - attr.ib( - type=str, - metadata={ - "position": 1, - "help": "output file", - "argstr": "", - }, - ), - ), - ( - "out_file", - attr.ib( - type=str, - metadata={ - "position": 2, - "argstr": "", - "output_file_template": "{orig_file}_cp", - "help": "output file", - }, - ), - ), - ], - bases=(ShellDef,), - ) + @shell.define + class Shelly2(ShellDef["Shelly2.Outputs"]): + orig_file: str = shell.arg( + position=1, + help="output file", + argstr="", + ) + + class Outputs(ShellOutputs): + out_file: File = shell.outarg( + position=2, + argstr="", + path_template="{orig_file}_cp", + help="output file", + ) + + @workflow.define(outputs=["touch_file", "out1", "cp_file", "out2"]) + def Workflow(cmd1, cmd2, args): - wf.add( - ShellDef( - name="shelly1", - input_spec=my_input_spec1, - executable=wf.lzin.cmd1, - args=wf.lzin.args, + shelly1 = workflow.add( + Shelly1( + executable=cmd1, + additional_args=args, + ) ) - ) - wf.add( - ShellDef( - name="shelly2", - input_spec=my_input_spec2, - executable=wf.lzin.cmd2, - orig_file=wf.shelly1.lzout.file, + shelly2 = workflow.add( + Shelly2( + executable=cmd2, + orig_file=shelly1.file, + ) ) - ) - wf.set_output( - [ - ("touch_file", wf.shelly1.lzout.file), - ("out1", wf.shelly1.lzout.stdout), - ("cp_file", wf.shelly2.lzout.out_file), - ("out2", wf.shelly2.lzout.stdout), - ] - ) + return shelly1.file, shelly1.stdout, shelly2.out_file, shelly2.stdout - with Submitter(worker=plugin) as sub: - wf(submitter=sub) + wf = Workflow(cmd1="touch", cmd2="cp", args=File.mock("newfile.txt")) - res = wf.result() - assert res.output.out1 == "" - assert res.output.touch_file.fspath.exists() - assert res.output.out2 == "" - assert res.output.cp_file.fspath.exists() + with Submitter(plugin=plugin) as sub: + res = sub(wf) + + assert res.outputs.out1 == "" + assert res.outputs.touch_file.fspath.exists() + assert res.outputs.out2 == "" + assert res.outputs.cp_file.fspath.exists() def test_wf_shell_cmd_state_1(plugin, tmp_path): """a workflow with 2 tasks and splitter on the wf level, - first one has input with output_file_template (str, uses wf.lzin), + first one has input with path_template (str, uses wf.lzin), that is passed to the second task """ - wf = Workflow( - name="wf", input_spec=["cmd1", "cmd2", "args"], cache_dir=tmp_path - ).split("args", args=["newfile_1.txt", "newfile_2.txt"]) - - wf.inputs.cmd1 = "touch" - wf.inputs.cmd2 = "cp" - - my_input_spec1 = SpecInfo( - name="Input", - fields=[ - ( - "file", - attr.ib( - type=str, - metadata={ - "output_file_template": "{args}", - "help": "output file", - }, - ), + + @shell.define + class Shelly1(ShellDef["Shelly1.Outputs"]): + class Outputs(ShellOutputs): + file: File = shell.outarg( + path_template="{args}", + help="output file", ) - ], - bases=(ShellDef,), - ) - my_input_spec2 = SpecInfo( - name="Input", - fields=[ - ( - "orig_file", - attr.ib( - type=str, - metadata={ - "position": 1, - "help": "output file", - "argstr": "", - }, - ), - ), - ( - "out_file", - attr.ib( - type=str, - metadata={ - "position": 2, - "argstr": "", - "output_file_template": "{orig_file}_copy", - "help": "output file", - }, - ), - ), - ], - bases=(ShellDef,), - ) + @shell.define + class Shelly2(ShellDef["Shelly2.Outputs"]): + orig_file: str = shell.arg( + position=1, + help="output file", + argstr="", + ) - wf.add( - ShellDef( - name="shelly1", - input_spec=my_input_spec1, - executable=wf.lzin.cmd1, - args=wf.lzin.args, + class Outputs(ShellOutputs): + out_file: File = shell.outarg( + position=2, + argstr="", + path_template="{orig_file}_copy", + help="output file", + ) + + @workflow.define(outputs=["touch_file", "out1", "cp_file", "out2"]) + def Workflow(cmd1, cmd2, args): + + shelly1 = workflow.add( + Shelly1( + executable=cmd1, + additional_args=args, + ) ) - ) - wf.add( - ShellDef( - name="shelly2", - input_spec=my_input_spec2, - executable=wf.lzin.cmd2, - orig_file=wf.shelly1.lzout.file, + shelly2 = workflow.add( + Shelly2( + executable=cmd2, + orig_file=shelly1.file, + ) ) - ) - wf.set_output( - [ - ("touch_file", wf.shelly1.lzout.file), - ("out1", wf.shelly1.lzout.stdout), - ("cp_file", wf.shelly2.lzout.out_file), - ("out2", wf.shelly2.lzout.stdout), - ] + return shelly1.file, shelly1.stdout, shelly2.out_file, shelly2.stdout + + wf = Workflow(cmd1="touch", cmd2="cp").split( + args=[File.mock("newfile_1.txt"), File.mock("newfile_2.txt")] ) - with Submitter(worker=plugin) as sub: - wf(submitter=sub) + with Submitter(plugin=plugin, cache_dir=tmp_path) as sub: + res = sub(wf) - res_l = wf.result() - for i, res in enumerate(res_l): - assert res.output.out1 == "" - assert res.output.touch_file.fspath.exists() - assert res.output.touch_file.fspath.parent == wf.output_dir[i] - assert res.output.out2 == "" - assert res.output.cp_file.fspath.exists() - assert res.output.cp_file.fspath.parent == wf.output_dir[i] + for i in range(2): + assert res.outputs.out1[i] == "" + assert res.outputs.touch_file[i].fspath.exists() + assert res.outputs.touch_file[i].fspath.parent.parent == tmp_path[i] + assert res.outputs.out2[i] == "" + assert res.outputs.cp_file[i].fspath.exists() + assert res.outputs.cp_file[i].fspath.parent.parent == tmp_path[i] def test_wf_shell_cmd_ndst_1(plugin, tmp_path): """a workflow with 2 tasks and a splitter on the node level, - first one has input with output_file_template (str, uses wf.lzin), + first one has input with path_template (str, uses wf.lzin), that is passed to the second task """ - wf = Workflow(name="wf", input_spec=["cmd1", "cmd2", "args"]) - - wf.inputs.cmd1 = "touch" - wf.inputs.cmd2 = "cp" - wf.inputs.args = ["newfile_1.txt", "newfile_2.txt"] - wf.cache_dir = tmp_path - - my_input_spec1 = SpecInfo( - name="Input", - fields=[ - ( - "file", - attr.ib( - type=str, - metadata={ - "output_file_template": "{args}", - "help": "output file", - }, - ), + + @shell.define + class Shelly1(ShellDef["Shelly1.Outputs"]): + class Outputs(ShellOutputs): + file: File = shell.outarg( + path_template="{args}", + help="output file", ) - ], - bases=(ShellDef,), - ) - my_input_spec2 = SpecInfo( - name="Input", - fields=[ - ( - "orig_file", - attr.ib( - type=str, - metadata={ - "position": 1, - "help": "output file", - "argstr": "", - }, - ), - ), - ( - "out_file", - attr.ib( - type=str, - metadata={ - "position": 2, - "argstr": "", - "output_file_template": "{orig_file}_copy", - "help": "output file", - }, - ), - ), - ], - bases=(ShellDef,), - ) + @shell.define + class Shelly2(ShellDef["Shelly2.Outputs"]): + orig_file: str = shell.arg( + position=1, + help="output file", + argstr="", + ) - wf.add( - ShellDef( - name="shelly1", - input_spec=my_input_spec1, - executable=wf.lzin.cmd1, - ).split("args", args=wf.lzin.args) - ) - wf.add( - ShellDef( - name="shelly2", - input_spec=my_input_spec2, - executable=wf.lzin.cmd2, - orig_file=wf.shelly1.lzout.file, + class Outputs(ShellOutputs): + out_file: File = shell.outarg( + position=2, + argstr="", + path_template="{orig_file}_copy", + help="output file", + ) + + @workflow.define(outputs=["touch_file", "out1", "cp_file", "out2"]) + def Workflow(cmd1, cmd2, args): + + shelly1 = workflow.add( + Shelly1( + executable=cmd1, + ).split("args", args=args) ) - ) + shelly2 = workflow.add( + Shelly2( + executable=cmd2, + orig_file=shelly1.file, + ) + ) + + return shelly1.file, shelly1.stdout, shelly2.out_file, shelly2.stdout - wf.set_output( - [ - ("touch_file", wf.shelly1.lzout.file), - ("out1", wf.shelly1.lzout.stdout), - ("cp_file", wf.shelly2.lzout.out_file), - ("out2", wf.shelly2.lzout.stdout), - ] + wf = Workflow( + cmd1="touch", + cmd2="cp", + args=[File.mock("newfile_1.txt"), File.mock("newfile_2.txt")], ) - with Submitter(worker=plugin) as sub: - wf(submitter=sub) + with Submitter(plugin=plugin) as sub: + res = sub(wf) - res = wf.result() - assert res.output.out1 == ["", ""] - assert all([file.fspath.exists() for file in res.output.touch_file]) - assert res.output.out2 == ["", ""] - assert all([file.fspath.exists() for file in res.output.cp_file]) + assert res.outputs.out1 == ["", ""] + assert all([file.fspath.exists() for file in res.outputs.touch_file]) + assert res.outputs.out2 == ["", ""] + assert all([file.fspath.exists() for file in res.outputs.cp_file]) # customised output definition @@ -2659,19 +1993,18 @@ def test_shell_cmd_outputspec_1(plugin, results_function, tmp_path): """ customised output_spec, adding files to the output, providing specific pathname """ - cmd = ["touch", "newfile_tmp.txt"] - my_output_spec = SpecInfo( - name="Output", - fields=[("newfile", File, "newfile_tmp.txt")], - bases=(ShellOutputs,), - ) - shelly = ShellDef( - name="shelly", executable=cmd, output_spec=my_output_spec, cache_dir=tmp_path + cmd = ["touch", File.mock("newfile_tmp.txt")] + Shelly = shell.define( + cmd, + outputs=[ + shell.outarg(name="newfile", type=File, path_template="newfile_tmp.txt") + ], ) + shelly = Shelly() - res = results_function(shelly, plugin) - assert res.output.stdout == "" - assert res.output.newfile.fspath.exists() + outputs = results_function(shelly, plugin=plugin, cache_dir=tmp_path) + assert outputs.stdout == "" + assert outputs.newfile.fspath.exists() @pytest.mark.parametrize("results_function", [run_no_submitter, run_submitter]) @@ -2680,18 +2013,20 @@ def test_shell_cmd_outputspec_1a(plugin, results_function, tmp_path): customised output_spec, adding files to the output, providing specific pathname """ cmd = ["touch", "newfile_tmp.txt"] - my_output_spec = SpecInfo( - name="Output", - fields=[("newfile", attr.ib(type=File, default="newfile_tmp.txt"))], - bases=(ShellOutputs,), - ) - shelly = ShellDef( - name="shelly", executable=cmd, output_spec=my_output_spec, cache_dir=tmp_path - ) - res = results_function(shelly, plugin) - assert res.output.stdout == "" - assert res.output.newfile.fspath.exists() + @shell.define + class Shelly(ShellDef["Shelly.Outputs"]): + + executable = cmd + + class Outputs(ShellOutputs): + newfile: File = shell.outarg(path_template="newfile_tmp.txt") + + shelly = Shelly() + + outputs = results_function(shelly, plugin=plugin, cache_dir=tmp_path) + assert outputs.stdout == "" + assert outputs.newfile.fspath.exists() def test_shell_cmd_outputspec_1b_exception(plugin, tmp_path): @@ -2699,17 +2034,19 @@ def test_shell_cmd_outputspec_1b_exception(plugin, tmp_path): customised output_spec, adding files to the output, providing specific pathname """ cmd = ["touch", "newfile_tmp.txt"] - my_output_spec = SpecInfo( - name="Output", - fields=[("newfile", File, "newfile_tmp_.txt")], - bases=(ShellOutputs,), - ) - shelly = ShellDef( - name="shelly", executable=cmd, output_spec=my_output_spec, cache_dir=tmp_path - ) + + @shell.define + class Shelly(ShellDef["Shelly.Outputs"]): + + executable = cmd + + class Outputs(ShellOutputs): + newfile: File = shell.outarg(path_template="newfile_tmp_.txt") + + shelly = Shelly() with pytest.raises(Exception) as exinfo: - with Submitter(worker=plugin) as sub: + with Submitter(plugin=plugin) as sub: shelly(submitter=sub) assert "does not exist" in str(exinfo.value) @@ -2721,18 +2058,20 @@ def test_shell_cmd_outputspec_2(plugin, results_function, tmp_path): using a wildcard in default """ cmd = ["touch", "newfile_tmp.txt"] - my_output_spec = SpecInfo( - name="Output", - fields=[("newfile", File, "newfile_*.txt")], - bases=(ShellOutputs,), - ) - shelly = ShellDef( - name="shelly", executable=cmd, output_spec=my_output_spec, cache_dir=tmp_path - ) - res = results_function(shelly, plugin) - assert res.output.stdout == "" - assert res.output.newfile.fspath.exists() + @shell.define + class Shelly(ShellDef["Shelly.Outputs"]): + + executable = cmd + + class Outputs(ShellOutputs): + newfile: File = shell.outarg(path_template="newfile_*.txt") + + shelly = Shelly() + + outputs = results_function(shelly, plugin=plugin, cache_dir=tmp_path) + assert outputs.stdout == "" + assert outputs.newfile.fspath.exists() def test_shell_cmd_outputspec_2a_exception(plugin, tmp_path): @@ -2741,17 +2080,19 @@ def test_shell_cmd_outputspec_2a_exception(plugin, tmp_path): using a wildcard in default """ cmd = ["touch", "newfile_tmp.txt"] - my_output_spec = SpecInfo( - name="Output", - fields=[("newfile", File, "newfile_*K.txt")], - bases=(ShellOutputs,), - ) - shelly = ShellDef( - name="shelly", executable=cmd, output_spec=my_output_spec, cache_dir=tmp_path - ) + + @shell.define + class Shelly(ShellDef["Shelly.Outputs"]): + + executable = cmd + + class Outputs(ShellOutputs): + newfile: File = "newfile_*K.txt" + + shelly = Shelly() with pytest.raises(Exception) as excinfo: - with Submitter(worker=plugin) as sub: + with Submitter(plugin=plugin) as sub: shelly(submitter=sub) assert "no file matches" in str(excinfo.value) @@ -2763,20 +2104,22 @@ def test_shell_cmd_outputspec_3(plugin, results_function, tmp_path): using a wildcard in default, should collect two files """ cmd = ["touch", "newfile_tmp1.txt", "newfile_tmp2.txt"] - my_output_spec = SpecInfo( - name="Output", - fields=[("newfile", MultiOutputFile, "newfile_*.txt")], - bases=(ShellOutputs,), - ) - shelly = ShellDef( - name="shelly", executable=cmd, output_spec=my_output_spec, cache_dir=tmp_path - ) - res = results_function(shelly, plugin) - assert res.output.stdout == "" + @shell.define + class Shelly(ShellDef["Shelly.Outputs"]): + + executable = cmd + + class Outputs(ShellOutputs): + newfile: MultiOutputFile = "newfile_*.txt" + + shelly = Shelly() + + outputs = results_function(shelly, plugin=plugin, cache_dir=tmp_path) + assert outputs.stdout == "" # newfile is a list - assert len(res.output.newfile) == 2 - assert all([file.fspath.exists() for file in res.output.newfile]) + assert len(outputs.newfile) == 2 + assert all([file.fspath.exists() for file in outputs.newfile]) @pytest.mark.parametrize("results_function", [run_no_submitter, run_submitter]) @@ -2792,29 +2135,25 @@ def gather_output(field, output_dir): if field.name == "newfile": return list(Path(output_dir).expanduser().glob("newfile*.txt")) - my_output_spec = SpecInfo( - name="Output", - fields=[ - ( - "newfile", - attr.ib(type=MultiOutputFile, metadata={"callable": gather_output}), - ) - ], - bases=(ShellOutputs,), - ) - shelly = ShellDef( - name="shelly", executable=cmd, output_spec=my_output_spec, cache_dir=tmp_path - ) + @shell.define + class Shelly(ShellDef["Shelly.Outputs"]): + + executable = cmd + + class Outputs(ShellOutputs): + newfile: MultiOutputFile = shell.outarg(callable=gather_output) + + shelly = Shelly() - res = results_function(shelly, plugin) - assert res.output.stdout == "" + outputs = results_function(shelly, plugin=plugin, cache_dir=tmp_path) + assert outputs.stdout == "" # newfile is a list - assert len(res.output.newfile) == 2 - assert all([file.fspath.exists() for file in res.output.newfile]) + assert len(outputs.newfile) == 2 + assert all([file.fspath.exists() for file in outputs.newfile]) assert ( - shelly.output_names - == shelly._generated_output_names - == ["return_code", "stdout", "stderr", "newfile"] + get_output_names(shelly) + == shelly._generated_output_names(outputs.stdout, outputs.stderr) + == ["newfile", "return_code", "stderr", "stdout"] ) @@ -2831,25 +2170,22 @@ def gather_output(executable, output_dir): files = executable[1:] return [Path(output_dir) / file for file in files] - my_output_spec = SpecInfo( - name="Output", - fields=[ - ( - "newfile", - attr.ib(type=MultiOutputFile, metadata={"callable": gather_output}), - ) - ], - bases=(ShellOutputs,), - ) - shelly = ShellDef( - name="shelly", executable=cmd, output_spec=my_output_spec, cache_dir=tmp_path - ) + @shell.define + class Shelly(ShellDef["Shelly.Outputs"]): + + executable = cmd + + class Outputs(ShellOutputs): - res = results_function(shelly, plugin) - assert res.output.stdout == "" + newfile: MultiOutputFile = shell.arg(callable=gather_output) + + shelly = Shelly() + + outputs = results_function(shelly, plugin=plugin, cache_dir=tmp_path) + assert outputs.stdout == "" # newfile is a list - assert len(res.output.newfile) == 2 - assert all([file.fspath.exists() for file in res.output.newfile]) + assert len(outputs.newfile) == 2 + assert all([file.fspath.exists() for file in outputs.newfile]) def test_shell_cmd_outputspec_5b_error(): @@ -2864,12 +2200,15 @@ def gather_output(executable, output_dir, ble): files = executable[1:] return [Path(output_dir) / file for file in files] - my_output_spec = SpecInfo( - name="Output", - fields=[("newfile", attr.ib(type=File, metadata={"callable": gather_output}))], - bases=(ShellOutputs,), - ) - shelly = ShellDef(name="shelly", executable=cmd, output_spec=my_output_spec) + @shell.define + class Shelly(ShellDef["Shelly.Outputs"]): + + executable = cmd + + class Outputs(ShellOutputs): + newfile: File = shell.outarg(callable=gather_output) + + shelly = Shelly() with pytest.raises(AttributeError, match="ble"): shelly() @@ -2877,99 +2216,89 @@ def gather_output(executable, output_dir, ble): @pytest.mark.parametrize("results_function", [run_no_submitter, run_submitter]) def test_shell_cmd_outputspec_5c(plugin, results_function, tmp_path): """ - Customised output definition defined as a class, + Customised output defined as a class, using a static function to collect output files. """ - @attr.s(kw_only=True) - class MyOutputDef(ShellOutputs): - @staticmethod - def gather_output(executable, output_dir): - files = executable[1:] - return [Path(output_dir) / file for file in files] + @shell.define + class Shelly(ShellDef["Shelly.Outputs"]): - newfile: MultiOutputFile = attr.ib(metadata={"callable": gather_output}) + executable = ["touch", "newfile_tmp1.txt", "newfile_tmp2.txt"] - shelly = ShellDef( - name="shelly", - executable=["touch", "newfile_tmp1.txt", "newfile_tmp2.txt"], - output_spec=SpecInfo(name="Output", bases=(MyOutputDef,)), - cache_dir=tmp_path, - ) + class Outputs(ShellOutputs): + + @staticmethod + def gather_output(executable, output_dir): + files = executable[1:] + return [Path(output_dir) / file for file in files] + + newfile: MultiOutputFile = shell.arg(callable=gather_output) + + shelly = Shelly() - res = results_function(shelly, plugin) - assert res.output.stdout == "" + outputs = results_function(shelly, plugin=plugin, cache_dir=tmp_path) + assert outputs.stdout == "" # newfile is a list - assert len(res.output.newfile) == 2 - assert all([file.exists() for file in res.output.newfile]) + assert len(outputs.newfile) == 2 + assert all([file.exists() for file in outputs.newfile]) @pytest.mark.parametrize("results_function", [run_no_submitter, run_submitter]) def test_shell_cmd_outputspec_6(plugin, results_function, tmp_path): """ - providing output name by providing output_file_template + providing output name by providing path_template (similar to the previous example, but not touching input_spec) """ cmd = "touch" args = "newfile_tmp.txt" - my_output_spec = SpecInfo( - name="Output", - fields=[ - ( - "out1", - attr.ib( - type=File, - metadata={ - "output_file_template": "{args}", - "help": "output file", - }, - ), + @shell.define + class Shelly(ShellDef["Shelly.Outputs"]): + + executable = cmd + + class Outputs(ShellOutputs): + + out1: File = shell.ouarg( + path_template="{args}", + help="output file", ) - ], - bases=(ShellOutputs,), - ) - shelly = ShellDef( - name="shelly", + shelly = Shelly( executable=cmd, - args=args, - output_spec=my_output_spec, - cache_dir=tmp_path, + additional_args=args, ) - res = results_function(shelly, plugin) - assert res.output.stdout == "" - assert res.output.out1.fspath.exists() + outputs = results_function(shelly, plugin=plugin, cache_dir=tmp_path) + assert outputs.stdout == "" + assert outputs.out1.fspath.exists() def test_shell_cmd_outputspec_6a(): """ - providing output name by providing output_file_template + providing output name by providing path_template (using shorter syntax) """ cmd = "touch" args = "newfile_tmp.txt" - my_output_spec = SpecInfo( - name="Output", - fields=[ - ( - "out1", - File, - {"output_file_template": "{args}", "help": "output file"}, + @shell.define + class Shelly(ShellDef["Shelly.Outputs"]): + + executable = cmd + + class Outputs(ShellOutputs): + + out1: File = shell.outarg( + path_template="{args}", + help="output file", ) - ], - bases=(ShellOutputs,), - ) - shelly = ShellDef( - name="shelly", executable=cmd, args=args, output_spec=my_output_spec - ) + shelly = Shelly(additional_args=args) - res = shelly() - assert res.output.stdout == "" - assert res.output.out1.fspath.exists() + outputs = shelly() + assert outputs.stdout == "" + assert outputs.out1.fspath.exists() @pytest.mark.parametrize("results_function", [run_no_submitter, run_submitter]) @@ -2984,67 +2313,36 @@ def test_shell_cmd_outputspec_7(tmp_path, plugin, results_function): cmd = "bash" new_files_id = ["1", "2", "3"] - my_input_spec = SpecInfo( - name="Input", - fields=[ - ( - "script", - attr.ib( - type=File, - metadata={ - "help": "script file", - "mandatory": True, - "position": 1, - "argstr": "", - }, - ), - ), - ( - "files_id", - attr.ib( - type=MultiInputObj, - metadata={ - "position": 2, - "argstr": "...", - "sep": " ", - "help": "list of name indices", - "mandatory": True, - }, - ), - ), - ], - bases=(ShellDef,), - ) + @shell.define + class Shelly(ShellDef["Shelly.Outputs"]): + executable = cmd + script: File = shell.arg( + help="script file", + position=1, + argstr="", + ) + files_id: MultiInputObj = shell.arg( + position=2, + argstr="...", + sep=" ", + help="list of name indices", + ) + + class Outputs(ShellOutputs): - my_output_spec = SpecInfo( - name="Output", - fields=[ - ( - "new_files", - attr.ib( - type=MultiOutputFile, - metadata={ - "output_file_template": "file{files_id}.txt", - "help": "output file", - }, - ), + new_files: MultiOutputFile = shell.outarg( + path_template="file{files_id}.txt", + help="output file", ) - ], - bases=(ShellOutputs,), - ) - shelly = ShellDef( - name="shelly", - executable=cmd, - input_spec=my_input_spec, - output_spec=my_output_spec, + shelly = Shelly( script=file, files_id=new_files_id, ) - res = results_function(shelly, "serial") - assert res.output.stdout == "" - for file in res.output.new_files: + outputs = results_function(shelly, cache_dir=tmp_path) + assert outputs.stdout == "" + for file in outputs.new_files: assert file.fspath.exists() @@ -3060,60 +2358,29 @@ def test_shell_cmd_outputspec_7a(tmp_path, plugin, results_function): cmd = "bash" new_files_id = "1" - my_input_spec = SpecInfo( - name="Input", - fields=[ - ( - "script", - attr.ib( - type=File, - metadata={ - "help": "script file", - "mandatory": True, - "position": 1, - "argstr": "", - }, - ), - ), - ( - "files_id", - attr.ib( - type=MultiInputObj, - metadata={ - "position": 2, - "argstr": "...", - "sep": " ", - "help": "list of name indices", - "mandatory": True, - }, - ), - ), - ], - bases=(ShellDef,), - ) + @shell.define + class Shelly(ShellDef["Shelly.Outputs"]): + executable = cmd + script: File = shell.arg( + help="script file", + position=1, + argstr="", + ) + files_id: MultiInputObj = shell.arg( + position=2, + argstr="...", + sep=" ", + help="list of name indices", + ) + + class Outputs(ShellOutputs): - my_output_spec = SpecInfo( - name="Output", - fields=[ - ( - "new_files", - attr.ib( - type=MultiOutputFile, - metadata={ - "output_file_template": "file{files_id}.txt", - "help": "output file", - }, - ), + new_files: MultiOutputFile = shell.outarg( + path_template="file{files_id}.txt", + help="output file", ) - ], - bases=(ShellOutputs,), - ) - shelly = ShellDef( - name="shelly", - executable=cmd, - input_spec=my_input_spec, - output_spec=my_output_spec, + shelly = Shelly( script=file, files_id=new_files_id, ) @@ -3121,9 +2388,9 @@ def test_shell_cmd_outputspec_7a(tmp_path, plugin, results_function): # XXX: Figure out why this fails with "cf". Occurs in CI when using Ubuntu + Python >= 3.10 # (but not when using macOS + Python >= 3.10). Same error occurs in test_shell_cmd_inputspec_11 # see https://github.com/nipype/pydra/issues/671 - res = results_function(shelly, "serial") - assert res.output.stdout == "" - assert res.output.new_files.fspath.exists() + outputs = results_function(shelly, "serial") + assert outputs.stdout == "" + assert outputs.new_files.fspath.exists() @pytest.mark.parametrize("results_function", [run_no_submitter, run_submitter]) @@ -3144,48 +2411,32 @@ def get_file_index(stdout): def get_stderr(stderr): return f"stderr: {stderr}" - my_output_spec = SpecInfo( - name="Output", - fields=[ - ( - "out1", - attr.ib( - type=File, - metadata={ - "output_file_template": "{args}", - "help": "output file", - }, - ), - ), - ( - "out_file_index", - attr.ib( - type=int, - metadata={"help": "output file", "callable": get_file_index}, - ), - ), - ( - "stderr_field", - attr.ib( - type=str, - metadata={ - "help": "The standard error output", - "callable": get_stderr, - }, - ), - ), - ], - bases=(ShellOutputs,), - ) + @shell.define + class Shelly(ShellDef["Shelly.Outputs"]): - shelly = ShellDef( - name="shelly", executable=cmd, output_spec=my_output_spec, cache_dir=tmp_path - ).split("args", args=args) + executable = cmd + + class Outputs(ShellOutputs): + + out1: File = shell.outarg( + path_template="{args}", + help="output file", + ) + out_file_index: int = shell.arg( + help="output file", + callable=get_file_index, + ) + stderr_field: str = shell.arg( + help="The standard error output", + callable=get_stderr, + ) + + shelly = Shelly().split("additional_args", args=args) - results = results_function(shelly, plugin) - for index, res in enumerate(results): - assert res.output.out_file_index == index + 1 - assert res.output.stderr_field == f"stderr: {res.output.stderr}" + outputs = results_function(shelly, plugin=plugin, cache_dir=tmp_path) + for index in range(2): + assert outputs.out_file_index[index] == index + 1 + assert outputs.stderr_field[index] == f"stderr: {outputs.stderr}" def test_shell_cmd_outputspec_8b_error(): @@ -3196,19 +2447,16 @@ def test_shell_cmd_outputspec_8b_error(): cmd = "echo" args = ["newfile_1.txt", "newfile_2.txt"] - my_output_spec = SpecInfo( - name="Output", - fields=[ - ( - "out", - attr.ib(type=int, metadata={"help": "output file", "value": "val"}), - ) - ], - bases=(ShellOutputs,), - ) - shelly = ShellDef(name="shelly", executable=cmd, output_spec=my_output_spec).split( - "args", args=args - ) + @shell.define + class Shelly(ShellDef["Shelly.Outputs"]): + + executable = cmd + + class Outputs(ShellOutputs): + + out: int = shell.arg(help="output file", value="val") + + shelly = Shelly().split("additional_args", args=args) with pytest.raises(Exception) as e: shelly() assert "has to have a callable" in str(e.value) @@ -3226,32 +2474,21 @@ def get_lowest_directory(directory_path): cmd = "mkdir" args = [f"{tmp_path}/dir1", f"{tmp_path}/dir2"] - my_output_spec = SpecInfo( - name="Output", - fields=[ - ( - "resultsDir", - attr.ib( - type=Directory, - metadata={ - "output_file_template": "{args}", - "help": "output file", - }, - ), + @shell.define + class Shelly(ShellDef["Shelly.Outputs"]): + + executable = cmd + + class Outputs(ShellOutputs): + + resultsDir: Directory = shell.outarg( + path_template="{args}", + help="output file", ) - ], - bases=(ShellOutputs,), - ) - shelly = ShellDef( - name="shelly", - executable=cmd, - output_spec=my_output_spec, - resultsDir="outdir", - cache_dir=tmp_path, - ).split("args", args=args) + shelly = Shelly(resultsDir="outdir").split(additional_args=args) - results_function(shelly, plugin) + results_function(shelly, plugin=plugin, cache_dir=tmp_path) for index, arg_dir in enumerate(args): assert Path(Path(tmp_path) / Path(arg_dir)).exists() assert get_lowest_directory(arg_dir) == f"/dir{index+1}" @@ -3269,99 +2506,66 @@ def get_lowest_directory(directory_path): cmd = "mkdir" - my_input_spec = SpecInfo( - name="Input", - fields=[ - ( - "resultsDir", - attr.ib( - type=str, - metadata={ - "position": 1, - "help": "new directory", - "argstr": "", - }, - ), - ) - ], - bases=(ShellDef,), - ) + @shell.define + class Shelly(ShellDef["Shelly.Outputs"]): + executable = cmd + resultsDir: str = shell.arg( + position=1, + help="new directory", + argstr="", + ) - my_output_spec = SpecInfo( - name="Output", - fields=[ - ( - "resultsDir", - attr.ib( - type=Directory, - metadata={ - "output_file_template": "{resultsDir}", - "help": "output file", - }, - ), + class Outputs(ShellOutputs): + + resultsDir: Directory = shell.outarg( + path_template="{resultsDir}", + help="output file", ) - ], - bases=(ShellOutputs,), - ) - shelly = ShellDef( - name=cmd, - executable=cmd, - input_spec=my_input_spec, - output_spec=my_output_spec, - cache_dir=tmp_path, - resultsDir="test", # Path(tmp_path) / "test" TODO: Not working without absolute path support - ) + shelly = Shelly(resultsDir="test") assert ( - shelly.output_names + get_output_names(shelly) == shelly._generated_output_names - == ["return_code", "stdout", "stderr", "resultsDir"] + == ["resultsDir", "return_code", "stderr", "stdout"] ) - res = results_function(shelly, plugin) + outputs = results_function(shelly, plugin=plugin, cache_dir=tmp_path) print("Cache_dirr:", shelly.cache_dir) - assert (shelly.output_dir / Path("test")).exists() - assert get_lowest_directory(res.output.resultsDir) == get_lowest_directory( - shelly.output_dir / Path("test") + output_dir = next(tmp_path.iterdir()) + assert (output_dir / Path("test")).exists() + assert get_lowest_directory(outputs.resultsDir) == get_lowest_directory( + output_dir / Path("test") ) @pytest.mark.parametrize("results_function", [run_no_submitter, run_submitter]) def test_shell_cmd_state_outputspec_1(plugin, results_function, tmp_path): """ - providing output name by providing output_file_template + providing output name by providing path_template splitter for a field that is used in the template """ cmd = "touch" args = ["newfile_1.txt", "newfile_2.txt"] - my_output_spec = SpecInfo( - name="Output", - fields=[ - ( - "out1", - attr.ib( - type=File, - metadata={ - "output_file_template": "{args}", - "help": "output file", - }, - ), + @shell.define + class Shelly(ShellDef["Shelly.Outputs"]): + + executable = cmd + + class Outputs(ShellOutputs): + + out1: File = shell.outarg( + path_template="{args}", + help="output file", ) - ], - bases=(ShellOutputs,), - ) - shelly = ShellDef( - name="shelly", + shelly = Shelly( executable=cmd, - output_spec=my_output_spec, - cache_dir=tmp_path, ).split("args", args=args) - res = results_function(shelly, plugin) + outputs = results_function(shelly, plugin=plugin, cache_dir=tmp_path) for i in range(len(args)): - assert res[i].output.stdout == "" - assert res[i].output.out1.fspath.exists() + assert outputs.stdout[i] == "" + assert outputs.out1[i].fspath.exists() # customised output_spec for tasks in workflows @@ -3374,28 +2578,29 @@ def test_shell_cmd_outputspec_wf_1(plugin, tmp_path): """ cmd = ["touch", "newfile_tmp.txt"] - wf = Workflow(name="wf", input_spec=["cmd"]) - wf.inputs.cmd = cmd - wf.cache_dir = tmp_path - - my_output_spec = SpecInfo( - name="Output", - fields=[("newfile", File, "newfile_tmp.txt")], - bases=(ShellOutputs,), - ) - wf.add(ShellDef(name="shelly", executable=wf.lzin.cmd, output_spec=my_output_spec)) - wf.set_output( - [("stdout", wf.shelly.lzout.stdout), ("newfile", wf.shelly.lzout.newfile)] - ) - with Submitter(worker=plugin) as sub: - wf(submitter=sub) + @shell.define + class Shelly(ShellDef["Shelly.Outputs"]): + + executable = cmd + + class Outputs(ShellOutputs): + newfile: File = shell.outarg(path_template="newfile_tmp.txt") - res = wf.result() - assert res.output.stdout == "" - assert res.output.newfile.fspath.exists() + @workflow.define(outputs=["stdout", "newfile"]) + def Workflow(cmd): + shelly = workflow.add(Shelly()) + return shelly.stdout, shelly.newfile + + wf = Workflow() + + with Submitter(plugin=plugin, cache_dir=tmp_path) as sub: + res = sub(wf) + + assert res.outputs.stdout == "" + assert res.outputs.newfile.fspath.exists() # checking if the file was copied to the wf dir - assert res.output.newfile.fspath.parent == wf.output_dir + assert res.outputs.newfile.fspath.parent.parent == tmp_path def test_shell_cmd_inputspec_outputspec_1(): @@ -3403,52 +2608,26 @@ def test_shell_cmd_inputspec_outputspec_1(): customised input_spec and output_spec, output_spec uses input_spec fields in templates """ cmd = ["touch", "newfile_tmp.txt"] - my_input_spec = SpecInfo( - name="Input", - fields=[ - ( - "file1", - str, - {"help": "1st creadted file", "argstr": "", "position": 1}, - ), - ( - "file2", - str, - {"help": "2nd creadted file", "argstr": "", "position": 2}, - ), - ], - bases=(ShellDef,), - ) - my_output_spec = SpecInfo( - name="Output", - fields=[ - ( - "newfile1", - File, - {"output_file_template": "{file1}", "help": "newfile 1"}, - ), - ( - "newfile2", - File, - {"output_file_template": "{file2}", "help": "newfile 2"}, - ), - ], - bases=(ShellOutputs,), - ) - shelly = ShellDef( - name="shelly", - executable=cmd, - input_spec=my_input_spec, - output_spec=my_output_spec, + @shell.define + class Shelly(ShellDef["Shelly.Outputs"]): + file1: File = shell.arg(help="1st creadted file", argstr="", position=1) + file2: File = shell.arg(help="2nd creadted file", argstr="", position=2) + + class Outputs(ShellOutputs): + newfile1: File = shell.outarg(path_template="{file1}", help="newfile 1") + newfile2: File = shell.outarg(path_template="{file2}", help="newfile 2") + + executable = cmd + + shelly = Shelly( + file1=File.mock("new_file_1.txt"), file2=File.mock("new_file_2.txt") ) - shelly.definition.file1 = "new_file_1.txt" - shelly.definition.file2 = "new_file_2.txt" - res = shelly() - assert res.output.stdout == "" - assert res.output.newfile1.fspath.exists() - assert res.output.newfile2.fspath.exists() + outputs = shelly() + assert outputs.stdout == "" + assert outputs.newfile1.fspath.exists() + assert outputs.newfile2.fspath.exists() def test_shell_cmd_inputspec_outputspec_1a(): @@ -3457,52 +2636,28 @@ def test_shell_cmd_inputspec_outputspec_1a(): file2 is used in a template for newfile2, but it is not provided, so newfile2 is set to NOTHING """ cmd = ["touch", "newfile_tmp.txt"] - my_input_spec = SpecInfo( - name="Input", - fields=[ - ( - "file1", - str, - {"help": "1st creadted file", "argstr": "", "position": 1}, - ), - ( - "file2", - str, - {"help": "2nd creadted file", "argstr": "", "position": 2}, - ), - ], - bases=(ShellDef,), - ) - my_output_spec = SpecInfo( - name="Output", - fields=[ - ( - "newfile1", - File, - {"output_file_template": "{file1}", "help": "newfile 1"}, - ), - ( - "newfile2", - File, - {"output_file_template": "{file2}", "help": "newfile 2"}, - ), - ], - bases=(ShellOutputs,), - ) - shelly = ShellDef( - name="shelly", + @shell.define + class Shelly(ShellDef["Shelly.Outputs"]): + executable = cmd + file1: str = shell.arg(help="1st creadted file", argstr="", position=1) + file2: str = shell.arg(help="2nd creadted file", argstr="", position=2) + + class Outputs(ShellOutputs): + + newfile1: File = shell.outarg(path_template="{file1}", help="newfile 1") + newfile2: File = shell.outarg(path_template="{file2}", help="newfile 2") + + shelly = Shelly( executable=cmd, - input_spec=my_input_spec, - output_spec=my_output_spec, ) - shelly.definition.file1 = "new_file_1.txt" + shelly.file1 = File.mock("new_file_1.txt") - res = shelly() - assert res.output.stdout == "" - assert res.output.newfile1.fspath.exists() + outputs = shelly() + assert outputs.stdout == "" + assert outputs.newfile1.fspath.exists() # newfile2 is not created, since file2 is not provided - assert res.output.newfile2 is attr.NOTHING + assert outputs.newfile2 is attr.NOTHING def test_shell_cmd_inputspec_outputspec_2(): @@ -3510,66 +2665,40 @@ def test_shell_cmd_inputspec_outputspec_2(): customised input_spec and output_spec, output_spec uses input_spec fields in the requires filed """ cmd = ["touch", "newfile_tmp.txt"] - my_input_spec = SpecInfo( - name="Input", - fields=[ - ( - "file1", - str, - {"help": "1st creadted file", "argstr": "", "position": 1}, - ), - ( - "file2", - str, - {"help": "2nd creadted file", "argstr": "", "position": 2}, - ), - ], - bases=(ShellDef,), - ) - my_output_spec = SpecInfo( - name="Output", - fields=[ - ( - "newfile1", - File, - { - "output_file_template": "{file1}", - "help": "newfile 1", - "requires": ["file1"], - }, - ), - ( - "newfile2", - File, - { - "output_file_template": "{file2}", - "help": "newfile 1", - "requires": ["file1", "file2"], - }, - ), - ], - bases=(ShellOutputs,), - ) - shelly = ShellDef( - name="shelly", - executable=cmd, - input_spec=my_input_spec, - output_spec=my_output_spec, + @shell.define + class Shelly(ShellDef["Shelly.Outputs"]): + executable = cmd + file1: str = shell.arg(help="1st creadted file", argstr="", position=1) + file2: str = shell.arg(help="2nd creadted file", argstr="", position=2) + + class Outputs(ShellOutputs): + + newfile1: File = shell.outarg( + path_template="{file1}", + help="newfile 1", + requires=["file1"], + ) + newfile2: File = shell.outarg( + path_template="{file2}", + help="newfile 1", + requires=["file1", "file2"], + ) + + shelly = Shelly( + file1=File.mock("new_file_1.txt"), file2=File.mock("new_file_2.txt") ) - shelly.definition.file1 = "new_file_1.txt" - shelly.definition.file2 = "new_file_2.txt" # all fields from output_spec should be in output_names and _generated_output_names assert ( - shelly.output_names + get_output_names(shelly) == shelly._generated_output_names - == ["return_code", "stdout", "stderr", "newfile1", "newfile2"] + == ["newfile1", "newfile2", "return_code", "stderr", "stdout"] ) - res = shelly() - assert res.output.stdout == "" - assert res.output.newfile1.fspath.exists() - assert res.output.newfile2.fspath.exists() + outputs = shelly() + assert outputs.stdout == "" + assert outputs.newfile1.fspath.exists() + assert outputs.newfile2.fspath.exists() def test_shell_cmd_inputspec_outputspec_2a(): @@ -3577,73 +2706,50 @@ def test_shell_cmd_inputspec_outputspec_2a(): customised input_spec and output_spec, output_spec uses input_spec fields in the requires filed """ cmd = ["touch", "newfile_tmp.txt"] - my_input_spec = SpecInfo( - name="Input", - fields=[ - ( - "file1", - str, - {"help": "1st creadted file", "argstr": "", "position": 1}, - ), - ( - "file2", - str, - {"help": "2nd creadted file", "argstr": "", "position": 2}, - ), - ], - bases=(ShellDef,), - ) - my_output_spec = SpecInfo( - name="Output", - fields=[ - ( - "newfile1", - File, - { - "output_file_template": "{file1}", - "help": "newfile 1", - "requires": ["file1"], - }, - ), - ( - "newfile2", - File, - { - "output_file_template": "{file2}", - "help": "newfile 1", - "requires": ["file1", "file2"], - }, - ), - ], - bases=(ShellOutputs,), - ) - shelly = ShellDef( - name="shelly", + @shell.define + class Shelly(ShellDef["Shelly.Outputs"]): + executable = cmd + file1: str = shell.arg(help="1st creadted file", argstr="", position=1) + file2: str = shell.arg(help="2nd creadted file", argstr="", position=2) + + class Outputs(ShellOutputs): + + newfile1: File = shell.outarg( + path_template="{file1}", + help="newfile 1", + requires=["file1"], + ) + newfile2: File = shell.outarg( + path_template="{file2}", + help="newfile 1", + requires=["file1", "file2"], + ) + + shelly = Shelly( executable=cmd, - input_spec=my_input_spec, - output_spec=my_output_spec, ) - shelly.definition.file1 = "new_file_1.txt" + shelly.file1 = File.mock("new_file_1.txt") # _generated_output_names should know that newfile2 will not be generated - assert shelly.output_names == [ - "return_code", - "stdout", - "stderr", + assert get_output_names(shelly) == [ "newfile1", "newfile2", - ] - assert shelly._generated_output_names == [ "return_code", - "stdout", "stderr", + "stdout", + ] + + outputs = shelly() + assert shelly._generated_output_names(outputs.stdout, outputs.stderr) == [ "newfile1", + "return_code", + "stderr", + "stdout", ] - res = shelly() - assert res.output.stdout == "" - assert res.output.newfile1.fspath.exists() - assert res.output.newfile2 is attr.NOTHING + assert outputs.stdout == "" + assert outputs.newfile1.fspath.exists() + assert outputs.newfile2 is attr.NOTHING def test_shell_cmd_inputspec_outputspec_3(): @@ -3652,58 +2758,34 @@ def test_shell_cmd_inputspec_outputspec_3(): adding one additional input that is not in the template, but in the requires field, """ cmd = ["touch", "newfile_tmp.txt"] - my_input_spec = SpecInfo( - name="Input", - fields=[ - ( - "file1", - str, - {"help": "1st creadted file", "argstr": "", "position": 1}, - ), - ( - "file2", - str, - {"help": "2nd creadted file", "argstr": "", "position": 2}, - ), - ("additional_inp", int, {"help": "additional inp"}), - ], - bases=(ShellDef,), - ) - my_output_spec = SpecInfo( - name="Output", - fields=[ - ( - "newfile1", - File, - {"output_file_template": "{file1}", "help": "newfile 1"}, - ), - ( - "newfile2", - File, - { - "output_file_template": "{file2}", - "help": "newfile 1", - "requires": ["file1", "additional_inp"], - }, - ), - ], - bases=(ShellOutputs,), - ) - shelly = ShellDef( - name="shelly", + @shell.define + class Shelly(ShellDef["Shelly.Outputs"]): + executable = cmd + file1: str = shell.arg(help="1st creadted file", argstr="", position=1) + file2: str = shell.arg(help="2nd creadted file", argstr="", position=2) + additional_inp: int = shell.arg(help="additional inp") + + class Outputs(ShellOutputs): + + newfile1: File = shell.outarg(path_template="{file1}", help="newfile 1") + newfile2: File = shell.outarg( + path_template="{file2}", + help="newfile 1", + requires=["file1", "additional_inp"], + ) + + shelly = Shelly( executable=cmd, - input_spec=my_input_spec, - output_spec=my_output_spec, ) - shelly.definition.file1 = "new_file_1.txt" - shelly.definition.file2 = "new_file_2.txt" - shelly.definition.additional_inp = 2 + shelly.file1 = File.mock("new_file_1.txt") + shelly.file2 = File.mock("new_file_2.txt") + shelly.additional_inp = 2 - res = shelly() - assert res.output.stdout == "" - assert res.output.newfile1.fspath.exists() - assert res.output.newfile2.fspath.exists() + outputs = shelly() + assert outputs.stdout == "" + assert outputs.newfile1.fspath.exists() + assert outputs.newfile2.fspath.exists() def test_shell_cmd_inputspec_outputspec_3a(): @@ -3713,72 +2795,48 @@ def test_shell_cmd_inputspec_outputspec_3a(): the additional input not provided, so the output is NOTHING """ cmd = ["touch", "newfile_tmp.txt"] - my_input_spec = SpecInfo( - name="Input", - fields=[ - ( - "file1", - str, - {"help": "1st creadted file", "argstr": "", "position": 1}, - ), - ( - "file2", - str, - {"help": "2nd creadted file", "argstr": "", "position": 2}, - ), - ("additional_inp", str, {"help": "additional inp"}), - ], - bases=(ShellDef,), - ) - my_output_spec = SpecInfo( - name="Output", - fields=[ - ( - "newfile1", - File, - {"output_file_template": "{file1}", "help": "newfile 1"}, - ), - ( - "newfile2", - File, - { - "output_file_template": "{file2}", - "help": "newfile 1", - "requires": ["file1", "additional_inp"], - }, - ), - ], - bases=(ShellOutputs,), - ) - shelly = ShellDef( - name="shelly", + @shell.define + class Shelly(ShellDef["Shelly.Outputs"]): + executable = cmd + file1: str = shell.arg(help="1st creadted file", argstr="", position=1) + file2: str = shell.arg(help="2nd creadted file", argstr="", position=2) + additional_inp: str = shell.arg(help="additional inp") + + class Outputs(ShellOutputs): + + newfile1: File = shell.outarg(path_template="{file1}", help="newfile 1") + newfile2: File = shell.outarg( + path_template="{file2}", + help="newfile 1", + requires=["file1", "additional_inp"], + ) + + shelly = Shelly( executable=cmd, - input_spec=my_input_spec, - output_spec=my_output_spec, ) - shelly.definition.file1 = "new_file_1.txt" - shelly.definition.file2 = "new_file_2.txt" + shelly.file1 = File.mock("new_file_1.txt") + shelly.file2 = File.mock("new_file_2.txt") # _generated_output_names should know that newfile2 will not be generated - assert shelly.output_names == [ - "return_code", - "stdout", - "stderr", + assert get_output_names(shelly) == [ "newfile1", "newfile2", + "return_code", + "stderr", + "stdout", ] assert shelly._generated_output_names == [ + "newfile1", "return_code", - "stdout", "stderr", - "newfile1", + "stdout", ] - res = shelly() - assert res.output.stdout == "" - assert res.output.newfile1.fspath.exists() + outputs = shelly() + assert outputs.stdout == "" + assert outputs.newfile1.fspath.exists() # additional input not provided so no newfile2 set (even if the file was created) - assert res.output.newfile2 is attr.NOTHING + assert outputs.newfile2 is attr.NOTHING def test_shell_cmd_inputspec_outputspec_4(): @@ -3787,52 +2845,37 @@ def test_shell_cmd_inputspec_outputspec_4(): adding one additional input to the requires together with a list of the allowed values, """ cmd = ["touch", "newfile_tmp.txt"] - my_input_spec = SpecInfo( - name="Input", - fields=[ - ( - "file1", - str, - {"help": "1st creadted file", "argstr": "", "position": 1}, - ), - ("additional_inp", int, {"help": "additional inp"}), - ], - bases=(ShellDef,), - ) - my_output_spec = SpecInfo( - name="Output", - fields=[ - ( - "newfile1", - File, - { - "output_file_template": "{file1}", - "help": "newfile 1", - "requires": ["file1", ("additional_inp", [2, 3])], - }, + @shell.define + class Shelly(ShellDef["Shelly.Outputs"]): + executable = cmd + file1: str = shell.arg(help="1st creadted file", argstr="", position=1) + additional_inp: int = shell.arg(help="additional inp") + + class Outputs(ShellOutputs): + + newfile1: File = shell.outarg( + path_template="{file1}", + help="newfile 1", + requires=["file1", ("additional_inp", [2, 3])], ) - ], - bases=(ShellOutputs,), - ) - shelly = ShellDef( - name="shelly", + + shelly = Shelly( executable=cmd, - input_spec=my_input_spec, - output_spec=my_output_spec, ) - shelly.definition.file1 = "new_file_1.txt" - shelly.definition.additional_inp = 2 + shelly.file1 = File.mock("new_file_1.txt") + shelly.additional_inp = 2 # _generated_output_names should be the same as output_names + + outputs = shelly() assert ( - shelly.output_names - == shelly._generated_output_names - == ["return_code", "stdout", "stderr", "newfile1"] + get_output_names(shelly) + == shelly._generated_output_names(outputs.stdout, outputs.stderr) + == ["newfile1", "return_code", "stderr", "stdout"] ) - res = shelly() - assert res.output.stdout == "" - assert res.output.newfile1.fspath.exists() + assert outputs.stdout == "" + assert outputs.newfile1.fspath.exists() def test_shell_cmd_inputspec_outputspec_4a(): @@ -3842,47 +2885,31 @@ def test_shell_cmd_inputspec_outputspec_4a(): the input is set to a value that is not in the list, so output is NOTHING """ cmd = ["touch", "newfile_tmp.txt"] - my_input_spec = SpecInfo( - name="Input", - fields=[ - ( - "file1", - str, - {"help": "1st creadted file", "argstr": "", "position": 1}, - ), - ("additional_inp", int, {"help": "additional inp"}), - ], - bases=(ShellDef,), - ) - my_output_spec = SpecInfo( - name="Output", - fields=[ - ( - "newfile1", - File, - { - "output_file_template": "{file1}", - "help": "newfile 1", - "requires": ["file1", ("additional_inp", [2, 3])], - }, + @shell.define + class Shelly(ShellDef["Shelly.Outputs"]): + executable = cmd + file1: str = shell.arg(help="1st creadted file", argstr="", position=1) + additional_inp: int = shell.arg(help="additional inp") + + class Outputs(ShellOutputs): + + newfile1: File = shell.outarg( + path_template="{file1}", + help="newfile 1", + requires=["file1", ("additional_inp", [2, 3])], ) - ], - bases=(ShellOutputs,), - ) - shelly = ShellDef( - name="shelly", + + shelly = Shelly( executable=cmd, - input_spec=my_input_spec, - output_spec=my_output_spec, ) - shelly.definition.file1 = "new_file_1.txt" + shelly.file1 = File.mock("new_file_1.txt") # the value is not in the list from requires - shelly.definition.additional_inp = 1 + shelly.additional_inp = 1 - res = shelly() - assert res.output.stdout == "" - assert res.output.newfile1 is attr.NOTHING + outputs = shelly() + assert outputs.stdout == "" + assert outputs.newfile1 is attr.NOTHING def test_shell_cmd_inputspec_outputspec_5(): @@ -3892,51 +2919,35 @@ def test_shell_cmd_inputspec_outputspec_5(): the firs element of the requires list has all the fields set """ cmd = ["touch", "newfile_tmp.txt"] - my_input_spec = SpecInfo( - name="Input", - fields=[ - ( - "file1", - str, - {"help": "1st creadted file", "argstr": "", "position": 1}, - ), - ("additional_inp_A", int, {"help": "additional inp A"}), - ("additional_inp_B", str, {"help": "additional inp B"}), - ], - bases=(ShellDef,), - ) - my_output_spec = SpecInfo( - name="Output", - fields=[ - ( - "newfile1", - File, - { - "output_file_template": "{file1}", - "help": "newfile 1", - # requires is a list of list so it's treated as el[0] OR el[1] OR... - "requires": [ - ["file1", "additional_inp_A"], - ["file1", "additional_inp_B"], - ], - }, + @shell.define + class Shelly(ShellDef["Shelly.Outputs"]): + executable = cmd + file1: str = shell.arg(help="1st creadted file", argstr="", position=1) + additional_inp_A: int = shell.arg(help="additional inp A") + additional_inp_B: str = shell.arg(help="additional inp B") + + class Outputs(ShellOutputs): + + newfile1: File = shell.outarg( + path_template="{file1}", + help="newfile 1", + # requires is a list of list so it's treated as el[0] OR el[1] OR... + requires=[ + ["file1", "additional_inp_A"], + ["file1", "additional_inp_B"], + ], ) - ], - bases=(ShellOutputs,), - ) - shelly = ShellDef( - name="shelly", + + shelly = Shelly( executable=cmd, - input_spec=my_input_spec, - output_spec=my_output_spec, ) - shelly.definition.file1 = "new_file_1.txt" - shelly.definition.additional_inp_A = 2 + shelly.file1 = File.mock("new_file_1.txt") + shelly.additional_inp_A = 2 - res = shelly() - assert res.output.stdout == "" - assert res.output.newfile1.fspath.exists() + outputs = shelly() + assert outputs.stdout == "" + assert outputs.newfile1.fspath.exists() def test_shell_cmd_inputspec_outputspec_5a(): @@ -3946,51 +2957,35 @@ def test_shell_cmd_inputspec_outputspec_5a(): the second element of the requires list (i.e. additional_inp_B) has all the fields set """ cmd = ["touch", "newfile_tmp.txt"] - my_input_spec = SpecInfo( - name="Input", - fields=[ - ( - "file1", - str, - {"help": "1st creadted file", "argstr": "", "position": 1}, - ), - ("additional_inp_A", str, {"help": "additional inp A"}), - ("additional_inp_B", int, {"help": "additional inp B"}), - ], - bases=(ShellDef,), - ) - my_output_spec = SpecInfo( - name="Output", - fields=[ - ( - "newfile1", - File, - { - "output_file_template": "{file1}", - "help": "newfile 1", - # requires is a list of list so it's treated as el[0] OR el[1] OR... - "requires": [ - ["file1", "additional_inp_A"], - ["file1", "additional_inp_B"], - ], - }, + @shell.define + class Shelly(ShellDef["Shelly.Outputs"]): + executable = cmd + file1: str = shell.arg(help="1st creadted file", argstr="", position=1) + additional_inp_A: str = shell.arg(help="additional inp A") + additional_inp_B: int = shell.arg(help="additional inp B") + + class Outputs(ShellOutputs): + + newfile1: File = shell.outarg( + path_template="{file1}", + help="newfile 1", + # requires is a list of list so it's treated as el[0] OR el[1] OR... + requires=[ + ["file1", "additional_inp_A"], + ["file1", "additional_inp_B"], + ], ) - ], - bases=(ShellOutputs,), - ) - shelly = ShellDef( - name="shelly", + + shelly = Shelly( executable=cmd, - input_spec=my_input_spec, - output_spec=my_output_spec, ) - shelly.definition.file1 = "new_file_1.txt" - shelly.definition.additional_inp_B = 2 + shelly.file1 = File.mock("new_file_1.txt") + shelly.additional_inp_B = 2 - res = shelly() - assert res.output.stdout == "" - assert res.output.newfile1.fspath.exists() + outputs = shelly() + assert outputs.stdout == "" + assert outputs.newfile1.fspath.exists() def test_shell_cmd_inputspec_outputspec_5b(): @@ -4000,51 +2995,35 @@ def test_shell_cmd_inputspec_outputspec_5b(): neither of the list from requirements has all the fields set, so the output is NOTHING """ cmd = ["touch", "newfile_tmp.txt"] - my_input_spec = SpecInfo( - name="Input", - fields=[ - ( - "file1", - str, - {"help": "1st creadted file", "argstr": "", "position": 1}, - ), - ("additional_inp_A", str, {"help": "additional inp A"}), - ("additional_inp_B", str, {"help": "additional inp B"}), - ], - bases=(ShellDef,), - ) - my_output_spec = SpecInfo( - name="Output", - fields=[ - ( - "newfile1", - File, - { - "output_file_template": "{file1}", - "help": "newfile 1", - # requires is a list of list so it's treated as el[0] OR el[1] OR... - "requires": [ - ["file1", "additional_inp_A"], - ["file1", "additional_inp_B"], - ], - }, + @shell.define + class Shelly(ShellDef["Shelly.Outputs"]): + executable = cmd + file1: str = shell.arg(help="1st creadted file", argstr="", position=1) + additional_inp_A: str = shell.arg(help="additional inp A") + additional_inp_B: str = shell.arg(help="additional inp B") + + class Outputs(ShellOutputs): + + newfile1: File = shell.outarg( + path_template="{file1}", + help="newfile 1", + # requires is a list of list so it's treated as el[0] OR el[1] OR... + requires=[ + ["file1", "additional_inp_A"], + ["file1", "additional_inp_B"], + ], ) - ], - bases=(ShellOutputs,), - ) - shelly = ShellDef( - name="shelly", + + shelly = Shelly( executable=cmd, - input_spec=my_input_spec, - output_spec=my_output_spec, ) - shelly.definition.file1 = "new_file_1.txt" + shelly.file1 = File.mock("new_file_1.txt") - res = shelly() - assert res.output.stdout == "" + outputs = shelly() + assert outputs.stdout == "" # neither additional_inp_A nor additional_inp_B is set, so newfile1 is NOTHING - assert res.output.newfile1 is attr.NOTHING + assert outputs.newfile1 is attr.NOTHING def test_shell_cmd_inputspec_outputspec_6_except(): @@ -4053,42 +3032,25 @@ def test_shell_cmd_inputspec_outputspec_6_except(): requires has invalid syntax - exception is raised """ cmd = ["touch", "newfile_tmp.txt"] - my_input_spec = SpecInfo( - name="Input", - fields=[ - ( - "file1", - str, - {"help": "1st creadted file", "argstr": "", "position": 1}, - ), - ("additional_inp_A", str, {"help": "additional inp A"}), - ], - bases=(ShellDef,), - ) - my_output_spec = SpecInfo( - name="Output", - fields=[ - ( - "newfile1", - File, - { - "output_file_template": "{file1}", - "help": "newfile 1", - # requires has invalid syntax - "requires": [["file1", "additional_inp_A"], "file1"], - }, + @shell.define + class Shelly(ShellDef["Shelly.Outputs"]): + executable = cmd + file1: str = shell.arg(help="1st creadted file", argstr="", position=1) + additional_inp_A: str = shell.arg(help="additional inp A") + + class Outputs(ShellOutputs): + newfile1: File = shell.outarg( + path_template="{file1}", + help="newfile 1", + # requires has invalid syntax + requires=[["file1", "additional_inp_A"], "file1"], ) - ], - bases=(ShellOutputs,), - ) - shelly = ShellDef( - name="shelly", + + shelly = Shelly( executable=cmd, - input_spec=my_input_spec, - output_spec=my_output_spec, ) - shelly.definition.file1 = "new_file_1.txt" + shelly.file1 = File.mock("new_file_1.txt") with pytest.raises(Exception, match="requires field can be"): shelly() @@ -4117,270 +3079,141 @@ def change_name(file): name, ext = os.path.splitext(file) return f"{name}_brain.{ext}" - bet_input_spec = SpecInfo( - name="Input", - # TODO: change the position?? - fields=[ - ( - "in_file", - attr.ib( - type=File, - metadata={ - "help": "input file to skull strip", - "position": 1, - "mandatory": True, - "argstr": "", - }, - ), - ), - ( - "out_file", - attr.ib( - type=str, - metadata={ - "help": "name of output skull stripped image", - "position": 2, - "argstr": "", - "output_file_template": "{in_file}_brain", - }, - ), - ), - ( - "outline", - attr.ib( - type=bool, - metadata={ - "help": "create surface outline image", - "argstr": "-o", - }, - ), - ), - ( - "mask", - attr.ib( - type=bool, - metadata={ - "help": "create binary mask image", - "argstr": "-m", - }, - ), - ), - ( - "skull", - attr.ib( - type=bool, - metadata={"help": "create skull image", "argstr": "-s"}, - ), - ), - ( - "no_output", - attr.ib( - type=bool, - metadata={ - "help": "Don't generate segmented output", - "argstr": "-n", - }, - ), - ), - ( - "frac", - attr.ib( - type=float, - metadata={ - "help": "fractional intensity threshold", - "argstr": "-f", - }, - ), - ), - ( - "vertical_gradient", - attr.ib( - type=float, - metadata={ - "help": "vertical gradient in fractional intensity threshold (-1, 1)", - "argstr": "-g", - "allowed_values": {"min_val": -1, "max_val": 1}, - }, - ), - ), - ( - "radius", - attr.ib(type=int, metadata={"argstr": "-r", "help": "head radius"}), - ), - ( - "center", - attr.ib( - type=ty.List[int], - metadata={ - "help": "center of gravity in voxels", - "argstr": "-c", - "allowed_values": {"min_value": 0, "max_value": 3}, - }, - ), - ), - ( - "threshold", - attr.ib( - type=bool, - metadata={ - "argstr": "-t", - "help": "apply thresholding to segmented brain image and mask", - }, - ), - ), - ( - "mesh", - attr.ib( - type=bool, - metadata={ - "argstr": "-e", - "help": "generate a vtk mesh brain surface", - }, - ), - ), - ( - "robust", - attr.ib( - type=bool, - metadata={ - "help": "robust brain centre estimation (iterates BET several times)", - "argstr": "-R", - "xor": _xor_inputs, - }, - ), - ), - ( - "padding", - attr.ib( - type=bool, - metadata={ - "help": "improve BET if FOV is very small in Z (by temporarily padding end slices", - "argstr": "-Z", - "xor": _xor_inputs, - }, - ), - ), - ( - "remove_eyes", - attr.ib( - type=bool, - metadata={ - "help": "eye & optic nerve cleanup (can be useful in SIENA)", - "argstr": "-S", - "xor": _xor_inputs, - }, - ), - ), - ( - "surfaces", - attr.ib( - type=bool, - metadata={ - "help": "run bet2 and then betsurf to get additional skull and scalp surfaces (includes registrations)", - "argstr": "-A", - "xor": _xor_inputs, - }, - ), - ), - ( - "t2_guided", - attr.ib( - type=ty.Union[File, str], - metadata={ - "help": "as with creating surfaces, when also feeding in non-brain-extracted T2 (includes registrations)", - "argstr": "-A2", - "xor": _xor_inputs, - }, - ), - ), - ( - "functional", - attr.ib( - type=bool, - metadata={ - "argstr": "-F", - "xor": _xor_inputs, - "help": "apply to 4D fMRI data", - }, - ), - ), - ( - "reduce_bias", - attr.ib( - type=bool, - metadata={ - "argstr": "-B", - "xor": _xor_inputs, - "help": "bias field and neck cleanup", - }, - ), - ), - # ("number_classes", int, attr.ib(metadata={"help": 'number of tissue-type classes', "argstr": '-n', - # "allowed_values": {"min_val": 1, "max_val": 10}})), - # ("output_biasfield", bool, - # attr.ib(metadata={"help": 'output estimated bias field', "argstr": '-b'})), - # ("output_biascorrected", bool, - # attr.ib(metadata={"help": 'output restored image (bias-corrected image)', "argstr": '-B'})), - ], - bases=(ShellDef,), - ) + @shell.define + class Shelly(ShellDef["Shelly.Outputs"]): + executable = "bet" + in_file: File = shell.arg( + help="input file to skull strip", + position=1, + argstr="", + ) + + outline: bool = shell.arg( + help="create surface outline image", + argstr="-o", + ) + mask: bool = shell.arg( + help="create binary mask image", + argstr="-m", + ) + skull: bool = shell.arg( + help="create skull image", + argstr="-s", + ) + no_output: bool = shell.arg( + help="Don't generate segmented output", + argstr="-n", + ) + frac: float = shell.arg( + help="fractional intensity threshold", + argstr="-f", + ) + vertical_gradient: float = shell.arg( + help="vertical gradient in fractional intensity threshold (-1, 1)", + argstr="-g", + allowed_values={"min_val": -1, "max_val": 1}, + ) + radius: int = shell.arg(argstr="-r", help="head radius") + center: ty.List[int] = shell.arg( + help="center of gravity in voxels", + argstr="-c", + allowed_values={"min_value": 0, "max_value": 3}, + ) + threshold: bool = shell.arg( + argstr="-t", + help="apply thresholding to segmented brain image and mask", + ) + mesh: bool = shell.arg( + argstr="-e", + help="generate a vtk mesh brain surface", + ) + robust: bool = shell.arg( + help="robust brain centre estimation (iterates BET several times)", + argstr="-R", + xor=_xor_inputs, + ) + padding: bool = shell.arg( + help="improve BET if FOV is very small in Z (by temporarily padding end slices", + argstr="-Z", + xor=_xor_inputs, + ) + remove_eyes: bool = shell.arg( + help="eye & optic nerve cleanup (can be useful in SIENA)", + argstr="-S", + xor=_xor_inputs, + ) + surfaces: bool = shell.arg( + help="run bet2 and then betsurf to get additional skull and scalp surfaces (includes registrations)", + argstr="-A", + xor=_xor_inputs, + ) + t2_guided: ty.Union[File, str] = shell.arg( + help="as with creating surfaces, when also feeding in non-brain-extracted T2 (includes registrations)", + argstr="-A2", + xor=_xor_inputs, + ) + functional: bool = shell.arg( + argstr="-F", + xor=_xor_inputs, + help="apply to 4D fMRI data", + ) + reduce_bias: bool = shell.arg( + argstr="-B", + xor=_xor_inputs, + help="bias field and neck cleanup", + ) + + class Outputs(ShellOutputs): + out_file: File = shell.outarg( + help="name of output skull stripped image", + position=2, + argstr="", + path_template="{in_file}_brain", + ) + + # ("number_classes", int, attr.ib(metadata={help='number of tissue-type classes', argstr='-n', + # allowed_values={"min_val": 1, max_val=10}})), + # ("output_biasfield", bool, + # attr.ib(metadata={help='output estimated bias field', argstr='-b'})), + # ("output_biascorrected", bool, + # attr.ib(metadata={help='output restored image (bias-corrected image)', argstr='-B'})), # TODO: not sure why this has to be string in_file = data_tests_dir / "test.nii.gz" # separate command into exec + args - shelly = ShellDef( - name="bet_task", executable="bet", in_file=in_file, input_spec=bet_input_spec - ) - out_file = shelly.output_dir / "test_brain.nii.gz" - assert shelly.definition.executable == "bet" + shelly = Shelly(in_file=in_file) + out_file = next(tmp_path.iterdir()) / "test_brain.nii.gz" + assert shelly.executable == "bet" assert shelly.cmdline == f"bet {in_file} {out_file}" - # res = shelly(plugin="cf") + # outputs = shelly(plugin="cf") def test_shell_cmd_optional_output_file1(tmp_path): """ Test to see that 'unused' doesn't complain about not having an output passed to it """ - my_cp_spec = SpecInfo( - name="Input", - fields=[ - ( - "input", - attr.ib(type=File, metadata={"argstr": "", "help": "input file"}), - ), - ( - "output", - attr.ib( - type=Path, - metadata={ - "argstr": "", - "output_file_template": "out.txt", - "help": "output file", - }, - ), - ), - ( - "unused", - attr.ib( - type=ty.Union[Path, bool], - default=False, - metadata={ - "argstr": "--not-used", - "output_file_template": "out.txt", - "help": "dummy output", - }, - ), - ), - ], - bases=(ShellDef,), - ) - my_cp = ShellDef( - name="my_cp", - executable="cp", - input_spec=my_cp_spec, - ) + @shell.define + class Shelly(ShellDef["Shelly.Outputs"]): + input: File = shell.arg(argstr="", help="input file") + + executable = "cp" + + class Outputs(ShellOutputs): + output: File = shell.outarg( + argstr="", + path_template="out.txt", + help="output file", + ) + unused: File | None = shell.outarg( + default=False, + argstr="--not-used", + path_template="out.txt", + help="dummy output", + ) + + my_cp = ShellDef() file1 = tmp_path / "file1.txt" file1.write_text("foo") outputs = my_cp(input=file1, unused=False) @@ -4391,34 +3224,21 @@ def test_shell_cmd_optional_output_file2(tmp_path): """ Test to see that 'unused' doesn't complain about not having an output passed to it """ - my_cp_spec = SpecInfo( - name="Input", - fields=[ - ( - "input", - attr.ib(type=File, metadata={"argstr": "", "help": "input file"}), - ), - ( - "output", - attr.ib( - type=ty.Union[Path, bool], - default=False, - metadata={ - "argstr": "", - "output_file_template": "out.txt", - "help": "dummy output", - }, - ), - ), - ], - bases=(ShellDef,), - ) - my_cp = ShellDef( - name="my_cp", - executable="cp", - input_spec=my_cp_spec, - ) + @shell.define + class Shelly(ShellDef["Shelly.Outputs"]): + executable = "cp" + + input: File = shell.arg(argstr="", help="input file") + + class Outputs(ShellOutputs): + output: File = shell.outarg( + argstr="", + path_template="out.txt", + help="dummy output", + ) + + my_cp = Shelly() file1 = tmp_path / "file1.txt" file1.write_text("foo") outputs = my_cp(input=file1, output=True) @@ -4433,253 +3253,128 @@ def test_shell_cmd_optional_output_file2(tmp_path): def test_shell_cmd_non_existing_outputs_1(tmp_path): """Checking that non existing output files do not return a phantom path, but return NOTHING instead""" - input_spec = SpecInfo( - name="Input", - fields=[ - ( - "out_name", - attr.ib( - type=str, - metadata={ - "help": """ - base name of the pretend outputs. - """, - "mandatory": True, - }, - ), + + @shell.define + class Shelly(ShellDef["Shelly.Outputs"]): + executable = "echo" + out_name: str = shell.arg( + help=""" + base name of the pretend outputs. + """, + ) + + class Outputs(ShellOutputs): + out_1: File = shell.outarg( + help="fictional output #1", + path_template="{out_name}_1.nii", + ) + out_2: File = shell.outarg( + help="fictional output #2", + path_template="{out_name}_2.nii", ) - ], - bases=(ShellDef,), - ) - out_spec = SpecInfo( - name="Output", - fields=[ - ( - "out_1", - attr.ib( - type=File, - metadata={ - "help": "fictional output #1", - "output_file_template": "{out_name}_1.nii", - }, - ), - ), - ( - "out_2", - attr.ib( - type=File, - metadata={ - "help": "fictional output #2", - "output_file_template": "{out_name}_2.nii", - }, - ), - ), - ], - bases=(ShellOutputs,), - ) - shelly = ShellDef( - cache_dir=tmp_path, - executable="echo", - input_spec=input_spec, - output_spec=out_spec, + shelly = Shelly( out_name="test", ) - shelly() - res = shelly.result() - assert res.output.out_1 == attr.NOTHING and res.output.out_2 == attr.NOTHING + outputs = shelly() + assert outputs.out_1 == attr.NOTHING and outputs.out_2 == attr.NOTHING def test_shell_cmd_non_existing_outputs_2(tmp_path): """Checking that non existing output files do not return a phantom path, but return NOTHING instead. This test has one existing and one non existing output file. """ - input_spec = SpecInfo( - name="Input", - fields=[ - ( - "out_name", - attr.ib( - type=str, - metadata={ - "help": """ - base name of the pretend outputs. - """, - "mandatory": True, - "argstr": "{out_name}_1.nii", - }, - ), + + @shell.define + class Shelly(ShellDef["Shelly.Outputs"]): + executable = "touch" + out_name: str = shell.arg( + help=""" + base name of the pretend outputs. + """, + argstr="{out_name}_1.nii", + ) + + class Outputs(ShellOutputs): + out_1: File = shell.outarg( + help="fictional output #1", + path_template="{out_name}_1.nii", + ) + out_2: File | None = shell.outarg( + help="fictional output #2", + path_template="{out_name}_2.nii", ) - ], - bases=(ShellDef,), - ) - out_spec = SpecInfo( - name="Output", - fields=[ - ( - "out_1", - attr.ib( - type=File, - metadata={ - "help": "fictional output #1", - "output_file_template": "{out_name}_1.nii", - }, - ), - ), - ( - "out_2", - attr.ib( - type=File, - metadata={ - "help": "fictional output #2", - "output_file_template": "{out_name}_2.nii", - }, - ), - ), - ], - bases=(ShellOutputs,), - ) - shelly = ShellDef( - cache_dir=tmp_path, - executable="touch", - input_spec=input_spec, - output_spec=out_spec, - out_name="test", - ) - shelly() - res = shelly.result() + shelly = Shelly(out_name="test") + outputs = shelly() # the first output file is created - assert res.output.out_1.fspath == Path(shelly.output_dir) / Path("test_1.nii") - assert res.output.out_1.fspath.exists() + assert outputs.out_1.fspath == next(tmp_path.iterdir()) / "test_1.nii" + assert outputs.out_1.fspath.exists() # the second output file is not created - assert res.output.out_2 == attr.NOTHING + assert outputs.out_2 is None def test_shell_cmd_non_existing_outputs_3(tmp_path): """Checking that non existing output files do not return a phantom path, but return NOTHING instead. This test has an existing mandatory output and another non existing output file. """ - input_spec = SpecInfo( - name="Input", - fields=[ - ( - "out_name", - attr.ib( - type=str, - metadata={ - "help": """ - base name of the pretend outputs. - """, - "mandatory": True, - "argstr": "{out_name}_1.nii", - }, - ), + + @shell.define + class Shelly(ShellDef["Shelly.Outputs"]): + executable = "touch" + out_name: str = shell.arg( + help=""" + base name of the pretend outputs. + """, + argstr="{out_name}_1.nii", + ) + + class Outputs(ShellOutputs): + out_1: File = shell.outarg( + help="fictional output #1", + path_template="{out_name}_1.nii", + ) + out_2: File = shell.outarg( + help="fictional output #2", + path_template="{out_name}_2.nii", ) - ], - bases=(ShellDef,), - ) - out_spec = SpecInfo( - name="Output", - fields=[ - ( - "out_1", - attr.ib( - type=File, - metadata={ - "help": "fictional output #1", - "output_file_template": "{out_name}_1.nii", - "mandatory": True, - }, - ), - ), - ( - "out_2", - attr.ib( - type=File, - metadata={ - "help": "fictional output #2", - "output_file_template": "{out_name}_2.nii", - }, - ), - ), - ], - bases=(ShellOutputs,), - ) - shelly = ShellDef( - cache_dir=tmp_path, - executable="touch", - input_spec=input_spec, - output_spec=out_spec, - out_name="test", - ) - shelly() - res = shelly.result() + shelly = Shelly(out_name="test") + + outputs = shelly() # the first output file is created - assert res.output.out_1.fspath == Path(shelly.output_dir) / Path("test_1.nii") - assert res.output.out_1.fspath.exists() + assert outputs.out_1.fspath == next(tmp_path.iterdir()) / "test_1.nii" + assert outputs.out_1.fspath.exists() # the second output file is not created - assert res.output.out_2 == attr.NOTHING + assert outputs.out_2 == attr.NOTHING def test_shell_cmd_non_existing_outputs_4(tmp_path): """Checking that non existing output files do not return a phantom path, but return NOTHING instead. This test has an existing mandatory output and another non existing mandatory output file.""" - input_spec = SpecInfo( - name="Input", - fields=[ - ( - "out_name", - attr.ib( - type=str, - metadata={ - "help": """ + + @shell.define + class Shelly(ShellDef["Shelly.Outputs"]): + executable = "touch" + out_name: str = shell.arg( + help=""" base name of the pretend outputs. """, - "mandatory": True, - "argstr": "{out_name}_1.nii", - }, - ), + argstr="{out_name}_1.nii", + ) + + class Outputs(ShellOutputs): + out_1: File = shell.outarg( + help="fictional output #1", + path_template="{out_name}_1.nii", + ) + out_2: File = shell.outarg( + help="fictional output #2", + path_template="{out_name}_2.nii", ) - ], - bases=(ShellDef,), - ) - out_spec = SpecInfo( - name="Output", - fields=[ - ( - "out_1", - attr.ib( - type=File, - metadata={ - "help": "fictional output #1", - "output_file_template": "{out_name}_1.nii", - "mandatory": True, - }, - ), - ), - ( - "out_2", - attr.ib( - type=File, - metadata={ - "help": "fictional output #2", - "output_file_template": "{out_name}_2.nii", - "mandatory": True, - }, - ), - ), - ], - bases=(ShellOutputs,), - ) - shelly = ShellDef( - cache_dir=tmp_path, - executable="touch", - input_spec=input_spec, - output_spec=out_spec, + shelly = Shelly( out_name="test", ) # An exception should be raised because the second mandatory output does not exist @@ -4687,113 +3382,63 @@ def test_shell_cmd_non_existing_outputs_4(tmp_path): shelly() assert "mandatory output for variable out_2 does not exist" == str(excinfo.value) # checking if the first output was created - assert (Path(shelly.output_dir) / Path("test_1.nii")).exists() + assert (next(tmp_path.iterdir()) / "test_1.nii").exists() def test_shell_cmd_non_existing_outputs_multi_1(tmp_path): """This test looks if non existing files of an multiOuputFile are also set to NOTHING""" - input_spec = SpecInfo( - name="Input", - fields=[ - ( - "out_name", - attr.ib( - type=MultiInputObj, - metadata={ - "help": """ + + @shell.define + class Shelly(ShellDef["Shelly.Outputs"]): + executable = "echo" + out_name: MultiInputObj = shell.arg( + help=""" base name of the pretend outputs. """, - "mandatory": True, - "argstr": "...", - }, - ), + argstr="...", + ) + + class Outputs(ShellOutputs): + out_list: MultiOutputFile = shell.outarg( + help="fictional output #1", + path_template="{out_name}", ) - ], - bases=(ShellDef,), - ) - out_spec = SpecInfo( - name="Output", - fields=[ - ( - "out_list", - attr.ib( - type=MultiOutputFile, - metadata={ - "help": "fictional output #1", - "output_file_template": "{out_name}", - }, - ), - ), - ], - bases=(ShellOutputs,), - ) - shelly = ShellDef( - cache_dir=tmp_path, - executable="echo", - input_spec=input_spec, - output_spec=out_spec, - out_name=["test_1.nii", "test_2.nii"], - ) - shelly() - res = shelly.result() + shelly = Shelly(out_name=["test_1.nii", "test_2.nii"]) + + outputs = shelly() # checking if the outputs are Nothing - assert res.output.out_list[0] == attr.NOTHING - assert res.output.out_list[1] == attr.NOTHING + assert outputs.out_list[0] == attr.NOTHING + assert outputs.out_list[1] == attr.NOTHING def test_shell_cmd_non_existing_outputs_multi_2(tmp_path): """This test looks if non existing files of an multiOutputFile are also set to NOTHING. It checks that it also works if one file of the multiOutputFile actually exists.""" - input_spec = SpecInfo( - name="Input", - fields=[ - ( - "out_name", - attr.ib( - type=MultiInputObj, - metadata={ - "help": """ + + @shell.define + class Shelly(ShellDef["Shelly.Outputs"]): + executable = "touch" + out_name: MultiInputObj = shell.arg( + help=""" base name of the pretend outputs. """, - "sep": " test_1_real.nii", # hacky way of creating an extra file with that name - "mandatory": True, - "argstr": "...", - }, - ), + sep=" test_1_real.nii", # hacky way of creating an extra file with that name + argstr="...", + ) + + class Outputs(ShellOutputs): + out_list: MultiOutputFile = shell.outarg( + help="fictional output #1", + path_template="{out_name}_real.nii", ) - ], - bases=(ShellDef,), - ) - out_spec = SpecInfo( - name="Output", - fields=[ - ( - "out_list", - attr.ib( - type=MultiOutputFile, - metadata={ - "help": "fictional output #1", - "output_file_template": "{out_name}_real.nii", - }, - ), - ), - ], - bases=(ShellOutputs,), - ) - shelly = ShellDef( - cache_dir=tmp_path, - executable="touch", - input_spec=input_spec, - output_spec=out_spec, - out_name=["test_1", "test_2"], - ) - shelly() - res = shelly.result() + shelly = Shelly(out_name=["test_1", "test_2"]) + + outputs = shelly() # checking if the outputs are Nothing - assert res.output.out_list[0] == File(Path(shelly.output_dir) / "test_1_real.nii") - assert res.output.out_list[1] == attr.NOTHING + assert outputs.out_list[0] == File(next(tmp_path.iterdir()) / "test_1_real.nii") + assert outputs.out_list[1] == attr.NOTHING @pytest.mark.xfail( @@ -4805,67 +3450,38 @@ def test_shell_cmd_non_existing_outputs_multi_2(tmp_path): def test_shellspec_formatter_1(tmp_path): """test the input callable 'formatter'.""" - def spec_info(formatter): - return SpecInfo( - name="Input", - fields=[ - ( - "in1", - attr.ib( - type=str, - metadata={ - "help": """ - just a dummy name - """, - "mandatory": True, - }, - ), - ), - ( - "in2", - attr.ib( - type=str, - metadata={ - "help": """ - just a dummy name - """, - "mandatory": True, - }, - ), - ), - ( - "together", - attr.ib( - type=ty.List, - metadata={ - "help": """ - combines in1 and in2 into a list - """, - # When providing a formatter all other metadata options are discarded. - "formatter": formatter, - }, - ), - ), - ], - bases=(ShellDef,), - ) - def formatter_1(inputs): print("FORMATTER:", inputs) return f"-t [{inputs['in1']}, {inputs['in2']}]" - input_spec = spec_info(formatter_1) - shelly = ShellDef(executable="exec", input_spec=input_spec, in1="i1", in2="i2") + def make_shelly(formatter): + @shell.define + class Shelly(ShellDef["Shelly.Outputs"]): + executable = "exec" + in1: str = shell.arg( + help=""" + just a dummy name + """, + ) + in2: str = shell.arg( + help=""" + just a dummy name + """, + ) + together: ty.List = shell.arg( + help=""" + combines in1 and in2 into a list + """, + # When providing a formatter all other metadata options are discarded. + formatter=formatter, + ) + + Shelly = make_shelly(formatter=formatter_1) + shelly = Shelly(in1="i1", in2="i2") assert shelly.cmdline == "exec -t [i1, i2]" # testing that the formatter can overwrite a provided value for together. - shelly = ShellDef( - executable="exec", - input_spec=input_spec, - in1="i1", - in2="i2", - together=[1], - ) + shelly = Shelly(in1="i1", in2="i2", together=[1]) assert shelly.cmdline == "exec -t [i1, i2]" # asking for specific inputs @@ -4873,18 +3489,18 @@ def formatter_2(in1, in2): print("FORMATTER:", in1, in2) return f"-t [{in1}, {in2}]" - input_spec = spec_info(formatter_2) + Shelly = make_shelly(formatter_2) - shelly = ShellDef(executable="exec", input_spec=input_spec, in1="i1", in2="i2") + shelly = Shelly(in1="i1", in2="i2") assert shelly.cmdline == "exec -t [i1, i2]" def formatter_3(in1, in3): print("FORMATTER:", in1, in3) return f"-t [{in1}, {in3}]" - input_spec = spec_info(formatter_3) + Shelly = make_shelly(formatter_3) - shelly = ShellDef(executable="exec", input_spec=input_spec, in1="i1", in2="i2") + shelly = Shelly(in1="i1", in2="i2") with pytest.raises(Exception) as excinfo: shelly.cmdline assert ( @@ -4898,11 +3514,9 @@ def formatter_5(field): # formatter must return a string return field - input_spec = spec_info(formatter_5) + Shelly = make_shelly(formatter_5) - shelly = ShellDef( - executable="exec", - input_spec=input_spec, + shelly = Shelly( in1="i1", in2="i2", # together="-t test", @@ -4915,63 +3529,34 @@ def formatter_4(field): # formatter must return a string return "" - input_spec = spec_info(formatter_4) + Shelly = make_shelly(formatter_4) - shelly = ShellDef(executable="exec", input_spec=input_spec, in1="i1", in2="i2") + shelly = Shelly(in1="i1", in2="i2") assert shelly.cmdline == "exec" def test_shellspec_formatter_splitter_2(tmp_path): """test the input callable 'formatter' when a splitter is used on an argument of the formatter.""" - def spec_info(formatter): - return SpecInfo( - name="Input", - fields=[ - ( - "in1", - attr.ib( - type=str, - metadata={ - "help": "in1", - }, - ), - ), - ( - "in2", - attr.ib( - type=str, - metadata={ - "help": "in2", - }, - ), - ), - ( - "together", - attr.ib( - type=ty.List, - metadata={ - "help": """ - uses in1 - """, - # When providing a formatter all other metadata options are discarded. - "formatter": formatter, - }, - ), - ), - ], - bases=(ShellDef,), - ) - # asking for specific inputs def formatter_1(in1, in2): return f"-t [{in1} {in2}]" - input_spec = spec_info(formatter_1) + @shell.define + class Shelly(ShellDef["Shelly.Outputs"]): + executable = "executable" + in1: str = shell.arg(help="in1") + in2: str = shell.arg(help="in2") + together: ty.List = shell.arg( + help=""" + uses in1 + """, + # When providing a formatter all other metadata options are discarded. + formatter=formatter_1, + ) + in1 = ["in11", "in12"] - shelly = ShellDef( - name="f", executable="executable", input_spec=input_spec, in2="in2" - ).split("in1", in1=in1) + shelly = Shelly(in2="in2").split("in1", in1=in1) assert shelly is not None # results = shelly.cmdline @@ -5004,21 +3589,14 @@ def test_shellcommand_error_msg(tmp_path): ), ) - input_spec = SpecInfo( - name="Input", - fields=[ - ( - "in1", - str, - {"help": "a dummy string", "argstr": "", "mandatory": True}, - ), - ], - bases=(ShellDef,), - ) + @shell.define + class Shelly(ShellDef["Shelly.Outputs"]): - shelly = ShellDef( - name="err_msg", executable=str(script_path), input_spec=input_spec, in1="hello" - ) + executable = script_path + + in1: str = shell.arg(help="a dummy string", argstr="") + + shelly = Shelly(in1="hello") with pytest.raises(RuntimeError) as excinfo: shelly() diff --git a/pydra/engine/tests/test_shelltask_inputspec.py b/pydra/engine/tests/test_shelltask_inputspec.py index 272231715..3972cbc3d 100644 --- a/pydra/engine/tests/test_shelltask_inputspec.py +++ b/pydra/engine/tests/test_shelltask_inputspec.py @@ -1,57 +1,60 @@ import typing as ty from pathlib import Path -import attr +import attrs import pytest from pydra.engine.specs import ShellOutputs, ShellDef from fileformats.generic import File from pydra.design import shell +from pydra.utils.typing import MultiInputObj +from .utils import get_output_names def test_shell_cmd_execargs_1(): # separate command into exec + args - shelly = ShellDef(executable="executable", args="arg") + Shelly = shell.define(["executable", "arg"]) + shelly = Shelly() assert shelly.cmdline == "executable arg" - assert shelly.name == "ShellTask_noname" def test_shell_cmd_execargs_2(): # separate command into exec + args - shelly = ShellDef(executable=["cmd_1", "cmd_2"], args="arg") + Shelly = shell.define(["cmd_1", "cmd_2", "arg"]) + shelly = Shelly() assert shelly.cmdline == "cmd_1 cmd_2 arg" def test_shell_cmd_inputs_1(): """additional input with provided position""" - my_input_spec = SpecInfo( - name="Input", - fields=[ - ( - "inpA", - attr.ib( - type=str, - metadata={"position": 1, "help": "inp1", "argstr": ""}, - ), - ) - ], - bases=(ShellDef,), - ) - shelly = ShellDef( - executable="executable", args="arg", inpA="inp1", input_spec=my_input_spec + @shell.define + class Shelly(ShellDef["Shelly.Outputs"]): + class Outputs(ShellOutputs): + pass + + executable = "executable" + inpA: str = shell.arg(position=1, help="inp1", argstr="") + + shelly = Shelly( + additional_args=["arg"], + inpA="inp1", ) assert shelly.cmdline == "executable inp1 arg" def test_shell_cmd_inputs_1a(): """additional input without provided position""" - my_input_spec = SpecInfo( - name="Input", - fields=[("inpA", attr.ib(type=str, metadata={"help": "inpA", "argstr": ""}))], - bases=(ShellDef,), - ) - shelly = ShellDef( - executable="executable", args="arg", inpA="inpNone1", input_spec=my_input_spec + @shell.define + class Shelly(ShellDef["Shelly.Outputs"]): + class Outputs(ShellOutputs): + pass + + executable = "executable" + inpA: str = shell.arg(help="inpA", argstr="") + + shelly = Shelly( + additional_args=["arg"], + inpA="inpNone1", ) # inp1 should be the first one after executable assert shelly.cmdline == "executable inpNone1 arg" @@ -59,102 +62,62 @@ def test_shell_cmd_inputs_1a(): def test_shell_cmd_inputs_1b(): """additional input with negative position""" - my_input_spec = SpecInfo( - name="Input", - fields=[ - ( - "inpA", - attr.ib( - type=str, - metadata={"position": -1, "help": "inpA", "argstr": ""}, - ), - ) - ], - bases=(ShellDef,), - ) + + @shell.define + class Shelly(ShellDef["Shelly.Outputs"]): + class Outputs(ShellOutputs): + pass + + executable = "executable" + inpA: str = shell.arg(position=-1, help="inpA", argstr="") # separate command into exec + args - shelly = ShellDef( - executable="executable", args="arg", inpA="inp-1", input_spec=my_input_spec + shelly = Shelly( + additional_args=["arg"], + inpA="inp-1", ) # inp1 should be last before arg assert shelly.cmdline == "executable inp-1 arg" -def test_shell_cmd_inputs_1_st(): - """additional input with provided position, checking cmdline when splitter""" - my_input_spec = SpecInfo( - name="Input", - fields=[ - ( - "inpA", - attr.ib( - type=str, - metadata={"position": 1, "help": "inp1", "argstr": ""}, - ), - ) - ], - bases=(ShellDef,), - ) +def test_shell_cmd_inputs_2(): + """additional inputs with provided positions""" - ShellDef( - name="shelly", - executable="executable", - args="arg", - input_spec=my_input_spec, - ).split("inpA", inpA=["inp1", "inp2"]) - # cmdline should be a list - # assert shelly.cmdline[0] == "executable inp1 arg" - # assert shelly.cmdline[1] == "executable inp2 arg" + @shell.define + class Shelly(ShellDef["Shelly.Outputs"]): + class Outputs(ShellOutputs): + pass + executable = "executable" -def test_shell_cmd_inputs_2(): - """additional inputs with provided positions""" - my_input_spec = SpecInfo( - name="Input", - fields=[ - ( - "inpA", - attr.ib( - type=str, - metadata={"position": 2, "help": "inpA", "argstr": ""}, - ), - ), - ( - "inpB", - attr.ib( - type=str, - metadata={"position": 1, "help": "inpN", "argstr": ""}, - ), - ), - ], - bases=(ShellDef,), - ) + inpA: str = shell.arg(position=2, help="inpA", argstr="") + inpB: str = shell.arg(position=1, help="inpN", argstr="") # separate command into exec + args - shelly = ShellDef( - executable="executable", inpB="inp1", inpA="inp2", input_spec=my_input_spec + shelly = Shelly( + inpB="inp1", + inpA="inp2", ) assert shelly.cmdline == "executable inp1 inp2" def test_shell_cmd_inputs_2a(): """additional inputs without provided positions""" - my_input_spec = SpecInfo( - name="Input", - fields=[ - ("inpA", attr.ib(type=str, metadata={"help": "inpA", "argstr": ""})), - ("inpB", attr.ib(type=str, metadata={"help": "inpB", "argstr": ""})), - ], - bases=(ShellDef,), - ) + + @shell.define + class Shelly(ShellDef["Shelly.Outputs"]): + class Outputs(ShellOutputs): + pass + + executable = "executable" + + inpA: str = shell.arg(help="inpA", argstr="") + inpB: str = shell.arg(help="inpB", argstr="") # separate command into exec + args - shelly = ShellDef( - executable="executable", + shelly = Shelly( inpA="inpNone1", inpB="inpNone2", - input_spec=my_input_spec, ) # position taken from the order in input definition assert shelly.cmdline == "executable inpNone1 inpNone2" @@ -162,95 +125,41 @@ def test_shell_cmd_inputs_2a(): def test_shell_cmd_inputs_2_err(): """additional inputs with provided positions (exception due to the duplication)""" - my_input_spec = SpecInfo( - name="Input", - fields=[ - ( - "inpA", - attr.ib( - type=str, - metadata={"position": 1, "help": "inpA", "argstr": ""}, - ), - ), - ( - "inpB", - attr.ib( - type=str, - metadata={"position": 1, "help": "inpB", "argstr": ""}, - ), - ), - ], - bases=(ShellDef,), - ) - shelly = ShellDef( - executable="executable", inpA="inp1", inpB="inp2", input_spec=my_input_spec - ) with pytest.raises(Exception) as e: - shelly.cmdline - assert "1 is already used" in str(e.value) + @shell.define + class Shelly(ShellDef["Shelly.Outputs"]): + class Outputs(ShellOutputs): + pass -def test_shell_cmd_inputs_2_noerr(): - """additional inputs with provided positions - (duplication of the position doesn't lead to error, since only one field has value) - """ - my_input_spec = SpecInfo( - name="Input", - fields=[ - ( - "inpA", - attr.ib( - type=str, - metadata={"position": 1, "help": "inpA", "argstr": ""}, - ), - ), - ( - "inpB", - attr.ib( - type=str, - metadata={"position": 1, "help": "inpB", "argstr": ""}, - ), - ), - ], - bases=(ShellDef,), - ) + executable = "executable" + + inpA: str = shell.arg(position=1, help="inpA", argstr="") + inpB: str = shell.arg(position=1, help="inpB", argstr="") - shelly = ShellDef(executable="executable", inpA="inp1", input_spec=my_input_spec) - shelly.cmdline + assert "Multiple fields have the overlapping positions" in str(e.value) def test_shell_cmd_inputs_3(): """additional inputs: positive pos, negative pos and no pos""" - my_input_spec = SpecInfo( - name="Input", - fields=[ - ( - "inpA", - attr.ib( - type=str, - metadata={"position": 1, "help": "inpA", "argstr": ""}, - ), - ), - ( - "inpB", - attr.ib( - type=str, - metadata={"position": -1, "help": "inpB", "argstr": ""}, - ), - ), - ("inpC", attr.ib(type=str, metadata={"help": "inpC", "argstr": ""})), - ], - bases=(ShellDef,), - ) + + @shell.define + class Shelly(ShellDef["Shelly.Outputs"]): + class Outputs(ShellOutputs): + pass + + executable = "executable" + + inpA: str = shell.arg(position=1, help="inpA", argstr="") + inpB: str = shell.arg(position=-1, help="inpB", argstr="") + inpC: str = shell.arg(help="inpC", argstr="") # separate command into exec + args - shelly = ShellDef( - executable="executable", + shelly = Shelly( inpA="inp1", inpB="inp-1", inpC="inpNone", - input_spec=my_input_spec, ) # input without position should be between positive an negative positions assert shelly.cmdline == "executable inp1 inpNone inp-1" @@ -258,1312 +167,916 @@ def test_shell_cmd_inputs_3(): def test_shell_cmd_inputs_argstr_1(): """additional string inputs with argstr""" - my_input_spec = SpecInfo( - name="Input", - fields=[ - ( - "inpA", - attr.ib( - type=str, - metadata={"position": 1, "help": "inpA", "argstr": "-v"}, - ), - ) - ], - bases=(ShellDef,), - ) - shelly = ShellDef(executable="executable", inpA="inp1", input_spec=my_input_spec) + @shell.define + class Shelly(ShellDef["Shelly.Outputs"]): + class Outputs(ShellOutputs): + pass + + executable = "executable" + + inpA: str = shell.arg(position=1, help="inpA", argstr="-v") + + shelly = Shelly(inpA="inp1") # flag used before inp1 assert shelly.cmdline == "executable -v inp1" def test_shell_cmd_inputs_argstr_2(): """additional bool inputs with argstr""" - my_input_spec = SpecInfo( - name="Input", - fields=[ - ( - "inpA", - attr.ib( - type=bool, - metadata={"position": 1, "help": "inpA", "argstr": "-v"}, - ), - ) - ], - bases=(ShellDef,), - ) + + @shell.define + class Shelly(ShellDef["Shelly.Outputs"]): + class Outputs(ShellOutputs): + pass + + executable = "executable" + + inpA: bool = shell.arg(position=1, help="inpA", argstr="-v") # separate command into exec + args - shelly = ShellDef( - executable="executable", args="arg", inpA=True, input_spec=my_input_spec - ) + shelly = Shelly(additional_args=["arg"], inpA=True) # a flag is used without any additional argument assert shelly.cmdline == "executable -v arg" def test_shell_cmd_inputs_list_1(): """providing list as an additional input, no sep, no argstr""" - my_input_spec = SpecInfo( - name="Input", - fields=[ - ( - "inpA", - attr.ib( - type=ty.List[str], - metadata={"position": 2, "help": "inpA", "argstr": ""}, - ), - ) - ], - bases=(ShellDef,), - ) - shelly = ShellDef( - executable="executable", inpA=["el_1", "el_2", "el_3"], input_spec=my_input_spec - ) + @shell.define + class Shelly(ShellDef["Shelly.Outputs"]): + class Outputs(ShellOutputs): + pass + + executable = "executable" + + inpA: ty.List[str] = shell.arg(position=2, help="inpA", argstr="", sep=" ") + + shelly = Shelly(inpA=["el_1", "el_2", "el_3"]) # multiple elements assert shelly.cmdline == "executable el_1 el_2 el_3" def test_shell_cmd_inputs_list_2(): """providing list as an additional input, no sep, but argstr""" - my_input_spec = SpecInfo( - name="Input", - fields=[ - ( - "inpA", - attr.ib( - type=ty.List[str], - metadata={"position": 2, "help": "inpA", "argstr": "-v"}, - ), - ) - ], - bases=(ShellDef,), - ) - shelly = ShellDef( - executable="executable", inpA=["el_1", "el_2", "el_3"], input_spec=my_input_spec - ) + @shell.define + class Shelly(ShellDef["Shelly.Outputs"]): + class Outputs(ShellOutputs): + pass + + executable = "executable" + + inpA: ty.List[str] = shell.arg(position=2, help="inpA", argstr="-v", sep=" ") + + shelly = Shelly(inpA=["el_1", "el_2", "el_3"]) assert shelly.cmdline == "executable -v el_1 el_2 el_3" def test_shell_cmd_inputs_list_3(): """providing list as an additional input, no sep, argstr with ...""" - my_input_spec = SpecInfo( - name="Input", - fields=[ - ( - "inpA", - attr.ib( - type=ty.List[str], - metadata={"position": 2, "help": "inpA", "argstr": "-v..."}, - ), - ) - ], - bases=(ShellDef,), - ) - shelly = ShellDef( - executable="executable", inpA=["el_1", "el_2", "el_3"], input_spec=my_input_spec - ) + @shell.define + class Shelly(ShellDef["Shelly.Outputs"]): + class Outputs(ShellOutputs): + pass + + executable = "executable" + + inpA: ty.List[str] = shell.arg(position=2, help="inpA", argstr="-v...", sep=" ") + + shelly = Shelly(inpA=["el_1", "el_2", "el_3"]) # a flag is repeated assert shelly.cmdline == "executable -v el_1 -v el_2 -v el_3" def test_shell_cmd_inputs_list_sep_1(): """providing list as an additional input:, sep, no argstr""" - my_input_spec = SpecInfo( - name="Input", - fields=[ - ( - "inpA", - attr.ib( - type=MultiInputObj[str], - metadata={ - "position": 1, - "help": "inpA", - "sep": ",", - "argstr": "", - }, - ), - ) - ], - bases=(ShellDef,), - ) - shelly = ShellDef( - executable="executable", - inpA=["aaa", "bbb", "ccc"], - input_spec=my_input_spec, - ) + @shell.define + class Shelly(ShellDef["Shelly.Outputs"]): + class Outputs(ShellOutputs): + pass + + executable = "executable" + + inpA: list[str] = shell.arg( + position=1, + help="inpA", + sep=",", + argstr="", + ) + + shelly = Shelly(inpA=["aaa", "bbb", "ccc"]) # separated by commas assert shelly.cmdline == "executable aaa,bbb,ccc" def test_shell_cmd_inputs_list_sep_2(): """providing list as an additional input:, sep, and argstr""" - my_input_spec = SpecInfo( - name="Input", - fields=[ - ( - "inpA", - attr.ib( - type=MultiInputObj[str], - metadata={ - "position": 1, - "help": "inpA", - "sep": ",", - "argstr": "-v", - }, - ), - ) - ], - bases=(ShellDef,), - ) - shelly = ShellDef( - executable="executable", - inpA=["aaa", "bbb", "ccc"], - input_spec=my_input_spec, - ) + @shell.define + class Shelly(ShellDef["Shelly.Outputs"]): + class Outputs(ShellOutputs): + pass + + executable = "executable" + + inpA: list[str] = shell.arg( + position=1, + help="inpA", + sep=",", + argstr="-v", + ) + + shelly = Shelly(inpA=["aaa", "bbb", "ccc"]) # a flag is used once assert shelly.cmdline == "executable -v aaa,bbb,ccc" def test_shell_cmd_inputs_list_sep_2a(): """providing list as an additional input:, sep, and argstr with f-string""" - my_input_spec = SpecInfo( - name="Input", - fields=[ - ( - "inpA", - attr.ib( - type=MultiInputObj[str], - metadata={ - "position": 1, - "help": "inpA", - "sep": ",", - "argstr": "-v {inpA}", - }, - ), - ) - ], - bases=(ShellDef,), - ) - shelly = ShellDef( - executable="executable", - inpA=["aaa", "bbb", "ccc"], - input_spec=my_input_spec, - ) + @shell.define + class Shelly(ShellDef["Shelly.Outputs"]): + class Outputs(ShellOutputs): + pass + + executable = "executable" + + inpA: list[str] = shell.arg( + position=1, + help="inpA", + sep=",", + argstr="-v {inpA}", + ) + + shelly = Shelly(inpA=["aaa", "bbb", "ccc"]) # a flag is used once assert shelly.cmdline == "executable -v aaa,bbb,ccc" def test_shell_cmd_inputs_list_sep_3(): """providing list as an additional input:, sep, argstr with ...""" - my_input_spec = SpecInfo( - name="Input", - fields=[ - ( - "inpA", - attr.ib( - type=MultiInputObj[str], - metadata={ - "position": 1, - "help": "inpA", - "sep": ",", - "argstr": "-v...", - }, - ), - ) - ], - bases=(ShellDef,), - ) - shelly = ShellDef( - executable="executable", - inpA=["aaa", "bbb", "ccc"], - input_spec=my_input_spec, - ) + @shell.define + class Shelly(ShellDef["Shelly.Outputs"]): + class Outputs(ShellOutputs): + pass + + executable = "executable" + + inpA: list[str] = shell.arg( + position=1, + help="inpA", + sep=",", + argstr="-v...", + ) + + shelly = Shelly(inpA=["aaa", "bbb", "ccc"]) # a flag is repeated assert shelly.cmdline == "executable -v aaa, -v bbb, -v ccc" def test_shell_cmd_inputs_list_sep_3a(): """providing list as an additional input:, sep, argstr with ... and f-string""" - my_input_spec = SpecInfo( - name="Input", - fields=[ - ( - "inpA", - attr.ib( - type=MultiInputObj[str], - metadata={ - "position": 1, - "help": "inpA", - "sep": ",", - "argstr": "-v {inpA}...", - }, - ), - ) - ], - bases=(ShellDef,), - ) - shelly = ShellDef( - executable="executable", - inpA=["aaa", "bbb", "ccc"], - input_spec=my_input_spec, - ) + @shell.define + class Shelly(ShellDef["Shelly.Outputs"]): + class Outputs(ShellOutputs): + pass + + executable = "executable" + + inpA: list[str] = shell.arg( + position=1, + help="inpA", + sep=",", + argstr="-v {inpA}...", + ) + + shelly = Shelly(inpA=["aaa", "bbb", "ccc"]) # a flag is repeated assert shelly.cmdline == "executable -v aaa, -v bbb, -v ccc" def test_shell_cmd_inputs_sep_4(): """providing 1-el list as an additional input:, sep, argstr with ...,""" - my_input_spec = SpecInfo( - name="Input", - fields=[ - ( - "inpA", - attr.ib( - type=MultiInputObj[str], - metadata={ - "position": 1, - "help": "inpA", - "sep": ",", - "argstr": "-v...", - }, - ), - ) - ], - bases=(ShellDef,), - ) - shelly = ShellDef(executable="executable", inpA=["aaa"], input_spec=my_input_spec) + @shell.define + class Shelly(ShellDef["Shelly.Outputs"]): + class Outputs(ShellOutputs): + pass + + executable = "executable" + + inpA: MultiInputObj[str] = shell.arg( + position=1, + help="inpA", + argstr="-v...", + ) + + shelly = Shelly(inpA=["aaa"]) assert shelly.cmdline == "executable -v aaa" def test_shell_cmd_inputs_sep_4a(): """providing str instead of list as an additional input:, sep, argstr with ...""" - my_input_spec = SpecInfo( - name="Input", - fields=[ - ( - "inpA", - attr.ib( - type=str, - metadata={ - "position": 1, - "help": "inpA", - "sep": ",", - "argstr": "-v...", - }, - ), - ) - ], - bases=(ShellDef,), - ) - shelly = ShellDef(executable="executable", inpA="aaa", input_spec=my_input_spec) + @shell.define + class Shelly(ShellDef["Shelly.Outputs"]): + class Outputs(ShellOutputs): + pass + + executable = "executable" + + inpA: str = shell.arg( + position=1, + help="inpA", + argstr="-v...", + ) + + shelly = Shelly(inpA="aaa") assert shelly.cmdline == "executable -v aaa" def test_shell_cmd_inputs_format_1(): """additional inputs with argstr that has string formatting""" - my_input_spec = SpecInfo( - name="Input", - fields=[ - ( - "inpA", - attr.ib( - type=str, - metadata={ - "position": 1, - "help": "inpA", - "argstr": "-v {inpA}", - }, - ), - ) - ], - bases=(ShellDef,), - ) - shelly = ShellDef(executable="executable", inpA="aaa", input_spec=my_input_spec) + @shell.define + class Shelly(ShellDef["Shelly.Outputs"]): + class Outputs(ShellOutputs): + pass + + executable = "executable" + + inpA: str = shell.arg( + position=1, + help="inpA", + argstr="-v {inpA}", + ) + + shelly = Shelly(inpA="aaa") assert shelly.cmdline == "executable -v aaa" def test_shell_cmd_inputs_format_2(): """additional inputs with argstr that has string formatting and ...""" - my_input_spec = SpecInfo( - name="Input", - fields=[ - ( - "inpA", - attr.ib( - type=MultiInputObj[str], - metadata={ - "position": 1, - "help": "inpA", - "argstr": "-v {inpA}...", - }, - ), - ) - ], - bases=(ShellDef,), - ) - shelly = ShellDef( - executable="executable", - inpA=["el_1", "el_2"], - input_spec=my_input_spec, - ) + @shell.define + class Shelly(ShellDef["Shelly.Outputs"]): + class Outputs(ShellOutputs): + pass + + executable = "executable" + + inpA: MultiInputObj[str] = shell.arg( + position=1, + help="inpA", + argstr="-v {inpA}...", + ) + + shelly = Shelly(inpA=["el_1", "el_2"]) assert shelly.cmdline == "executable -v el_1 -v el_2" def test_shell_cmd_inputs_format_3(): """adding float formatting for argstr with input field""" - my_input_spec = SpecInfo( - name="Input", - fields=[ - ( - "inpA", - attr.ib( - type=float, - metadata={ - "position": 1, - "help": "inpA", - "argstr": "-v {inpA:.5f}", - }, - ), - ) - ], - bases=(ShellDef,), - ) - shelly = ShellDef(executable="executable", inpA=0.007, input_spec=my_input_spec) + @shell.define + class Shelly(ShellDef["Shelly.Outputs"]): + class Outputs(ShellOutputs): + pass + + executable = "executable" + + inpA: float = shell.arg( + position=1, + help="inpA", + argstr="-v {inpA:.5f}", + ) + + shelly = Shelly(inpA=0.007) assert shelly.cmdline == "executable -v 0.00700" def test_shell_cmd_inputs_mandatory_1(): """additional inputs with mandatory=True""" - my_input_spec = SpecInfo( - name="Input", - fields=[ - ( - "inpA", - attr.ib( - type=str, - metadata={ - "position": 1, - "help": "inpA", - "argstr": "", - "mandatory": True, - }, - ), - ) - ], - bases=(ShellDef,), - ) - shelly = ShellDef(executable="executable", input_spec=my_input_spec) + @shell.define + class Shelly(ShellDef["Shelly.Outputs"]): + class Outputs(ShellOutputs): + pass + + executable = "executable" + + inpA: str = shell.arg( + position=1, + help="inpA", + argstr="", + ) + + shelly = Shelly() with pytest.raises(Exception) as e: shelly.cmdline - assert "mandatory" in str(e.value) + assert "mandatory" in str(e.value).lower() def test_shell_cmd_inputs_not_given_1(): - my_input_spec = SpecInfo( - name="Input", - fields=[ - ( - "arg1", - attr.ib( - type=MultiInputObj, - metadata={ - "argstr": "--arg1", - "help": "Command line argument 1", - }, - ), - ), - ( - "arg2", - attr.ib( - type=MultiInputObj, - metadata={ - "argstr": "--arg2", - "help": "Command line argument 2", - }, - ), - ), - ( - "arg3", - attr.ib( - type=File, - metadata={ - "argstr": "--arg3", - "help": "Command line argument 3", - }, - ), - ), - ], - bases=(ShellDef,), - ) - shelly = ShellDef(name="shelly", executable="executable", input_spec=my_input_spec) + @shell.define + class Shelly(ShellDef["Shelly.Outputs"]): + class Outputs(ShellOutputs): + pass - shelly.definition.arg2 = "argument2" + executable = "executable" + + arg1: MultiInputObj = shell.arg( + argstr="--arg1", + default=attrs.Factory(list), + help="Command line argument 1", + ) + arg2: MultiInputObj = shell.arg( + argstr="--arg2", + help="Command line argument 2", + ) + arg3: File | None = shell.arg( + argstr="--arg3", + default=None, + help="Command line argument 3", + ) + + shelly = Shelly() + + shelly.arg2 = "argument2" assert shelly.cmdline == "executable --arg2 argument2" def test_shell_cmd_inputs_template_1(): - """additional inputs, one uses output_file_template (and argstr)""" - my_input_spec = SpecInfo( - name="Input", - fields=[ - ( - "inpA", - attr.ib( - type=str, - metadata={ - "position": 1, - "help": "inpA", - "argstr": "", - "mandatory": True, - }, - ), - ), - ( - "outA", - attr.ib( - type=str, - metadata={ - "position": 2, - "help": "outA", - "argstr": "-o", - "output_file_template": "{inpA}_out", - }, - ), - ), - ], - bases=(ShellDef,), - ) + """additional inputs, one uses path_template (and argstr)""" + + @shell.define + class Shelly(ShellDef["Shelly.Outputs"]): + class Outputs(ShellOutputs): + outA: File = shell.outarg( + position=2, + help="outA", + argstr="-o", + path_template="{inpA}_out", + ) + + executable = "executable" + + inpA: str = shell.arg( + position=1, + help="inpA", + argstr="", + ) - shelly = ShellDef(executable="executable", input_spec=my_input_spec, inpA="inpA") + shelly = Shelly(inpA="inpA") # outA has argstr in the metadata fields, so it's a part of the command line # the full path will be use din the command line - assert shelly.cmdline == f"executable inpA -o {shelly.output_dir / 'inpA_out'}" + assert shelly.cmdline == f"executable inpA -o {Path.cwd() / 'inpA_out'}" # checking if outA in the output fields - assert shelly.output_names == ["return_code", "stdout", "stderr", "outA"] - - -def test_shell_cmd_inputs_template_1a(): - """additional inputs, one uses output_file_template (without argstr)""" - my_input_spec = SpecInfo( - name="Input", - fields=[ - ( - "inpA", - attr.ib( - type=str, - metadata={ - "position": 1, - "help": "inpA", - "argstr": "", - "mandatory": True, - }, - ), - ), - ( - "outA", - attr.ib( - type=str, - metadata={ - "help": "outA", - "output_file_template": "{inpA}_out", - }, - ), - ), - ], - bases=(ShellDef,), - ) - - shelly = ShellDef(executable="executable", input_spec=my_input_spec, inpA="inpA") - # outA has no argstr in metadata, so it's not a part of the command line - assert shelly.cmdline == "executable inpA" + assert get_output_names(shelly) == ["outA", "return_code", "stderr", "stdout"] # TODO: after deciding how we use requires/templates def test_shell_cmd_inputs_template_2(): - """additional inputs, one uses output_file_template (and argstr, but input not provided)""" - my_input_spec = SpecInfo( - name="Input", - fields=[ - ( - "inpB", - attr.ib( - type=str, - metadata={"position": 1, "help": "inpB", "argstr": ""}, - ), - ), - ( - "outB", - attr.ib( - type=str, - metadata={ - "position": 2, - "help": "outB", - "argstr": "-o", - "output_file_template": "{inpB}_out", - }, - ), - ), - ], - bases=(ShellDef,), - ) + """additional inputs, one uses path_template (and argstr, but input not provided)""" + + @shell.define + class Shelly(ShellDef["Shelly.Outputs"]): + class Outputs(ShellOutputs): + outB: File | None = shell.outarg( + position=2, + help="outB", + argstr="-o", + path_template="{inpB}_out", + ) - shelly = ShellDef(executable="executable", input_spec=my_input_spec) + executable = "executable" + + inpB: File | None = shell.arg(position=1, help="inpB", argstr="", default=None) + + shelly = Shelly() # inpB not in the inputs, so no outB in the command line assert shelly.cmdline == "executable" # checking if outB in the output fields - assert shelly.output_names == ["return_code", "stdout", "stderr", "outB"] + assert get_output_names(shelly) == ["outB", "return_code", "stderr", "stdout"] def test_shell_cmd_inputs_template_3(tmp_path): - """additional inputs with output_file_template and an additional + """additional inputs with path_template and an additional read-only fields that combine two outputs together in the command line """ inpA = tmp_path / "inpA" inpB = tmp_path / "inpB" Path.touch(inpA) Path.touch(inpB) - my_input_spec = SpecInfo( - name="Input", - fields=[ - ( - "inpA", - attr.ib( - type=str, - metadata={ - "position": 1, - "help": "inpA", - "argstr": "", - "mandatory": True, - }, - ), - ), - ( - "inpB", - attr.ib( - type=str, - metadata={ - "position": 2, - "help": "inpB", - "argstr": "", - "mandatory": True, - }, - ), - ), - ( - "outA", - attr.ib( - type=str, - metadata={ - "help": "outA", - "output_file_template": "{inpA}_out", - }, - ), - ), - ( - "outB", - attr.ib( - type=str, - metadata={ - "help": "outB", - "output_file_template": "{inpB}_out", - }, - ), - ), - ( - "outAB", - attr.ib( - type=str, - metadata={ - "position": -1, - "help": "outAB", - "argstr": "-o {outA} {outB}", - "readonly": True, - }, - ), - ), - ], - bases=(ShellDef,), - ) - shelly = ShellDef( - executable="executable", input_spec=my_input_spec, inpA=inpA, inpB=inpB - ) + @shell.define + class Shelly(ShellDef["Shelly.Outputs"]): + class Outputs(ShellOutputs): + + outA: File = shell.outarg( + help="outA", + path_template="{inpA}_out", + ) + outB: File = shell.outarg( + help="outB", + path_template="{inpB}_out", + ) + + executable = "executable" + + inpA: str = shell.arg( + position=1, + help="inpA", + argstr="", + ) + inpB: str = shell.arg( + position=2, + help="inpB", + argstr="", + ) + outAB: str = shell.arg( + position=-1, + help="outAB", + argstr="-o {outA} {outB}", + readonly=True, + ) + + shelly = Shelly(inpA=inpA, inpB=inpB) # using syntax from the outAB field assert ( shelly.cmdline - == f"executable {tmp_path / 'inpA'} {tmp_path / 'inpB'} -o {shelly.output_dir / 'inpA_out'} {str(shelly.output_dir / 'inpB_out')}" + == f"executable {tmp_path / 'inpA'} {tmp_path / 'inpB'} -o {Path.cwd() / 'inpA_out'} {str(Path.cwd() / 'inpB_out')}" ) # checking if outA and outB in the output fields (outAB should not be) - assert shelly.output_names == ["return_code", "stdout", "stderr", "outA", "outB"] + assert get_output_names(shelly) == [ + "return_code", + "stdout", + "stderr", + "outA", + "outB", + ] def test_shell_cmd_inputs_template_3a(): - """additional inputs with output_file_template and an additional + """additional inputs with path_template and an additional read-only fields that combine two outputs together in the command line testing a different order within the input definition """ - my_input_spec = SpecInfo( - name="Input", - fields=[ - ( - "inpA", - attr.ib( - type=str, - metadata={ - "position": 1, - "help": "inpA", - "argstr": "", - "mandatory": True, - }, - ), - ), - ( - "inpB", - attr.ib( - type=str, - metadata={ - "position": 2, - "help": "inpB", - "argstr": "", - "mandatory": True, - }, - ), - ), - ( - "outAB", - attr.ib( - type=str, - metadata={ - "position": -1, - "help": "outAB", - "argstr": "-o {outA} {outB}", - "readonly": True, - }, - ), - ), - ( - "outA", - attr.ib( - type=str, - metadata={ - "help": "outA", - "output_file_template": "{inpA}_out", - }, - ), - ), - ( - "outB", - attr.ib( - type=str, - metadata={ - "help": "outB", - "output_file_template": "{inpB}_out", - }, - ), - ), - ], - bases=(ShellDef,), - ) - shelly = ShellDef( - executable="executable", input_spec=my_input_spec, inpA="inpA", inpB="inpB" - ) + @shell.define + class Shelly(ShellDef["Shelly.Outputs"]): + class Outputs(ShellOutputs): + + outA: File = shell.outarg( + help="outA", + path_template="{inpA}_out", + ) + outB: File = shell.outarg( + help="outB", + path_template="{inpB}_out", + ) + + executable = "executable" + + inpA: str = shell.arg( + position=1, + help="inpA", + argstr="", + ) + inpB: str = shell.arg( + position=2, + help="inpB", + argstr="", + ) + outAB: str = shell.arg( + position=-1, + help="outAB", + argstr="-o {outA} {outB}", + readonly=True, + ) + + shelly = Shelly(inpA="inpA", inpB="inpB") # using syntax from the outAB field assert ( shelly.cmdline - == f"executable inpA inpB -o {shelly.output_dir / 'inpA_out'} {str(shelly.output_dir / 'inpB_out')}" + == f"executable inpA inpB -o {Path.cwd() / 'inpA_out'} {str(Path.cwd() / 'inpB_out')}" ) # checking if outA and outB in the output fields (outAB should not be) - assert shelly.output_names == ["return_code", "stdout", "stderr", "outA", "outB"] + assert get_output_names(shelly) == [ + "return_code", + "stdout", + "stderr", + "outA", + "outB", + ] # TODO: after deciding how we use requires/templates def test_shell_cmd_inputs_template_4(): - """additional inputs with output_file_template and an additional + """additional inputs with path_template and an additional read-only fields that combine two outputs together in the command line - one output_file_template can't be resolved - no inpB is provided + one path_template can't be resolved - no inpB is provided """ - my_input_spec = SpecInfo( - name="Input", - fields=[ - ( - "inpA", - attr.ib( - type=str, - metadata={ - "position": 1, - "help": "inpA", - "argstr": "", - "mandatory": True, - }, - ), - ), - ( - "inpB", - attr.ib( - type=str, - metadata={"position": 2, "help": "inpB", "argstr": ""}, - ), - ), - ( - "outAB", - attr.ib( - type=str, - metadata={ - "position": -1, - "help": "outAB", - "argstr": "-o {outA} {outB}", - "readonly": True, - }, - ), - ), - ( - "outA", - attr.ib( - type=str, - metadata={ - "help": "outA", - "output_file_template": "{inpA}_out", - }, - ), - ), - ( - "outB", - attr.ib( - type=str, - metadata={ - "help": "outB", - "output_file_template": "{inpB}_out", - }, - ), - ), - ], - bases=(ShellDef,), - ) - shelly = ShellDef(executable="executable", input_spec=my_input_spec, inpA="inpA") + @shell.define + class Shelly(ShellDef["Shelly.Outputs"]): + class Outputs(ShellOutputs): + outA: File = shell.outarg( + help="outA", + path_template="{inpA}_out", + ) + outB: str = shell.arg( + help="outB", + path_template="{inpB}_out", + ) + + executable = "executable" + + inpA: str = shell.arg( + position=1, + help="inpA", + argstr="", + ) + inpB: str = shell.arg(position=2, help="inpB", argstr="") + outAB: str = shell.arg( + position=-1, + help="outAB", + argstr="-o {outA} {outB}", + readonly=True, + ) + + shelly = Shelly(inpA="inpA") # inpB is not provided so outB not in the command line - assert shelly.cmdline == f"executable inpA -o {shelly.output_dir / 'inpA_out'}" - assert shelly.output_names == ["return_code", "stdout", "stderr", "outA", "outB"] + assert shelly.cmdline == f"executable inpA -o {Path.cwd() / 'inpA_out'}" + assert get_output_names(shelly) == [ + "return_code", + "stdout", + "stderr", + "outA", + "outB", + ] def test_shell_cmd_inputs_template_5_ex(): """checking if the exception is raised for read-only fields when input is set""" - my_input_spec = SpecInfo( - name="Input", - fields=[ - ( - "outAB", - attr.ib( - type=str, - metadata={ - "position": -1, - "help": "outAB", - "argstr": "-o", - "readonly": True, - }, - ), - ) - ], - bases=(ShellDef,), - ) - shelly = ShellDef(executable="executable", input_spec=my_input_spec, outAB="outAB") + @shell.define + class Shelly(ShellDef["Shelly.Outputs"]): + class Outputs(ShellOutputs): + pass + + executable = "executable" + + outAB: str = shell.arg( + position=-1, + help="outAB", + argstr="-o", + readonly=True, + ) + + shelly = Shelly(outAB="outAB") with pytest.raises(Exception) as e: shelly.cmdline assert "read only" in str(e.value) def test_shell_cmd_inputs_template_6(): - """additional inputs with output_file_template that has type ty.Union[str, bool] + """additional inputs with path_template that has type ty.Union[str, bool] no default is set, so if nothing is provided as an input, the output is used whenever the template can be formatted (the same way as for templates that has type=str) """ - my_input_spec = SpecInfo( - name="Input", - fields=[ - ( - "inpA", - attr.ib( - type=str, - metadata={ - "position": 1, - "help": "inpA", - "argstr": "", - "mandatory": True, - }, - ), - ), - ( - "outA", - attr.ib( - type=ty.Union[str, bool], - metadata={ - "position": 2, - "help": "outA", - "argstr": "-o", - "output_file_template": "{inpA}_out", - }, - ), - ), - ], - bases=(ShellDef,), - ) + + @shell.define + class Shelly(ShellDef["Shelly.Outputs"]): + class Outputs(ShellOutputs): + outA: File = shell.outarg( + position=2, + help="outA", + argstr="-o", + path_template="{inpA}_out", + ) + + executable = "executable" + + inpA: File = shell.arg( + position=1, + help="inpA", + argstr="", + ) # no input for outA (and no default value), so the output is created whenever the # template can be formatted (the same way as for templates that has type=str) - shelly = ShellDef(executable="executable", input_spec=my_input_spec, inpA="inpA") - assert shelly.cmdline == f"executable inpA -o {shelly.output_dir / 'inpA_out'}" + inpA = File.mock("inpA") + shelly = Shelly(inpA=inpA) + assert shelly.cmdline == f"executable inpA -o {Path.cwd() / 'inpA_out'}" # a string is provided for outA, so this should be used as the outA value - shelly = ShellDef( - executable="executable", input_spec=my_input_spec, inpA="inpA", outA="outA" - ) + shelly = Shelly(inpA=inpA, outA="outA") assert shelly.cmdline == "executable inpA -o outA" # True is provided for outA, so the formatted template should be used as outA value - shelly = ShellDef( - executable="executable", input_spec=my_input_spec, inpA="inpA", outA=True - ) - assert shelly.cmdline == f"executable inpA -o {shelly.output_dir / 'inpA_out'}" + shelly = Shelly(inpA=inpA, outA=True) + assert shelly.cmdline == f"executable inpA -o {Path.cwd() / 'inpA_out'}" # False is provided for outA, so the outA shouldn't be used - shelly = ShellDef( - executable="executable", input_spec=my_input_spec, inpA="inpA", outA=False - ) + shelly = Shelly(inpA=inpA, outA=False) assert shelly.cmdline == "executable inpA" def test_shell_cmd_inputs_template_6a(): - """additional inputs with output_file_template that has type ty.Union[str, bool] + """additional inputs with path_template that has type ty.Union[str, bool] and default is set to False, so if nothing is provided as an input, the output is not used """ - my_input_spec = SpecInfo( - name="Input", - fields=[ - ( - "inpA", - attr.ib( - type=str, - metadata={ - "position": 1, - "help": "inpA", - "argstr": "", - "mandatory": True, - }, - ), - ), - ( - "outA", - attr.ib( - type=ty.Union[str, bool], - default=False, - metadata={ - "position": 2, - "help": "outA", - "argstr": "-o", - "output_file_template": "{inpA}_out", - }, - ), - ), - ], - bases=(ShellDef,), - ) + + @shell.define + class Shelly(ShellDef["Shelly.Outputs"]): + class Outputs(ShellOutputs): + outA: File | None = shell.outarg( + position=2, + help="outA", + argstr="-o", + path_template="{inpA}_out", + ) + + executable = "executable" + + inpA: str = shell.arg( + position=1, + help="inpA", + argstr="", + ) # no input for outA, but default is False, so the outA shouldn't be used - shelly = ShellDef(executable="executable", input_spec=my_input_spec, inpA="inpA") + shelly = Shelly(inpA="inpA") assert shelly.cmdline == "executable inpA" # a string is provided for outA, so this should be used as the outA value - shelly = ShellDef( - executable="executable", input_spec=my_input_spec, inpA="inpA", outA="outA" - ) + shelly = Shelly(inpA="inpA", outA="outA") assert shelly.cmdline == "executable inpA -o outA" # True is provided for outA, so the formatted template should be used as outA value - shelly = ShellDef( - executable="executable", input_spec=my_input_spec, inpA="inpA", outA=True - ) - assert shelly.cmdline == f"executable inpA -o {shelly.output_dir / 'inpA_out'}" + shelly = Shelly(inpA="inpA", outA=True) + assert shelly.cmdline == f"executable inpA -o {Path.cwd() / 'inpA_out'}" # False is provided for outA, so the outA shouldn't be used - shelly = ShellDef( - executable="executable", input_spec=my_input_spec, inpA="inpA", outA=False - ) + shelly = Shelly(inpA="inpA", outA=False) assert shelly.cmdline == "executable inpA" def test_shell_cmd_inputs_template_7(tmp_path: Path): - """additional inputs uses output_file_template with a suffix (no extension) + """additional inputs uses path_template with a suffix (no extension) no keep_extension is used """ - my_input_spec = SpecInfo( - name="Input", - fields=[ - ( - "inpA", - attr.ib( - type=File, - metadata={ - "position": 1, - "help": "inpA", - "argstr": "", - "mandatory": True, - }, - ), - ), - ( - "outA", - attr.ib( - type=str, - metadata={ - "position": 2, - "help": "outA", - "argstr": "", - "output_file_template": "{inpA}_out", - }, - ), - ), - ], - bases=(ShellDef,), - ) + + @shell.define + class Shelly(ShellDef["Shelly.Outputs"]): + class Outputs(ShellOutputs): + outA: File = shell.outarg( + position=2, + help="outA", + argstr="", + path_template="{inpA}_out", + ) + + executable = "executable" + + inpA: File = shell.arg( + position=1, + help="inpA", + argstr="", + ) inpA_file = tmp_path / "a_file.txt" inpA_file.write_text("content") - shelly = ShellDef(executable="executable", input_spec=my_input_spec, inpA=inpA_file) + shelly = Shelly(inpA=inpA_file) # outA should be formatted in a way that that .txt goes to the end assert ( shelly.cmdline - == f"executable {tmp_path / 'a_file.txt'} {shelly.output_dir / 'a_file_out.txt'}" + == f"executable {tmp_path / 'a_file.txt'} {Path.cwd() / 'a_file_out.txt'}" ) def test_shell_cmd_inputs_template_7a(tmp_path: Path): - """additional inputs uses output_file_template with a suffix (no extension) + """additional inputs uses path_template with a suffix (no extension) keep_extension is True (as default) """ - my_input_spec = SpecInfo( - name="Input", - fields=[ - ( - "inpA", - attr.ib( - type=File, - metadata={ - "position": 1, - "help": "inpA", - "argstr": "", - "mandatory": True, - }, - ), - ), - ( - "outA", - attr.ib( - type=str, - metadata={ - "position": 2, - "help": "outA", - "argstr": "", - "keep_extension": True, - "output_file_template": "{inpA}_out", - }, - ), - ), - ], - bases=(ShellDef,), - ) + + @shell.define + class Shelly(ShellDef["Shelly.Outputs"]): + class Outputs(ShellOutputs): + outA: File = shell.outarg( + position=2, + help="outA", + argstr="", + keep_extension=True, + path_template="{inpA}_out", + ) + + executable = "executable" + + inpA: File = shell.arg( + position=1, + help="inpA", + argstr="", + ) inpA_file = tmp_path / "a_file.txt" inpA_file.write_text("content") - shelly = ShellDef(executable="executable", input_spec=my_input_spec, inpA=inpA_file) + shelly = Shelly(inpA=inpA_file) # outA should be formatted in a way that that .txt goes to the end assert ( shelly.cmdline - == f"executable {tmp_path / 'a_file.txt'} {shelly.output_dir / 'a_file_out.txt'}" + == f"executable {tmp_path / 'a_file.txt'} {Path.cwd() / 'a_file_out.txt'}" ) def test_shell_cmd_inputs_template_7b(tmp_path: Path): - """additional inputs uses output_file_template with a suffix (no extension) + """additional inputs uses path_template with a suffix (no extension) keep extension is False (so the extension is removed when creating the output) """ - my_input_spec = SpecInfo( - name="Input", - fields=[ - ( - "inpA", - attr.ib( - type=File, - metadata={ - "position": 1, - "help": "inpA", - "argstr": "", - "mandatory": True, - }, - ), - ), - ( - "outA", - attr.ib( - type=str, - metadata={ - "position": 2, - "help": "outA", - "argstr": "", - "keep_extension": False, - "output_file_template": "{inpA}_out", - }, - ), - ), - ], - bases=(ShellDef,), - ) + + @shell.define + class Shelly(ShellDef["Shelly.Outputs"]): + class Outputs(ShellOutputs): + outA: File = shell.outarg( + position=2, + help="outA", + argstr="", + keep_extension=False, + path_template="{inpA}_out", + ) + + executable = "executable" + + inpA: File = shell.arg( + position=1, + help="inpA", + argstr="", + ) inpA_file = tmp_path / "a_file.txt" inpA_file.write_text("content") - shelly = ShellDef(executable="executable", input_spec=my_input_spec, inpA=inpA_file) + shelly = Shelly(inpA=inpA_file) # outA should be formatted in a way that that .txt goes to the end assert ( shelly.cmdline - == f"executable {tmp_path / 'a_file.txt'} {shelly.output_dir / 'a_file_out'}" + == f"executable {tmp_path / 'a_file.txt'} {Path.cwd() / 'a_file_out'}" ) def test_shell_cmd_inputs_template_8(tmp_path: Path): - """additional inputs uses output_file_template with a suffix and an extension""" - my_input_spec = SpecInfo( - name="Input", - fields=[ - ( - "inpA", - attr.ib( - type=File, - metadata={ - "position": 1, - "help": "inpA", - "argstr": "", - "mandatory": True, - }, - ), - ), - ( - "outA", - attr.ib( - type=str, - metadata={ - "position": 2, - "help": "outA", - "argstr": "", - "output_file_template": "{inpA}_out.txt", - }, - ), - ), - ], - bases=(ShellDef,), - ) + """additional inputs uses path_template with a suffix and an extension""" + + @shell.define + class Shelly(ShellDef["Shelly.Outputs"]): + class Outputs(ShellOutputs): + outA: File = shell.outarg( + position=2, + help="outA", + argstr="", + path_template="{inpA}_out.txt", + ) + + executable = "executable" + + inpA: File = shell.arg( + position=1, + help="inpA", + argstr="", + ) inpA_file = tmp_path / "a_file.t" inpA_file.write_text("content") - shelly = ShellDef(executable="executable", input_spec=my_input_spec, inpA=inpA_file) + shelly = Shelly(inpA=inpA_file) # outA should be formatted in a way that inpA extension is removed and the template extension is used assert ( shelly.cmdline - == f"executable {tmp_path / 'a_file.t'} {shelly.output_dir / 'a_file_out.txt'}" + == f"executable {tmp_path / 'a_file.t'} {Path.cwd() / 'a_file_out.txt'}" ) def test_shell_cmd_inputs_template_9(tmp_path: Path): - """additional inputs, one uses output_file_template with two fields: + """additional inputs, one uses path_template with two fields: one File and one ints - the output should be recreated from the template """ - my_input_spec = SpecInfo( - name="Input", - fields=[ - ( - "inpA", - attr.ib( - type=File, - metadata={ - "position": 1, - "help": "inpA", - "argstr": "", - "mandatory": True, - }, - ), - ), - ( - "inpInt", - attr.ib( - type=int, - metadata={ - "position": 2, - "help": "inp int", - "argstr": "-i", - "mandatory": True, - }, - ), - ), - ( - "outA", - attr.ib( - type=str, - metadata={ - "position": 3, - "help": "outA", - "argstr": "-o", - "output_file_template": "{inpA}_{inpInt}_out.txt", - }, - ), - ), - ], - bases=(ShellDef,), - ) + + @shell.define + class Shelly(ShellDef["Shelly.Outputs"]): + class Outputs(ShellOutputs): + outA: File = shell.outarg( + position=3, + help="outA", + argstr="-o", + path_template="{inpA}_{inpInt}_out.txt", + ) + + executable = "executable" + + inpA: File = shell.arg( + position=1, + help="inpA", + argstr="", + ) + inpInt: int = shell.arg( + position=2, + help="inp int", + argstr="-i", + ) inpA_file = tmp_path / "inpA.t" inpA_file.write_text("content") - shelly = ShellDef( - executable="executable", input_spec=my_input_spec, inpA=inpA_file, inpInt=3 - ) + shelly = Shelly(inpA=inpA_file, inpInt=3) assert ( shelly.cmdline - == f"executable {tmp_path / 'inpA.t'} -i 3 -o {shelly.output_dir / 'inpA_3_out.txt'}" + == f"executable {tmp_path / 'inpA.t'} -i 3 -o {Path.cwd() / 'inpA_3_out.txt'}" ) # checking if outA in the output fields - assert shelly.output_names == ["return_code", "stdout", "stderr", "outA"] + assert get_output_names(shelly) == ["outA", "return_code", "stderr", "stdout"] def test_shell_cmd_inputs_template_9a(tmp_path: Path): - """additional inputs, one uses output_file_template with two fields: + """additional inputs, one uses path_template with two fields: one file and one string without extension - should be fine """ - my_input_spec = SpecInfo( - name="Input", - fields=[ - ( - "inpA", - attr.ib( - type=File, - metadata={ - "position": 1, - "help": "inpA", - "argstr": "", - "mandatory": True, - }, - ), - ), - ( - "inpStr", - attr.ib( - type=str, - metadata={ - "position": 2, - "help": "inp str", - "argstr": "-i", - "mandatory": True, - }, - ), - ), - ( - "outA", - attr.ib( - type=str, - metadata={ - "position": 3, - "help": "outA", - "argstr": "-o", - "output_file_template": "{inpA}_{inpStr}_out.txt", - }, - ), - ), - ], - bases=(ShellDef,), - ) + + @shell.define + class Shelly(ShellDef["Shelly.Outputs"]): + class Outputs(ShellOutputs): + outA: File = shell.outarg( + position=3, + help="outA", + argstr="-o", + path_template="{inpA}_{inpStr}_out.txt", + ) + + executable = "executable" + + inpA: File = shell.arg( + position=1, + help="inpA", + argstr="", + ) + inpStr: str = shell.arg( + position=2, + help="inp str", + argstr="-i", + ) inpA_file = tmp_path / "inpA.t" inpA_file.write_text("content") - shelly = ShellDef( - executable="executable", input_spec=my_input_spec, inpA=inpA_file, inpStr="hola" - ) + shelly = Shelly(inpA=inpA_file, inpStr="hola") assert ( shelly.cmdline - == f"executable {tmp_path / 'inpA.t'} -i hola -o {shelly.output_dir / 'inpA_hola_out.txt'}" + == f"executable {tmp_path / 'inpA.t'} -i hola -o {Path.cwd() / 'inpA_hola_out.txt'}" ) # checking if outA in the output fields - assert shelly.output_names == ["return_code", "stdout", "stderr", "outA"] + assert get_output_names(shelly) == ["outA", "return_code", "stderr", "stdout"] def test_shell_cmd_inputs_template_9b_err(tmp_path: Path): - """output_file_template with two fields that are both Files, + """path_template with two fields that are both Files, an exception should be raised """ - my_input_spec = SpecInfo( - name="Input", - fields=[ - ( - "inpA", - attr.ib( - type=File, - metadata={ - "position": 1, - "help": "inpA", - "argstr": "", - "mandatory": True, - }, - ), - ), - ( - "inpFile", - attr.ib( - type=File, - metadata={ - "position": 2, - "help": "inp file", - "argstr": "-i", - "mandatory": True, - }, - ), - ), - ( - "outA", - attr.ib( - type=str, - metadata={ - "position": 3, - "help": "outA", - "argstr": "-o", - "output_file_template": "{inpA}_{inpFile}_out.txt", - }, - ), - ), - ], - bases=(ShellDef,), - ) + + @shell.define + class Shelly(ShellDef["Shelly.Outputs"]): + class Outputs(ShellOutputs): + + outA: File = shell.outarg( + position=3, + help="outA", + argstr="-o", + path_template="{inpA}_{inpFile}_out.txt", + ) + + executable = "executable" + + inpA: File = shell.arg( + position=1, + help="inpA", + argstr="", + ) + inpFile: File = shell.arg( + position=2, + help="inp file", + argstr="-i", + ) inpA_file = tmp_path / "inpA.t" inpA_file.write_text("content") @@ -1571,9 +1084,7 @@ def test_shell_cmd_inputs_template_9b_err(tmp_path: Path): inpFile_file = tmp_path / "inpFile.t" inpFile_file.write_text("content") - shelly = ShellDef( - executable="executable", - input_spec=my_input_spec, + shelly = Shelly( inpA=inpA_file, inpFile=inpFile_file, ) @@ -1583,58 +1094,38 @@ def test_shell_cmd_inputs_template_9b_err(tmp_path: Path): def test_shell_cmd_inputs_template_9c_err(tmp_path: Path): - """output_file_template with two fields: a file and a string with extension, + """path_template with two fields: a file and a string with extension, that should be used as an additional file and the exception should be raised """ - my_input_spec = SpecInfo( - name="Input", - fields=[ - ( - "inpA", - attr.ib( - type=File, - metadata={ - "position": 1, - "help": "inpA", - "argstr": "", - "mandatory": True, - }, - ), - ), - ( - "inpStr", - attr.ib( - type=str, - metadata={ - "position": 2, - "help": "inp str with extension", - "argstr": "-i", - "mandatory": True, - }, - ), - ), - ( - "outA", - attr.ib( - type=str, - metadata={ - "position": 3, - "help": "outA", - "argstr": "-o", - "output_file_template": "{inpA}_{inpStr}_out.txt", - }, - ), - ), - ], - bases=(ShellDef,), - ) + + @shell.define + class Shelly(ShellDef["Shelly.Outputs"]): + class Outputs(ShellOutputs): + + outA: File = shell.outarg( + position=3, + help="outA", + argstr="-o", + path_template="{inpA}_{inpStr}_out.txt", + ) + + executable = "executable" + + inpA: File = shell.arg( + position=1, + help="inpA", + argstr="", + ) + inpStr: str = shell.arg( + position=2, + help="inp str with extension", + argstr="-i", + ) inpA_file = tmp_path / "inpA.t" inpA_file.write_text("content") - shelly = ShellDef( - executable="executable", - input_spec=my_input_spec, + shelly = Shelly( inpA=inpA_file, inpStr="hola.txt", ) @@ -1645,98 +1136,69 @@ def test_shell_cmd_inputs_template_9c_err(tmp_path: Path): def test_shell_cmd_inputs_template_10(): - """output_file_template uses a float field with formatting""" - my_input_spec = SpecInfo( - name="Input", - fields=[ - ( - "inpA", - attr.ib( - type=float, - metadata={ - "position": 1, - "help": "inpA", - "argstr": "{inpA:.1f}", - "mandatory": True, - }, - ), - ), - ( - "outA", - attr.ib( - type=str, - metadata={ - "position": 2, - "help": "outA", - "argstr": "-o", - "output_file_template": "file_{inpA:.1f}_out", - }, - ), - ), - ], - bases=(ShellDef,), - ) + """path_template uses a float field with formatting""" + + @shell.define + class Shelly(ShellDef["Shelly.Outputs"]): + class Outputs(ShellOutputs): + + outA: File = shell.outarg( + position=2, + help="outA", + argstr="-o", + path_template="file_{inpA:.1f}_out", + ) + + executable = "executable" + + inpA: float = shell.arg( + position=1, + help="inpA", + argstr="{inpA:.1f}", + ) - shelly = ShellDef(executable="executable", input_spec=my_input_spec, inpA=3.3456) + shelly = Shelly(inpA=3.3456) # outA has argstr in the metadata fields, so it's a part of the command line # the full path will be use din the command line - assert shelly.cmdline == f"executable 3.3 -o {shelly.output_dir / 'file_3.3_out'}" + assert shelly.cmdline == f"executable 3.3 -o {Path.cwd() / 'file_3.3_out'}" # checking if outA in the output fields - assert shelly.output_names == ["return_code", "stdout", "stderr", "outA"] + assert get_output_names(shelly) == ["outA", "return_code", "stderr", "stdout"] def test_shell_cmd_inputs_template_requires_1(): """Given an input definition with a templated output file subject to required fields, ensure the field is set only when all requirements are met.""" - my_input_spec = SpecInfo( - name="Input", - fields=[ - ( - "in_file", - attr.ib( - type=str, - metadata={ - "help": "input file", - "mandatory": True, - "argstr": "", - }, - ), - ), - ( - "with_tpl", - attr.ib( - type=bool, - metadata={"help": "enable template"}, - ), - ), - ( - "out_file", - attr.ib( - type=str, - metadata={ - "help": "output file", - "argstr": "--tpl", - "output_file_template": "tpl.{in_file}", - "requires": {"with_tpl"}, - }, - ), - ), - ], - bases=(ShellDef,), - ) + @shell.define + class Shelly(ShellDef["Shelly.Outputs"]): + class Outputs(ShellOutputs): + + out_file: File | None = shell.outarg( + help="output file", + argstr="--tpl", + path_template="tpl.{in_file}", + requires={"with_tpl"}, + ) + + executable = "executable" + + in_file: str = shell.arg( + help="input file", + argstr="", + ) + with_tpl: bool = shell.arg(help="enable template", default=False) # When requirements are not met. - shelly = ShellDef(executable="cmd", input_spec=my_input_spec, in_file="in.file") + shelly = Shelly(executable="cmd", in_file="in.file") assert "--tpl" not in shelly.cmdline # When requirements are met. - shelly.definition.with_tpl = True + shelly.with_tpl = True assert "tpl.in.file" in shelly.cmdline def test_shell_cmd_inputs_template_function_1(): - """one input field uses output_file_template that is a simple function + """one input field uses path_template that is a simple function this can be easily done by simple template as in test_shell_cmd_inputs_template_1 """ @@ -1744,44 +1206,32 @@ def test_shell_cmd_inputs_template_function_1(): def template_fun(inputs): return "{inpA}_out" - my_input_spec = SpecInfo( - name="Input", - fields=[ - ( - "inpA", - attr.ib( - type=str, - metadata={ - "position": 1, - "help": "inpA", - "argstr": "", - "mandatory": True, - }, - ), - ), - ( - "outA", - attr.ib( - type=str, - metadata={ - "position": 2, - "help": "outA", - "argstr": "-o", - "output_file_template": template_fun, - }, - ), - ), - ], - bases=(ShellDef,), - ) + @shell.define + class Shelly(ShellDef["Shelly.Outputs"]): + class Outputs(ShellOutputs): - shelly = ShellDef(executable="executable", input_spec=my_input_spec, inpA="inpA") + outA: File = shell.outarg( + position=2, + help="outA", + argstr="-o", + path_template=template_fun, + ) - assert shelly.cmdline == f"executable inpA -o {shelly.output_dir / 'inpA_out'}" + executable = "executable" + + inpA: str = shell.arg( + position=1, + help="inpA", + argstr="", + ) + + shelly = Shelly(inpA="inpA") + + assert shelly.cmdline == f"executable inpA -o {Path.cwd() / 'inpA_out'}" def test_shell_cmd_inputs_template_function_2(): - """one input field uses output_file_template that is a function, + """one input field uses path_template that is a function, depending on a value of an input it returns different template """ @@ -1792,104 +1242,34 @@ def template_fun(inputs): else: return "{inpA}_odd" - my_input_spec = SpecInfo( - name="Input", - fields=[ - ( - "inpA", - attr.ib( - type=str, - metadata={ - "position": 1, - "help": "inpA", - "argstr": "", - "mandatory": True, - }, - ), - ), - ( - "inpB", - attr.ib( - type=int, - metadata={ - "help": "inpB", - "mandatory": True, - }, - ), - ), - ( - "outA", - attr.ib( - type=str, - metadata={ - "position": 2, - "help": "outA", - "argstr": "-o", - "output_file_template": template_fun, - }, - ), - ), - ], - bases=(ShellDef,), - ) + @shell.define + class Shelly(ShellDef["Shelly.Outputs"]): + class Outputs(ShellOutputs): - shelly = ShellDef( - executable="executable", - input_spec=my_input_spec, - inpA="inpA", - inpB=1, - ) + outA: File = shell.outarg( + position=2, + help="outA", + argstr="-o", + path_template=template_fun, + ) - assert shelly.cmdline == f"executable inpA -o {shelly.output_dir / 'inpA_odd'}" + executable = "executable" + inpA: str = shell.arg( + position=1, + help="inpA", + argstr="", + ) + inpB: int = shell.arg( + help="inpB", + ) -def test_shell_cmd_inputs_template_1_st(): - """additional inputs, one uses output_file_template (and argstr) - testing cmdline when splitter defined - """ - my_input_spec = SpecInfo( - name="Input", - fields=[ - ( - "inpA", - attr.ib( - type=str, - metadata={ - "position": 1, - "help": "inpA", - "argstr": "", - "mandatory": True, - }, - ), - ), - ( - "outA", - attr.ib( - type=str, - metadata={ - "position": 2, - "help": "outA", - "argstr": "-o", - "output_file_template": "{inpA}_out", - }, - ), - ), - ], - bases=(ShellDef,), + shelly = Shelly( + inpA="inpA", + inpB=1, ) - inpA = ["inpA_1", "inpA_2"] - ShellDef( - name="f", - executable="executable", - input_spec=my_input_spec, - ).split("inpA", inpA=inpA) - - # cmdline_list = shelly.cmdline - # assert len(cmdline_list) == 2 - # for i in range(2): - # path_out = Path(shelly.output_dir[i]) / f"{inpA[i]}_out" - # assert cmdline_list[i] == f"executable {inpA[i]} -o {path_out}" + assert shelly.cmdline == f"executable inpA -o {Path.cwd() / 'inpA_odd'}" # TODO: after deciding how we use requires/templates @@ -1897,227 +1277,140 @@ def test_shell_cmd_inputs_denoise_image( tmp_path, ): """example from #279""" - my_input_spec = SpecInfo( - name="Input", - fields=[ - ( - "image_dimensionality", - attr.ib( - type=int, - metadata={ - "help": """ + + @shell.define + class Shelly(ShellDef["Shelly.Outputs"]): + class Outputs(ShellOutputs): + + correctedImage: File | None = shell.outarg( + help=""" + The output consists of the noise corrected version of the input image. + Optionally, one can also output the estimated noise image. """, + path_template="{inputImageFilename}_out", + ) + noiseImage: File | None = shell.outarg( + help=""" + The output consists of the noise corrected version of the input image. + Optionally, one can also output the estimated noise image. """, + path_template="{inputImageFilename}_noise", + ) + + executable = "executable" + + image_dimensionality: int | None = shell.arg( + help=""" 2/3/4 This option forces the image to be treated as a specified-dimensional image. If not specified, the program tries to infer the dimensionality from the input image. """, - "allowed_values": [2, 3, 4], - "argstr": "-d", - }, - ), - ), - ( - "inputImageFilename", - attr.ib( - type=File, - metadata={ - "help": "A scalar image is expected as input for noise correction.", - "argstr": "-i", - "mandatory": True, - }, - ), - ), - ( - "noise_model", - attr.ib( - type=str, - metadata={ - "help": """ - Rician/(Gaussian) - Employ a Rician or Gaussian noise model. - """, - "allowed_values": ["Rician", "Gaussian"], - "argstr": "-n", - }, - ), - ), - ( - "maskImageFilename", - attr.ib( - type=str, - metadata={ - "help": "If a mask image is specified, denoising is only performed in the mask region.", - "argstr": "-x", - }, - ), - ), - ( - "shrink_factor", - attr.ib( - type=int, - default=1, - metadata={ - "help": """ - (1)/2/3/... - Running noise correction on large images can be time consuming. - To lessen computation time, the input image can be resampled. - The shrink factor, specified as a single integer, describes this - resampling. Shrink factor = 1 is the default. - """, - "argstr": "-s", - }, - ), - ), - ( - "patch_radius", - attr.ib( - type=int, - default=1, - metadata={ - "help": "Patch radius. Default = 1x1x1", - "argstr": "-p", - }, - ), - ), - ( - "search_radius", - attr.ib( - type=int, - default=2, - metadata={ - "help": "Search radius. Default = 2x2x2.", - "argstr": "-r", - }, - ), - ), - ( - "correctedImage", - attr.ib( - type=str, - metadata={ - "help": """ - The output consists of the noise corrected version of the input image. - Optionally, one can also output the estimated noise image. - """, - "output_file_template": "{inputImageFilename}_out", - }, - ), - ), - ( - "noiseImage", - attr.ib( - type=ty.Union[str, bool], - default=False, - metadata={ - "help": """ - The output consists of the noise corrected version of the input image. - Optionally, one can also output the estimated noise image. - """, - "output_file_template": "{inputImageFilename}_noise", - }, - ), - ), - ( - "output", - attr.ib( - type=str, - metadata={ - "help": "Combined output", - "argstr": "-o [{correctedImage}, {noiseImage}]", - "position": -1, - "readonly": True, - }, - ), - ), - ( - "version", - attr.ib( - type=bool, - default=False, - metadata={ - "help": "Get Version Information.", - "argstr": "--version", - }, - ), - ), - ( - "verbose", - attr.ib( - type=int, - default=0, - metadata={"help": "(0)/1. Verbose output. ", "argstr": "-v"}, - ), - ), - ( - "help_short", - attr.ib( - type=bool, - default=False, - metadata={ - "help": "Print the help menu (short version)", - "argstr": "-h", - }, - ), - ), - ( - "help", - attr.ib( - type=int, - metadata={ - "help": "Print the help menu.", - "argstr": "--help", - }, - ), - ), - ], - bases=(ShellDef,), - ) + allowed_values=[2, 3, 4, None], + default=None, + argstr="-d", + ) + inputImageFilename: File = shell.arg( + help="A scalar image is expected as input for noise correction.", + argstr="-i", + ) + noise_model: str | None = shell.arg( + default=None, + help=""" Rician/(Gaussian) Employ a Rician or Gaussian noise model. """, + allowed_values=["Rician", "Gaussian"], + argstr="-n", + ) + maskImageFilename: str | None = shell.arg( + default=None, + help="If a mask image is specified, denoising is only performed in the mask region.", + argstr="-x", + ) + shrink_factor: int = shell.arg( + default=1, + help=""" + (1)/2/3/... + Running noise correction on large images can be time consuming. + To lessen computation time, the input image can be resampled. + The shrink factor, specified as a single integer, describes this + resampling. Shrink factor = 1 is the default. """, + argstr="-s", + ) + patch_radius: int = shell.arg( + default=1, + help="Patch radius. Default = 1x1x1", + argstr="-p", + ) + search_radius: int = shell.arg( + default=2, + help="Search radius. Default = 2x2x2.", + argstr="-r", + ) + output: str | None = shell.arg( + default=None, + help="Combined output", + argstr="-o [{correctedImage}, {noiseImage}]", + position=-1, + readonly=True, + ) + version: bool = shell.arg( + default=False, + help="Get Version Information.", + argstr="--version", + ) + verbose: int = shell.arg(default=0, help="(0)/1. Verbose output. ", argstr="-v") + help_short: bool = shell.arg( + default=False, + help="Print the help menu (short version)", + argstr="-h", + ) + help: int | None = shell.arg( + default=None, + help="Print the help menu.", + argstr="--help", + ) my_input_file = tmp_path / "a_file.ext" my_input_file.write_text("content") # no input provided - shelly = ShellDef(executable="DenoiseImage", input_spec=my_input_spec) + shelly = Shelly( + executable="DenoiseImage", + ) with pytest.raises(Exception) as e: shelly.cmdline - assert "mandatory" in str(e.value) + assert "mandatory" in str(e.value).lower() # input file name, noiseImage is not set, so using default value False - shelly = ShellDef( + shelly = Shelly( executable="DenoiseImage", inputImageFilename=my_input_file, - input_spec=my_input_spec, ) assert ( shelly.cmdline - == f"DenoiseImage -i {tmp_path / 'a_file.ext'} -s 1 -p 1 -r 2 -o [{shelly.output_dir / 'a_file_out.ext'}]" + == f"DenoiseImage -i {tmp_path / 'a_file.ext'} -s 1 -p 1 -r 2 -o [{Path.cwd() / 'a_file_out.ext'}]" ) # input file name, noiseImage is set to True, so template is used in the output - shelly = ShellDef( + shelly = Shelly( executable="DenoiseImage", inputImageFilename=my_input_file, - input_spec=my_input_spec, noiseImage=True, ) assert ( shelly.cmdline == f"DenoiseImage -i {tmp_path / 'a_file.ext'} -s 1 -p 1 -r 2 " - f"-o [{shelly.output_dir / 'a_file_out.ext'}, {str(shelly.output_dir / 'a_file_noise.ext')}]" + f"-o [{Path.cwd() / 'a_file_out.ext'}, {str(Path.cwd() / 'a_file_noise.ext')}]" ) # input file name and help_short - shelly = ShellDef( + shelly = Shelly( executable="DenoiseImage", inputImageFilename=my_input_file, help_short=True, - input_spec=my_input_spec, ) assert ( shelly.cmdline - == f"DenoiseImage -i {tmp_path / 'a_file.ext'} -s 1 -p 1 -r 2 -h -o [{shelly.output_dir / 'a_file_out.ext'}]" + == f"DenoiseImage -i {tmp_path / 'a_file.ext'} -s 1 -p 1 -r 2 -h -o [{Path.cwd() / 'a_file_out.ext'}]" ) - assert shelly.output_names == [ + assert get_output_names(shelly) == [ "return_code", "stdout", "stderr", @@ -2126,23 +1419,21 @@ def test_shell_cmd_inputs_denoise_image( ] # adding image_dimensionality that has allowed_values [2, 3, 4] - shelly = ShellDef( + shelly = Shelly( executable="DenoiseImage", inputImageFilename=my_input_file, - input_spec=my_input_spec, image_dimensionality=2, ) assert ( shelly.cmdline - == f"DenoiseImage -d 2 -i {tmp_path / 'a_file.ext'} -s 1 -p 1 -r 2 -o [{shelly.output_dir / 'a_file_out.ext'}]" + == f"DenoiseImage -d 2 -i {tmp_path / 'a_file.ext'} -s 1 -p 1 -r 2 -o [{Path.cwd() / 'a_file_out.ext'}]" ) # adding image_dimensionality that has allowed_values [2, 3, 4] and providing 5 - exception should be raised with pytest.raises(ValueError) as excinfo: - shelly = ShellDef( + shelly = Shelly( executable="DenoiseImage", inputImageFilename=my_input_file, - input_spec=my_input_spec, image_dimensionality=5, ) assert "value of image_dimensionality" in str(excinfo.value) @@ -2152,7 +1443,7 @@ def test_shell_cmd_inputs_denoise_image( @shell.define -class SimpleTaskXor(ShellDef["SimpleTaskXor.Outputs"]): +class SimpleXor(ShellDef["SimpleTaskXor.Outputs"]): input_1: str = shell.arg( help="help", @@ -2177,28 +1468,28 @@ class Outputs(ShellOutputs): def test_task_inputs_mandatory_with_xOR_one_mandatory_is_OK(): """input definition with mandatory inputs""" - task = SimpleTaskXor() - task.definition.input_1 = "Input1" - task.definition.input_2 = attr.NOTHING - task.definition.check_fields_input_spec() + simple_xor = SimpleXor() + simple_xor.input_1 = "Input1" + simple_xor.input_2 = attrs.NOTHING + simple_xor._check_rules() def test_task_inputs_mandatory_with_xOR_one_mandatory_out_3_is_OK(): """input definition with mandatory inputs""" - task = SimpleTaskXor() - task.definition.input_1 = attr.NOTHING - task.definition.input_2 = attr.NOTHING - task.definition.input_3 = True - task.definition.check_fields_input_spec() + simple_xor = SimpleXor() + simple_xor.input_1 = attrs.NOTHING + simple_xor.input_2 = attrs.NOTHING + simple_xor.input_3 = True + simple_xor._check_rules() def test_task_inputs_mandatory_with_xOR_zero_mandatory_raises_error(): """input definition with mandatory inputs""" - task = SimpleTaskXor() - task.definition.input_1 = attr.NOTHING - task.definition.input_2 = attr.NOTHING + simple_xor = SimpleXor() + simple_xor.input_1 = attrs.NOTHING + simple_xor.input_2 = attrs.NOTHING with pytest.raises(Exception) as excinfo: - task.definition.check_fields_input_spec() + simple_xor._check_rules() assert "input_1 is mandatory" in str(excinfo.value) assert "no alternative provided by ['input_2', 'input_3']" in str(excinfo.value) assert excinfo.type is AttributeError @@ -2206,25 +1497,25 @@ def test_task_inputs_mandatory_with_xOR_zero_mandatory_raises_error(): def test_task_inputs_mandatory_with_xOR_two_mandatories_raises_error(): """input definition with mandatory inputs""" - task = SimpleTaskXor() - task.definition.input_1 = "Input1" - task.definition.input_2 = True + simple_xor = SimpleXor() + simple_xor.input_1 = "Input1" + simple_xor.input_2 = True with pytest.raises(Exception) as excinfo: - task.definition.check_fields_input_spec() + simple_xor._check_rules() assert "input_1 is mutually exclusive with ['input_2']" in str(excinfo.value) assert excinfo.type is AttributeError def test_task_inputs_mandatory_with_xOR_3_mandatories_raises_error(): """input definition with mandatory inputs""" - task = SimpleTaskXor() - task.definition.input_1 = "Input1" - task.definition.input_2 = True - task.definition.input_3 = False + simple_xor = SimpleXor() + simple_xor.input_1 = "Input1" + simple_xor.input_2 = True + simple_xor.input_3 = False with pytest.raises(Exception) as excinfo: - task.definition.check_fields_input_spec() + simple_xor._check_rules() assert "input_1 is mutually exclusive with ['input_2', 'input_3']" in str( excinfo.value ) diff --git a/pydra/engine/tests/test_task.py b/pydra/engine/tests/test_task.py index fcdf1d246..153d0a3e9 100644 --- a/pydra/engine/tests/test_task.py +++ b/pydra/engine/tests/test_task.py @@ -896,7 +896,7 @@ def TestFunc(a: int, b: float = 0.1): data = json.load(f) if "@type" in data: if "AssociatedWith" in data: - assert "TestFunc" in data["Label"] + assert "main" in data["Label"] if "@type" in data: if data["@type"] == "input": @@ -931,7 +931,7 @@ def test_audit_shellcommandtask(tmpdir): if "@type" in data: if "AssociatedWith" in data: - assert "shelly" in data["Label"] + assert "main" == data["Label"] if "@type" in data: if data["@type"] == "input": @@ -1332,7 +1332,8 @@ def FunError(x): raise Exception("Error from the function") with pytest.raises(Exception, match="Error from the function") as exinfo: - FunError(x=3)(worker="cf", cache_dir=tmpdir) + with Submitter(worker="cf", cache_dir=tmpdir) as sub: + sub(FunError(x=3), raise_errors=True) # getting error file from the error message error_file_match = ( @@ -1364,8 +1365,9 @@ def Workflow(x_list): return fun_error.out wf = Workflow(x_list=[3, 4]) - with pytest.raises(Exception, match="Task 'fun_error' raised an error") as exinfo: - wf(worker="cf") + with pytest.raises(Exception, match="Task 'fun_error' raised an error.*") as exinfo: + with Submitter(worker="cf", cache_dir=tmpdir) as sub: + sub(wf, raise_errors=True) # getting error file from the error message error_file_match = ( diff --git a/pydra/engine/tests/test_tasks_files.py b/pydra/engine/tests/test_tasks_files.py index daf846b31..96a4f940a 100644 --- a/pydra/engine/tests/test_tasks_files.py +++ b/pydra/engine/tests/test_tasks_files.py @@ -5,22 +5,22 @@ import typing as ty from ..submitter import Submitter -from pydra.design import python +from pydra.design import python, workflow from fileformats.generic import File, Directory @python.define -def dir_count_file(dirpath): +def DirCountFile(dirpath: Directory) -> int: return len(os.listdir(dirpath)) @python.define -def dir_count_file_annot(dirpath: Directory): +def DirCountFileAnnot(dirpath: Directory) -> int: return len(os.listdir(dirpath)) @python.define -def file_add2(file): +def FileAdd2(file: File) -> File: array_inp = np.load(file) array_out = array_inp + 2 cwd = os.getcwd() @@ -31,7 +31,7 @@ def file_add2(file): @python.define -def file_mult(file): +def FileMult(file: File) -> File: array_inp = np.load(file) array_out = 10 * array_inp cwd = os.getcwd() @@ -41,7 +41,7 @@ def file_mult(file): @python.define -def file_add2_annot(file: File) -> ty.NamedTuple("Output", [("out", File)]): +def FileAdd2Annot(file: File) -> File: array_inp = np.load(file) array_out = array_inp + 2 cwd = os.getcwd() @@ -52,7 +52,7 @@ def file_add2_annot(file: File) -> ty.NamedTuple("Output", [("out", File)]): @python.define -def file_mult_annot(file: File) -> ty.NamedTuple("Output", [("out", File)]): +def FileMultAnnot(file: File) -> File: array_inp = np.load(file) array_out = 10 * array_inp cwd = os.getcwd() @@ -68,36 +68,38 @@ def test_task_1(tmpdir): # creating abs path file = os.path.join(os.getcwd(), "arr1.npy") np.save(file, arr) - nn = file_add2(name="add2", file=file) + nn = FileAdd2(file=file) with Submitter(worker="cf") as sub: - sub(nn) + res = sub(nn) # checking the results - results = nn.result() - res = np.load(results.output.out) - assert res == np.array([4]) + + result = np.load(res.outputs.out) + assert result == np.array([4]) def test_wf_1(tmpdir): """workflow with 2 tasks that take file as an input and give file as an aoutput""" - wf = Workflow(name="wf_1", input_spec=["file_orig"]) - wf.add(file_add2(name="add2", file=wf.lzin.file_orig)) - wf.add(file_mult(name="mult", file=wf.add2.lzout.out)) - wf.set_output([("out", wf.mult.lzout.out)]) + + @workflow.define + def Workflow(file_orig: File): + add2 = workflow.add(FileAdd2(file=file_orig)) + mult = workflow.add(FileMult(file=add2.out)) + return mult.out os.chdir(tmpdir) arr = np.array([2, 3]) # creating abs path file_orig = os.path.join(os.getcwd(), "arr_orig.npy") np.save(file_orig, arr) - wf.inputs.file_orig = file_orig + wf = Workflow(file_orig=file_orig) with Submitter(worker="cf") as sub: - sub(wf) + res = sub(wf) - assert wf.output_dir.exists() - file_output = wf.result().output.out + assert res.output_dir.exists() + file_output = res.outputs.out assert Path(file_output).exists() # loading results array_out = np.load(file_output) @@ -111,15 +113,15 @@ def test_file_annotation_1(tmpdir): # creating abs path file = os.path.join(os.getcwd(), "arr1.npy") np.save(file, arr) - nn = file_add2_annot(name="add2", file=file) + nn = FileAdd2Annot(file=file) with Submitter(worker="cf") as sub: - sub(nn) + res = sub(nn) # checking the results - results = nn.result() - res = np.load(results.output.out) - assert res == np.array([4]) + assert res.errored is False, " ".join(res.errors["error message"]) + arr = np.load(res.outputs.out) + assert arr == np.array([4]) def test_broken_file(tmpdir): @@ -127,13 +129,12 @@ def test_broken_file(tmpdir): os.chdir(tmpdir) file = os.path.join(os.getcwd(), "non_existent.npy") - nn = file_add2(name="add2", file=file) with pytest.raises(FileNotFoundError): with Submitter(worker="cf") as sub: - sub(nn) + sub(FileAdd2(file=file)) with pytest.raises(FileNotFoundError, match="do not exist"): - file_add2_annot(name="add2_annot", file=file) + FileAdd2Annot(file=file) def test_broken_file_link(tmpdir): @@ -149,31 +150,27 @@ def test_broken_file_link(tmpdir): os.symlink(file, file_link) os.remove(file) - nn = file_add2(name="add2", file=file_link) # raises error inside task # unless variable is defined as a File pydra will treat it as a string with pytest.raises(FileNotFoundError): with Submitter(worker="cf") as sub: - sub(nn) + sub(FileAdd2(file=file_link)) with pytest.raises(FileNotFoundError, match="do not exist"): - file_add2_annot(name="add2_annot", file=file_link) + FileAdd2Annot(file=file_link) def test_broken_dir(): """Test how broken directories are handled during hashing""" - # dirpath doesn't exist - nn = dir_count_file(name="listdir", dirpath="/broken_dir_path/") - # raises error inside task # unless variable is defined as a File pydra will treat it as a string with pytest.raises(FileNotFoundError): with Submitter(worker="cf") as sub: - sub(nn) + sub(DirCountFile(dirpath="/broken_dir_path/")) # raises error before task is run with pytest.raises(FileNotFoundError): - dir_count_file_annot(name="listdir", dirpath="/broken_dir_path/") + DirCountFileAnnot(dirpath="/broken_dir_path/") def test_broken_dir_link1(tmpdir): @@ -187,34 +184,10 @@ def test_broken_dir_link1(tmpdir): os.symlink(dir1, dir1_link) os.rmdir(dir1) - nn = dir_count_file(name="listdir", dirpath=Path(dir1)) # raises error while running task with pytest.raises(FileNotFoundError): with Submitter(worker="cf") as sub: - sub(nn) + sub(DirCountFile(dirpath=Path(dir1))) with pytest.raises(FileNotFoundError): - dir_count_file_annot(name="listdir", dirpath=Path(dir1)) - - -def test_broken_dir_link2(tmpdir): - # valid dirs with broken symlink(s) are hashed - dir2 = tmpdir.join("dir2") - os.mkdir(dir2) - file1 = dir2.join("file1") - file2 = dir2.join("file2") - file1.open("w+").close() - file2.open("w+").close() - - file1_link = dir2.join("file1_link") - os.symlink(file1, file1_link) - os.remove(file1) # file1_link is broken - - nn = dir_count_file(name="listdir", dirpath=dir2) - # does not raises error because pydra treats dirpath as a string - with Submitter(worker="cf") as sub: - sub(nn) - - nn2 = dir_count_file_annot(name="listdir", dirpath=str(dir2)) - with Submitter(worker="cf") as sub: - sub(nn2) + DirCountFileAnnot(dirpath=Path(dir1)) diff --git a/pydra/engine/tests/test_workflow.py b/pydra/engine/tests/test_workflow.py index 1478c417a..f42b08db6 100644 --- a/pydra/engine/tests/test_workflow.py +++ b/pydra/engine/tests/test_workflow.py @@ -1,5 +1,7 @@ import pytest -import shutil, os, sys +import shutil +import os +import sys import time import typing as ty import attr @@ -8,8 +10,8 @@ Add2, Add2Wait, Multiply, - MultiplyList, - MultiplyMixed, + # MultiplyList, + # MultiplyMixed, Power, Ten, Identity, @@ -33,209 +35,126 @@ ListMultSum, DOT_FLAG, ) -from ..submitter import Submitter -from pydra.design import python -from ..specs import ShellDef +from pydra.engine.submitter import Submitter +from pydra.design import python, workflow from pydra.utils import exc_info_matches -def test_wf_specinfo_input_spec(): - input_spec = SpecInfo( - name="Input", - fields=[ - ("a", str, "", {"mandatory": True}), - ("b", dict, {"foo": 1, "bar": False}, {"mandatory": False}), - ], - bases=(BaseDef,), - ) - wf = Workflow( - name="workflow", - input_spec=input_spec, - ) - for x in ["a", "b", "_graph_checksums"]: - assert hasattr(wf.inputs, x) - assert wf.inputs.a == "" - assert wf.inputs.b == {"foo": 1, "bar": False} - bad_input_spec = SpecInfo( - name="Input", - fields=[ - ("a", str, {"mandatory": True}), - ], - bases=(ShellDef,), - ) - with pytest.raises( - ValueError, match="Provided SpecInfo must have BaseDef as its base." - ): - Workflow(name="workflow", input_spec=bad_input_spec) - - -def test_wf_dict_input_and_output_spec(): - definition = { - "a": str, - "b": ty.Dict[str, ty.Union[int, bool]], - } - wf = Workflow( - name="workflow", - input_spec=definition, - output_spec=definition, - ) - wf.add( - Identity2Flds( - name="identity", - x1=wf.lzin.a, - x2=wf.lzin.b, - ) - ) - wf.set_output( - [ - ("a", wf.identity.lzout.out1), - ("b", wf.identity.lzout.out2), - ] - ) - for x in ["a", "b", "_graph_checksums"]: - assert hasattr(wf.inputs, x) - wf.inputs.a = "any-string" - wf.inputs.b = {"foo": 1, "bar": False} - - with pytest.raises(TypeError) as exc_info: - wf.inputs.a = 1.0 - assert exc_info_matches(exc_info, "Cannot coerce 1.0 into ") - - with pytest.raises(TypeError) as exc_info: - wf.inputs.b = {"foo": 1, "bar": "bad-value"} - assert exc_info_matches( - exc_info, "Could not coerce object, 'bad-value', to any of the union types" - ) - - outputs = wf() - assert outputs.a == "any-string" - assert outputs.b == {"foo": 1, "bar": False} - - -def test_wf_name_conflict1(): - """raise error when workflow name conflicts with a class attribute or method""" - with pytest.raises(ValueError) as excinfo1: - Workflow(name="result", input_spec=["x"]) - assert "Cannot use names of attributes or methods" in str(excinfo1.value) - with pytest.raises(ValueError) as excinfo2: - Workflow(name="done", input_spec=["x"]) - assert "Cannot use names of attributes or methods" in str(excinfo2.value) - - -def test_wf_name_conflict2(): - """raise error when a task with the same name is already added to workflow""" - wf = Workflow(name="wf_1", input_spec=["x"]) - wf.add(Add2(name="task_name", x=wf.lzin.x)) - with pytest.raises(ValueError) as excinfo: - wf.add(Identity(name="task_name", x=3)) - assert "Another task named task_name is already added" in str(excinfo.value) - - def test_wf_no_output(plugin, tmpdir): """Raise error when output isn't set with set_output""" - wf = Workflow(name="wf_1", input_spec=["x"], cache_dir=tmpdir) - wf.add(Add2(name="add2", x=wf.lzin.x)) - wf.inputs.x = 2 + + @workflow.define + def Workflow(x): + workflow.add(Add2(x=x)) + + wf = Workflow(x=2) with pytest.raises(ValueError) as excinfo: - with Submitter(worker=plugin) as sub: + with Submitter(worker=plugin, cache_dir=tmpdir) as sub: sub(wf) assert "Workflow output cannot be None" in str(excinfo.value) def test_wf_1(plugin, tmpdir): """workflow with one task and no splitter""" - wf = Workflow(name="wf_1", input_spec=["x"]) - wf.add(Add2(name="add2", x=wf.lzin.x)) - wf.set_output([("out", wf.add2.lzout.out)]) - wf.inputs.x = 2 - wf.cache_dir = tmpdir - - checksum_before = wf.checksum - with Submitter(worker=plugin) as sub: - sub(wf) - assert wf.checksum == checksum_before - results = wf.result() - assert 4 == results.output.out - assert wf.output_dir.exists() + @workflow.define + def Workflow(x): + add2 = workflow.add(Add2(x=x)) + return add2.out + + wf = Workflow(x=2) + + checksum_before = wf._checksum + with Submitter(worker=plugin, cache_dir=tmpdir) as sub: + results = sub(wf) + + assert not results.errored, "\n".join(results.errors["error message"]) + + assert wf._checksum == checksum_before + + assert 4 == results.outputs.out def test_wf_1a_outpastuple(plugin, tmpdir): """workflow with one task and no splitter set_output takes a tuple """ - wf = Workflow(name="wf_1", input_spec=["x"]) - wf.add(Add2(name="add2", x=wf.lzin.x)) - wf.set_output(("out", wf.add2.lzout.out)) - wf.inputs.x = 2 - wf.plugin = plugin - wf.cache_dir = tmpdir - with Submitter(worker=plugin) as sub: - sub(wf) + @workflow.define + def Workflow(x): + add2 = workflow.add(Add2(x=x)) + return add2.out + + wf = Workflow(x=2) + + with Submitter(worker=plugin, cache_dir=tmpdir) as sub: + results = sub(wf) + + assert not results.errored, "\n".join(results.errors["error message"]) - results = wf.result() - assert 4 == results.output.out - assert wf.output_dir.exists() + assert 4 == results.outputs.out def test_wf_1_call_subm(plugin, tmpdir): """using wf.__call_ with submitter""" - wf = Workflow(name="wf_1", input_spec=["x"]) - wf.add(Add2(name="add2", x=wf.lzin.x)) - wf.set_output([("out", wf.add2.lzout.out)]) - wf.inputs.x = 2 - wf.cache_dir = tmpdir - with Submitter(worker=plugin) as sub: - wf(submitter=sub) + @workflow.define + def Workflow(x): + add2 = workflow.add(Add2(x=x)) + return add2.out - results = wf.result() - assert 4 == results.output.out - assert wf.output_dir.exists() + wf = Workflow(x=2) + + with Submitter(worker=plugin, cache_dir=tmpdir) as sub: + results = sub(wf) + + assert not results.errored, "\n".join(results.errors["error message"]) + + assert 4 == results.outputs.out def test_wf_1_call_plug(plugin, tmpdir): """using wf.__call_ with plugin""" - wf = Workflow(name="wf_1", input_spec=["x"]) - wf.add(Add2(name="add2", x=wf.lzin.x)) - wf.set_output([("out", wf.add2.lzout.out)]) - wf.inputs.x = 2 - wf.plugin = plugin - wf.cache_dir = tmpdir - wf(plugin=plugin) + @workflow.define + def Workflow(x): + add2 = workflow.add(Add2(x=x)) + return add2.out + + wf = Workflow(x=2) - results = wf.result() - assert 4 == results.output.out - assert wf.output_dir.exists() + outputs = wf(plugin=plugin) + + assert 4 == outputs.out def test_wf_1_call_noplug_nosubm(plugin, tmpdir): """using wf.__call_ without plugin or submitter""" - wf = Workflow(name="wf_1", input_spec=["x"]) - wf.add(Add2(name="add2", x=wf.lzin.x)) - wf.set_output([("out", wf.add2.lzout.out)]) - wf.inputs.x = 2 - wf.cache_dir = tmpdir - wf() - results = wf.result() - assert 4 == results.output.out - assert wf.output_dir.exists() + @workflow.define + def Workflow(x): + add2 = workflow.add(Add2(x=x)) + return add2.out + + wf = Workflow(x=2) + + outputs = wf() + + assert 4 == outputs.out def test_wf_1_call_exception(plugin, tmpdir): """using wf.__call_ with plugin and submitter - should raise an exception""" - wf = Workflow(name="wf_1", input_spec=["x"]) - wf.add(Add2(name="add2", x=wf.lzin.x)) - wf.set_output([("out", wf.add2.lzout.out)]) - wf.inputs.x = 2 - wf.plugin = plugin - wf.cache_dir = tmpdir - - with Submitter(worker=plugin) as sub: + + @workflow.define + def Workflow(x): + add2 = workflow.add(Add2(x=x)) + return add2.out + + wf = Workflow(x=2) + + with Submitter(worker=plugin, cache_dir=tmpdir) as sub: with pytest.raises(Exception) as e: wf(submitter=sub, plugin=plugin) assert "Defify submitter OR plugin" in str(e.value) @@ -243,62 +162,68 @@ def test_wf_1_call_exception(plugin, tmpdir): def test_wf_1_inp_in_call(tmpdir): """Defining input in __call__""" - wf = Workflow(name="wf_1", input_spec=["x"], cache_dir=tmpdir) - wf.add(Add2(name="add2", x=wf.lzin.x)) - wf.set_output([("out", wf.add2.lzout.out)]) - wf.inputs.x = 1 + + @workflow.define + def Workflow(x): + add2 = workflow.add(Add2(x=x)) + return add2.out + + wf = Workflow(x=1) results = wf(x=2) - assert 4 == results.output.out + assert 4 == results.outputs.out def test_wf_1_upd_in_run(tmpdir): """Updating input in __call__""" - wf = Workflow(name="wf_1", input_spec=["x"], cache_dir=tmpdir) - wf.add(Add2(name="add2", x=wf.lzin.x)) - wf.set_output([("out", wf.add2.lzout.out)]) - wf.inputs.x = 1 + + @workflow.define + def Workflow(x): + add2 = workflow.add(Add2(x=x)) + return add2.out + + wf = Workflow(x=1) results = wf(x=2) - assert 4 == results.output.out + assert 4 == results.outputs.out def test_wf_2(plugin, tmpdir): """workflow with 2 tasks, no splitter""" - wf = Workflow(name="wf_2", input_spec=["x", "y"]) - wf.add(Multiply(name="mult", x=wf.lzin.x, y=wf.lzin.y)) - wf.add(Add2(name="add2", x=wf.mult.lzout.out)) - wf.set_output([("out", wf.add2.lzout.out)]) - wf.inputs.x = 2 - wf.inputs.y = 3 - wf.cache_dir = tmpdir - - with Submitter(worker=plugin) as sub: - sub(wf) - assert wf.output_dir.exists() - results = wf.result() - assert 8 == results.output.out + @workflow.define + def Workflow(x, y): + mult = workflow.add(Multiply(x=x, y=y)) + add2 = workflow.add(Add2(x=mult.out)) + return add2.out + + wf = Workflow(x=2, y=3) + + with Submitter(worker=plugin, cache_dir=tmpdir) as sub: + results = sub(wf) + + assert not results.errored, "\n".join(results.errors["error message"]) + + assert 8 == results.outputs.out def test_wf_2a(plugin, tmpdir): """workflow with 2 tasks, no splitter creating add2_task first (before calling add method), """ - wf = Workflow(name="wf_2", input_spec=["x", "y"]) - wf.add(Multiply(name="mult", x=wf.lzin.x, y=wf.lzin.y)) - add2_task = Add2(name="add2") - add2_task.inputs.x = wf.mult.lzout.out - wf.add(add2_task) - wf.set_output([("out", wf.add2.lzout.out)]) - wf.inputs.x = 2 - wf.inputs.y = 3 - wf.cache_dir = tmpdir - - with Submitter(worker=plugin) as sub: - sub(wf) - results = wf.result() - assert 8 == results.output.out - assert wf.output_dir.exists() + @workflow.define + def Workflow(x, y): + mult = workflow.add(Multiply(x=x, y=y)) + add2 = workflow.add(Add2(x=mult.out)) + return add2.out + + wf = Workflow(x=2, y=3) + + with Submitter(worker=plugin, cache_dir=tmpdir) as sub: + results = sub(wf) + + assert not results.errored, "\n".join(results.errors["error message"]) + + assert 8 == results.outputs.out def test_wf_2b(plugin, tmpdir): @@ -306,92 +231,87 @@ def test_wf_2b(plugin, tmpdir): creating add2_task first (before calling add method), adding inputs.x after add method """ - wf = Workflow(name="wf_2", input_spec=["x", "y"]) - wf.add(Multiply(name="mult", x=wf.lzin.x, y=wf.lzin.y)) - add2_task = Add2(name="add2") - wf.add(add2_task) - add2_task.inputs.x = wf.mult.lzout.out - wf.set_output([("out", wf.add2.lzout.out)]) - wf.inputs.x = 2 - wf.inputs.y = 3 - wf.cache_dir = tmpdir - - with Submitter(worker=plugin) as sub: - sub(wf) - results = wf.result() - assert 8 == results.output.out + @workflow.define + def Workflow(x, y): + mult = workflow.add(Multiply(x=x, y=y)) + add2 = workflow.add(Add2(x=mult.out)) + return add2.out + + wf = Workflow(x=2, y=3) - assert wf.output_dir.exists() + with Submitter(worker=plugin, cache_dir=tmpdir) as sub: + results = sub(wf) + + assert not results.errored, "\n".join(results.errors["error message"]) + + assert 8 == results.outputs.out def test_wf_2c_multoutp(plugin, tmpdir): """workflow with 2 tasks, no splitter setting multiple outputs for the workflow """ - wf = Workflow(name="wf_2", input_spec=["x", "y"]) - wf.add(Multiply(name="mult", x=wf.lzin.x, y=wf.lzin.y)) - add2_task = Add2(name="add2") - add2_task.inputs.x = wf.mult.lzout.out - wf.add(add2_task) - # setting multiple output (from both nodes) - wf.set_output([("out_add2", wf.add2.lzout.out), ("out_mult", wf.mult.lzout.out)]) - wf.inputs.x = 2 - wf.inputs.y = 3 - wf.cache_dir = tmpdir - - with Submitter(worker=plugin) as sub: - sub(wf) - results = wf.result() + @workflow.define(outputs=["out_add2", "out_mult"]) + def Workflow(x, y): + mult = workflow.add(Multiply(x=x, y=y)) + add2 = workflow.add(Add2(x=mult.out)) + return add2.out, mult.out + + wf = Workflow(x=2, y=3) + + with Submitter(worker=plugin, cache_dir=tmpdir) as sub: + results = sub(wf) + + assert not results.errored, "\n".join(results.errors["error message"]) + # checking outputs from both nodes - assert 6 == results.output.out_mult - assert 8 == results.output.out_add2 - assert wf.output_dir.exists() + assert 6 == results.outputs.out_mult + assert 8 == results.outputs.out_add2 def test_wf_2d_outpasdict(plugin, tmpdir): """workflow with 2 tasks, no splitter setting multiple outputs using a dictionary """ - wf = Workflow(name="wf_2", input_spec=["x", "y"]) - wf.add(Multiply(name="mult", x=wf.lzin.x, y=wf.lzin.y)) - add2_task = Add2(name="add2") - add2_task.inputs.x = wf.mult.lzout.out - wf.add(add2_task) - # setting multiple output (from both nodes) - wf.set_output({"out_add2": wf.add2.lzout.out, "out_mult": wf.mult.lzout.out}) - wf.inputs.x = 2 - wf.inputs.y = 3 - wf.cache_dir = tmpdir - - with Submitter(worker=plugin) as sub: - sub(wf) - results = wf.result() + @workflow.define(outputs=["out_add2", "out_mult"]) + def Workflow(x, y): + mult = workflow.add(Multiply(x=x, y=y)) + add2 = workflow.add(Add2(x=mult.out)) + return add2.out, mult.out + + wf = Workflow(x=2, y=3) + + with Submitter(worker=plugin, cache_dir=tmpdir) as sub: + results = sub(wf) + + assert not results.errored, "\n".join(results.errors["error message"]) + # checking outputs from both nodes - assert 6 == results.output.out_mult - assert 8 == results.output.out_add2 - assert wf.output_dir.exists() + assert 6 == results.outputs.out_mult + assert 8 == results.outputs.out_add2 @pytest.mark.flaky(reruns=3) # when dask def test_wf_3(plugin_dask_opt, tmpdir): """testing None value for an input""" - wf = Workflow(name="wf_3", input_spec=["x", "y"]) - wf.add(FunAddVarNone(name="addvar", a=wf.lzin.x, b=wf.lzin.y)) - wf.add(Add2(name="add2", x=wf.addvar.lzout.out)) - wf.set_output([("out", wf.add2.lzout.out)]) - wf.inputs.x = 2 - wf.inputs.y = None - wf.cache_dir = tmpdir + + @workflow.define + def Workflow(x, y): + addvar = workflow.add(FunAddVarNone(a=x, b=y)) + add2 = workflow.add(Add2(x=addvar.out)) + return add2.out + + wf = Workflow(x=2, y=None) with Submitter(worker=plugin_dask_opt) as sub: - sub(wf) + results = sub(wf) - assert wf.output_dir.exists() - results = wf.result() - assert 4 == results.output.out + assert not results.errored, "\n".join(results.errors["error message"]) + + assert 4 == results.outputs.out @pytest.mark.xfail(reason="the task error doesn't propagate") @@ -399,36 +319,38 @@ def test_wf_3a_exception(plugin, tmpdir): """testinh wf without set input, attr.NOTHING should be set and the function should raise an exception """ - wf = Workflow(name="wf_3", input_spec=["x", "y"]) - wf.add(FunAddVarNone(name="addvar", a=wf.lzin.x, b=wf.lzin.y)) - wf.add(Add2(name="add2", x=wf.addvar.lzout.out)) - wf.set_output([("out", wf.add2.lzout.out)]) - wf.inputs.x = 2 - wf.inputs.y = attr.NOTHING - wf.plugin = plugin - wf.cache_dir = tmpdir + + @workflow.define + def Workflow(x, y): + addvar = workflow.add(FunAddVarNone(a=x, b=y)) + add2 = workflow.add(Add2(x=addvar.out)) + return add2.out + + wf = Workflow(x=2, y=attr.NOTHING) with pytest.raises(TypeError) as excinfo: - with Submitter(worker=plugin) as sub: + with Submitter(worker=plugin, cache_dir=tmpdir) as sub: sub(wf) assert "unsupported" in str(excinfo.value) def test_wf_4(plugin, tmpdir): """wf with a task that doesn't set one input and use the function default value""" - wf = Workflow(name="wf_4", input_spec=["x", "y"]) - wf.add(FunAddVarDefault(name="addvar", a=wf.lzin.x)) - wf.add(Add2(name="add2", x=wf.addvar.lzout.out)) - wf.set_output([("out", wf.add2.lzout.out)]) - wf.inputs.x = 2 - wf.cache_dir = tmpdir - - with Submitter(worker=plugin) as sub: - sub(wf) - assert wf.output_dir.exists() - results = wf.result() - assert 5 == results.output.out + @workflow.define + def Workflow(x, y): + addvar = workflow.add(FunAddVarDefault(a=x)) + add2 = workflow.add(Add2(x=addvar.out)) + return add2.out + + wf = Workflow(x=2) + + with Submitter(worker=plugin, cache_dir=tmpdir) as sub: + results = sub(wf) + + assert not results.errored, "\n".join(results.errors["error message"]) + + assert 5 == results.outputs.out def test_wf_4a(plugin, tmpdir): @@ -436,196 +358,191 @@ def test_wf_4a(plugin, tmpdir): the unset input is send to the task input, so the task should use the function default value """ - wf = Workflow(name="wf_4a", input_spec=["x", "y"]) - wf.add(FunAddVarDefault(name="addvar", a=wf.lzin.x, y=wf.lzin.y)) - wf.add(Add2(name="add2", x=wf.addvar.lzout.out)) - wf.set_output([("out", wf.add2.lzout.out)]) - wf.inputs.x = 2 - wf.cache_dir = tmpdir - with Submitter(worker=plugin) as sub: - sub(wf) + @workflow.define + def Workflow(x, y): + addvar = workflow.add(FunAddVarDefault(a=x, y=y)) + add2 = workflow.add(Add2(x=addvar.out)) + return add2.out + + wf = Workflow(x=2) - assert wf.output_dir.exists() - results = wf.result() - assert 5 == results.output.out + with Submitter(worker=plugin, cache_dir=tmpdir) as sub: + results = sub(wf) + + assert not results.errored, "\n".join(results.errors["error message"]) + + assert 5 == results.outputs.out def test_wf_5(plugin, tmpdir): """wf with two outputs connected to the task outputs one set_output """ - wf = Workflow(name="wf_5", input_spec=["x", "y"], x=3, y=2) - wf.add(FunAddSubVar(name="addsub", a=wf.lzin.x, b=wf.lzin.y)) - wf.set_output([("out_sum", wf.addsub.lzout.sum), ("out_sub", wf.addsub.lzout.sub)]) - wf.cache_dir = tmpdir - with Submitter(worker=plugin) as sub: - sub(wf) + @workflow.define(outputs=["out_sum", "out_sub"]) + def Workflow(x, y): + addsub = workflow.add(FunAddSubVar(a=x, b=y)) + return addsub.sum, addsub.sub + + wf = Workflow(x=3, y=2) - results = wf.result() - assert 5 == results.output.out_sum - assert 1 == results.output.out_sub + with Submitter(worker=plugin, cache_dir=tmpdir) as sub: + results = sub(wf) + + assert not results.errored, "\n".join(results.errors["error message"]) + + assert 5 == results.outputs.out_sum + assert 1 == results.outputs.out_sub def test_wf_5a(plugin, tmpdir): """wf with two outputs connected to the task outputs, set_output set twice """ - wf = Workflow(name="wf_5", input_spec=["x", "y"], x=3, y=2) - wf.add(FunAddSubVar(name="addsub", a=wf.lzin.x, b=wf.lzin.y)) - wf.set_output([("out_sum", wf.addsub.lzout.sum)]) - wf.set_output([("out_sub", wf.addsub.lzout.sub)]) - wf.cache_dir = tmpdir - - with Submitter(worker=plugin) as sub: - sub(wf) - results = wf.result() - assert 5 == results.output.out_sum - assert 1 == results.output.out_sub + @workflow.define + def Workflow(x, y): + addsub = workflow.add(FunAddSubVar(a=x, b=y)) + return addsub.sum # out_sum + return addsub.sub # out_sub + wf = Workflow(x=3, y=2) + with Submitter(worker=plugin, cache_dir=tmpdir) as sub: + results = sub(wf) -def test_wf_5b_exception(tmpdir): - """set_output used twice with the same name - exception should be raised""" - wf = Workflow(name="wf_5", input_spec=["x", "y"], x=3, y=2) - wf.add(FunAddSubVar(name="addsub", a=wf.lzin.x, b=wf.lzin.y)) - wf.set_output([("out", wf.addsub.lzout.sum)]) - wf.cache_dir = tmpdir + assert not results.errored, "\n".join(results.errors["error message"]) - with pytest.raises(Exception, match="are already set"): - wf.set_output([("out", wf.addsub.lzout.sub)]) + assert 5 == results.outputs.out_sum + assert 1 == results.outputs.out_sub def test_wf_6(plugin, tmpdir): """wf with two tasks and two outputs connected to both tasks, one set_output """ - wf = Workflow(name="wf_6", input_spec=["x", "y"], x=2, y=3) - wf.add(Multiply(name="mult", x=wf.lzin.x, y=wf.lzin.y)) - wf.add(Add2(name="add2", x=wf.mult.lzout.out)) - wf.set_output([("out1", wf.mult.lzout.out), ("out2", wf.add2.lzout.out)]) - wf.cache_dir = tmpdir - with Submitter(worker=plugin) as sub: - sub(wf) + @workflow.define(outputs=["out1", "out2"]) + def Workflow(x, y): + mult = workflow.add(Multiply(x=x, y=y)) + add2 = workflow.add(Add2(x=mult.out)) + return mult.out, add2.out # + + wf = Workflow(x=2, y=3) + + with Submitter(worker=plugin, cache_dir=tmpdir) as sub: + results = sub(wf) + + assert not results.errored, "\n".join(results.errors["error message"]) - assert wf.output_dir.exists() - results = wf.result() - assert 6 == results.output.out1 - assert 8 == results.output.out2 + assert 6 == results.outputs.out1 + assert 8 == results.outputs.out2 def test_wf_6a(plugin, tmpdir): """wf with two tasks and two outputs connected to both tasks, set_output used twice """ - wf = Workflow(name="wf_6", input_spec=["x", "y"], x=2, y=3) - wf.add(Multiply(name="mult", x=wf.lzin.x, y=wf.lzin.y)) - wf.add(Add2(name="add2", x=wf.mult.lzout.out)) - wf.set_output([("out1", wf.mult.lzout.out)]) - wf.set_output([("out2", wf.add2.lzout.out)]) - wf.cache_dir = tmpdir - with Submitter(worker=plugin) as sub: - sub(wf) + @workflow.define(outputs=["out1", "out2"]) + def Workflow(x, y): + mult = workflow.add(Multiply(x=x, y=y)) + add2 = workflow.add(Add2(x=mult.out)) + return mult.out, add2.out + + wf = Workflow(x=2, y=3) + + with Submitter(worker=plugin, cache_dir=tmpdir) as sub: + results = sub(wf) + + assert not results.errored, "\n".join(results.errors["error message"]) - assert wf.output_dir.exists() - results = wf.result() - assert 6 == results.output.out1 - assert 8 == results.output.out2 + assert 6 == results.outputs.out1 + assert 8 == results.outputs.out2 def test_wf_st_1(plugin, tmpdir): """Workflow with one task, a splitter for the workflow""" - wf = Workflow(name="wf_spl_1", input_spec=["x"]) - wf.add(Add2(name="add2", x=wf.lzin.x)) - wf.split("x", x=[1, 2]) - wf.set_output([("out", wf.add2.lzout.out)]) - wf.cache_dir = tmpdir + @workflow.define + def Workflow(x): + add2 = workflow.add(Add2(x=x).split("x", x=x)) - checksum_before = wf.checksum - with Submitter(worker="serial") as sub: - sub(wf) + return add2.out + + wf = Workflow(x=[1, 2]) + + checksum_before = wf._checksum + with Submitter(cache_dir=tmpdir) as sub: + results = sub(wf) + + assert not results.errored, "\n".join(results.errors["error message"]) + + assert wf._checksum == checksum_before - assert wf.checksum == checksum_before - results = wf.result() # expected: [({"test7.x": 1}, 3), ({"test7.x": 2}, 4)] - assert results[0].output.out == 3 - assert results[1].output.out == 4 - # checking all directories - assert wf.output_dir - for odir in wf.output_dir: - assert odir.exists() + assert results.outputs.out[0] == 3 + assert results.outputs.out[1] == 4 def test_wf_st_1_call_subm(plugin, tmpdir): """Workflow with one task, a splitter for the workflow""" - wf = Workflow(name="wf_spl_1", input_spec=["x"]) - wf.add(Add2(name="add2", x=wf.lzin.x)) - wf.split("x", x=[1, 2]) - wf.set_output([("out", wf.add2.lzout.out)]) - wf.cache_dir = tmpdir + @workflow.define + def Workflow(x): + add2 = workflow.add(Add2(x=x).split("x", x=x)) + + return add2.out - with Submitter(worker=plugin) as sub: - wf(submitter=sub) + wf = Workflow(x=[1, 2]) + + with Submitter(worker=plugin, cache_dir=tmpdir) as sub: + results = sub(wf) + + assert not results.errored, "\n".join(results.errors["error message"]) - results = wf.result() # expected: [({"test7.x": 1}, 3), ({"test7.x": 2}, 4)] - assert results[0].output.out == 3 - assert results[1].output.out == 4 - # checking all directories - assert wf.output_dir - for odir in wf.output_dir: - assert odir.exists() + assert results.outputs.out[0] == 3 + assert results.outputs.out[1] == 4 def test_wf_st_1_call_plug(plugin, tmpdir): """Workflow with one task, a splitter for the workflow using Workflow.__call__(plugin) """ - wf = Workflow(name="wf_spl_1", input_spec=["x"]) - wf.add(Add2(name="add2", x=wf.lzin.x)) - wf.split("x", x=[1, 2]) - wf.set_output([("out", wf.add2.lzout.out)]) - wf.cache_dir = tmpdir + @workflow.define + def Workflow(x): + add2 = workflow.add(Add2(x=x).split("x", x=x)) - wf(plugin=plugin) + return add2.out + + wf = Workflow(x=[1, 2]) + + outputs = wf(plugin=plugin) - results = wf.result() # expected: [({"test7.x": 1}, 3), ({"test7.x": 2}, 4)] - assert results[0].output.out == 3 - assert results[1].output.out == 4 - # checking all directories - assert wf.output_dir - for odir in wf.output_dir: - assert odir.exists() + assert outputs.out[0] == 3 + assert outputs.out[1] == 4 def test_wf_st_1_call_selfplug(plugin, tmpdir): """Workflow with one task, a splitter for the workflow using Workflow.__call__() and using self.plugin """ - wf = Workflow(name="wf_spl_1", input_spec=["x"]) - wf.add(Add2(name="add2", x=wf.lzin.x)) - wf.split("x", x=[1, 2]) - wf.set_output([("out", wf.add2.lzout.out)]) - wf.plugin = plugin - wf.cache_dir = tmpdir + @workflow.define + def Workflow(x): + add2 = workflow.add(Add2(x=x).split("x", x=x)) + return add2.out + + wf = Workflow(x=[1, 2]) + + outputs = wf() - wf() - results = wf.result() # expected: [({"test7.x": 1}, 3), ({"test7.x": 2}, 4)] - assert results[0].output.out == 3 - assert results[1].output.out == 4 - # checking all directories - assert wf.output_dir - for odir in wf.output_dir: - assert odir.exists() + assert outputs.out[0] == 3 + assert outputs.out[1] == 4 def test_wf_st_1_call_noplug_nosubm(plugin, tmpdir): @@ -633,130 +550,132 @@ def test_wf_st_1_call_noplug_nosubm(plugin, tmpdir): using Workflow.__call__() without plugin and submitter (a submitter should be created within the __call__ function) """ - wf = Workflow(name="wf_spl_1", input_spec=["x"]) - wf.add(Add2(name="add2", x=wf.lzin.x)) - wf.split("x", x=[1, 2]) - wf.set_output([("out", wf.add2.lzout.out)]) - wf.cache_dir = tmpdir + @workflow.define + def Workflow(x): + add2 = workflow.add(Add2(x=x).split("x", x=x)) + return add2.out + + wf = Workflow(x=[1, 2]) + + outputs = wf() - wf() - results = wf.result() # expected: [({"test7.x": 1}, 3), ({"test7.x": 2}, 4)] - assert results[0].output.out == 3 - assert results[1].output.out == 4 - # checking all directories - assert wf.output_dir - for odir in wf.output_dir: - assert odir.exists() + assert outputs.out[0] == 3 + assert outputs.out[1] == 4 def test_wf_st_1_inp_in_call(tmpdir): """Defining input in __call__""" - wf = Workflow(name="wf_spl_1", input_spec=["x"], cache_dir=tmpdir).split( - "x", x=[1, 2] - ) - wf.add(Add2(name="add2", x=wf.lzin.x)) - wf.set_output([("out", wf.add2.lzout.out)]) + + @workflow.define + def Workflow(x): + add2 = workflow.add(Add2(x=x)) + return add2.out + + wf = Workflow().split("x", x=[1, 2]) results = wf() - assert results[0].output.out == 3 - assert results[1].output.out == 4 + assert results.outputs.out[0] == 3 + assert results.outputs.out[1] == 4 def test_wf_st_1_upd_inp_call(tmpdir): """Updating input in __call___""" - wf = Workflow(name="wf_spl_1", input_spec=["x"], cache_dir=tmpdir).split( - "x", x=[11, 22] - ) - wf.add(Add2(name="add2", x=wf.lzin.x)) - wf.set_output([("out", wf.add2.lzout.out)]) + + @workflow.define + def Workflow(x): + add2 = workflow.add(Add2(x=x)) + return add2.out + + wf = Workflow().split("x", x=[11, 22]) results = wf(x=[1, 2]) - assert results[0].output.out == 3 - assert results[1].output.out == 4 + assert results.outputs.out[0] == 3 + assert results.outputs.out[1] == 4 def test_wf_st_noinput_1(plugin, tmpdir): """Workflow with one task, a splitter for the workflow""" - wf = Workflow(name="wf_spl_1", input_spec=["x"]) - wf.add(Add2(name="add2", x=wf.lzin.x)) - wf.split("x", x=[]) - wf.set_output([("out", wf.add2.lzout.out)]) - wf.plugin = plugin - wf.cache_dir = tmpdir + @workflow.define + def Workflow(x): + add2 = workflow.add(Add2(x=x).split("x", x=x)) + return add2.out - checksum_before = wf.checksum - with Submitter(worker=plugin) as sub: - sub(wf) + wf = Workflow(x=[]) + + checksum_before = wf._checksum + with Submitter(worker=plugin, cache_dir=tmpdir) as sub: + results = sub(wf) + + assert not results.errored, "\n".join(results.errors["error message"]) + + assert wf._checksum == checksum_before - assert wf.checksum == checksum_before - results = wf.result() assert results == [] - # checking all directories - assert wf.output_dir == [] def test_wf_ndst_1(plugin, tmpdir): """workflow with one task, a splitter on the task level""" - wf = Workflow(name="wf_spl_1", input_spec=["x"]) - wf.add(Add2(name="add2").split("x", x=wf.lzin.x)) - wf.inputs.x = [1, 2] - wf.set_output([("out", wf.add2.lzout.out)]) - wf.cache_dir = tmpdir - - checksum_before = wf.checksum - with Submitter(worker=plugin) as sub: - sub(wf) - assert wf.checksum == checksum_before - results = wf.result() + @workflow.define + def Workflow(x): + add2 = workflow.add(Add2().split("x", x=x)) + return add2.out + + wf = Workflow(x=[1, 2]) + + checksum_before = wf._checksum + with Submitter(worker=plugin, cache_dir=tmpdir) as sub: + results = sub(wf) + + assert not results.errored, "\n".join(results.errors["error message"]) + + assert wf._checksum == checksum_before + # expected: [({"test7.x": 1}, 3), ({"test7.x": 2}, 4)] - assert results.output.out == [3, 4] - assert wf.output_dir.exists() + assert results.outputs.out == [3, 4] def test_wf_ndst_updatespl_1(plugin, tmpdir): """workflow with one task, a splitter on the task level is added *after* calling add """ - wf = Workflow(name="wf_spl_1", input_spec=["x"]) - wf.add(Add2(name="add2")) - wf.inputs.x = [1, 2] - wf.add2.split("x", x=wf.lzin.x) - wf.set_output([("out", wf.add2.lzout.out)]) - wf.cache_dir = tmpdir - with Submitter(worker=plugin) as sub: - sub(wf) + @workflow.define + def Workflow(x): + add2 = workflow.add(Add2(name="add2").split("x", x=x)) + return add2.out - results = wf.result() - # expected: [({"test7.x": 1}, 3), ({"test7.x": 2}, 4)] - assert results.output.out == [3, 4] - assert wf.output_dir.exists() + wf = Workflow(x=[1, 2]) - assert wf.output_dir.exists() + with Submitter(worker=plugin, cache_dir=tmpdir) as sub: + results = sub(wf) + + assert not results.errored, "\n".join(results.errors["error message"]) + + # expected: [({"test7.x": 1}, 3), ({"test7.x": 2}, 4)] + assert results.outputs.out == [3, 4] def test_wf_ndst_updatespl_1a(plugin, tmpdir): """workflow with one task (initialize before calling add), a splitter on the task level is added *after* calling add """ - wf = Workflow(name="wf_spl_1", input_spec=["x"]) - task_add2 = Add2(name="add2", x=wf.lzin.x) - wf.add(task_add2) - task_add2.split("x", x=[1, 2]) - wf.set_output([("out", wf.add2.lzout.out)]) - wf.cache_dir = tmpdir - with Submitter(worker=plugin) as sub: - sub(wf) + @workflow.define + def Workflow(x): + add2 = workflow.add(Add2().split("x", x=x)) + return add2.out - results = wf.result() - # expected: [({"test7.x": 1}, 3), ({"test7.x": 2}, 4)] - assert results.output.out == [3, 4] - assert wf.output_dir.exists() + wf = Workflow(x=[1, 2]) + + with Submitter(worker=plugin, cache_dir=tmpdir) as sub: + results = sub(wf) - assert wf.output_dir.exists() + assert not results.errored, "\n".join(results.errors["error message"]) + + # expected: [({"test7.x": 1}, 3), ({"test7.x": 2}, 4)] + assert results.outputs.out == [3, 4] def test_wf_ndst_updateinp_1(plugin, tmpdir): @@ -764,80 +683,81 @@ def test_wf_ndst_updateinp_1(plugin, tmpdir): a splitter on the task level, updating input of the task after calling add """ - wf = Workflow(name="wf_spl_1", input_spec=["x", "y"]) - wf.add(Add2(name="add2", x=wf.lzin.x)) - wf.inputs.x = [1, 2] - wf.inputs.y = [11, 12] - wf.add2.split("x", x=wf.lzin.y) - wf.set_output([("out", wf.add2.lzout.out)]) - wf.cache_dir = tmpdir - with Submitter(worker=plugin) as sub: - sub(wf) + @workflow.define + def Workflow(x, y): + add2 = workflow.add(Add2(x=x).split("x", x=y)) + return add2.out + + wf = Workflow(x=[1, 2], y=[11, 12]) - results = wf.result() - assert results.output.out == [13, 14] - assert wf.output_dir.exists() + with Submitter(worker=plugin, cache_dir=tmpdir) as sub: + results = sub(wf) - assert wf.output_dir.exists() + assert not results.errored, "\n".join(results.errors["error message"]) + + assert results.outputs.out == [13, 14] def test_wf_ndst_noinput_1(plugin, tmpdir): """workflow with one task, a splitter on the task level""" - wf = Workflow(name="wf_spl_1", input_spec=["x"]) - wf.add(Add2(name="add2").split("x", x=wf.lzin.x)) - wf.inputs.x = [] - wf.set_output([("out", wf.add2.lzout.out)]) - wf.cache_dir = tmpdir - - checksum_before = wf.checksum - with Submitter(worker=plugin) as sub: - sub(wf) - assert wf.checksum == checksum_before - results = wf.result() + @workflow.define + def Workflow(x): + add2 = workflow.add(Add2().split("x", x=x)) + return add2.out + + wf = Workflow(x=[]) + + checksum_before = wf._checksum + with Submitter(worker=plugin, cache_dir=tmpdir) as sub: + results = sub(wf) + + assert not results.errored, "\n".join(results.errors["error message"]) + + assert wf._checksum == checksum_before - assert results.output.out == [] - assert wf.output_dir.exists() + assert results.outputs.out == [] def test_wf_st_2(plugin, tmpdir): """workflow with one task, splitters and combiner for workflow""" - wf = Workflow(name="wf_st_2", input_spec=["x"]) - wf.add(Add2(name="add2", x=wf.lzin.x)) - wf.split("x", x=[1, 2]).combine(combiner="x") - wf.set_output([("out", wf.add2.lzout.out)]) - wf.cache_dir = tmpdir + @workflow.define + def Workflow(x): + add2 = workflow.add(Add2(x=x)) - with Submitter(worker=plugin) as sub: - sub(wf) + return add2.out + + wf = Workflow().split("x", x=[1, 2]).combine(combiner="x") + + with Submitter(worker=plugin, cache_dir=tmpdir) as sub: + results = sub(wf) + + assert not results.errored, "\n".join(results.errors["error message"]) - results = wf.result() # expected: [({"test7.x": 1}, 3), ({"test7.x": 2}, 4)] - assert results[0].output.out == 3 - assert results[1].output.out == 4 - # checking all directories - assert wf.output_dir - for odir in wf.output_dir: - assert odir.exists() + assert results.outputs.out[0] == 3 + assert results.outputs.out[1] == 4 def test_wf_ndst_2(plugin, tmpdir): """workflow with one task, splitters and combiner on the task level""" - wf = Workflow(name="wf_ndst_2", input_spec=["x"]) - wf.add(Add2(name="add2").split("x", x=wf.lzin.x).combine(combiner="x")) - wf.inputs.x = [1, 2] - wf.set_output([("out", wf.add2.lzout.out)]) - wf.cache_dir = tmpdir - with Submitter(worker=plugin) as sub: - sub(wf) + @workflow.define + def Workflow(x): + add2 = workflow.add(Add2().split("x", x=x).combine(combiner="x")) + return add2.out + + wf = Workflow(x=[1, 2]) + + with Submitter(worker=plugin, cache_dir=tmpdir) as sub: + results = sub(wf) + + assert not results.errored, "\n".join(results.errors["error message"]) - results = wf.result() # expected: [({"test7.x": 1}, 3), ({"test7.x": 2}, 4)] - assert results.output.out == [3, 4] - assert wf.output_dir.exists() + assert results.outputs.out == [3, 4] # workflows with structures A -> B @@ -845,15 +765,20 @@ def test_wf_ndst_2(plugin, tmpdir): def test_wf_st_3(plugin, tmpdir): """workflow with 2 tasks, splitter on wf level""" - wf = Workflow(name="wfst_3", input_spec=["x", "y"]) - wf.add(Multiply(name="mult", x=wf.lzin.x, y=wf.lzin.y)) - wf.add(Add2(name="add2", x=wf.mult.lzout.out)) - wf.split(("x", "y"), x=[1, 2], y=[11, 12]) - wf.set_output([("out", wf.add2.lzout.out)]) - wf.cache_dir = tmpdir - - with Submitter(worker=plugin) as sub: - sub(wf) + + @workflow.define + def Workflow(x, y): + mult = workflow.add(Multiply(x=x, y=y)) + add2 = workflow.add(Add2(x=mult.out)) + + return add2.out + + wf = Workflow().split(("x", "y"), x=[1, 2], y=[11, 12]) + + with Submitter(worker=plugin, cache_dir=tmpdir) as sub: + results = sub(wf) + + assert not results.errored, "\n".join(results.errors["error message"]) expected = [ ({"wfst_3.x": 1, "wfst_3.y": 11}, 13), @@ -864,7 +789,6 @@ def test_wf_st_3(plugin, tmpdir): ({"wfst_3.x": 1, "wfst_3.y": 1}, 26), ] - results = wf.result() for i, res in enumerate(expected): assert results[i].output.out == res[1] @@ -882,240 +806,219 @@ def test_wf_st_3(plugin, tmpdir): for i, res in enumerate(expected_ind): assert (results_verb_ind[i][0], results_verb_ind[i][1].output.out) == res - # checking all directories - assert wf.output_dir - for odir in wf.output_dir: - assert odir.exists() - def test_wf_ndst_3(plugin, tmpdir): """Test workflow with 2 tasks, splitter on a task level""" - wf = Workflow(name="wf_ndst_3", input_spec=["x", "y"]) - wf.add(Multiply(name="mult").split(("x", "y"), x=wf.lzin.x, y=wf.lzin.y)) - wf.add(Add2(name="add2", x=wf.mult.lzout.out)) - wf.inputs.x = [1, 2] - wf.inputs.y = [11, 12] - wf.set_output([("out", wf.add2.lzout.out)]) - wf.cache_dir = tmpdir - - with Submitter(worker=plugin) as sub: - sub(wf) - results = wf.result() + @workflow.define + def Workflow(x, y): + mult = workflow.add(Multiply().split(("x", "y"), x=x, y=y)) + add2 = workflow.add(Add2(x=mult.out)) + return add2.out + + wf = Workflow(x=[1, 2], y=[11, 12]) + + with Submitter(worker=plugin, cache_dir=tmpdir) as sub: + results = sub(wf) + + assert not results.errored, "\n".join(results.errors["error message"]) + # expected: [({"test7.x": 1, "test7.y": 11}, 13), ({"test7.x": 2, "test.y": 12}, 26)] - assert results.output.out == [13, 26] - # checking the output directory - assert wf.output_dir.exists() + assert results.outputs.out == [13, 26] def test_wf_st_4(plugin, tmpdir): """workflow with two tasks, scalar splitter and combiner for the workflow""" - wf = Workflow(name="wf_st_4", input_spec=["x", "y"]) - wf.add(Multiply(name="mult", x=wf.lzin.x, y=wf.lzin.y)) - wf.add(Add2(name="add2", x=wf.mult.lzout.out)) - wf.split(("x", "y"), x=[1, 2], y=[11, 12]) - wf.combine("x") - wf.set_output([("out", wf.add2.lzout.out)]) - wf.cache_dir = tmpdir + @workflow.define + def Workflow(x, y): + mult = workflow.add(Multiply(x=x, y=y)) + add2 = workflow.add(Add2(x=mult.out)) - with Submitter(worker=plugin) as sub: - sub(wf) + return add2.out + + wf = Workflow().split(("x", "y"), x=[1, 2], y=[11, 12]).combine("x") + with Submitter(worker=plugin, cache_dir=tmpdir) as sub: + results = sub(wf) + + assert not results.errored, "\n".join(results.errors["error message"]) - results = wf.result() # expected: [ # ({"test7.x": 1, "test7.y": 11}, 13), ({"test7.x": 2, "test.y": 12}, 26) # ] - assert results[0].output.out == 13 - assert results[1].output.out == 26 - # checking all directories - assert wf.output_dir - for odir in wf.output_dir: - assert odir.exists() + assert results.outputs.out[0] == 13 + assert results.outputs.out[1] == 26 def test_wf_ndst_4(plugin, tmpdir): """workflow with two tasks, scalar splitter and combiner on tasks level""" - wf = Workflow(name="wf_ndst_4", input_spec=["a", "b"]) - wf.add(Multiply(name="mult").split(("x", "y"), x=wf.lzin.a, y=wf.lzin.b)) - wf.add(Add2(name="add2", x=wf.mult.lzout.out).combine("mult.x")) - wf.set_output([("out", wf.add2.lzout.out)]) - wf.cache_dir = tmpdir - wf.inputs.a = [1, 2] - wf.inputs.b = [11, 12] + @workflow.define + def Workflow(a, b): + mult = workflow.add(Multiply().split(("x", "y"), x=a, y=b)) + add2 = workflow.add(Add2(x=mult.out).combine("mult.x")) - with Submitter(worker=plugin) as sub: - sub(wf) + return add2.out + + wf = Workflow(a=[1, 2], b=[11, 12]) + + with Submitter(worker=plugin, cache_dir=tmpdir) as sub: + results = sub(wf) + + assert not results.errored, "\n".join(results.errors["error message"]) - results = wf.result() # expected: [ # ({"test7.x": 1, "test7.y": 11}, 13), ({"test7.x": 2, "test.y": 12}, 26) # ] - assert results.output.out == [13, 26] - # checking the output directory - assert wf.output_dir.exists() + assert results.outputs.out == [13, 26] def test_wf_st_5(plugin, tmpdir): """workflow with two tasks, outer splitter and no combiner""" - wf = Workflow(name="wf_st_5", input_spec=["x", "y"]) - wf.add(Multiply(name="mult", x=wf.lzin.x, y=wf.lzin.y)) - wf.add(Add2(name="add2", x=wf.mult.lzout.out)) - wf.split(["x", "y"], x=[1, 2], y=[11, 12]) - wf.set_output([("out", wf.add2.lzout.out)]) - wf.cache_dir = tmpdir + @workflow.define + def Workflow(x, y): + mult = workflow.add(Multiply(x=x, y=y)) + add2 = workflow.add(Add2(x=mult.out).split(["x", "y"], x=x, y=y)) - with Submitter(worker=plugin) as sub: - sub(wf) + return add2.out - results = wf.result() - assert results[0].output.out == 13 - assert results[1].output.out == 14 - assert results[2].output.out == 24 - assert results[3].output.out == 26 - # checking all directories - assert wf.output_dir - for odir in wf.output_dir: - assert odir.exists() + wf = Workflow(x=[1, 2], y=[11, 12]) + + with Submitter(worker=plugin, cache_dir=tmpdir) as sub: + results = sub(wf) + + assert not results.errored, "\n".join(results.errors["error message"]) + + assert results.outputs.out[0] == 13 + assert results.outputs.out[1] == 14 + assert results.outputs.out[2] == 24 + assert results.outputs.out[3] == 26 def test_wf_ndst_5(plugin, tmpdir): """workflow with two tasks, outer splitter on tasks level and no combiner""" - wf = Workflow(name="wf_ndst_5", input_spec=["x", "y"]) - wf.add(Multiply(name="mult").split(["x", "y"], x=wf.lzin.x, y=wf.lzin.y)) - wf.add(Add2(name="add2", x=wf.mult.lzout.out)) - wf.inputs.x = [1, 2] - wf.inputs.y = [11, 12] - wf.set_output([("out", wf.add2.lzout.out)]) - wf.cache_dir = tmpdir - - with Submitter(worker=plugin) as sub: - sub(wf) - results = wf.result() - assert results.output.out[0] == 13 - assert results.output.out[1] == 14 - assert results.output.out[2] == 24 - assert results.output.out[3] == 26 - # checking the output directory - assert wf.output_dir.exists() + @workflow.define + def Workflow(x, y): + mult = workflow.add(Multiply().split(["x", "y"], x=x, y=y)) + add2 = workflow.add(Add2(x=mult.out)) + return add2.out + + wf = Workflow(x=[1, 2], y=[11, 12]) + + with Submitter(worker=plugin, cache_dir=tmpdir) as sub: + results = sub(wf) + + assert not results.errored, "\n".join(results.errors["error message"]) + + assert results.outputs.out[0] == 13 + assert results.outputs.out[1] == 14 + assert results.outputs.out[2] == 24 + assert results.outputs.out[3] == 26 def test_wf_st_6(plugin, tmpdir): """workflow with two tasks, outer splitter and combiner for the workflow""" - wf = Workflow(name="wf_st_6", input_spec=["x", "y"]) - wf.add(Multiply(name="mult", x=wf.lzin.x, y=wf.lzin.y)) - wf.add(Add2(name="add2", x=wf.mult.lzout.out)) - wf.split(["x", "y"], x=[1, 2, 3], y=[11, 12]) - wf.combine("x") - wf.set_output([("out", wf.add2.lzout.out)]) - wf.cache_dir = tmpdir + @workflow.define + def Workflow(x, y): + mult = workflow.add(Multiply(x=x, y=y)) + add2 = workflow.add(Add2(x=mult.out)) - with Submitter(worker=plugin) as sub: - sub(wf) + return add2.out - results = wf.result() - assert results[0][0].output.out == 13 - assert results[0][1].output.out == 24 - assert results[0][2].output.out == 35 - assert results[1][0].output.out == 14 - assert results[1][1].output.out == 26 - assert results[1][2].output.out == 38 - # checking all directories - assert wf.output_dir - for odir in wf.output_dir: - assert odir.exists() + wf = Workflow().split(["x", "y"], x=[1, 2, 3], y=[11, 12]).combine("x") + + with Submitter(worker=plugin, cache_dir=tmpdir) as sub: + results = sub(wf) + + assert not results.errored, "\n".join(results.errors["error message"]) + + assert results.outputs.out[0][0] == 13 + assert results.outputs.out[0][1] == 24 + assert results.outputs.out[0][2] == 35 + assert results.outputs.out[1][0] == 14 + assert results.outputs.out[1][1] == 26 + assert results.outputs.out[1][2] == 38 def test_wf_ndst_6(plugin, tmpdir): """workflow with two tasks, outer splitter and combiner on tasks level""" - wf = Workflow(name="wf_ndst_6", input_spec=["x", "y"]) - wf.add(Multiply(name="mult").split(["x", "y"], x=wf.lzin.x, y=wf.lzin.y)) - wf.add(Add2(name="add2", x=wf.mult.lzout.out).combine("mult.x")) - wf.inputs.x = [1, 2, 3] - wf.inputs.y = [11, 12] - wf.set_output([("out", wf.add2.lzout.out)]) - wf.cache_dir = tmpdir - - with Submitter(worker=plugin) as sub: - sub(wf) - results = wf.result() - assert results.output.out[0] == [13, 24, 35] - assert results.output.out[1] == [14, 26, 38] + @workflow.define + def Workflow(x, y): + mult = workflow.add(Multiply().split(["x", "y"], x=x, y=y)) + add2 = workflow.add(Add2(x=mult.out).combine("mult.x")) + return add2.out + + wf = Workflow(x=[1, 2, 3], y=[11, 12]) - # checking the output directory - assert wf.output_dir.exists() + with Submitter(worker=plugin, cache_dir=tmpdir) as sub: + results = sub(wf) + + assert not results.errored, "\n".join(results.errors["error message"]) + + assert results.outputs.out[0] == [13, 24, 35] + assert results.outputs.out[1] == [14, 26, 38] def test_wf_ndst_7(plugin, tmpdir): """workflow with two tasks, outer splitter and (full) combiner for first node only""" - wf = Workflow(name="wf_ndst_6", input_spec=["x", "y"]) - wf.add(Multiply(name="mult").split("x", x=wf.lzin.x, y=wf.lzin.y).combine("x")) - wf.add(Identity(name="iden", x=wf.mult.lzout.out)) - wf.inputs.x = [1, 2, 3] - wf.inputs.y = 11 - wf.set_output([("out", wf.iden.lzout.out)]) - wf.cache_dir = tmpdir - - with Submitter(worker=plugin) as sub: - sub(wf) - results = wf.result() - assert results.output.out == [11, 22, 33] + @workflow.define + def Workflow(x, y): + mult = workflow.add(Multiply().split("x", x=x, y=y).combine("x")) + iden = workflow.add(Identity(x=mult.out)) + return iden.out + + wf = Workflow(x=[1, 2, 3], y=11) - # checking the output directory - assert wf.output_dir.exists() + with Submitter(worker=plugin, cache_dir=tmpdir) as sub: + results = sub(wf) + + assert not results.errored, "\n".join(results.errors["error message"]) + + assert results.outputs.out == [11, 22, 33] def test_wf_ndst_8(plugin, tmpdir): """workflow with two tasks, outer splitter and (partial) combiner for first task only""" - wf = Workflow(name="wf_ndst_6", input_spec=["x", "y"]) - wf.add( - Multiply(name="mult").split(["x", "y"], x=wf.lzin.x, y=wf.lzin.y).combine("x") - ) - wf.add(Identity(name="iden", x=wf.mult.lzout.out)) - wf.inputs.x = [1, 2, 3] - wf.inputs.y = [11, 12] - wf.set_output([("out", wf.iden.lzout.out)]) - wf.cache_dir = tmpdir - with Submitter(worker=plugin) as sub: - sub(wf) + @workflow.define + def Workflow(x, y): + mult = workflow.add(Multiply().split(["x", "y"], x=x, y=y).combine("x")) + iden = workflow.add(Identity(x=mult.out)) + return iden.out - results = wf.result() - assert results.output.out[0] == [11, 22, 33] - assert results.output.out[1] == [12, 24, 36] + wf = Workflow(x=[1, 2, 3], y=[11, 12]) - # checking the output directory - assert wf.output_dir.exists() + with Submitter(worker=plugin, cache_dir=tmpdir) as sub: + results = sub(wf) + + assert not results.errored, "\n".join(results.errors["error message"]) + + assert results.outputs.out[0] == [11, 22, 33] + assert results.outputs.out[1] == [12, 24, 36] def test_wf_ndst_9(plugin, tmpdir): """workflow with two tasks, outer splitter and (full) combiner for first task only""" - wf = Workflow(name="wf_ndst_6", input_spec=["x", "y"]) - wf.add( - Multiply(name="mult") - .split(["x", "y"], x=wf.lzin.x, y=wf.lzin.y) - .combine(["x", "y"]) - ) - wf.add(Identity(name="iden", x=wf.mult.lzout.out)) - wf.inputs.x = [1, 2, 3] - wf.inputs.y = [11, 12] - wf.set_output([("out", wf.iden.lzout.out)]) - wf.cache_dir = tmpdir - with Submitter(worker=plugin) as sub: - sub(wf) + @workflow.define + def Workflow(x, y): + mult = workflow.add(Multiply().split(["x", "y"], x=x, y=y).combine(["x", "y"])) + iden = workflow.add(Identity(x=mult.out)) + return iden.out - results = wf.result() - assert results.output.out == [11, 12, 22, 24, 33, 36] + wf = Workflow(x=[1, 2, 3], y=[11, 12]) - # checking the output directory - assert wf.output_dir.exists() + with Submitter(worker=plugin, cache_dir=tmpdir) as sub: + results = sub(wf) + + assert not results.errored, "\n".join(results.errors["error message"]) + + assert results.outputs.out == [11, 12, 22, 24, 33, 36] # workflows with structures A -> B -> C @@ -1123,17 +1026,20 @@ def test_wf_ndst_9(plugin, tmpdir): def test_wf_3sernd_ndst_1(plugin, tmpdir): """workflow with three "serial" tasks, checking if the splitter is propagating""" - wf = Workflow(name="wf_3sernd_ndst_1", input_spec=["x", "y"]) - wf.add(Multiply(name="mult").split(["x", "y"], x=wf.lzin.x, y=wf.lzin.y)) - wf.add(Add2(name="add2_1st", x=wf.mult.lzout.out)) - wf.add(Add2(name="add2_2nd", x=wf.add2_1st.lzout.out)) - wf.inputs.x = [1, 2] - wf.inputs.y = [11, 12] - wf.set_output([("out", wf.add2_2nd.lzout.out)]) - wf.cache_dir = tmpdir - - with Submitter(worker=plugin) as sub: - sub(wf) + + @workflow.define + def Workflow(x, y): + mult = workflow.add(Multiply().split(["x", "y"], x=x, y=y)) + add2_1st = workflow.add(Add2(x=mult.out), name="add2_1st") + add2_2nd = workflow.add(Add2(x=add2_1st.out), name="add2_2nd") + return add2_2nd.out + + wf = Workflow(x=[1, 2], y=[11, 12]) + + with Submitter(worker=plugin, cache_dir=tmpdir) as sub: + results = sub(wf) + + assert not results.errored, "\n".join(results.errors["error message"]) # splitter from the first task should propagate to all tasks, # splitter_rpn should be the same in all tasks @@ -1147,13 +1053,10 @@ def test_wf_3sernd_ndst_1(plugin, tmpdir): == wf.add2_2nd.state.splitter_rpn ) - results = wf.result() - assert results.output.out[0] == 15 - assert results.output.out[1] == 16 - assert results.output.out[2] == 26 - assert results.output.out[3] == 28 - # checking the output directory - assert wf.output_dir.exists() + assert results.outputs.out[0] == 15 + assert results.outputs.out[1] == 16 + assert results.outputs.out[2] == 26 + assert results.outputs.out[3] == 28 def test_wf_3sernd_ndst_1a(plugin, tmpdir): @@ -1162,17 +1065,20 @@ def test_wf_3sernd_ndst_1a(plugin, tmpdir): first task has a splitter that propagates to the 2nd task, and the 2nd task is adding one more input to the splitter """ - wf = Workflow(name="wf_3sernd_ndst_1", input_spec=["x", "y"]) - wf.add(Add2(name="add2_1st").split("x", x=wf.lzin.x)) - wf.add(Multiply(name="mult", x=wf.add2_1st.lzout.out).split("y", y=wf.lzin.y)) - wf.add(Add2(name="add2_2nd", x=wf.mult.lzout.out)) - wf.inputs.x = [1, 2] - wf.inputs.y = [11, 12] - wf.set_output([("out", wf.add2_2nd.lzout.out)]) - wf.cache_dir = tmpdir - with Submitter(worker=plugin) as sub: - sub(wf) + @workflow.define + def Workflow(x, y): + add2_1st = workflow.add(Add2().split("x", x=x), name="add2_1st") + mult = workflow.add(Multiply(x=add2_1st.out).split("y", y=y)) + add2_2nd = workflow.add(Add2(x=mult.out), name="add2_2nd") + return add2_2nd.out + + wf = Workflow(x=[1, 2], y=[11, 12]) + + with Submitter(worker=plugin, cache_dir=tmpdir) as sub: + results = sub(wf) + + assert not results.errored, "\n".join(results.errors["error message"]) # splitter from the 1st task should propagate and the 2nd task should add one more # splitter_rpn for the 2nd and the 3rd task should be the same @@ -1185,13 +1091,10 @@ def test_wf_3sernd_ndst_1a(plugin, tmpdir): == wf.add2_2nd.state.splitter_rpn ) - results = wf.result() - assert results.output.out[0] == 35 - assert results.output.out[1] == 38 - assert results.output.out[2] == 46 - assert results.output.out[3] == 50 - # checking the output directory - assert wf.output_dir.exists() + assert results.outputs.out[0] == 35 + assert results.outputs.out[1] == 38 + assert results.outputs.out[2] == 46 + assert results.outputs.out[3] == 50 # workflows with structures A -> C, B -> C @@ -1202,27 +1105,26 @@ def test_wf_3nd_st_1(plugin_dask_opt, tmpdir): """workflow with three tasks, third one connected to two previous tasks, splitter on the workflow level """ - wf = Workflow(name="wf_st_7", input_spec=["x", "y"]) - wf.add(Add2(name="add2x", x=wf.lzin.x)) - wf.add(Add2(name="add2y", x=wf.lzin.y)) - wf.add(Multiply(name="mult", x=wf.add2x.lzout.out, y=wf.add2y.lzout.out)) - wf.split(["x", "y"], x=[1, 2, 3], y=[11, 12]) - wf.set_output([("out", wf.mult.lzout.out)]) - wf.cache_dir = tmpdir + @workflow.define + def Workflow(x, y): + add2x = workflow.add(Add2(x=x), name="add2x") + add2y = workflow.add(Add2(x=y), name="add2y") + mult = workflow.add(Multiply(x=add2x.out, y=add2y.out)) + + return mult.out + + wf = Workflow().split(["x", "y"], x=[1, 2, 3], y=[11, 12]) with Submitter(worker=plugin_dask_opt) as sub: - sub(wf) + results = sub(wf) + + assert not results.errored, "\n".join(results.errors["error message"]) - results = wf.result() assert len(results) == 6 - assert results[0].output.out == 39 - assert results[1].output.out == 42 - assert results[5].output.out == 70 - # checking all directories - assert wf.output_dir - for odir in wf.output_dir: - assert odir.exists() + assert results.outputs.out[0] == 39 + assert results.outputs.out[1] == 42 + assert results.outputs.out[5] == 70 @pytest.mark.flaky(reruns=3) # when dask @@ -1230,320 +1132,297 @@ def test_wf_3nd_ndst_1(plugin_dask_opt, tmpdir): """workflow with three tasks, third one connected to two previous tasks, splitter on the tasks levels """ - wf = Workflow(name="wf_ndst_7", input_spec=["x", "y"]) - wf.add(Add2(name="add2x").split("x", x=wf.lzin.x)) - wf.add(Add2(name="add2y").split("x", x=wf.lzin.y)) - wf.add(Multiply(name="mult", x=wf.add2x.lzout.out, y=wf.add2y.lzout.out)) - wf.inputs.x = [1, 2, 3] - wf.inputs.y = [11, 12] - wf.set_output([("out", wf.mult.lzout.out)]) - wf.cache_dir = tmpdir + + @workflow.define + def Workflow(x, y): + add2x = workflow.add(Add2().split("x", x=x), name="add2x") + add2y = workflow.add(Add2().split("x", x=y), name="add2y") + mult = workflow.add(Multiply(x=add2x.out, y=add2y.out)) + return mult.out + + wf = Workflow(x=[1, 2, 3], y=[11, 12]) with Submitter(worker=plugin_dask_opt) as sub: - sub(wf) + results = sub(wf) - results = wf.result() - assert len(results.output.out) == 6 - assert results.output.out == [39, 42, 52, 56, 65, 70] - # checking the output directory - assert wf.output_dir.exists() + assert not results.errored, "\n".join(results.errors["error message"]) + + assert len(results.outputs.out) == 6 + assert results.outputs.out == [39, 42, 52, 56, 65, 70] def test_wf_3nd_st_2(plugin, tmpdir): """workflow with three tasks, third one connected to two previous tasks, splitter and partial combiner on the workflow level """ - wf = Workflow(name="wf_st_8", input_spec=["x", "y"]) - wf.add(Add2(name="add2x", x=wf.lzin.x)) - wf.add(Add2(name="add2y", x=wf.lzin.y)) - wf.add(Multiply(name="mult", x=wf.add2x.lzout.out, y=wf.add2y.lzout.out)) - wf.split(["x", "y"], x=[1, 2, 3], y=[11, 12]).combine("x") - wf.set_output([("out", wf.mult.lzout.out)]) - wf.cache_dir = tmpdir + @workflow.define + def Workflow(x, y): + add2x = workflow.add(Add2(x=x), name="add2x") + add2y = workflow.add(Add2(x=y), name="add2y") + mult = workflow.add(Multiply(x=add2x.out, y=add2y.out)) + return mult.out - with Submitter(worker=plugin) as sub: - sub(wf) + wf = Workflow().split(["x", "y"], x=[1, 2, 3], y=[11, 12]).combine("x") + + with Submitter(worker=plugin, cache_dir=tmpdir) as sub: + results = sub(wf) + + assert not results.errored, "\n".join(results.errors["error message"]) - results = wf.result() assert len(results) == 2 - assert results[0][0].output.out == 39 - assert results[0][1].output.out == 52 - assert results[0][2].output.out == 65 - assert results[1][0].output.out == 42 - assert results[1][1].output.out == 56 - assert results[1][2].output.out == 70 - # checking all directories - assert wf.output_dir - for odir in wf.output_dir: - assert odir.exists() + assert results.outputs.out[0][0] == 39 + assert results.outputs.out[0][1] == 52 + assert results.outputs.out[0][2] == 65 + assert results.outputs.out[1][0] == 42 + assert results.outputs.out[1][1] == 56 + assert results.outputs.out[1][2] == 70 def test_wf_3nd_ndst_2(plugin, tmpdir): """workflow with three tasks, third one connected to two previous tasks, splitter and partial combiner on the tasks levels """ - wf = Workflow(name="wf_ndst_8", input_spec=["x", "y"]) - wf.add(Add2(name="add2x").split("x", x=wf.lzin.x)) - wf.add(Add2(name="add2y").split("x", x=wf.lzin.y)) - wf.add( - Multiply(name="mult", x=wf.add2x.lzout.out, y=wf.add2y.lzout.out).combine( - "add2x.x" - ) - ) - wf.inputs.x = [1, 2, 3] - wf.inputs.y = [11, 12] - wf.set_output([("out", wf.mult.lzout.out)]) - wf.cache_dir = tmpdir - with Submitter(worker="serial") as sub: - sub(wf) + @workflow.define + def Workflow(x, y): + add2x = workflow.add(Add2().split("x", x=x), name="add2x") + add2y = workflow.add(Add2().split("x", x=y), name="add2y") + mult = workflow.add(Multiply(x=add2x.out, y=add2y.out).combine("add2x.x")) + return mult.out + + wf = Workflow(x=[1, 2, 3], y=[11, 12]) + + with Submitter(cache_dir=tmpdir) as sub: + results = sub(wf) + + assert not results.errored, "\n".join(results.errors["error message"]) - results = wf.result() - assert len(results.output.out) == 2 - assert results.output.out[0] == [39, 52, 65] - assert results.output.out[1] == [42, 56, 70] - # checking the output directory - assert wf.output_dir.exists() + assert len(results.outputs.out) == 2 + assert results.outputs.out[0] == [39, 52, 65] + assert results.outputs.out[1] == [42, 56, 70] def test_wf_3nd_st_3(plugin, tmpdir): """workflow with three tasks, third one connected to two previous tasks, splitter and partial combiner (from the second task) on the workflow level """ - wf = Workflow(name="wf_st_9", input_spec=["x", "y"]) - wf.add(Add2(name="add2x", x=wf.lzin.x)) - wf.add(Add2(name="add2y", x=wf.lzin.y)) - wf.add(Multiply(name="mult", x=wf.add2x.lzout.out, y=wf.add2y.lzout.out)) - wf.split(["x", "y"], x=[1, 2, 3], y=[11, 12]).combine("y") - wf.set_output([("out", wf.mult.lzout.out)]) - wf.cache_dir = tmpdir + @workflow.define + def Workflow(x, y): + add2x = workflow.add(Add2(x=x), name="add2x") + add2y = workflow.add(Add2(x=y), name="add2y") + mult = workflow.add(Multiply(x=add2x.out, y=add2y.out)) + return mult.out - with Submitter(worker=plugin) as sub: - sub(wf) + wf = Workflow().split(["x", "y"], x=[1, 2, 3], y=[11, 12]).combine("y") + + with Submitter(worker=plugin, cache_dir=tmpdir) as sub: + results = sub(wf) + + assert not results.errored, "\n".join(results.errors["error message"]) - results = wf.result() assert len(results) == 3 - assert results[0][0].output.out == 39 - assert results[0][1].output.out == 42 - assert results[1][0].output.out == 52 - assert results[1][1].output.out == 56 - assert results[2][0].output.out == 65 - assert results[2][1].output.out == 70 - # checking all directories - assert wf.output_dir - for odir in wf.output_dir: - assert odir.exists() + assert results.outputs.out[0][0] == 39 + assert results.outputs.out[0][1] == 42 + assert results.outputs.out[1][0] == 52 + assert results.outputs.out[1][1] == 56 + assert results.outputs.out[2][0] == 65 + assert results.outputs.out[2][1] == 70 def test_wf_3nd_ndst_3(plugin, tmpdir): """workflow with three tasks, third one connected to two previous tasks, splitter and partial combiner (from the second task) on the tasks levels """ - wf = Workflow(name="wf_ndst_9", input_spec=["x", "y"]) - wf.add(Add2(name="add2x").split("x", x=wf.lzin.x)) - wf.add(Add2(name="add2y").split("x", x=wf.lzin.y)) - wf.add( - Multiply(name="mult", x=wf.add2x.lzout.out, y=wf.add2y.lzout.out).combine( - "add2y.x" - ) - ) - wf.inputs.x = [1, 2, 3] - wf.inputs.y = [11, 12] - wf.set_output([("out", wf.mult.lzout.out)]) - wf.cache_dir = tmpdir - with Submitter(worker=plugin) as sub: - sub(wf) + @workflow.define + def Workflow(x, y): + add2x = workflow.add(Add2().split("x", x=x), name="add2x") + add2y = workflow.add(Add2().split("x", x=y), name="add2y") + mult = workflow.add(Multiply(x=add2x.out, y=add2y.out).combine("add2y.x")) + return mult.out - results = wf.result() - assert len(results.output.out) == 3 - assert results.output.out[0] == [39, 42] - assert results.output.out[1] == [52, 56] - assert results.output.out[2] == [65, 70] - # checking the output directory - assert wf.output_dir.exists() + wf = Workflow(x=[1, 2, 3], y=[11, 12]) + + with Submitter(worker="debug", cache_dir=tmpdir) as sub: + results = sub(wf) + + assert not results.errored, "\n".join(results.errors["error message"]) + + assert len(results.outputs.out) == 3 + assert results.outputs.out[0] == [39, 42] + assert results.outputs.out[1] == [52, 56] + assert results.outputs.out[2] == [65, 70] def test_wf_3nd_st_4(plugin, tmpdir): """workflow with three tasks, third one connected to two previous tasks, splitter and full combiner on the workflow level """ - wf = Workflow(name="wf_st_10", input_spec=["x", "y"]) - wf.add(Add2(name="add2x", x=wf.lzin.x)) - wf.add(Add2(name="add2y", x=wf.lzin.y)) - wf.add(Multiply(name="mult", x=wf.add2x.lzout.out, y=wf.add2y.lzout.out)) - wf.split(["x", "y"], x=[1, 2, 3], y=[11, 12]).combine(["x", "y"]) - wf.set_output([("out", wf.mult.lzout.out)]) - wf.plugin = plugin - wf.cache_dir = tmpdir - with Submitter(worker=plugin) as sub: - sub(wf) + @workflow.define + def Workflow(x, y): + add2x = workflow.add(Add2(x=x), name="add2x") + add2y = workflow.add(Add2(x=y), name="add2y") + mult = workflow.add(Multiply(x=add2x.out, y=add2y.out)) + return mult.out + + wf = Workflow().split(["x", "y"], x=[1, 2, 3], y=[11, 12]).combine(["x", "y"]) + + with Submitter(worker=plugin, cache_dir=tmpdir) as sub: + results = sub(wf) + + assert not results.errored, "\n".join(results.errors["error message"]) - results = wf.result() assert len(results) == 6 - assert results[0].output.out == 39 - assert results[1].output.out == 42 - assert results[2].output.out == 52 - assert results[3].output.out == 56 - assert results[4].output.out == 65 - assert results[5].output.out == 70 - # checking all directories - assert wf.output_dir - for odir in wf.output_dir: - assert odir.exists() + assert results.outputs.out[0] == 39 + assert results.outputs.out[1] == 42 + assert results.outputs.out[2] == 52 + assert results.outputs.out[3] == 56 + assert results.outputs.out[4] == 65 + assert results.outputs.out[5] == 70 def test_wf_3nd_ndst_4(plugin, tmpdir): """workflow with three tasks, third one connected to two previous tasks, splitter and full combiner on the tasks levels """ - wf = Workflow(name="wf_ndst_10", input_spec=["x", "y"]) - wf.add(Add2(name="add2x").split("x", x=wf.lzin.x)) - wf.add(Add2(name="add2y").split("x", x=wf.lzin.y)) - wf.add( - Multiply(name="mult", x=wf.add2x.lzout.out, y=wf.add2y.lzout.out).combine( - ["add2x.x", "add2y.x"] + + @workflow.define + def Workflow(x, y): + add2x = workflow.add(Add2().split("x", x=x), name="add2x") + add2y = workflow.add(Add2().split("x", x=y), name="add2y") + mult = workflow.add( + Multiply(x=add2x.out, y=add2y.out).combine(["add2x.x", "add2y.x"]) ) - ) - wf.inputs.x = [1, 2, 3] - wf.inputs.y = [11, 12] - wf.set_output([("out", wf.mult.lzout.out)]) - wf.cache_dir = tmpdir + return mult.out - with Submitter(worker=plugin) as sub: - sub(wf) + wf = Workflow(x=[1, 2, 3], y=[11, 12]) + + with Submitter(worker=plugin, cache_dir=tmpdir) as sub: + results = sub(wf) + + assert not results.errored, "\n".join(results.errors["error message"]) # assert wf.output_dir.exists() - results = wf.result() - assert len(results.output.out) == 6 - assert results.output.out == [39, 42, 52, 56, 65, 70] - # checking the output directory - assert wf.output_dir.exists() + assert len(results.outputs.out) == 6 + assert results.outputs.out == [39, 42, 52, 56, 65, 70] def test_wf_3nd_st_5(plugin, tmpdir): """workflow with three tasks (A->C, B->C) and three fields in the splitter, splitter and partial combiner (from the second task) on the workflow level """ - wf = Workflow(name="wf_st_9", input_spec=["x", "y", "z"]) - wf.add(Add2(name="add2x", x=wf.lzin.x)) - wf.add(Add2(name="add2y", x=wf.lzin.y)) - wf.add( - FunAddVar3( - name="addvar", a=wf.add2x.lzout.out, b=wf.add2y.lzout.out, c=wf.lzin.z - ) + + @workflow.define + def Workflow(x, y, z): + add2x = workflow.add(Add2(x=x), name="add2x") + add2y = workflow.add(Add2(x=y), name="add2y") + addvar = workflow.add(FunAddVar3(a=add2x.out, b=add2y.out, c=z)) + return addvar.out + + wf = ( + Workflow() + .split(["x", "y", "z"], x=[2, 3], y=[11, 12], z=[10, 100]) + .combine("y") ) - wf.split(["x", "y", "z"], x=[2, 3], y=[11, 12], z=[10, 100]).combine("y") - wf.set_output([("out", wf.addvar.lzout.out)]) - wf.plugin = plugin - wf.cache_dir = tmpdir + with Submitter(worker=plugin, cache_dir=tmpdir) as sub: + results = sub(wf) - with Submitter(worker=plugin) as sub: - sub(wf) + assert not results.errored, "\n".join(results.errors["error message"]) - results = wf.result() assert len(results) == 4 - assert results[0][0].output.out == 27 - assert results[0][1].output.out == 28 - assert results[1][0].output.out == 117 - assert results[1][1].output.out == 118 - assert results[2][0].output.out == 28 - assert results[2][1].output.out == 29 - assert results[3][0].output.out == 118 - assert results[3][1].output.out == 119 - - # checking all directories - assert wf.output_dir - for odir in wf.output_dir: - assert odir.exists() + assert results.outputs.out[0][0] == 27 + assert results.outputs.out[0][1] == 28 + assert results.outputs.out[1][0] == 117 + assert results.outputs.out[1][1] == 118 + assert results.outputs.out[2][0] == 28 + assert results.outputs.out[2][1] == 29 + assert results.outputs.out[3][0] == 118 + assert results.outputs.out[3][1] == 119 def test_wf_3nd_ndst_5(plugin, tmpdir): """workflow with three tasks (A->C, B->C) and three fields in the splitter, all tasks have splitters and the last one has a partial combiner (from the 2nd) """ - wf = Workflow(name="wf_st_9", input_spec=["x", "y", "z"]) - wf.add(Add2(name="add2x").split("x", x=wf.lzin.x)) - wf.add(Add2(name="add2y").split("x", x=wf.lzin.y)) - wf.add( - FunAddVar3(name="addvar", a=wf.add2x.lzout.out, b=wf.add2y.lzout.out) - .split("c", c=wf.lzin.z) - .combine("add2x.x") - ) - wf.inputs.x = [2, 3] - wf.inputs.y = [11, 12] - wf.inputs.z = [10, 100] - wf.set_output([("out", wf.addvar.lzout.out)]) - wf.cache_dir = tmpdir + @workflow.define + def Workflow(x, y, z): + add2x = workflow.add(Add2().split("x", x=x), name="add2x") + add2y = workflow.add(Add2().split("x", x=y), name="add2y") + addvar = workflow.add( + FunAddVar3(a=add2x.out, b=add2y.out).split("c", c=z).combine("add2x.x") + ) - with Submitter(worker=plugin) as sub: - sub(wf) + return addvar.out - results = wf.result() - assert len(results.output.out) == 4 - assert results.output.out[0] == [27, 28] - assert results.output.out[1] == [117, 118] - assert results.output.out[2] == [28, 29] - assert results.output.out[3] == [118, 119] + wf = Workflow(x=[2, 3], y=[11, 12], z=[10, 100]) + + with Submitter(worker=plugin, cache_dir=tmpdir) as sub: + results = sub(wf) + + assert not results.errored, "\n".join(results.errors["error message"]) + + assert len(results.outputs.out) == 4 + assert results.outputs.out[0] == [27, 28] + assert results.outputs.out[1] == [117, 118] + assert results.outputs.out[2] == [28, 29] + assert results.outputs.out[3] == [118, 119] # checking all directories - assert wf.output_dir.exists() def test_wf_3nd_ndst_6(plugin, tmpdir): """workflow with three tasks, third one connected to two previous tasks, the third one uses scalar splitter from the previous ones and a combiner """ - wf = Workflow(name="wf_ndst_9", input_spec=["x", "y"]) - wf.add(Add2(name="add2x").split("x", x=wf.lzin.x)) - wf.add(Add2(name="add2y").split("x", x=wf.lzin.y)) - wf.add( - Multiply(name="mult", x=wf.add2x.lzout.out, y=wf.add2y.lzout.out) - .split(("_add2x", "_add2y")) - .combine("add2y.x") - ) - wf.inputs.x = [1, 2] - wf.inputs.y = [11, 12] - wf.set_output([("out", wf.mult.lzout.out)]) - wf.cache_dir = tmpdir - with Submitter(worker=plugin) as sub: - sub(wf) + @workflow.define + def Workflow(x, y): + add2x = workflow.add(Add2().split("x", x=x), name="add2x") + add2y = workflow.add(Add2().split("x", x=y), name="add2y") + mult = workflow.add( + Multiply(x=add2x.out, y=add2y.out) + .split(("_add2x", "_add2y")) + .combine("add2y.x") + ) + return mult.out + + wf = Workflow(x=[1, 2], y=[11, 12]) + + with Submitter(worker=plugin, cache_dir=tmpdir) as sub: + results = sub(wf) - results = wf.result() - assert results.output.out == [39, 56] - # checking the output directory - assert wf.output_dir.exists() + assert not results.errored, "\n".join(results.errors["error message"]) + + assert results.outputs.out == [39, 56] def test_wf_3nd_ndst_7(plugin, tmpdir): """workflow with three tasks, third one connected to two previous tasks, the third one uses scalar splitter from the previous ones """ - wf = Workflow(name="wf_ndst_9", input_spec=["x"]) - wf.add(Add2(name="add2x").split("x", x=wf.lzin.x)) - wf.add(Add2(name="add2y").split("x", x=wf.lzin.x)) - wf.add( - Multiply(name="mult", x=wf.add2x.lzout.out, y=wf.add2y.lzout.out).split( - ("_add2x", "_add2y") + + @workflow.define + def Workflow(x): + add2x = workflow.add(Add2().split("x", x=x), name="add2x") + add2y = workflow.add(Add2().split("x", x=x), name="add2y") + mult = workflow.add( + Multiply(x=add2x.out, y=add2y.out).split(("_add2x", "_add2y")) ) - ) - wf.inputs.x = [1, 2] - wf.set_output([("out", wf.mult.lzout.out)]) - wf.cache_dir = tmpdir + return mult.out - with Submitter(worker=plugin) as sub: - sub(wf) + wf = Workflow(x=[1, 2]) - results = wf.result() - assert results.output.out == [9, 16] - # checking the output directory - assert wf.output_dir.exists() + with Submitter(worker=plugin, cache_dir=tmpdir) as sub: + results = sub(wf) + + assert not results.errored, "\n".join(results.errors["error message"]) + + assert results.outputs.out == [9, 16] # workflows with structures A -> B -> C with multiple connections @@ -1551,47 +1430,40 @@ def test_wf_3nd_ndst_7(plugin, tmpdir): def test_wf_3nd_8(tmpdir): """workflow with three tasks A->B->C vs two tasks A->C with multiple connections""" - wf = Workflow(name="wf", input_spec=["zip"], cache_dir=tmpdir) - wf.inputs.zip = [["test1", "test3", "test5"], ["test2", "test4", "test6"]] - wf.add(Identity2Flds(name="iden2flds_1", x2="Hoi").split("x1", x1=wf.lzin.zip)) + @workflow.define(outputs=["out1", "out2", "out1a", "out2a"]) + def Workflow(zip): + + iden2flds_1 = workflow.add( + Identity2Flds(x2="Hoi").split("x1", x1=zip), name="iden2flds_1" + ) - wf.add(Identity(name="identity", x=wf.iden2flds_1.lzout.out1)) + identity = workflow.add(Identity(x=iden2flds_1.out1)) - wf.add( - Identity2Flds( - name="iden2flds_2", x1=wf.identity.lzout.out, x2=wf.iden2flds_1.lzout.out2 + iden2flds_2 = workflow.add( + Identity2Flds(x1=identity.out, x2=iden2flds_1.out2), name="iden2flds_2" ) - ) - wf.add( - Identity2Flds( - name="iden2flds_2a", - x1=wf.iden2flds_1.lzout.out1, - x2=wf.iden2flds_1.lzout.out2, + iden2flds_2a = workflow.add( + Identity2Flds( + x1=iden2flds_1.out1, + x2=iden2flds_1.out2, + ) ) - ) - wf.set_output( - [ - ("out1", wf.iden2flds_2.lzout.out1), - ("out2", wf.iden2flds_2.lzout.out2), - ("out1a", wf.iden2flds_2a.lzout.out1), - ("out2a", wf.iden2flds_2a.lzout.out2), - ] - ) + return iden2flds_2.out1, iden2flds_2.out2, iden2flds_2a.out1, iden2flds_2a.out2 - with Submitter(worker="cf") as sub: - sub(wf) + wf = Workflow(zip=[["test1", "test3", "test5"], ["test2", "test4", "test6"]]) - res = wf.result() + with Submitter(worker="cf") as sub: + res = sub(wf) assert ( - res.output.out1 - == res.output.out1a + res.outputs.out1 + == res.outputs.out1a == [["test1", "test3", "test5"], ["test2", "test4", "test6"]] ) - assert res.output.out2 == res.output.out2a == ["Hoi", "Hoi"] + assert res.outputs.out2 == res.output.out2a == ["Hoi", "Hoi"] # workflows with Left and Right part in splitters A -> B (L&R parts of the splitter) @@ -1602,27 +1474,27 @@ def test_wf_ndstLR_1(plugin, tmpdir): The second task has its own simple splitter and the Left part from the first task should be added """ - wf = Workflow(name="wf_ndst_3", input_spec=["x", "y"]) - wf.add(Add2(name="add2").split("x", x=wf.lzin.x)) - wf.add(Multiply(name="mult", x=wf.add2.lzout.out).split("y", y=wf.lzin.y)) - wf.inputs.x = [1, 2] - wf.inputs.y = [11, 12] - wf.set_output([("out", wf.mult.lzout.out)]) - wf.cache_dir = tmpdir - with Submitter(worker=plugin) as sub: - sub(wf) + @workflow.define + def Workflow(x, y): + add2 = workflow.add(Add2().split("x", x=x)) + mult = workflow.add(Multiply(x=add2.out).split("y", y=y)) + return mult.out + + wf = Workflow(x=[1, 2], y=[11, 12]) + + with Submitter(worker=plugin, cache_dir=tmpdir) as sub: + results = sub(wf) + + assert not results.errored, "\n".join(results.errors["error message"]) # checking if the splitter is created properly assert wf.mult.state.splitter == ["_add2", "mult.y"] assert wf.mult.state.splitter_rpn == ["add2.x", "mult.y", "*"] - results = wf.result() # expected: [({"add2.x": 1, "mult.y": 11}, 33), ({"add2.x": 1, "mult.y": 12}, 36), # ({"add2.x": 2, "mult.y": 11}, 44), ({"add2.x": 2, "mult.y": 12}, 48)] - assert results.output.out == [33, 36, 44, 48] - # checking the output directory - assert wf.output_dir.exists() + assert results.outputs.out == [33, 36, 44, 48] def test_wf_ndstLR_1a(plugin, tmpdir): @@ -1630,29 +1502,27 @@ def test_wf_ndstLR_1a(plugin, tmpdir): The second task has splitter that has Left part (from previous state) and the Right part (it's own splitter) """ - wf = Workflow(name="wf_ndst_3", input_spec=["x", "y"]) - wf.add(Add2(name="add2").split("x", x=wf.lzin.x)) - wf.add( - Multiply(name="mult").split(["_add2", "y"], x=wf.add2.lzout.out, y=wf.lzin.y) - ) - wf.inputs.x = [1, 2] - wf.inputs.y = [11, 12] - wf.set_output([("out", wf.mult.lzout.out)]) - wf.cache_dir = tmpdir - with Submitter(worker=plugin) as sub: - sub(wf) + @workflow.define + def Workflow(x, y): + add2 = workflow.add(Add2().split("x", x=x)) + mult = workflow.add(Multiply().split(["_add2", "y"], x=add2.out, y=y)) + return mult.out + + wf = Workflow(x=[1, 2], y=[11, 12]) + + with Submitter(worker=plugin, cache_dir=tmpdir) as sub: + results = sub(wf) + + assert not results.errored, "\n".join(results.errors["error message"]) # checking if the splitter is created properly assert wf.mult.state.splitter == ["_add2", "mult.y"] assert wf.mult.state.splitter_rpn == ["add2.x", "mult.y", "*"] - results = wf.result() # expected: [({"add2.x": 1, "mult.y": 11}, 33), ({"add2.x": 1, "mult.y": 12}, 36), # ({"add2.x": 2, "mult.y": 11}, 44), ({"add2.x": 2, "mult.y": 12}, 48)] - assert results.output.out == [33, 36, 44, 48] - # checking the output directory - assert wf.output_dir.exists() + assert results.outputs.out == [33, 36, 44, 48] def test_wf_ndstLR_2(plugin, tmpdir): @@ -1660,33 +1530,30 @@ def test_wf_ndstLR_2(plugin, tmpdir): The second task has its own outer splitter and the Left part from the first task should be added """ - wf = Workflow(name="wf_ndst_3", input_spec=["x", "y", "z"]) - wf.add(Add2(name="add2").split("x", x=wf.lzin.x)) - wf.add( - FunAddVar3(name="addvar", a=wf.add2.lzout.out).split( - ["b", "c"], b=wf.lzin.y, c=wf.lzin.z - ) - ) - wf.inputs.x = [1, 2, 3] - wf.inputs.y = [10, 20] - wf.inputs.z = [100, 200] - wf.set_output([("out", wf.addvar.lzout.out)]) - wf.cache_dir = tmpdir - with Submitter(worker=plugin) as sub: - sub(wf) + @workflow.define + def Workflow(x, y, z): + add2 = workflow.add(Add2().split("x", x=x)) + addvar = workflow.add(FunAddVar3(a=add2.out).split(["b", "c"], b=y, c=z)) + return addvar.out + + wf = Workflow(x=[1, 2, 3], y=[10, 20], z=[100, 200]) + + with Submitter(worker=plugin, cache_dir=tmpdir) as sub: + results = sub(wf) + + assert not results.errored, "\n".join(results.errors["error message"]) # checking if the splitter is created properly assert wf.addvar.state.splitter == ["_add2", ["addvar.b", "addvar.c"]] assert wf.addvar.state.splitter_rpn == ["add2.x", "addvar.b", "addvar.c", "*", "*"] - results = wf.result() # expected: [({"add2.x": 1, "mult.b": 10, "mult.c": 100}, 113), # ({"add2.x": 1, "mult.b": 10, "mult.c": 200}, 213), # ({"add2.x": 1, "mult.b": 20, "mult.c": 100}, 123), # ({"add2.x": 1, "mult.b": 20, "mult.c": 200}, 223), # ...] - assert results.output.out == [ + assert results.outputs.out == [ 113, 213, 123, @@ -1700,8 +1567,6 @@ def test_wf_ndstLR_2(plugin, tmpdir): 125, 225, ] - # checking the output directory - assert wf.output_dir.exists() def test_wf_ndstLR_2a(plugin, tmpdir): @@ -1709,33 +1574,33 @@ def test_wf_ndstLR_2a(plugin, tmpdir): The second task has splitter that has Left part (from previous state) and the Right part (it's own outer splitter) """ - wf = Workflow(name="wf_ndst_3", input_spec=["x", "y", "z"]) - wf.add(Add2(name="add2").split("x", x=wf.lzin.x)) - wf.add( - FunAddVar3(name="addvar", a=wf.add2.lzout.out).split( - ["_add2", ["b", "c"]], b=wf.lzin.y, c=wf.lzin.z + + @workflow.define + def Workflow(x, y, z): + add2 = workflow.add(Add2().split("x", x=x)) + addvar = workflow.add( + FunAddVar3(a=add2.out).split(["_add2", ["b", "c"]], b=y, c=z) ) - ) - wf.inputs.x = [1, 2, 3] - wf.inputs.y = [10, 20] - wf.inputs.z = [100, 200] - wf.set_output([("out", wf.addvar.lzout.out)]) - wf.cache_dir = tmpdir - with Submitter(worker=plugin) as sub: - sub(wf) + return addvar.out + + wf = Workflow(x=[1, 2, 3], y=[10, 20], z=[100, 200]) + + with Submitter(worker=plugin, cache_dir=tmpdir) as sub: + results = sub(wf) + + assert not results.errored, "\n".join(results.errors["error message"]) # checking if the splitter is created properly assert wf.addvar.state.splitter == ["_add2", ["addvar.b", "addvar.c"]] assert wf.addvar.state.splitter_rpn == ["add2.x", "addvar.b", "addvar.c", "*", "*"] - results = wf.result() # expected: [({"add2.x": 1, "mult.b": 10, "mult.c": 100}, 113), # ({"add2.x": 1, "mult.b": 10, "mult.c": 200}, 213), # ({"add2.x": 1, "mult.b": 20, "mult.c": 100}, 123), # ({"add2.x": 1, "mult.b": 20, "mult.c": 200}, 223), # ...] - assert results.output.out == [ + assert results.outputs.out == [ 113, 213, 123, @@ -1749,8 +1614,6 @@ def test_wf_ndstLR_2a(plugin, tmpdir): 125, 225, ] - # checking the output directory - assert wf.output_dir.exists() # workflows with inner splitters A -> B (inner spl) @@ -1760,74 +1623,75 @@ def test_wf_ndstinner_1(plugin, tmpdir): """workflow with 2 tasks, the second task has inner splitter """ - wf = Workflow(name="wf_st_3", input_spec={"x": int}) - wf.add(ListOutput(name="list", x=wf.lzin.x)) - wf.add(Add2(name="add2").split("x", x=wf.list.lzout.out)) - wf.inputs.x = 1 - wf.set_output([("out_list", wf.list.lzout.out), ("out", wf.add2.lzout.out)]) - wf.cache_dir = tmpdir - with Submitter(worker=plugin) as sub: - sub(wf) + @workflow.define(outputs=["out_list", "out"]) + def Workflow(x: int): + list = workflow.add(ListOutput(x=x)) + add2 = workflow.add(Add2().split("x", x=list.out)) + return list.out, add2.out + + wf = Workflow(x=1) # + + with Submitter(worker=plugin, cache_dir=tmpdir) as sub: + results = sub(wf) + + assert not results.errored, "\n".join(results.errors["error message"]) assert wf.add2.state.splitter == "add2.x" assert wf.add2.state.splitter_rpn == ["add2.x"] - results = wf.result() - assert results.output.out_list == [1, 2, 3] - assert results.output.out == [3, 4, 5] - - assert wf.output_dir.exists() + assert results.outputs.out_list == [1, 2, 3] + assert results.outputs.out == [3, 4, 5] def test_wf_ndstinner_2(plugin, tmpdir): """workflow with 2 tasks, the second task has two inputs and inner splitter from one of the input """ - wf = Workflow(name="wf_st_3", input_spec=["x", "y"]) - wf.add(ListOutput(name="list", x=wf.lzin.x)) - wf.add(Multiply(name="mult", y=wf.lzin.y).split("x", x=wf.list.lzout.out)) - wf.inputs.x = 1 - wf.inputs.y = 10 - wf.set_output([("out_list", wf.list.lzout.out), ("out", wf.mult.lzout.out)]) - wf.cache_dir = tmpdir - with Submitter(worker=plugin) as sub: - sub(wf) + @workflow.define(outputs=["out_list", "out"]) + def Workflow(x, y): + list = workflow.add(ListOutput(x=x)) + mult = workflow.add(Multiply(y=y).split("x", x=list.out)) + return list.out, mult.out + + wf = Workflow(x=1, y=10) # + + with Submitter(worker=plugin, cache_dir=tmpdir) as sub: + results = sub(wf) + + assert not results.errored, "\n".join(results.errors["error message"]) assert wf.mult.state.splitter == "mult.x" assert wf.mult.state.splitter_rpn == ["mult.x"] - results = wf.result() - assert results.output.out_list == [1, 2, 3] - assert results.output.out == [10, 20, 30] - - assert wf.output_dir.exists() + assert results.outputs.out_list == [1, 2, 3] + assert results.outputs.out == [10, 20, 30] def test_wf_ndstinner_3(plugin, tmpdir): """workflow with 2 tasks, the second task has two inputs and outer splitter that includes an inner field """ - wf = Workflow(name="wf_st_3", input_spec=["x", "y"]) - wf.add(ListOutput(name="list", x=wf.lzin.x)) - wf.add(Multiply(name="mult").split(["x", "y"], x=wf.list.lzout.out, y=wf.lzin.y)) - wf.inputs.x = 1 - wf.inputs.y = [10, 100] - wf.set_output([("out_list", wf.list.lzout.out), ("out", wf.mult.lzout.out)]) - wf.cache_dir = tmpdir - with Submitter(worker=plugin) as sub: - sub(wf) + @workflow.define(outputs=["out_list", "out"]) + def Workflow(x, y): + list = workflow.add(ListOutput(x=x)) + mult = workflow.add(Multiply().split(["x", "y"], x=list.out, y=y)) + return list.out, mult.out + + wf = Workflow(x=1, y=[10, 100]) + + with Submitter(worker=plugin, cache_dir=tmpdir) as sub: + results = sub(wf) + + assert not results.errored, "\n".join(results.errors["error message"]) assert wf.mult.state.splitter == ["mult.x", "mult.y"] assert wf.mult.state.splitter_rpn == ["mult.x", "mult.y", "*"] - results = wf.result() - assert results.output.out_list == [1, 2, 3] - assert results.output.out == [10, 100, 20, 200, 30, 300] - - assert wf.output_dir.exists() + assert results.outputs.out_list == [1, 2, 3] + assert results.outputs.out == [10, 100, 20, 200, 30, 300] def test_wf_ndstinner_4(plugin, tmpdir): @@ -1835,28 +1699,28 @@ def test_wf_ndstinner_4(plugin, tmpdir): the second task has two inputs and inner splitter from one of the input, the third task has no its own splitter """ - wf = Workflow(name="wf_st_3", input_spec=["x", "y"]) - wf.add(ListOutput(name="list", x=wf.lzin.x)) - wf.add(Multiply(name="mult", y=wf.lzin.y).split("x", x=wf.list.lzout.out)) - wf.add(Add2(name="add2", x=wf.mult.lzout.out)) - wf.inputs.x = 1 - wf.inputs.y = 10 - wf.set_output([("out_list", wf.list.lzout.out), ("out", wf.add2.lzout.out)]) - wf.cache_dir = tmpdir - with Submitter(worker=plugin) as sub: - sub(wf) + @workflow.define(outputs=["out_list", "out"]) + def Workflow(x, y): + list = workflow.add(ListOutput(x=x)) + mult = workflow.add(Multiply(y=y).split("x", x=list.out)) + add2 = workflow.add(Add2(x=mult.out)) + return list.out, add2.out + + wf = Workflow(x=1, y=10) + + with Submitter(worker=plugin, cache_dir=tmpdir) as sub: + results = sub(wf) + + assert not results.errored, "\n".join(results.errors["error message"]) assert wf.mult.state.splitter == "mult.x" assert wf.mult.state.splitter_rpn == ["mult.x"] assert wf.add2.state.splitter == "_mult" assert wf.add2.state.splitter_rpn == ["mult.x"] - results = wf.result() - assert results.output.out_list == [1, 2, 3] - assert results.output.out == [12, 22, 32] - - assert wf.output_dir.exists() + assert results.outputs.out_list == [1, 2, 3] + assert results.outputs.out == [12, 22, 32] def test_wf_ndstinner_5(plugin, tmpdir): @@ -1866,25 +1730,20 @@ def test_wf_ndstinner_5(plugin, tmpdir): there is a inner_cont_dim) the third task has no new splitter """ - wf = Workflow(name="wf_5", input_spec=["x", "y", "b"]) - wf.add(ListOutput(name="list").split("x", x=wf.lzin.x)) - wf.add(Multiply(name="mult").split(["y", "x"], x=wf.list.lzout.out, y=wf.lzin.y)) - wf.add(FunAddVar(name="addvar", a=wf.mult.lzout.out).split("b", b=wf.lzin.b)) - wf.inputs.x = [1, 2] - wf.inputs.y = [10, 100] - wf.inputs.b = [3, 5] - - wf.set_output( - [ - ("out_list", wf.list.lzout.out), - ("out_mult", wf.mult.lzout.out), - ("out_add", wf.addvar.lzout.out), - ] - ) - wf.cache_dir = tmpdir - with Submitter(worker=plugin) as sub: - sub(wf) + @workflow.define(outputs=["out_list", "out_mult", "out_add"]) + def Workflow(x, y, b): + list = workflow.add(ListOutput().split("x", x=x)) + mult = workflow.add(Multiply().split(["y", "x"], x=list.out, y=y)) + addvar = workflow.add(FunAddVar(a=mult.out).split("b", b=b)) + return list.out, mult.out, addvar.out + + wf = Workflow(x=[1, 2], y=[10, 100], b=[3, 5]) + + with Submitter(worker=plugin, cache_dir=tmpdir) as sub: + results = sub(wf) + + assert not results.errored, "\n".join(results.errors["error message"]) assert wf.mult.state.splitter == ["_list", ["mult.y", "mult.x"]] assert wf.mult.state.splitter_rpn == ["list.x", "mult.y", "mult.x", "*", "*"] @@ -1899,9 +1758,8 @@ def test_wf_ndstinner_5(plugin, tmpdir): "*", ] - results = wf.result() - assert results.output.out_list == [[1, 2, 3], [2, 4, 6]] - assert results.output.out_mult == [ + assert results.outputs.out_list == [[1, 2, 3], [2, 4, 6]] + assert results.outputs.out_mult == [ 10, 20, 30, @@ -1915,7 +1773,7 @@ def test_wf_ndstinner_5(plugin, tmpdir): 400, 600, ] - assert results.output.out_add == [ + assert results.outputs.out_add == [ 13, 15, 23, @@ -1942,54 +1800,50 @@ def test_wf_ndstinner_5(plugin, tmpdir): 605, ] - assert wf.output_dir.exists() - # workflow that have some single values as the input def test_wf_st_singl_1(plugin, tmpdir): """workflow with two tasks, only one input is in the splitter and combiner""" - wf = Workflow(name="wf_st_5", input_spec=["x", "y"]) - wf.add(Multiply(name="mult", x=wf.lzin.x, y=wf.lzin.y)) - wf.add(Add2(name="add2", x=wf.mult.lzout.out)) - wf.split("x", x=[1, 2], y=11) - wf.combine("x") - wf.set_output([("out", wf.add2.lzout.out)]) - wf.cache_dir = tmpdir + @workflow.define + def Workflow(x, y): + mult = workflow.add(Multiply(x=x, y=y)) + add2 = workflow.add(Add2(x=mult.out)) - with Submitter(worker=plugin) as sub: - sub(wf) + return add2.out - results = wf.result() - assert results[0].output.out == 13 - assert results[1].output.out == 24 - # checking all directories - assert wf.output_dir - for odir in wf.output_dir: - assert odir.exists() + wf = Workflow().split("x", x=[1, 2], y=11).combine("x") + + with Submitter(worker=plugin, cache_dir=tmpdir) as sub: + results = sub(wf) + + assert not results.errored, "\n".join(results.errors["error message"]) + + assert results.outputs.out[0] == 13 + assert results.outputs.out[1] == 24 def test_wf_ndst_singl_1(plugin, tmpdir): """workflow with two tasks, outer splitter and combiner on tasks level; only one input is part of the splitter, the other is a single value """ - wf = Workflow(name="wf_ndst_5", input_spec=["x", "y"]) - wf.add(Multiply(name="mult", y=wf.lzin.y).split("x", x=wf.lzin.x)) - wf.add(Add2(name="add2", x=wf.mult.lzout.out).combine("mult.x")) - wf.inputs.x = [1, 2] - wf.inputs.y = 11 - wf.set_output([("out", wf.add2.lzout.out)]) - wf.cache_dir = tmpdir - with Submitter(worker=plugin) as sub: - sub(wf) + @workflow.define + def Workflow(x, y): + mult = workflow.add(Multiply(y=y).split("x", x=x), name="mult") + add2 = workflow.add(Add2(x=mult.out).combine("mult.x")) + return add2.out - results = wf.result() - assert results.output.out == [13, 24] - # checking the output directory - assert wf.output_dir.exists() + wf = Workflow(x=[1, 2], y=11) + + with Submitter(worker=plugin, cache_dir=tmpdir) as sub: + results = sub(wf) + + assert not results.errored, "\n".join(results.errors["error message"]) + + assert results.outputs.out == [13, 24] def test_wf_st_singl_2(plugin, tmpdir): @@ -1997,27 +1851,25 @@ def test_wf_st_singl_2(plugin, tmpdir): splitter on the workflow level only one input is part of the splitter, the other is a single value """ - wf = Workflow(name="wf_st_6", input_spec=["x", "y"]) - wf.add(Add2(name="add2x", x=wf.lzin.x)) - wf.add(Add2(name="add2y", x=wf.lzin.y)) - wf.add(Multiply(name="mult", x=wf.add2x.lzout.out, y=wf.add2y.lzout.out)) - wf.split("x", x=[1, 2, 3], y=11) - wf.set_output([("out", wf.mult.lzout.out)]) - wf.cache_dir = tmpdir + @workflow.define + def Workflow(x, y): + add2x = workflow.add(Add2(x=x), name="add2x") + add2y = workflow.add(Add2(x=y), name="add2y") + mult = workflow.add(Multiply(x=add2x.out, y=add2y.out)) + return mult.out - with Submitter(worker=plugin) as sub: - sub(wf) + wf = Workflow().split("x", x=[1, 2, 3], y=11) + + with Submitter(worker=plugin, cache_dir=tmpdir) as sub: + results = sub(wf) + + assert not results.errored, "\n".join(results.errors["error message"]) - results = wf.result() assert len(results) == 3 - assert results[0].output.out == 39 - assert results[1].output.out == 52 - assert results[2].output.out == 65 - # checking all directories - assert wf.output_dir - for odir in wf.output_dir: - assert odir.exists() + assert results.outputs.out[0] == 39 + assert results.outputs.out[1] == 52 + assert results.outputs.out[2] == 65 def test_wf_ndst_singl_2(plugin, tmpdir): @@ -2025,23 +1877,23 @@ def test_wf_ndst_singl_2(plugin, tmpdir): splitter on the tasks levels only one input is part of the splitter, the other is a single value """ - wf = Workflow(name="wf_ndst_6", input_spec=["x", "y"]) - wf.add(Add2(name="add2x").split("x", x=wf.lzin.x)) - wf.add(Add2(name="add2y", x=wf.lzin.y)) - wf.add(Multiply(name="mult", x=wf.add2x.lzout.out, y=wf.add2y.lzout.out)) - wf.inputs.x = [1, 2, 3] - wf.inputs.y = 11 - wf.set_output([("out", wf.mult.lzout.out)]) - wf.cache_dir = tmpdir - with Submitter(worker=plugin) as sub: - sub(wf) + @workflow.define + def Workflow(x, y): + add2x = workflow.add(Add2().split("x", x=x), name="add2x") + add2y = workflow.add(Add2(x=y), name="add2y") + mult = workflow.add(Multiply(x=add2x.out, y=add2y.out)) + return mult.out + + wf = Workflow(x=[1, 2, 3], y=11) + + with Submitter(worker=plugin, cache_dir=tmpdir) as sub: + results = sub(wf) + + assert not results.errored, "\n".join(results.errors["error message"]) - results = wf.result() - assert len(results.output.out) == 3 - assert results.output.out == [39, 52, 65] - # checking the output directory - assert wf.output_dir.exists() + assert len(results.outputs.out) == 3 + assert results.outputs.out == [39, 52, 65] # workflows with structures wf(A) @@ -2051,23 +1903,25 @@ def test_wfasnd_1(plugin, tmpdir): """workflow as a node workflow-node with one task and no splitter """ - wfnd = Workflow(name="wfnd", input_spec=["x"]) - wfnd.add(Add2(name="add2", x=wfnd.lzin.x)) - wfnd.set_output([("out", wfnd.add2.lzout.out)]) - wfnd.inputs.x = 2 - wf = Workflow(name="wf", input_spec=["x"]) - wf.add(wfnd) - wf.set_output([("out", wf.wfnd.lzout.out)]) - wf.cache_dir = tmpdir + @workflow.define + def Wfnd(x): + add2 = workflow.add(Add2(x=x)) + return add2.out - with Submitter(worker=plugin) as sub: - sub(wf) + @workflow.define + def Workflow(x): + wfnd = workflow.add(Wfnd(x)) + return wfnd.out + + wf = Workflow(x=2) + + with Submitter(worker=plugin, cache_dir=tmpdir) as sub: + results = sub(wf) + + assert not results.errored, "\n".join(results.errors["error message"]) - results = wf.result() - assert results.output.out == 4 - # checking the output directory - assert wf.output_dir.exists() + assert results.outputs.out == 4 def test_wfasnd_wfinp_1(plugin, tmpdir): @@ -2075,25 +1929,28 @@ def test_wfasnd_wfinp_1(plugin, tmpdir): workflow-node with one task and no splitter input set for the main workflow """ - wf = Workflow(name="wf", input_spec=["x"]) - wfnd = Workflow(name="wfnd", input_spec=["x"], x=wf.lzin.x) - wfnd.add(Add2(name="add2", x=wfnd.lzin.x)) - wfnd.set_output([("out", wfnd.add2.lzout.out)]) - wf.add(wfnd) - wf.inputs.x = 2 - wf.set_output([("out", wf.wfnd.lzout.out)]) - wf.cache_dir = tmpdir + @workflow.define + def Wfnd(x): + add2 = workflow.add(Add2(x=x)) + return add2.out - checksum_before = wf.checksum - with Submitter(worker=plugin) as sub: - sub(wf) + @workflow.define + def Workflow(x): + wfnd = workflow.add(Wfnd(x=x)) + return wfnd.out + + wf = Workflow(x=2) + + checksum_before = wf._checksum + with Submitter(worker=plugin, cache_dir=tmpdir) as sub: + results = sub(wf) + + assert not results.errored, "\n".join(results.errors["error message"]) - assert wf.checksum == checksum_before - results = wf.result() - assert results.output.out == 4 - # checking the output directory - assert wf.output_dir.exists() + assert wf._checksum == checksum_before + + assert results.outputs.out == 4 def test_wfasnd_wfndupdate(plugin, tmpdir): @@ -2102,22 +1959,24 @@ def test_wfasnd_wfndupdate(plugin, tmpdir): wfasnode input is updated to use the main workflow input """ - wfnd = Workflow(name="wfnd", input_spec=["x"], x=2) - wfnd.add(Add2(name="add2", x=wfnd.lzin.x)) - wfnd.set_output([("out", wfnd.add2.lzout.out)]) + @workflow.define + def Wfnd(x): + add2 = workflow.add(Add2(x=x)) + return add2.out - wf = Workflow(name="wf", input_spec=["x"], x=3) - wfnd.inputs.x = wf.lzin.x - wf.add(wfnd) - wf.set_output([("out", wf.wfnd.lzout.out)]) - wf.cache_dir = tmpdir + @workflow.define + def Workflow(x): + wfnd = workflow.add(Wfnd(x)) + return wfnd.out - with Submitter(worker=plugin) as sub: - sub(wf) + wf = Workflow(x=3) + + with Submitter(worker=plugin, cache_dir=tmpdir) as sub: + results = sub(wf) + + assert not results.errored, "\n".join(results.errors["error message"]) - results = wf.result() - assert results.output.out == 5 - assert wf.output_dir.exists() + assert results.outputs.out == 5 def test_wfasnd_wfndupdate_rerun(plugin, tmpdir): @@ -2127,42 +1986,44 @@ def test_wfasnd_wfndupdate_rerun(plugin, tmpdir): updated to use the main workflow input """ - wfnd = Workflow(name="wfnd", input_spec=["x"], x=2) - wfnd.add(Add2(name="add2", x=wfnd.lzin.x)) - wfnd.set_output([("out", wfnd.add2.lzout.out)]) - wfnd.cache_dir = tmpdir - with Submitter(worker=plugin) as sub: + @workflow.define + def Wfnd(x): + add2 = workflow.add(Add2(x=x)) + return add2.out + + wfnd = Wfnd(x=2) + + with Submitter(worker=plugin, cache_dir=tmpdir) as sub: sub(wfnd) - wf = Workflow(name="wf", input_spec=["x"], x=3) - # trying to set before - wfnd.inputs.x = wf.lzin.x - wf.add(wfnd) - # trying to set after add... - wf.wfnd.inputs.x = wf.lzin.x - wf.set_output([("out", wf.wfnd.lzout.out)]) - wf.cache_dir = tmpdir + @workflow.define + def Workflow(x): + wfnd = workflow.add(Wfnd(x)) + return wfnd.out - with Submitter(worker=plugin) as sub: - sub(wf) + wf = Workflow(x=3) + + with Submitter(worker=plugin, cache_dir=tmpdir) as sub: + results = sub(wf) - results = wf.result() - assert results.output.out == 5 - assert wf.output_dir.exists() + assert not results.errored, "\n".join(results.errors["error message"]) + + assert results.outputs.out == 5 # adding another layer of workflow - wf_o = Workflow(name="wf_o", input_spec=["x"], x=4) - wf.inputs.x = wf_o.lzin.x - wf_o.add(wf) - wf_o.set_output([("out", wf_o.wf.lzout.out)]) - wf_o.cache_dir = tmpdir + @workflow.define + def WorkflowO(x): + wf = workflow.add(Workflow(x=3)) + return wf.out + + wf_o = WorkflowO(x=4) - with Submitter(worker=plugin) as sub: - sub(wf_o) + with Submitter(worker=plugin, cache_dir=tmpdir) as sub: + results = sub(wf_o) - results = wf_o.result() - assert results.output.out == 6 - assert wf_o.output_dir.exists() + assert not results.errored, "\n".join(results.errors["error message"]) + + assert results.outputs.out == 6 def test_wfasnd_st_1(plugin, tmpdir): @@ -2170,25 +2031,28 @@ def test_wfasnd_st_1(plugin, tmpdir): workflow-node with one task, splitter for wfnd """ - wfnd = Workflow(name="wfnd", input_spec=["x"]) - wfnd.add(Add2(name="add2", x=wfnd.lzin.x)) - wfnd.set_output([("out", wfnd.add2.lzout.out)]) - wfnd.split("x", x=[2, 4]) - wf = Workflow(name="wf", input_spec=["x"]) - wf.add(wfnd) - wf.set_output([("out", wf.wfnd.lzout.out)]) - wf.cache_dir = tmpdir + @workflow.define + def Wfnd(x): + add2 = workflow.add(Add2(x=x)) + return add2.out - checksum_before = wf.checksum - with Submitter(worker=plugin) as sub: - sub(wf) + @workflow.define + def Workflow(x): + wfnd = workflow.add(Wfnd(x=x).split(x=x)) + return wfnd.out + + wf = Workflow(x=[2, 4]) + + checksum_before = wf._checksum + with Submitter(worker=plugin, cache_dir=tmpdir) as sub: + results = sub(wf) + + assert not results.errored, "\n".join(results.errors["error message"]) + + assert wf._checksum == checksum_before - assert wf.checksum == checksum_before - results = wf.result() - assert results.output.out == [4, 6] - # checking the output directory - assert wf.output_dir.exists() + assert results.outputs.out == [4, 6] def test_wfasnd_st_updatespl_1(plugin, tmpdir): @@ -2196,23 +2060,25 @@ def test_wfasnd_st_updatespl_1(plugin, tmpdir): workflow-node with one task, splitter for wfnd is set after add """ - wfnd = Workflow(name="wfnd", input_spec=["x"]) - wfnd.add(Add2(name="add2", x=wfnd.lzin.x)) - wfnd.set_output([("out", wfnd.add2.lzout.out)]) - wf = Workflow(name="wf", input_spec=["x"]) - wf.add(wfnd) - wfnd.split("x", x=[2, 4]) - wf.set_output([("out", wf.wfnd.lzout.out)]) - wf.cache_dir = tmpdir + @workflow.define + def Wfnd(x): + add2 = workflow.add(Add2(x=x)) + return add2.out - with Submitter(worker=plugin) as sub: - sub(wf) + @workflow.define + def Workflow(x): + wfnd = workflow.add(Wfnd(x=x).split(x=x)) + return wfnd.out + + wf = Workflow(x=[2, 4]) + + with Submitter(worker=plugin, cache_dir=tmpdir) as sub: + results = sub(wf) + + assert not results.errored, "\n".join(results.errors["error message"]) - results = wf.result() - assert results.output.out == [4, 6] - # checking the output directory - assert wf.output_dir.exists() + assert results.outputs.out == [4, 6] def test_wfasnd_ndst_1(plugin, tmpdir): @@ -2220,25 +2086,25 @@ def test_wfasnd_ndst_1(plugin, tmpdir): workflow-node with one task, splitter for node """ - wfnd = Workflow(name="wfnd", input_spec=["x"]) - wfnd.add(Add2(name="add2").split("x", x=wfnd.lzin.x)) - wfnd.set_output([("out", wfnd.add2.lzout.out)]) - # TODO: without this the test is failing - wfnd.plugin = plugin - wfnd.inputs.x = [2, 4] - wf = Workflow(name="wf", input_spec=["x"]) - wf.add(wfnd) - wf.set_output([("out", wf.wfnd.lzout.out)]) - wf.cache_dir = tmpdir + @workflow.define + def Wfnd(x): + add2 = workflow.add(Add2().split("x", x=x)) + return add2.out - with Submitter(worker=plugin) as sub: - sub(wf) + @workflow.define + def Workflow(x): + wfnd = workflow.add(Wfnd(x=x)) + return wfnd.out + + wf = Workflow(x=[2, 4]) + + with Submitter(worker=plugin, cache_dir=tmpdir) as sub: + results = sub(wf) + + assert not results.errored, "\n".join(results.errors["error message"]) - results = wf.result() - assert results.output.out == [4, 6] - # checking the output directory - assert wf.output_dir.exists() + assert results.outputs.out == [4, 6] def test_wfasnd_ndst_updatespl_1(plugin, tmpdir): @@ -2246,23 +2112,25 @@ def test_wfasnd_ndst_updatespl_1(plugin, tmpdir): workflow-node with one task, splitter for node added after add """ - wfnd = Workflow(name="wfnd", input_spec=["x"]) - wfnd.add(Add2(name="add2", x=wfnd.lzin.x)) - wfnd.add2.split("x", x=[2, 4]) - wfnd.set_output([("out", wfnd.add2.lzout.out)]) - wf = Workflow(name="wf", input_spec=["x"]) - wf.add(wfnd) - wf.set_output([("out", wf.wfnd.lzout.out)]) - wf.cache_dir = tmpdir + @workflow.define + def Wfnd(x): + add2 = workflow.add(Add2().split("x", x=x)) + return add2.out - with Submitter(worker=plugin) as sub: - sub(wf) + @workflow.define + def Workflow(x): + wfnd = workflow.add(Wfnd(x=x)) + return wfnd.out + + wf = Workflow(x=[2, 4]) + + with Submitter(worker=plugin, cache_dir=tmpdir) as sub: + results = sub(wf) + + assert not results.errored, "\n".join(results.errors["error message"]) - results = wf.result() - assert results.output.out == [4, 6] - # checking the output directory - assert wf.output_dir.exists() + assert results.outputs.out == [4, 6] def test_wfasnd_wfst_1(plugin, tmpdir): @@ -2270,25 +2138,27 @@ def test_wfasnd_wfst_1(plugin, tmpdir): workflow-node with one task, splitter for the main workflow """ - wf = Workflow(name="wf", input_spec=["x"], cache_dir=tmpdir) - wfnd = Workflow(name="wfnd", input_spec=["x"], x=wf.lzin.x) - wfnd.add(Add2(name="add2", x=wfnd.lzin.x)) - wfnd.set_output([("out", wfnd.add2.lzout.out)]) - wf.add(wfnd) - wf.split("x", x=[2, 4]) - wf.set_output([("out", wf.wfnd.lzout.out)]) + @workflow.define + def Wfnd(x): + add2 = workflow.add(Add2(x=x)) + return add2.out - with Submitter(worker=plugin) as sub: - sub(wf) + @workflow.define + def Workflow(x): + wfnd = workflow.add(Wfnd(x=x)) + return wfnd.out + + wf = Workflow().split("x", x=[2, 4]) + + with Submitter(worker=plugin, cache_dir=tmpdir) as sub: + results = sub(wf) + + assert not results.errored, "\n".join(results.errors["error message"]) # assert wf.output_dir.exists() - results = wf.result() - assert results[0].output.out == 4 - assert results[1].output.out == 6 - # checking all directories - assert wf.output_dir - for odir in wf.output_dir: - assert odir.exists() + + assert results.outputs.out[0] == 4 + assert results.outputs.out[1] == 6 # workflows with structures wf(A) -> B @@ -2299,24 +2169,27 @@ def test_wfasnd_st_2(plugin, tmpdir): the main workflow has two tasks, splitter for wfnd """ - wfnd = Workflow(name="wfnd", input_spec=["x", "y"]) - wfnd.add(Multiply(name="mult", x=wfnd.lzin.x, y=wfnd.lzin.y)) - wfnd.set_output([("out", wfnd.mult.lzout.out)]) - wfnd.split(("x", "y"), x=[2, 4], y=[1, 10]) - wf = Workflow(name="wf_st_3", input_spec=["x", "y"]) - wf.add(wfnd) - wf.add(Add2(name="add2", x=wf.wfnd.lzout.out)) - wf.set_output([("out", wf.add2.lzout.out)]) - wf.cache_dir = tmpdir + @workflow.define + def Wfnd(x, y): + mult = workflow.add(Multiply().split(("x", "y"), x=x, y=y)) + return mult.out - with Submitter(worker=plugin) as sub: - sub(wf) + @workflow.define + def Workflow(x, y): + wfnd = workflow.add(Wfnd(x=x, y=y)) + add2 = workflow.add(Add2(x=wfnd.out)) + return add2.out + + wf = Workflow(x=[2, 4], y=[1, 10]) + + with Submitter(worker=plugin, cache_dir=tmpdir) as sub: + results = sub(wf) + + assert not results.errored, "\n".join(results.errors["error message"]) # assert wf.output_dir.exists() - results = wf.result() - assert results.output.out == [4, 42] - # checking the output directory - assert wf.output_dir.exists() + + assert results.outputs.out == [4, 42] def test_wfasnd_wfst_2(plugin, tmpdir): @@ -2324,27 +2197,28 @@ def test_wfasnd_wfst_2(plugin, tmpdir): the main workflow has two tasks, splitter for the main workflow """ - wf = Workflow(name="wf_st_3", input_spec=["x", "y"]) - wfnd = Workflow(name="wfnd", input_spec=["x", "y"], x=wf.lzin.x, y=wf.lzin.y) - wfnd.add(Multiply(name="mult", x=wfnd.lzin.x, y=wfnd.lzin.y)) - wfnd.set_output([("out", wfnd.mult.lzout.out)]) - wf.add(wfnd) - wf.add(Add2(name="add2", x=wf.wfnd.lzout.out)) - wf.split(("x", "y"), x=[2, 4], y=[1, 10]) - wf.set_output([("out", wf.add2.lzout.out)]) - wf.cache_dir = tmpdir + @workflow.define + def Wfnd(x, y): + mult = workflow.add(Multiply(x=x, y=y)) + return mult.out - with Submitter(worker=plugin) as sub: - sub(wf) + @workflow.define + def Workflow(x, y): + wfnd = workflow.add(Wfnd(x=x, y=y)) + add2 = workflow.add(Add2(x=wfnd.out)) + return add2.out + + wf = Workflow().split(("x", "y"), x=[2, 4], y=[1, 10]) + + with Submitter(worker=plugin, cache_dir=tmpdir) as sub: + results = sub(wf) + + assert not results.errored, "\n".join(results.errors["error message"]) # assert wf.output_dir.exists() - results = wf.result() - assert results[0].output.out == 4 - assert results[1].output.out == 42 - # checking all directories - assert wf.output_dir - for odir in wf.output_dir: - assert odir.exists() + + assert results.outputs.out[0] == 4 + assert results.outputs.out[1] == 42 # workflows with structures A -> wf(B) @@ -2355,26 +2229,27 @@ def test_wfasnd_ndst_3(plugin, tmpdir): the main workflow has two tasks, splitter for the first task """ - wf = Workflow(name="wf_st_3", input_spec=["x", "y"]) - wf.add(Multiply(name="mult").split(("x", "y"), x=wf.lzin.x, y=wf.lzin.y)) - wf.inputs.x = [2, 4] - wf.inputs.y = [1, 10] - wfnd = Workflow(name="wfnd", input_spec=["x"], x=wf.mult.lzout.out) - wfnd.add(Add2(name="add2", x=wfnd.lzin.x)) - wfnd.set_output([("out", wfnd.add2.lzout.out)]) - wf.add(wfnd) + @workflow.define + def Wfnd(x): + add2 = workflow.add(Add2(x=x)) + return add2.out - wf.set_output([("out", wf.wfnd.lzout.out)]) - wf.cache_dir = tmpdir + @workflow.define + def Workflow(x, y): + mult = workflow.add(Multiply().split(("x", "y"), x=x, y=y)) + wfnd = workflow.add(Wfnd(mult.out)) + return wfnd.out - with Submitter(worker="serial") as sub: - sub(wf) + wf = Workflow(x=[2, 4], y=[1, 10]) + + with Submitter(cache_dir=tmpdir) as sub: + results = sub(wf) + + assert not results.errored, "\n".join(results.errors["error message"]) # assert wf.output_dir.exists() - results = wf.result() - assert results.output.out == [4, 42] - # checking the output directory - assert wf.output_dir.exists() + + assert results.outputs.out == [4, 42] def test_wfasnd_wfst_3(plugin, tmpdir): @@ -2382,29 +2257,30 @@ def test_wfasnd_wfst_3(plugin, tmpdir): the main workflow has two tasks, splitter for the main workflow """ - wf = Workflow(name="wf_st_3", input_spec=["x", "y"]) - wf.add(Multiply(name="mult", x=wf.lzin.x, y=wf.lzin.y)) - wf.split(("x", "y"), x=[2, 4], y=[1, 10]) - wfnd = Workflow(name="wfnd", input_spec=["x"], x=wf.mult.lzout.out) - wfnd.add(Add2(name="add2", x=wfnd.lzin.x)) - wfnd.set_output([("out", wfnd.add2.lzout.out)]) - wf.add(wfnd) + @workflow.define + def Wfnd(x): + add2 = workflow.add(Add2(x=x)) + return add2.out - wf.set_output([("out", wf.wfnd.lzout.out)]) - wf.plugin = plugin - wf.cache_dir = tmpdir + @workflow.define + def Workflow(x, y): + mult = workflow.add(Multiply(x=x, y=y)) - with Submitter(worker=plugin) as sub: - sub(wf) + wfnd = workflow.add(Wfnd(mult.out)) + + return wfnd.out + + wf = Workflow().split(("x", "y"), x=[2, 4], y=[1, 10]) + + with Submitter(worker=plugin, cache_dir=tmpdir) as sub: + results = sub(wf) + + assert not results.errored, "\n".join(results.errors["error message"]) # assert wf.output_dir.exists() - results = wf.result() - assert results[0].output.out == 4 - assert results[1].output.out == 42 - # checking all directories - assert wf.output_dir - for odir in wf.output_dir: - assert odir.exists() + + assert results.outputs.out[0] == 4 + assert results.outputs.out[1] == 42 # workflows with structures wfns(A->B) @@ -2414,24 +2290,26 @@ def test_wfasnd_4(plugin, tmpdir): """workflow as a node workflow-node with two tasks and no splitter """ - wfnd = Workflow(name="wfnd", input_spec=["x"]) - wfnd.add(Add2(name="add2_1st", x=wfnd.lzin.x)) - wfnd.add(Add2(name="add2_2nd", x=wfnd.add2_1st.lzout.out)) - wfnd.set_output([("out", wfnd.add2_2nd.lzout.out)]) - wfnd.inputs.x = 2 - wf = Workflow(name="wf", input_spec=["x"]) - wf.add(wfnd) - wf.set_output([("out", wf.wfnd.lzout.out)]) - wf.cache_dir = tmpdir + @workflow.define + def Wfnd(x): + add2_1st = workflow.add(Add2(x=x), name="add2_1st") + add2_2nd = workflow.add(Add2(x=add2_1st.out), name="add2_2nd") + return add2_2nd.out - with Submitter(worker=plugin) as sub: - sub(wf) + @workflow.define + def Workflow(x): + wfnd = workflow.add(Wfnd(x=2)) + return wfnd.out - results = wf.result() - assert results.output.out == 6 - # checking the output directory - assert wf.output_dir.exists() + wf = Workflow(x=2) + + with Submitter(worker=plugin, cache_dir=tmpdir) as sub: + results = sub(wf) + + assert not results.errored, "\n".join(results.errors["error message"]) + + assert results.outputs.out == 6 def test_wfasnd_ndst_4(plugin, tmpdir): @@ -2439,24 +2317,26 @@ def test_wfasnd_ndst_4(plugin, tmpdir): workflow-node with two tasks, splitter for node """ - wfnd = Workflow(name="wfnd", input_spec=["x"]) - wfnd.add(Add2(name="add2_1st").split("x", x=wfnd.lzin.x)) - wfnd.add(Add2(name="add2_2nd", x=wfnd.add2_1st.lzout.out)) - wfnd.set_output([("out", wfnd.add2_2nd.lzout.out)]) - wfnd.inputs.x = [2, 4] - wf = Workflow(name="wf", input_spec=["x"]) - wf.add(wfnd) - wf.set_output([("out", wf.wfnd.lzout.out)]) - wf.cache_dir = tmpdir + @workflow.define + def Wfnd(x): + add2_1st = workflow.add(Add2().split(x=x), name="add2_1st") + add2_2nd = workflow.add(Add2(x=add2_1st.out), name="add2_2nd") + return add2_2nd.out - with Submitter(worker=plugin) as sub: - sub(wf) + @workflow.define + def Workflow(x): + wfnd = workflow.add(Wfnd(x=x)) + return wfnd.out - results = wf.result() - assert results.output.out == [6, 8] - # checking the output directory - assert wf.output_dir.exists() + wf = Workflow(x=[2, 4]) + + with Submitter(worker=plugin, cache_dir=tmpdir) as sub: + results = sub(wf) + + assert not results.errored, "\n".join(results.errors["error message"]) + + assert results.outputs.out == [6, 8] def test_wfasnd_wfst_4(plugin, tmpdir): @@ -2464,26 +2344,28 @@ def test_wfasnd_wfst_4(plugin, tmpdir): workflow-node with two tasks, splitter for the main workflow """ - wf = Workflow(name="wf", input_spec=["x"], cache_dir=tmpdir) - wfnd = Workflow(name="wfnd", input_spec=["x"], x=wf.lzin.x) - wfnd.add(Add2(name="add2_1st", x=wfnd.lzin.x)) - wfnd.add(Add2(name="add2_2nd", x=wfnd.add2_1st.lzout.out)) - wfnd.set_output([("out", wfnd.add2_2nd.lzout.out)]) - wf.add(wfnd) - wf.split("x", x=[2, 4]) - wf.set_output([("out", wf.wfnd.lzout.out)]) + @workflow.define + def Wfnd(x): + add2_1st = workflow.add(Add2(x=x), name="add2_1st") + add2_2nd = workflow.add(Add2(x=add2_1st.out), name="add2_2nd") + return add2_2nd.out - with Submitter(worker=plugin) as sub: - sub(wf) + @workflow.define + def Workflow(x): + wfnd = workflow.add(Wfnd(x=x)) + return wfnd.out + + wf = Workflow().split("x", x=[2, 4]) + + with Submitter(worker=plugin, cache_dir=tmpdir) as sub: + results = sub(wf) + + assert not results.errored, "\n".join(results.errors["error message"]) # assert wf.output_dir.exists() - results = wf.result() - assert results[0].output.out == 6 - assert results[1].output.out == 8 - # checking all directories - assert wf.output_dir - for odir in wf.output_dir: - assert odir.exists() + + assert results.outputs.out[0] == 6 + assert results.outputs.out[1] == 8 # Testing caching @@ -2494,19 +2376,20 @@ def test_wf_nostate_cachedir(plugin, tmpdir): """wf with provided cache_dir using pytest tmpdir""" cache_dir = tmpdir.mkdir("test_wf_cache_1") - wf = Workflow(name="wf_2", input_spec=["x", "y"], cache_dir=cache_dir) - wf.add(Multiply(name="mult", x=wf.lzin.x, y=wf.lzin.y)) - wf.add(Add2(name="add2", x=wf.mult.lzout.out)) - wf.set_output([("out", wf.add2.lzout.out)]) - wf.inputs.x = 2 - wf.inputs.y = 3 + @workflow.define + def Workflow(x, y): + mult = workflow.add(Multiply(x=x, y=y)) + add2 = workflow.add(Add2(x=mult.out)) + return add2.out - with Submitter(worker=plugin) as sub: - sub(wf) + wf = Workflow(x=2, y=3) + + with Submitter(worker=plugin, cache_dir=tmpdir) as sub: + results = sub(wf) - assert wf.output_dir.exists() - results = wf.result() - assert 8 == results.output.out + assert not results.errored, "\n".join(results.errors["error message"]) + + assert 8 == results.outputs.out shutil.rmtree(cache_dir) @@ -2518,19 +2401,20 @@ def test_wf_nostate_cachedir_relativepath(tmpdir, plugin): cache_dir = "test_wf_cache_2" tmpdir.mkdir(cache_dir) - wf = Workflow(name="wf_2", input_spec=["x", "y"], cache_dir=cache_dir) - wf.add(Multiply(name="mult", x=wf.lzin.x, y=wf.lzin.y)) - wf.add(Add2(name="add2", x=wf.mult.lzout.out)) - wf.set_output([("out", wf.add2.lzout.out)]) - wf.inputs.x = 2 - wf.inputs.y = 3 + @workflow.define + def Workflow(x, y): + mult = workflow.add(Multiply(x=x, y=y)) + add2 = workflow.add(Add2(x=mult.out)) + return add2.out - with Submitter(worker=plugin) as sub: - sub(wf) + wf = Workflow(x=2, y=3) + + with Submitter(worker=plugin, cache_dir=tmpdir) as sub: + results = sub(wf) + + assert not results.errored, "\n".join(results.errors["error message"]) - assert wf.output_dir.exists() - results = wf.result() - assert 8 == results.output.out + assert 8 == results.outputs.out shutil.rmtree(cache_dir) @@ -2544,40 +2428,42 @@ def test_wf_nostate_cachelocations(plugin, tmpdir): cache_dir1 = tmpdir.mkdir("test_wf_cache3") cache_dir2 = tmpdir.mkdir("test_wf_cache4") - wf1 = Workflow(name="wf", input_spec=["x", "y"], cache_dir=cache_dir1) - wf1.add(Multiply(name="mult", x=wf1.lzin.x, y=wf1.lzin.y)) - wf1.add(Add2Wait(name="add2", x=wf1.mult.lzout.out)) - wf1.set_output([("out", wf1.add2.lzout.out)]) - wf1.inputs.x = 2 - wf1.inputs.y = 3 + @workflow.define + def Workflow1(x, y): + mult = workflow.add(Multiply(x=x, y=y)) + add2 = workflow.add(Add2Wait(x=mult.out)) + return add2.out + + wf1 = Workflow1(x=2, y=3) t0 = time.time() - with Submitter(worker=plugin) as sub: - sub(wf1) + with Submitter(worker=plugin, cache_dir=cache_dir1) as sub: + results1 = sub(wf1) + + assert not results1.errored, "\n".join(results1.errors["error message"]) t1 = time.time() - t0 - results1 = wf1.result() - assert 8 == results1.output.out + assert 8 == results1.outputs.out - wf2 = Workflow( - name="wf", - input_spec=["x", "y"], - cache_dir=cache_dir2, - cache_locations=cache_dir1, - ) - wf2.add(Multiply(name="mult", x=wf2.lzin.x, y=wf2.lzin.y)) - wf2.add(Add2Wait(name="add2", x=wf2.mult.lzout.out)) - wf2.set_output([("out", wf2.add2.lzout.out)]) - wf2.inputs.x = 2 - wf2.inputs.y = 3 + @workflow.define + def Workflow2(x, y): + + mult = workflow.add(Multiply(x=x, y=y)) + add2 = workflow.add(Add2Wait(x=mult.out)) + return add2.out + + wf2 = Workflow2(x=2, y=3) t0 = time.time() - with Submitter(worker=plugin) as sub: - sub(wf2) + with Submitter( + worker=plugin, cache_dir=cache_dir2, cache_locations=cache_dir1 + ) as sub: + results2 = sub(wf2) + + assert not results2.errored, "\n".join(results2.errors["error message"]) t2 = time.time() - t0 - results2 = wf2.result() - assert 8 == results2.output.out + assert 8 == results2.outputs.out # checking execution time (for unix and cf) # for win and dask/slurm the time for dir creation etc. might take much longer @@ -2600,42 +2486,42 @@ def test_wf_nostate_cachelocations_a(plugin, tmpdir): cache_dir1 = tmpdir.mkdir("test_wf_cache3") cache_dir2 = tmpdir.mkdir("test_wf_cache4") - wf1 = Workflow(name="wf1", input_spec=["x", "y"], cache_dir=cache_dir1) - wf1.add(Multiply(name="mult", x=wf1.lzin.x, y=wf1.lzin.y)) - wf1.add(Add2Wait(name="add2", x=wf1.mult.lzout.out)) - wf1.set_output([("out", wf1.add2.lzout.out)]) - wf1.inputs.x = 2 - wf1.inputs.y = 3 - wf1.plugin = plugin + @workflow.define + def Workflow1(x, y): + mult = workflow.add(Multiply(x=x, y=y)) + add2 = workflow.add(Add2Wait(x=mult.out)) + return add2.out + + wf1 = Workflow1(x=2, y=3) t0 = time.time() - with Submitter(worker=plugin) as sub: - sub(wf1) + with Submitter(worker=plugin, cache_dir=cache_dir1) as sub: + results1 = sub(wf1) + + assert not results1.errored, "\n".join(results1.errors["error message"]) t1 = time.time() - t0 - results1 = wf1.result() - assert 8 == results1.output.out + assert 8 == results1.outputs.out - wf2 = Workflow( - name="wf2", - input_spec=["x", "y"], - cache_dir=cache_dir2, - cache_locations=cache_dir1, - ) - wf2.add(Multiply(name="mult", x=wf2.lzin.x, y=wf2.lzin.y)) - wf2.add(Add2Wait(name="add2", x=wf2.mult.lzout.out)) - wf2.set_output([("out", wf2.add2.lzout.out)]) - wf2.inputs.x = 2 - wf2.inputs.y = 3 - wf2.plugin = plugin + @workflow.define + def Workflow2(x, y): + + mult = workflow.add(Multiply(x=x, y=y)) + add2 = workflow.add(Add2Wait(x=mult.out)) + return add2.out + + wf2 = Workflow2(x=2, y=3) t0 = time.time() - with Submitter(worker=plugin) as sub: - sub(wf2) + with Submitter( + worker=plugin, cache_dir=cache_dir2, cache_locations=cache_dir1 + ) as sub: + results2 = sub(wf2) + + assert not results2.errored, "\n".join(results2.errors["error message"]) t2 = time.time() - t0 - results2 = wf2.result() - assert 8 == results2.output.out + assert 8 == results2.outputs.out # for win and dask/slurm the time for dir creation etc. might take much longer if not sys.platform.startswith("win") and plugin == "cf": @@ -2645,8 +2531,7 @@ def test_wf_nostate_cachelocations_a(plugin, tmpdir): assert t2 < max(1, t1 - 1) # checking if both wf.output_dir are created - assert wf1.output_dir.exists() - assert wf2.output_dir.exists() + assert results1.output_dir != results2.output_dir @pytest.mark.flaky(reruns=3) @@ -2660,44 +2545,42 @@ def test_wf_nostate_cachelocations_b(plugin, tmpdir): cache_dir1 = tmpdir.mkdir("test_wf_cache3") cache_dir2 = tmpdir.mkdir("test_wf_cache4") - wf1 = Workflow(name="wf", input_spec=["x", "y"], cache_dir=cache_dir1) - wf1.add(Multiply(name="mult", x=wf1.lzin.x, y=wf1.lzin.y)) - wf1.add(Add2Wait(name="add2", x=wf1.mult.lzout.out)) - wf1.set_output([("out", wf1.add2.lzout.out)]) - wf1.inputs.x = 2 - wf1.inputs.y = 3 - wf1.plugin = plugin + @workflow.define + def Workflow1(x, y): + mult = workflow.add(Multiply(x=x, y=y)) + add2 = workflow.add(Add2Wait(x=mult.out)) + return add2.out + + wf1 = Workflow1(x=2, y=3) t0 = time.time() - with Submitter(worker=plugin) as sub: - sub(wf1) + with Submitter(worker=plugin, cache_dir=cache_dir1) as sub: + results1 = sub(wf1) + + assert not results1.errored, "\n".join(results1.errors["error message"]) t1 = time.time() - t0 - results1 = wf1.result() - assert 8 == results1.output.out + assert 8 == results1.outputs.out - wf2 = Workflow( - name="wf", - input_spec=["x", "y"], - cache_dir=cache_dir2, - cache_locations=cache_dir1, - ) - wf2.add(Multiply(name="mult", x=wf2.lzin.x, y=wf2.lzin.y)) - wf2.add(Add2Wait(name="add2", x=wf2.mult.lzout.out)) - wf2.set_output([("out", wf2.add2.lzout.out)]) - # additional output - wf2.set_output([("out_pr", wf2.add2.lzout.out)]) - wf2.inputs.x = 2 - wf2.inputs.y = 3 - wf2.plugin = plugin + @workflow.define("out_pr") + def Workflow2(x, y): + + mult = workflow.add(Multiply(x=x, y=y)) + add2 = workflow.add(Add2Wait(x=mult.out)) + return add2.out + + wf2 = Workflow2(x=2, y=3) t0 = time.time() - with Submitter(worker=plugin) as sub: - sub(wf2) + with Submitter( + worker=plugin, cache_dir=cache_dir2, cache_locations=cache_dir1 + ) as sub: + results2 = sub(wf2) + + assert not results2.errored, "\n".join(results2.errors["error message"]) t2 = time.time() - t0 - results2 = wf2.result() - assert 8 == results2.output.out == results2.output.out_pr + assert 8 == results2.outputs.out == results2.outputs.out_pr # for win and dask/slurm the time for dir creation etc. might take much longer if not sys.platform.startswith("win") and plugin == "cf": @@ -2706,8 +2589,7 @@ def test_wf_nostate_cachelocations_b(plugin, tmpdir): assert t2 < max(1, t1 - 1) # checking if the second wf didn't run again - assert wf1.output_dir.exists() - assert wf2.output_dir.exists() + assert results1.output_dir != results2.output_dir @pytest.mark.flaky(reruns=3) @@ -2721,42 +2603,41 @@ def test_wf_nostate_cachelocations_setoutputchange(plugin, tmpdir): cache_dir1 = tmpdir.mkdir("test_wf_cache3") cache_dir2 = tmpdir.mkdir("test_wf_cache4") - wf1 = Workflow(name="wf", input_spec=["x", "y"], cache_dir=cache_dir1) - wf1.add(Multiply(name="mult", x=wf1.lzin.x, y=wf1.lzin.y)) - wf1.add(Add2Wait(name="add2", x=wf1.mult.lzout.out)) - wf1.set_output([("out1", wf1.add2.lzout.out)]) - wf1.inputs.x = 2 - wf1.inputs.y = 3 - wf1.plugin = plugin + @workflow.define(outputs=["out1"]) + def Workflow1(x, y): + mult = workflow.add(Multiply(x=x, y=y)) + add2 = workflow.add(Add2Wait(x=mult.out)) + return add2.out # out1 + + wf1 = Workflow1(x=2, y=3) t0 = time.time() - with Submitter(worker=plugin) as sub: - sub(wf1) + with Submitter(worker=plugin, cache_dir=cache_dir1) as sub: + results1 = sub(wf1) + + assert not results1.errored, "\n".join(results1.errors["error message"]) t1 = time.time() - t0 - results1 = wf1.result() - assert 8 == results1.output.out1 + assert 8 == results1.outputs.out1 - wf2 = Workflow( - name="wf", - input_spec=["x", "y"], - cache_dir=cache_dir2, - cache_locations=cache_dir1, - ) - wf2.add(Multiply(name="mult", x=wf2.lzin.x, y=wf2.lzin.y)) - wf2.add(Add2Wait(name="add2", x=wf2.mult.lzout.out)) - wf2.set_output([("out2", wf2.add2.lzout.out)]) - wf2.inputs.x = 2 - wf2.inputs.y = 3 - wf2.plugin = plugin + @workflow.define(outputs=["out2"]) + def Workflow2(x, y): + mult = workflow.add(Multiply(x=x, y=y)) + add2 = workflow.add(Add2Wait(x=mult.out)) + return add2.out # out2 + + wf2 = Workflow2(x=2, y=3) t0 = time.time() - with Submitter(worker=plugin) as sub: - sub(wf2) + with Submitter( + worker=plugin, cache_dir=cache_dir2, cache_locations=cache_dir1 + ) as sub: + results2 = sub(wf2) + + assert not results2.errored, "\n".join(results2.errors["error message"]) t2 = time.time() - t0 - results2 = wf2.result() - assert 8 == results2.output.out2 + assert 8 == results2.outputs.out2 # for win and dask/slurm the time for dir creation etc. might take much longer if not sys.platform.startswith("win") and plugin == "cf": @@ -2766,8 +2647,7 @@ def test_wf_nostate_cachelocations_setoutputchange(plugin, tmpdir): assert t2 < max(1, t1 - 1) # both wf output_dirs should be created - assert wf1.output_dir.exists() - assert wf2.output_dir.exists() + assert results1.output_dir != results2.output_dir @pytest.mark.flaky(reruns=3) @@ -2778,42 +2658,41 @@ def test_wf_nostate_cachelocations_setoutputchange_a(plugin, tmpdir): cache_dir1 = tmpdir.mkdir("test_wf_cache3") cache_dir2 = tmpdir.mkdir("test_wf_cache4") - wf1 = Workflow(name="wf1", input_spec=["x", "y"], cache_dir=cache_dir1) - wf1.add(Multiply(name="mult", x=wf1.lzin.x, y=wf1.lzin.y)) - wf1.add(Add2Wait(name="add2", x=wf1.mult.lzout.out)) - wf1.set_output([("out1", wf1.add2.lzout.out)]) - wf1.inputs.x = 2 - wf1.inputs.y = 3 - wf1.plugin = plugin + @workflow.define(outputs=["out1"]) + def Workflow1(x, y): + mult = workflow.add(Multiply(x=x, y=y)) + add2 = workflow.add(Add2Wait(x=mult.out)) + return add2.out # out1 + + wf1 = Workflow1(x=2, y=3) t0 = time.time() - with Submitter(worker=plugin) as sub: - sub(wf1) + with Submitter(worker=plugin, cache_dir=cache_dir1) as sub: + results1 = sub(wf1) + + assert not results1.errored, "\n".join(results1.errors["error message"]) t1 = time.time() - t0 - results1 = wf1.result() - assert 8 == results1.output.out1 + assert 8 == results1.outputs.out1 - wf2 = Workflow( - name="wf2", - input_spec=["x", "y"], - cache_dir=cache_dir2, - cache_locations=cache_dir1, - ) - wf2.add(Multiply(name="mult", x=wf2.lzin.x, y=wf2.lzin.y)) - wf2.add(Add2Wait(name="add2", x=wf2.mult.lzout.out)) - wf2.set_output([("out2", wf2.add2.lzout.out)]) - wf2.inputs.x = 2 - wf2.inputs.y = 3 - wf2.plugin = plugin + @workflow.define(outputs=["out2"]) + def Workflow2(x, y): + mult = workflow.add(Multiply(x=x, y=y)) + add2 = workflow.add(Add2Wait(x=mult.out)) + return add2.out + + wf2 = Workflow2(x=2, y=3) t0 = time.time() - with Submitter(worker=plugin) as sub: - sub(wf2) + with Submitter( + worker=plugin, cache_dir=cache_dir2, cache_locations=cache_dir1 + ) as sub: + results2 = sub(wf2) + + assert not results2.errored, "\n".join(results2.errors["error message"]) t2 = time.time() - t0 - results2 = wf2.result() - assert 8 == results2.output.out2 + assert 8 == results2.outputs.out2 # for win and dask/slurm the time for dir creation etc. might take much longer if not sys.platform.startswith("win") and plugin == "cf": @@ -2822,8 +2701,7 @@ def test_wf_nostate_cachelocations_setoutputchange_a(plugin, tmpdir): assert t2 < max(1, t1 - 1) # both wf output_dirs should be created - assert wf1.output_dir.exists() - assert wf2.output_dir.exists() + assert results1.output_dir != results2.output_dir @pytest.mark.flaky(reruns=3) @@ -2836,42 +2714,40 @@ def test_wf_nostate_cachelocations_forcererun(plugin, tmpdir): cache_dir1 = tmpdir.mkdir("test_wf_cache3") cache_dir2 = tmpdir.mkdir("test_wf_cache4") - wf1 = Workflow(name="wf", input_spec=["x", "y"], cache_dir=cache_dir1) - wf1.add(Multiply(name="mult", x=wf1.lzin.x, y=wf1.lzin.y)) - wf1.add(Add2Wait(name="add2", x=wf1.mult.lzout.out)) - wf1.set_output([("out", wf1.add2.lzout.out)]) - wf1.inputs.x = 2 - wf1.inputs.y = 3 - wf1.plugin = plugin + @workflow.define + def Workflow1(x, y): + mult = workflow.add(Multiply(x=x, y=y)) + add2 = workflow.add(Add2Wait(x=mult.out)) + return add2.out + + wf1 = Workflow1(x=2, y=3) t0 = time.time() - with Submitter(worker=plugin) as sub: - sub(wf1) + with Submitter(worker=plugin, cache_dir=cache_dir1) as sub: + results1 = sub(wf1) + + assert not results1.errored, "\n".join(results1.errors["error message"]) t1 = time.time() - t0 - results1 = wf1.result() - assert 8 == results1.output.out + assert 8 == results1.outputs.out - wf2 = Workflow( - name="wf", - input_spec=["x", "y"], - cache_dir=cache_dir2, - cache_locations=cache_dir1, - ) - wf2.add(Multiply(name="mult", x=wf2.lzin.x, y=wf2.lzin.y)) - wf2.add(Add2Wait(name="add2", x=wf2.mult.lzout.out)) - wf2.set_output([("out", wf2.add2.lzout.out)]) - wf2.inputs.x = 2 - wf2.inputs.y = 3 - wf2.plugin = plugin + @workflow.define + def Workflow2(x, y): + + mult = workflow.add(Multiply(x=x, y=y)) + add2 = workflow.add(Add2Wait(x=mult.out)) + return add2.out + + wf2 = Workflow2(x=2, y=3) t0 = time.time() - with Submitter(worker=plugin) as sub: - sub(wf2, rerun=True) + with Submitter(worker=plugin, cache_dir=cache_dir2) as sub: + results2 = sub(wf2, rerun=True) + + assert not results2.errored, "\n".join(results2.errors["error message"]) t2 = time.time() - t0 - results2 = wf2.result() - assert 8 == results2.output.out + assert 8 == results2.outputs.out # for win and dask/slurm the time for dir creation etc. might take much longer if not sys.platform.startswith("win") and plugin == "cf": @@ -2880,8 +2756,7 @@ def test_wf_nostate_cachelocations_forcererun(plugin, tmpdir): assert t2 > 2 # checking if the second wf didn't run again - assert wf1.output_dir.exists() - assert wf2.output_dir.exists() + assert results1.output_dir != results2.output_dir @pytest.mark.flaky(reruns=3) @@ -2894,47 +2769,44 @@ def test_wf_nostate_cachelocations_wftaskrerun_propagateTrue(plugin, tmpdir): cache_dir1 = tmpdir.mkdir("test_wf_cache3") cache_dir2 = tmpdir.mkdir("test_wf_cache4") - wf1 = Workflow(name="wf", input_spec=["x", "y"], cache_dir=cache_dir1) - wf1.add(Multiply(name="mult", x=wf1.lzin.x, y=wf1.lzin.y)) - wf1.add(Add2Wait(name="add2", x=wf1.mult.lzout.out)) - wf1.set_output([("out", wf1.add2.lzout.out)]) - wf1.inputs.x = 2 - wf1.inputs.y = 3 - wf1.plugin = plugin + @workflow.define + def Workflow1(x, y): + mult = workflow.add(Multiply(x=x, y=y)) + add2 = workflow.add(Add2Wait(x=mult.out)) + return add2.out + + wf1 = Workflow1(x=2, y=3) t0 = time.time() - with Submitter(worker=plugin) as sub: - sub(wf1) + with Submitter(worker=plugin, cache_dir=cache_dir1) as sub: + results1 = sub(wf1) + + assert not results1.errored, "\n".join(results1.errors["error message"]) t1 = time.time() - t0 - results1 = wf1.result() - assert 8 == results1.output.out + assert 8 == results1.outputs.out - wf2 = Workflow( - name="wf", - input_spec=["x", "y"], - cache_dir=cache_dir2, - cache_locations=cache_dir1, - rerun=True, # wh has to be rerun (default for propagate_rerun is True) - ) - wf2.add(Multiply(name="mult", x=wf2.lzin.x, y=wf2.lzin.y)) - wf2.add(Add2Wait(name="add2", x=wf2.mult.lzout.out)) - wf2.set_output([("out", wf2.add2.lzout.out)]) - wf2.inputs.x = 2 - wf2.inputs.y = 3 - wf2.plugin = plugin + @workflow.define + def Workflow2(x, y): + mult = workflow.add(Multiply(x=x, y=y)) + add2 = workflow.add(Add2Wait(x=mult.out)) + return add2.out + + wf2 = Workflow2(x=2, y=3) t0 = time.time() - with Submitter(worker=plugin) as sub: - sub(wf2) + with Submitter( + worker=plugin, cache_dir=cache_dir2, cache_locations=cache_dir1 + ) as sub: + results2 = sub(wf2, rerun=True) + + assert not results2.errored, "\n".join(results2.errors["error message"]) t2 = time.time() - t0 - results2 = wf2.result() - assert 8 == results2.output.out + assert 8 == results2.outputs.out # checking if the second wf runs again - assert wf1.output_dir.exists() - assert wf2.output_dir.exists() + assert results1.output_dir != results2.output_dir # everything has to be recomputed assert len(list(Path(cache_dir1).glob("F*"))) == 2 @@ -2957,48 +2829,44 @@ def test_wf_nostate_cachelocations_wftaskrerun_propagateFalse(plugin, tmpdir): cache_dir1 = tmpdir.mkdir("test_wf_cache3") cache_dir2 = tmpdir.mkdir("test_wf_cache4") - wf1 = Workflow(name="wf", input_spec=["x", "y"], cache_dir=cache_dir1) - wf1.add(Multiply(name="mult", x=wf1.lzin.x, y=wf1.lzin.y)) - wf1.add(Add2Wait(name="add2", x=wf1.mult.lzout.out)) - wf1.set_output([("out", wf1.add2.lzout.out)]) - wf1.inputs.x = 2 - wf1.inputs.y = 3 - wf1.plugin = plugin + @workflow.define + def Workflow1(x, y): + mult = workflow.add(Multiply(x=x, y=y)) + add2 = workflow.add(Add2Wait(x=mult.out)) + return add2.out + + wf1 = Workflow1(x=2, y=3) t0 = time.time() - with Submitter(worker=plugin) as sub: - sub(wf1) + with Submitter(worker=plugin, cache_dir=cache_dir1) as sub: + results1 = sub(wf1) + + assert not results1.errored, "\n".join(results1.errors["error message"]) t1 = time.time() - t0 - results1 = wf1.result() - assert 8 == results1.output.out + assert 8 == results1.outputs.out - wf2 = Workflow( - name="wf", - input_spec=["x", "y"], - cache_dir=cache_dir2, - cache_locations=cache_dir1, - rerun=True, # wh has to be rerun - propagate_rerun=False, # but rerun doesn't propagate to the tasks - ) - wf2.add(Multiply(name="mult", x=wf2.lzin.x, y=wf2.lzin.y)) - wf2.add(Add2Wait(name="add2", x=wf2.mult.lzout.out)) - wf2.set_output([("out", wf2.add2.lzout.out)]) - wf2.inputs.x = 2 - wf2.inputs.y = 3 - wf2.plugin = plugin + @workflow.define + def Workflow2(x, y): + mult = workflow.add(Multiply(x=x, y=y)) + add2 = workflow.add(Add2Wait(x=mult.out)) + return add2.out + + wf2 = Workflow2(x=2, y=3) t0 = time.time() - with Submitter(worker=plugin) as sub: - sub(wf2) + with Submitter( + worker=plugin, cache_dir=cache_dir2, cache_locations=cache_dir1 + ) as sub: + results2 = sub(wf2, rerun=True, propagate_rerun=False) + + assert not results2.errored, "\n".join(results2.errors["error message"]) t2 = time.time() - t0 - results2 = wf2.result() - assert 8 == results2.output.out + assert 8 == results2.outputs.out # checking if the second wf runs again - assert wf1.output_dir.exists() - assert wf2.output_dir.exists() + assert results1.output_dir != results2.output_dir # for win and dask/slurm the time for dir creation etc. might take much longer if not sys.platform.startswith("win") and plugin == "cf": @@ -3021,48 +2889,44 @@ def test_wf_nostate_cachelocations_taskrerun_wfrerun_propagateFalse(plugin, tmpd cache_dir1 = tmpdir.mkdir("test_wf_cache3") cache_dir2 = tmpdir.mkdir("test_wf_cache4") - wf1 = Workflow(name="wf", input_spec=["x", "y"], cache_dir=cache_dir1) - wf1.add(Multiply(name="mult", x=wf1.lzin.x, y=wf1.lzin.y)) - wf1.add(Add2Wait(name="add2", x=wf1.mult.lzout.out)) - wf1.set_output([("out", wf1.add2.lzout.out)]) - wf1.inputs.x = 2 - wf1.inputs.y = 3 - wf1.plugin = plugin + @workflow.define + def Workflow1(x, y): + mult = workflow.add(Multiply(x=x, y=y)) + add2 = workflow.add(Add2Wait(x=mult.out)) + return add2.out + + wf1 = Workflow1(x=2, y=3) t0 = time.time() - with Submitter(worker=plugin) as sub: - sub(wf1) + with Submitter(worker=plugin, cache_dir=cache_dir1) as sub: + results1 = sub(wf1) + + assert not results1.errored, "\n".join(results1.errors["error message"]) t1 = time.time() - t0 - results1 = wf1.result() - assert 8 == results1.output.out + assert 8 == results1.outputs.out - wf2 = Workflow( - name="wf", - input_spec=["x", "y"], - cache_dir=cache_dir2, - cache_locations=cache_dir1, - rerun=True, - propagate_rerun=False, # rerun will not be propagated to each task - ) - wf2.add(Multiply(name="mult", x=wf2.lzin.x, y=wf2.lzin.y)) - # rerun on the task level needed (wf.propagate_rerun is False) - wf2.add(Add2Wait(name="add2", x=wf2.mult.lzout.out, rerun=True)) - wf2.set_output([("out", wf2.add2.lzout.out)]) - wf2.inputs.x = 2 - wf2.inputs.y = 3 - wf2.plugin = plugin + @workflow.define + def Workflow2(x, y): + mult = workflow.add(Multiply(x=x, y=y)) + # rerun on the task level needed (wf.propagate_rerun is False) + add2 = workflow.add(Add2Wait(x=mult.out, rerun=True)) + return add2.out + + wf2 = Workflow2(x=2, y=3) t0 = time.time() - with Submitter(worker=plugin) as sub: - sub(wf2) + with Submitter( + worker=plugin, cache_dir=cache_dir2, cache_locations=cache_dir1 + ) as sub: + results2 = sub( + wf2, rerun=True, propagate_rerun=False + ) # rerun will not be propagated to each task) t2 = time.time() - t0 - results2 = wf2.result() - assert 8 == results2.output.out + assert 8 == results2.outputs.out - assert wf1.output_dir.exists() - assert wf2.output_dir.exists() + assert results1.output_dir != results2.output_dir # the second task should be recomputed assert len(list(Path(cache_dir1).glob("F*"))) == 2 assert len(list(Path(cache_dir2).glob("F*"))) == 1 @@ -3084,40 +2948,41 @@ def test_wf_nostate_nodecachelocations(plugin, tmpdir): cache_dir1 = tmpdir.mkdir("test_wf_cache3") cache_dir2 = tmpdir.mkdir("test_wf_cache4") - wf1 = Workflow(name="wf", input_spec=["x"], cache_dir=cache_dir1) - wf1.add(Ten(name="ten", x=wf1.lzin.x)) - wf1.add(Add2(name="add2", x=wf1.ten.lzout.out)) - wf1.set_output([("out", wf1.add2.lzout.out)]) - wf1.inputs.x = 3 - wf1.plugin = plugin + @workflow.define + def Workflow1(x): + ten = workflow.add(Ten(x=x)) + add2 = workflow.add(Add2(x=ten.out)) + return add2.out - with Submitter(worker=plugin) as sub: - sub(wf1) + wf1 = Workflow1(x=3) - results1 = wf1.result() - assert 12 == results1.output.out + with Submitter(worker=plugin, cache_dir=cache_dir1) as sub: + results1 = sub(wf1) - wf2 = Workflow( - name="wf", - input_spec=["x", "y"], - cache_dir=cache_dir2, - cache_locations=cache_dir1, - ) - wf2.add(Ten(name="ten", x=wf2.lzin.x)) - wf2.add(Add2(name="add2", x=wf2.ten.lzout.out)) - wf2.set_output([("out", wf2.add2.lzout.out)]) - wf2.inputs.x = 2 - wf2.plugin = plugin + assert not results1.errored, "\n".join(results1.errors["error message"]) + + assert 12 == results1.outputs.out + + @workflow.define + def Workflow2(x, y): + + ten = workflow.add(Ten(x=x)) + add2 = workflow.add(Add2(x=ten.out)) + return add2.out - with Submitter(worker=plugin) as sub: - sub(wf2) + wf2 = Workflow2(x=2) - results2 = wf2.result() - assert 12 == results2.output.out + with Submitter( + worker=plugin, cache_dir=cache_dir2, cache_locations=cache_dir1 + ) as sub: + results2 = sub(wf2) + + assert not results2.errored, "\n".join(results2.errors["error message"]) + + assert 12 == results2.outputs.out # checking if the second wf runs again, but runs only one task - assert wf1.output_dir.exists() - assert wf2.output_dir.exists() + assert results1.output_dir != results2.output_dir # the second wf should rerun one task assert len(list(Path(cache_dir1).glob("F*"))) == 2 assert len(list(Path(cache_dir2).glob("F*"))) == 1 @@ -3133,37 +2998,41 @@ def test_wf_nostate_nodecachelocations_upd(plugin, tmpdir): cache_dir1 = tmpdir.mkdir("test_wf_cache3") cache_dir2 = tmpdir.mkdir("test_wf_cache4") - wf1 = Workflow(name="wf", input_spec=["x"], cache_dir=cache_dir1) - wf1.add(Ten(name="ten", x=wf1.lzin.x)) - wf1.add(Add2(name="add2", x=wf1.ten.lzout.out)) - wf1.set_output([("out", wf1.add2.lzout.out)]) - wf1.inputs.x = 3 - wf1.plugin = plugin - - with Submitter(worker=plugin) as sub: - sub(wf1) - - results1 = wf1.result() - assert 12 == results1.output.out - - wf2 = Workflow(name="wf", input_spec=["x", "y"], cache_dir=cache_dir2) - wf2.add(Ten(name="ten", x=wf2.lzin.x)) - wf2.add(Add2(name="add2", x=wf2.ten.lzout.out)) - wf2.set_output([("out", wf2.add2.lzout.out)]) - wf2.inputs.x = 2 - wf2.plugin = plugin + @workflow.define + def Workflow1(x): + ten = workflow.add(Ten(x=x)) + add2 = workflow.add(Add2(x=ten.out)) + return add2.out + + wf1 = Workflow1(x=3) + + with Submitter(worker=plugin, cache_dir=cache_dir1) as sub: + results1 = sub(wf1) + + assert not results1.errored, "\n".join(results1.errors["error message"]) + + assert 12 == results1.outputs.out + + @workflow.define + def Workflow2(x, y): + ten = workflow.add(Ten(x=x)) + add2 = workflow.add(Add2(x=ten.out)) + return add2.out + + wf2 = Workflow2(x=2) + # updating cache_locations after adding the tasks wf2.cache_locations = cache_dir1 - with Submitter(worker=plugin) as sub: - sub(wf2) + with Submitter(worker=plugin, cache_dir=cache_dir2) as sub: + results2 = sub(wf2) + + assert not results2.errored, "\n".join(results2.errors["error message"]) - results2 = wf2.result() - assert 12 == results2.output.out + assert 12 == results2.outputs.out # checking if the second wf runs again, but runs only one task - assert wf1.output_dir.exists() - assert wf2.output_dir.exists() + assert results1.output_dir != results2.output_dir # the second wf should have only one task run assert len(list(Path(cache_dir1).glob("F*"))) == 2 assert len(list(Path(cache_dir2).glob("F*"))) == 1 @@ -3178,42 +3047,44 @@ def test_wf_state_cachelocations(plugin, tmpdir): cache_dir1 = tmpdir.mkdir("test_wf_cache3") cache_dir2 = tmpdir.mkdir("test_wf_cache4") - wf1 = Workflow(name="wf", input_spec=["x", "y"], cache_dir=cache_dir1) - wf1.add(Multiply(name="mult", x=wf1.lzin.x, y=wf1.lzin.y)) - wf1.add(Add2Wait(name="add2", x=wf1.mult.lzout.out)) - wf1.set_output([("out", wf1.add2.lzout.out)]) - wf1.split(splitter=("x", "y"), x=[2, 20], y=[3, 4]) - wf1.plugin = plugin + @workflow.define + def Workflow1(x, y): + mult = workflow.add(Multiply(x=x, y=y)) + add2 = workflow.add(Add2Wait(x=mult.out)) + return add2.out + + wf1 = Workflow1().split(splitter=("x", "y"), x=[2, 20], y=[3, 4]) t0 = time.time() - with Submitter(worker=plugin) as sub: - sub(wf1) + with Submitter(worker=plugin, cache_dir=cache_dir1) as sub: + results1 = sub(wf1) + + assert not results1.errored, "\n".join(results1.errors["error message"]) t1 = time.time() - t0 - results1 = wf1.result() - assert results1[0].output.out == 8 - assert results1[1].output.out == 82 + assert results1.outputs.out[0] == 8 + assert results1.outputs.out[1] == 82 - wf2 = Workflow( - name="wf", - input_spec=["x", "y"], - cache_dir=cache_dir2, - cache_locations=cache_dir1, - ) - wf2.add(Multiply(name="mult", x=wf2.lzin.x, y=wf2.lzin.y)) - wf2.add(Add2Wait(name="add2", x=wf2.mult.lzout.out)) - wf2.set_output([("out", wf2.add2.lzout.out)]) - wf2.split(splitter=("x", "y"), x=[2, 20], y=[3, 4]) - wf2.plugin = plugin + @workflow.define + def Workflow2(x, y): + + mult = workflow.add(Multiply(x=x, y=y)) + add2 = workflow.add(Add2Wait(x=mult.out)) + return add2.out + + wf2 = Workflow2().split(splitter=("x", "y"), x=[2, 20], y=[3, 4]) t0 = time.time() - with Submitter(worker=plugin) as sub: - sub(wf2) + with Submitter( + worker=plugin, cache_dir=cache_dir2, cache_locations=cache_dir1 + ) as sub: + results2 = sub(wf2) + + assert not results2.errored, "\n".join(results2.errors["error message"]) t2 = time.time() - t0 - results2 = wf2.result() - assert results2[0].output.out == 8 - assert results2[1].output.out == 82 + assert results2.outputs.out[0] == 8 + assert results2.outputs.out[1] == 82 # for win and dask/slurm the time for dir creation etc. might take much longer if not sys.platform.startswith("win") and plugin == "cf": @@ -3242,42 +3113,42 @@ def test_wf_state_cachelocations_forcererun(plugin, tmpdir): cache_dir1 = tmpdir.mkdir("test_wf_cache3") cache_dir2 = tmpdir.mkdir("test_wf_cache4") - wf1 = Workflow(name="wf", input_spec=["x", "y"], cache_dir=cache_dir1) - wf1.add(Multiply(name="mult", x=wf1.lzin.x, y=wf1.lzin.y)) - wf1.add(Add2Wait(name="add2", x=wf1.mult.lzout.out)) - wf1.set_output([("out", wf1.add2.lzout.out)]) - wf1.split(splitter=("x", "y"), x=[2, 20], y=[3, 4]) - wf1.plugin = plugin + @workflow.define + def Workflow1(x, y): + mult = workflow.add(Multiply(x=x, y=y)) + add2 = workflow.add(Add2Wait(x=mult.out)) + return add2.out + + wf1 = Workflow1().split(splitter=("x", "y"), x=[2, 20], y=[3, 4]) t0 = time.time() - with Submitter(worker=plugin) as sub: - sub(wf1) + with Submitter(worker=plugin, cache_dir=cache_dir1) as sub: + results1 = sub(wf1) + + assert not results1.errored, "\n".join(results1.errors["error message"]) t1 = time.time() - t0 - results1 = wf1.result() - assert results1[0].output.out == 8 - assert results1[1].output.out == 82 + assert results1.outputs.out[0] == 8 + assert results1.outputs.out[1] == 82 - wf2 = Workflow( - name="wf", - input_spec=["x", "y"], - cache_dir=cache_dir2, - cache_locations=cache_dir1, - ) - wf2.add(Multiply(name="mult", x=wf2.lzin.x, y=wf2.lzin.y)) - wf2.add(Add2Wait(name="add2", x=wf2.mult.lzout.out)) - wf2.set_output([("out", wf2.add2.lzout.out)]) - wf2.split(splitter=("x", "y"), x=[2, 20], y=[3, 4]) - wf2.plugin = plugin + @workflow.define + def Workflow2(x, y): + + mult = workflow.add(Multiply(x=x, y=y)) + add2 = workflow.add(Add2Wait(x=mult.out)) + return add2.out + + wf2 = Workflow2().split(splitter=("x", "y"), x=[2, 20], y=[3, 4]) t0 = time.time() - with Submitter(worker=plugin) as sub: - sub(wf2, rerun=True) + with Submitter(worker=plugin, cache_dir=cache_dir2) as sub: + results2 = sub(wf2, rerun=True) + + assert not results2.errored, "\n".join(results2.errors["error message"]) t2 = time.time() - t0 - results2 = wf2.result() - assert results2[0].output.out == 8 - assert results2[1].output.out == 82 + assert results2.outputs.out[0] == 8 + assert results2.outputs.out[1] == 82 # for win and dask/slurm the time for dir creation etc. might take much longer if not sys.platform.startswith("win") and plugin == "cf": @@ -3307,43 +3178,43 @@ def test_wf_state_cachelocations_updateinp(plugin, tmpdir): cache_dir1 = tmpdir.mkdir("test_wf_cache3") cache_dir2 = tmpdir.mkdir("test_wf_cache4") - wf1 = Workflow(name="wf", input_spec=["x", "y"], cache_dir=cache_dir1) - wf1.add(Multiply(name="mult", x=wf1.lzin.x, y=wf1.lzin.y)) - wf1.add(Add2Wait(name="add2", x=wf1.mult.lzout.out)) - wf1.set_output([("out", wf1.add2.lzout.out)]) - wf1.split(splitter=("x", "y"), x=[2, 20], y=[3, 4]) - wf1.plugin = plugin + @workflow.define + def Workflow1(x, y): + mult = workflow.add(Multiply(x=x, y=y)) + add2 = workflow.add(Add2Wait(x=mult.out)) + return add2.out + + wf1 = Workflow1().split(splitter=("x", "y"), x=[2, 20], y=[3, 4]) t0 = time.time() - with Submitter(worker=plugin) as sub: - sub(wf1) + with Submitter(worker=plugin, cache_dir=cache_dir1) as sub: + results1 = sub(wf1) + + assert not results1.errored, "\n".join(results1.errors["error message"]) t1 = time.time() - t0 - results1 = wf1.result() - assert results1[0].output.out == 8 - assert results1[1].output.out == 82 + assert results1.outputs.out[0] == 8 + assert results1.outputs.out[1] == 82 - wf2 = Workflow( - name="wf", - input_spec=["x", "y"], - cache_dir=cache_dir2, - cache_locations=cache_dir1, - ) - wf2.add(Multiply(name="mult", x=wf2.lzin.x, y=wf2.lzin.y)) - wf2.add(Add2Wait(name="add2", x=wf2.mult.lzout.out)) - wf2.set_output([("out", wf2.add2.lzout.out)]) - wf2.split(splitter=("x", "y"), x=[2, 20], y=[3, 4]) - wf2.plugin = plugin - wf2.mult.inputs.y = wf2.lzin.y + @workflow.define + def Workflow2(x, y): + mult = workflow.add(Multiply(x=x, y=y)) + add2 = workflow.add(Add2Wait(x=mult.out)) + return add2.out + + wf2 = Workflow2().split(splitter=("x", "y"), x=[2, 20], y=[3, 4]) t0 = time.time() - with Submitter(worker=plugin) as sub: - sub(wf2) + with Submitter( + worker=plugin, cache_dir=cache_dir2, cache_locations=cache_dir1 + ) as sub: + results2 = sub(wf2) + + assert not results2.errored, "\n".join(results2.errors["error message"]) t2 = time.time() - t0 - results2 = wf2.result() - assert results2[0].output.out == 8 - assert results2[1].output.out == 82 + assert results2.outputs.out[0] == 8 + assert results2.outputs.out[1] == 82 # for win and dask/slurm the time for dir creation etc. might take much longer if not sys.platform.startswith("win") and plugin == "cf": @@ -3371,38 +3242,39 @@ def test_wf_state_n_nostate_cachelocations(plugin, tmpdir): cache_dir1 = tmpdir.mkdir("test_wf_cache3") cache_dir2 = tmpdir.mkdir("test_wf_cache4") - wf1 = Workflow(name="wf", input_spec=["x", "y"], cache_dir=cache_dir1) - wf1.add(Multiply(name="mult", x=wf1.lzin.x, y=wf1.lzin.y)) - wf1.add(Add2Wait(name="add2", x=wf1.mult.lzout.out)) - wf1.set_output([("out", wf1.add2.lzout.out)]) - wf1.inputs.x = 2 - wf1.inputs.y = 3 - wf1.plugin = plugin - - with Submitter(worker=plugin) as sub: - sub(wf1) - - results1 = wf1.result() - assert results1.output.out == 8 - - wf2 = Workflow( - name="wf", - input_spec=["x", "y"], - cache_dir=cache_dir2, - cache_locations=cache_dir1, - ) - wf2.add(Multiply(name="mult", x=wf2.lzin.x, y=wf2.lzin.y)) - wf2.add(Add2Wait(name="add2", x=wf2.mult.lzout.out)) - wf2.set_output([("out", wf2.add2.lzout.out)]) - wf2.split(splitter=("x", "y"), x=[2, 20], y=[3, 4]) - wf2.plugin = plugin + @workflow.define + def Workflow1(x, y): + mult = workflow.add(Multiply(x=x, y=y)) + add2 = workflow.add(Add2Wait(x=mult.out)) + return add2.out + + wf1 = Workflow1(x=2, y=3) + + with Submitter(worker=plugin, cache_dir=cache_dir1) as sub: + results1 = sub(wf1) + + assert not results1.errored, "\n".join(results1.errors["error message"]) + + assert results1.outputs.out == 8 - with Submitter(worker=plugin) as sub: - sub(wf2) + @workflow.define + def Workflow2(x, y): - results2 = wf2.result() - assert results2[0].output.out == 8 - assert results2[1].output.out == 82 + mult = workflow.add(Multiply(x=x, y=y)) + add2 = workflow.add(Add2Wait(x=mult.out)) + return add2.out + + wf2 = Workflow2().split(splitter=("x", "y"), x=[2, 20], y=[3, 4]) + + with Submitter( + worker=plugin, cache_dir=cache_dir2, cache_locations=cache_dir1 + ) as sub: + results2 = sub(wf2) + + assert not results2.errored, "\n".join(results2.errors["error message"]) + + assert results2.outputs.out[0] == 8 + assert results2.outputs.out[1] == 82 # checking the directory from the first wf assert wf1.output_dir.exists() @@ -3422,43 +3294,43 @@ def test_wf_nostate_cachelocations_updated(plugin, tmpdir): cache_dir1_empty = tmpdir.mkdir("test_wf_cache3_empty") cache_dir2 = tmpdir.mkdir("test_wf_cache4") - wf1 = Workflow(name="wf", input_spec=["x", "y"], cache_dir=cache_dir1) - wf1.add(Multiply(name="mult", x=wf1.lzin.x, y=wf1.lzin.y)) - wf1.add(Add2Wait(name="add2", x=wf1.mult.lzout.out)) - wf1.set_output([("out", wf1.add2.lzout.out)]) - wf1.inputs.x = 2 - wf1.inputs.y = 3 - wf1.plugin = plugin + @workflow.define + def Workflow1(x, y): + mult = workflow.add(Multiply(x=x, y=y)) + add2 = workflow.add(Add2Wait(x=mult.out)) + return add2.out + + wf1 = Workflow1(x=2, y=3) t0 = time.time() - with Submitter(worker=plugin) as sub: - sub(wf1) + with Submitter(worker=plugin, cache_dir=cache_dir1) as sub: + results1 = sub(wf1) + + assert not results1.errored, "\n".join(results1.errors["error message"]) t1 = time.time() - t0 - results1 = wf1.result() - assert 8 == results1.output.out + assert 8 == results1.outputs.out - wf2 = Workflow( - name="wf", - input_spec=["x", "y"], - cache_dir=cache_dir2, - cache_locations=cache_dir1, - ) - wf2.add(Multiply(name="mult", x=wf2.lzin.x, y=wf2.lzin.y)) - wf2.add(Add2Wait(name="add2", x=wf2.mult.lzout.out)) - wf2.set_output([("out", wf2.add2.lzout.out)]) - wf2.inputs.x = 2 - wf2.inputs.y = 3 - wf2.plugin = plugin + @workflow.define + def Workflow2(x, y): + + mult = workflow.add(Multiply(x=x, y=y)) + add2 = workflow.add(Add2Wait(x=mult.out)) + return add2.out + + wf2 = Workflow2(x=2, y=3) t0 = time.time() # changing cache_locations to non-existing dir - with Submitter(worker=plugin) as sub: - sub(wf2, cache_locations=cache_dir1_empty) + with Submitter( + worker=plugin, cache_dir=cache_dir2, cache_locations=cache_dir1_empty + ) as sub: + results2 = sub(wf2) + + assert not results2.errored, "\n".join(results2.errors["error message"]) t2 = time.time() - t0 - results2 = wf2.result() - assert 8 == results2.output.out + assert 8 == results2.outputs.out # for win and dask/slurm the time for dir creation etc. might take much longer if not sys.platform.startswith("win") and plugin == "cf": @@ -3467,8 +3339,7 @@ def test_wf_nostate_cachelocations_updated(plugin, tmpdir): assert t2 > 2 # checking if both wf run - assert wf1.output_dir.exists() - assert wf2.output_dir.exists() + assert results1.output_dir != results2.output_dir @pytest.mark.flaky(reruns=3) @@ -3481,43 +3352,42 @@ def test_wf_nostate_cachelocations_recompute(plugin, tmpdir): cache_dir1 = tmpdir.mkdir("test_wf_cache3") cache_dir2 = tmpdir.mkdir("test_wf_cache4") - wf1 = Workflow(name="wf", input_spec=["x", "y"], cache_dir=cache_dir1) - wf1.add(Multiply(name="mult", x=wf1.lzin.x, y=wf1.lzin.y)) - wf1.add(Add2(name="add2", x=wf1.mult.lzout.out)) - wf1.set_output([("out", wf1.add2.lzout.out)]) - wf1.inputs.x = 2 - wf1.inputs.y = 3 - wf1.plugin = plugin - - with Submitter(worker=plugin) as sub: - sub(wf1) - - results1 = wf1.result() - assert 8 == results1.output.out - - wf2 = Workflow( - name="wf", - input_spec=["x", "y"], - cache_dir=cache_dir2, - cache_locations=cache_dir1, - ) - # different argument assignment - wf2.add(Multiply(name="mult", x=wf2.lzin.y, y=wf2.lzin.x)) - wf2.add(Add2(name="add2", x=wf2.mult.lzout.out)) - wf2.set_output([("out", wf2.add2.lzout.out)]) - wf2.inputs.x = 2 - wf2.inputs.y = 3 - wf2.plugin = plugin + @workflow.define + def Workflow1(x, y): + mult = workflow.add(Multiply(x=x, y=y)) + add2 = workflow.add(Add2(x=mult.out)) + return add2.out + + wf1 = Workflow1(x=2, y=3) + + with Submitter(worker=plugin, cache_dir=cache_dir1) as sub: + results1 = sub(wf1) + + assert not results1.errored, "\n".join(results1.errors["error message"]) - with Submitter(worker=plugin) as sub: - sub(wf2) + assert 8 == results1.outputs.out - results2 = wf2.result() - assert 8 == results2.output.out + @workflow.define + def Workflow2(x, y): + + # different argument assignment + mult = workflow.add(Multiply(x=y, y=x)) + add2 = workflow.add(Add2(x=mult.out)) + return add2.out + + wf2 = Workflow2(x=2, y=3) + + with Submitter( + worker=plugin, cache_dir=cache_dir2, cache_locations=cache_dir1 + ) as sub: + results2 = sub(wf2) + + assert not results2.errored, "\n".join(results2.errors["error message"]) + + assert 8 == results2.outputs.out # checking if both dir exists - assert wf1.output_dir.exists() - assert wf2.output_dir.exists() + assert results1.output_dir != results2.output_dir # the second wf should have only one task run assert len(list(Path(cache_dir1).glob("F*"))) == 2 @@ -3533,46 +3403,42 @@ def test_wf_ndstate_cachelocations(plugin, tmpdir): cache_dir1 = tmpdir.mkdir("test_wf_cache3") cache_dir2 = tmpdir.mkdir("test_wf_cache4") - wf1 = Workflow(name="wf", input_spec=["x", "y"], cache_dir=cache_dir1) - wf1.add( - Multiply(name="mult").split(splitter=("x", "y"), x=wf1.lzin.x, y=wf1.lzin.y) - ) - wf1.add(Add2Wait(name="add2", x=wf1.mult.lzout.out)) - wf1.set_output([("out", wf1.add2.lzout.out)]) - wf1.inputs.x = [2, 20] - wf1.inputs.y = [3, 4] - wf1.plugin = plugin + @workflow.define + def Workflow1(x, y): + mult = workflow.add(Multiply().split(splitter=("x", "y"), x=x, y=y)) + add2 = workflow.add(Add2Wait(x=mult.out)) + return add2.out + + wf1 = Workflow1(x=[2, 20], y=[3, 4]) t0 = time.time() - with Submitter(worker=plugin) as sub: - sub(wf1) + with Submitter(worker=plugin, cache_dir=cache_dir1) as sub: + results1 = sub(wf1) + + assert not results1.errored, "\n".join(results1.errors["error message"]) t1 = time.time() - t0 - results1 = wf1.result() - assert results1.output.out == [8, 82] + assert results1.outputs.out == [8, 82] - wf2 = Workflow( - name="wf", - input_spec=["x", "y"], - cache_dir=cache_dir2, - cache_locations=cache_dir1, - ) - wf2.add( - Multiply(name="mult").split(splitter=("x", "y"), x=wf2.lzin.x, y=wf2.lzin.y) - ) - wf2.add(Add2Wait(name="add2", x=wf2.mult.lzout.out)) - wf2.set_output([("out", wf2.add2.lzout.out)]) - wf2.inputs.x = [2, 20] - wf2.inputs.y = [3, 4] - wf2.plugin = plugin + @workflow.define + def Workflow2(x, y): + + mult = workflow.add(Multiply().split(splitter=("x", "y"), x=x, y=y)) + add2 = workflow.add(Add2Wait(x=mult.out)) + return add2.out + + wf2 = Workflow2(x=[2, 20], y=[3, 4]) t0 = time.time() - with Submitter(worker=plugin) as sub: - sub(wf2) + with Submitter( + worker=plugin, cache_dir=cache_dir2, cache_locations=cache_dir1 + ) as sub: + results2 = sub(wf2) + + assert not results2.errored, "\n".join(results2.errors["error message"]) t2 = time.time() - t0 - results2 = wf2.result() - assert results2.output.out == [8, 82] + assert results2.outputs.out == [8, 82] # for win and dask/slurm the time for dir creation etc. might take much longer if not sys.platform.startswith("win") and plugin == "cf": @@ -3598,46 +3464,40 @@ def test_wf_ndstate_cachelocations_forcererun(plugin, tmpdir): cache_dir1 = tmpdir.mkdir("test_wf_cache3") cache_dir2 = tmpdir.mkdir("test_wf_cache4") - wf1 = Workflow(name="wf", input_spec=["x", "y"], cache_dir=cache_dir1) - wf1.add( - Multiply(name="mult").split(splitter=("x", "y"), x=wf1.lzin.x, y=wf1.lzin.y) - ) - wf1.add(Add2Wait(name="add2", x=wf1.mult.lzout.out)) - wf1.set_output([("out", wf1.add2.lzout.out)]) - wf1.inputs.x = [2, 20] - wf1.inputs.y = [3, 4] - wf1.plugin = plugin + @workflow.define + def Workflow1(x, y): + mult = workflow.add(Multiply().split(splitter=("x", "y"), x=x, y=y)) + add2 = workflow.add(Add2Wait(x=mult.out)) + return add2.out + + wf1 = Workflow1(x=[2, 20], y=[3, 4]) t0 = time.time() - with Submitter(worker=plugin) as sub: - sub(wf1) + with Submitter(worker=plugin, cache_dir=cache_dir1) as sub: + results1 = sub(wf1) + + assert not results1.errored, "\n".join(results1.errors["error message"]) t1 = time.time() - t0 - results1 = wf1.result() - assert results1.output.out == [8, 82] + assert results1.outputs.out == [8, 82] - wf2 = Workflow( - name="wf", - input_spec=["x", "y"], - cache_dir=cache_dir2, - cache_locations=cache_dir1, - ) - wf2.add( - Multiply(name="mult").split(splitter=("x", "y"), x=wf2.lzin.x, y=wf2.lzin.y) - ) - wf2.add(Add2Wait(name="add2", x=wf2.mult.lzout.out)) - wf2.set_output([("out", wf2.add2.lzout.out)]) - wf2.inputs.x = [2, 20] - wf2.inputs.y = [3, 4] - wf2.plugin = plugin + @workflow.define + def Workflow2(x, y): + + mult = workflow.add(Multiply().split(splitter=("x", "y"), x=x, y=y)) + add2 = workflow.add(Add2Wait(x=mult.out)) + return add2.out + + wf2 = Workflow2(x=[2, 20], y=[3, 4]) t0 = time.time() - with Submitter(worker=plugin) as sub: - sub(wf2, rerun=True) + with Submitter(worker=plugin, cache_dir=cache_dir2) as sub: + results2 = sub(wf2, rerun=True) + + assert not results2.errored, "\n".join(results2.errors["error message"]) t2 = time.time() - t0 - results2 = wf2.result() - assert results2.output.out == [8, 82] + assert results2.outputs.out == [8, 82] # for win and dask/slurm the time for dir creation etc. might take much longer if not sys.platform.startswith("win") and plugin == "cf": @@ -3661,45 +3521,42 @@ def test_wf_ndstate_cachelocations_updatespl(plugin, tmpdir): cache_dir1 = tmpdir.mkdir("test_wf_cache3") cache_dir2 = tmpdir.mkdir("test_wf_cache4") - wf1 = Workflow(name="wf", input_spec=["x", "y"], cache_dir=cache_dir1) - wf1.add( - Multiply(name="mult").split(splitter=("x", "y"), x=wf1.lzin.x, y=wf1.lzin.y) - ) - wf1.add(Add2Wait(name="add2", x=wf1.mult.lzout.out)) - wf1.set_output([("out", wf1.add2.lzout.out)]) - wf1.inputs.x = [2, 20] - wf1.inputs.y = [3, 4] - wf1.plugin = plugin + @workflow.define + def Workflow1(x, y): + mult = workflow.add(Multiply().split(splitter=("x", "y"), x=x, y=y)) + add2 = workflow.add(Add2Wait(x=mult.out)) + return add2.out + + wf1 = Workflow1(x=[2, 20], y=[3, 4]) t0 = time.time() - with Submitter(worker=plugin) as sub: - sub(wf1) + with Submitter(worker=plugin, cache_dir=cache_dir1) as sub: + results1 = sub(wf1) + + assert not results1.errored, "\n".join(results1.errors["error message"]) t1 = time.time() - t0 - results1 = wf1.result() - assert results1.output.out == [8, 82] + assert results1.outputs.out == [8, 82] - wf2 = Workflow( - name="wf", - input_spec=["x", "y"], - cache_dir=cache_dir2, - cache_locations=cache_dir1, - ) - wf2.add(Multiply(name="mult")) - wf2.mult.split(splitter=("x", "y"), x=wf2.lzin.x, y=wf2.lzin.y) - wf2.add(Add2Wait(name="add2", x=wf2.mult.lzout.out)) - wf2.set_output([("out", wf2.add2.lzout.out)]) - wf2.inputs.x = [2, 20] - wf2.inputs.y = [3, 4] - wf2.plugin = plugin + @workflow.define + def Workflow2(x, y): + + mult = workflow.add(Multiply().split(splitter=("x", "y"), x=x, y=y)) + add2 = workflow.add(Add2Wait(x=mult.out)) + return add2.out + + wf2 = Workflow2(x=[2, 20], y=[3, 4]) t0 = time.time() - with Submitter(worker=plugin) as sub: - sub(wf2) + with Submitter( + worker=plugin, cache_dir=cache_dir2, cache_locations=cache_dir1 + ) as sub: + results2 = sub(wf2) + + assert not results2.errored, "\n".join(results2.errors["error message"]) t2 = time.time() - t0 - results2 = wf2.result() - assert results2.output.out == [8, 82] + assert results2.outputs.out == [8, 82] # for win and dask/slurm the time for dir creation etc. might take much longer if not sys.platform.startswith("win") and plugin == "cf": @@ -3724,46 +3581,42 @@ def test_wf_ndstate_cachelocations_recompute(plugin, tmpdir): cache_dir1 = tmpdir.mkdir("test_wf_cache3") cache_dir2 = tmpdir.mkdir("test_wf_cache4") - wf1 = Workflow(name="wf", input_spec=["x", "y"], cache_dir=cache_dir1) - wf1.add( - Multiply(name="mult").split(splitter=("x", "y"), x=wf1.lzin.x, y=wf1.lzin.y) - ) - wf1.add(Add2Wait(name="add2", x=wf1.mult.lzout.out)) - wf1.set_output([("out", wf1.add2.lzout.out)]) - wf1.inputs.x = [2, 20] - wf1.inputs.y = [3, 4] - wf1.plugin = plugin + @workflow.define + def Workflow1(x, y): + mult = workflow.add(Multiply().split(splitter=("x", "y"), x=x, y=y)) + add2 = workflow.add(Add2Wait(x=mult.out)) + return add2.out + + wf1 = Workflow1(x=[2, 20], y=[3, 4]) t0 = time.time() - with Submitter(worker=plugin) as sub: - sub(wf1) + with Submitter(worker=plugin, cache_dir=cache_dir1) as sub: + results1 = sub(wf1) + + assert not results1.errored, "\n".join(results1.errors["error message"]) t1 = time.time() - t0 - results1 = wf1.result() - assert results1.output.out == [8, 82] + assert results1.outputs.out == [8, 82] - wf2 = Workflow( - name="wf", - input_spec=["x", "y"], - cache_dir=cache_dir2, - cache_locations=cache_dir1, - ) - wf2.add( - Multiply(name="mult").split(splitter=["x", "y"], x=wf2.lzin.x, y=wf2.lzin.y) - ) - wf2.add(Add2Wait(name="add2", x=wf2.mult.lzout.out)) - wf2.set_output([("out", wf2.add2.lzout.out)]) - wf2.inputs.x = [2, 20] - wf2.inputs.y = [3, 4] - wf2.plugin = plugin + @workflow.define + def Workflow2(x, y): + + mult = workflow.add(Multiply().split(splitter=["x", "y"], x=x, y=y)) + add2 = workflow.add(Add2Wait(x=mult.out)) + return add2.out + + wf2 = Workflow2(x=[2, 20], y=[3, 4]) t0 = time.time() - with Submitter(worker=plugin) as sub: - sub(wf2) + with Submitter( + worker=plugin, cache_dir=cache_dir2, cache_locations=cache_dir1 + ) as sub: + results2 = sub(wf2) + + assert not results2.errored, "\n".join(results2.errors["error message"]) t2 = time.time() - t0 - results2 = wf2.result() - assert results2.output.out == [8, 10, 62, 82] + assert results2.outputs.out == [8, 10, 62, 82] # for win and dask/slurm the time for dir creation etc. might take much longer if not sys.platform.startswith("win") and plugin == "cf": @@ -3787,21 +3640,22 @@ def test_wf_nostate_runtwice_usecache(plugin, tmpdir): """ cache_dir1 = tmpdir.mkdir("test_wf_cache3") - wf1 = Workflow(name="wf", input_spec=["x", "y"], cache_dir=cache_dir1) - wf1.add(Multiply(name="mult", x=wf1.lzin.x, y=wf1.lzin.y)) - wf1.add(Add2Wait(name="add2", x=wf1.mult.lzout.out)) - wf1.set_output([("out", wf1.add2.lzout.out)]) - wf1.inputs.x = 2 - wf1.inputs.y = 3 - wf1.plugin = plugin + @workflow.define + def Workflow1(x, y): + mult = workflow.add(Multiply(x=x, y=y)) + add2 = workflow.add(Add2Wait(x=mult.out)) + return add2.out + + wf1 = Workflow1(x=2, y=3) t0 = time.time() - with Submitter(worker=plugin) as sub: - sub(wf1) + with Submitter(worker=plugin, cache_dir=cache_dir1) as sub: + results1 = sub(wf1) + + assert not results1.errored, "\n".join(results1.errors["error message"]) t1 = time.time() - t0 - results1 = wf1.result() - assert 8 == results1.output.out + assert 8 == results1.outputs.out # checkoing output_dir after the first run assert wf1.output_dir.exists() @@ -3810,12 +3664,13 @@ def test_wf_nostate_runtwice_usecache(plugin, tmpdir): # running workflow the second time t0 = time.time() - with Submitter(worker=plugin) as sub: - sub(wf1) + with Submitter(worker=plugin, cache_dir=cache_dir1) as sub: + results1 = sub(wf1) + + assert not results1.errored, "\n".join(results1.errors["error message"]) t2 = time.time() - t0 - results1 = wf1.result() - assert 8 == results1.output.out + assert 8 == results1.outputs.out # checking if no new directory is created assert cache_dir_content == os.listdir(wf1.cache_dir) @@ -3833,21 +3688,23 @@ def test_wf_state_runtwice_usecache(plugin, tmpdir): """ cache_dir1 = tmpdir.mkdir("test_wf_cache3") - wf1 = Workflow(name="wf", input_spec=["x", "y"], cache_dir=cache_dir1) - wf1.add(Multiply(name="mult", x=wf1.lzin.x, y=wf1.lzin.y)) - wf1.add(Add2Wait(name="add2", x=wf1.mult.lzout.out)) - wf1.set_output([("out", wf1.add2.lzout.out)]) - wf1.split(splitter=("x", "y"), x=[2, 20], y=[3, 30]) - wf1.plugin = plugin + @workflow.define + def Workflow1(x, y): + mult = workflow.add(Multiply(x=x, y=y)) + add2 = workflow.add(Add2Wait(x=mult.out)) + return add2.out + + wf1 = Workflow1().split(splitter=("x", "y"), x=[2, 20], y=[3, 30]) t0 = time.time() - with Submitter(worker=plugin) as sub: - sub(wf1) + with Submitter(worker=plugin, cache_dir=cache_dir1) as sub: + results1 = sub(wf1) + + assert not results1.errored, "\n".join(results1.errors["error message"]) t1 = time.time() - t0 - results1 = wf1.result() - assert 8 == results1[0].output.out - assert 602 == results1[1].output.out + assert 8 == results1.outputs.out[0] + assert 602 == results1.outputs.out[1] # checkoing output_dir after the first run assert [odir.exists() for odir in wf1.output_dir] @@ -3857,13 +3714,14 @@ def test_wf_state_runtwice_usecache(plugin, tmpdir): # running workflow the second time t0 = time.time() - with Submitter(worker=plugin) as sub: - sub(wf1) + with Submitter(worker=plugin, cache_dir=cache_dir1) as sub: + results1 = sub(wf1) + + assert not results1.errored, "\n".join(results1.errors["error message"]) t2 = time.time() - t0 - results1 = wf1.result() - assert 8 == results1[0].output.out - assert 602 == results1[1].output.out + assert 8 == results1.outputs.out[0] + assert 602 == results1.outputs.out[1] # checking if no new directory is created assert cache_dir_content == os.listdir(wf1.cache_dir) # for win and dask/slurm the time for dir creation etc. might take much longer @@ -3875,11 +3733,13 @@ def test_wf_state_runtwice_usecache(plugin, tmpdir): @pytest.fixture def create_tasks(): - wf = Workflow(name="wf", input_spec=["x"]) - wf.inputs.x = 1 - wf.add(Add2(name="t1", x=wf.lzin.x)) - wf.add(Multiply(name="t2", x=wf.t1.lzout.out, y=2)) - wf.set_output([("out", wf.t2.lzout.out)]) + @workflow.define + def Workflow(x): + t1 = workflow.add(Add2(x=x)) + t2 = workflow.add(Multiply(x=t1.out, y=2)) + return t2.out + + wf = Workflow(x=1) t1 = wf.name2obj["t1"] t2 = wf.name2obj["t2"] return wf, t1, t2 @@ -3907,25 +3767,25 @@ def test_cache_propagation2(tmpdir, create_tasks): def test_cache_propagation3(tmpdir, create_tasks): """Shared cache_dir with state""" wf, t1, t2 = create_tasks - wf.split("x", x=[1, 2]) + wf = wf.split("x", x=[1, 2]) wf.cache_dir = (tmpdir / "shared").strpath wf(plugin="cf") assert wf.cache_dir == t1.cache_dir == t2.cache_dir def test_workflow_combine1(tmpdir): - wf1 = Workflow(name="wf1", input_spec=["a", "b"], a=[1, 2], b=[2, 3]) - wf1.add(Power(name="power").split(["a", "b"], a=wf1.lzin.a, b=wf1.lzin.b)) - wf1.add(Identity(name="identity1", x=wf1.power.lzout.out).combine("power.a")) - wf1.add(Identity(name="identity2", x=wf1.identity1.lzout.out).combine("power.b")) - wf1.set_output( - { - "out_pow": wf1.power.lzout.out, - "out_iden1": wf1.identity1.lzout.out, - "out_iden2": wf1.identity2.lzout.out, - } - ) - wf1.cache_dir = tmpdir + @workflow.define(outputs=["out_pow", "out_iden1", "out_iden2"]) + def Workflow1(a, b): + power = workflow.add(Power().split(["a", "b"], a=a, b=b)) + identity1 = workflow.add( + Identity(x=power.out).combine("power.a"), name="identity1" + ) + identity2 = workflow.add( + Identity(x=identity1.out).combine("power.b"), name="identity2" + ) + return power.out, identity1.out, identity2.out + + wf1 = Workflow1(a=[1, 2], b=[2, 3]) outputs = wf1() assert outputs.out_pow == [1, 1, 4, 8] @@ -3934,106 +3794,108 @@ def test_workflow_combine1(tmpdir): def test_workflow_combine2(tmpdir): - wf1 = Workflow(name="wf1", input_spec=["a", "b"], a=[1, 2], b=[2, 3]) - wf1.add( - Power(name="power").split(["a", "b"], a=wf1.lzin.a, b=wf1.lzin.b).combine("a") - ) - wf1.add(Identity(name="identity", x=wf1.power.lzout.out).combine("power.b")) - wf1.set_output({"out_pow": wf1.power.lzout.out, "out_iden": wf1.identity.lzout.out}) - wf1.cache_dir = tmpdir - outputs = wf1() + @workflow.define(outputs=["out_pow", "out_iden"]) + def Workflow1(a, b): + power = workflow.add(Power().split(["a", "b"], a=a, b=b).combine("a")) + identity = workflow.add(Identity(x=power.out).combine("power.b")) + return power.out, identity.out + + wf1 = Workflow1(a=[1, 2], b=[2, 3]) + outputs = wf1(cache_dir=tmpdir) assert outputs.out_pow == [[1, 4], [1, 8]] assert outputs.out_iden == [[1, 4], [1, 8]] -# testing lzout.all to collect all of the results and let PythonTask deal with it +# g.all to collect all of the results and let PythonTask deal with it def test_wf_lzoutall_1(plugin, tmpdir): """workflow with 2 tasks, no splitter passing entire result object to add2_sub2_res function - by using lzout.all syntax + using.all syntax """ - wf = Workflow(name="wf_2", input_spec=["x", "y"]) - wf.add(Multiply(name="mult", x=wf.lzin.x, y=wf.lzin.y)) - wf.add(Add2Sub2Res(name="add_sub", res=wf.mult.lzout.all_)) - wf.set_output([("out", wf.add_sub.lzout.out_add)]) - wf.inputs.x = 2 - wf.inputs.y = 3 - wf.cache_dir = tmpdir - with Submitter(worker=plugin) as sub: - sub(wf) + @workflow.define + def Workflow(x, y): + mult = workflow.add(Multiply(x=x, y=y)) + add_sub = workflow.add(Add2Sub2Res(res=mult.all_)) + return add_sub.out_add + + wf = Workflow(x=2, y=3) + + with Submitter(worker=plugin, cache_dir=tmpdir) as sub: + results = sub(wf) + + assert not results.errored, "\n".join(results.errors["error message"]) - assert wf.output_dir.exists() - results = wf.result() - assert 8 == results.output.out + assert 8 == results.outputs.out def test_wf_lzoutall_1a(plugin, tmpdir): """workflow with 2 tasks, no splitter passing entire result object to add2_res function - by using lzout.all syntax in the node connections and for wf output + using.all syntax in the node connections and for wf output """ - wf = Workflow(name="wf_2", input_spec=["x", "y"]) - wf.add(Multiply(name="mult", x=wf.lzin.x, y=wf.lzin.y)) - wf.add(Add2Sub2Res(name="add_sub", res=wf.mult.lzout.all_)) - wf.set_output([("out_all", wf.add_sub.lzout.all_)]) - wf.inputs.x = 2 - wf.inputs.y = 3 - wf.cache_dir = tmpdir - with Submitter(worker=plugin) as sub: - sub(wf) + @workflow.define + def Workflow(x, y): + mult = workflow.add(Multiply(x=x, y=y)) + add_sub = workflow.add(Add2Sub2Res(res=mult.all_)) + return add_sub.all_ # out_all + + wf = Workflow(x=2, y=3) + + with Submitter(worker=plugin, cache_dir=tmpdir) as sub: + results = sub(wf) + + assert not results.errored, "\n".join(results.errors["error message"]) - assert wf.output_dir.exists() - results = wf.result() - assert results.output.out_all == {"out_add": 8, "out_sub": 4} + assert results.outputs.out_all == {"out_add": 8, "out_sub": 4} def test_wf_lzoutall_st_1(plugin, tmpdir): """workflow with 2 tasks, no splitter passing entire result object to add2_res function - by using lzout.all syntax - """ - wf = Workflow(name="wf_2", input_spec=["x", "y"]) - wf.add(Multiply(name="mult").split(["x", "y"], x=wf.lzin.x, y=wf.lzin.y)) - wf.add(Add2Sub2Res(name="add_sub", res=wf.mult.lzout.all_)) - wf.set_output([("out_add", wf.add_sub.lzout.out_add)]) - wf.inputs.x = [2, 20] - wf.inputs.y = [3, 30] - wf.plugin = plugin - wf.cache_dir = tmpdir - - with Submitter(worker=plugin) as sub: - sub(wf) + using.all syntax + """ + + @workflow.define + def Workflow(x, y): + mult = workflow.add(Multiply().split(["x", "y"], x=x, y=y)) + add_sub = workflow.add(Add2Sub2Res(res=mult.all_)) + return add_sub.out_add # out_add + + wf = Workflow(x=[2, 20], y=[3, 30]) - assert wf.output_dir.exists() - results = wf.result() - assert results.output.out_add == [8, 62, 62, 602] + with Submitter(worker=plugin, cache_dir=tmpdir) as sub: + results = sub(wf) + + assert not results.errored, "\n".join(results.errors["error message"]) + + assert results.outputs.out_add == [8, 62, 62, 602] def test_wf_lzoutall_st_1a(plugin, tmpdir): """workflow with 2 tasks, no splitter passing entire result object to add2_res function - by using lzout.all syntax - """ - wf = Workflow(name="wf_2", input_spec=["x", "y"]) - wf.add(Multiply(name="mult").split(["x", "y"], x=wf.lzin.x, y=wf.lzin.y)) - wf.add(Add2Sub2Res(name="add_sub", res=wf.mult.lzout.all_)) - wf.set_output([("out_all", wf.add_sub.lzout.all_)]) - wf.inputs.x = [2, 20] - wf.inputs.y = [3, 30] - wf.plugin = plugin - wf.cache_dir = tmpdir - - with Submitter(worker=plugin) as sub: - sub(wf) + using.all syntax + """ + + @workflow.define + def Workflow(x, y): + mult = workflow.add(Multiply().split(["x", "y"], x=x, y=y)) + add_sub = workflow.add(Add2Sub2Res(res=mult.all_)) + return add_sub.all_ # out_all + + wf = Workflow(x=[2, 20], y=[3, 30]) + + with Submitter(worker=plugin, cache_dir=tmpdir) as sub: + results = sub(wf) + + assert not results.errored, "\n".join(results.errors["error message"]) - assert wf.output_dir.exists() - results = wf.result() - assert results.output.out_all == [ + assert results.outputs.out_all == [ {"out_add": 8, "out_sub": 4}, {"out_add": 62, "out_sub": 58}, {"out_add": 62, "out_sub": 58}, @@ -4044,26 +3906,24 @@ def test_wf_lzoutall_st_1a(plugin, tmpdir): def test_wf_lzoutall_st_2(plugin, tmpdir): """workflow with 2 tasks, no splitter passing entire result object to add2_res function - by using lzout.all syntax + using.all syntax """ - wf = Workflow(name="wf_2", input_spec=["x", "y"]) - wf.add( - Multiply(name="mult").split(["x", "y"], x=wf.lzin.x, y=wf.lzin.y).combine("x") - ) - wf.add(Add2Sub2ResList(name="add_sub", res=wf.mult.lzout.all_)) - wf.set_output([("out_add", wf.add_sub.lzout.out_add)]) - wf.inputs.x = [2, 20] - wf.inputs.y = [3, 30] - wf.plugin = plugin - wf.cache_dir = tmpdir - - with Submitter(worker=plugin) as sub: - sub(wf) - assert wf.output_dir.exists() - results = wf.result() - assert results.output.out_add[0] == [8, 62] - assert results.output.out_add[1] == [62, 602] + @workflow.define + def Workflow(x, y): + mult = workflow.add(Multiply().split(["x", "y"], x=x, y=y).combine("x")) + add_sub = workflow.add(Add2Sub2ResList(res=mult.all_)) + return add_sub.out_add # out_add + + wf = Workflow(x=[2, 20], y=[3, 30]) + + with Submitter(worker=plugin, cache_dir=tmpdir) as sub: + results = sub(wf) + + assert not results.errored, "\n".join(results.errors["error message"]) + + assert results.outputs.out_add[0] == [8, 62] + assert results.outputs.out_add[1] == [62, 602] @pytest.mark.xfail( @@ -4076,25 +3936,23 @@ def test_wf_lzoutall_st_2(plugin, tmpdir): def test_wf_lzoutall_st_2a(plugin, tmpdir): """workflow with 2 tasks, no splitter passing entire result object to add2_res function - by using lzout.all syntax + using.all syntax """ - wf = Workflow(name="wf_2", input_spec=["x", "y"]) - wf.add( - Multiply(name="mult").split(["x", "y"], x=wf.lzin.x, y=wf.lzin.y).combine("x") - ) - wf.add(Add2Sub2ResList(name="add_sub", res=wf.mult.lzout.all_)) - wf.set_output([("out_all", wf.add_sub.lzout.all_)]) - wf.inputs.x = [2, 20] - wf.inputs.y = [3, 30] - wf.plugin = plugin - wf.cache_dir = tmpdir - - with Submitter(worker=plugin) as sub: - sub(wf) - assert wf.output_dir.exists() - results = wf.result() - assert results.output.out_all == [ + @workflow.define + def Workflow(x, y): + mult = workflow.add(Multiply().split(["x", "y"], x=x, y=y).combine("x")) + add_sub = workflow.add(Add2Sub2ResList(res=mult.all_)) + return add_sub.all_ # out_all + + wf = Workflow(x=[2, 20], y=[3, 30]) + + with Submitter(worker=plugin, cache_dir=tmpdir) as sub: + results = sub(wf) + + assert not results.errored, "\n".join(results.errors["error message"]) + + assert results.outputs.out_all == [ {"out_add": [8, 62], "out_sub": [4, 58]}, {"out_add": [62, 602], "out_sub": [58, 598]}, ] @@ -4105,18 +3963,21 @@ def test_wf_lzoutall_st_2a(plugin, tmpdir): def test_wf_resultfile_1(plugin, tmpdir): """workflow with a file in the result, file should be copied to the wf dir""" - wf = Workflow(name="wf_file_1", input_spec=["x"], cache_dir=tmpdir) - wf.add(FunWriteFile(name="writefile", filename=wf.lzin.x)) - wf.inputs.x = "file_1.txt" - wf.plugin = plugin - wf.set_output([("wf_out", wf.writefile.lzout.out)]) - with Submitter(worker=plugin) as sub: - sub(wf) + @workflow.define + def Workflow(x): + writefile = workflow.add(FunWriteFile(filename=x)) + + return writefile.out # wf_out + + wf = Workflow(x="file_1.txt") + with Submitter(worker=plugin, cache_dir=tmpdir) as sub: + results = sub(wf) + + assert not results.errored, "\n".join(results.errors["error message"]) - results = wf.result() # checking if the file exists and if it is in the Workflow directory - wf_out = results.output.wf_out.fspath + wf_out = results.outputs.wf_out.fspath wf_out.exists() assert wf_out == wf.output_dir / "file_1.txt" @@ -4125,19 +3986,22 @@ def test_wf_resultfile_2(plugin, tmpdir): """workflow with a list of files in the wf result, all files should be copied to the wf dir """ - wf = Workflow(name="wf_file_1", input_spec=["x"], cache_dir=tmpdir) - wf.add(FunWriteFileList(name="writefile", filename_list=wf.lzin.x)) + + @workflow.define + def Workflow(x): + writefile = workflow.add(FunWriteFileList(filename_list=x)) + + return writefile.out # wf_out + file_list = ["file_1.txt", "file_2.txt", "file_3.txt"] - wf.inputs.x = file_list - wf.plugin = plugin - wf.set_output([("wf_out", wf.writefile.lzout.out)]) + wf = Workflow(x=file_list) + with Submitter(worker=plugin, cache_dir=tmpdir) as sub: + results = sub(wf) - with Submitter(worker=plugin) as sub: - sub(wf) + assert not results.errored, "\n".join(results.errors["error message"]) - results = wf.result() # checking if the file exists and if it is in the Workflow directory - for ii, file in enumerate(results.output.wf_out): + for ii, file in enumerate(results.outputs.wf_out): assert file.fspath.exists() assert file.fspath == wf.output_dir / file_list[ii] @@ -4146,19 +4010,22 @@ def test_wf_resultfile_3(plugin, tmpdir): """workflow with a dictionaries of files in the wf result, all files should be copied to the wf dir """ - wf = Workflow(name="wf_file_1", input_spec=["x"], cache_dir=tmpdir) - wf.add(FunWriteFileList2Dict(name="writefile", filename_list=wf.lzin.x)) + + @workflow.define + def Workflow(x): + writefile = workflow.add(FunWriteFileList2Dict(filename_list=x)) + + return writefile.out # wf_out + file_list = ["file_1.txt", "file_2.txt", "file_3.txt"] - wf.inputs.x = file_list - wf.plugin = plugin - wf.set_output([("wf_out", wf.writefile.lzout.out)]) + wf = Workflow(x=file_list) + with Submitter(worker=plugin, cache_dir=tmpdir) as sub: + results = sub(wf) - with Submitter(worker=plugin) as sub: - sub(wf) + assert not results.errored, "\n".join(results.errors["error message"]) - results = wf.result() # checking if the file exists and if it is in the Workflow directory - for key, val in results.output.wf_out.items(): + for key, val in results.outputs.wf_out.items(): if key == "random_int": assert val == 20 else: @@ -4169,15 +4036,17 @@ def test_wf_resultfile_3(plugin, tmpdir): def test_wf_upstream_error1(plugin, tmpdir): """workflow with two tasks, task2 dependent on an task1 which raised an error""" - wf = Workflow(name="wf", input_spec=["x"], cache_dir=tmpdir) - wf.add(FunAddVarDefaultNoType(name="addvar1", a=wf.lzin.x)) - wf.inputs.x = "hi" # TypeError for adding str and int - wf.plugin = plugin - wf.add(FunAddVarDefaultNoType(name="addvar2", a=wf.addvar1.lzout.out)) - wf.set_output([("out", wf.addvar2.lzout.out)]) + + @workflow.define + def Workflow(x): + addvar1 = workflow.add(FunAddVarDefaultNoType(a=x), name="addvar1") + addvar2 = workflow.add(FunAddVarDefaultNoType(a=addvar1.out), name="addvar2") + return addvar2.out + + wf = Workflow(x="hi") # TypeError for adding str and int with pytest.raises(ValueError) as excinfo: - with Submitter(worker=plugin) as sub: + with Submitter(worker=plugin, cache_dir=tmpdir) as sub: sub(wf) assert "addvar1" in str(excinfo.value) assert "raised an error" in str(excinfo.value) @@ -4187,15 +4056,20 @@ def test_wf_upstream_error2(plugin, tmpdir): """task2 dependent on task1, task1 errors, workflow-level split on task 1 goal - workflow finish running, one output errors but the other doesn't """ - wf = Workflow(name="wf", input_spec=["x"], cache_dir=tmpdir) - wf.add(FunAddVarDefaultNoType(name="addvar1", a=wf.lzin.x)) - wf.split("x", x=[1, "hi"]) # workflow-level split TypeError for adding str and int - wf.plugin = plugin - wf.add(FunAddVarDefaultNoType(name="addvar2", a=wf.addvar1.lzout.out)) - wf.set_output([("out", wf.addvar2.lzout.out)]) + + @workflow.define + def Workflow(x): + addvar1 = workflow.add(FunAddVarDefaultNoType(a=x), name="addvar1") + + addvar2 = workflow.add(FunAddVarDefaultNoType(a=addvar1.out), name="addvar2") + return addvar2.out + + wf = Workflow().split( + "x", x=[1, "hi"] + ) # workflow-level split TypeError for adding str and int with pytest.raises(Exception) as excinfo: - with Submitter(worker=plugin) as sub: + with Submitter(worker=plugin, cache_dir=tmpdir) as sub: sub(wf) assert "addvar1" in str(excinfo.value) assert "raised an error" in str(excinfo.value) @@ -4206,16 +4080,17 @@ def test_wf_upstream_error3(plugin, tmpdir): """task2 dependent on task1, task1 errors, task-level split on task 1 goal - workflow finish running, one output errors but the other doesn't """ - wf = Workflow(name="wf", input_spec=["x"], cache_dir=tmpdir) - wf.add(FunAddVarDefaultNoType(name="addvar1")) - wf.inputs.x = [1, "hi"] # TypeError for adding str and int - wf.addvar1.split("a", a=wf.lzin.x) # task-level split - wf.plugin = plugin - wf.add(FunAddVarDefaultNoType(name="addvar2", a=wf.addvar1.lzout.out)) - wf.set_output([("out", wf.addvar2.lzout.out)]) + @workflow.define + def Workflow(x): + addvar1 = workflow.add(FunAddVarDefaultNoType().split("a", a=x), name="addvar1") + + addvar2 = workflow.add(FunAddVarDefaultNoType(a=addvar1.out), name="addvar2") + return addvar2.out + + wf = Workflow(x=[1, "hi"]) # TypeError for adding str and int with pytest.raises(Exception) as excinfo: - with Submitter(worker=plugin) as sub: + with Submitter(worker=plugin, cache_dir=tmpdir) as sub: sub(wf) assert "addvar1" in str(excinfo.value) assert "raised an error" in str(excinfo.value) @@ -4223,14 +4098,16 @@ def test_wf_upstream_error3(plugin, tmpdir): def test_wf_upstream_error4(plugin, tmpdir): """workflow with one task, which raises an error""" - wf = Workflow(name="wf", input_spec=["x"], cache_dir=tmpdir) - wf.add(FunAddVarDefaultNoType(name="addvar1", a=wf.lzin.x)) - wf.inputs.x = "hi" # TypeError for adding str and int - wf.plugin = plugin - wf.set_output([("out", wf.addvar1.lzout.out)]) + @workflow.define + def Workflow(x): + addvar1 = workflow.add(FunAddVarDefaultNoType(a=x)) + + return addvar1.out + + wf = Workflow(x="hi") # TypeError for adding str and int with pytest.raises(Exception) as excinfo: - with Submitter(worker=plugin) as sub: + with Submitter(worker=plugin, cache_dir=tmpdir) as sub: sub(wf) assert "raised an error" in str(excinfo.value) assert "addvar1" in str(excinfo.value) @@ -4238,18 +4115,21 @@ def test_wf_upstream_error4(plugin, tmpdir): def test_wf_upstream_error5(plugin, tmpdir): """nested workflow with one task, which raises an error""" - wf_main = Workflow(name="wf_main", input_spec=["x"], cache_dir=tmpdir) - wf = Workflow(name="wf", input_spec=["x"], x=wf_main.lzin.x) - wf.add(FunAddVarDefaultNoType(name="addvar1", a=wf.lzin.x)) - wf.plugin = plugin - wf.set_output([("wf_out", wf.addvar1.lzout.out)]) - wf_main.add(wf) - wf_main.inputs.x = "hi" # TypeError for adding str and int - wf_main.set_output([("out", wf_main.wf.lzout.wf_out)]) + @workflow.define + def Workflow(x): + addvar1 = workflow.add(FunAddVarDefaultNoType(a=x)) + return addvar1.out # wf_out + + @workflow.define + def WfMain(x): + wf = workflow.add(Workflow(x=x)) + return wf.out + + wf_main = WfMain(x="hi") # TypeError for adding str and int with pytest.raises(Exception) as excinfo: - with Submitter(worker=plugin) as sub: + with Submitter(worker=plugin, cache_dir=tmpdir) as sub: sub(wf_main) assert "addvar1" in str(excinfo.value) @@ -4258,19 +4138,23 @@ def test_wf_upstream_error5(plugin, tmpdir): def test_wf_upstream_error6(plugin, tmpdir): """nested workflow with two tasks, the first one raises an error""" - wf_main = Workflow(name="wf_main", input_spec=["x"], cache_dir=tmpdir) - wf = Workflow(name="wf", input_spec=["x"], x=wf_main.lzin.x) - wf.add(FunAddVarDefaultNoType(name="addvar1", a=wf.lzin.x)) - wf.add(FunAddVarDefaultNoType(name="addvar2", a=wf.addvar1.lzout.out)) - wf.plugin = plugin - wf.set_output([("wf_out", wf.addvar2.lzout.out)]) - wf_main.add(wf) - wf_main.inputs.x = "hi" # TypeError for adding str and int - wf_main.set_output([("out", wf_main.wf.lzout.wf_out)]) + @workflow.define + def Workflow(x): + addvar1 = workflow.add(FunAddVarDefaultNoType(a=x), name="addvar1") + addvar2 = workflow.add(FunAddVarDefaultNoType(a=addvar1.out), name="addvar2") + + return addvar2.out # wf_out + + @workflow.define + def WfMain(x): + wf = workflow.add(Workflow(x=x)) + return wf.out + + wf_main = WfMain(x="hi") # TypeError for adding str and int with pytest.raises(Exception) as excinfo: - with Submitter(worker=plugin) as sub: + with Submitter(worker=plugin, cache_dir=tmpdir) as sub: sub(wf_main) assert "addvar1" in str(excinfo.value) @@ -4282,16 +4166,19 @@ def test_wf_upstream_error7(plugin, tmpdir): workflow with three sequential tasks, the first task raises an error the last task is set as the workflow output """ - wf = Workflow(name="wf", input_spec=["x"], cache_dir=tmpdir) - wf.add(FunAddVarDefaultNoType(name="addvar1", a=wf.lzin.x)) - wf.inputs.x = "hi" # TypeError for adding str and int - wf.plugin = plugin - wf.add(FunAddVarDefaultNoType(name="addvar2", a=wf.addvar1.lzout.out)) - wf.add(FunAddVarDefaultNoType(name="addvar3", a=wf.addvar2.lzout.out)) - wf.set_output([("out", wf.addvar3.lzout.out)]) + + @workflow.define + def Workflow(x): + addvar1 = workflow.add(FunAddVarDefaultNoType(a=x), name="addvar1") + + addvar2 = workflow.add(FunAddVarDefaultNoType(a=addvar1.out), name="addvar2") + addvar3 = workflow.add(FunAddVarDefaultNoType(a=addvar2.out), name="addvar3") + return addvar3.out + + wf = Workflow(x="hi") # TypeError for adding str and int with pytest.raises(ValueError) as excinfo: - with Submitter(worker=plugin) as sub: + with Submitter(worker=plugin, cache_dir=tmpdir) as sub: sub(wf) assert "addvar1" in str(excinfo.value) assert "raised an error" in str(excinfo.value) @@ -4304,16 +4191,18 @@ def test_wf_upstream_error7a(plugin, tmpdir): workflow with three sequential tasks, the first task raises an error the second task is set as the workflow output """ - wf = Workflow(name="wf", input_spec=["x"], cache_dir=tmpdir) - wf.add(FunAddVarDefaultNoType(name="addvar1", a=wf.lzin.x)) - wf.inputs.x = "hi" # TypeError for adding str and int - wf.plugin = plugin - wf.add(FunAddVarDefaultNoType(name="addvar2", a=wf.addvar1.lzout.out)) - wf.add(FunAddVarDefaultNoType(name="addvar3", a=wf.addvar2.lzout.out)) - wf.set_output([("out", wf.addvar2.lzout.out)]) + @workflow.define + def Workflow(x): + addvar1 = workflow.add(FunAddVarDefaultNoType(a=x), name="addvar1") + + addvar2 = workflow.add(FunAddVarDefaultNoType(a=addvar1.out), name="addvar2") + addvar3 = workflow.add(FunAddVarDefaultNoType(a=addvar2.out), name="addvar3") + return addvar3.out + + wf = Workflow(x="hi") # TypeError for adding str and int with pytest.raises(ValueError) as excinfo: - with Submitter(worker=plugin) as sub: + with Submitter(worker=plugin, cache_dir=tmpdir) as sub: sub(wf) assert "addvar1" in str(excinfo.value) assert "raised an error" in str(excinfo.value) @@ -4326,16 +4215,18 @@ def test_wf_upstream_error7b(plugin, tmpdir): workflow with three sequential tasks, the first task raises an error the second and the third tasks are set as the workflow output """ - wf = Workflow(name="wf", input_spec=["x"], cache_dir=tmpdir) - wf.add(FunAddVarDefaultNoType(name="addvar1", a=wf.lzin.x)) - wf.inputs.x = "hi" # TypeError for adding str and int - wf.plugin = plugin - wf.add(FunAddVarDefaultNoType(name="addvar2", a=wf.addvar1.lzout.out)) - wf.add(FunAddVarDefaultNoType(name="addvar3", a=wf.addvar2.lzout.out)) - wf.set_output([("out1", wf.addvar2.lzout.out), ("out2", wf.addvar3.lzout.out)]) + @workflow.define(outputs=["out1", "out2"]) + def Workflow(x): + addvar1 = workflow.add(FunAddVarDefaultNoType(a=x), name="addvar1") + + addvar2 = workflow.add(FunAddVarDefaultNoType(a=addvar1.out), name="addvar2") + addvar3 = workflow.add(FunAddVarDefaultNoType(a=addvar2.out), name="addvar3") + return addvar2.out, addvar3.out # + + wf = Workflow(x="hi") # TypeError for adding str and int with pytest.raises(ValueError) as excinfo: - with Submitter(worker=plugin) as sub: + with Submitter(worker=plugin, cache_dir=tmpdir) as sub: sub(wf) assert "addvar1" in str(excinfo.value) assert "raised an error" in str(excinfo.value) @@ -4345,16 +4236,18 @@ def test_wf_upstream_error7b(plugin, tmpdir): def test_wf_upstream_error8(plugin, tmpdir): """workflow with three tasks, the first one raises an error, so 2 others are removed""" - wf = Workflow(name="wf", input_spec=["x"], cache_dir=tmpdir) - wf.add(FunAddVarDefaultNoType(name="addvar1", a=wf.lzin.x)) - wf.inputs.x = "hi" # TypeError for adding str and int - wf.plugin = plugin - wf.add(FunAddVarDefaultNoType(name="addvar2", a=wf.addvar1.lzout.out)) - wf.add(FunAddTwo(name="addtwo", a=wf.addvar1.lzout.out)) - wf.set_output([("out1", wf.addvar2.lzout.out), ("out2", wf.addtwo.lzout.out)]) + @workflow.define(outputs=["out1", "out2"]) + def Workflow(x): + addvar1 = workflow.add(FunAddVarDefaultNoType(a=x), name="addvar1") + + addvar2 = workflow.add(FunAddVarDefaultNoType(a=addvar1.out), name="addvar2") + addtwo = workflow.add(FunAddTwo(a=addvar1.out)) + return addvar2.out, addtwo.out # + + wf = Workflow(x="hi") # TypeError for adding str and int with pytest.raises(ValueError) as excinfo: - with Submitter(worker=plugin) as sub: + with Submitter(worker=plugin, cache_dir=tmpdir) as sub: sub(wf) assert "addvar1" in str(excinfo.value) @@ -4369,19 +4262,21 @@ def test_wf_upstream_error9(plugin, tmpdir): one branch has an error, the second is fine the errored branch is connected to the workflow output """ - wf = Workflow(name="wf", input_spec=["x"], cache_dir=tmpdir) - wf.add(FunAddVarDefaultNoType(name="addvar1", a=wf.lzin.x)) - wf.inputs.x = 2 - wf.add(FunAddVarNoType(name="err", a=wf.addvar1.lzout.out, b="hi")) - wf.add(FunAddVarDefaultNoType(name="follow_err", a=wf.err.lzout.out)) - wf.add(FunAddTwoNoType(name="addtwo", a=wf.addvar1.lzout.out)) - wf.add(FunAddVarDefaultNoType(name="addvar2", a=wf.addtwo.lzout.out)) - wf.set_output([("out1", wf.follow_err.lzout.out)]) + @workflow.define + def Workflow(x): + addvar1 = workflow.add(FunAddVarDefaultNoType(a=x), name="addvar1") + + err = workflow.add(FunAddVarNoType(a=addvar1.out, b="hi"), name="err") + follow_err = workflow.add(FunAddVarDefaultNoType(a=err.out), name="follow_err") - wf.plugin = plugin + addtwo = workflow.add(FunAddTwoNoType(a=addvar1.out), name="addtwo") + workflow.add(FunAddVarDefaultNoType(a=addtwo.out)) + return follow_err.out # out1 + + wf = Workflow(x=2) with pytest.raises(ValueError) as excinfo: - with Submitter(worker=plugin) as sub: + with Submitter(worker=plugin, cache_dir=tmpdir) as sub: sub(wf) assert "err" in str(excinfo.value) assert "raised an error" in str(excinfo.value) @@ -4396,18 +4291,21 @@ def test_wf_upstream_error9a(plugin, tmpdir): the branch without error is connected to the workflow output so the workflow finished clean """ - wf = Workflow(name="wf", input_spec=["x"], cache_dir=tmpdir) - wf.add(FunAddVarDefault(name="addvar1", a=wf.lzin.x)) - wf.inputs.x = 2 - wf.add(FunAddVarNoType(name="err", a=wf.addvar1.lzout.out, b="hi")) - wf.add(FunAddVarDefault(name="follow_err", a=wf.err.lzout.out)) - wf.add(FunAddTwoNoType(name="addtwo", a=wf.addvar1.lzout.out)) - wf.add(FunAddVarDefault(name="addvar2", a=wf.addtwo.lzout.out)) - wf.set_output([("out1", wf.addvar2.lzout.out)]) # , ("out2", wf.addtwo.lzout.out)]) + @workflow.define + def Workflow(x): + addvar1 = workflow.add(FunAddVarDefault(a=x), name="addvar1") + + err = workflow.add(FunAddVarNoType(a=addvar1.out, b="hi"), name="err") + workflow.add(FunAddVarDefault(a=err.out)) + + addtwo = workflow.add(FunAddTwoNoType(a=addvar1.out), name="addtwo") + addvar2 = workflow.add(FunAddVarDefault(a=addtwo.out), name="addvar2") + return addvar2.out # out1 # , ("out2", addtwo.out)]) + + wf = Workflow(x=2) - wf.plugin = plugin - with Submitter(worker=plugin) as sub: + with Submitter(worker=plugin, cache_dir=tmpdir) as sub: sub(wf) assert wf.err._errored is True assert wf.follow_err._errored == ["err"] @@ -4419,19 +4317,21 @@ def test_wf_upstream_error9b(plugin, tmpdir): one branch has an error, the second is fine both branches are connected to the workflow output """ - wf = Workflow(name="wf", input_spec=["x"], cache_dir=tmpdir) - wf.add(FunAddVarDefaultNoType(name="addvar1", a=wf.lzin.x)) - wf.inputs.x = 2 - wf.add(FunAddVarNoType(name="err", a=wf.addvar1.lzout.out, b="hi")) - wf.add(FunAddVarDefaultNoType(name="follow_err", a=wf.err.lzout.out)) - wf.add(FunAddTwoNoType(name="addtwo", a=wf.addvar1.lzout.out)) - wf.add(FunAddVarDefaultNoType(name="addvar2", a=wf.addtwo.lzout.out)) - wf.set_output([("out1", wf.follow_err.lzout.out), ("out2", wf.addtwo.lzout.out)]) + @workflow.define(outputs=["out1", "out2"]) + def Workflow(x): + addvar1 = workflow.add(FunAddVarDefaultNoType(a=x), name="addvar1") - wf.plugin = plugin + err = workflow.add(FunAddVarNoType(a=addvar1.out, b="hi"), name="err") + follow_err = workflow.add(FunAddVarDefaultNoType(a=err.out), name="follow_err") + + addtwo = workflow.add(FunAddTwoNoType(a=addvar1.out), name="addtwo") + addvar2 = workflow.add(FunAddVarDefaultNoType(a=addtwo.out), name="addvar2") + return follow_err.out, addvar2.out + + wf = Workflow(x=2) with pytest.raises(ValueError) as excinfo: - with Submitter(worker=plugin) as sub: + with Submitter(worker=plugin, cache_dir=tmpdir) as sub: sub(wf) assert "err" in str(excinfo.value) assert "raised an error" in str(excinfo.value) @@ -4468,12 +4368,15 @@ def exporting_graphs(wf, name): @pytest.mark.parametrize("splitter", [None, "x"]) def test_graph_1(tmpdir, splitter): """creating a set of graphs, wf with two nodes""" - wf = Workflow(name="wf", input_spec=["x", "y"], cache_dir=tmpdir) - wf.add(Multiply(name="mult_1", x=wf.lzin.x, y=wf.lzin.y)) - wf.add(Multiply(name="mult_2", x=wf.lzin.x, y=wf.lzin.x)) - wf.add(Add2(name="add2", x=wf.mult_1.lzout.out)) - wf.set_output([("out", wf.add2.lzout.out)]) - wf.split(splitter, x=[1, 2]) + + @workflow.define + def Workflow(x, y): + mult_1 = workflow.add(Multiply(x=x, y=y), name="mult_1") + workflow.add(Multiply(x=x, y=x), name="mult_2") + add2 = workflow.add(Add2(x=mult_1.out), name="add2") + return add2.out + + wf = Workflow().split(splitter, x=[1, 2]) # simple graph dotfile_s = wf.create_dotfile() @@ -4510,11 +4413,15 @@ def test_graph_1st(tmpdir): """creating a set of graphs, wf with two nodes some nodes have splitters, should be marked with blue color """ - wf = Workflow(name="wf", input_spec=["x", "y"], cache_dir=tmpdir) - wf.add(Multiply(name="mult_1", y=wf.lzin.y).split("x", x=wf.lzin.x)) - wf.add(Multiply(name="mult_2", x=wf.lzin.x, y=wf.lzin.x)) - wf.add(Add2(name="add2", x=wf.mult_1.lzout.out)) - wf.set_output([("out", wf.add2.lzout.out)]) + + @workflow.define + def Workflow(x, y): + mult_1 = workflow.add(Multiply(y=y).split("x", x=x), name="mult_1") + workflow.add(Multiply(x=x, y=x), name="mult_2") + add2 = workflow.add(Add2(x=mult_1.out), name="add2") + return add2.out + + wf = Workflow(x=[1, 2], y=2) # simple graph dotfile_s = wf.create_dotfile() @@ -4551,11 +4458,15 @@ def test_graph_1st_cmb(tmpdir): the first one has a splitter, the second has a combiner, so the third one is stateless first two nodes should be blue and the arrow between them should be blue """ - wf = Workflow(name="wf", input_spec=["x", "y"], cache_dir=tmpdir) - wf.add(Multiply(name="mult", y=wf.lzin.y).split("x", x=wf.lzin.x)) - wf.add(Add2(name="add2", x=wf.mult.lzout.out).combine("mult.x")) - wf.add(ListSum(name="sum", x=wf.add2.lzout.out)) - wf.set_output([("out", wf.sum.lzout.out)]) + + @workflow.define + def Workflow(x, y): + mult = workflow.add(Multiply(y=y).split("x", x=x), name="mult") + add2 = workflow.add(Add2(x=mult.out).combine("mult.x"), name="add2") + sum = workflow.add(ListSum(x=add2.out), name="sum") + return sum.out + + wf = Workflow(x=[1, 2], y=2) # simple graph dotfile_s = wf.create_dotfile() dotstr_s_lines = dotfile_s.read_text().split("\n") @@ -4590,12 +4501,18 @@ def test_graph_1st_cmb(tmpdir): def test_graph_2(tmpdir): """creating a graph, wf with one workflow as a node""" - wf = Workflow(name="wf", input_spec=["x"], cache_dir=tmpdir) - wfnd = Workflow(name="wfnd", input_spec=["x"], x=wf.lzin.x) - wfnd.add(Add2(name="add2", x=wfnd.lzin.x)) - wfnd.set_output([("out", wfnd.add2.lzout.out)]) - wf.add(wfnd) - wf.set_output([("out", wf.wfnd.lzout.out)]) + + @workflow.define + def Wfnd(x): + add2 = workflow.add(Add2(x=x), name="add2") + return add2.out + + @workflow.define + def Workflow(x): + wfnd = workflow.add(Wfnd(x=x), name="wfnd") + return wfnd.out + + wf = Workflow(x=2) # simple graph dotfile_s = wf.create_dotfile() @@ -4624,12 +4541,18 @@ def test_graph_2st(tmpdir): """creating a set of graphs, wf with one workflow as a node the inner workflow has a state, so should be blue """ - wf = Workflow(name="wf", input_spec=["x"], cache_dir=tmpdir) - wfnd = Workflow(name="wfnd", input_spec=["x"]).split("x", x=wf.lzin.x) - wfnd.add(Add2(name="add2", x=wfnd.lzin.x)) - wfnd.set_output([("out", wfnd.add2.lzout.out)]) - wf.add(wfnd) - wf.set_output([("out", wf.wfnd.lzout.out)]) + + @workflow.define + def Wfnd(x): + add2 = workflow.add(Add2(x=x), name="add2") + return add2.out + + @workflow.define + def Workflow(x): + wfnd = workflow.add(Wfnd(x=x).split("x", x=x), name="wfnd") + return wfnd.out + + wf = Workflow(x=[1, 2]) # simple graph dotfile_s = wf.create_dotfile() @@ -4658,14 +4581,19 @@ def test_graph_2st(tmpdir): def test_graph_3(tmpdir): """creating a set of graphs, wf with two nodes (one node is a workflow)""" - wf = Workflow(name="wf", input_spec=["x", "y"], cache_dir=tmpdir) - wf.add(Multiply(name="mult", x=wf.lzin.x, y=wf.lzin.y)) - wfnd = Workflow(name="wfnd", input_spec=["x"], x=wf.mult.lzout.out) - wfnd.add(Add2(name="add2", x=wfnd.lzin.x)) - wfnd.set_output([("out", wfnd.add2.lzout.out)]) - wf.add(wfnd) - wf.set_output([("out", wf.wfnd.lzout.out)]) + @workflow.define + def Wfnd(x): + add2 = workflow.add(Add2(x=x), name="add2") + return add2.out + + @workflow.define + def Workflow(x, y): + mult = workflow.add(Multiply(x=x, y=y), name="mult") + wfnd = workflow.add(Wfnd(x=mult.out), name="wfnd") + return wfnd.out + + wf = Workflow(x=2) # simple graph dotfile_s = wf.create_dotfile() @@ -4700,14 +4628,19 @@ def test_graph_3st(tmpdir): the first node has a state and it should be passed to the second node (blue node and a wfasnd, and blue arrow from the node to the wfasnd) """ - wf = Workflow(name="wf", input_spec=["x", "y"], cache_dir=tmpdir) - wf.add(Multiply(name="mult", y=wf.lzin.y).split("x", x=wf.lzin.x)) - wfnd = Workflow(name="wfnd", input_spec=["x"], x=wf.mult.lzout.out) - wfnd.add(Add2(name="add2", x=wfnd.lzin.x)) - wfnd.set_output([("out", wfnd.add2.lzout.out)]) - wf.add(wfnd) - wf.set_output([("out", wf.wfnd.lzout.out)]) + @workflow.define + def Wfnd(x): + add2 = workflow.add(Add2(x=x), name="add2") + return add2.out + + @workflow.define + def Workflow(x, y): + mult = workflow.add(Multiply(y=y).split("x", x=x), name="mult") + wfnd = workflow.add(Wfnd(x=mult.out), name="wfnd") + return wfnd.out + + wf = Workflow(x=[1, 2], y=2) # simple graph dotfile_s = wf.create_dotfile() @@ -4741,14 +4674,20 @@ def test_graph_4(tmpdir): """creating a set of graphs, wf with two nodes (one node is a workflow with two nodes inside). Connection from the node to the inner workflow. """ - wf = Workflow(name="wf", input_spec=["x", "y"], cache_dir=tmpdir) - wf.add(Multiply(name="mult", x=wf.lzin.x, y=wf.lzin.y)) - wfnd = Workflow(name="wfnd", input_spec=["x"], x=wf.mult.lzout.out) - wfnd.add(Add2(name="add2_a", x=wfnd.lzin.x)) - wfnd.add(Add2(name="add2_b", x=wfnd.add2_a.lzout.out)) - wfnd.set_output([("out", wfnd.add2_b.lzout.out)]) - wf.add(wfnd) - wf.set_output([("out", wf.wfnd.lzout.out)]) + + @workflow.define + def Wfnd(x): + add2_a = workflow.add(Add2(x=x), name="add2_a") + add2_b = workflow.add(Add2(x=add2_a.out), name="add2_b") + return add2_b.out + + @workflow.define + def Workflow(x, y): + mult = workflow.add(Multiply(x=x, y=y), name="mult") + wfnd = workflow.add(Wfnd(x=mult.out), name="wfnd") + return wfnd.out + + wf = Workflow(x=2, y=3) # simple graph dotfile_s = wf.create_dotfile() @@ -4784,14 +4723,20 @@ def test_graph_5(tmpdir): """creating a set of graphs, wf with two nodes (one node is a workflow with two nodes inside). Connection from the inner workflow to the node. """ - wf = Workflow(name="wf", input_spec=["x", "y"], cache_dir=tmpdir) - wfnd = Workflow(name="wfnd", input_spec=["x"], x=wf.lzin.x) - wfnd.add(Add2(name="add2_a", x=wfnd.lzin.x)) - wfnd.add(Add2(name="add2_b", x=wfnd.add2_a.lzout.out)) - wfnd.set_output([("out", wfnd.add2_b.lzout.out)]) - wf.add(wfnd) - wf.add(Multiply(name="mult", x=wf.wfnd.lzout.out, y=wf.lzin.y)) - wf.set_output([("out", wf.mult.lzout.out)]) + + @workflow.define + def Wfnd(x): + add2_a = workflow.add(Add2(x=x), name="add2_a") + add2_b = workflow.add(Add2(x=add2_a.out), name="add2_b") + return add2_b.out + + @workflow.define + def Workflow(x, y): + wfnd = workflow.add(Wfnd(x=x), name="wfnd") + mult = workflow.add(Multiply(x=wfnd.out, y=y), name="mult") + return mult.out + + wf = Workflow(x=2, y=3) # simple graph dotfile_s = wf.create_dotfile() @@ -4834,19 +4779,19 @@ def test_duplicate_input_on_split_wf(tmpdir): def printer(a): return a - wf = Workflow(name="wf", input_spec=["text"], cache_dir=tmpdir) - wf.split(("text"), text=text) + @workflow.define + def Workflow(text): + printer1 = workflow.add(printer(a=text)) + return printer1.out # out1 - wf.add(printer(name="printer1", a=wf.lzin.text)) - - wf.set_output([("out1", wf.printer1.lzout.out)]) + wf = Workflow().split(text=text) with Submitter(worker="cf", n_procs=6) as sub: - sub(wf) + results = sub(wf) - res = wf.result() + assert not results.errored, "\n".join(results.errors["error message"]) - assert res[0].output.out1 == "test" and res[1].output.out1 == "test" + assert results.output.out1[0] == "test" and results.output.out1[0] == "test" @pytest.mark.timeout(40) @@ -4858,48 +4803,40 @@ def test_inner_outer_wf_duplicate(tmpdir): start_list = [3, 4] @python.define - def one_arg(start_number): + def OneArg(start_number): for k in range(10): start_number += 1 return start_number @python.define - def one_arg_inner(start_number): + def OneArgInner(start_number): for k in range(10): start_number += 1 return start_number - # Outer workflow - test_outer = Workflow( - name="test_outer", - input_spec=["start_number", "task_name", "dummy"], - cache_dir=tmpdir, - dummy=1, - ) - # Splitting on both arguments - test_outer.split( - ["start_number", "task_name"], start_number=start_list, task_name=task_list - ) - # Inner Workflow - test_inner = Workflow(name="test_inner", input_spec=["start_number1"]) - test_inner.add( - one_arg_inner(name="Ilevel1", start_number=test_inner.lzin.start_number1) - ) - test_inner.set_output([("res", test_inner.Ilevel1.lzout.out)]) + @workflow.define(outputs=["res"]) + def InnerWf(start_number1): + inner_level1 = workflow.add(OneArgInner(start_number=start_number1)) + return inner_level1.out - # Outer workflow has two nodes plus the inner workflow - test_outer.add(one_arg(name="level1", start_number=test_outer.lzin.start_number)) - test_outer.add(test_inner) - test_inner.inputs.start_number1 = test_outer.level1.lzout.out + # Outer workflow has two nodes plus the inner workflow - test_outer.set_output([("res2", test_outer.test_inner.lzout.res)]) + # Outer workflow + @workflow.define(outputs=["res2"]) + def OuterWf(start_number, task_name, dummy): + level1 = workflow.add(OneArg(start_number=start_number)) + inner = workflow.add(InnerWf(start_number1=level1.out)) + return inner.res + + test_outer = OuterWf(dummy=1).split( + ["start_number", "task_name"], start_number=start_list, task_name=task_list + ) with Submitter(worker="cf") as sub: - sub(test_outer) + res = sub(test_outer) - res = test_outer.result() - assert res[0].output.res2 == 23 and res[1].output.res2 == 23 + assert res.output.res2[0] == 23 and res.output.res2[1] == 23 def test_rerun_errored(tmpdir, capfd): @@ -4907,7 +4844,7 @@ def test_rerun_errored(tmpdir, capfd): Only the errored tasks and workflow should be rerun""" @python.define - def pass_odds(x): + def PassOdds(x): if x % 2 == 0: print(f"x%2 = {x % 2} (error)\n") raise Exception("even error") @@ -4915,9 +4852,12 @@ def pass_odds(x): print(f"x%2 = {x % 2}\n") return x - wf = Workflow(name="wf", input_spec=["x"], cache_dir=tmpdir) - wf.add(pass_odds(name="pass_odds").split("x", x=[1, 2, 3, 4, 5])) - wf.set_output([("out", wf.pass_odds.lzout.out)]) + @workflow.define + def Workflow(x): + pass_odds = workflow.add(PassOdds().split("x", x=x)) + return pass_odds.out + + wf = Workflow(x=[1, 2, 3, 4, 5]) with pytest.raises(Exception): wf() @@ -4943,83 +4883,87 @@ def pass_odds(x): def test_wf_state_arrays(): - wf = Workflow( - name="test", - input_spec={"x": ty.List[int], "y": int}, - output_spec={"alpha": int, "beta": ty.List[int]}, - ) + @workflow.define(outputs={"alpha": int, "beta": ty.List[int]}) + def Workflow(x: ty.List[int], y: int): - wf.add( # Split over workflow input "x" on "scalar" input - ListMultSum( - in_list=wf.lzin.x, - name="A", - ).split(scalar=wf.lzin.x) - ) + A = workflow.add( # Split over workflow input "x" on "scalar" input + ListMultSum( + in_list=x, + ).split(scalar=x) + ) - wf.add( # Workflow is still split over "x", combined over "x" on out - ListMultSum( - name="B", - scalar=wf.A.lzout.sum, - in_list=wf.A.lzout.products, - ).combine("A.scalar") - ) + B = workflow.add( # Workflow is still split over "x", combined over "x" on out + ListMultSum( + scalar=A.sum, + in_list=A.products, + ).combine("A.scalar") + ) - wf.add( # Workflow " - ListMultSum( - name="C", - scalar=wf.lzin.y, - in_list=wf.B.lzout.sum, + C = workflow.add( # Workflow " + ListMultSum( + scalar=y, + in_list=B.sum, + ) ) - ) - wf.add( # Workflow is split again, this time over C.products - ListMultSum( - name="D", - in_list=wf.lzin.x, + D = workflow.add( # Workflow is split again, this time over C.products + ListMultSum( + in_list=x, + ) + .split(scalar=C.products) + .combine("scalar") ) - .split(scalar=wf.C.lzout.products) - .combine("scalar") - ) - wf.add( # Workflow is finally combined again into a single node - ListMultSum(name="E", scalar=wf.lzin.y, in_list=wf.D.lzout.sum) - ) + E = workflow.add( # Workflow is finally combined again into a single node + ListMultSum(scalar=y, in_list=D.sum) + ) + + return E.sum, E.products - wf.set_output([("alpha", wf.E.lzout.sum), ("beta", wf.E.lzout.products)]) + wf = Workflow(x=[1, 2, 3, 4], y=10) - results = wf(x=[1, 2, 3, 4], y=10) - assert results.output.alpha == 3000000 - assert results.output.beta == [100000, 400000, 900000, 1600000] + results = wf() + assert results.outputs.alpha == 3000000 + assert results.outputs.beta == [100000, 400000, 900000, 1600000] def test_wf_input_output_typing(): - wf = Workflow( - name="test", - input_spec={"x": int, "y": ty.List[int]}, - output_spec={"alpha": int, "beta": ty.List[int]}, - ) - with pytest.raises(TypeError) as exc_info: + @workflow.define(outputs={"alpha": int, "beta": ty.List[int]}) + def MismatchInputWf(x: int, y: ty.List[int]): ListMultSum( - scalar=wf.lzin.y, - in_list=wf.lzin.y, + scalar=y, + in_list=y, name="A", ) + + with pytest.raises(TypeError) as exc_info: + MismatchInputWf(x=1, y=[1, 2, 3]) exc_info_matches(exc_info, "Cannot coerce into ") - wf.add( # Split over workflow input "x" on "scalar" input - ListMultSum( - scalar=wf.lzin.x, - in_list=wf.lzin.y, - name="A", + @workflow.define(outputs={"alpha": int, "beta": ty.List[int]}) + def MismatchOutputWf(x: int, y: ty.List[int]): + A = workflow.add( # Split over workflow input "x" on "scalar" input + ListMultSum( + scalar=x, + in_list=y, + ) ) - ) + return A.products, A.products with pytest.raises(TypeError, match="don't match their declared types"): - wf.set_output( - [ - ("alpha", wf.A.lzout.products), - ] + MismatchOutputWf(x=1, y=[1, 2, 3]) + + @workflow.define(outputs={"alpha": int, "beta": ty.List[int]}) + def Workflow(x: int, y: ty.List[int]): + A = workflow.add( # Split over workflow input "x" on "scalar" input + ListMultSum( + scalar=x, + in_list=y, + ) ) + return A.sum, A.products - wf.set_output([("alpha", wf.A.lzout.sum), ("beta", wf.A.lzout.products)]) + outputs = Workflow(x=10, y=[1, 2, 3, 4])() + assert outputs.sum == 10 + assert outputs.products == [10, 20, 30, 40] diff --git a/pydra/engine/tests/utils.py b/pydra/engine/tests/utils.py index 47ba21e4c..4bdae926a 100644 --- a/pydra/engine/tests/utils.py +++ b/pydra/engine/tests/utils.py @@ -9,6 +9,7 @@ import subprocess as sp import pytest from fileformats.generic import File +from pydra.engine.helpers import list_fields from pydra.engine.specs import ShellDef from ..submitter import Submitter from pydra.design import workflow, python @@ -38,6 +39,10 @@ ) +def get_output_names(task): + return sorted(f.name for f in list_fields(task.Outputs)) + + def run_no_submitter( shell_def: ShellDef, cache_dir: Path | None = None, diff --git a/pydra/utils/tests/utils.py b/pydra/utils/tests/utils.py index 12cfa74c7..b178f2df2 100644 --- a/pydra/utils/tests/utils.py +++ b/pydra/utils/tests/utils.py @@ -63,7 +63,6 @@ class SpecificShellTask(specs.ShellDef["SpecificShellTask.Outputs"]): help="the input file", argstr="", copy_mode="copy", - sep=" ", ) class Outputs(specs.ShellOutputs): @@ -86,7 +85,6 @@ class OtherSpecificShellTask(ShellDef): help="the input file", argstr="", copy_mode="copy", - sep=" ", ) class Outputs(specs.ShellOutputs): diff --git a/pydra/utils/typing.py b/pydra/utils/typing.py index 6c538efaa..d7aa09309 100644 --- a/pydra/utils/typing.py +++ b/pydra/utils/typing.py @@ -1058,6 +1058,12 @@ def optional_type(type_: type) -> type: return type_ +def is_multi_input(type_: type) -> bool: + """Check if the type is a MultiInputObj""" + type_ = optional_type(type_) + return MultiInputObj in (type_, ty.get_origin(type_)) + + def is_fileset_or_union(type_: type, allow_none: bool | None = None) -> bool: """Check if the type is a FileSet or a Union containing a FileSet