Merge pull request #649 from crim-ca/cwl-workflow-steps

.pylintrc

@@ -95,6 +95,7 @@ disable=C0111,missing-docstring,
         R0401,cyclic-import,
         R0801,duplicate-code,
         R0901,too-many-ancestors,
+        R0902,too-many-instance-attributes,
         R0904,too-many-public-methods,
         R0912,too-many-branches,
         R0914,too-many-locals,

CHANGES.rst

@@ -12,10 +12,35 @@ Changes
 
 Changes:
 --------
-- No change.
-
-Fixes:
-------
+- Add `CWL` ``MultipleInputFeatureRequirement`` support.
+- Add `CWL` ``SubworkflowFeatureRequirement`` support.
+- Add `CWL` ``Workflow`` explicit schema validation of its ``steps``.
+- Remove "unknown" definitions in `CWL` ``requirements``. Only fully defined and resolved definitions will be allowed.
+  If an unsupported `CWL` requirement by `Weaver` must be provided (but is a valid definition supported by ``cwltool``),
+  it must now be provided through ``hints`` to succeed schema validation.
+- Improve support of `CWL` output definition using ``loadContents`` to an ``outputBinding.glob`` reference to
+  load the ``File`` contents into a ``string`` output.
+- Improve support of `CWL` JavaScript expressions within intermediate steps of a ``Workflow`` to collect output results
+  from relevant sources with better data manipulation flexibility.
+- Modify signature of ``weaver.processes.wps_process_base.WpsProcessInterface`` to allow better reuse of the
+  common operations shared by derived `CWL` ``Workflow`` steps implemented by ``ESGFProcess``, ``Wps1Process``,
+  ``Wps3Process`` and ``OGCAPIRemoteProcessBase``.
+- Refactor ``ESGFProcess`` to use the common operations of `CWL` ``Workflow`` steps defined by ``WpsProcessInterface``.
+
+Fixes:
+------
+- Fix invalid `CWL` schema definition for ``ScatterFeatureRequirement`` that directly
+  contained the corresponding fields ``scatter`` and ``scatterMethod``, instead of the expected
+  definition within a `Workflow Step <https://www.commonwl.org/v1.2/Workflow.html#WorkflowStep>`_.
+- Fix `CWL` ``requirements`` schema definition using ``OneOf`` and the ``discriminator`` property that could sometime
+  drop a definition when it only contained an empty mapping ``{}``, and that the corresponding requirement allows it.
+- Fix ``weaver.wps_restapi.colander_extras.AnyOfKeywordSchema`` not allowing distinct `JSON` structure ``type`` to be
+  combined simultaneously.
+- Fix `CWL` ``Workflow`` not retrieving output results when returned directly as literal data from a remote `Process`.
+- Fix `CWL` ``Workflow`` potentially failing tool resolution for a local step `Process` if ``hints`` where omitted.
+- Fix `CWL` ``Workflow`` resolution of step ``requirements`` from one of the `Weaver` application types
+  (i.e.: ``builtin``, ``docker``, ``ESGF-CWT``, ``OGCAPI``, ``WPS1``) due to ``cwltool`` namespace adding a
+  prefixed URI.
 - Pin ``requests!=2.32`` to avoid issue with ``docker-py`` custom adapter not (yet) supporting it
   (relates to `psf/requests#6710 <https://github.com/psf/requests/pull/6710>`_
   and `docker/docker-py#3257 <https://github.com/docker/docker-py/pull/3257>`_).

tests/functional/application-packages/Echo/echo.cwl

@@ -15,3 +15,4 @@ outputs:
     # note: format omitted on purpose to let Weaver Process/CWL resolution generate the IANA namespace mapping
     outputBinding:
       glob: "stdout.log"
+stdout: stdout.log

tests/functional/application-packages/PassthroughExpressions/deploy.yml

@@ -0,0 +1,7 @@
+processDescription:
+  process:
+    id: PassthroughExpressions
+executionUnit:
+  # note: This does not work by itself! The test suite injects the file dynamically.
+  - test: "tests/functional/application-packages/PassthroughExpressions/package.cwl"
+deploymentProfileName: "http://www.opengis.net/profiles/eoc/dockerizedApplication"

tests/functional/application-packages/PassthroughExpressions/execute.yml

@@ -0,0 +1,3 @@
+inputs:
+  message: msg
+  code: 123456

tests/functional/application-packages/PassthroughExpressions/package.cwl

@@ -0,0 +1,34 @@
+cwlVersion: v1.2
+class: CommandLineTool
+baseCommand: "true"
+requirements:
+  - class: InlineJavascriptRequirement
+  - class: DockerRequirement
+    dockerPull: docker.io/debian:stable-slim
+inputs:
+  message:
+    type: string
+  code:
+    type: int
+  autogen:
+    type: float
+    default: 3.1416
+    inputBinding:
+      valueFrom: ${ return self; }
+outputs:
+  message:
+    type: string
+    outputBinding:
+      outputEval: ${ return inputs.message }
+  code:
+    type: int
+    outputBinding:
+      outputEval: $(inputs.code)
+  number:
+    type: float
+    outputBinding:
+      outputEval: $(inputs.autogen)
+  integer:
+    type: int
+    outputBinding:
+      outputEval: ${ return parseInt(inputs.autogen) }

tests/functional/application-packages/ReadFile/deploy.yml

@@ -19,5 +19,5 @@ outputTransmission:
   - reference
 executionUnit:
   # note: This does not work by itself! The test suite injects the file dynamically.
-  - href: "tests/functional/application-packages/ReadValue/package.cwl"
+  - href: "tests/functional/application-packages/ReadFile/package.cwl"
 deploymentProfileName: "http://www.opengis.net/profiles/eoc/dockerizedApplication"

tests/functional/application-packages/ReadFile/execute.yml

@@ -0,0 +1,3 @@
+# WARNING:
+#   This is a pseudo-definition only to avoid unnecessary requests to external definitions for this application.
+{}

tests/functional/application-packages/ReadFile/package.cwl

@@ -16,3 +16,4 @@ outputs:
       glob: "stdout.log"
       loadContents: true
       outputEval: $(self[0].contents)
+stdout: stdout.log

tests/functional/application-packages/WorkflowDirectoryListing/execute.yml

@@ -0,0 +1,5 @@
+# WARNING:
+#   This is a pseudo-definition only to avoid unnecessary requests to external definitions for this Workflow.
+#   The actual execution body is overridden by the test itself.
+#   See: 'tests.functional.test_workflow.WorkflowTestCase.test_workflow_directory_input_output_chaining'
+{}

tests/functional/application-packages/WorkflowPassthroughExpressions/deploy.yml

@@ -0,0 +1,7 @@
+processDescription:
+  process:
+    id: WorkflowPassthroughExpressions
+executionUnit:
+  # note: This does not work by itself! The test suite injects the file dynamically.
+  - test: "tests/functional/application-packages/WorkflowPassthroughExpressions/package.cwl"
+deploymentProfileName: "http://www.opengis.net/profiles/eoc/workflow"

tests/functional/application-packages/WorkflowPassthroughExpressions/execute.yml

@@ -0,0 +1,3 @@
+inputs:
+  message: msg
+  code: 123456

tests/functional/application-packages/WorkflowPassthroughExpressions/package.cwl

@@ -0,0 +1,58 @@
+cwlVersion: v1.2
+class: Workflow
+requirements:
+  - class: InlineJavascriptRequirement
+
+inputs:
+  message:
+    type: string
+  code:
+    type: int
+
+outputs:
+  message1:
+    type: string
+    outputSource: first/message
+  message2:
+    type: string
+    outputSource: second/message
+  code1:
+    type: int
+    outputSource: first/code
+  code2:
+    type: int
+    outputSource: second/code
+  number1:
+    type: float
+    outputSource: first/number
+  number2:
+    type: float
+    outputSource: second/number
+  integer1:
+    type: int
+    outputSource: first/integer
+  integer2:
+    type: int
+    outputSource: second/integer
+
+steps:
+  first:
+    run: PassthroughExpressions.cwl
+    in:
+      message: message
+      code: code
+    out:
+      - message
+      - code
+      - number
+      - integer
+  second:
+    run: PassthroughExpressions.cwl
+    in:
+      message: first/message
+      code: first/code
+    out:
+      - message
+      - code
+      - number
+      - integer

tests/functional/application-packages/WorkflowStepMerge/deploy.yml

@@ -0,0 +1,7 @@
+processDescription:
+  process:
+    id: WorkflowStepMerge
+executionUnit:
+  # note: This does not work by itself! The test suite injects the file dynamically.
+  - test: "tests/functional/application-packages/WorkflowStepMerge/package.cwl"
+deploymentProfileName: "http://www.opengis.net/profiles/eoc/workflow"

tests/functional/application-packages/WorkflowStepMerge/execute.yml

@@ -0,0 +1,3 @@
+inputs:
+  message1: Hello
+  message2: World

tests/functional/application-packages/WorkflowStepMerge/package.cwl

@@ -0,0 +1,36 @@
+cwlVersion: v1.2
+class: Workflow
+inputs:
+  message1:
+    type: string
+  message2:
+    type: string
+outputs:
+  output:
+    type: File
+    outputSource: concat/output
+requirements:
+  - class: MultipleInputFeatureRequirement
+  - class: InlineJavascriptRequirement
+  - class: StepInputExpressionRequirement
+  - class: SubworkflowFeatureRequirement
+steps:
+  msg1:
+    # evaluate 'SubworkflowFeatureRequirement'
+    run: WorkflowChainStrings.cwl
+    in:
+      message: message1
+    out: [output]
+  concat:
+    run: Echo.cwl
+    in:
+      message:
+        # evaluate 'MultipleInputFeatureRequirement'
+        source:
+          - msg1/output
+          - message2
+        # evaluate 'linkMerge' (schema definition)
+        # below value is the default that would be used if omitted
+        linkMerge: merge_nested
+        valueFrom: "${ return self.map(txt => txt.trim()).join(); }"
+    out: [output]