common-workflow-language · tetron · Nov 22, 2015 · Oct 14, 2015 · Oct 14, 2015 · Oct 14, 2015
diff --git a/cwltool/builder.py b/cwltool/builder.py
@@ -156,4 +156,7 @@ def generate_arg(self, binding):
         return [a for a in args if a is not None]
 
     def do_eval(self, ex, context=None, pull_image=True):
-        return expression.do_eval(ex, self.job, self.requirements, self.outdir, self.tmpdir, context=context, pull_image=pull_image)
+        return expression.do_eval(ex, self.job, self.requirements,
+                                  self.outdir, self.tmpdir,
+                                  self.resources,
+                                  context=context, pull_image=pull_image)
diff --git a/cwltool/expression.py b/cwltool/expression.py
@@ -12,10 +12,10 @@
 
 _logger = logging.getLogger("cwltool")
 
-def exeval(ex, jobinput, requirements, outdir, tmpdir, context, pull_image):
+def exeval(ex, jobinput, requirements, outdir, tmpdir, context, pull_image, resources):
     if ex["engine"] == "https://w3id.org/cwl/cwl#JsonPointer":
         try:
-            obj = {"job": jobinput, "context": context, "outdir": outdir, "tmpdir": tmpdir}
+            obj = {"job": jobinput, "context": context, "outdir": outdir, "tmpdir": tmpdir, "resources": resources}
             return schema_salad.ref_resolver.resolve_json_pointer(obj, ex["script"])
         except ValueError as v:
             raise WorkflowException("%s in %s" % (v,  obj))
@@ -52,6 +52,7 @@ class DR(object):
                 "context": context,
                 "outdir": outdir,
                 "tmpdir": tmpdir,
+                "resources": resources
             }
 
             _logger.debug("Invoking expression engine %s with %s",
@@ -72,8 +73,8 @@ class DR(object):
 
     raise WorkflowException("Unknown expression engine '%s'" % ex["engine"])
 
-def do_eval(ex, jobinput, requirements, outdir, tmpdir, context=None, pull_image=True):
+def do_eval(ex, jobinput, requirements, outdir, tmpdir, resources, context=None, pull_image=True):
     if isinstance(ex, dict) and "engine" in ex and "script" in ex:
-        return exeval(ex, jobinput, requirements, outdir, tmpdir, context, pull_image)
+        return exeval(ex, jobinput, requirements, outdir, tmpdir, context, pull_image, resources)
     else:
         return ex
diff --git a/cwltool/main.py b/cwltool/main.py
@@ -299,6 +299,7 @@ def load_tool(argsworkflow, updateonly, strict, makeTool, debug, print_pre=False
 def main(args=None,
          executor=single_job_executor,
          makeTool=workflow.defaultMakeTool,
+         selectResources=None,
          parser=None,
          stdin=sys.stdin,
          stdout=sys.stdout,
@@ -434,7 +435,8 @@ def main(args=None,
                        tmpdir_prefix=args.tmpdir_prefix,
                        rm_tmpdir=args.rm_tmpdir,
                        makeTool=makeTool,
-                       move_outputs=args.move_outputs
+                       move_outputs=args.move_outputs,
+                       select_resources=selectResources
                        )
         # This is the workflow output, it needs to be written
         stdout.write(json.dumps(out, indent=4))

diff --git a/cwltool/process.py b/cwltool/process.py
@@ -186,8 +186,48 @@ def _init_job(self, joborder, input_basedir, **kwargs):
 
         builder.bindings.extend(builder.bind_input(self.inputs_record_schema, builder.job))
 
+        builder.resources = {}
+        builder.resources = self.evalResources(builder, kwargs)
+
         return builder
 
+    def evalResources(self, builder, kwargs):
+        resourceReq, _ = self.get_requirement("ResourceRequirement")
+        request = {
+            "coresMin": 1,
+            "coresMax": 1,
+            "ramMin": 1024,
+            "ramMax": 1024,
+            "tmpdirMin": 1024,
+            "tmpdirMax": 1024,
+            "outdirMin": 1024,
+            "outdirMax": 1024
+        }
+        for a in ("cores", "ram", "tmpdir", "outdir"):
+            mn = None
+            mx = None
+            if resourceReq.get(a+"Min"):
+                mn = builder.do_eval(resourceReq[a+"Min"])
+            if resourceReq.get(a+"Max"):
+                mx = builder.do_eval(resourceReq[a+"Max"])
+            if mn is None:
+                mn = mx
+            elif mx is None:
+                mx = mn
+
+            if mn:
+                request[a+"Min"] = mn
+                request[a+"Max"] = mx
+
+        if kwargs.get("select_resources"):
+            return kwargs["select_resources"](request)
+        else:
+            return {
+                "cores": request["coresMin"],
+                "ram":   request["ramMin"],
+                "tmpdir": request["tmpdirMin"],
+                "outdir": request["outdirMin"],
+            }
 
     def validate_hints(self, hints, strict):
         for r in hints:

diff --git a/cwltool/schemas/draft-3/cwl-avro.yml b/cwltool/schemas/draft-3/cwl-avro.yml
@@ -363,6 +363,11 @@
           the designated temporary directory that will be used when executing
           the tool.  Null if not applicable.
 
+        - **resources**: When used in the context of a CommandLineTool, an object
+          describing the resources allocated to run the tool.  These fields are "cores",
+          "ram", "tmpdir", "outdir".  See ResourceRequirement for details.
+
+
       * On standard output, print a single JSON value (string, number, array, object,
         boolean, or null) for the return value.
 
@@ -2044,3 +2049,62 @@
         the expression engine.  The semantics of this field are defined by the
         underlying expression engine.  Intended for uses such as providing
         function definitions that will be called from CWL expressions.
+
+
+- type: record
+  name: ResourceRequirement
+  extends: "#ProcessRequirement"
+  doc: |
+    Specify basic hardware resource requirements.
+
+    "min" is the minimum amount of a resource that must be reserved to schedule
+    a job. If "min" cannot be satisfied, the job should not be run.
+
+    "max" is the maximum amount of a resource that the job shall be permitted
+    to use. If a node has sufficient resources, multiple jobs may be scheduled
+    on a single node provided each job's "max" resource requirements are
+    met. If a job attempts to exceed its "max" resource allocation, an
+    implementation may deny additional resources, which may result in job
+    failure.
+
+    If "min" is specified but "max" is not, then "max" == "min"
+    If "max" is specified by "min" is not, then "min" == "max".
+
+    It is an error if max < min.
+
+    It is an error if the value of any of these fields is negative.
+
+    If neither "min" nor "max" is specified for a resource, an implementation may provide a default.
+
+  fields:
+    - name: coresMin
+      type: ["null", long, string, "#Expression"]
+      doc: Minimum reserved number of CPU cores
+
+    - name: coresMax
+      type: ["null", int, string, "#Expression"]
+      doc: Maximum reserved number of CPU cores
+
+    - name: ramMin
+      type: ["null", long, string, "#Expression"]
+      doc: Minimum reserved RAM in mebibytes (2**20)
+
+    - name: ramMax
+      type: ["null", long, string, "#Expression"]
+      doc: Maximum reserved RAM in mebibytes (2**20)
+
+    - name: tmpdirMin
+      type: ["null", long, string, "#Expression"]
+      doc: Minimum reserved filesystem based storage for the designated temporary directory, in mebibytes (2**20)
+
+    - name: tmpdirMax
+      type: ["null", long, string, "#Expression"]
+      doc: Maximum reserved filesystem based storage for the designated temporary directory, in mebibytes (2**20)
+
+    - name: outdirMin
+      type: ["null", long, string, "#Expression"]
+      doc: Minimum reserved filesystem based storage for the designated output directory, in mebibytes (2**20)
+
+    - name: outdirMax
+      type: ["null", long, string, "#Expression"]
+      doc: Maximum reserved filesystem based storage for the designated output directory, in mebibytes (2**20)
diff --git a/cwltool/schemas/draft-3/draft-3/add-lines-wf.cwl b/cwltool/schemas/draft-3/draft-3/add-lines-wf.cwl
@@ -1 +1 @@
-../draft-2/add-lines-wf.cwl
+../../draft-2/draft-2/add-lines-wf.cwl
diff --git a/cwltool/schemas/draft-3/draft-3/binding-test.cwl b/cwltool/schemas/draft-3/draft-3/binding-test.cwl
@@ -1 +1 @@
-../draft-2/binding-test.cwl
+../../draft-2/draft-2/binding-test.cwl
diff --git a/cwltool/schemas/draft-3/draft-3/bwa-mem-job.json b/cwltool/schemas/draft-3/draft-3/bwa-mem-job.json
@@ -1 +1 @@
-../draft-2/bwa-mem-job.json
+../../draft-2/draft-2/bwa-mem-job.json
diff --git a/cwltool/schemas/draft-3/draft-3/cat-job.json b/cwltool/schemas/draft-3/draft-3/cat-job.json
@@ -1 +1 @@
-../draft-2/cat-job.json
+../../draft-2/draft-2/cat-job.json
diff --git a/cwltool/schemas/draft-3/draft-3/cat-n-job.json b/cwltool/schemas/draft-3/draft-3/cat-n-job.json
@@ -1 +1 @@
-../draft-2/cat-n-job.json
+../../draft-2/draft-2/cat-n-job.json
diff --git a/cwltool/schemas/draft-3/draft-3/cat1-tool.cwl b/cwltool/schemas/draft-3/draft-3/cat1-tool.cwl
@@ -1 +1 @@
-../draft-2/cat1-tool.cwl
+../../draft-2/draft-2/cat1-tool.cwl
diff --git a/cwltool/schemas/draft-3/draft-3/cat2-tool.cwl b/cwltool/schemas/draft-3/draft-3/cat2-tool.cwl
@@ -1 +1 @@
-../draft-2/cat2-tool.cwl
+../../draft-2/draft-2/cat2-tool.cwl
diff --git a/cwltool/schemas/draft-3/draft-3/cat3-tool.cwl b/cwltool/schemas/draft-3/draft-3/cat3-tool.cwl
@@ -1 +1 @@
-../draft-2/cat3-tool.cwl
+../../draft-2/draft-2/cat3-tool.cwl
diff --git a/cwltool/schemas/draft-3/draft-3/cat4-tool.cwl b/cwltool/schemas/draft-3/draft-3/cat4-tool.cwl
@@ -1 +1 @@
-../draft-2/cat4-tool.cwl
+../../draft-2/draft-2/cat4-tool.cwl
diff --git a/cwltool/schemas/draft-3/draft-3/cat5-tool.cwl b/cwltool/schemas/draft-3/draft-3/cat5-tool.cwl
@@ -1 +1 @@
-../draft-2/cat5-tool.cwl
+../../draft-2/draft-2/cat5-tool.cwl
diff --git a/cwltool/schemas/draft-3/draft-3/count-lines3-job.json b/cwltool/schemas/draft-3/draft-3/count-lines3-job.json
@@ -1 +1 @@
-../draft-2/count-lines3-job.json
+../../draft-2/draft-2/count-lines3-job.json
diff --git a/cwltool/schemas/draft-3/draft-3/count-lines4-job.json b/cwltool/schemas/draft-3/draft-3/count-lines4-job.json
@@ -1 +1 @@
-../draft-2/count-lines4-job.json
+../../draft-2/draft-2/count-lines4-job.json
diff --git a/cwltool/schemas/draft-3/draft-3/count-lines6-job.json b/cwltool/schemas/draft-3/draft-3/count-lines6-job.json
@@ -1 +1 @@
-../draft-2/count-lines6-job.json
+../../draft-2/draft-2/count-lines6-job.json
diff --git a/cwltool/schemas/draft-3/draft-3/echo-tool.cwl b/cwltool/schemas/draft-3/draft-3/echo-tool.cwl
@@ -1 +1 @@
-../draft-2/echo-tool.cwl
+../../draft-2/draft-2/echo-tool.cwl
diff --git a/cwltool/schemas/draft-3/draft-3/empty.json b/cwltool/schemas/draft-3/draft-3/empty.json
@@ -1 +1 @@
-../draft-2/empty.json
+../../draft-2/draft-2/empty.json
diff --git a/cwltool/schemas/draft-3/draft-3/env-job.json b/cwltool/schemas/draft-3/draft-3/env-job.json
@@ -1 +1 @@
-../draft-2/env-job.json
+../../draft-2/draft-2/env-job.json
diff --git a/cwltool/schemas/draft-3/draft-3/env-tool1.cwl b/cwltool/schemas/draft-3/draft-3/env-tool1.cwl
@@ -1 +1 @@
-../draft-2/env-tool1.cwl
+../../draft-2/draft-2/env-tool1.cwl
diff --git a/cwltool/schemas/draft-3/draft-3/env-tool2.cwl b/cwltool/schemas/draft-3/draft-3/env-tool2.cwl
@@ -1 +1 @@
-../draft-2/env-tool2.cwl
+../../draft-2/draft-2/env-tool2.cwl
diff --git a/cwltool/schemas/draft-3/draft-3/hello.txt b/cwltool/schemas/draft-3/draft-3/hello.txt
@@ -1 +1 @@
-../draft-2/hello.txt
+../../draft-2/draft-2/hello.txt
diff --git a/cwltool/schemas/draft-3/draft-3/index.py b/cwltool/schemas/draft-3/draft-3/index.py
@@ -1 +1 @@
-../draft-2/index.py
+../../draft-2/draft-2/index.py
diff --git a/cwltool/schemas/draft-3/draft-3/node-engine.cwl b/cwltool/schemas/draft-3/draft-3/node-engine.cwl
@@ -1 +1 @@
-../draft-2/node-engine.cwl
+../../draft-2/draft-2/node-engine.cwl
diff --git a/cwltool/schemas/draft-3/draft-3/number.txt b/cwltool/schemas/draft-3/draft-3/number.txt
@@ -1 +1 @@
-../draft-2/number.txt
+../../draft-2/draft-2/number.txt
diff --git a/cwltool/schemas/draft-3/draft-3/parseInt-job.json b/cwltool/schemas/draft-3/draft-3/parseInt-job.json
@@ -1 +1 @@
-../draft-2/parseInt-job.json
+../../draft-2/draft-2/parseInt-job.json
diff --git a/cwltool/schemas/draft-3/draft-3/rename-job.json b/cwltool/schemas/draft-3/draft-3/rename-job.json
@@ -1 +1 @@
-../draft-2/rename-job.json
+../../draft-2/draft-2/rename-job.json
diff --git a/cwltool/schemas/draft-3/draft-3/rename.cwl b/cwltool/schemas/draft-3/draft-3/rename.cwl
@@ -1 +1 @@
-../draft-2/rename.cwl
+../../draft-2/draft-2/rename.cwl
diff --git a/cwltool/schemas/draft-3/draft-3/revsort-job.json b/cwltool/schemas/draft-3/draft-3/revsort-job.json
@@ -1 +1 @@
-../draft-2/revsort-job.json
+../../draft-2/draft-2/revsort-job.json
diff --git a/cwltool/schemas/draft-3/draft-3/revtool.cwl b/cwltool/schemas/draft-3/draft-3/revtool.cwl
@@ -1 +1 @@
-../draft-2/revtool.cwl
+../../draft-2/draft-2/revtool.cwl
diff --git a/cwltool/schemas/draft-3/draft-3/scatter-job1.json b/cwltool/schemas/draft-3/draft-3/scatter-job1.json
@@ -1 +1 @@
-../draft-2/scatter-job1.json
+../../draft-2/draft-2/scatter-job1.json
diff --git a/cwltool/schemas/draft-3/draft-3/scatter-job2.json b/cwltool/schemas/draft-3/draft-3/scatter-job2.json
@@ -1 +1 @@
-../draft-2/scatter-job2.json
+../../draft-2/draft-2/scatter-job2.json
diff --git a/cwltool/schemas/draft-3/draft-3/scatter-wf1.cwl b/cwltool/schemas/draft-3/draft-3/scatter-wf1.cwl
@@ -1 +1 @@
-../draft-2/scatter-wf1.cwl
+../../draft-2/draft-2/scatter-wf1.cwl
diff --git a/cwltool/schemas/draft-3/draft-3/scatter-wf2.cwl b/cwltool/schemas/draft-3/draft-3/scatter-wf2.cwl
@@ -1 +1 @@
-../draft-2/scatter-wf2.cwl
+../../draft-2/draft-2/scatter-wf2.cwl
diff --git a/cwltool/schemas/draft-3/draft-3/search-job.json b/cwltool/schemas/draft-3/draft-3/search-job.json
@@ -1 +1 @@
-../draft-2/search-job.json
+../../draft-2/draft-2/search-job.json
diff --git a/cwltool/schemas/draft-3/draft-3/search.py b/cwltool/schemas/draft-3/draft-3/search.py
@@ -1 +1 @@
-../draft-2/search.py
+../../draft-2/draft-2/search.py
diff --git a/cwltool/schemas/draft-3/draft-3/sorttool.cwl b/cwltool/schemas/draft-3/draft-3/sorttool.cwl
@@ -1 +1 @@
-../draft-2/sorttool.cwl
+../../draft-2/draft-2/sorttool.cwl
diff --git a/cwltool/schemas/draft-3/draft-3/tmap-job.json b/cwltool/schemas/draft-3/draft-3/tmap-job.json
@@ -1 +1 @@
-../draft-2/tmap-job.json
+../../draft-2/draft-2/tmap-job.json
diff --git a/cwltool/schemas/draft-3/draft-3/underscore.js b/cwltool/schemas/draft-3/draft-3/underscore.js
@@ -1 +1 @@
-../draft-2/underscore.js
+../../draft-2/draft-2/underscore.js
diff --git a/cwltool/schemas/draft-3/draft-3/wc-job.json b/cwltool/schemas/draft-3/draft-3/wc-job.json
@@ -1 +1 @@
-../draft-2/wc-job.json
+../../draft-2/draft-2/wc-job.json
diff --git a/cwltool/schemas/draft-3/draft-3/wc-tool.cwl b/cwltool/schemas/draft-3/draft-3/wc-tool.cwl
@@ -1 +1 @@
-../draft-2/wc-tool.cwl
+../../draft-2/draft-2/wc-tool.cwl
diff --git a/cwltool/schemas/draft-3/draft-3/whale.txt b/cwltool/schemas/draft-3/draft-3/whale.txt
@@ -1 +1 @@
-../draft-2/whale.txt
+../../draft-2/draft-2/whale.txt
diff --git a/node-expr-engine/cwlNodeEngine.js b/node-expr-engine/cwlNodeEngine.js
@@ -34,6 +34,7 @@ process.stdin.on('end', function() {
 
     fn += "var $job = " + JSON.stringify(j.job) + ";\n";
     fn += "var $self = " + JSON.stringify(j.context) + ";\n"
+    fn += "var $resources = " + JSON.stringify(j.resources) + ";\n"
 
     fn += "(function()" + exp + ")()";
Original file line number	Diff line number	Diff line change
		@@ -1 +1 @@
		../draft-2/add-lines-wf.cwl
		../../draft-2/draft-2/add-lines-wf.cwl
Original file line number	Diff line number	Diff line change
		@@ -1 +1 @@
		../draft-2/binding-test.cwl
		../../draft-2/draft-2/binding-test.cwl
Original file line number	Diff line number	Diff line change
		@@ -1 +1 @@
		../draft-2/bwa-mem-job.json
		../../draft-2/draft-2/bwa-mem-job.json
Original file line number	Diff line number	Diff line change
		@@ -1 +1 @@
		../draft-2/cat-job.json
		../../draft-2/draft-2/cat-job.json
Original file line number	Diff line number	Diff line change
		@@ -1 +1 @@
		../draft-2/cat-n-job.json
		../../draft-2/draft-2/cat-n-job.json
Original file line number	Diff line number	Diff line change
		@@ -1 +1 @@
		../draft-2/cat1-tool.cwl
		../../draft-2/draft-2/cat1-tool.cwl
Original file line number	Diff line number	Diff line change
		@@ -1 +1 @@
		../draft-2/cat2-tool.cwl
		../../draft-2/draft-2/cat2-tool.cwl
Original file line number	Diff line number	Diff line change
		@@ -1 +1 @@
		../draft-2/cat3-tool.cwl
		../../draft-2/draft-2/cat3-tool.cwl
Original file line number	Diff line number	Diff line change
		@@ -1 +1 @@
		../draft-2/cat4-tool.cwl
		../../draft-2/draft-2/cat4-tool.cwl
Original file line number	Diff line number	Diff line change
		@@ -1 +1 @@
		../draft-2/cat5-tool.cwl
		../../draft-2/draft-2/cat5-tool.cwl