Add javascript code snippet validation

MANIFEST.in

@@ -19,6 +19,9 @@ include cwltool/schemas/v1.1.0-dev1/salad/schema_salad/metaschema/*.yml
 include cwltool/schemas/v1.1.0-dev1/salad/schema_salad/metaschema/*.md
 include cwltool/cwlNodeEngine.js
 include cwltool/cwlNodeEngineJSConsole.js
+include cwltool/cwlNodeEngineWithContext.js
 include cwltool/extensions.yml
+include cwltool/jshint/jshint_wrapper.js
+include cwltool/jshint/jshint.js
 global-exclude *~
 global-exclude *.pyc

cwltool/argparser.py

@@ -152,6 +152,11 @@ def arg_parser():  # type: () -> argparse.ArgumentParser
                         "timestamps to the errors, warnings, and "
                         "notifications.")
     parser.add_argument("--js-console", action="store_true", help="Enable javascript console output")
+    parser.add_argument("--disable-js-validation", action="store_true", help="Disable javascript validation.")
+    parser.add_argument("--js-hint-options-file",
+                        type=Text,
+                        help="File of options to pass to jshint."
+                        "This includes the added option \"includewarnings\". ")
     dockergroup = parser.add_mutually_exclusive_group()
     dockergroup.add_argument("--user-space-docker-cmd", metavar="CMD",
                         help="(Linux/OS X only) Specify a user space docker "

cwltool/cwlNodeEngineWithContext.js

@@ -0,0 +1,30 @@
+"use strict";
+process.stdin.setEncoding("utf8");
+var incoming = "";
+var firstInput = true;
+var context;
+
+process.stdin.on("data", function(chunk) {
+  incoming += chunk;
+  var i = incoming.indexOf("\n");
+  if (i > -1) {
+    try{
+      var fn = JSON.parse(incoming.substr(0, i));
+      incoming = incoming.substr(i+1);
+      if(firstInput){
+        context = require("vm").runInNewContext(fn, {});
+        firstInput = false;
+      }
+      else{
+        process.stdout.write(JSON.stringify(require("vm").runInNewContext(fn, context)) + "\n");
+      }
+    }
+    catch(e){
+      console.error(e)
+    }
+    /*strings to indicate the process has finished*/
+    console.log("r1cepzbhUTxtykz5XTC4");
+    console.error("r1cepzbhUTxtykz5XTC4");
+  }
+});
+process.stdin.on("end", process.exit);

cwltool/jshint/jshint_wrapper.js

@@ -0,0 +1,47 @@
+"use strict";
+// set a global object, in order for jshint to work
+var global = this;
+
+function validateJS(input) {
+  var jshintGlobalsObj = {};
+  input.globals.forEach(function (global) {
+    jshintGlobalsObj[global] = true;
+  })
+  var includewarnings;
+
+  if (input.options.includewarnings !== undefined) {
+    includewarnings = input.options.includewarnings;
+    delete input.options.includewarnings;
+  }
+
+  JSHINT(
+    input.code,
+    input.options,
+    jshintGlobalsObj
+  )
+
+  var jshintData = JSHINT.data();
+  if (jshintData.errors !== undefined) {
+    if (includewarnings !== undefined) {
+      jshintData.errors = jshintData.errors.filter(function (error) {
+        return includewarnings.indexOf(error.code) !== -1 || error.code[0] == "E";
+      })
+    }
+
+    jshintData.errors.forEach(function (error) {
+      if (error.code == "W104" || error.code == "W119") {
+        if (error.code == "W104"){
+          var jslint_suffix = " (use 'esversion: 6') or Mozilla JS extensions (use moz)."
+        }
+        else{
+          var jslint_suffix = " (use 'esversion: 6')"
+        }
+
+        error.reason = error.reason.slice(0, -jslint_suffix.length - 1) +
+          ". CWL only supports ES5.1";
+      }
+    })
+  }
+
+  return jshintData;
+}

cwltool/load_tool.py

@@ -25,6 +25,7 @@
 
 from . import process, update
 from .errors import WorkflowException
+from .validate_js import validate_js_expressions
 from .process import Process, shortname, get_schema
 from .update import ALLUPDATES
 
@@ -188,6 +189,8 @@ def validate_document(document_loader,  # type: Loader
                       skip_schemas=None,  # type: bool
                       overrides=None,  # type: List[Dict]
                       metadata=None,  # type: Optional[Dict]
+                      should_validate_js=True, # type: bool
+                      validate_js_options=None # type: Dict
                       ):
     # type: (...) -> Tuple[Loader, Names, Union[Dict[Text, Any], List[Dict[Text, Any]]], Dict[Text, Any], Text]
     """Validate a CWL document."""
@@ -281,6 +284,9 @@ def validate_document(document_loader,  # type: Loader
     if overrides:
         new_metadata[u"cwltool:overrides"] = overrides
 
+    if workflowobj.get("class") is not None and should_validate_js:
+        validate_js_expressions(processobj, avsc_names.names[workflowobj["class"]], validate_js_options)
+
     return document_loader, avsc_names, processobj, new_metadata, uri
 
 

cwltool/main.py

@@ -471,13 +471,25 @@ def main(argsl=None,  # type: List[str]
                 printdeps(workflowobj, document_loader, stdout, args.relative_deps, uri)
                 return 0
 
+            if args.js_hint_options_file is not None:
+                try:
+                    with open(args.js_hint_options_file) as options_file:
+                        validate_js_options = json.load(options_file)
+                except (OSError, ValueError) as e:
+                    _logger.error("Failed to read options file %s" % args.js_hint_options_file)
+                    raise e
+            else:
+                validate_js_options = None
+
             document_loader, avsc_names, processobj, metadata, uri \
                 = validate_document(document_loader, workflowobj, uri,
                                     enable_dev=args.enable_dev, strict=args.strict,
                                     preprocess_only=args.print_pre or args.pack,
                                     fetcher_constructor=fetcher_constructor,
                                     skip_schemas=args.skip_schemas,
-                                    overrides=overrides)
+                                    overrides=overrides,
+                                    should_validate_js=not args.disable_js_validation,
+                                    validate_js_options=validate_js_options)
 
             if args.print_pre:
                 stdout.write(json.dumps(processobj, indent=4))

cwltool/sandboxjs.py

@@ -10,7 +10,7 @@
 import sys
 from io import BytesIO
 from typing import Any, Dict, List, Mapping, Text, Tuple, Union
-from .utils import onWindows
+from .utils import onWindows, get_data
 from pkg_resources import resource_stream
 
 import six
@@ -54,16 +54,8 @@ def check_js_threshold_version(working_alias):
         return False
 
 
-def new_js_proc(force_docker_pull=False, js_console=False):
-    # type: (bool, bool) -> subprocess.Popen
-
-    cwl_node_engine_js = 'cwlNodeEngine.js'
-    if js_console:
-        cwl_node_engine_js = 'cwlNodeEngineJSConsole.js'
-        _logger.warn("Running with support for javascript console in expressions (DO NOT USE IN PRODUCTION)")
-
-    res = resource_stream(__name__, cwl_node_engine_js)
-    nodecode = res.read().decode('utf-8')
+def new_js_proc(js_text, force_docker_pull=False):
+    # type: (Text, bool) -> subprocess.Popen
 
     required_node_version, docker = (False,)*2
     nodejs = None
@@ -73,7 +65,7 @@ def new_js_proc(force_docker_pull=False, js_console=False):
             if subprocess.check_output([n, "--eval", "process.stdout.write('t')"]).decode('utf-8') != "t":
                 continue
             else:
-                nodejs = subprocess.Popen([n, "--eval", nodecode],
+                nodejs = subprocess.Popen([n, "--eval", js_text],
                                           stdin=subprocess.PIPE,
                                           stdout=subprocess.PIPE,
                                           stderr=subprocess.PIPE)
@@ -104,7 +96,7 @@ def new_js_proc(force_docker_pull=False, js_console=False):
             nodejs = subprocess.Popen(["docker", "run",
                                        "--attach=STDIN", "--attach=STDOUT", "--attach=STDERR",
                                        "--sig-proxy=true", "--interactive",
-                                       "--rm", nodeimg, "node", "--eval", nodecode],
+                                       "--rm", nodeimg, "node", "--eval", js_text],
                                       stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
             docker = True
         except OSError as e:
@@ -118,9 +110,9 @@ def new_js_proc(force_docker_pull=False, js_console=False):
     # docker failed and nodejs not on system
     if nodejs is None:
         raise JavascriptException(
-            u"cwltool requires Node.js engine to evaluate Javascript "
-            "expressions, but couldn't find it.  Tried %s, docker run "
-            "node:slim" % u", ".join(trynodes))
+            u"cwltool requires Node.js engine to evaluate and validate "
+            u"Javascript expressions, but couldn't find it.  Tried %s, "
+            u"docker run node:slim" % u", ".join(trynodes))
 
     # docker failed, but nodejs is installed on system but the version is below the required version
     if docker is False and required_node_version is False:
@@ -131,15 +123,39 @@ def new_js_proc(force_docker_pull=False, js_console=False):
     return nodejs
 
 
-def execjs(js, jslib, timeout=None, force_docker_pull=False, debug=False, js_console=False):  # type: (Union[Mapping, Text], Any, int, bool, bool, bool) -> JSON
+def exec_js_process(js_text, timeout=None, js_console=False, context=None, force_docker_pull=False, debug=False):
+    # type: (Text, int, bool, Text, bool, bool) -> Tuple[int, Text, Text]
+
+    if not hasattr(localdata, "procs"):
+        localdata.procs = {}
+
+    if js_console and context is not None:
+        raise NotImplementedError("js_console=True and context not implemented")
+
+    if js_console:
+        js_engine = 'cwlNodeEngineJSConsole.js'
+        _logger.warn("Running with support for javascript console in expressions (DO NOT USE IN PRODUCTION)")
+    elif context is not None:
+        js_engine = "cwlNodeEngineWithContext.js"
+    else:
+        js_engine = 'cwlNodeEngine.js'
 
-    if not hasattr(localdata, "proc") or localdata.proc.poll() is not None or onWindows():
-        localdata.proc = new_js_proc(force_docker_pull=force_docker_pull, js_console=js_console)
+    if localdata.procs.get(js_engine) is None \
+            or localdata.procs[js_engine].poll() is not None \
+            or onWindows():
+        res = resource_stream(__name__, js_engine)
+        js_engine_code = res.read().decode('utf-8')
 
-    nodejs = localdata.proc
+        localdata.procs[js_engine] = new_js_proc(js_engine_code, force_docker_pull=force_docker_pull)
+        if context is not None:
+            output = exec_js_process(context, timeout=timeout, js_console=js_console, context=context, force_docker_pull=force_docker_pull, debug=debug)
+            if output != (0, "", ""):
+                # restart to reset the context
+                localdata.procs[js_engine].kill()
+                del localdata.procs[js_engine]
+                return output
 
-    fn = u"\"use strict\";\n%s\n(function()%s)()" %\
-         (jslib, js if isinstance(js, six.string_types) and len(js) > 1 and js[0] == '{' else ("{return (%s);}" % js))
+    nodejs = localdata.procs[js_engine]
 
     killed = []
 
@@ -157,7 +173,7 @@ def terminate():
     tm = threading.Timer(timeout, terminate)
     tm.start()
 
-    stdin_buf = BytesIO((json.dumps(fn) + "\n").encode('utf-8'))
+    stdin_buf = BytesIO((json.dumps(js_text) + "\n").encode('utf-8'))
     stdout_buf = BytesIO()
     stderr_buf = BytesIO()
 
@@ -167,8 +183,8 @@ def terminate():
     PROCESS_FINISHED_STR = "r1cepzbhUTxtykz5XTC4\n"
 
     def process_finished(): # type: () -> bool
-        return stdout_buf.getvalue().decode().endswith(PROCESS_FINISHED_STR) and \
-            stderr_buf.getvalue().decode().endswith(PROCESS_FINISHED_STR)
+        return stdout_buf.getvalue().decode('utf-8').endswith(PROCESS_FINISHED_STR) and \
+            stderr_buf.getvalue().decode('utf-8').endswith(PROCESS_FINISHED_STR)
 
     # On windows system standard input/output are not handled properly by select module
     # (modules like  pywin32, msvcrt, gevent don't work either)
@@ -235,7 +251,7 @@ def get_error(error_queue):
 
                 if nodejs.stderr in rselect:
                     if not error_queue.empty():
-                        stderr_buf.write(error_queue.get())                
+                        stderr_buf.write(error_queue.get())
 
                 if process_finished() and error_queue.empty() and output_queue.empty():
                     finished = True
@@ -265,6 +281,50 @@ def get_error(error_queue):
     stdoutdata = stdout_buf.getvalue()[:-len(PROCESS_FINISHED_STR) - 1]
     stderrdata = stderr_buf.getvalue()[:-len(PROCESS_FINISHED_STR) - 1]
 
+    nodejs.poll()
+
+    if nodejs.poll() not in (None, 0):
+        if killed:
+            returncode = -1
+        else:
+            returncode = nodejs.returncode
+    else:
+        returncode = 0
+        # On windows currently a new instance of nodejs process is used due to problem with blocking on read operation on windows
+        if onWindows():
+            nodejs.kill()
+
+    return returncode, stdoutdata.decode('utf-8'), stderrdata.decode('utf-8')
+
+def code_fragment_to_js(js, jslib=""):
+    # type: (Text, Text) -> Text
+    if isinstance(js, six.string_types) and len(js) > 1 and js[0] == '{':
+        inner_js = js
+    else:
+        inner_js = "{return (%s);}" % js
+
+    return u"\"use strict\";\n%s\n(function()%s)()" % (jslib, inner_js)
+
+def execjs(js, jslib, timeout=None, force_docker_pull=False, debug=False, js_console=False):
+    # type: (Text, Text, int, bool, bool, bool) -> JSON
+
+    fn = code_fragment_to_js(js, jslib)
+
+    returncode, stdout, stderr = exec_js_process(
+        fn, timeout=timeout, js_console=js_console, force_docker_pull=force_docker_pull, debug=debug)
+
+    if js_console:
+        if len(stderr) > 0:
+            _logger.info("Javascript console output:")
+            _logger.info("----------------------------------------")
+            _logger.info('\n'.join(re.findall(r'^[[](?:log|err)[]].*$', stderr, flags=re.MULTILINE)))
+            _logger.info("----------------------------------------")
+
+    def stdfmt(data):  # type: (Text) -> Text
+        if "\n" in data:
+            return "\n" + data.strip()
+        return data
+
     def fn_linenum():  # type: () -> Text
         lines = fn.splitlines()
         ofs = 0
@@ -274,38 +334,21 @@ def fn_linenum():  # type: () -> Text
             lines = lines[-maxlines:]
         return u"\n".join(u"%02i %s" % (i + ofs + 1, b) for i, b in enumerate(lines))
 
-    def stdfmt(data):  # type: (Text) -> Text
-        if "\n" in data:
-            return "\n" + data.strip()
-        return data
-
-    nodejs.poll()
-
-    if js_console:
-        if len(stderrdata) > 0:
-            _logger.info("Javascript console output:")
-            _logger.info("----------------------------------------")
-            _logger.info('\n'.join(re.findall(r'^[[](?:log|err)[]].*$', stderrdata.decode('utf-8'), flags=re.MULTILINE)))
-            _logger.info("----------------------------------------")
-
-    if debug:
-        info = u"returncode was: %s\nscript was:\n%s\nstdout was: %s\nstderr was: %s\n" %\
-               (nodejs.returncode, fn_linenum(), stdfmt(stdoutdata.decode('utf-8')), stdfmt(stderrdata.decode('utf-8')))
-    else:
-        info = u"Javascript expression was: %s\nstdout was: %s\nstderr was: %s" %\
-               (js, stdfmt(stdoutdata.decode('utf-8')), stdfmt(stderrdata.decode('utf-8')))
+    if returncode != 0:
+        if debug:
+            info = u"returncode was: %s\nscript was:\n%s\nstdout was: %s\nstderr was: %s\n" %\
+                (returncode, fn_linenum(), stdfmt(stdout), stdfmt(stderr))
+        else:
+            info = u"Javascript expression was: %s\nstdout was: %s\nstderr was: %s" %\
+                (js, stdfmt(stdout), stdfmt(stderr))
 
-    if nodejs.poll() not in (None, 0):
-        if killed:
+        if returncode == -1:
             raise JavascriptException(u"Long-running script killed after %s seconds: %s" % (timeout, info))
         else:
             raise JavascriptException(info)
-    else:
-        try:
-            # On windows currently a new instance of nodejs process is used due to problem with blocking on read operation on windows
-            if onWindows():
-                nodejs.kill()
-            return json.loads(stdoutdata.decode('utf-8'))
-        except ValueError as e:
-            raise JavascriptException(u"%s\nscript was:\n%s\nstdout was: '%s'\nstderr was: '%s'\n" %
-                                      (e, fn_linenum(), stdoutdata, stderrdata))
+
+    try:
+        return json.loads(stdout)
+    except ValueError as e:
+        raise JavascriptException(u"%s\nscript was:\n%s\nstdout was: '%s'\nstderr was: '%s'\n" %
+                                    (e, fn_linenum(), stdout, stderr))