Merge pull request #3017 from matthewrmshin/7.8.x-patch2

7.8.x patch2

bin/cylc-check-versions

@@ -43,7 +43,7 @@ import cylc.flags
 from cylc.option_parsers import CylcOptionParser as COP
 from cylc.version import CYLC_VERSION
 from cylc.config import SuiteConfig
-from cylc.subprocpool import SuiteProcPool
+from cylc.subprocpool import SubProcPool
 from cylc.suite_srv_files_mgr import SuiteSrvFilesManager
 from cylc.task_remote_mgr import TaskRemoteMgr
 from cylc.templatevars import load_template_vars
@@ -73,7 +73,7 @@ def main():
             config.get_config(['runtime', name, 'remote', 'owner']),
             config.get_config(['runtime', name, 'remote', 'host'])))
     task_remote_mgr = TaskRemoteMgr(
-        suite, SuiteProcPool(), suite_srv_files_mgr)
+        suite, SubProcPool(), suite_srv_files_mgr)
     for _, host_str in account_set:
         task_remote_mgr.remote_host_select(host_str)
     accounts = []

bin/cylc-submit

@@ -43,7 +43,7 @@ from cylc.cfgspec.glbl_cfg import glbl_cfg
 from cylc.config import SuiteConfig
 from cylc.cycling.loader import get_point
 import cylc.flags
-from cylc.subprocpool import SuiteProcPool
+from cylc.subprocpool import SubProcPool
 from cylc.option_parsers import CylcOptionParser as COP
 from cylc.suite_db_mgr import SuiteDatabaseManager
 from cylc.broadcast_mgr import BroadcastMgr
@@ -105,7 +105,7 @@ def main():
 
     # Initialise job submit environment
     glbl_cfg().create_cylc_run_tree(suite)
-    pool = SuiteProcPool()
+    pool = SubProcPool()
     db_mgr = SuiteDatabaseManager()
     task_job_mgr = TaskJobManager(
         suite, pool, db_mgr, suite_srv_mgr,

lib/cylc/cylc-review/template/cycles.html

@@ -38,8 +38,8 @@
                 <div class="input-group">
                   <label for="order">Sort Order</label>
                   <select name="order" title="Sort Order" class="form-control">
-                  {% for k, v in [("time_desc", "new-&gt;old"),
-                                  ("time_asc", "old-&gt;new")] -%}
+                  {% for k, v in [("time_desc", "new-&gt;old" | safe ),
+                                  ("time_asc", "old-&gt;new" | safe )] -%}
                     <option
                     {% if order and order == k -%}selected="selected"{% endif -%}
                     value="{{k}}">{{v}}</option>
@@ -140,8 +140,8 @@
 {% for entry in entries -%}
   {% set cycle_in_url = entry.cycle|replace('+', '%2B') -%}
   {% set task_jobs_url = (
-      script ~ "/taskjobs/" ~ user ~ "?&amp;suite=" ~
-      suite|replace('+', '%2F') ~ "&amp;cycles=" ~ cycle_in_url
+      (script ~ "/taskjobs/" ~ user ~ "?&amp;suite=" ~
+       suite|replace('+', '%2F') ~ "&amp;cycles=" ~ cycle_in_url) | safe
   ) -%}
     <tr class="entry">
 
@@ -162,21 +162,22 @@
   {# task and jobs states -#}
   {% for state, icon, label, title_, unit1, unit2, url_arg in [
       ("active", "play", "info", "active", "task", "tasks",
-       "&amp;task_status=" ~
-       task_status_groups["active"]|join("&amp;task_status=")),
+       ("&amp;task_status=" ~
+        task_status_groups["active"]|join("&amp;task_status=")) | safe ),
       ("job_active", "play-circle", "info", "active", "job", "jobs",
-       "&amp;job_status=submitted,running"),
+       "&amp;job_status=submitted,running" | safe ),
       ("success", "ok", "success", "succeeded", "task", "tasks",
-       "&amp;task_status=" ~
-       task_status_groups["success"]|join("&amp;task_status=")),
+       ("&amp;task_status=" ~
+        task_status_groups["success"]|join("&amp;task_status=")) | safe ),
       ("job_success", "ok-circle", "success", "succeeded", "job", "jobs",
-       "&amp;job_status=succeeded"),
+       "&amp;job_status=succeeded" | safe ),
       ("fail", "remove", "danger", "failed", "task", "tasks",
-       "&amp;task_status=" ~
-       task_status_groups["fail"]|join("&amp;task_status=")),
+       ("&amp;task_status=" ~
+        task_status_groups["fail"]|join("&amp;task_status=")) | safe ),
       ("job_fail", "remove-circle", "danger", "failed", "job", "jobs",
-       "&amp;job_status=submission-failed,failed"),
+       "&amp;job_status=submission-failed,failed" | safe ),
   ] -%}
+
     {% set n_state = entry.n_states[state] -%}
     {% set unit = unit1 -%}
     {% if n_state -%}
@@ -206,8 +207,9 @@
     <td>
   {% if entry.has_log_job_tar_gz -%}
     {% set download = (
-        script ~ "/view/" ~ user ~ "?&amp;suite=" ~ suite|replace('+', '%2F') ~
-        "&amp;path=log/job-" ~ cycle_in_url ~ ".tar.gz&amp;mode=download"
+        (script ~ "/view/" ~ user ~ "?&amp;suite=" ~
+         suite|replace('+', '%2F') ~ "&amp;path=log/job-" ~ cycle_in_url ~
+         ".tar.gz&amp;mode=download") | safe
     ) -%}
       <a href="{{download}}"
       download="{{user}}-{{suite|replace("/", "%2F")}}-log-job-{{entry.cycle}}.tar.gz">

lib/cylc/cylc-review/template/job-entry.html

@@ -5,10 +5,12 @@
 {% endif -%}
 {% set cycle_str = entry.cycle|replace('+', '%2B') -%}
 {% set taskjobs_link = (
-    script ~ "/taskjobs/" ~ user ~ "?&suite=" ~ suite|replace('+', '%2F') ~ no_fuzzy_time_str
+    (script ~ "/taskjobs/" ~ user ~ "?&suite=" ~
+     suite|replace('+', '%2F') ~ no_fuzzy_time_str) | safe
 ) -%}
 {% set view_link = (
-    script ~ "/view/" ~ user ~ "?&suite=" ~ suite|replace('+', '%2F') ~ no_fuzzy_time_str
+    (script ~ "/view/" ~ user ~ "?&suite=" ~ suite|replace('+', '%2F') ~
+     no_fuzzy_time_str) | safe
 ) -%}
 <tr class="entry"><!-- entry row -->
   <td>
@@ -17,22 +19,22 @@
       {% set icon = "ok" %}
       {% set label_class = "label-success" %}
       {% set url_arg = (
-        "&amp;task_status=" ~
-        task_status_groups["success"]|join("&amp;task_status=")
+        ("&amp;task_status=" ~
+         task_status_groups["success"]|join("&amp;task_status=")) | safe
       ) -%}
     {% elif entry.task_status in ["failed", "submission failed"] -%}
       {% set icon = "remove" %}
       {% set label_class = "label-danger" %}
       {% set url_arg = (
-        "&amp;task_status=" ~
-        task_status_groups["fail"]|join("&amp;task_status=")
+        ("&amp;task_status=" ~
+         task_status_groups["fail"]|join("&amp;task_status=")) | safe
       ) -%}
     {% else -%}
       {% set icon = "play" %}
       {% set label_class = "label-info" %}
       {% set url_arg = (
-        "&amp;task_status=" ~
-        task_status_groups["active"]|join("&amp;task_status=")
+        ("&amp;task_status=" ~
+         task_status_groups["active"]|join("&amp;task_status=")) | safe
       ) -%}
     {% endif -%}
     <small>
@@ -47,8 +49,8 @@
     <!-- entry: submit_status, run_status -->
     <small>
     {% set link = (
-        script ~ "/taskjobs/" ~ user ~ "?&amp;suite=" ~ suite|replace('+', '%2F') ~
-        no_fuzzy_time_str
+        (script ~ "/taskjobs/" ~ user ~ "?&amp;suite=" ~
+         suite|replace('+', '%2F') ~ no_fuzzy_time_str) | safe
     ) -%}
     {% if entry.run_status == 0 -%}
       <a href="{{taskjobs_link}}&amp;job_status=succeeded"
@@ -161,7 +163,7 @@
         <ul class="list-inline">
           {% for key, log in entry.logs|dictsort if not log.seq_key -%}
             {% if key in ["job.out", "job.err"] -%}
-              {% set key_str = "<strong>" ~ key ~ "</strong>" -%}
+              {% set key_str = ("<strong>" ~ key ~ "</strong>") | safe -%}
             {% else -%}
               {% set key_str = key -%}
             {% endif -%}

lib/cylc/cylc-review/template/suites.html

@@ -96,8 +96,8 @@
                   <label for="order">Sort Order</label>
                   <select name="order" title="Sort Order" class="form-control">
                   {% for k, v in [
-                      ("time_desc", "new-&gt;old"),
-                      ("time_asc", "old-&gt;new"),
+                      ("time_desc", "new-&gt;old" | safe ),
+                      ("time_asc", "old-&gt;new" | safe ),
                       ("name_asc", "a-z suite name"),
                       ("name_desc", "z-a suite name"),
                   ] -%}

lib/cylc/cylc-review/template/taskjobs.html

@@ -24,8 +24,8 @@
         <fieldset class="container-fluid">
           <div class="row">
 {% for key, name, value in [
-    ("cycles",
-     "Cycles (before, after or patterns): &lt;CYCLE | &gt;CYCLE | GLOB ...", cycles),
+    ("cycles", "Cycles (before, after or patterns):" ~
+     "&lt;CYCLE | &gt;CYCLE | GLOB ..." | safe , cycles),
     ("tasks", "Task Name Globs", tasks),
 ] -%}
             <div class="form-group col-sm-12 col-md-6">
@@ -107,28 +107,28 @@
               <select id="order" name="order" title="Sort Order"
               class="form-control">
 {% for k, v in [
-    ("time_desc", "new-&gt;old"),
-    ("time_asc", "old-&gt;new"),
-    ("cycle_desc_name_asc", "new-&gt;old cycle, a-z task name"),
-    ("cycle_desc_name_desc", "new-&gt;old cycle, z-a task name"),
-    ("cycle_asc_name_asc", "old-&gt;new cycle, a-z task name"),
-    ("cycle_asc_name_desc", "old-&gt;new cycle, z-a task name"),
-    ("name_asc_cycle_desc", "a-z task name, new-&gt;old cycle"),
-    ("name_desc_cycle_desc", "z-a task name, new-&gt;old cycle"),
-    ("name_asc_cycle_asc", "a-z task name, old-&gt;new cycle"),
-    ("name_desc_cycle_asc", "z-a task name, old-&gt;new cycle"),
-    ("time_submit_desc", "submit time, new-&gt;old"),
-    ("time_submit_asc", "submit time, old-&gt;new"),
-    ("time_run_desc", "run start time, new-&gt;old"),
-    ("time_run_asc", "run start time, old-&gt;new"),
-    ("time_run_exit_desc", "run exit time, new-&gt;old"),
-    ("time_run_exit_asc", "run exit time, old-&gt;new"),
-    ("duration_queue_desc", "queue duration, long-&gt;short"),
-    ("duration_queue_asc", "queue duration, short-&gt;long"),
-    ("duration_run_desc", "run duration, long-&gt;short"),
-    ("duration_run_asc", "run duration, short-&gt;long"),
-    ("duration_queue_run_desc", "queue+run duration, long-&gt;short"),
-    ("duration_queue_run_asc", "queue+run duration, short-&gt;long"),
+    ("time_desc", "new-&gt;old" | safe ),
+    ("time_asc", "old-&gt;new" | safe ),
+    ("cycle_desc_name_asc", "new-&gt;old cycle, a-z task name" | safe ),
+    ("cycle_desc_name_desc", "new-&gt;old cycle, z-a task name" | safe ),
+    ("cycle_asc_name_asc", "old-&gt;new cycle, a-z task name" | safe ),
+    ("cycle_asc_name_desc", "old-&gt;new cycle, z-a task name" | safe ),
+    ("name_asc_cycle_desc", "a-z task name, new-&gt;old cycle" | safe ),
+    ("name_desc_cycle_desc", "z-a task name, new-&gt;old cycle" | safe ),
+    ("name_asc_cycle_asc", "a-z task name, old-&gt;new cycle" | safe ),
+    ("name_desc_cycle_asc", "z-a task name, old-&gt;new cycle" | safe ),
+    ("time_submit_desc", "submit time, new-&gt;old" | safe ),
+    ("time_submit_asc", "submit time, old-&gt;new" | safe ),
+    ("time_run_desc", "run start time, new-&gt;old" | safe ),
+    ("time_run_asc", "run start time, old-&gt;new" | safe ),
+    ("time_run_exit_desc", "run exit time, new-&gt;old" | safe ),
+    ("time_run_exit_asc", "run exit time, old-&gt;new" | safe ),
+    ("duration_queue_desc", "queue duration, long-&gt;short" | safe ),
+    ("duration_queue_asc", "queue duration, short-&gt;long" | safe ),
+    ("duration_run_desc", "run duration, long-&gt;short" | safe ),
+    ("duration_run_asc", "run duration, short-&gt;long" | safe ),
+    ("duration_queue_run_desc", "queue+run duration, long-&gt;short" | safe ),
+    ("duration_queue_run_asc", "queue+run duration, short-&gt;long" | safe ),
 ] -%}
                 <option
 {% if order and order == k -%}

lib/cylc/cylc-review/template/view.html

@@ -71,9 +71,9 @@ <h1>{{path}}</h1>
     'WARNING', '<span class="nocode text-warning">WARNING</span>') -%}
 {% endif -%}
 {% if mode == "tags" -%}
-{{line}}
+{{line|safe}}
 {% else -%}
-{{line|urlise}}
+{{ (line|urlise) | safe }}
 {% endif -%}
 {% endfor -%}
 </pre>

lib/cylc/review.py

@@ -78,8 +78,13 @@ def __init__(self, *args, **kwargs):
         if self.host_name and "." in self.host_name:
             self.host_name = self.host_name.split(".", 1)[0]
         self.cylc_version = CYLC_VERSION
-        template_env = jinja2.Environment(loader=jinja2.FileSystemLoader(
-            get_util_home("lib", "cylc", "cylc-review", "template")))
+        # Autoescape markup to prevent code injection from user inputs.
+        template_env = jinja2.Environment(
+            loader=jinja2.FileSystemLoader(
+                get_util_home("lib", "cylc", "cylc-review", "template")),
+            autoescape=jinja2.select_autoescape(
+                enabled_extensions=('html', 'xml'), default_for_string=True),
+        )
         template_env.filters['urlise'] = self.url2hyperlink
         self.template_env = template_env
 

lib/cylc/scheduler.py

@@ -50,7 +50,7 @@
 from cylc.network.httpserver import HTTPServer
 from cylc.profiler import Profiler
 from cylc.state_summary_mgr import StateSummaryMgr
-from cylc.subprocpool import SuiteProcPool
+from cylc.subprocpool import SubProcPool
 from cylc.suite_db_mgr import SuiteDatabaseManager
 from cylc.suite_events import (
     SuiteEventContext, SuiteEventError, SuiteEventHandler)
@@ -355,7 +355,7 @@ def configure(self):
         self.profiler.log_memory("scheduler.py: start configure")
 
         # Start up essential services
-        self.proc_pool = SuiteProcPool()
+        self.proc_pool = SubProcPool()
         self.state_summary_mgr = StateSummaryMgr()
         self.command_queue = Queue()
         self.message_queue = Queue()

lib/cylc/subprocctx.py

@@ -47,8 +47,9 @@ class SubProcContext(object):
                     Default return code.
                 shell (boolean):
                     Launch command with "/bin/sh"?
-                stdin_file_paths (list):
+                stdin_files (list):
                     Files with content to send to command's STDIN.
+                    Can be file paths or opened file handles.
                 stdin_str (str):
                     Content to send to command's STDIN.
         .err (str):
@@ -84,9 +85,9 @@ def __str__(self):
             value = getattr(self, attr, None)
             if value is not None and str(value).strip():
                 mesg = ''
-                if attr == 'cmd' and self.cmd_kwargs.get('stdin_file_paths'):
+                if attr == 'cmd' and self.cmd_kwargs.get('stdin_files'):
                     mesg += 'cat'
-                    for file_path in self.cmd_kwargs.get('stdin_file_paths'):
+                    for file_path in self.cmd_kwargs.get('stdin_files'):
                         mesg += ' ' + quote(file_path)
                     mesg += ' | '
                 if attr == 'cmd' and isinstance(value, list):

lib/cylc/subprocpool.py

@@ -91,16 +91,16 @@ def run_function(func_name, json_args, json_kwargs, src_dir):
     sys.stdout.write(json.dumps(res))
 
 
-class SuiteProcPool(object):
+class SubProcPool(object):
     """Manage queueing and pooling of subprocesses.
 
     This is mainly used by the main loop of the suite server program, although
-    the SuiteProcPool.run_command can be used as a standalone utility function
+    the SubProcPool.run_command can be used as a standalone utility function
     to run the command in a cylc.subprocctx.SubProcContext.
 
     A command to run under a subprocess in the pool is expected to be wrapped
     using a cylc.subprocctx.SubProcContext object. The caller will add the
-    context object using the SuiteProcPool.put_command method. A callback can
+    context object using the SubProcPool.put_command method. A callback can
     be specified to notify the caller on exit of the subprocess.
 
     A command launched by the pool is expected to write to STDOUT and STDERR.
@@ -311,15 +311,20 @@ def _poll_proc_pipes(self, proc, ctx):
     def _run_command_init(cls, ctx, callback=None, callback_args=None):
         """Prepare and launch shell command in ctx."""
         try:
-            if ctx.cmd_kwargs.get('stdin_file_paths'):
-                if len(ctx.cmd_kwargs['stdin_file_paths']) > 1:
+            if ctx.cmd_kwargs.get('stdin_files'):
+                if len(ctx.cmd_kwargs['stdin_files']) > 1:
                     stdin_file = TemporaryFile()
-                    for file_path in ctx.cmd_kwargs['stdin_file_paths']:
-                        stdin_file.write(open(file_path, 'rb').read())
+                    for file_ in ctx.cmd_kwargs['stdin_files']:
+                        if hasattr(file_, 'read'):
+                            stdin_file.write(file_.read())
+                        else:
+                            stdin_file.write(open(file_, 'rb').read())
                     stdin_file.seek(0)
+                elif hasattr(ctx.cmd_kwargs['stdin_files'][0], 'read'):
+                    stdin_file = ctx.cmd_kwargs['stdin_files'][0]
                 else:
                     stdin_file = open(
-                        ctx.cmd_kwargs['stdin_file_paths'][0], 'rb')
+                        ctx.cmd_kwargs['stdin_files'][0], 'rb')
             elif ctx.cmd_kwargs.get('stdin_str'):
                 stdin_file = TemporaryFile()
                 stdin_file.write(ctx.cmd_kwargs.get('stdin_str'))