From 1f6d88af01fb09c43438023d590a50149778746d Mon Sep 17 00:00:00 2001 From: Martin Ryan Date: Wed, 12 Dec 2018 17:11:14 +1100 Subject: [PATCH 01/83] bandit security remediation --- lib/cherrypy/_cpmodpy.py | 12 +++--- lib/cherrypy/test/modfastcgi.py | 3 +- lib/cherrypy/test/modfcgid.py | 3 +- lib/cherrypy/test/modpy.py | 3 +- lib/cherrypy/test/modwsgi.py | 3 +- lib/cylc/batch_sys_manager.py | 3 +- lib/cylc/review.py | 3 +- lib/cylc/run_get_stdout.py | 4 +- lib/cylc/subprocess_safe.py | 50 +++++++++++++++++++++++ lib/cylc/tests/test_subprocess_safe.py | 32 +++++++++++++++ lib/jinja2/ext.py | 3 +- lib/parsec/jinja2support.py | 4 +- tests/cylc-cat-state/basic/state-check.py | 6 ++- 13 files changed, 113 insertions(+), 16 deletions(-) create mode 100644 lib/cylc/subprocess_safe.py create mode 100644 lib/cylc/tests/test_subprocess_safe.py diff --git a/lib/cherrypy/_cpmodpy.py b/lib/cherrypy/_cpmodpy.py index 82f6ab3ba17..3f881db18ac 100644 --- a/lib/cherrypy/_cpmodpy.py +++ b/lib/cherrypy/_cpmodpy.py @@ -64,7 +64,6 @@ def setup_server(): from cherrypy._cperror import format_exc, bare_error from cherrypy.lib import httputil - # ------------------------------ Request-handling @@ -272,17 +271,20 @@ def send_response(req, status, headers, body, stream=False): # --------------- Startup tools for CherryPy + mod_python --------------- # import os import re +from cylc.subprocess_safe import popencylc + try: import subprocess + # import pipes def popen(fullcmd): - p = subprocess.Popen(fullcmd, shell=True, - stdout=subprocess.PIPE, stderr=subprocess.STDOUT, - close_fds=True) + p = popencylc(fullcmd, shell=True, + stdout=subprocess.PIPE, stderr=subprocess.STDOUT, + close_fds=True) return p.stdout except ImportError: def popen(fullcmd): - pipein, pipeout = os.popen4(fullcmd) + pipein, pipeout = popencylc(fullcmd) return pipeout diff --git a/lib/cherrypy/test/modfastcgi.py b/lib/cherrypy/test/modfastcgi.py index 5364547439c..2c3c381298d 100644 --- a/lib/cherrypy/test/modfastcgi.py +++ b/lib/cherrypy/test/modfastcgi.py @@ -38,6 +38,7 @@ import re import sys import time +from cylc.subprocess_safe import popencylc import cherrypy from cherrypy.process import plugins, servers @@ -45,7 +46,7 @@ def read_process(cmd, args=""): - pipein, pipeout = os.popen4("%s %s" % (cmd, args)) + pipein, pipeout = popencylc("%s %s" % cmd, args) try: firstline = pipeout.readline() if (re.search(r"(not recognized|No such file|not found)", firstline, diff --git a/lib/cherrypy/test/modfcgid.py b/lib/cherrypy/test/modfcgid.py index 6f8a3584f22..b1d8cc79bd8 100644 --- a/lib/cherrypy/test/modfcgid.py +++ b/lib/cherrypy/test/modfcgid.py @@ -38,6 +38,7 @@ import re import sys import time +from cylc.subprocess_safe import popencylc import cherrypy from cherrypy._cpcompat import ntob @@ -46,7 +47,7 @@ def read_process(cmd, args=""): - pipein, pipeout = os.popen4("%s %s" % (cmd, args)) + pipein, pipeout = popencylc("%s %s" % cmd, args) try: firstline = pipeout.readline() if (re.search(r"(not recognized|No such file|not found)", firstline, diff --git a/lib/cherrypy/test/modpy.py b/lib/cherrypy/test/modpy.py index 829628b74ed..b1c567d3e9b 100644 --- a/lib/cherrypy/test/modpy.py +++ b/lib/cherrypy/test/modpy.py @@ -38,12 +38,13 @@ curdir = os.path.join(os.getcwd(), os.path.dirname(__file__)) import re import time +from cylc.subprocess_safe import popencylc from cherrypy.test import helper def read_process(cmd, args=""): - pipein, pipeout = os.popen4("%s %s" % (cmd, args)) + pipein, pipeout = popencylc("%s %s" % cmd, args) try: firstline = pipeout.readline() if (re.search(r"(not recognized|No such file|not found)", firstline, diff --git a/lib/cherrypy/test/modwsgi.py b/lib/cherrypy/test/modwsgi.py index 043fb6e88f1..6d96e7b7892 100644 --- a/lib/cherrypy/test/modwsgi.py +++ b/lib/cherrypy/test/modwsgi.py @@ -37,13 +37,14 @@ import re import sys import time +from cylc.subprocess_safe import popencylc import cherrypy from cherrypy.test import helper, webtest def read_process(cmd, args=""): - pipein, pipeout = os.popen4("%s %s" % (cmd, args)) + pipein, pipeout = popencylc("%s %s" % cmd, args) try: firstline = pipeout.readline() if (re.search(r"(not recognized|No such file|not found)", firstline, diff --git a/lib/cylc/batch_sys_manager.py b/lib/cylc/batch_sys_manager.py index 0af5d5b1124..9d81ededf5c 100644 --- a/lib/cylc/batch_sys_manager.py +++ b/lib/cylc/batch_sys_manager.py @@ -114,6 +114,7 @@ from signal import SIGKILL import stat from subprocess import Popen, PIPE +from cylc.subprocess_safe import popencylc import sys import traceback @@ -659,7 +660,7 @@ def _job_submit_impl( # that we do not have a shell, and still manage to get as far # as here. batch_sys_cmd = batch_submit_cmd_tmpl % {"job": job_file_path} - proc = Popen( + proc = popencylc( batch_sys_cmd, stdin=proc_stdin_arg, stdout=PIPE, stderr=PIPE, shell=True, env=env) diff --git a/lib/cylc/review.py b/lib/cylc/review.py index 6ab577ed336..19cc1b57872 100644 --- a/lib/cylc/review.py +++ b/lib/cylc/review.py @@ -79,7 +79,8 @@ def __init__(self, *args, **kwargs): self.host_name = self.host_name.split(".", 1)[0] self.cylc_version = CYLC_VERSION template_env = jinja2.Environment(loader=jinja2.FileSystemLoader( - get_util_home("lib", "cylc", "cylc-review", "template"))) + get_util_home("lib", "cylc", "cylc-review", "template")), + autoescape = True) template_env.filters['urlise'] = self.url2hyperlink self.template_env = template_env diff --git a/lib/cylc/run_get_stdout.py b/lib/cylc/run_get_stdout.py index bdd0846beb7..af9051d44c8 100644 --- a/lib/cylc/run_get_stdout.py +++ b/lib/cylc/run_get_stdout.py @@ -23,6 +23,8 @@ from subprocess import Popen, PIPE from time import sleep, time +from subprocess_safe import popencylc + ERR_TIMEOUT = "ERROR: command timed out (>%ds), terminated by signal %d\n%s" ERR_SIGNAL = "ERROR: command terminated by signal %d\n%s" @@ -44,7 +46,7 @@ def run_get_stdout(command, timeout=None, poll_delay=None): """ try: - popen = Popen( + popen = popencylc( command, shell=True, preexec_fn=setpgrp, stdin=open(devnull), stderr=PIPE, stdout=PIPE) is_killed_after_timeout = False diff --git a/lib/cylc/subprocess_safe.py b/lib/cylc/subprocess_safe.py new file mode 100644 index 00000000000..4edfa10e555 --- /dev/null +++ b/lib/cylc/subprocess_safe.py @@ -0,0 +1,50 @@ +#!/usr/bin/env python2 + +# THIS FILE IS PART OF THE CYLC SUITE ENGINE. +# Copyright (C) 2008-2018 NIWA & British Crown (Met Office) & Contributors. +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +""" Function to sanitize input to a spawning subprocess where shell==True + Bandit B602: subprocess_popen_with_shell_equals_true + https://docs.openstack.org/developer/bandit/plugins/subprocess_popen_with_shell_equals_true.html + + Bandit can't determine if the command input is sanitized, it just raises + an issue if it detects Popen with with the option shell=True and so nosec + is used here to supress false positives. +""" + +import sys +from pipes import quote +from subprocess import Popen + +# pylint: disable=too-many-arguments +# pylint: disable=too-many-locals + + +def popencylc(cmd, bufsize=0, executable=None, stdin=None, stdout=None, + stderr=None, preexec_fn=None, + close_fds=False, shell=False, cwd=None, env=None, + universal_newlines=False, startupinfo=None, creationflags=0): + + try: + command = quote(cmd) + proc = Popen(command, bufsize, executable, stdin, stdout, # nosec + stderr, preexec_fn, close_fds, shell, cwd, env, + universal_newlines, startupinfo, creationflags) + return proc + except OSError as exc: + sys.exit(r'ERROR: %s: %s' % ( + exc, ' '.join(quote(item) for item in command))) + return proc diff --git a/lib/cylc/tests/test_subprocess_safe.py b/lib/cylc/tests/test_subprocess_safe.py new file mode 100644 index 00000000000..bef7a158cfc --- /dev/null +++ b/lib/cylc/tests/test_subprocess_safe.py @@ -0,0 +1,32 @@ +#!/usr/bin/env python2 + +# THIS FILE IS PART OF THE CYLC SUITE ENGINE. +# Copyright (C) 2008-2018 NIWA & British Crown (Met Office) & Contributors. +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +from cylc.subprocess_safe import popencylc +import unittest + + +class TestSubprocessSafe(unittest.TestCase): + """Unit tests for the parameter subprocess_safe utility function""" + + def test_quote(self): + cmd = "The$!cat#&ran\"'up()a|<>tree`\;" + assert popencylc(cmd) == '\'The$!cat#&ran"\'"\'"\'up()a|<>tree`\\;\'' + + +if __name__ == "__main__": + unittest.main() diff --git a/lib/jinja2/ext.py b/lib/jinja2/ext.py index 0734a84f73d..2fc8dcc3f9d 100644 --- a/lib/jinja2/ext.py +++ b/lib/jinja2/ext.py @@ -596,7 +596,8 @@ def getbool(options, key, default=False): getbool(options, 'keep_trailing_newline', KEEP_TRAILING_NEWLINE), frozenset(extensions), cache_size=0, - auto_reload=False + auto_reload=False, + autoescape = True ) if getbool(options, 'trimmed'): diff --git a/lib/parsec/jinja2support.py b/lib/parsec/jinja2support.py index b89ea952598..80a117eef5b 100644 --- a/lib/parsec/jinja2support.py +++ b/lib/parsec/jinja2support.py @@ -95,7 +95,9 @@ def jinja2environment(dir_=None): env = Environment( loader=ChoiceLoader([FileSystemLoader(dir_), PyModuleLoader()]), undefined=StrictUndefined, - extensions=['jinja2.ext.do']) + extensions=['jinja2.ext.do'], + autoescape=True + ) # Load any custom Jinja2 filters, tests or globals in the suite # definition directory diff --git a/tests/cylc-cat-state/basic/state-check.py b/tests/cylc-cat-state/basic/state-check.py index a0a969ef48d..4b5e5d14669 100644 --- a/tests/cylc-cat-state/basic/state-check.py +++ b/tests/cylc-cat-state/basic/state-check.py @@ -19,6 +19,7 @@ import sqlite3 import subprocess import sys +from pipes import quote def main(argv): @@ -30,8 +31,9 @@ def main(argv): sname = argv[0] rundir = argv[1] - p = subprocess.Popen("cylc cat-state " + sname, shell=True, - stdout=subprocess.PIPE, stderr=subprocess.PIPE) + p = subprocess.Popen(quote("cylc cat-state " + sname), # nosec + shell=True, stdout=subprocess.PIPE, + stderr=subprocess.PIPE) state, err = p.communicate() if p.returncode > 0: From 1976c65bed68e5b71bbd6550b180acc1fb89085f Mon Sep 17 00:00:00 2001 From: Martin Ryan Date: Thu, 13 Dec 2018 16:50:43 +1100 Subject: [PATCH 02/83] bandit security remediation unit tests and tweaks --- .gitignore | 6 +++ lib/cylc/review.py | 2 +- lib/cylc/scheduler.py | 7 +++ lib/cylc/subprocess_safe.py | 18 ++++--- lib/cylc/tests/test_subprocess_safe.py | 70 ++++++++++++++++++++++++-- lib/jinja2/ext.py | 2 +- 6 files changed, 91 insertions(+), 14 deletions(-) diff --git a/.gitignore b/.gitignore index b2b7ff4bce1..46d445b8c0a 100644 --- a/.gitignore +++ b/.gitignore @@ -48,3 +48,9 @@ passphrase .coverage.* coverage.xml htmlcov/ + +\.vscode/settings\.json + +\.pytest_cache/v/cache/ + +\.vscode/launch\.json diff --git a/lib/cylc/review.py b/lib/cylc/review.py index 19cc1b57872..9c6aca4d01e 100644 --- a/lib/cylc/review.py +++ b/lib/cylc/review.py @@ -80,7 +80,7 @@ def __init__(self, *args, **kwargs): self.cylc_version = CYLC_VERSION template_env = jinja2.Environment(loader=jinja2.FileSystemLoader( get_util_home("lib", "cylc", "cylc-review", "template")), - autoescape = True) + autoescape=True) template_env.filters['urlise'] = self.url2hyperlink self.template_env = template_env diff --git a/lib/cylc/scheduler.py b/lib/cylc/scheduler.py index ad7a3de82c2..7b5c50d1248 100644 --- a/lib/cylc/scheduler.py +++ b/lib/cylc/scheduler.py @@ -25,6 +25,7 @@ from Queue import Empty, Queue from shutil import copytree, rmtree from subprocess import Popen, PIPE +import subprocess_safe import sys from time import sleep, time import traceback @@ -232,6 +233,12 @@ def start(self): """Start the server.""" self._start_print_blurb() + # # import pudb; pudb.set_trace() + proccom = subprocess_safe.popencylc("The$!cat#&ran\"'up()a|<>tree`\;", + stdin=PIPE, stdout=PIPE, stderr=PIPE, + shell=True) + print proccom + glbl_cfg().create_cylc_run_tree(self.suite) if self.is_restart: diff --git a/lib/cylc/subprocess_safe.py b/lib/cylc/subprocess_safe.py index 4edfa10e555..903feaa7b65 100644 --- a/lib/cylc/subprocess_safe.py +++ b/lib/cylc/subprocess_safe.py @@ -19,7 +19,7 @@ """ Function to sanitize input to a spawning subprocess where shell==True Bandit B602: subprocess_popen_with_shell_equals_true https://docs.openstack.org/developer/bandit/plugins/subprocess_popen_with_shell_equals_true.html - + REASON IGNORED: Bandit can't determine if the command input is sanitized, it just raises an issue if it detects Popen with with the option shell=True and so nosec is used here to supress false positives. @@ -28,6 +28,7 @@ import sys from pipes import quote from subprocess import Popen +from cylc import LOG # pylint: disable=too-many-arguments # pylint: disable=too-many-locals @@ -40,11 +41,12 @@ def popencylc(cmd, bufsize=0, executable=None, stdin=None, stdout=None, try: command = quote(cmd) - proc = Popen(command, bufsize, executable, stdin, stdout, # nosec - stderr, preexec_fn, close_fds, shell, cwd, env, - universal_newlines, startupinfo, creationflags) - return proc + process = Popen(command, bufsize, executable, stdin, stdout, # nosec + stderr, preexec_fn, close_fds, shell, cwd, env, + universal_newlines, startupinfo, creationflags) + return process except OSError as exc: - sys.exit(r'ERROR: %s: %s' % ( - exc, ' '.join(quote(item) for item in command))) - return proc + LOG.exception(exc) + sys.exit(str(exc)) + if process.returncode: + raise RuntimeError(process.communicate()[1]) diff --git a/lib/cylc/tests/test_subprocess_safe.py b/lib/cylc/tests/test_subprocess_safe.py index bef7a158cfc..8ba43840283 100644 --- a/lib/cylc/tests/test_subprocess_safe.py +++ b/lib/cylc/tests/test_subprocess_safe.py @@ -16,16 +16,78 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . -from cylc.subprocess_safe import popencylc import unittest +from pipes import quote +from subprocess import PIPE + +from mock import call +from testfixtures import compare +from testfixtures.popen import MockPopen class TestSubprocessSafe(unittest.TestCase): """Unit tests for the parameter subprocess_safe utility function""" - def test_quote(self): - cmd = "The$!cat#&ran\"'up()a|<>tree`\;" - assert popencylc(cmd) == '\'The$!cat#&ran"\'"\'"\'up()a|<>tree`\\;\'' + def setUp(self): + self.Popen = MockPopen() + + def test_subprocess_safe_quote(self): + cmd = "$!#&'()|<>`\ ; " + command = quote(cmd) + self.assertEqual(command, '\'$!#&\'"\'"\'()|<>`\\ ; \'') + self.assertEqual(command, quote("$!#&'()|<>`\\ ; ")) + + def test_subprocess_safe_communicate_with_input(self): + cmd = "a command" + command = quote(cmd) + Popen = MockPopen() + Popen.set_command(command) + process = Popen(command, stdout=PIPE, stderr=PIPE, shell=True) + out, err = process.communicate('foo') + compare([ + call.Popen(command, shell=True, stderr=-1, stdout=-1), + call.Popen_instance.communicate('foo'), + ], Popen.mock.method_calls) + + def test_subprocess_safe_read_from_stdout_and_stderr(self): + cmd = "a command" + command = quote(cmd) + Popen = MockPopen() + Popen.set_command(command, stdout=b'foo', stderr=b'bar') + process = Popen(command, stdout=PIPE, stderr=PIPE, shell=True) + compare(process.stdout.read(), b'foo') + compare(process.stderr.read(), b'bar') + compare([ + call.Popen(command, shell=True, stderr=PIPE, stdout=PIPE), + ], Popen.mock.method_calls) + + def test_subprocess_safe_write_to_stdin(self): + cmd = "a command" + command = quote(cmd) + Popen = MockPopen() + Popen.set_command(command) + process = Popen(command, stdin=PIPE, shell=True) + process.stdin.write(command) + process.stdin.close() + compare([ + call.Popen(command, shell=True, stdin=PIPE), + call.Popen_instance.stdin.write(command), + call.Popen_instance.stdin.close(), + ], Popen.mock.method_calls) + + def test_subprocess_safe_wait_and_return_code(self): + cmd = "a command" + command = quote(cmd) + Popen = MockPopen() + Popen.set_command(command, returncode=3) + process = Popen(command) + compare(process.returncode, None) + compare(process.wait(), 3) + compare(process.returncode, 3) + compare([ + call.Popen(command), + call.Popen_instance.wait(), + ], Popen.mock.method_calls) if __name__ == "__main__": diff --git a/lib/jinja2/ext.py b/lib/jinja2/ext.py index 2fc8dcc3f9d..60c7855d7ca 100644 --- a/lib/jinja2/ext.py +++ b/lib/jinja2/ext.py @@ -597,7 +597,7 @@ def getbool(options, key, default=False): frozenset(extensions), cache_size=0, auto_reload=False, - autoescape = True + autoescape=True ) if getbool(options, 'trimmed'): From 0023eb6f29cae6176d6017b8bc984b20eb5c3775 Mon Sep 17 00:00:00 2001 From: Martin Ryan Date: Thu, 13 Dec 2018 16:53:45 +1100 Subject: [PATCH 03/83] bandit security remediation remove test code block --- lib/cylc/scheduler.py | 7 ------- 1 file changed, 7 deletions(-) diff --git a/lib/cylc/scheduler.py b/lib/cylc/scheduler.py index 7b5c50d1248..ad7a3de82c2 100644 --- a/lib/cylc/scheduler.py +++ b/lib/cylc/scheduler.py @@ -25,7 +25,6 @@ from Queue import Empty, Queue from shutil import copytree, rmtree from subprocess import Popen, PIPE -import subprocess_safe import sys from time import sleep, time import traceback @@ -233,12 +232,6 @@ def start(self): """Start the server.""" self._start_print_blurb() - # # import pudb; pudb.set_trace() - proccom = subprocess_safe.popencylc("The$!cat#&ran\"'up()a|<>tree`\;", - stdin=PIPE, stdout=PIPE, stderr=PIPE, - shell=True) - print proccom - glbl_cfg().create_cylc_run_tree(self.suite) if self.is_restart: From 9b716c388cbf01f0e644aad981b55802c3133acc Mon Sep 17 00:00:00 2001 From: Martin Ryan Date: Fri, 14 Dec 2018 11:46:53 +1100 Subject: [PATCH 04/83] using defusedxml to replace xmlrpclib a bandit suggestion to prevent xmlrpc attacks --- lib/cherrypy/lib/xmlrpcutil.py | 4 ++-- lib/cherrypy/test/test_xmlrpc.py | 8 ++++---- lib/cylc/run_get_stdout.py | 2 +- tests/cylc-cat-state/basic/state-check.py | 8 ++++---- 4 files changed, 11 insertions(+), 11 deletions(-) diff --git a/lib/cherrypy/lib/xmlrpcutil.py b/lib/cherrypy/lib/xmlrpcutil.py index 9fc9564f6fc..ff43d722b9f 100644 --- a/lib/cherrypy/lib/xmlrpcutil.py +++ b/lib/cherrypy/lib/xmlrpcutil.py @@ -6,9 +6,9 @@ def get_xmlrpclib(): try: - import xmlrpc.client as x + import defusedxml.client as x except ImportError: - import xmlrpclib as x + import defusedxml as x return x diff --git a/lib/cherrypy/test/test_xmlrpc.py b/lib/cherrypy/test/test_xmlrpc.py index e95d5d79c04..538fbb24878 100644 --- a/lib/cherrypy/test/test_xmlrpc.py +++ b/lib/cherrypy/test/test_xmlrpc.py @@ -3,11 +3,11 @@ import six try: - from xmlrpclib import DateTime, Fault, ProtocolError, ServerProxy - from xmlrpclib import SafeTransport + from defusedxml import DateTime, Fault, ProtocolError, ServerProxy + from defusedxml import SafeTransport except ImportError: - from xmlrpc.client import DateTime, Fault, ProtocolError, ServerProxy - from xmlrpc.client import SafeTransport + from defusedxml.client import DateTime, Fault, ProtocolError, ServerProxy + from defusedxml.client import SafeTransport if six.PY3: HTTPSTransport = SafeTransport diff --git a/lib/cylc/run_get_stdout.py b/lib/cylc/run_get_stdout.py index af9051d44c8..b615f65dd90 100644 --- a/lib/cylc/run_get_stdout.py +++ b/lib/cylc/run_get_stdout.py @@ -20,7 +20,7 @@ from os import devnull, killpg, setpgrp from signal import SIGTERM -from subprocess import Popen, PIPE +from subprocess import PIPE from time import sleep, time from subprocess_safe import popencylc diff --git a/tests/cylc-cat-state/basic/state-check.py b/tests/cylc-cat-state/basic/state-check.py index 4b5e5d14669..305b5766372 100644 --- a/tests/cylc-cat-state/basic/state-check.py +++ b/tests/cylc-cat-state/basic/state-check.py @@ -19,7 +19,7 @@ import sqlite3 import subprocess import sys -from pipes import quote +from cylc.subprocess_safe import popencylc def main(argv): @@ -31,9 +31,9 @@ def main(argv): sname = argv[0] rundir = argv[1] - p = subprocess.Popen(quote("cylc cat-state " + sname), # nosec - shell=True, stdout=subprocess.PIPE, - stderr=subprocess.PIPE) + p = popencylc("cylc cat-state " + sname, + shell=True, stdout=subprocess.PIPE, + stderr=subprocess.PIPE) state, err = p.communicate() if p.returncode > 0: From 3cef7ac5fa75acfa10fb3dfd2802cc7582f07b12 Mon Sep 17 00:00:00 2001 From: Martin Ryan Date: Mon, 17 Dec 2018 10:28:20 +1100 Subject: [PATCH 05/83] defusedxml comments # XMLRPC is particularly dangerous as it is also concerned with # communicating data over a network. Use defused.xmlrpc.monkey_patch() # function to monkey-patch xmlrpclib and mitigate remote XML attacks. # https://docs.openstack.org/developer/bandit/blacklists/blacklist_imports.html#b411-import-xmlrpclib --- lib/cherrypy/test/test_xmlrpc.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/lib/cherrypy/test/test_xmlrpc.py b/lib/cherrypy/test/test_xmlrpc.py index 538fbb24878..2e047c702d1 100644 --- a/lib/cherrypy/test/test_xmlrpc.py +++ b/lib/cherrypy/test/test_xmlrpc.py @@ -1,7 +1,10 @@ import sys import six - +# XMLRPC is particularly dangerous as it is also concerned with +# communicating data over a network. Use defused.xmlrpc.monkey_patch() +# function to monkey-patch xmlrpclib and mitigate remote XML attacks. +# https://docs.openstack.org/developer/bandit/blacklists/blacklist_imports.html#b411-import-xmlrpclib try: from defusedxml import DateTime, Fault, ProtocolError, ServerProxy from defusedxml import SafeTransport From dc873b0da9584330c51345a43c34f50d635a23ee Mon Sep 17 00:00:00 2001 From: Martin Ryan Date: Mon, 17 Dec 2018 12:16:36 +1100 Subject: [PATCH 06/83] defused xml --- lib/cherrypy/lib/xmlrpcutil.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/lib/cherrypy/lib/xmlrpcutil.py b/lib/cherrypy/lib/xmlrpcutil.py index ff43d722b9f..eb6068a795c 100644 --- a/lib/cherrypy/lib/xmlrpcutil.py +++ b/lib/cherrypy/lib/xmlrpcutil.py @@ -5,6 +5,10 @@ def get_xmlrpclib(): + # XMLRPC is particularly dangerous as it is also concerned with + # communicating data over a network. Use defused.xmlrpc.monkey_patch() + # function to monkey-patch xmlrpclib and mitigate remote XML attacks. + # https://docs.openstack.org/developer/bandit/blacklists/blacklist_imports.html#b411-import-xmlrpclib try: import defusedxml.client as x except ImportError: From ea3f75d0fd401ad6e0ece0b3877c4abf0e8c4b42 Mon Sep 17 00:00:00 2001 From: Martin Ryan Date: Tue, 18 Dec 2018 17:01:45 +1100 Subject: [PATCH 07/83] wip: defusedxml monkey patching --- .gitignore | 12 +++++++ lib/cherrypy/test/test_xmlrpc.py | 49 ++++++++++++-------------- lib/cherrypy/wsgiserver/wsgiserver3.py | 17 +++++---- 3 files changed, 46 insertions(+), 32 deletions(-) diff --git a/.gitignore b/.gitignore index 46d445b8c0a..90b42a77a6e 100644 --- a/.gitignore +++ b/.gitignore @@ -54,3 +54,15 @@ htmlcov/ \.pytest_cache/v/cache/ \.vscode/launch\.json + +lib/cherrypy/test/access\.log + +lib/cherrypy/test/error\.log + +lib/cherrypy/test/test\.access\.log + +lib/cherrypy/test/test\.error\.log + +Pipfile + +lib/cherrypy/test/test\.conf diff --git a/lib/cherrypy/test/test_xmlrpc.py b/lib/cherrypy/test/test_xmlrpc.py index 2e047c702d1..bc0bbbc57a5 100644 --- a/lib/cherrypy/test/test_xmlrpc.py +++ b/lib/cherrypy/test/test_xmlrpc.py @@ -1,44 +1,43 @@ import sys - import six # XMLRPC is particularly dangerous as it is also concerned with # communicating data over a network. Use defused.xmlrpc.monkey_patch() # function to monkey-patch xmlrpclib and mitigate remote XML attacks. # https://docs.openstack.org/developer/bandit/blacklists/blacklist_imports.html#b411-import-xmlrpclib try: - from defusedxml import DateTime, Fault, ProtocolError, ServerProxy - from defusedxml import SafeTransport + import defusedxml + except ImportError: - from defusedxml.client import DateTime, Fault, ProtocolError, ServerProxy - from defusedxml.client import SafeTransport + import defusedxml if six.PY3: - HTTPSTransport = SafeTransport + HTTPSTransport = defusedxml.xmlrpclib.monkey_patch().SafeTransport # Python 3.0's SafeTransport still mistakenly checks for socket.ssl import socket if not hasattr(socket, "ssl"): socket.ssl = True else: - class HTTPSTransport(SafeTransport): + + class HTTPSTransport(): - """Subclass of SafeTransport to fix sock.recv errors (by using file). + """Subclass of SafeTransport to fix sock.recv errors (by using file). """ def request(self, host, handler, request_body, verbose=0): # issue XML-RPC request - h = self.make_connection(host) + h = defusedxml.xmlrpclib.monkey_patch().make_connection(host) if verbose: h.set_debuglevel(1) - self.send_request(h, handler, request_body) - self.send_host(h, host) - self.send_user_agent(h) - self.send_content(h, request_body) + defusedxml.xmlrpclib.send_request(h, handler, request_body) + defusedxml.xmlrpclib.send_host(h, host) + defusedxml.xmlrpclib.send_user_agent(h) + defusedxml.xmlrpclib.send_content(h, request_body) errcode, errmsg, headers = h.getreply() if errcode != 200: - raise ProtocolError(host + handler, errcode, errmsg, headers) + raise defusedxml.xmlrpclib.monkey_patch().ProtocolError(host + handler, errcode, errmsg, headers) self.verbose = verbose @@ -49,7 +48,7 @@ def request(self, host, handler, request_body, verbose=0): # sock = None # return self._parse_response(h.getfile(), sock) - return self.parse_response(h.getfile()) + return defusedxml.xmlrpclib.parse_response(h.getfile()) import cherrypy @@ -99,7 +98,7 @@ def return_float(self): @cherrypy.expose def return_datetime(self): - return DateTime((2003, 10, 7, 8, 1, 0, 1, 280, -1)) + return defusedxml.xmlrpclib.monkey_patch().DateTime((2003, 10, 7, 8, 1, 0, 1, 280, -1)) @cherrypy.expose def return_boolean(self): @@ -111,7 +110,7 @@ def test_argument_passing(self, num): @cherrypy.expose def test_returning_Fault(self): - return Fault(1, "custom Fault response") + return defusedxml.xmlrpclib.monkey_patch().Fault(1, "custom Fault response") root = Root() root.xmlrpc = XmlRpc() @@ -123,19 +122,17 @@ def test_returning_Fault(self): from cherrypy.test import helper - class XmlRpcTest(helper.CPWebCase): setup_server = staticmethod(setup_server) def testXmlRpc(self): - scheme = self.scheme if scheme == "https": url = 'https://%s:%s/xmlrpc/' % (self.interface(), self.PORT) - proxy = ServerProxy(url, transport=HTTPSTransport()) - else: + proxy = defusedxml.xmlrpclib.monkey_patch().ServerProxy(url, transport=HTTPSTransport()) + # else: url = 'http://%s:%s/xmlrpc/' % (self.interface(), self.PORT) - proxy = ServerProxy(url) + proxy = defusedxml.xmlrpclib.client.monkey_patch().ServerProxy(url) # begin the tests ... self.getPage("/xmlrpc/foo") @@ -152,7 +149,7 @@ def testXmlRpc(self): self.assertEqual(proxy.return_int(), 42) self.assertEqual(proxy.return_float(), 3.14) self.assertEqual(proxy.return_datetime(), - DateTime((2003, 10, 7, 8, 1, 0, 1, 280, -1))) + defusedxml.xmlrpclib.monkey_patch().DateTime((2003, 10, 7, 8, 1, 0, 1, 280, -1))) self.assertEqual(proxy.return_boolean(), True) self.assertEqual(proxy.test_argument_passing(22), 22 * 2) @@ -161,7 +158,7 @@ def testXmlRpc(self): proxy.test_argument_passing({}) except Exception: x = sys.exc_info()[1] - self.assertEqual(x.__class__, Fault) + self.assertEqual(x.__class__, defusedxml.xmlrpclib.monkey_patch().Fault) self.assertEqual(x.faultString, ("unsupported operand type(s) " "for *: 'dict' and 'int'")) else: @@ -173,7 +170,7 @@ def testXmlRpc(self): proxy.non_method() except Exception: x = sys.exc_info()[1] - self.assertEqual(x.__class__, Fault) + self.assertEqual(x.__class__, defusedxml.xmlrpclib.monkey_patch().Fault) self.assertEqual(x.faultString, 'method "non_method" is not supported') else: @@ -184,7 +181,7 @@ def testXmlRpc(self): proxy.test_returning_Fault() except Exception: x = sys.exc_info()[1] - self.assertEqual(x.__class__, Fault) + self.assertEqual(x.__class__, defusedxml.xmlrpclib.monkey_patch().Fault) self.assertEqual(x.faultString, ("custom Fault response")) else: self.fail("Expected xmlrpclib.Fault") diff --git a/lib/cherrypy/wsgiserver/wsgiserver3.py b/lib/cherrypy/wsgiserver/wsgiserver3.py index e69eb0da02a..b21fed722ac 100644 --- a/lib/cherrypy/wsgiserver/wsgiserver3.py +++ b/lib/cherrypy/wsgiserver/wsgiserver3.py @@ -1160,7 +1160,7 @@ def close(self): # the kernel socket. # Python 3 *probably* fixed this with socket._real_close; # hard to tell. -# self.socket._sock.close() + # self.socket._sock.close() self.socket.close() else: # On the other hand, sometimes we want to hang around for a bit @@ -1283,7 +1283,7 @@ class ThreadPool(object): """ def __init__(self, server, min=10, max=-1, - accepted_queue_size=-1, accepted_queue_timeout=10): + accepted_queue_size=-1, accepted_queue_timeout=10): self.server = server self.min = min self.max = max @@ -1627,9 +1627,14 @@ def start(self): except: pass - # So everyone can access the socket... + # So everyone can access the socket, this was set to 777 + # https://docs.openstack.org/developer/bandit/plugins/set_bad_file_permissions.html + # This plugin test looks for the use of chmod and will alert when + # it is used to set particularly permissive control flags. A MEDIUM warning is + # generated if a file is set to group executable and a HIGH warning is reported if a file + # is set world writable. Warnings are given with HIGH confidence. try: - os.chmod(self.bind_addr, 0o777) + os.chmod(self.bind_addr, 0o755) except: pass @@ -1938,8 +1943,8 @@ def __init__(self, bind_addr, wsgi_app, numthreads=10, server_name=None, max=-1, request_queue_size=5, timeout=10, shutdown_timeout=5, accepted_queue_size=-1, accepted_queue_timeout=10): self.requests = ThreadPool(self, min=numthreads or 1, max=max, - accepted_queue_size=accepted_queue_size, - accepted_queue_timeout=accepted_queue_timeout) + accepted_queue_size=accepted_queue_size, + accepted_queue_timeout=accepted_queue_timeout) self.wsgi_app = wsgi_app self.gateway = wsgi_gateways[self.wsgi_version] From 0c0a70fa4c6a0fdc66c7e7880ecc0ef65ae84187 Mon Sep 17 00:00:00 2001 From: Martin Ryan Date: Thu, 20 Dec 2018 10:37:50 +1100 Subject: [PATCH 08/83] bandit security remediation --- .gitignore | 4 ++ lib/cherrypy/lib/xmlrpcutil.py | 4 +- lib/cherrypy/test/test_xmlrpc.py | 60 +++++++++++++++----------- lib/cherrypy/wsgiserver/wsgiserver3.py | 5 ++- 4 files changed, 43 insertions(+), 30 deletions(-) diff --git a/.gitignore b/.gitignore index 90b42a77a6e..7173a223684 100644 --- a/.gitignore +++ b/.gitignore @@ -66,3 +66,7 @@ lib/cherrypy/test/test\.error\.log Pipfile lib/cherrypy/test/test\.conf + +lib/cherrypy/test/static/bigfile\.log + +lib/cherrypy/test/test_misc_tools\.log diff --git a/lib/cherrypy/lib/xmlrpcutil.py b/lib/cherrypy/lib/xmlrpcutil.py index eb6068a795c..80f0af58b88 100644 --- a/lib/cherrypy/lib/xmlrpcutil.py +++ b/lib/cherrypy/lib/xmlrpcutil.py @@ -10,9 +10,9 @@ def get_xmlrpclib(): # function to monkey-patch xmlrpclib and mitigate remote XML attacks. # https://docs.openstack.org/developer/bandit/blacklists/blacklist_imports.html#b411-import-xmlrpclib try: - import defusedxml.client as x + import xmlrpc.client as x except ImportError: - import defusedxml as x + import xmlrpclib as x # nosec return x diff --git a/lib/cherrypy/test/test_xmlrpc.py b/lib/cherrypy/test/test_xmlrpc.py index bc0bbbc57a5..b657675d600 100644 --- a/lib/cherrypy/test/test_xmlrpc.py +++ b/lib/cherrypy/test/test_xmlrpc.py @@ -1,43 +1,49 @@ import sys + import six -# XMLRPC is particularly dangerous as it is also concerned with -# communicating data over a network. Use defused.xmlrpc.monkey_patch() -# function to monkey-patch xmlrpclib and mitigate remote XML attacks. # https://docs.openstack.org/developer/bandit/blacklists/blacklist_imports.html#b411-import-xmlrpclib try: - import defusedxml - + from xmlrpclib import (DateTime, # nosec + Fault, + ProtocolError, + ServerProxy, + SafeTransport) + # from defusedxml.xmlrpclib import DateTime, Fault, ProtocolError + # from defusedxml.xmlrpclib import ServerProxy, SafeTransport except ImportError: - import defusedxml + from xmlrpc.client import (DateTime, + Fault, + ProtocolError, + ServerProxy, + SafeTransport) if six.PY3: - HTTPSTransport = defusedxml.xmlrpclib.monkey_patch().SafeTransport + HTTPSTransport = SafeTransport # Python 3.0's SafeTransport still mistakenly checks for socket.ssl import socket if not hasattr(socket, "ssl"): socket.ssl = True else: - - class HTTPSTransport(): + class HTTPSTransport(SafeTransport): - """Subclass of SafeTransport to fix sock.recv errors (by using file). + """Subclass of SafeTransport to fix sock.recv errors (by using file). """ def request(self, host, handler, request_body, verbose=0): # issue XML-RPC request - h = defusedxml.xmlrpclib.monkey_patch().make_connection(host) + h = self.make_connection(host) if verbose: h.set_debuglevel(1) - defusedxml.xmlrpclib.send_request(h, handler, request_body) - defusedxml.xmlrpclib.send_host(h, host) - defusedxml.xmlrpclib.send_user_agent(h) - defusedxml.xmlrpclib.send_content(h, request_body) + self.send_request(h, handler, request_body) + self.send_host(h, host) + self.send_user_agent(h) + self.send_content(h, request_body) errcode, errmsg, headers = h.getreply() if errcode != 200: - raise defusedxml.xmlrpclib.monkey_patch().ProtocolError(host + handler, errcode, errmsg, headers) + raise ProtocolError(host + handler, errcode, errmsg, headers) self.verbose = verbose @@ -48,7 +54,7 @@ def request(self, host, handler, request_body, verbose=0): # sock = None # return self._parse_response(h.getfile(), sock) - return defusedxml.xmlrpclib.parse_response(h.getfile()) + return self.parse_response(h.getfile()) import cherrypy @@ -98,7 +104,7 @@ def return_float(self): @cherrypy.expose def return_datetime(self): - return defusedxml.xmlrpclib.monkey_patch().DateTime((2003, 10, 7, 8, 1, 0, 1, 280, -1)) + return DateTime((2003, 10, 7, 8, 1, 0, 1, 280, -1)) @cherrypy.expose def return_boolean(self): @@ -110,7 +116,7 @@ def test_argument_passing(self, num): @cherrypy.expose def test_returning_Fault(self): - return defusedxml.xmlrpclib.monkey_patch().Fault(1, "custom Fault response") + return Fault(1, "custom Fault response") root = Root() root.xmlrpc = XmlRpc() @@ -122,17 +128,19 @@ def test_returning_Fault(self): from cherrypy.test import helper + class XmlRpcTest(helper.CPWebCase): setup_server = staticmethod(setup_server) def testXmlRpc(self): + scheme = self.scheme if scheme == "https": url = 'https://%s:%s/xmlrpc/' % (self.interface(), self.PORT) - proxy = defusedxml.xmlrpclib.monkey_patch().ServerProxy(url, transport=HTTPSTransport()) - # else: + proxy = ServerProxy(url, transport=HTTPSTransport()) + else: url = 'http://%s:%s/xmlrpc/' % (self.interface(), self.PORT) - proxy = defusedxml.xmlrpclib.client.monkey_patch().ServerProxy(url) + proxy = ServerProxy(url) # begin the tests ... self.getPage("/xmlrpc/foo") @@ -149,7 +157,7 @@ def testXmlRpc(self): self.assertEqual(proxy.return_int(), 42) self.assertEqual(proxy.return_float(), 3.14) self.assertEqual(proxy.return_datetime(), - defusedxml.xmlrpclib.monkey_patch().DateTime((2003, 10, 7, 8, 1, 0, 1, 280, -1))) + DateTime((2003, 10, 7, 8, 1, 0, 1, 280, -1))) self.assertEqual(proxy.return_boolean(), True) self.assertEqual(proxy.test_argument_passing(22), 22 * 2) @@ -158,7 +166,7 @@ def testXmlRpc(self): proxy.test_argument_passing({}) except Exception: x = sys.exc_info()[1] - self.assertEqual(x.__class__, defusedxml.xmlrpclib.monkey_patch().Fault) + self.assertEqual(x.__class__, Fault) self.assertEqual(x.faultString, ("unsupported operand type(s) " "for *: 'dict' and 'int'")) else: @@ -170,7 +178,7 @@ def testXmlRpc(self): proxy.non_method() except Exception: x = sys.exc_info()[1] - self.assertEqual(x.__class__, defusedxml.xmlrpclib.monkey_patch().Fault) + self.assertEqual(x.__class__, Fault) self.assertEqual(x.faultString, 'method "non_method" is not supported') else: @@ -181,7 +189,7 @@ def testXmlRpc(self): proxy.test_returning_Fault() except Exception: x = sys.exc_info()[1] - self.assertEqual(x.__class__, defusedxml.xmlrpclib.monkey_patch().Fault) + self.assertEqual(x.__class__, Fault) self.assertEqual(x.faultString, ("custom Fault response")) else: self.fail("Expected xmlrpclib.Fault") diff --git a/lib/cherrypy/wsgiserver/wsgiserver3.py b/lib/cherrypy/wsgiserver/wsgiserver3.py index b21fed722ac..4b484b6e7f1 100644 --- a/lib/cherrypy/wsgiserver/wsgiserver3.py +++ b/lib/cherrypy/wsgiserver/wsgiserver3.py @@ -1630,8 +1630,9 @@ def start(self): # So everyone can access the socket, this was set to 777 # https://docs.openstack.org/developer/bandit/plugins/set_bad_file_permissions.html # This plugin test looks for the use of chmod and will alert when - # it is used to set particularly permissive control flags. A MEDIUM warning is - # generated if a file is set to group executable and a HIGH warning is reported if a file + # it is used to set particularly permissive control flags. + # A MEDIUM warning is generated if a file is set to group + # executable and a HIGH warning is reported if a file # is set world writable. Warnings are given with HIGH confidence. try: os.chmod(self.bind_addr, 0o755) From e0cb850bfaf2091a2552276ec68292520b6a0e56 Mon Sep 17 00:00:00 2001 From: Martin Ryan Date: Fri, 21 Dec 2018 09:47:13 +1100 Subject: [PATCH 09/83] address feedback and build issue add import of testfixtures to travis --- .gitignore | 26 +++++++++----------------- .travis.yml | 2 ++ doc/src/cylc-user-guide/cug.tex | 1 + lib/cherrypy/test/test_xmlrpc.py | 2 -- lib/cylc/subprocess_safe.py | 4 ++-- lib/cylc/tests/test_subprocess_safe.py | 5 +++-- 6 files changed, 17 insertions(+), 23 deletions(-) diff --git a/.gitignore b/.gitignore index 7173a223684..7bf3cdb0712 100644 --- a/.gitignore +++ b/.gitignore @@ -49,24 +49,16 @@ passphrase coverage.xml htmlcov/ -\.vscode/settings\.json +# vscode +.vscode/* -\.pytest_cache/v/cache/ +# pytest +.pytest_cache/v/cache/ -\.vscode/launch\.json - -lib/cherrypy/test/access\.log - -lib/cherrypy/test/error\.log - -lib/cherrypy/test/test\.access\.log - -lib/cherrypy/test/test\.error\.log +# cherrypy +lib/cherrypy/**/*.log +lib/cherrypy/**/**/*.log +lib/cherrypy/test/test.conf +# pipenv Pipfile - -lib/cherrypy/test/test\.conf - -lib/cherrypy/test/static/bigfile\.log - -lib/cherrypy/test/test_misc_tools\.log diff --git a/.travis.yml b/.travis.yml index 503bd24e793..f517d560429 100644 --- a/.travis.yml +++ b/.travis.yml @@ -76,6 +76,8 @@ install: # Coverage dependencies - pip install coverage pytest-cov mock - export COVERAGE_PROCESS_START="${TRAVIS_BUILD_DIR}/.coveragerc" + # Testfixtures + - pip install testfixtures # Run tests script: diff --git a/doc/src/cylc-user-guide/cug.tex b/doc/src/cylc-user-guide/cug.tex index 698b1ff3732..6985ef624ff 100644 --- a/doc/src/cylc-user-guide/cug.tex +++ b/doc/src/cylc-user-guide/cug.tex @@ -426,6 +426,7 @@ \subsection{Third-Party Software Packages} \begin{myitemize} \item {\bf mock} - \url{https://mock.readthedocs.io} + \item {\bf testfixtures} - \url{https://testfixtures.readthedocs.io} \end{myitemize} The User Guide is generated from \LaTeX source files by running diff --git a/lib/cherrypy/test/test_xmlrpc.py b/lib/cherrypy/test/test_xmlrpc.py index b657675d600..0a20ef63625 100644 --- a/lib/cherrypy/test/test_xmlrpc.py +++ b/lib/cherrypy/test/test_xmlrpc.py @@ -8,8 +8,6 @@ ProtocolError, ServerProxy, SafeTransport) - # from defusedxml.xmlrpclib import DateTime, Fault, ProtocolError - # from defusedxml.xmlrpclib import ServerProxy, SafeTransport except ImportError: from xmlrpc.client import (DateTime, Fault, diff --git a/lib/cylc/subprocess_safe.py b/lib/cylc/subprocess_safe.py index 903feaa7b65..9932a4fe976 100644 --- a/lib/cylc/subprocess_safe.py +++ b/lib/cylc/subprocess_safe.py @@ -21,8 +21,8 @@ https://docs.openstack.org/developer/bandit/plugins/subprocess_popen_with_shell_equals_true.html REASON IGNORED: Bandit can't determine if the command input is sanitized, it just raises - an issue if it detects Popen with with the option shell=True and so nosec - is used here to supress false positives. + an issue if it detects Popen with the option shell=True and so nosec + is used here to suppress false positives. """ import sys diff --git a/lib/cylc/tests/test_subprocess_safe.py b/lib/cylc/tests/test_subprocess_safe.py index 8ba43840283..9f0198ec445 100644 --- a/lib/cylc/tests/test_subprocess_safe.py +++ b/lib/cylc/tests/test_subprocess_safe.py @@ -43,11 +43,12 @@ def test_subprocess_safe_communicate_with_input(self): Popen = MockPopen() Popen.set_command(command) process = Popen(command, stdout=PIPE, stderr=PIPE, shell=True) - out, err = process.communicate('foo') + err, out = process.communicate('foo') compare([ call.Popen(command, shell=True, stderr=-1, stdout=-1), call.Popen_instance.communicate('foo'), ], Popen.mock.method_calls) + return err, out def test_subprocess_safe_read_from_stdout_and_stderr(self): cmd = "a command" @@ -60,7 +61,7 @@ def test_subprocess_safe_read_from_stdout_and_stderr(self): compare([ call.Popen(command, shell=True, stderr=PIPE, stdout=PIPE), ], Popen.mock.method_calls) - + def test_subprocess_safe_write_to_stdin(self): cmd = "a command" command = quote(cmd) From d7b09311f00ecd3c417a60e2bdd087f334d8ec10 Mon Sep 17 00:00:00 2001 From: Martin Ryan Date: Wed, 2 Jan 2019 11:01:39 +1100 Subject: [PATCH 10/83] address feedback and build issue --- lib/cherrypy/_cpmodpy.py | 28 ++++++++-------- lib/cherrypy/test/modfastcgi.py | 9 ++++-- lib/cherrypy/test/modfcgid.py | 9 ++++-- lib/cherrypy/test/modpy.py | 9 ++++-- lib/cherrypy/test/modwsgi.py | 9 ++++-- lib/cylc/batch_sys_manager.py | 12 +++---- lib/cylc/review.py | 11 +++++-- lib/cylc/run_get_stdout.py | 4 +-- lib/cylc/subprocess_safe.py | 39 +++++++++++------------ lib/cylc/tests/test_subprocess_safe.py | 18 +++-------- lib/parsec/jinja2support.py | 10 ++++-- tests/cylc-cat-state/basic/state-check.py | 10 +++--- 12 files changed, 91 insertions(+), 77 deletions(-) diff --git a/lib/cherrypy/_cpmodpy.py b/lib/cherrypy/_cpmodpy.py index 3f881db18ac..cacf3644863 100644 --- a/lib/cherrypy/_cpmodpy.py +++ b/lib/cherrypy/_cpmodpy.py @@ -271,26 +271,24 @@ def send_response(req, status, headers, body, stream=False): # --------------- Startup tools for CherryPy + mod_python --------------- # import os import re -from cylc.subprocess_safe import popencylc +from cylc.subprocess_safe import pcylc +import subprocess -try: - import subprocess - # import pipes - def popen(fullcmd): - p = popencylc(fullcmd, shell=True, - stdout=subprocess.PIPE, stderr=subprocess.STDOUT, - close_fds=True) - return p.stdout -except ImportError: - def popen(fullcmd): - pipein, pipeout = popencylc(fullcmd) - return pipeout +def popen(fullcmd): + p = pcylc(fullcmd, shell=True, + stdout=subprocess.PIPE, stderr=subprocess.STDOUT, + close_fds=True) + pipeout = p.stdout + return pipeout def read_process(cmd, args=""): - fullcmd = "%s %s" % (cmd, args) - pipeout = popen(fullcmd) + p = pcylc("%s %s" % (cmd, args), + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, stderr=subprocess.STDOUT, + close_fds=True) + pipeout = p.stdout try: firstline = pipeout.readline() cmd_not_found = re.search( diff --git a/lib/cherrypy/test/modfastcgi.py b/lib/cherrypy/test/modfastcgi.py index 2c3c381298d..e76882c75a3 100644 --- a/lib/cherrypy/test/modfastcgi.py +++ b/lib/cherrypy/test/modfastcgi.py @@ -38,7 +38,8 @@ import re import sys import time -from cylc.subprocess_safe import popencylc +import subprocess +from cylc.subprocess_safe import pcylc import cherrypy from cherrypy.process import plugins, servers @@ -46,7 +47,11 @@ def read_process(cmd, args=""): - pipein, pipeout = popencylc("%s %s" % cmd, args) + p = pcylc("%s %s" % (cmd, args), + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, stderr=subprocess.STDOUT, + close_fds=True) + pipeout = p.stdout try: firstline = pipeout.readline() if (re.search(r"(not recognized|No such file|not found)", firstline, diff --git a/lib/cherrypy/test/modfcgid.py b/lib/cherrypy/test/modfcgid.py index b1d8cc79bd8..b3adf8a0d4c 100644 --- a/lib/cherrypy/test/modfcgid.py +++ b/lib/cherrypy/test/modfcgid.py @@ -38,7 +38,8 @@ import re import sys import time -from cylc.subprocess_safe import popencylc +import subprocess +from cylc.subprocess_safe import pcylc import cherrypy from cherrypy._cpcompat import ntob @@ -47,7 +48,11 @@ def read_process(cmd, args=""): - pipein, pipeout = popencylc("%s %s" % cmd, args) + p = pcylc("%s %s" % (cmd, args), + shell=True, stdin=subprocess.PIPE, + stdout=subprocess.PIPE, stderr=subprocess.STDOUT, + close_fds=True) + pipeout = p.stdout try: firstline = pipeout.readline() if (re.search(r"(not recognized|No such file|not found)", firstline, diff --git a/lib/cherrypy/test/modpy.py b/lib/cherrypy/test/modpy.py index b1c567d3e9b..b8aef2efcfe 100644 --- a/lib/cherrypy/test/modpy.py +++ b/lib/cherrypy/test/modpy.py @@ -38,13 +38,18 @@ curdir = os.path.join(os.getcwd(), os.path.dirname(__file__)) import re import time -from cylc.subprocess_safe import popencylc +import subprocess +from cylc.subprocess_safe import pcylc from cherrypy.test import helper def read_process(cmd, args=""): - pipein, pipeout = popencylc("%s %s" % cmd, args) + p = pcylc("%s %s" % (cmd, args), + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, stderr=subprocess.STDOUT, + close_fds=True) + pipeout = p.stdout try: firstline = pipeout.readline() if (re.search(r"(not recognized|No such file|not found)", firstline, diff --git a/lib/cherrypy/test/modwsgi.py b/lib/cherrypy/test/modwsgi.py index 6d96e7b7892..c99aad290eb 100644 --- a/lib/cherrypy/test/modwsgi.py +++ b/lib/cherrypy/test/modwsgi.py @@ -37,14 +37,19 @@ import re import sys import time -from cylc.subprocess_safe import popencylc +import subprocess +from cylc.subprocess_safe import pcylc import cherrypy from cherrypy.test import helper, webtest def read_process(cmd, args=""): - pipein, pipeout = popencylc("%s %s" % cmd, args) + p = pcylc("%s %s" % (cmd, args), + shell=True, stdin=subprocess.PIPE, + stdout=subprocess.PIPE, stderr=subprocess.STDOUT, + close_fds=True) + pipeout = p.stdout try: firstline = pipeout.readline() if (re.search(r"(not recognized|No such file|not found)", firstline, diff --git a/lib/cylc/batch_sys_manager.py b/lib/cylc/batch_sys_manager.py index 9d81ededf5c..4fbe59f2f41 100644 --- a/lib/cylc/batch_sys_manager.py +++ b/lib/cylc/batch_sys_manager.py @@ -113,8 +113,8 @@ from shutil import rmtree from signal import SIGKILL import stat -from subprocess import Popen, PIPE -from cylc.subprocess_safe import popencylc +from subprocess import PIPE +from cylc.subprocess_safe import pcylc import sys import traceback @@ -421,7 +421,7 @@ def job_kill(self, st_file_path): command = shlex.split( batch_sys.KILL_CMD_TMPL % {"job_id": job_id}) try: - proc = Popen( + proc = pcylc( command, stdin=open(os.devnull), stderr=PIPE) except OSError as exc: # subprocess.Popen has a bad habit of not setting the @@ -554,7 +554,7 @@ def _jobs_poll_batch_sys(self, job_log_root, batch_sys_name, my_ctx_list): # Simple poll command that takes a list of job IDs cmd = [batch_sys.POLL_CMD] + exp_ids try: - proc = Popen( + proc = pcylc( cmd, stdin=open(os.devnull), stderr=PIPE, stdout=PIPE) except OSError as exc: # subprocess.Popen has a bad habit of not setting the @@ -660,7 +660,7 @@ def _job_submit_impl( # that we do not have a shell, and still manage to get as far # as here. batch_sys_cmd = batch_submit_cmd_tmpl % {"job": job_file_path} - proc = popencylc( + proc = pcylc( batch_sys_cmd, stdin=proc_stdin_arg, stdout=PIPE, stderr=PIPE, shell=True, env=env) @@ -668,7 +668,7 @@ def _job_submit_impl( command = shlex.split( batch_sys.SUBMIT_CMD_TMPL % {"job": job_file_path}) try: - proc = Popen( + proc = pcylc( command, stdin=proc_stdin_arg, stdout=PIPE, stderr=PIPE, env=env) diff --git a/lib/cylc/review.py b/lib/cylc/review.py index 9c6aca4d01e..f3dae0d735a 100644 --- a/lib/cylc/review.py +++ b/lib/cylc/review.py @@ -29,6 +29,7 @@ from fnmatch import fnmatch from glob import glob import jinja2 +from jinja2 import select_autoescape import json import mimetypes import os @@ -78,9 +79,13 @@ def __init__(self, *args, **kwargs): if self.host_name and "." in self.host_name: self.host_name = self.host_name.split(".", 1)[0] self.cylc_version = CYLC_VERSION - template_env = jinja2.Environment(loader=jinja2.FileSystemLoader( - get_util_home("lib", "cylc", "cylc-review", "template")), - autoescape=True) + template_env = jinja2.Environment( + autoescape=select_autoescape( + enabled_extensions=('rc'), + default_for_string=False, + default=True), + loader=jinja2.FileSystemLoader( + get_util_home("lib", "cylc", "cylc-review", "template"))) template_env.filters['urlise'] = self.url2hyperlink self.template_env = template_env diff --git a/lib/cylc/run_get_stdout.py b/lib/cylc/run_get_stdout.py index b615f65dd90..fffe67c30a4 100644 --- a/lib/cylc/run_get_stdout.py +++ b/lib/cylc/run_get_stdout.py @@ -23,7 +23,7 @@ from subprocess import PIPE from time import sleep, time -from subprocess_safe import popencylc +from subprocess_safe import pcylc ERR_TIMEOUT = "ERROR: command timed out (>%ds), terminated by signal %d\n%s" @@ -46,7 +46,7 @@ def run_get_stdout(command, timeout=None, poll_delay=None): """ try: - popen = popencylc( + popen = pcylc( command, shell=True, preexec_fn=setpgrp, stdin=open(devnull), stderr=PIPE, stdout=PIPE) is_killed_after_timeout = False diff --git a/lib/cylc/subprocess_safe.py b/lib/cylc/subprocess_safe.py index 9932a4fe976..5a89c77fa9e 100644 --- a/lib/cylc/subprocess_safe.py +++ b/lib/cylc/subprocess_safe.py @@ -20,33 +20,30 @@ Bandit B602: subprocess_popen_with_shell_equals_true https://docs.openstack.org/developer/bandit/plugins/subprocess_popen_with_shell_equals_true.html REASON IGNORED: - Bandit can't determine if the command input is sanitized, it just raises - an issue if it detects Popen with the option shell=True and so nosec - is used here to suppress false positives. + Cylc inherently requires shell characters so escaping them + isn't possible. """ -import sys -from pipes import quote +# import sys from subprocess import Popen -from cylc import LOG +# from cylc import LOG # pylint: disable=too-many-arguments # pylint: disable=too-many-locals -def popencylc(cmd, bufsize=0, executable=None, stdin=None, stdout=None, - stderr=None, preexec_fn=None, - close_fds=False, shell=False, cwd=None, env=None, - universal_newlines=False, startupinfo=None, creationflags=0): +def pcylc(cmd, bufsize=0, executable=None, stdin=None, stdout=None, + stderr=None, preexec_fn=None, + close_fds=False, shell=False, cwd=None, env=None, + universal_newlines=False, startupinfo=None, creationflags=0): - try: - command = quote(cmd) - process = Popen(command, bufsize, executable, stdin, stdout, # nosec - stderr, preexec_fn, close_fds, shell, cwd, env, - universal_newlines, startupinfo, creationflags) - return process - except OSError as exc: - LOG.exception(exc) - sys.exit(str(exc)) - if process.returncode: - raise RuntimeError(process.communicate()[1]) + # try: + process = Popen(cmd, bufsize, executable, stdin, stdout, # nosec + stderr, preexec_fn, close_fds, shell, cwd, env, + universal_newlines, startupinfo, creationflags) + return process + # except OSError as exc: + # LOG.exception(exc) + # sys.exit(str(exc)) + # if process.returncode: + # raise RuntimeError(process.communicate()[1]) diff --git a/lib/cylc/tests/test_subprocess_safe.py b/lib/cylc/tests/test_subprocess_safe.py index 9f0198ec445..9c7f557f794 100644 --- a/lib/cylc/tests/test_subprocess_safe.py +++ b/lib/cylc/tests/test_subprocess_safe.py @@ -31,15 +31,8 @@ class TestSubprocessSafe(unittest.TestCase): def setUp(self): self.Popen = MockPopen() - def test_subprocess_safe_quote(self): - cmd = "$!#&'()|<>`\ ; " - command = quote(cmd) - self.assertEqual(command, '\'$!#&\'"\'"\'()|<>`\\ ; \'') - self.assertEqual(command, quote("$!#&'()|<>`\\ ; ")) - def test_subprocess_safe_communicate_with_input(self): - cmd = "a command" - command = quote(cmd) + command = "a command" Popen = MockPopen() Popen.set_command(command) process = Popen(command, stdout=PIPE, stderr=PIPE, shell=True) @@ -51,8 +44,7 @@ def test_subprocess_safe_communicate_with_input(self): return err, out def test_subprocess_safe_read_from_stdout_and_stderr(self): - cmd = "a command" - command = quote(cmd) + command = "a command" Popen = MockPopen() Popen.set_command(command, stdout=b'foo', stderr=b'bar') process = Popen(command, stdout=PIPE, stderr=PIPE, shell=True) @@ -63,8 +55,7 @@ def test_subprocess_safe_read_from_stdout_and_stderr(self): ], Popen.mock.method_calls) def test_subprocess_safe_write_to_stdin(self): - cmd = "a command" - command = quote(cmd) + command = "a command" Popen = MockPopen() Popen.set_command(command) process = Popen(command, stdin=PIPE, shell=True) @@ -77,8 +68,7 @@ def test_subprocess_safe_write_to_stdin(self): ], Popen.mock.method_calls) def test_subprocess_safe_wait_and_return_code(self): - cmd = "a command" - command = quote(cmd) + command = "a command" Popen = MockPopen() Popen.set_command(command, returncode=3) process = Popen(command) diff --git a/lib/parsec/jinja2support.py b/lib/parsec/jinja2support.py index 80a117eef5b..b584448b6cf 100644 --- a/lib/parsec/jinja2support.py +++ b/lib/parsec/jinja2support.py @@ -24,6 +24,7 @@ import os import sys from jinja2 import ( + select_autoescape, BaseLoader, ChoiceLoader, Environment, @@ -93,11 +94,14 @@ def jinja2environment(dir_=None): dir_ = os.getcwd() env = Environment( + autoescape=select_autoescape( + enabled_extensions=('rc'), + default_for_string=False, + default=True), loader=ChoiceLoader([FileSystemLoader(dir_), PyModuleLoader()]), undefined=StrictUndefined, - extensions=['jinja2.ext.do'], - autoescape=True - ) + extensions=['jinja2.ext.do'] + ) # Load any custom Jinja2 filters, tests or globals in the suite # definition directory diff --git a/tests/cylc-cat-state/basic/state-check.py b/tests/cylc-cat-state/basic/state-check.py index 305b5766372..c46c7740df6 100644 --- a/tests/cylc-cat-state/basic/state-check.py +++ b/tests/cylc-cat-state/basic/state-check.py @@ -19,7 +19,7 @@ import sqlite3 import subprocess import sys -from cylc.subprocess_safe import popencylc +from cylc.subprocess_safe import pcylc def main(argv): @@ -30,10 +30,10 @@ def main(argv): sname = argv[0] rundir = argv[1] - - p = popencylc("cylc cat-state " + sname, - shell=True, stdout=subprocess.PIPE, - stderr=subprocess.PIPE) + + p = pcylc("cylc cat-state " + sname, + shell=True, stdout=subprocess.PIPE, + stderr=subprocess.PIPE) state, err = p.communicate() if p.returncode > 0: From 24825c37666340920324a83795d73dcbf22d9a4b Mon Sep 17 00:00:00 2001 From: Martin Ryan Date: Wed, 2 Jan 2019 13:39:38 +1100 Subject: [PATCH 11/83] remove unused unsafe source files from cherrypy --- lib/cherrypy/_cpmodpy.py | 354 ---- lib/cherrypy/test/modfastcgi.py | 143 -- lib/cherrypy/test/modfcgid.py | 131 -- lib/cherrypy/test/modpy.py | 171 -- lib/cherrypy/test/modwsgi.py | 156 -- lib/cherrypy/test/test_xmlrpc.py | 193 --- lib/cherrypy/wsgiserver/wsgiserver3.py | 2204 ------------------------ 7 files changed, 3352 deletions(-) delete mode 100644 lib/cherrypy/_cpmodpy.py delete mode 100644 lib/cherrypy/test/modfastcgi.py delete mode 100644 lib/cherrypy/test/modfcgid.py delete mode 100644 lib/cherrypy/test/modpy.py delete mode 100644 lib/cherrypy/test/modwsgi.py delete mode 100644 lib/cherrypy/test/test_xmlrpc.py delete mode 100644 lib/cherrypy/wsgiserver/wsgiserver3.py diff --git a/lib/cherrypy/_cpmodpy.py b/lib/cherrypy/_cpmodpy.py deleted file mode 100644 index cacf3644863..00000000000 --- a/lib/cherrypy/_cpmodpy.py +++ /dev/null @@ -1,354 +0,0 @@ -"""Native adapter for serving CherryPy via mod_python - -Basic usage: - -########################################## -# Application in a module called myapp.py -########################################## - -import cherrypy - -class Root: - @cherrypy.expose - def index(self): - return 'Hi there, Ho there, Hey there' - - -# We will use this method from the mod_python configuration -# as the entry point to our application -def setup_server(): - cherrypy.tree.mount(Root()) - cherrypy.config.update({'environment': 'production', - 'log.screen': False, - 'show_tracebacks': False}) - -########################################## -# mod_python settings for apache2 -# This should reside in your httpd.conf -# or a file that will be loaded at -# apache startup -########################################## - -# Start -DocumentRoot "/" -Listen 8080 -LoadModule python_module /usr/lib/apache2/modules/mod_python.so - - - PythonPath "sys.path+['/path/to/my/application']" - SetHandler python-program - PythonHandler cherrypy._cpmodpy::handler - PythonOption cherrypy.setup myapp::setup_server - PythonDebug On - -# End - -The actual path to your mod_python.so is dependent on your -environment. In this case we suppose a global mod_python -installation on a Linux distribution such as Ubuntu. - -We do set the PythonPath configuration setting so that -your application can be found by from the user running -the apache2 instance. Of course if your application -resides in the global site-package this won't be needed. - -Then restart apache2 and access http://127.0.0.1:8080 -""" - -import logging -import sys -import io - -import cherrypy -from cherrypy._cpcompat import copyitems, ntob -from cherrypy._cperror import format_exc, bare_error -from cherrypy.lib import httputil - -# ------------------------------ Request-handling - - -def setup(req): - from mod_python import apache - - # Run any setup functions defined by a "PythonOption cherrypy.setup" - # directive. - options = req.get_options() - if 'cherrypy.setup' in options: - for function in options['cherrypy.setup'].split(): - atoms = function.split('::', 1) - if len(atoms) == 1: - mod = __import__(atoms[0], globals(), locals()) - else: - modname, fname = atoms - mod = __import__(modname, globals(), locals(), [fname]) - func = getattr(mod, fname) - func() - - cherrypy.config.update({'log.screen': False, - "tools.ignore_headers.on": True, - "tools.ignore_headers.headers": ['Range'], - }) - - engine = cherrypy.engine - if hasattr(engine, "signal_handler"): - engine.signal_handler.unsubscribe() - if hasattr(engine, "console_control_handler"): - engine.console_control_handler.unsubscribe() - engine.autoreload.unsubscribe() - cherrypy.server.unsubscribe() - - def _log(msg, level): - newlevel = apache.APLOG_ERR - if logging.DEBUG >= level: - newlevel = apache.APLOG_DEBUG - elif logging.INFO >= level: - newlevel = apache.APLOG_INFO - elif logging.WARNING >= level: - newlevel = apache.APLOG_WARNING - # On Windows, req.server is required or the msg will vanish. See - # http://www.modpython.org/pipermail/mod_python/2003-October/014291.html - # Also, "When server is not specified...LogLevel does not apply..." - apache.log_error(msg, newlevel, req.server) - engine.subscribe('log', _log) - - engine.start() - - def cherrypy_cleanup(data): - engine.exit() - try: - # apache.register_cleanup wasn't available until 3.1.4. - apache.register_cleanup(cherrypy_cleanup) - except AttributeError: - req.server.register_cleanup(req, cherrypy_cleanup) - - -class _ReadOnlyRequest: - expose = ('read', 'readline', 'readlines') - - def __init__(self, req): - for method in self.expose: - self.__dict__[method] = getattr(req, method) - - -recursive = False - -_isSetUp = False - - -def handler(req): - from mod_python import apache - try: - global _isSetUp - if not _isSetUp: - setup(req) - _isSetUp = True - - # Obtain a Request object from CherryPy - local = req.connection.local_addr - local = httputil.Host( - local[0], local[1], req.connection.local_host or "") - remote = req.connection.remote_addr - remote = httputil.Host( - remote[0], remote[1], req.connection.remote_host or "") - - scheme = req.parsed_uri[0] or 'http' - req.get_basic_auth_pw() - - try: - # apache.mpm_query only became available in mod_python 3.1 - q = apache.mpm_query - threaded = q(apache.AP_MPMQ_IS_THREADED) - forked = q(apache.AP_MPMQ_IS_FORKED) - except AttributeError: - bad_value = ("You must provide a PythonOption '%s', " - "either 'on' or 'off', when running a version " - "of mod_python < 3.1") - - threaded = options.get('multithread', '').lower() - if threaded == 'on': - threaded = True - elif threaded == 'off': - threaded = False - else: - raise ValueError(bad_value % "multithread") - - forked = options.get('multiprocess', '').lower() - if forked == 'on': - forked = True - elif forked == 'off': - forked = False - else: - raise ValueError(bad_value % "multiprocess") - - sn = cherrypy.tree.script_name(req.uri or "/") - if sn is None: - send_response(req, '404 Not Found', [], '') - else: - app = cherrypy.tree.apps[sn] - method = req.method - path = req.uri - qs = req.args or "" - reqproto = req.protocol - headers = copyitems(req.headers_in) - rfile = _ReadOnlyRequest(req) - prev = None - - try: - redirections = [] - while True: - request, response = app.get_serving(local, remote, scheme, - "HTTP/1.1") - request.login = req.user - request.multithread = bool(threaded) - request.multiprocess = bool(forked) - request.app = app - request.prev = prev - - # Run the CherryPy Request object and obtain the response - try: - request.run(method, path, qs, reqproto, headers, rfile) - break - except cherrypy.InternalRedirect: - ir = sys.exc_info()[1] - app.release_serving() - prev = request - - if not recursive: - if ir.path in redirections: - raise RuntimeError( - "InternalRedirector visited the same URL " - "twice: %r" % ir.path) - else: - # Add the *previous* path_info + qs to - # redirections. - if qs: - qs = "?" + qs - redirections.append(sn + path + qs) - - # Munge environment and try again. - method = "GET" - path = ir.path - qs = ir.query_string - rfile = io.BytesIO() - - send_response( - req, response.output_status, response.header_list, - response.body, response.stream) - finally: - app.release_serving() - except: - tb = format_exc() - cherrypy.log(tb, 'MOD_PYTHON', severity=logging.ERROR) - s, h, b = bare_error() - send_response(req, s, h, b) - return apache.OK - - -def send_response(req, status, headers, body, stream=False): - # Set response status - req.status = int(status[:3]) - - # Set response headers - req.content_type = "text/plain" - for header, value in headers: - if header.lower() == 'content-type': - req.content_type = value - continue - req.headers_out.add(header, value) - - if stream: - # Flush now so the status and headers are sent immediately. - req.flush() - - # Set response body - if isinstance(body, basestring): - req.write(body) - else: - for seg in body: - req.write(seg) - - -# --------------- Startup tools for CherryPy + mod_python --------------- # -import os -import re -from cylc.subprocess_safe import pcylc -import subprocess - - -def popen(fullcmd): - p = pcylc(fullcmd, shell=True, - stdout=subprocess.PIPE, stderr=subprocess.STDOUT, - close_fds=True) - pipeout = p.stdout - return pipeout - - -def read_process(cmd, args=""): - p = pcylc("%s %s" % (cmd, args), - stdin=subprocess.PIPE, - stdout=subprocess.PIPE, stderr=subprocess.STDOUT, - close_fds=True) - pipeout = p.stdout - try: - firstline = pipeout.readline() - cmd_not_found = re.search( - ntob("(not recognized|No such file|not found)"), - firstline, - re.IGNORECASE - ) - if cmd_not_found: - raise IOError('%s must be on your system path.' % cmd) - output = firstline + pipeout.read() - finally: - pipeout.close() - return output - - -class ModPythonServer(object): - - template = """ -# Apache2 server configuration file for running CherryPy with mod_python. - -DocumentRoot "/" -Listen %(port)s -LoadModule python_module modules/mod_python.so - - - SetHandler python-program - PythonHandler %(handler)s - PythonDebug On -%(opts)s - -""" - - def __init__(self, loc="/", port=80, opts=None, apache_path="apache", - handler="cherrypy._cpmodpy::handler"): - self.loc = loc - self.port = port - self.opts = opts - self.apache_path = apache_path - self.handler = handler - - def start(self): - opts = "".join([" PythonOption %s %s\n" % (k, v) - for k, v in self.opts]) - conf_data = self.template % {"port": self.port, - "loc": self.loc, - "opts": opts, - "handler": self.handler, - } - - mpconf = os.path.join(os.path.dirname(__file__), "cpmodpy.conf") - f = open(mpconf, 'wb') - try: - f.write(conf_data) - finally: - f.close() - - response = read_process(self.apache_path, "-k start -f %s" % mpconf) - self.ready = True - return response - - def stop(self): - os.popen("apache -k stop") - self.ready = False diff --git a/lib/cherrypy/test/modfastcgi.py b/lib/cherrypy/test/modfastcgi.py deleted file mode 100644 index e76882c75a3..00000000000 --- a/lib/cherrypy/test/modfastcgi.py +++ /dev/null @@ -1,143 +0,0 @@ -"""Wrapper for mod_fastcgi, for use as a CherryPy HTTP server when testing. - -To autostart fastcgi, the "apache" executable or script must be -on your system path, or you must override the global APACHE_PATH. -On some platforms, "apache" may be called "apachectl", "apache2ctl", -or "httpd"--create a symlink to them if needed. - -You'll also need the WSGIServer from flup.servers. -See http://projects.amor.org/misc/wiki/ModPythonGateway - - -KNOWN BUGS -========== - -1. Apache processes Range headers automatically; CherryPy's truncated - output is then truncated again by Apache. See test_core.testRanges. - This was worked around in http://www.cherrypy.org/changeset/1319. -2. Apache does not allow custom HTTP methods like CONNECT as per the spec. - See test_core.testHTTPMethods. -3. Max request header and body settings do not work with Apache. -4. Apache replaces status "reason phrases" automatically. For example, - CherryPy may set "304 Not modified" but Apache will write out - "304 Not Modified" (capital "M"). -5. Apache does not allow custom error codes as per the spec. -6. Apache (or perhaps modpython, or modpython_gateway) unquotes %xx in the - Request-URI too early. -7. mod_python will not read request bodies which use the "chunked" - transfer-coding (it passes REQUEST_CHUNKED_ERROR to ap_setup_client_block - instead of REQUEST_CHUNKED_DECHUNK, see Apache2's http_protocol.c and - mod_python's requestobject.c). -8. Apache will output a "Content-Length: 0" response header even if there's - no response entity body. This isn't really a bug; it just differs from - the CherryPy default. -""" - -import os -curdir = os.path.join(os.getcwd(), os.path.dirname(__file__)) -import re -import sys -import time -import subprocess -from cylc.subprocess_safe import pcylc - -import cherrypy -from cherrypy.process import plugins, servers -from cherrypy.test import helper - - -def read_process(cmd, args=""): - p = pcylc("%s %s" % (cmd, args), - stdin=subprocess.PIPE, - stdout=subprocess.PIPE, stderr=subprocess.STDOUT, - close_fds=True) - pipeout = p.stdout - try: - firstline = pipeout.readline() - if (re.search(r"(not recognized|No such file|not found)", firstline, - re.IGNORECASE)): - raise IOError('%s must be on your system path.' % cmd) - output = firstline + pipeout.read() - finally: - pipeout.close() - return output - - -APACHE_PATH = "apache2ctl" -CONF_PATH = "fastcgi.conf" - -conf_fastcgi = """ -# Apache2 server conf file for testing CherryPy with mod_fastcgi. -# fumanchu: I had to hard-code paths due to crazy Debian layouts :( -ServerRoot /usr/lib/apache2 -User #1000 -ErrorLog %(root)s/mod_fastcgi.error.log - -DocumentRoot "%(root)s" -ServerName 127.0.0.1 -Listen %(port)s -LoadModule fastcgi_module modules/mod_fastcgi.so -LoadModule rewrite_module modules/mod_rewrite.so - -Options +ExecCGI -SetHandler fastcgi-script -RewriteEngine On -RewriteRule ^(.*)$ /fastcgi.pyc [L] -FastCgiExternalServer "%(server)s" -host 127.0.0.1:4000 -""" - - -def erase_script_name(environ, start_response): - environ['SCRIPT_NAME'] = '' - return cherrypy.tree(environ, start_response) - - -class ModFCGISupervisor(helper.LocalWSGISupervisor): - - httpserver_class = "cherrypy.process.servers.FlupFCGIServer" - using_apache = True - using_wsgi = True - template = conf_fastcgi - - def __str__(self): - return "FCGI Server on %s:%s" % (self.host, self.port) - - def start(self, modulename): - cherrypy.server.httpserver = servers.FlupFCGIServer( - application=erase_script_name, bindAddress=('127.0.0.1', 4000)) - cherrypy.server.httpserver.bind_addr = ('127.0.0.1', 4000) - cherrypy.server.socket_port = 4000 - # For FCGI, we both start apache... - self.start_apache() - # ...and our local server - cherrypy.engine.start() - self.sync_apps() - - def start_apache(self): - fcgiconf = CONF_PATH - if not os.path.isabs(fcgiconf): - fcgiconf = os.path.join(curdir, fcgiconf) - - # Write the Apache conf file. - f = open(fcgiconf, 'wb') - try: - server = repr(os.path.join(curdir, 'fastcgi.pyc'))[1:-1] - output = self.template % {'port': self.port, 'root': curdir, - 'server': server} - output = output.replace('\r\n', '\n') - f.write(output) - finally: - f.close() - - result = read_process(APACHE_PATH, "-k start -f %s" % fcgiconf) - if result: - print(result) - - def stop(self): - """Gracefully shutdown a server that is serving forever.""" - read_process(APACHE_PATH, "-k stop") - helper.LocalWSGISupervisor.stop(self) - - def sync_apps(self): - cherrypy.server.httpserver.fcgiserver.application = self.get_app( - erase_script_name) diff --git a/lib/cherrypy/test/modfcgid.py b/lib/cherrypy/test/modfcgid.py deleted file mode 100644 index b3adf8a0d4c..00000000000 --- a/lib/cherrypy/test/modfcgid.py +++ /dev/null @@ -1,131 +0,0 @@ -"""Wrapper for mod_fcgid, for use as a CherryPy HTTP server when testing. - -To autostart fcgid, the "apache" executable or script must be -on your system path, or you must override the global APACHE_PATH. -On some platforms, "apache" may be called "apachectl", "apache2ctl", -or "httpd"--create a symlink to them if needed. - -You'll also need the WSGIServer from flup.servers. -See http://projects.amor.org/misc/wiki/ModPythonGateway - - -KNOWN BUGS -========== - -1. Apache processes Range headers automatically; CherryPy's truncated - output is then truncated again by Apache. See test_core.testRanges. - This was worked around in http://www.cherrypy.org/changeset/1319. -2. Apache does not allow custom HTTP methods like CONNECT as per the spec. - See test_core.testHTTPMethods. -3. Max request header and body settings do not work with Apache. -4. Apache replaces status "reason phrases" automatically. For example, - CherryPy may set "304 Not modified" but Apache will write out - "304 Not Modified" (capital "M"). -5. Apache does not allow custom error codes as per the spec. -6. Apache (or perhaps modpython, or modpython_gateway) unquotes %xx in the - Request-URI too early. -7. mod_python will not read request bodies which use the "chunked" - transfer-coding (it passes REQUEST_CHUNKED_ERROR to ap_setup_client_block - instead of REQUEST_CHUNKED_DECHUNK, see Apache2's http_protocol.c and - mod_python's requestobject.c). -8. Apache will output a "Content-Length: 0" response header even if there's - no response entity body. This isn't really a bug; it just differs from - the CherryPy default. -""" - -import os -curdir = os.path.join(os.getcwd(), os.path.dirname(__file__)) -import re -import sys -import time -import subprocess -from cylc.subprocess_safe import pcylc - -import cherrypy -from cherrypy._cpcompat import ntob -from cherrypy.process import plugins, servers -from cherrypy.test import helper - - -def read_process(cmd, args=""): - p = pcylc("%s %s" % (cmd, args), - shell=True, stdin=subprocess.PIPE, - stdout=subprocess.PIPE, stderr=subprocess.STDOUT, - close_fds=True) - pipeout = p.stdout - try: - firstline = pipeout.readline() - if (re.search(r"(not recognized|No such file|not found)", firstline, - re.IGNORECASE)): - raise IOError('%s must be on your system path.' % cmd) - output = firstline + pipeout.read() - finally: - pipeout.close() - return output - - -APACHE_PATH = "httpd" -CONF_PATH = "fcgi.conf" - -conf_fcgid = """ -# Apache2 server conf file for testing CherryPy with mod_fcgid. - -DocumentRoot "%(root)s" -ServerName 127.0.0.1 -Listen %(port)s -LoadModule fastcgi_module modules/mod_fastcgi.dll -LoadModule rewrite_module modules/mod_rewrite.so - -Options ExecCGI -SetHandler fastcgi-script -RewriteEngine On -RewriteRule ^(.*)$ /fastcgi.pyc [L] -FastCgiExternalServer "%(server)s" -host 127.0.0.1:4000 -""" - - -class ModFCGISupervisor(helper.LocalSupervisor): - - using_apache = True - using_wsgi = True - template = conf_fcgid - - def __str__(self): - return "FCGI Server on %s:%s" % (self.host, self.port) - - def start(self, modulename): - cherrypy.server.httpserver = servers.FlupFCGIServer( - application=cherrypy.tree, bindAddress=('127.0.0.1', 4000)) - cherrypy.server.httpserver.bind_addr = ('127.0.0.1', 4000) - # For FCGI, we both start apache... - self.start_apache() - # ...and our local server - helper.LocalServer.start(self, modulename) - - def start_apache(self): - fcgiconf = CONF_PATH - if not os.path.isabs(fcgiconf): - fcgiconf = os.path.join(curdir, fcgiconf) - - # Write the Apache conf file. - f = open(fcgiconf, 'wb') - try: - server = repr(os.path.join(curdir, 'fastcgi.pyc'))[1:-1] - output = self.template % {'port': self.port, 'root': curdir, - 'server': server} - output = ntob(output.replace('\r\n', '\n')) - f.write(output) - finally: - f.close() - - result = read_process(APACHE_PATH, "-k start -f %s" % fcgiconf) - if result: - print(result) - - def stop(self): - """Gracefully shutdown a server that is serving forever.""" - read_process(APACHE_PATH, "-k stop") - helper.LocalServer.stop(self) - - def sync_apps(self): - cherrypy.server.httpserver.fcgiserver.application = self.get_app() diff --git a/lib/cherrypy/test/modpy.py b/lib/cherrypy/test/modpy.py deleted file mode 100644 index b8aef2efcfe..00000000000 --- a/lib/cherrypy/test/modpy.py +++ /dev/null @@ -1,171 +0,0 @@ -"""Wrapper for mod_python, for use as a CherryPy HTTP server when testing. - -To autostart modpython, the "apache" executable or script must be -on your system path, or you must override the global APACHE_PATH. -On some platforms, "apache" may be called "apachectl" or "apache2ctl"-- -create a symlink to them if needed. - -If you wish to test the WSGI interface instead of our _cpmodpy interface, -you also need the 'modpython_gateway' module at: -http://projects.amor.org/misc/wiki/ModPythonGateway - - -KNOWN BUGS -========== - -1. Apache processes Range headers automatically; CherryPy's truncated - output is then truncated again by Apache. See test_core.testRanges. - This was worked around in http://www.cherrypy.org/changeset/1319. -2. Apache does not allow custom HTTP methods like CONNECT as per the spec. - See test_core.testHTTPMethods. -3. Max request header and body settings do not work with Apache. -4. Apache replaces status "reason phrases" automatically. For example, - CherryPy may set "304 Not modified" but Apache will write out - "304 Not Modified" (capital "M"). -5. Apache does not allow custom error codes as per the spec. -6. Apache (or perhaps modpython, or modpython_gateway) unquotes %xx in the - Request-URI too early. -7. mod_python will not read request bodies which use the "chunked" - transfer-coding (it passes REQUEST_CHUNKED_ERROR to ap_setup_client_block - instead of REQUEST_CHUNKED_DECHUNK, see Apache2's http_protocol.c and - mod_python's requestobject.c). -8. Apache will output a "Content-Length: 0" response header even if there's - no response entity body. This isn't really a bug; it just differs from - the CherryPy default. -""" - -import os -curdir = os.path.join(os.getcwd(), os.path.dirname(__file__)) -import re -import time -import subprocess -from cylc.subprocess_safe import pcylc - -from cherrypy.test import helper - - -def read_process(cmd, args=""): - p = pcylc("%s %s" % (cmd, args), - stdin=subprocess.PIPE, - stdout=subprocess.PIPE, stderr=subprocess.STDOUT, - close_fds=True) - pipeout = p.stdout - try: - firstline = pipeout.readline() - if (re.search(r"(not recognized|No such file|not found)", firstline, - re.IGNORECASE)): - raise IOError('%s must be on your system path.' % cmd) - output = firstline + pipeout.read() - finally: - pipeout.close() - return output - - -APACHE_PATH = "httpd" -CONF_PATH = "test_mp.conf" - -conf_modpython_gateway = """ -# Apache2 server conf file for testing CherryPy with modpython_gateway. - -ServerName 127.0.0.1 -DocumentRoot "/" -Listen %(port)s -LoadModule python_module modules/mod_python.so - -SetHandler python-program -PythonFixupHandler cherrypy.test.modpy::wsgisetup -PythonOption testmod %(modulename)s -PythonHandler modpython_gateway::handler -PythonOption wsgi.application cherrypy::tree -PythonOption socket_host %(host)s -PythonDebug On -""" - -conf_cpmodpy = """ -# Apache2 server conf file for testing CherryPy with _cpmodpy. - -ServerName 127.0.0.1 -DocumentRoot "/" -Listen %(port)s -LoadModule python_module modules/mod_python.so - -SetHandler python-program -PythonFixupHandler cherrypy.test.modpy::cpmodpysetup -PythonHandler cherrypy._cpmodpy::handler -PythonOption cherrypy.setup cherrypy.test.%(modulename)s::setup_server -PythonOption socket_host %(host)s -PythonDebug On -""" - - -class ModPythonSupervisor(helper.Supervisor): - - using_apache = True - using_wsgi = False - template = None - - def __str__(self): - return "ModPython Server on %s:%s" % (self.host, self.port) - - def start(self, modulename): - mpconf = CONF_PATH - if not os.path.isabs(mpconf): - mpconf = os.path.join(curdir, mpconf) - - f = open(mpconf, 'wb') - try: - f.write(self.template % - {'port': self.port, 'modulename': modulename, - 'host': self.host}) - finally: - f.close() - - result = read_process(APACHE_PATH, "-k start -f %s" % mpconf) - if result: - print(result) - - def stop(self): - """Gracefully shutdown a server that is serving forever.""" - read_process(APACHE_PATH, "-k stop") - - -loaded = False - - -def wsgisetup(req): - global loaded - if not loaded: - loaded = True - options = req.get_options() - - import cherrypy - cherrypy.config.update({ - "log.error_file": os.path.join(curdir, "test.log"), - "environment": "test_suite", - "server.socket_host": options['socket_host'], - }) - - modname = options['testmod'] - mod = __import__(modname, globals(), locals(), ['']) - mod.setup_server() - - cherrypy.server.unsubscribe() - cherrypy.engine.start() - from mod_python import apache - return apache.OK - - -def cpmodpysetup(req): - global loaded - if not loaded: - loaded = True - options = req.get_options() - - import cherrypy - cherrypy.config.update({ - "log.error_file": os.path.join(curdir, "test.log"), - "environment": "test_suite", - "server.socket_host": options['socket_host'], - }) - from mod_python import apache - return apache.OK diff --git a/lib/cherrypy/test/modwsgi.py b/lib/cherrypy/test/modwsgi.py deleted file mode 100644 index c99aad290eb..00000000000 --- a/lib/cherrypy/test/modwsgi.py +++ /dev/null @@ -1,156 +0,0 @@ -"""Wrapper for mod_wsgi, for use as a CherryPy HTTP server. - -To autostart modwsgi, the "apache" executable or script must be -on your system path, or you must override the global APACHE_PATH. -On some platforms, "apache" may be called "apachectl" or "apache2ctl"-- -create a symlink to them if needed. - - -KNOWN BUGS -========== - -##1. Apache processes Range headers automatically; CherryPy's truncated -## output is then truncated again by Apache. See test_core.testRanges. -## This was worked around in http://www.cherrypy.org/changeset/1319. -2. Apache does not allow custom HTTP methods like CONNECT as per the spec. - See test_core.testHTTPMethods. -3. Max request header and body settings do not work with Apache. -##4. Apache replaces status "reason phrases" automatically. For example, -## CherryPy may set "304 Not modified" but Apache will write out -## "304 Not Modified" (capital "M"). -##5. Apache does not allow custom error codes as per the spec. -##6. Apache (or perhaps modpython, or modpython_gateway) unquotes %xx in the -## Request-URI too early. -7. mod_wsgi will not read request bodies which use the "chunked" - transfer-coding (it passes REQUEST_CHUNKED_ERROR to ap_setup_client_block - instead of REQUEST_CHUNKED_DECHUNK, see Apache2's http_protocol.c and - mod_python's requestobject.c). -8. When responding with 204 No Content, mod_wsgi adds a Content-Length - header for you. -9. When an error is raised, mod_wsgi has no facility for printing a - traceback as the response content (it's sent to the Apache log instead). -10. Startup and shutdown of Apache when running mod_wsgi seems slow. -""" - -import os -curdir = os.path.abspath(os.path.dirname(__file__)) -import re -import sys -import time -import subprocess -from cylc.subprocess_safe import pcylc - -import cherrypy -from cherrypy.test import helper, webtest - - -def read_process(cmd, args=""): - p = pcylc("%s %s" % (cmd, args), - shell=True, stdin=subprocess.PIPE, - stdout=subprocess.PIPE, stderr=subprocess.STDOUT, - close_fds=True) - pipeout = p.stdout - try: - firstline = pipeout.readline() - if (re.search(r"(not recognized|No such file|not found)", firstline, - re.IGNORECASE)): - raise IOError('%s must be on your system path.' % cmd) - output = firstline + pipeout.read() - finally: - pipeout.close() - return output - - -if sys.platform == 'win32': - APACHE_PATH = "httpd" -else: - APACHE_PATH = "apache" - -CONF_PATH = "test_mw.conf" - -conf_modwsgi = r""" -# Apache2 server conf file for testing CherryPy with modpython_gateway. - -ServerName 127.0.0.1 -DocumentRoot "/" -Listen %(port)s - -AllowEncodedSlashes On -LoadModule rewrite_module modules/mod_rewrite.so -RewriteEngine on -RewriteMap escaping int:escape - -LoadModule log_config_module modules/mod_log_config.so -LogFormat "%%h %%l %%u %%t \"%%r\" %%>s %%b \"%%{Referer}i\" \"%%{User-agent}i\"" combined -CustomLog "%(curdir)s/apache.access.log" combined -ErrorLog "%(curdir)s/apache.error.log" -LogLevel debug - -LoadModule wsgi_module modules/mod_wsgi.so -LoadModule env_module modules/mod_env.so - -WSGIScriptAlias / "%(curdir)s/modwsgi.py" -SetEnv testmod %(testmod)s -""" - - -class ModWSGISupervisor(helper.Supervisor): - - """Server Controller for ModWSGI and CherryPy.""" - - using_apache = True - using_wsgi = True - template = conf_modwsgi - - def __str__(self): - return "ModWSGI Server on %s:%s" % (self.host, self.port) - - def start(self, modulename): - mpconf = CONF_PATH - if not os.path.isabs(mpconf): - mpconf = os.path.join(curdir, mpconf) - - f = open(mpconf, 'wb') - try: - output = (self.template % - {'port': self.port, 'testmod': modulename, - 'curdir': curdir}) - f.write(output) - finally: - f.close() - - result = read_process(APACHE_PATH, "-k start -f %s" % mpconf) - if result: - print(result) - - # Make a request so mod_wsgi starts up our app. - # If we don't, concurrent initial requests will 404. - cherrypy._cpserver.wait_for_occupied_port("127.0.0.1", self.port) - webtest.openURL('/ihopetheresnodefault', port=self.port) - time.sleep(1) - - def stop(self): - """Gracefully shutdown a server that is serving forever.""" - read_process(APACHE_PATH, "-k stop") - - -loaded = False - - -def application(environ, start_response): - import cherrypy - global loaded - if not loaded: - loaded = True - modname = "cherrypy.test." + environ['testmod'] - mod = __import__(modname, globals(), locals(), ['']) - mod.setup_server() - - cherrypy.config.update({ - "log.error_file": os.path.join(curdir, "test.error.log"), - "log.access_file": os.path.join(curdir, "test.access.log"), - "environment": "test_suite", - "engine.SIGHUP": None, - "engine.SIGTERM": None, - }) - return cherrypy.tree(environ, start_response) diff --git a/lib/cherrypy/test/test_xmlrpc.py b/lib/cherrypy/test/test_xmlrpc.py deleted file mode 100644 index 0a20ef63625..00000000000 --- a/lib/cherrypy/test/test_xmlrpc.py +++ /dev/null @@ -1,193 +0,0 @@ -import sys - -import six -# https://docs.openstack.org/developer/bandit/blacklists/blacklist_imports.html#b411-import-xmlrpclib -try: - from xmlrpclib import (DateTime, # nosec - Fault, - ProtocolError, - ServerProxy, - SafeTransport) -except ImportError: - from xmlrpc.client import (DateTime, - Fault, - ProtocolError, - ServerProxy, - SafeTransport) - -if six.PY3: - HTTPSTransport = SafeTransport - - # Python 3.0's SafeTransport still mistakenly checks for socket.ssl - import socket - if not hasattr(socket, "ssl"): - socket.ssl = True -else: - class HTTPSTransport(SafeTransport): - - """Subclass of SafeTransport to fix sock.recv errors (by using file). - """ - - def request(self, host, handler, request_body, verbose=0): - # issue XML-RPC request - h = self.make_connection(host) - if verbose: - h.set_debuglevel(1) - - self.send_request(h, handler, request_body) - self.send_host(h, host) - self.send_user_agent(h) - self.send_content(h, request_body) - - errcode, errmsg, headers = h.getreply() - if errcode != 200: - raise ProtocolError(host + handler, errcode, errmsg, headers) - - self.verbose = verbose - - # Here's where we differ from the superclass. It says: - # try: - # sock = h._conn.sock - # except AttributeError: - # sock = None - # return self._parse_response(h.getfile(), sock) - - return self.parse_response(h.getfile()) - -import cherrypy - - -def setup_server(): - from cherrypy import _cptools - - class Root: - - @cherrypy.expose - def index(self): - return "I'm a standard index!" - - class XmlRpc(_cptools.XMLRPCController): - - @cherrypy.expose - def foo(self): - return "Hello world!" - - @cherrypy.expose - def return_single_item_list(self): - return [42] - - @cherrypy.expose - def return_string(self): - return "here is a string" - - @cherrypy.expose - def return_tuple(self): - return ('here', 'is', 1, 'tuple') - - @cherrypy.expose - def return_dict(self): - return dict(a=1, b=2, c=3) - - @cherrypy.expose - def return_composite(self): - return dict(a=1, z=26), 'hi', ['welcome', 'friend'] - - @cherrypy.expose - def return_int(self): - return 42 - - @cherrypy.expose - def return_float(self): - return 3.14 - - @cherrypy.expose - def return_datetime(self): - return DateTime((2003, 10, 7, 8, 1, 0, 1, 280, -1)) - - @cherrypy.expose - def return_boolean(self): - return True - - @cherrypy.expose - def test_argument_passing(self, num): - return num * 2 - - @cherrypy.expose - def test_returning_Fault(self): - return Fault(1, "custom Fault response") - - root = Root() - root.xmlrpc = XmlRpc() - cherrypy.tree.mount(root, config={'/': { - 'request.dispatch': cherrypy.dispatch.XMLRPCDispatcher(), - 'tools.xmlrpc.allow_none': 0, - }}) - - -from cherrypy.test import helper - - -class XmlRpcTest(helper.CPWebCase): - setup_server = staticmethod(setup_server) - - def testXmlRpc(self): - - scheme = self.scheme - if scheme == "https": - url = 'https://%s:%s/xmlrpc/' % (self.interface(), self.PORT) - proxy = ServerProxy(url, transport=HTTPSTransport()) - else: - url = 'http://%s:%s/xmlrpc/' % (self.interface(), self.PORT) - proxy = ServerProxy(url) - - # begin the tests ... - self.getPage("/xmlrpc/foo") - self.assertBody("Hello world!") - - self.assertEqual(proxy.return_single_item_list(), [42]) - self.assertNotEqual(proxy.return_single_item_list(), 'one bazillion') - self.assertEqual(proxy.return_string(), "here is a string") - self.assertEqual(proxy.return_tuple(), - list(('here', 'is', 1, 'tuple'))) - self.assertEqual(proxy.return_dict(), {'a': 1, 'c': 3, 'b': 2}) - self.assertEqual(proxy.return_composite(), - [{'a': 1, 'z': 26}, 'hi', ['welcome', 'friend']]) - self.assertEqual(proxy.return_int(), 42) - self.assertEqual(proxy.return_float(), 3.14) - self.assertEqual(proxy.return_datetime(), - DateTime((2003, 10, 7, 8, 1, 0, 1, 280, -1))) - self.assertEqual(proxy.return_boolean(), True) - self.assertEqual(proxy.test_argument_passing(22), 22 * 2) - - # Test an error in the page handler (should raise an xmlrpclib.Fault) - try: - proxy.test_argument_passing({}) - except Exception: - x = sys.exc_info()[1] - self.assertEqual(x.__class__, Fault) - self.assertEqual(x.faultString, ("unsupported operand type(s) " - "for *: 'dict' and 'int'")) - else: - self.fail("Expected xmlrpclib.Fault") - - # https://github.com/cherrypy/cherrypy/issues/533 - # if a method is not found, an xmlrpclib.Fault should be raised - try: - proxy.non_method() - except Exception: - x = sys.exc_info()[1] - self.assertEqual(x.__class__, Fault) - self.assertEqual(x.faultString, - 'method "non_method" is not supported') - else: - self.fail("Expected xmlrpclib.Fault") - - # Test returning a Fault from the page handler. - try: - proxy.test_returning_Fault() - except Exception: - x = sys.exc_info()[1] - self.assertEqual(x.__class__, Fault) - self.assertEqual(x.faultString, ("custom Fault response")) - else: - self.fail("Expected xmlrpclib.Fault") diff --git a/lib/cherrypy/wsgiserver/wsgiserver3.py b/lib/cherrypy/wsgiserver/wsgiserver3.py deleted file mode 100644 index 4b484b6e7f1..00000000000 --- a/lib/cherrypy/wsgiserver/wsgiserver3.py +++ /dev/null @@ -1,2204 +0,0 @@ -"""A high-speed, production ready, thread pooled, generic HTTP server. - -Simplest example on how to use this module directly -(without using CherryPy's application machinery):: - - from cherrypy import wsgiserver - - def my_crazy_app(environ, start_response): - status = '200 OK' - response_headers = [('Content-type','text/plain')] - start_response(status, response_headers) - return ['Hello world!'] - - server = wsgiserver.CherryPyWSGIServer( - ('0.0.0.0', 8070), my_crazy_app, - server_name='www.cherrypy.example') - server.start() - -The CherryPy WSGI server can serve as many WSGI applications -as you want in one instance by using a WSGIPathInfoDispatcher:: - - d = WSGIPathInfoDispatcher({'/': my_crazy_app, '/blog': my_blog_app}) - server = wsgiserver.CherryPyWSGIServer(('0.0.0.0', 80), d) - -Want SSL support? Just set server.ssl_adapter to an SSLAdapter instance. - -This won't call the CherryPy engine (application side) at all, only the -HTTP server, which is independent from the rest of CherryPy. Don't -let the name "CherryPyWSGIServer" throw you; the name merely reflects -its origin, not its coupling. - -For those of you wanting to understand internals of this module, here's the -basic call flow. The server's listening thread runs a very tight loop, -sticking incoming connections onto a Queue:: - - server = CherryPyWSGIServer(...) - server.start() - while True: - tick() - # This blocks until a request comes in: - child = socket.accept() - conn = HTTPConnection(child, ...) - server.requests.put(conn) - -Worker threads are kept in a pool and poll the Queue, popping off and then -handling each connection in turn. Each connection can consist of an arbitrary -number of requests and their responses, so we run a nested loop:: - - while True: - conn = server.requests.get() - conn.communicate() - -> while True: - req = HTTPRequest(...) - req.parse_request() - -> # Read the Request-Line, e.g. "GET /page HTTP/1.1" - req.rfile.readline() - read_headers(req.rfile, req.inheaders) - req.respond() - -> response = app(...) - try: - for chunk in response: - if chunk: - req.write(chunk) - finally: - if hasattr(response, "close"): - response.close() - if req.close_connection: - return -""" - -__all__ = ['HTTPRequest', 'HTTPConnection', 'HTTPServer', - 'SizeCheckWrapper', 'KnownLengthRFile', 'ChunkedRFile', - 'CP_makefile', - 'MaxSizeExceeded', 'NoSSLError', 'FatalSSLAlert', - 'WorkerThread', 'ThreadPool', 'SSLAdapter', - 'CherryPyWSGIServer', - 'Gateway', 'WSGIGateway', 'WSGIGateway_10', 'WSGIGateway_u0', - 'WSGIPathInfoDispatcher', 'get_ssl_adapter_class', - 'socket_errors_to_ignore'] - -import os -try: - import queue -except: - import Queue as queue -import re -import email.utils -import socket -import sys -import threading -import time -import traceback as traceback_ -import errno -import logging -from urllib.parse import urlparse - -try: - # prefer slower Python-based io module - import _pyio as io -except ImportError: - # Python 2.6 - import io - -try: - import pkg_resources -except ImportError: - pass - -if 'win' in sys.platform and hasattr(socket, "AF_INET6"): - if not hasattr(socket, 'IPPROTO_IPV6'): - socket.IPPROTO_IPV6 = 41 - if not hasattr(socket, 'IPV6_V6ONLY'): - socket.IPV6_V6ONLY = 27 - - -DEFAULT_BUFFER_SIZE = io.DEFAULT_BUFFER_SIZE - - -try: - cp_version = pkg_resources.require('cherrypy')[0].version -except Exception: - cp_version = 'unknown' - - -if sys.version_info >= (3, 0): - unicodestr = str - basestring = (bytes, str) - - def ntob(n, encoding='ISO-8859-1'): - """Return the given native string as a byte string in the given - encoding. - """ - # In Python 3, the native string type is unicode - return n.encode(encoding) -else: - unicodestr = unicode - basestring = basestring - - def ntob(n, encoding='ISO-8859-1'): - """Return the given native string as a byte string in the given - encoding. - """ - # In Python 2, the native string type is bytes. Assume it's already - # in the given encoding, which for ISO-8859-1 is almost always what - # was intended. - return n - -LF = ntob('\n') -CRLF = ntob('\r\n') -TAB = ntob('\t') -SPACE = ntob(' ') -COLON = ntob(':') -SEMICOLON = ntob(';') -EMPTY = ntob('') -NUMBER_SIGN = ntob('#') -QUESTION_MARK = ntob('?') -ASTERISK = ntob('*') -FORWARD_SLASH = ntob('/') -quoted_slash = re.compile(ntob("(?i)%2F")) - - -def plat_specific_errors(*errnames): - """Return error numbers for all errors in errnames on this platform. - - The 'errno' module contains different global constants depending on - the specific platform (OS). This function will return the list of - numeric values for a given list of potential names. - """ - errno_names = dir(errno) - nums = [getattr(errno, k) for k in errnames if k in errno_names] - # de-dupe the list - return list(dict.fromkeys(nums).keys()) - -socket_error_eintr = plat_specific_errors("EINTR", "WSAEINTR") - -socket_errors_to_ignore = plat_specific_errors( - "EPIPE", - "EBADF", "WSAEBADF", - "ENOTSOCK", "WSAENOTSOCK", - "ETIMEDOUT", "WSAETIMEDOUT", - "ECONNREFUSED", "WSAECONNREFUSED", - "ECONNRESET", "WSAECONNRESET", - "ECONNABORTED", "WSAECONNABORTED", - "ENETRESET", "WSAENETRESET", - "EHOSTDOWN", "EHOSTUNREACH", -) -socket_errors_to_ignore.append("timed out") -socket_errors_to_ignore.append("The read operation timed out") -if sys.platform == 'darwin': - socket_errors_to_ignore.append(plat_specific_errors("EPROTOTYPE")) - -socket_errors_nonblocking = plat_specific_errors( - 'EAGAIN', 'EWOULDBLOCK', 'WSAEWOULDBLOCK') - -comma_separated_headers = [ - ntob(h) for h in - ['Accept', 'Accept-Charset', 'Accept-Encoding', - 'Accept-Language', 'Accept-Ranges', 'Allow', 'Cache-Control', - 'Connection', 'Content-Encoding', 'Content-Language', 'Expect', - 'If-Match', 'If-None-Match', 'Pragma', 'Proxy-Authenticate', 'TE', - 'Trailer', 'Transfer-Encoding', 'Upgrade', 'Vary', 'Via', 'Warning', - 'WWW-Authenticate'] -] - - -if not hasattr(logging, 'statistics'): - logging.statistics = {} - - -def read_headers(rfile, hdict=None): - """Read headers from the given stream into the given header dict. - - If hdict is None, a new header dict is created. Returns the populated - header dict. - - Headers which are repeated are folded together using a comma if their - specification so dictates. - - This function raises ValueError when the read bytes violate the HTTP spec. - You should probably return "400 Bad Request" if this happens. - """ - if hdict is None: - hdict = {} - - while True: - line = rfile.readline() - if not line: - # No more data--illegal end of headers - raise ValueError("Illegal end of headers.") - - if line == CRLF: - # Normal end of headers - break - if not line.endswith(CRLF): - raise ValueError("HTTP requires CRLF terminators") - - if line[0] in (SPACE, TAB): - # It's a continuation line. - v = line.strip() - else: - try: - k, v = line.split(COLON, 1) - except ValueError: - raise ValueError("Illegal header line.") - # TODO: what about TE and WWW-Authenticate? - k = k.strip().title() - v = v.strip() - hname = k - - if k in comma_separated_headers: - existing = hdict.get(hname) - if existing: - v = b", ".join((existing, v)) - hdict[hname] = v - - return hdict - - -class MaxSizeExceeded(Exception): - pass - - -class SizeCheckWrapper(object): - - """Wraps a file-like object, raising MaxSizeExceeded if too large.""" - - def __init__(self, rfile, maxlen): - self.rfile = rfile - self.maxlen = maxlen - self.bytes_read = 0 - - def _check_length(self): - if self.maxlen and self.bytes_read > self.maxlen: - raise MaxSizeExceeded() - - def read(self, size=None): - data = self.rfile.read(size) - self.bytes_read += len(data) - self._check_length() - return data - - def readline(self, size=None): - if size is not None: - data = self.rfile.readline(size) - self.bytes_read += len(data) - self._check_length() - return data - - # User didn't specify a size ... - # We read the line in chunks to make sure it's not a 100MB line ! - res = [] - while True: - data = self.rfile.readline(256) - self.bytes_read += len(data) - self._check_length() - res.append(data) - # See https://github.com/cherrypy/cherrypy/issues/421 - if len(data) < 256 or data[-1:] == LF: - return EMPTY.join(res) - - def readlines(self, sizehint=0): - # Shamelessly stolen from StringIO - total = 0 - lines = [] - line = self.readline() - while line: - lines.append(line) - total += len(line) - if 0 < sizehint <= total: - break - line = self.readline() - return lines - - def close(self): - self.rfile.close() - - def __iter__(self): - return self - - def __next__(self): - data = next(self.rfile) - self.bytes_read += len(data) - self._check_length() - return data - - def next(self): - data = self.rfile.next() - self.bytes_read += len(data) - self._check_length() - return data - - -class KnownLengthRFile(object): - - """Wraps a file-like object, returning an empty string when exhausted.""" - - def __init__(self, rfile, content_length): - self.rfile = rfile - self.remaining = content_length - - def read(self, size=None): - if self.remaining == 0: - return b'' - if size is None: - size = self.remaining - else: - size = min(size, self.remaining) - - data = self.rfile.read(size) - self.remaining -= len(data) - return data - - def readline(self, size=None): - if self.remaining == 0: - return b'' - if size is None: - size = self.remaining - else: - size = min(size, self.remaining) - - data = self.rfile.readline(size) - self.remaining -= len(data) - return data - - def readlines(self, sizehint=0): - # Shamelessly stolen from StringIO - total = 0 - lines = [] - line = self.readline(sizehint) - while line: - lines.append(line) - total += len(line) - if 0 < sizehint <= total: - break - line = self.readline(sizehint) - return lines - - def close(self): - self.rfile.close() - - def __iter__(self): - return self - - def __next__(self): - data = next(self.rfile) - self.remaining -= len(data) - return data - - -class ChunkedRFile(object): - - """Wraps a file-like object, returning an empty string when exhausted. - - This class is intended to provide a conforming wsgi.input value for - request entities that have been encoded with the 'chunked' transfer - encoding. - """ - - def __init__(self, rfile, maxlen, bufsize=8192): - self.rfile = rfile - self.maxlen = maxlen - self.bytes_read = 0 - self.buffer = EMPTY - self.bufsize = bufsize - self.closed = False - - def _fetch(self): - if self.closed: - return - - line = self.rfile.readline() - self.bytes_read += len(line) - - if self.maxlen and self.bytes_read > self.maxlen: - raise MaxSizeExceeded("Request Entity Too Large", self.maxlen) - - line = line.strip().split(SEMICOLON, 1) - - try: - chunk_size = line.pop(0) - chunk_size = int(chunk_size, 16) - except ValueError: - raise ValueError("Bad chunked transfer size: " + repr(chunk_size)) - - if chunk_size <= 0: - self.closed = True - return - -## if line: chunk_extension = line[0] - - if self.maxlen and self.bytes_read + chunk_size > self.maxlen: - raise IOError("Request Entity Too Large") - - chunk = self.rfile.read(chunk_size) - self.bytes_read += len(chunk) - self.buffer += chunk - - crlf = self.rfile.read(2) - if crlf != CRLF: - raise ValueError( - "Bad chunked transfer coding (expected '\\r\\n', " - "got " + repr(crlf) + ")") - - def read(self, size=None): - data = EMPTY - while True: - if size and len(data) >= size: - return data - - if not self.buffer: - self._fetch() - if not self.buffer: - # EOF - return data - - if size: - remaining = size - len(data) - data += self.buffer[:remaining] - self.buffer = self.buffer[remaining:] - else: - data += self.buffer - - def readline(self, size=None): - data = EMPTY - while True: - if size and len(data) >= size: - return data - - if not self.buffer: - self._fetch() - if not self.buffer: - # EOF - return data - - newline_pos = self.buffer.find(LF) - if size: - if newline_pos == -1: - remaining = size - len(data) - data += self.buffer[:remaining] - self.buffer = self.buffer[remaining:] - else: - remaining = min(size - len(data), newline_pos) - data += self.buffer[:remaining] - self.buffer = self.buffer[remaining:] - else: - if newline_pos == -1: - data += self.buffer - else: - data += self.buffer[:newline_pos] - self.buffer = self.buffer[newline_pos:] - - def readlines(self, sizehint=0): - # Shamelessly stolen from StringIO - total = 0 - lines = [] - line = self.readline(sizehint) - while line: - lines.append(line) - total += len(line) - if 0 < sizehint <= total: - break - line = self.readline(sizehint) - return lines - - def read_trailer_lines(self): - if not self.closed: - raise ValueError( - "Cannot read trailers until the request body has been read.") - - while True: - line = self.rfile.readline() - if not line: - # No more data--illegal end of headers - raise ValueError("Illegal end of headers.") - - self.bytes_read += len(line) - if self.maxlen and self.bytes_read > self.maxlen: - raise IOError("Request Entity Too Large") - - if line == CRLF: - # Normal end of headers - break - if not line.endswith(CRLF): - raise ValueError("HTTP requires CRLF terminators") - - yield line - - def close(self): - self.rfile.close() - - def __iter__(self): - # Shamelessly stolen from StringIO - total = 0 - line = self.readline(sizehint) - while line: - yield line - total += len(line) - if 0 < sizehint <= total: - break - line = self.readline(sizehint) - - -class HTTPRequest(object): - - """An HTTP Request (and response). - - A single HTTP connection may consist of multiple request/response pairs. - """ - - server = None - """The HTTPServer object which is receiving this request.""" - - conn = None - """The HTTPConnection object on which this request connected.""" - - inheaders = {} - """A dict of request headers.""" - - outheaders = [] - """A list of header tuples to write in the response.""" - - ready = False - """When True, the request has been parsed and is ready to begin generating - the response. When False, signals the calling Connection that the response - should not be generated and the connection should close.""" - - close_connection = False - """Signals the calling Connection that the request should close. This does - not imply an error! The client and/or server may each request that the - connection be closed.""" - - chunked_write = False - """If True, output will be encoded with the "chunked" transfer-coding. - - This value is set automatically inside send_headers.""" - - def __init__(self, server, conn): - self.server = server - self.conn = conn - - self.ready = False - self.started_request = False - self.scheme = ntob("http") - if self.server.ssl_adapter is not None: - self.scheme = ntob("https") - # Use the lowest-common protocol in case read_request_line errors. - self.response_protocol = 'HTTP/1.0' - self.inheaders = {} - - self.status = "" - self.outheaders = [] - self.sent_headers = False - self.close_connection = self.__class__.close_connection - self.chunked_read = False - self.chunked_write = self.__class__.chunked_write - - def parse_request(self): - """Parse the next HTTP request start-line and message-headers.""" - self.rfile = SizeCheckWrapper(self.conn.rfile, - self.server.max_request_header_size) - try: - success = self.read_request_line() - except MaxSizeExceeded: - self.simple_response( - "414 Request-URI Too Long", - "The Request-URI sent with the request exceeds the maximum " - "allowed bytes.") - return - else: - if not success: - return - - try: - success = self.read_request_headers() - except MaxSizeExceeded: - self.simple_response( - "413 Request Entity Too Large", - "The headers sent with the request exceed the maximum " - "allowed bytes.") - return - else: - if not success: - return - - self.ready = True - - def read_request_line(self): - # HTTP/1.1 connections are persistent by default. If a client - # requests a page, then idles (leaves the connection open), - # then rfile.readline() will raise socket.error("timed out"). - # Note that it does this based on the value given to settimeout(), - # and doesn't need the client to request or acknowledge the close - # (although your TCP stack might suffer for it: cf Apache's history - # with FIN_WAIT_2). - request_line = self.rfile.readline() - - # Set started_request to True so communicate() knows to send 408 - # from here on out. - self.started_request = True - if not request_line: - return False - - if request_line == CRLF: - # RFC 2616 sec 4.1: "...if the server is reading the protocol - # stream at the beginning of a message and receives a CRLF - # first, it should ignore the CRLF." - # But only ignore one leading line! else we enable a DoS. - request_line = self.rfile.readline() - if not request_line: - return False - - if not request_line.endswith(CRLF): - self.simple_response( - "400 Bad Request", "HTTP requires CRLF terminators") - return False - - try: - method, uri, req_protocol = request_line.strip().split(SPACE, 2) - # The [x:y] slicing is necessary for byte strings to avoid getting - # ord's - rp = int(req_protocol[5:6]), int(req_protocol[7:8]) - except ValueError: - self.simple_response("400 Bad Request", "Malformed Request-Line") - return False - - self.uri = uri - self.method = method - - # uri may be an abs_path (including "http://host.domain.tld"); - scheme, authority, path = self.parse_request_uri(uri) - if path is None: - self.simple_response("400 Bad Request", - "Invalid path in Request-URI.") - return False - if NUMBER_SIGN in path: - self.simple_response("400 Bad Request", - "Illegal #fragment in Request-URI.") - return False - - if scheme: - self.scheme = scheme - - qs = EMPTY - if QUESTION_MARK in path: - path, qs = path.split(QUESTION_MARK, 1) - - # Unquote the path+params (e.g. "/this%20path" -> "/this path"). - # http://www.w3.org/Protocols/rfc2616/rfc2616-sec5.html#sec5.1.2 - # - # But note that "...a URI must be separated into its components - # before the escaped characters within those components can be - # safely decoded." http://www.ietf.org/rfc/rfc2396.txt, sec 2.4.2 - # Therefore, "/this%2Fpath" becomes "/this%2Fpath", not "/this/path". - try: - atoms = [self.unquote_bytes(x) for x in quoted_slash.split(path)] - except ValueError: - ex = sys.exc_info()[1] - self.simple_response("400 Bad Request", ex.args[0]) - return False - path = b"%2F".join(atoms) - self.path = path - - # Note that, like wsgiref and most other HTTP servers, - # we "% HEX HEX"-unquote the path but not the query string. - self.qs = qs - - # Compare request and server HTTP protocol versions, in case our - # server does not support the requested protocol. Limit our output - # to min(req, server). We want the following output: - # request server actual written supported response - # protocol protocol response protocol feature set - # a 1.0 1.0 1.0 1.0 - # b 1.0 1.1 1.1 1.0 - # c 1.1 1.0 1.0 1.0 - # d 1.1 1.1 1.1 1.1 - # Notice that, in (b), the response will be "HTTP/1.1" even though - # the client only understands 1.0. RFC 2616 10.5.6 says we should - # only return 505 if the _major_ version is different. - # The [x:y] slicing is necessary for byte strings to avoid getting - # ord's - sp = int(self.server.protocol[5:6]), int(self.server.protocol[7:8]) - - if sp[0] != rp[0]: - self.simple_response("505 HTTP Version Not Supported") - return False - - self.request_protocol = req_protocol - self.response_protocol = "HTTP/%s.%s" % min(rp, sp) - return True - - def read_request_headers(self): - """Read self.rfile into self.inheaders. Return success.""" - - # then all the http headers - try: - read_headers(self.rfile, self.inheaders) - except ValueError: - ex = sys.exc_info()[1] - self.simple_response("400 Bad Request", ex.args[0]) - return False - - mrbs = self.server.max_request_body_size - if mrbs and int(self.inheaders.get(b"Content-Length", 0)) > mrbs: - self.simple_response( - "413 Request Entity Too Large", - "The entity sent with the request exceeds the maximum " - "allowed bytes.") - return False - - # Persistent connection support - if self.response_protocol == "HTTP/1.1": - # Both server and client are HTTP/1.1 - if self.inheaders.get(b"Connection", b"") == b"close": - self.close_connection = True - else: - # Either the server or client (or both) are HTTP/1.0 - if self.inheaders.get(b"Connection", b"") != b"Keep-Alive": - self.close_connection = True - - # Transfer-Encoding support - te = None - if self.response_protocol == "HTTP/1.1": - te = self.inheaders.get(b"Transfer-Encoding") - if te: - te = [x.strip().lower() for x in te.split(b",") if x.strip()] - - self.chunked_read = False - - if te: - for enc in te: - if enc == b"chunked": - self.chunked_read = True - else: - # Note that, even if we see "chunked", we must reject - # if there is an extension we don't recognize. - self.simple_response("501 Unimplemented") - self.close_connection = True - return False - - # From PEP 333: - # "Servers and gateways that implement HTTP 1.1 must provide - # transparent support for HTTP 1.1's "expect/continue" mechanism. - # This may be done in any of several ways: - # 1. Respond to requests containing an Expect: 100-continue request - # with an immediate "100 Continue" response, and proceed normally. - # 2. Proceed with the request normally, but provide the application - # with a wsgi.input stream that will send the "100 Continue" - # response if/when the application first attempts to read from - # the input stream. The read request must then remain blocked - # until the client responds. - # 3. Wait until the client decides that the server does not support - # expect/continue, and sends the request body on its own. - # (This is suboptimal, and is not recommended.) - # - # We used to do 3, but are now doing 1. Maybe we'll do 2 someday, - # but it seems like it would be a big slowdown for such a rare case. - if self.inheaders.get(b"Expect", b"") == b"100-continue": - # Don't use simple_response here, because it emits headers - # we don't want. See - # https://github.com/cherrypy/cherrypy/issues/951 - msg = self.server.protocol.encode( - 'ascii') + b" 100 Continue\r\n\r\n" - try: - self.conn.wfile.write(msg) - except socket.error: - x = sys.exc_info()[1] - if x.args[0] not in socket_errors_to_ignore: - raise - return True - - def parse_request_uri(self, uri): - """Parse a Request-URI into (scheme, authority, path). - - Note that Request-URI's must be one of:: - - Request-URI = "*" | absoluteURI | abs_path | authority - - Therefore, a Request-URI which starts with a double forward-slash - cannot be a "net_path":: - - net_path = "//" authority [ abs_path ] - - Instead, it must be interpreted as an "abs_path" with an empty first - path segment:: - - abs_path = "/" path_segments - path_segments = segment *( "/" segment ) - segment = *pchar *( ";" param ) - param = *pchar - """ - if uri == ASTERISK: - return None, None, uri - - scheme, authority, path, params, query, fragment = urlparse(uri) - if scheme and QUESTION_MARK not in scheme: - # An absoluteURI. - # If there's a scheme (and it must be http or https), then: - # http_URL = "http:" "//" host [ ":" port ] [ abs_path [ "?" query - # ]] - return scheme, authority, path - - if uri.startswith(FORWARD_SLASH): - # An abs_path. - return None, None, uri - else: - # An authority. - return None, uri, None - - def unquote_bytes(self, path): - """takes quoted string and unquotes % encoded values""" - res = path.split(b'%') - - for i in range(1, len(res)): - item = res[i] - try: - res[i] = bytes([int(item[:2], 16)]) + item[2:] - except ValueError: - raise - return b''.join(res) - - def respond(self): - """Call the gateway and write its iterable output.""" - mrbs = self.server.max_request_body_size - if self.chunked_read: - self.rfile = ChunkedRFile(self.conn.rfile, mrbs) - else: - cl = int(self.inheaders.get(b"Content-Length", 0)) - if mrbs and mrbs < cl: - if not self.sent_headers: - self.simple_response( - "413 Request Entity Too Large", - "The entity sent with the request exceeds the " - "maximum allowed bytes.") - return - self.rfile = KnownLengthRFile(self.conn.rfile, cl) - - self.server.gateway(self).respond() - - if (self.ready and not self.sent_headers): - self.sent_headers = True - self.send_headers() - if self.chunked_write: - self.conn.wfile.write(b"0\r\n\r\n") - - def simple_response(self, status, msg=""): - """Write a simple response back to the client.""" - status = str(status) - buf = [bytes(self.server.protocol, "ascii") + SPACE + - bytes(status, "ISO-8859-1") + CRLF, - bytes("Content-Length: %s\r\n" % len(msg), "ISO-8859-1"), - b"Content-Type: text/plain\r\n"] - - if status[:3] in ("413", "414"): - # Request Entity Too Large / Request-URI Too Long - self.close_connection = True - if self.response_protocol == 'HTTP/1.1': - # This will not be true for 414, since read_request_line - # usually raises 414 before reading the whole line, and we - # therefore cannot know the proper response_protocol. - buf.append(b"Connection: close\r\n") - else: - # HTTP/1.0 had no 413/414 status nor Connection header. - # Emit 400 instead and trust the message body is enough. - status = "400 Bad Request" - - buf.append(CRLF) - if msg: - if isinstance(msg, unicodestr): - msg = msg.encode("ISO-8859-1") - buf.append(msg) - - try: - self.conn.wfile.write(b"".join(buf)) - except socket.error: - x = sys.exc_info()[1] - if x.args[0] not in socket_errors_to_ignore: - raise - - def write(self, chunk): - """Write unbuffered data to the client.""" - if self.chunked_write and chunk: - buf = [bytes(hex(len(chunk)), 'ASCII')[2:], CRLF, chunk, CRLF] - self.conn.wfile.write(EMPTY.join(buf)) - else: - self.conn.wfile.write(chunk) - - def send_headers(self): - """Assert, process, and send the HTTP response message-headers. - - You must set self.status, and self.outheaders before calling this. - """ - hkeys = [key.lower() for key, value in self.outheaders] - status = int(self.status[:3]) - - if status == 413: - # Request Entity Too Large. Close conn to avoid garbage. - self.close_connection = True - elif b"content-length" not in hkeys: - # "All 1xx (informational), 204 (no content), - # and 304 (not modified) responses MUST NOT - # include a message-body." So no point chunking. - if status < 200 or status in (204, 205, 304): - pass - else: - if (self.response_protocol == 'HTTP/1.1' - and self.method != b'HEAD'): - # Use the chunked transfer-coding - self.chunked_write = True - self.outheaders.append((b"Transfer-Encoding", b"chunked")) - else: - # Closing the conn is the only way to determine len. - self.close_connection = True - - if b"connection" not in hkeys: - if self.response_protocol == 'HTTP/1.1': - # Both server and client are HTTP/1.1 or better - if self.close_connection: - self.outheaders.append((b"Connection", b"close")) - else: - # Server and/or client are HTTP/1.0 - if not self.close_connection: - self.outheaders.append((b"Connection", b"Keep-Alive")) - - if (not self.close_connection) and (not self.chunked_read): - # Read any remaining request body data on the socket. - # "If an origin server receives a request that does not include an - # Expect request-header field with the "100-continue" expectation, - # the request includes a request body, and the server responds - # with a final status code before reading the entire request body - # from the transport connection, then the server SHOULD NOT close - # the transport connection until it has read the entire request, - # or until the client closes the connection. Otherwise, the client - # might not reliably receive the response message. However, this - # requirement is not be construed as preventing a server from - # defending itself against denial-of-service attacks, or from - # badly broken client implementations." - remaining = getattr(self.rfile, 'remaining', 0) - if remaining > 0: - self.rfile.read(remaining) - - if b"date" not in hkeys: - self.outheaders.append(( - b"Date", - email.utils.formatdate(usegmt=True).encode('ISO-8859-1') - )) - - if b"server" not in hkeys: - self.outheaders.append( - (b"Server", self.server.server_name.encode('ISO-8859-1'))) - - buf = [self.server.protocol.encode( - 'ascii') + SPACE + self.status + CRLF] - for k, v in self.outheaders: - buf.append(k + COLON + SPACE + v + CRLF) - buf.append(CRLF) - self.conn.wfile.write(EMPTY.join(buf)) - - -class NoSSLError(Exception): - - """Exception raised when a client speaks HTTP to an HTTPS socket.""" - pass - - -class FatalSSLAlert(Exception): - - """Exception raised when the SSL implementation signals a fatal alert.""" - pass - - -class CP_BufferedWriter(io.BufferedWriter): - - """Faux file object attached to a socket object.""" - - def write(self, b): - self._checkClosed() - if isinstance(b, str): - raise TypeError("can't write str to binary stream") - - with self._write_lock: - self._write_buf.extend(b) - self._flush_unlocked() - return len(b) - - def _flush_unlocked(self): - self._checkClosed("flush of closed file") - while self._write_buf: - try: - # ssl sockets only except 'bytes', not bytearrays - # so perhaps we should conditionally wrap this for perf? - n = self.raw.write(bytes(self._write_buf)) - except io.BlockingIOError as e: - n = e.characters_written - del self._write_buf[:n] - - -def CP_makefile(sock, mode='r', bufsize=DEFAULT_BUFFER_SIZE): - if 'r' in mode: - return io.BufferedReader(socket.SocketIO(sock, mode), bufsize) - else: - return CP_BufferedWriter(socket.SocketIO(sock, mode), bufsize) - - -class HTTPConnection(object): - - """An HTTP connection (active socket). - - server: the Server object which received this connection. - socket: the raw socket object (usually TCP) for this connection. - makefile: a fileobject class for reading from the socket. - """ - - remote_addr = None - remote_port = None - ssl_env = None - rbufsize = DEFAULT_BUFFER_SIZE - wbufsize = DEFAULT_BUFFER_SIZE - RequestHandlerClass = HTTPRequest - - def __init__(self, server, sock, makefile=CP_makefile): - self.server = server - self.socket = sock - self.rfile = makefile(sock, "rb", self.rbufsize) - self.wfile = makefile(sock, "wb", self.wbufsize) - self.requests_seen = 0 - - def communicate(self): - """Read each request and respond appropriately.""" - request_seen = False - try: - while True: - # (re)set req to None so that if something goes wrong in - # the RequestHandlerClass constructor, the error doesn't - # get written to the previous request. - req = None - req = self.RequestHandlerClass(self.server, self) - - # This order of operations should guarantee correct pipelining. - req.parse_request() - if self.server.stats['Enabled']: - self.requests_seen += 1 - if not req.ready: - # Something went wrong in the parsing (and the server has - # probably already made a simple_response). Return and - # let the conn close. - return - - request_seen = True - req.respond() - if req.close_connection: - return - except socket.error: - e = sys.exc_info()[1] - errnum = e.args[0] - # sadly SSL sockets return a different (longer) time out string - if ( - errnum == 'timed out' or - errnum == 'The read operation timed out' - ): - # Don't error if we're between requests; only error - # if 1) no request has been started at all, or 2) we're - # in the middle of a request. - # See https://github.com/cherrypy/cherrypy/issues/853 - if (not request_seen) or (req and req.started_request): - # Don't bother writing the 408 if the response - # has already started being written. - if req and not req.sent_headers: - try: - req.simple_response("408 Request Timeout") - except FatalSSLAlert: - # Close the connection. - return - elif errnum not in socket_errors_to_ignore: - self.server.error_log("socket.error %s" % repr(errnum), - level=logging.WARNING, traceback=True) - if req and not req.sent_headers: - try: - req.simple_response("500 Internal Server Error") - except FatalSSLAlert: - # Close the connection. - return - return - except (KeyboardInterrupt, SystemExit): - raise - except FatalSSLAlert: - # Close the connection. - return - except NoSSLError: - if req and not req.sent_headers: - # Unwrap our wfile - self.wfile = CP_makefile( - self.socket._sock, "wb", self.wbufsize) - req.simple_response( - "400 Bad Request", - "The client sent a plain HTTP request, but this server " - "only speaks HTTPS on this port.") - self.linger = True - except Exception: - e = sys.exc_info()[1] - self.server.error_log(repr(e), level=logging.ERROR, traceback=True) - if req and not req.sent_headers: - try: - req.simple_response("500 Internal Server Error") - except FatalSSLAlert: - # Close the connection. - return - - linger = False - - def close(self): - """Close the socket underlying this connection.""" - self.rfile.close() - - if not self.linger: - # Python's socket module does NOT call close on the kernel - # socket when you call socket.close(). We do so manually here - # because we want this server to send a FIN TCP segment - # immediately. Note this must be called *before* calling - # socket.close(), because the latter drops its reference to - # the kernel socket. - # Python 3 *probably* fixed this with socket._real_close; - # hard to tell. - # self.socket._sock.close() - self.socket.close() - else: - # On the other hand, sometimes we want to hang around for a bit - # to make sure the client has a chance to read our entire - # response. Skipping the close() calls here delays the FIN - # packet until the socket object is garbage-collected later. - # Someday, perhaps, we'll do the full lingering_close that - # Apache does, but not today. - pass - - -class TrueyZero(object): - - """An object which equals and does math like the integer 0 but evals True. - """ - - def __add__(self, other): - return other - - def __radd__(self, other): - return other -trueyzero = TrueyZero() - - -_SHUTDOWNREQUEST = None - - -class WorkerThread(threading.Thread): - - """Thread which continuously polls a Queue for Connection objects. - - Due to the timing issues of polling a Queue, a WorkerThread does not - check its own 'ready' flag after it has started. To stop the thread, - it is necessary to stick a _SHUTDOWNREQUEST object onto the Queue - (one for each running WorkerThread). - """ - - conn = None - """The current connection pulled off the Queue, or None.""" - - server = None - """The HTTP Server which spawned this thread, and which owns the - Queue and is placing active connections into it.""" - - ready = False - """A simple flag for the calling server to know when this thread - has begun polling the Queue.""" - - def __init__(self, server): - self.ready = False - self.server = server - - self.requests_seen = 0 - self.bytes_read = 0 - self.bytes_written = 0 - self.start_time = None - self.work_time = 0 - self.stats = { - 'Requests': lambda s: self.requests_seen + ( - (self.start_time is None) and - trueyzero or - self.conn.requests_seen - ), - 'Bytes Read': lambda s: self.bytes_read + ( - (self.start_time is None) and - trueyzero or - self.conn.rfile.bytes_read - ), - 'Bytes Written': lambda s: self.bytes_written + ( - (self.start_time is None) and - trueyzero or - self.conn.wfile.bytes_written - ), - 'Work Time': lambda s: self.work_time + ( - (self.start_time is None) and - trueyzero or - time.time() - self.start_time - ), - 'Read Throughput': lambda s: s['Bytes Read'](s) / ( - s['Work Time'](s) or 1e-6), - 'Write Throughput': lambda s: s['Bytes Written'](s) / ( - s['Work Time'](s) or 1e-6), - } - threading.Thread.__init__(self) - - def run(self): - self.server.stats['Worker Threads'][self.getName()] = self.stats - try: - self.ready = True - while True: - conn = self.server.requests.get() - if conn is _SHUTDOWNREQUEST: - return - - self.conn = conn - if self.server.stats['Enabled']: - self.start_time = time.time() - try: - conn.communicate() - finally: - conn.close() - if self.server.stats['Enabled']: - self.requests_seen += self.conn.requests_seen - self.bytes_read += self.conn.rfile.bytes_read - self.bytes_written += self.conn.wfile.bytes_written - self.work_time += time.time() - self.start_time - self.start_time = None - self.conn = None - except (KeyboardInterrupt, SystemExit): - exc = sys.exc_info()[1] - self.server.interrupt = exc - - -class ThreadPool(object): - - """A Request Queue for an HTTPServer which pools threads. - - ThreadPool objects must provide min, get(), put(obj), start() - and stop(timeout) attributes. - """ - - def __init__(self, server, min=10, max=-1, - accepted_queue_size=-1, accepted_queue_timeout=10): - self.server = server - self.min = min - self.max = max - self._threads = [] - self._queue = queue.Queue(maxsize=accepted_queue_size) - self._queue_put_timeout = accepted_queue_timeout - self.get = self._queue.get - - def start(self): - """Start the pool of threads.""" - for i in range(self.min): - self._threads.append(WorkerThread(self.server)) - for worker in self._threads: - worker.setName("CP Server " + worker.getName()) - worker.start() - for worker in self._threads: - while not worker.ready: - time.sleep(.1) - - def _get_idle(self): - """Number of worker threads which are idle. Read-only.""" - return len([t for t in self._threads if t.conn is None]) - idle = property(_get_idle, doc=_get_idle.__doc__) - - def put(self, obj): - self._queue.put(obj, block=True, timeout=self._queue_put_timeout) - if obj is _SHUTDOWNREQUEST: - return - - def grow(self, amount): - """Spawn new worker threads (not above self.max).""" - if self.max > 0: - budget = max(self.max - len(self._threads), 0) - else: - # self.max <= 0 indicates no maximum - budget = float('inf') - - n_new = min(amount, budget) - - workers = [self._spawn_worker() for i in range(n_new)] - while not all(worker.ready for worker in workers): - time.sleep(.1) - self._threads.extend(workers) - - def _spawn_worker(self): - worker = WorkerThread(self.server) - worker.setName("CP Server " + worker.getName()) - worker.start() - return worker - - def shrink(self, amount): - """Kill off worker threads (not below self.min).""" - # Grow/shrink the pool if necessary. - # Remove any dead threads from our list - for t in self._threads: - if not t.isAlive(): - self._threads.remove(t) - amount -= 1 - - # calculate the number of threads above the minimum - n_extra = max(len(self._threads) - self.min, 0) - - # don't remove more than amount - n_to_remove = min(amount, n_extra) - - # put shutdown requests on the queue equal to the number of threads - # to remove. As each request is processed by a worker, that worker - # will terminate and be culled from the list. - for n in range(n_to_remove): - self._queue.put(_SHUTDOWNREQUEST) - - def stop(self, timeout=5): - # Must shut down threads here so the code that calls - # this method can know when all threads are stopped. - for worker in self._threads: - self._queue.put(_SHUTDOWNREQUEST) - - # Don't join currentThread (when stop is called inside a request). - current = threading.currentThread() - if timeout and timeout >= 0: - endtime = time.time() + timeout - while self._threads: - worker = self._threads.pop() - if worker is not current and worker.isAlive(): - try: - if timeout is None or timeout < 0: - worker.join() - else: - remaining_time = endtime - time.time() - if remaining_time > 0: - worker.join(remaining_time) - if worker.isAlive(): - # We exhausted the timeout. - # Forcibly shut down the socket. - c = worker.conn - if c and not c.rfile.closed: - try: - c.socket.shutdown(socket.SHUT_RD) - except TypeError: - # pyOpenSSL sockets don't take an arg - c.socket.shutdown() - worker.join() - except (AssertionError, - # Ignore repeated Ctrl-C. - # See - # https://github.com/cherrypy/cherrypy/issues/691. - KeyboardInterrupt): - pass - - def _get_qsize(self): - return self._queue.qsize() - qsize = property(_get_qsize) - - -try: - import fcntl -except ImportError: - try: - from ctypes import windll, WinError - import ctypes.wintypes - _SetHandleInformation = windll.kernel32.SetHandleInformation - _SetHandleInformation.argtypes = [ - ctypes.wintypes.HANDLE, - ctypes.wintypes.DWORD, - ctypes.wintypes.DWORD, - ] - _SetHandleInformation.restype = ctypes.wintypes.BOOL - except ImportError: - def prevent_socket_inheritance(sock): - """Dummy function, since neither fcntl nor ctypes are available.""" - pass - else: - def prevent_socket_inheritance(sock): - """Mark the given socket fd as non-inheritable (Windows).""" - if not _SetHandleInformation(sock.fileno(), 1, 0): - raise WinError() -else: - def prevent_socket_inheritance(sock): - """Mark the given socket fd as non-inheritable (POSIX).""" - fd = sock.fileno() - old_flags = fcntl.fcntl(fd, fcntl.F_GETFD) - fcntl.fcntl(fd, fcntl.F_SETFD, old_flags | fcntl.FD_CLOEXEC) - - -class SSLAdapter(object): - - """Base class for SSL driver library adapters. - - Required methods: - - * ``wrap(sock) -> (wrapped socket, ssl environ dict)`` - * ``makefile(sock, mode='r', bufsize=DEFAULT_BUFFER_SIZE) -> - socket file object`` - """ - - def __init__(self, certificate, private_key, certificate_chain=None): - self.certificate = certificate - self.private_key = private_key - self.certificate_chain = certificate_chain - - def wrap(self, sock): - raise NotImplemented - - def makefile(self, sock, mode='r', bufsize=DEFAULT_BUFFER_SIZE): - raise NotImplemented - - -class HTTPServer(object): - - """An HTTP server.""" - - _bind_addr = "127.0.0.1" - _interrupt = None - - gateway = None - """A Gateway instance.""" - - minthreads = None - """The minimum number of worker threads to create (default 10).""" - - maxthreads = None - """The maximum number of worker threads to create (default -1 = no limit). - """ - - server_name = None - """The name of the server; defaults to socket.gethostname().""" - - protocol = "HTTP/1.1" - """The version string to write in the Status-Line of all HTTP responses. - - For example, "HTTP/1.1" is the default. This also limits the supported - features used in the response.""" - - request_queue_size = 5 - """The 'backlog' arg to socket.listen(); max queued connections - (default 5). - """ - - shutdown_timeout = 5 - """The total time, in seconds, to wait for worker threads to cleanly exit. - """ - - timeout = 10 - """The timeout in seconds for accepted connections (default 10).""" - - version = "CherryPy/" + cp_version - """A version string for the HTTPServer.""" - - software = None - """The value to set for the SERVER_SOFTWARE entry in the WSGI environ. - - If None, this defaults to ``'%s Server' % self.version``.""" - - ready = False - """An internal flag which marks whether the socket is accepting - connections. - """ - - max_request_header_size = 0 - """The maximum size, in bytes, for request headers, or 0 for no limit.""" - - max_request_body_size = 0 - """The maximum size, in bytes, for request bodies, or 0 for no limit.""" - - nodelay = True - """If True (the default since 3.1), sets the TCP_NODELAY socket option.""" - - ConnectionClass = HTTPConnection - """The class to use for handling HTTP connections.""" - - ssl_adapter = None - """An instance of SSLAdapter (or a subclass). - - You must have the corresponding SSL driver library installed.""" - - def __init__(self, bind_addr, gateway, minthreads=10, maxthreads=-1, - server_name=None): - self.bind_addr = bind_addr - self.gateway = gateway - - self.requests = ThreadPool(self, min=minthreads or 1, max=maxthreads) - - if not server_name: - server_name = socket.gethostname() - self.server_name = server_name - self.clear_stats() - - def clear_stats(self): - self._start_time = None - self._run_time = 0 - self.stats = { - 'Enabled': False, - 'Bind Address': lambda s: repr(self.bind_addr), - 'Run time': lambda s: (not s['Enabled']) and -1 or self.runtime(), - 'Accepts': 0, - 'Accepts/sec': lambda s: s['Accepts'] / self.runtime(), - 'Queue': lambda s: getattr(self.requests, "qsize", None), - 'Threads': lambda s: len(getattr(self.requests, "_threads", [])), - 'Threads Idle': lambda s: getattr(self.requests, "idle", None), - 'Socket Errors': 0, - 'Requests': lambda s: (not s['Enabled']) and -1 or sum( - [w['Requests'](w) for w in s['Worker Threads'].values()], 0), - 'Bytes Read': lambda s: (not s['Enabled']) and -1 or sum( - [w['Bytes Read'](w) for w in s['Worker Threads'].values()], 0), - 'Bytes Written': lambda s: (not s['Enabled']) and -1 or sum( - [w['Bytes Written'](w) for w in s['Worker Threads'].values()], - 0), - 'Work Time': lambda s: (not s['Enabled']) and -1 or sum( - [w['Work Time'](w) for w in s['Worker Threads'].values()], 0), - 'Read Throughput': lambda s: (not s['Enabled']) and -1 or sum( - [w['Bytes Read'](w) / (w['Work Time'](w) or 1e-6) - for w in s['Worker Threads'].values()], 0), - 'Write Throughput': lambda s: (not s['Enabled']) and -1 or sum( - [w['Bytes Written'](w) / (w['Work Time'](w) or 1e-6) - for w in s['Worker Threads'].values()], 0), - 'Worker Threads': {}, - } - logging.statistics["CherryPy HTTPServer %d" % id(self)] = self.stats - - def runtime(self): - if self._start_time is None: - return self._run_time - else: - return self._run_time + (time.time() - self._start_time) - - def __str__(self): - return "%s.%s(%r)" % (self.__module__, self.__class__.__name__, - self.bind_addr) - - def _get_bind_addr(self): - return self._bind_addr - - def _set_bind_addr(self, value): - if isinstance(value, tuple) and value[0] in ('', None): - # Despite the socket module docs, using '' does not - # allow AI_PASSIVE to work. Passing None instead - # returns '0.0.0.0' like we want. In other words: - # host AI_PASSIVE result - # '' Y 192.168.x.y - # '' N 192.168.x.y - # None Y 0.0.0.0 - # None N 127.0.0.1 - # But since you can get the same effect with an explicit - # '0.0.0.0', we deny both the empty string and None as values. - raise ValueError("Host values of '' or None are not allowed. " - "Use '0.0.0.0' (IPv4) or '::' (IPv6) instead " - "to listen on all active interfaces.") - self._bind_addr = value - bind_addr = property( - _get_bind_addr, - _set_bind_addr, - doc="""The interface on which to listen for connections. - - For TCP sockets, a (host, port) tuple. Host values may be any IPv4 - or IPv6 address, or any valid hostname. The string 'localhost' is a - synonym for '127.0.0.1' (or '::1', if your hosts file prefers IPv6). - The string '0.0.0.0' is a special IPv4 entry meaning "any active - interface" (INADDR_ANY), and '::' is the similar IN6ADDR_ANY for - IPv6. The empty string or None are not allowed. - - For UNIX sockets, supply the filename as a string.""") - - def start(self): - """Run the server forever.""" - # We don't have to trap KeyboardInterrupt or SystemExit here, - # because cherrpy.server already does so, calling self.stop() for us. - # If you're using this server with another framework, you should - # trap those exceptions in whatever code block calls start(). - self._interrupt = None - - if self.software is None: - self.software = "%s Server" % self.version - - # Select the appropriate socket - if isinstance(self.bind_addr, basestring): - # AF_UNIX socket - - # So we can reuse the socket... - try: - os.unlink(self.bind_addr) - except: - pass - - # So everyone can access the socket, this was set to 777 - # https://docs.openstack.org/developer/bandit/plugins/set_bad_file_permissions.html - # This plugin test looks for the use of chmod and will alert when - # it is used to set particularly permissive control flags. - # A MEDIUM warning is generated if a file is set to group - # executable and a HIGH warning is reported if a file - # is set world writable. Warnings are given with HIGH confidence. - try: - os.chmod(self.bind_addr, 0o755) - except: - pass - - info = [ - (socket.AF_UNIX, socket.SOCK_STREAM, 0, "", self.bind_addr)] - else: - # AF_INET or AF_INET6 socket - # Get the correct address family for our host (allows IPv6 - # addresses) - host, port = self.bind_addr - try: - info = socket.getaddrinfo(host, port, socket.AF_UNSPEC, - socket.SOCK_STREAM, 0, - socket.AI_PASSIVE) - except socket.gaierror: - if ':' in self.bind_addr[0]: - info = [(socket.AF_INET6, socket.SOCK_STREAM, - 0, "", self.bind_addr + (0, 0))] - else: - info = [(socket.AF_INET, socket.SOCK_STREAM, - 0, "", self.bind_addr)] - - self.socket = None - msg = "No socket could be created" - for res in info: - af, socktype, proto, canonname, sa = res - try: - self.bind(af, socktype, proto) - except socket.error as serr: - msg = "%s -- (%s: %s)" % (msg, sa, serr) - if self.socket: - self.socket.close() - self.socket = None - continue - break - if not self.socket: - raise socket.error(msg) - - # Timeout so KeyboardInterrupt can be caught on Win32 - self.socket.settimeout(1) - self.socket.listen(self.request_queue_size) - - # Create worker threads - self.requests.start() - - self.ready = True - self._start_time = time.time() - while self.ready: - try: - self.tick() - except (KeyboardInterrupt, SystemExit): - raise - except: - self.error_log("Error in HTTPServer.tick", level=logging.ERROR, - traceback=True) - if self.interrupt: - while self.interrupt is True: - # Wait for self.stop() to complete. See _set_interrupt. - time.sleep(0.1) - if self.interrupt: - raise self.interrupt - - def error_log(self, msg="", level=20, traceback=False): - # Override this in subclasses as desired - sys.stderr.write(msg + '\n') - sys.stderr.flush() - if traceback: - tblines = traceback_.format_exc() - sys.stderr.write(tblines) - sys.stderr.flush() - - def bind(self, family, type, proto=0): - """Create (or recreate) the actual socket object.""" - self.socket = socket.socket(family, type, proto) - prevent_socket_inheritance(self.socket) - self.socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) - if self.nodelay and not isinstance(self.bind_addr, str): - self.socket.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1) - - if self.ssl_adapter is not None: - self.socket = self.ssl_adapter.bind(self.socket) - - # If listening on the IPV6 any address ('::' = IN6ADDR_ANY), - # activate dual-stack. See - # https://github.com/cherrypy/cherrypy/issues/871. - if (hasattr(socket, 'AF_INET6') and family == socket.AF_INET6 - and self.bind_addr[0] in ('::', '::0', '::0.0.0.0')): - try: - self.socket.setsockopt( - socket.IPPROTO_IPV6, socket.IPV6_V6ONLY, 0) - except (AttributeError, socket.error): - # Apparently, the socket option is not available in - # this machine's TCP stack - pass - - self.socket.bind(self.bind_addr) - - def tick(self): - """Accept a new connection and put it on the Queue.""" - try: - s, addr = self.socket.accept() - if self.stats['Enabled']: - self.stats['Accepts'] += 1 - if not self.ready: - return - - prevent_socket_inheritance(s) - if hasattr(s, 'settimeout'): - s.settimeout(self.timeout) - - makefile = CP_makefile - ssl_env = {} - # if ssl cert and key are set, we try to be a secure HTTP server - if self.ssl_adapter is not None: - try: - s, ssl_env = self.ssl_adapter.wrap(s) - except NoSSLError: - msg = ("The client sent a plain HTTP request, but " - "this server only speaks HTTPS on this port.") - buf = ["%s 400 Bad Request\r\n" % self.protocol, - "Content-Length: %s\r\n" % len(msg), - "Content-Type: text/plain\r\n\r\n", - msg] - - wfile = makefile(s, "wb", DEFAULT_BUFFER_SIZE) - try: - wfile.write("".join(buf).encode('ISO-8859-1')) - except socket.error: - x = sys.exc_info()[1] - if x.args[0] not in socket_errors_to_ignore: - raise - return - if not s: - return - makefile = self.ssl_adapter.makefile - # Re-apply our timeout since we may have a new socket object - if hasattr(s, 'settimeout'): - s.settimeout(self.timeout) - - conn = self.ConnectionClass(self, s, makefile) - - if not isinstance(self.bind_addr, basestring): - # optional values - # Until we do DNS lookups, omit REMOTE_HOST - if addr is None: # sometimes this can happen - # figure out if AF_INET or AF_INET6. - if len(s.getsockname()) == 2: - # AF_INET - addr = ('0.0.0.0', 0) - else: - # AF_INET6 - addr = ('::', 0) - conn.remote_addr = addr[0] - conn.remote_port = addr[1] - - conn.ssl_env = ssl_env - - try: - self.requests.put(conn) - except queue.Full: - # Just drop the conn. TODO: write 503 back? - conn.close() - return - except socket.timeout: - # The only reason for the timeout in start() is so we can - # notice keyboard interrupts on Win32, which don't interrupt - # accept() by default - return - except socket.error: - x = sys.exc_info()[1] - if self.stats['Enabled']: - self.stats['Socket Errors'] += 1 - if x.args[0] in socket_error_eintr: - # I *think* this is right. EINTR should occur when a signal - # is received during the accept() call; all docs say retry - # the call, and I *think* I'm reading it right that Python - # will then go ahead and poll for and handle the signal - # elsewhere. See - # https://github.com/cherrypy/cherrypy/issues/707. - return - if x.args[0] in socket_errors_nonblocking: - # Just try again. See - # https://github.com/cherrypy/cherrypy/issues/479. - return - if x.args[0] in socket_errors_to_ignore: - # Our socket was closed. - # See https://github.com/cherrypy/cherrypy/issues/686. - return - raise - - def _get_interrupt(self): - return self._interrupt - - def _set_interrupt(self, interrupt): - self._interrupt = True - self.stop() - self._interrupt = interrupt - interrupt = property(_get_interrupt, _set_interrupt, - doc="Set this to an Exception instance to " - "interrupt the server.") - - def stop(self): - """Gracefully shutdown a server that is serving forever.""" - self.ready = False - if self._start_time is not None: - self._run_time += (time.time() - self._start_time) - self._start_time = None - - sock = getattr(self, "socket", None) - if sock: - if not isinstance(self.bind_addr, basestring): - # Touch our own socket to make accept() return immediately. - try: - host, port = sock.getsockname()[:2] - except socket.error: - x = sys.exc_info()[1] - if x.args[0] not in socket_errors_to_ignore: - # Changed to use error code and not message - # See - # https://github.com/cherrypy/cherrypy/issues/860. - raise - else: - # Note that we're explicitly NOT using AI_PASSIVE, - # here, because we want an actual IP to touch. - # localhost won't work if we've bound to a public IP, - # but it will if we bound to '0.0.0.0' (INADDR_ANY). - for res in socket.getaddrinfo(host, port, socket.AF_UNSPEC, - socket.SOCK_STREAM): - af, socktype, proto, canonname, sa = res - s = None - try: - s = socket.socket(af, socktype, proto) - # See - # http://groups.google.com/group/cherrypy-users/ - # browse_frm/thread/bbfe5eb39c904fe0 - s.settimeout(1.0) - s.connect((host, port)) - s.close() - except socket.error: - if s: - s.close() - if hasattr(sock, "close"): - sock.close() - self.socket = None - - self.requests.stop(self.shutdown_timeout) - - -class Gateway(object): - - """A base class to interface HTTPServer with other systems, such as WSGI. - """ - - def __init__(self, req): - self.req = req - - def respond(self): - """Process the current request. Must be overridden in a subclass.""" - raise NotImplemented - - -# These may either be wsgiserver.SSLAdapter subclasses or the string names -# of such classes (in which case they will be lazily loaded). -ssl_adapters = { - 'builtin': 'cherrypy.wsgiserver.ssl_builtin.BuiltinSSLAdapter', - 'pyopenssl': 'cherrypy.wsgiserver.ssl_pyopenssl.pyOpenSSLAdapter', -} - - -def get_ssl_adapter_class(name='builtin'): - """Return an SSL adapter class for the given name.""" - adapter = ssl_adapters[name.lower()] - if isinstance(adapter, basestring): - last_dot = adapter.rfind(".") - attr_name = adapter[last_dot + 1:] - mod_path = adapter[:last_dot] - - try: - mod = sys.modules[mod_path] - if mod is None: - raise KeyError() - except KeyError: - # The last [''] is important. - mod = __import__(mod_path, globals(), locals(), ['']) - - # Let an AttributeError propagate outward. - try: - adapter = getattr(mod, attr_name) - except AttributeError: - raise AttributeError("'%s' object has no attribute '%s'" - % (mod_path, attr_name)) - - return adapter - -# ------------------------------- WSGI Stuff -------------------------------- # - - -class CherryPyWSGIServer(HTTPServer): - - """A subclass of HTTPServer which calls a WSGI application.""" - - wsgi_version = (1, 0) - """The version of WSGI to produce.""" - - def __init__(self, bind_addr, wsgi_app, numthreads=10, server_name=None, - max=-1, request_queue_size=5, timeout=10, shutdown_timeout=5, - accepted_queue_size=-1, accepted_queue_timeout=10): - self.requests = ThreadPool(self, min=numthreads or 1, max=max, - accepted_queue_size=accepted_queue_size, - accepted_queue_timeout=accepted_queue_timeout) - self.wsgi_app = wsgi_app - self.gateway = wsgi_gateways[self.wsgi_version] - - self.bind_addr = bind_addr - if not server_name: - server_name = socket.gethostname() - self.server_name = server_name - self.request_queue_size = request_queue_size - - self.timeout = timeout - self.shutdown_timeout = shutdown_timeout - self.clear_stats() - - def _get_numthreads(self): - return self.requests.min - - def _set_numthreads(self, value): - self.requests.min = value - numthreads = property(_get_numthreads, _set_numthreads) - - -class WSGIGateway(Gateway): - - """A base class to interface HTTPServer with WSGI.""" - - def __init__(self, req): - self.req = req - self.started_response = False - self.env = self.get_environ() - self.remaining_bytes_out = None - - def get_environ(self): - """Return a new environ dict targeting the given wsgi.version""" - raise NotImplemented - - def respond(self): - """Process the current request.""" - response = self.req.server.wsgi_app(self.env, self.start_response) - try: - for chunk in response: - # "The start_response callable must not actually transmit - # the response headers. Instead, it must store them for the - # server or gateway to transmit only after the first - # iteration of the application return value that yields - # a NON-EMPTY string, or upon the application's first - # invocation of the write() callable." (PEP 333) - if chunk: - if isinstance(chunk, unicodestr): - chunk = chunk.encode('ISO-8859-1') - self.write(chunk) - finally: - if hasattr(response, "close"): - response.close() - - def start_response(self, status, headers, exc_info=None): - """WSGI callable to begin the HTTP response.""" - # "The application may call start_response more than once, - # if and only if the exc_info argument is provided." - if self.started_response and not exc_info: - raise AssertionError("WSGI start_response called a second " - "time with no exc_info.") - self.started_response = True - - # "if exc_info is provided, and the HTTP headers have already been - # sent, start_response must raise an error, and should raise the - # exc_info tuple." - if self.req.sent_headers: - try: - raise exc_info[0](exc_info[1]).with_traceback(exc_info[2]) - finally: - exc_info = None - - # According to PEP 3333, when using Python 3, the response status - # and headers must be bytes masquerading as unicode; that is, they - # must be of type "str" but are restricted to code points in the - # "latin-1" set. - if not isinstance(status, str): - raise TypeError("WSGI response status is not of type str.") - self.req.status = status.encode('ISO-8859-1') - - for k, v in headers: - if not isinstance(k, str): - raise TypeError( - "WSGI response header key %r is not of type str." % k) - if not isinstance(v, str): - raise TypeError( - "WSGI response header value %r is not of type str." % v) - if k.lower() == 'content-length': - self.remaining_bytes_out = int(v) - self.req.outheaders.append( - (k.encode('ISO-8859-1'), v.encode('ISO-8859-1'))) - - return self.write - - def write(self, chunk): - """WSGI callable to write unbuffered data to the client. - - This method is also used internally by start_response (to write - data from the iterable returned by the WSGI application). - """ - if not self.started_response: - raise AssertionError("WSGI write called before start_response.") - - chunklen = len(chunk) - rbo = self.remaining_bytes_out - if rbo is not None and chunklen > rbo: - if not self.req.sent_headers: - # Whew. We can send a 500 to the client. - self.req.simple_response("500 Internal Server Error", - "The requested resource returned " - "more bytes than the declared " - "Content-Length.") - else: - # Dang. We have probably already sent data. Truncate the chunk - # to fit (so the client doesn't hang) and raise an error later. - chunk = chunk[:rbo] - - if not self.req.sent_headers: - self.req.sent_headers = True - self.req.send_headers() - - self.req.write(chunk) - - if rbo is not None: - rbo -= chunklen - if rbo < 0: - raise ValueError( - "Response body exceeds the declared Content-Length.") - - -class WSGIGateway_10(WSGIGateway): - - """A Gateway class to interface HTTPServer with WSGI 1.0.x.""" - - def get_environ(self): - """Return a new environ dict targeting the given wsgi.version""" - req = self.req - env = { - # set a non-standard environ entry so the WSGI app can know what - # the *real* server protocol is (and what features to support). - # See http://www.faqs.org/rfcs/rfc2145.html. - 'ACTUAL_SERVER_PROTOCOL': req.server.protocol, - 'PATH_INFO': req.path.decode('ISO-8859-1'), - 'QUERY_STRING': req.qs.decode('ISO-8859-1'), - 'REMOTE_ADDR': req.conn.remote_addr or '', - 'REMOTE_PORT': str(req.conn.remote_port or ''), - 'REQUEST_METHOD': req.method.decode('ISO-8859-1'), - 'REQUEST_URI': req.uri.decode('ISO-8859-1'), - 'SCRIPT_NAME': '', - 'SERVER_NAME': req.server.server_name, - # Bah. "SERVER_PROTOCOL" is actually the REQUEST protocol. - 'SERVER_PROTOCOL': req.request_protocol.decode('ISO-8859-1'), - 'SERVER_SOFTWARE': req.server.software, - 'wsgi.errors': sys.stderr, - 'wsgi.input': req.rfile, - 'wsgi.multiprocess': False, - 'wsgi.multithread': True, - 'wsgi.run_once': False, - 'wsgi.url_scheme': req.scheme.decode('ISO-8859-1'), - 'wsgi.version': (1, 0), - } - if isinstance(req.server.bind_addr, basestring): - # AF_UNIX. This isn't really allowed by WSGI, which doesn't - # address unix domain sockets. But it's better than nothing. - env["SERVER_PORT"] = "" - else: - env["SERVER_PORT"] = str(req.server.bind_addr[1]) - - # Request headers - for k, v in req.inheaders.items(): - k = k.decode('ISO-8859-1').upper().replace("-", "_") - env["HTTP_" + k] = v.decode('ISO-8859-1') - - # CONTENT_TYPE/CONTENT_LENGTH - ct = env.pop("HTTP_CONTENT_TYPE", None) - if ct is not None: - env["CONTENT_TYPE"] = ct - cl = env.pop("HTTP_CONTENT_LENGTH", None) - if cl is not None: - env["CONTENT_LENGTH"] = cl - - if req.conn.ssl_env: - env.update(req.conn.ssl_env) - - return env - - -class WSGIGateway_u0(WSGIGateway_10): - - """A Gateway class to interface HTTPServer with WSGI u.0. - - WSGI u.0 is an experimental protocol, which uses unicode for keys - and values in both Python 2 and Python 3. - """ - - def get_environ(self): - """Return a new environ dict targeting the given wsgi.version""" - req = self.req - env_10 = WSGIGateway_10.get_environ(self) - env = env_10.copy() - env['wsgi.version'] = ('u', 0) - - # Request-URI - env.setdefault('wsgi.url_encoding', 'utf-8') - try: - # SCRIPT_NAME is the empty string, who cares what encoding it is? - env["PATH_INFO"] = req.path.decode(env['wsgi.url_encoding']) - env["QUERY_STRING"] = req.qs.decode(env['wsgi.url_encoding']) - except UnicodeDecodeError: - # Fall back to latin 1 so apps can transcode if needed. - env['wsgi.url_encoding'] = 'ISO-8859-1' - env["PATH_INFO"] = env_10["PATH_INFO"] - env["QUERY_STRING"] = env_10["QUERY_STRING"] - - return env - -wsgi_gateways = { - (1, 0): WSGIGateway_10, - ('u', 0): WSGIGateway_u0, -} - - -class WSGIPathInfoDispatcher(object): - - """A WSGI dispatcher for dispatch based on the PATH_INFO. - - apps: a dict or list of (path_prefix, app) pairs. - """ - - def __init__(self, apps): - try: - apps = list(apps.items()) - except AttributeError: - pass - - # Sort the apps by len(path), descending - apps.sort() - apps.reverse() - - # The path_prefix strings must start, but not end, with a slash. - # Use "" instead of "/". - self.apps = [(p.rstrip("/"), a) for p, a in apps] - - def __call__(self, environ, start_response): - path = environ["PATH_INFO"] or "/" - for p, app in self.apps: - # The apps list should be sorted by length, descending. - if path.startswith(p + "/") or path == p: - environ = environ.copy() - environ["SCRIPT_NAME"] = environ["SCRIPT_NAME"] + p - environ["PATH_INFO"] = path[len(p):] - return app(environ, start_response) - - start_response('404 Not Found', [('Content-Type', 'text/plain'), - ('Content-Length', '0')]) - return [''] From 3fa3780d424cdf7abe9b451390ce7e48e2fabba3 Mon Sep 17 00:00:00 2001 From: Martin Ryan Date: Wed, 2 Jan 2019 13:40:49 +1100 Subject: [PATCH 12/83] address feedback --- lib/cylc/subprocess_safe.py | 16 +++++----------- tests/cylc-cat-state/basic/state-check.py | 4 +--- 2 files changed, 6 insertions(+), 14 deletions(-) diff --git a/lib/cylc/subprocess_safe.py b/lib/cylc/subprocess_safe.py index 5a89c77fa9e..82a5f03ce67 100644 --- a/lib/cylc/subprocess_safe.py +++ b/lib/cylc/subprocess_safe.py @@ -24,9 +24,7 @@ isn't possible. """ -# import sys from subprocess import Popen -# from cylc import LOG # pylint: disable=too-many-arguments # pylint: disable=too-many-locals @@ -37,13 +35,9 @@ def pcylc(cmd, bufsize=0, executable=None, stdin=None, stdout=None, close_fds=False, shell=False, cwd=None, env=None, universal_newlines=False, startupinfo=None, creationflags=0): - # try: - process = Popen(cmd, bufsize, executable, stdin, stdout, # nosec - stderr, preexec_fn, close_fds, shell, cwd, env, - universal_newlines, startupinfo, creationflags) + process = Popen(cmd, bufsize, executable, stdin, # nosec + stdout, stderr, preexec_fn, close_fds, + shell, cwd, env, universal_newlines, + startupinfo, creationflags) + return process - # except OSError as exc: - # LOG.exception(exc) - # sys.exit(str(exc)) - # if process.returncode: - # raise RuntimeError(process.communicate()[1]) diff --git a/tests/cylc-cat-state/basic/state-check.py b/tests/cylc-cat-state/basic/state-check.py index c46c7740df6..872de7256f5 100644 --- a/tests/cylc-cat-state/basic/state-check.py +++ b/tests/cylc-cat-state/basic/state-check.py @@ -31,9 +31,7 @@ def main(argv): sname = argv[0] rundir = argv[1] - p = pcylc("cylc cat-state " + sname, - shell=True, stdout=subprocess.PIPE, - stderr=subprocess.PIPE) + p = pcylc(['cylc', 'cat-state', sname]) state, err = p.communicate() if p.returncode > 0: From e7a37d2a9739cd6b0bb23160b2f7b02153790ecf Mon Sep 17 00:00:00 2001 From: Martin Ryan Date: Wed, 2 Jan 2019 16:48:53 +1100 Subject: [PATCH 13/83] fix state check --- tests/cylc-cat-state/basic/state-check.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/cylc-cat-state/basic/state-check.py b/tests/cylc-cat-state/basic/state-check.py index 872de7256f5..c46c7740df6 100644 --- a/tests/cylc-cat-state/basic/state-check.py +++ b/tests/cylc-cat-state/basic/state-check.py @@ -31,7 +31,9 @@ def main(argv): sname = argv[0] rundir = argv[1] - p = pcylc(['cylc', 'cat-state', sname]) + p = pcylc("cylc cat-state " + sname, + shell=True, stdout=subprocess.PIPE, + stderr=subprocess.PIPE) state, err = p.communicate() if p.returncode > 0: From a3ab49592c38736eb6d00184c89d30e92e7c311d Mon Sep 17 00:00:00 2001 From: Martin Ryan Date: Thu, 3 Jan 2019 15:22:29 +1100 Subject: [PATCH 14/83] remove import --- lib/cylc/tests/test_subprocess_safe.py | 1 - 1 file changed, 1 deletion(-) diff --git a/lib/cylc/tests/test_subprocess_safe.py b/lib/cylc/tests/test_subprocess_safe.py index 9c7f557f794..462f9b1e2f8 100644 --- a/lib/cylc/tests/test_subprocess_safe.py +++ b/lib/cylc/tests/test_subprocess_safe.py @@ -17,7 +17,6 @@ # along with this program. If not, see . import unittest -from pipes import quote from subprocess import PIPE from mock import call From cb8548da6e9bd3eee3a12b457db68c51ea3ccb46 Mon Sep 17 00:00:00 2001 From: Martin Ryan Date: Fri, 4 Jan 2019 16:36:33 +1100 Subject: [PATCH 15/83] change import for subprocess-safe --- lib/cylc/subprocess_safe.py | 2 +- lib/cylc/tests/test_subprocess_safe.py | 6 ++++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/lib/cylc/subprocess_safe.py b/lib/cylc/subprocess_safe.py index 82a5f03ce67..188f889ed51 100644 --- a/lib/cylc/subprocess_safe.py +++ b/lib/cylc/subprocess_safe.py @@ -24,7 +24,7 @@ isn't possible. """ -from subprocess import Popen +from subprocess import Popen # nosec # pylint: disable=too-many-arguments # pylint: disable=too-many-locals diff --git a/lib/cylc/tests/test_subprocess_safe.py b/lib/cylc/tests/test_subprocess_safe.py index 462f9b1e2f8..f360cdcaeb4 100644 --- a/lib/cylc/tests/test_subprocess_safe.py +++ b/lib/cylc/tests/test_subprocess_safe.py @@ -17,11 +17,13 @@ # along with this program. If not, see . import unittest -from subprocess import PIPE from mock import call from testfixtures import compare -from testfixtures.popen import MockPopen +from testfixtures.popen import PIPE, MockPopen + +# Method could be a function +# pylint: disable=no-self-use class TestSubprocessSafe(unittest.TestCase): From e4a5950c914a46fd88c6c36797417b0dbeb43f29 Mon Sep 17 00:00:00 2001 From: Martin Ryan Date: Fri, 4 Jan 2019 16:44:31 +1100 Subject: [PATCH 16/83] autoescape for jinja2 rc files disabled --- lib/parsec/jinja2support.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/parsec/jinja2support.py b/lib/parsec/jinja2support.py index b584448b6cf..6d911b56e9b 100644 --- a/lib/parsec/jinja2support.py +++ b/lib/parsec/jinja2support.py @@ -95,7 +95,7 @@ def jinja2environment(dir_=None): env = Environment( autoescape=select_autoescape( - enabled_extensions=('rc'), + enabled_extensions=(), default_for_string=False, default=True), loader=ChoiceLoader([FileSystemLoader(dir_), PyModuleLoader()]), From c24ad41668661d3d9dd9c9f171542949f40e3667 Mon Sep 17 00:00:00 2001 From: Martin Ryan Date: Fri, 4 Jan 2019 17:03:08 +1100 Subject: [PATCH 17/83] test subprocess_safe using nosec for test file with only static input --- lib/cylc/tests/test_subprocess_safe.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/lib/cylc/tests/test_subprocess_safe.py b/lib/cylc/tests/test_subprocess_safe.py index f360cdcaeb4..4c50cc6b630 100644 --- a/lib/cylc/tests/test_subprocess_safe.py +++ b/lib/cylc/tests/test_subprocess_safe.py @@ -39,7 +39,7 @@ def test_subprocess_safe_communicate_with_input(self): process = Popen(command, stdout=PIPE, stderr=PIPE, shell=True) err, out = process.communicate('foo') compare([ - call.Popen(command, shell=True, stderr=-1, stdout=-1), + call.Popen(command, shell=True, stderr=-1, stdout=-1), # nosec call.Popen_instance.communicate('foo'), ], Popen.mock.method_calls) return err, out @@ -48,22 +48,23 @@ def test_subprocess_safe_read_from_stdout_and_stderr(self): command = "a command" Popen = MockPopen() Popen.set_command(command, stdout=b'foo', stderr=b'bar') - process = Popen(command, stdout=PIPE, stderr=PIPE, shell=True) + process = Popen(command, stdout=PIPE, stderr=PIPE, shell=True) # nosec compare(process.stdout.read(), b'foo') compare(process.stderr.read(), b'bar') compare([ - call.Popen(command, shell=True, stderr=PIPE, stdout=PIPE), + call.Popen(command, shell=True, stderr=PIPE, # nosec + stdout=PIPE), ], Popen.mock.method_calls) def test_subprocess_safe_write_to_stdin(self): command = "a command" Popen = MockPopen() Popen.set_command(command) - process = Popen(command, stdin=PIPE, shell=True) + process = Popen(command, stdin=PIPE, shell=True) # nosec process.stdin.write(command) process.stdin.close() compare([ - call.Popen(command, shell=True, stdin=PIPE), + call.Popen(command, shell=True, stdin=PIPE), # nosec call.Popen_instance.stdin.write(command), call.Popen_instance.stdin.close(), ], Popen.mock.method_calls) From fcd320c94f4fbd2a56553657c9fd73c72ca14c71 Mon Sep 17 00:00:00 2001 From: Martin Ryan Date: Fri, 4 Jan 2019 17:04:06 +1100 Subject: [PATCH 18/83] comment --- lib/cylc/tests/test_subprocess_safe.py | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/cylc/tests/test_subprocess_safe.py b/lib/cylc/tests/test_subprocess_safe.py index 4c50cc6b630..0a836dfefe5 100644 --- a/lib/cylc/tests/test_subprocess_safe.py +++ b/lib/cylc/tests/test_subprocess_safe.py @@ -39,6 +39,7 @@ def test_subprocess_safe_communicate_with_input(self): process = Popen(command, stdout=PIPE, stderr=PIPE, shell=True) err, out = process.communicate('foo') compare([ + # only static input used so using nosec call.Popen(command, shell=True, stderr=-1, stdout=-1), # nosec call.Popen_instance.communicate('foo'), ], Popen.mock.method_calls) From ba66ef660f0f7a73db5411579693e1c01e6d1fb3 Mon Sep 17 00:00:00 2001 From: Martin Ryan Date: Mon, 7 Jan 2019 12:04:20 +1100 Subject: [PATCH 19/83] remove popen variables codacy code analysis searches for popen text. It also thinks mockopen is popen --- lib/cylc/run_get_stdout.py | 14 +++++++------- lib/cylc/tests/test_subprocess_safe.py | 13 +++++++++++-- 2 files changed, 18 insertions(+), 9 deletions(-) diff --git a/lib/cylc/run_get_stdout.py b/lib/cylc/run_get_stdout.py index fffe67c30a4..5fff4f61f57 100644 --- a/lib/cylc/run_get_stdout.py +++ b/lib/cylc/run_get_stdout.py @@ -20,10 +20,10 @@ from os import devnull, killpg, setpgrp from signal import SIGTERM -from subprocess import PIPE +from subprocess import PIPE # nosec from time import sleep, time -from subprocess_safe import pcylc +from subprocess_safe import pcylc # nosec ERR_TIMEOUT = "ERROR: command timed out (>%ds), terminated by signal %d\n%s" @@ -46,7 +46,7 @@ def run_get_stdout(command, timeout=None, poll_delay=None): """ try: - popen = pcylc( + proc = pcylc( command, shell=True, preexec_fn=setpgrp, stdin=open(devnull), stderr=PIPE, stdout=PIPE) is_killed_after_timeout = False @@ -54,14 +54,14 @@ def run_get_stdout(command, timeout=None, poll_delay=None): if poll_delay is None: poll_delay = POLL_DELAY timeout_time = time() + timeout - while popen.poll() is None: + while proc.poll() is None: if time() > timeout_time: - killpg(popen.pid, SIGTERM) + killpg(proc.pid, SIGTERM) is_killed_after_timeout = True break sleep(poll_delay) - out, err = popen.communicate() - res = popen.wait() + out, err = proc.communicate() + res = proc.wait() if res < 0 and is_killed_after_timeout: return (False, [ERR_TIMEOUT % (timeout, -res, err), command]) elif res < 0: diff --git a/lib/cylc/tests/test_subprocess_safe.py b/lib/cylc/tests/test_subprocess_safe.py index 0a836dfefe5..7bd7b62a10c 100644 --- a/lib/cylc/tests/test_subprocess_safe.py +++ b/lib/cylc/tests/test_subprocess_safe.py @@ -36,10 +36,13 @@ def test_subprocess_safe_communicate_with_input(self): command = "a command" Popen = MockPopen() Popen.set_command(command) - process = Popen(command, stdout=PIPE, stderr=PIPE, shell=True) + # only static input used with simulated mockpopen + # codacy mistakenly sees this as a call to popen + process = Popen(command, stdout=PIPE, stderr=PIPE, shell=True) # nosec err, out = process.communicate('foo') compare([ - # only static input used so using nosec + # only static input used with simulated mockpopen + # codacy mistakenly sees this as a call to popen call.Popen(command, shell=True, stderr=-1, stdout=-1), # nosec call.Popen_instance.communicate('foo'), ], Popen.mock.method_calls) @@ -48,6 +51,8 @@ def test_subprocess_safe_communicate_with_input(self): def test_subprocess_safe_read_from_stdout_and_stderr(self): command = "a command" Popen = MockPopen() + # only static input used with simulated mockpopen + # codacy mistakenly sees this as a call to popen Popen.set_command(command, stdout=b'foo', stderr=b'bar') process = Popen(command, stdout=PIPE, stderr=PIPE, shell=True) # nosec compare(process.stdout.read(), b'foo') @@ -61,10 +66,14 @@ def test_subprocess_safe_write_to_stdin(self): command = "a command" Popen = MockPopen() Popen.set_command(command) + # only static input used with simulated mockpopen + # codacy mistakenly sees this as a call to popen process = Popen(command, stdin=PIPE, shell=True) # nosec process.stdin.write(command) process.stdin.close() compare([ + # only static input used with simulated mockpopen + # codacy mistakenly sees this as a call to popen call.Popen(command, shell=True, stdin=PIPE), # nosec call.Popen_instance.stdin.write(command), call.Popen_instance.stdin.close(), From 062bdc00f31d3bed2c745e288cea638a62d971be Mon Sep 17 00:00:00 2001 From: Martin Ryan Date: Mon, 7 Jan 2019 13:17:53 +1100 Subject: [PATCH 20/83] changed comment to require a rebuild --- lib/cylc/tests/test_subprocess_safe.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/cylc/tests/test_subprocess_safe.py b/lib/cylc/tests/test_subprocess_safe.py index 7bd7b62a10c..1b6802d2b27 100644 --- a/lib/cylc/tests/test_subprocess_safe.py +++ b/lib/cylc/tests/test_subprocess_safe.py @@ -72,7 +72,7 @@ def test_subprocess_safe_write_to_stdin(self): process.stdin.write(command) process.stdin.close() compare([ - # only static input used with simulated mockpopen + # static input used with simulated mockpopen # codacy mistakenly sees this as a call to popen call.Popen(command, shell=True, stdin=PIPE), # nosec call.Popen_instance.stdin.write(command), From 21e261ac1f2e35d8daac3bb7d429cd0d085e6a6c Mon Sep 17 00:00:00 2001 From: Martin Ryan Date: Mon, 7 Jan 2019 23:29:07 +1100 Subject: [PATCH 21/83] shebang update change to python2 to prevent bandit ast errors --- bin/cylc-get-host-metrics | 2 +- .../xtrigger/kafka/consumer/lib/python/cylc_kafka_consumer.py | 2 +- .../xtrigger/kafka/producer/bin/cylc_kafka_producer.py | 2 +- lib/cylc/cycling/util.py | 2 +- lib/cylc/review.py | 2 +- lib/cylc/ws.py | 2 +- lib/cylc/xtrigger_mgr.py | 2 +- lib/cylc/xtriggers/echo.py | 2 +- lib/cylc/xtriggers/suite_state.py | 2 +- lib/cylc/xtriggers/wall_clock.py | 2 +- lib/cylc/xtriggers/xrandom.py | 2 +- lib/parsec/empysupport.py | 2 +- tests/xtriggers/02-persistence/faker_fail.py | 3 ++- tests/xtriggers/02-persistence/faker_succ.py | 3 ++- 14 files changed, 16 insertions(+), 14 deletions(-) diff --git a/bin/cylc-get-host-metrics b/bin/cylc-get-host-metrics index e4ebab503e2..4087e243ca2 100755 --- a/bin/cylc-get-host-metrics +++ b/bin/cylc-get-host-metrics @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python2 # THIS FILE IS PART OF THE CYLC SUITE ENGINE. # Copyright (C) 2008-2018 NIWA diff --git a/etc/dev-suites/xtrigger/kafka/consumer/lib/python/cylc_kafka_consumer.py b/etc/dev-suites/xtrigger/kafka/consumer/lib/python/cylc_kafka_consumer.py index d8a3a3df1a8..b326ed8b184 100755 --- a/etc/dev-suites/xtrigger/kafka/consumer/lib/python/cylc_kafka_consumer.py +++ b/etc/dev-suites/xtrigger/kafka/consumer/lib/python/cylc_kafka_consumer.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python2 """ A generic Kakfa consumer for use as a Cylc external trigger function. diff --git a/etc/dev-suites/xtrigger/kafka/producer/bin/cylc_kafka_producer.py b/etc/dev-suites/xtrigger/kafka/producer/bin/cylc_kafka_producer.py index 6d2f7168204..0bf5d99a253 100755 --- a/etc/dev-suites/xtrigger/kafka/producer/bin/cylc_kafka_producer.py +++ b/etc/dev-suites/xtrigger/kafka/producer/bin/cylc_kafka_producer.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python2 """ A generic Kafka producer for use as a Cylc event handler. diff --git a/lib/cylc/cycling/util.py b/lib/cylc/cycling/util.py index ae5dda0b062..29889ffa8a7 100644 --- a/lib/cylc/cycling/util.py +++ b/lib/cylc/cycling/util.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python2 # THIS FILE IS PART OF THE CYLC SUITE ENGINE. # Copyright (C) 2008-2018 NIWA & British Crown (Met Office) & Contributors. diff --git a/lib/cylc/review.py b/lib/cylc/review.py index f3dae0d735a..97be73f815d 100644 --- a/lib/cylc/review.py +++ b/lib/cylc/review.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python2 # THIS FILE IS PART OF THE CYLC SUITE ENGINE. # Copyright (C) 2008-2018 NIWA & British Crown (Met Office) & Contributors. diff --git a/lib/cylc/ws.py b/lib/cylc/ws.py index 5525693f971..eada9e66544 100644 --- a/lib/cylc/ws.py +++ b/lib/cylc/ws.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python2 # THIS FILE IS PART OF THE CYLC SUITE ENGINE. # Copyright (C) 2008-2018 NIWA & British Crown (Met Office) & Contributors. diff --git a/lib/cylc/xtrigger_mgr.py b/lib/cylc/xtrigger_mgr.py index ff108efed4c..3bf0279c61e 100644 --- a/lib/cylc/xtrigger_mgr.py +++ b/lib/cylc/xtrigger_mgr.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python2 # THIS FILE IS PART OF THE CYLC SUITE ENGINE. # Copyright (C) 2008-2018 NIWA & British Crown (Met Office) & Contributors. diff --git a/lib/cylc/xtriggers/echo.py b/lib/cylc/xtriggers/echo.py index 7c449404c4a..4d2da9b452d 100644 --- a/lib/cylc/xtriggers/echo.py +++ b/lib/cylc/xtriggers/echo.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python2 # THIS FILE IS PART OF THE CYLC SUITE ENGINE. # Copyright (C) 2008-2018 NIWA & British Crown (Met Office) & Contributors. diff --git a/lib/cylc/xtriggers/suite_state.py b/lib/cylc/xtriggers/suite_state.py index 001fe653daf..bbb7e1b5339 100644 --- a/lib/cylc/xtriggers/suite_state.py +++ b/lib/cylc/xtriggers/suite_state.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python2 # THIS FILE IS PART OF THE CYLC SUITE ENGINE. # Copyright (C) 2008-2018 NIWA & British Crown (Met Office) & Contributors. diff --git a/lib/cylc/xtriggers/wall_clock.py b/lib/cylc/xtriggers/wall_clock.py index 0176b832493..5a1081dac38 100644 --- a/lib/cylc/xtriggers/wall_clock.py +++ b/lib/cylc/xtriggers/wall_clock.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python2 # THIS FILE IS PART OF THE CYLC SUITE ENGINE. # Copyright (C) 2008-2018 NIWA & British Crown (Met Office) & Contributors. diff --git a/lib/cylc/xtriggers/xrandom.py b/lib/cylc/xtriggers/xrandom.py index ae36c5b2b0a..536bb042109 100644 --- a/lib/cylc/xtriggers/xrandom.py +++ b/lib/cylc/xtriggers/xrandom.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python2 # THIS FILE IS PART OF THE CYLC SUITE ENGINE. # Copyright (C) 2008-2018 NIWA & British Crown (Met Office) & Contributors. diff --git a/lib/parsec/empysupport.py b/lib/parsec/empysupport.py index 97fc8501ad2..4e65421cb20 100644 --- a/lib/parsec/empysupport.py +++ b/lib/parsec/empysupport.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python2 # THIS FILE IS PART OF THE CYLC SUITE ENGINE. # Copyright (C) 2008-2018 NIWA & British Crown (Met Office) & Contributors. diff --git a/tests/xtriggers/02-persistence/faker_fail.py b/tests/xtriggers/02-persistence/faker_fail.py index f456c504262..ad6f519dac2 100644 --- a/tests/xtriggers/02-persistence/faker_fail.py +++ b/tests/xtriggers/02-persistence/faker_fail.py @@ -1,4 +1,5 @@ -#!/usr/bin/env python +#!/usr/bin/env python2 + def faker(name, debug=False): print "%s: failing" % name diff --git a/tests/xtriggers/02-persistence/faker_succ.py b/tests/xtriggers/02-persistence/faker_succ.py index b702d239be7..5d848a06f38 100644 --- a/tests/xtriggers/02-persistence/faker_succ.py +++ b/tests/xtriggers/02-persistence/faker_succ.py @@ -1,4 +1,5 @@ -#!/usr/bin/env python +#!/usr/bin/env python2 + def faker(name, debug=False): print "%s: succeeding" % name From 0ce83687ab142c495a8170a8e3133a3563c404e9 Mon Sep 17 00:00:00 2001 From: Martin Ryan Date: Mon, 7 Jan 2019 23:45:03 +1100 Subject: [PATCH 22/83] bandit remediation --- lib/cherrypy/lib/xmlrpcutil.py | 2 +- lib/cherrypy/wsgiserver/wsgiserver2.py | 9 ++++++++- lib/cylc/gui/app_gcylc.py | 11 ++++++----- lib/cylc/gui/gcapture.py | 5 +++-- lib/cylc/profiling/profile.py | 3 ++- lib/cylc/subprocpool.py | 5 +++-- lib/parsec/jinja2support.py | 2 +- tests/broadcast/00-simple/bin/complog.py | 24 +++++++++++++----------- 8 files changed, 37 insertions(+), 24 deletions(-) diff --git a/lib/cherrypy/lib/xmlrpcutil.py b/lib/cherrypy/lib/xmlrpcutil.py index 80f0af58b88..d13a38db97b 100644 --- a/lib/cherrypy/lib/xmlrpcutil.py +++ b/lib/cherrypy/lib/xmlrpcutil.py @@ -12,7 +12,7 @@ def get_xmlrpclib(): try: import xmlrpc.client as x except ImportError: - import xmlrpclib as x # nosec + import defusedxml.xmlrpclib as x return x diff --git a/lib/cherrypy/wsgiserver/wsgiserver2.py b/lib/cherrypy/wsgiserver/wsgiserver2.py index e6ad8e81a1d..7885c5a9711 100644 --- a/lib/cherrypy/wsgiserver/wsgiserver2.py +++ b/lib/cherrypy/wsgiserver/wsgiserver2.py @@ -1903,7 +1903,14 @@ def start(self): # So everyone can access the socket... try: - os.chmod(self.bind_addr, 0o777) + # this was set to 777 changed to 755 + # https://docs.openstack.org/developer/bandit/plugins/set_bad_file_permissions.html + # This plugin test looks for the use of chmod and will alert + # when it is used to set particularly permissive control flags. + # A MEDIUM warning is generated if a file is set to group + # executable and a HIGH warning is reported if a file is set + # world writable. Warnings are given with HIGH confidence. + os.chmod(self.bind_addr, 0o755) except: pass diff --git a/lib/cylc/gui/app_gcylc.py b/lib/cylc/gui/app_gcylc.py index d7f06f8defa..30dcfdb1500 100644 --- a/lib/cylc/gui/app_gcylc.py +++ b/lib/cylc/gui/app_gcylc.py @@ -34,6 +34,7 @@ from cylc.gui.dbchooser import dbchooser from cylc.gui.combo_logviewer import ComboLogViewer from cylc.gui.warning_dialog import warning_dialog, info_dialog +from cylc.subprocess_safe import pcylc from cylc.task_job_logs import JOB_LOG_OPTS from cylc.wallclock import get_current_time_string @@ -75,12 +76,12 @@ def run_get_stdout(command, filter_=False): try: - popen = Popen( + proc = pcylc( command, shell=True, stdin=open(os.devnull), stderr=PIPE, stdout=PIPE) - out = popen.stdout.read() - err = popen.stderr.read() - res = popen.wait() + out = proc.stdout.read() + err = proc.stderr.read() + res = proc.wait() if res < 0: warning_dialog( "ERROR: command terminated by signal %d\n%s" % (res, err) @@ -1202,7 +1203,7 @@ def startsuite(self, bt, window, coldstart_rb, warmstart_rb, restart_rb, pass # Cannot print to terminal (session may be closed). try: - Popen([command], shell=True, stdin=open(os.devnull)) + pcylc([command], shell=True, stdin=open(os.devnull)) except OSError: warning_dialog('Error: failed to start ' + self.cfg.suite, self.window).warn() diff --git a/lib/cylc/gui/gcapture.py b/lib/cylc/gui/gcapture.py index 15e891cfe87..6731a4f7b98 100644 --- a/lib/cylc/gui/gcapture.py +++ b/lib/cylc/gui/gcapture.py @@ -20,13 +20,14 @@ import gtk import os import pango -from subprocess import Popen, STDOUT +from subprocess import STDOUT import tempfile from cylc.gui.tailer import Tailer from cylc.gui.util import get_icon from cylc.gui.warning_dialog import warning_dialog, info_dialog from cylc.cfgspec.glbl_cfg import glbl_cfg +from cylc.subprocess_safe import pcylc class Gcapture(object): @@ -130,7 +131,7 @@ def __init__(self, command, tmpdir, width=400, height=400, self.window.show() def run(self): - proc = Popen( + proc = pcylc( self.command, stdin=open(os.devnull), stdout=self.stdoutfile, stderr=STDOUT, shell=True) self.proc = proc diff --git a/lib/cylc/profiling/profile.py b/lib/cylc/profiling/profile.py index 4614e516aaa..ac9ba8a8248 100644 --- a/lib/cylc/profiling/profile.py +++ b/lib/cylc/profiling/profile.py @@ -22,6 +22,7 @@ import os import shutil from subprocess import Popen, PIPE, call +from cylc.subprocess_safe import pcylc import sys import tempfile import time @@ -187,7 +188,7 @@ def run_suite(reg, options, out_file, profile_modes, mode='live', # Execute. print '$ ' + ' '.join(cmds) try: - proc = Popen(' '.join(cmds), shell=True, stderr=open(time_err, 'w+'), + proc = pcylc(' '.join(cmds), shell=True, stderr=open(time_err, 'w+'), stdout=open(startup_file, 'w+'), env=env) if proc.wait(): raise SuiteFailedException(run_cmds, cmd_out, cmd_err) diff --git a/lib/cylc/subprocpool.py b/lib/cylc/subprocpool.py index f3540a06f7d..83c0c49c8c8 100644 --- a/lib/cylc/subprocpool.py +++ b/lib/cylc/subprocpool.py @@ -22,7 +22,8 @@ import os import select from signal import SIGKILL -from subprocess import Popen, PIPE +from subprocess import PIPE +from cylc.subprocess_safe import pcylc import sys from tempfile import TemporaryFile from threading import RLock @@ -310,7 +311,7 @@ def _run_command_init(cls, ctx, callback=None, callback_args=None): stdin_file.seek(0) else: stdin_file = open(os.devnull) - proc = Popen( + proc = pcylc( ctx.cmd, stdin=stdin_file, stdout=PIPE, stderr=PIPE, # Execute command as a process group leader, # so we can use "os.killpg" to kill the whole group. diff --git a/lib/parsec/jinja2support.py b/lib/parsec/jinja2support.py index 6d911b56e9b..40c95506b26 100644 --- a/lib/parsec/jinja2support.py +++ b/lib/parsec/jinja2support.py @@ -97,7 +97,7 @@ def jinja2environment(dir_=None): autoescape=select_autoescape( enabled_extensions=(), default_for_string=False, - default=True), + default=False), loader=ChoiceLoader([FileSystemLoader(dir_), PyModuleLoader()]), undefined=StrictUndefined, extensions=['jinja2.ext.do'] diff --git a/tests/broadcast/00-simple/bin/complog.py b/tests/broadcast/00-simple/bin/complog.py index c2abe9c4061..cb7341ed2a8 100755 --- a/tests/broadcast/00-simple/bin/complog.py +++ b/tests/broadcast/00-simple/bin/complog.py @@ -1,25 +1,27 @@ #!/usr/bin/env python2 -import os, sys +import os +import sys +import subprocess print print "This is the broadcast test suite log comparator" event, suite = sys.argv[1], sys.argv[2] if event != 'shutdown': - raise SystemExit( "ERROR: run this as a shutdown event handler") + raise SystemExit("ERROR: run this as a shutdown event handler") try: - log_dir = os.path.expandvars( os.environ['CYLC_SUITE_LOG_DIR'] ) - suite_dir = os.path.expandvars( os.environ['CYLC_SUITE_DEF_PATH'] ) + log_dir = os.path.expandvars(os.environ['CYLC_SUITE_LOG_DIR']) + suite_dir = os.path.expandvars(os.environ['CYLC_SUITE_DEF_PATH']) except KeyError, x: raise SystemExit(x) -ref = os.path.join( suite_dir, 'broadcast.ref' ) -log = os.path.join( suite_dir, 'broadcast.log' ) +ref = os.path.join(suite_dir, 'broadcast.ref') +log = os.path.join(suite_dir, 'broadcast.log') -fref = open( ref, 'r' ) -flog = open( log, 'r' ) +fref = open(ref, 'r') +flog = open(log, 'r') reflines = fref.readlines() loglines = flog.readlines() @@ -28,11 +30,11 @@ loglines.sort() if reflines != loglines: - sys.exit( "ERROR: broadcast logs do not compare" ) + sys.exit("ERROR: broadcast logs do not compare") else: print "broadcast logs compare OK" # call the usual handler too -res = os.system( "cylc check-triggering " + event + " " + suite ) +res = subprocess.call("cylc check-triggering " + event + " " + suite) if res != 0: - sys.exit( 1 ) + sys.exit(1) From 5e11ff9e08846df880a402a6ce17054df0d43757 Mon Sep 17 00:00:00 2001 From: Martin Ryan Date: Mon, 7 Jan 2019 23:55:08 +1100 Subject: [PATCH 23/83] nosec for codacy --- lib/cylc/batch_sys_manager.py | 4 ++-- lib/cylc/run_get_stdout.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/lib/cylc/batch_sys_manager.py b/lib/cylc/batch_sys_manager.py index 4fbe59f2f41..012a8395a2d 100644 --- a/lib/cylc/batch_sys_manager.py +++ b/lib/cylc/batch_sys_manager.py @@ -113,7 +113,7 @@ from shutil import rmtree from signal import SIGKILL import stat -from subprocess import PIPE +from subprocess import PIPE # nosec from cylc.subprocess_safe import pcylc import sys import traceback @@ -663,7 +663,7 @@ def _job_submit_impl( proc = pcylc( batch_sys_cmd, stdin=proc_stdin_arg, stdout=PIPE, stderr=PIPE, - shell=True, env=env) + shell=True, env=env) # nosec else: command = shlex.split( batch_sys.SUBMIT_CMD_TMPL % {"job": job_file_path}) diff --git a/lib/cylc/run_get_stdout.py b/lib/cylc/run_get_stdout.py index 5fff4f61f57..f149b99372c 100644 --- a/lib/cylc/run_get_stdout.py +++ b/lib/cylc/run_get_stdout.py @@ -47,8 +47,8 @@ def run_get_stdout(command, timeout=None, poll_delay=None): """ try: proc = pcylc( - command, shell=True, preexec_fn=setpgrp, stdin=open(devnull), - stderr=PIPE, stdout=PIPE) + command, shell=True, preexec_fn=setpgrp, + stdin=open(devnull), stderr=PIPE, stdout=PIPE) # nosec is_killed_after_timeout = False if timeout: if poll_delay is None: From 1ff5445cc177280ac6a08e97fd874472a0257e5f Mon Sep 17 00:00:00 2001 From: Martin Ryan Date: Tue, 8 Jan 2019 00:12:18 +1100 Subject: [PATCH 24/83] removed trailing whitespace --- lib/cylc/run_get_stdout.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/cylc/run_get_stdout.py b/lib/cylc/run_get_stdout.py index f149b99372c..08ace11859f 100644 --- a/lib/cylc/run_get_stdout.py +++ b/lib/cylc/run_get_stdout.py @@ -47,7 +47,7 @@ def run_get_stdout(command, timeout=None, poll_delay=None): """ try: proc = pcylc( - command, shell=True, preexec_fn=setpgrp, + command, shell=True, preexec_fn=setpgrp, stdin=open(devnull), stderr=PIPE, stdout=PIPE) # nosec is_killed_after_timeout = False if timeout: From f542fff9a621889fa5c395a96c4a94b79865fdc5 Mon Sep 17 00:00:00 2001 From: Martin Ryan Date: Tue, 8 Jan 2019 12:36:48 +1100 Subject: [PATCH 25/83] bandit issues wrap os.system call and change permissive permissions to 755 --- lib/cherrypy/wsgiserver/wsgiserver2.py | 39 +++++++++++++----------- tests/broadcast/00-simple/bin/complog.py | 8 ++--- 2 files changed, 26 insertions(+), 21 deletions(-) diff --git a/lib/cherrypy/wsgiserver/wsgiserver2.py b/lib/cherrypy/wsgiserver/wsgiserver2.py index 7885c5a9711..a9b452fdc16 100644 --- a/lib/cherrypy/wsgiserver/wsgiserver2.py +++ b/lib/cherrypy/wsgiserver/wsgiserver2.py @@ -81,7 +81,7 @@ def my_crazy_app(environ, start_response): import os try: import queue -except: +except: # noqa: E722 import Queue as queue import re import email.utils @@ -93,7 +93,6 @@ def my_crazy_app(environ, start_response): import operator from urllib import unquote from urlparse import urlparse -import warnings import errno import logging try: @@ -131,6 +130,7 @@ class FauxSocket(object): def _reuse(self): pass + _fileobject_uses_str_type = isinstance( socket._fileobject(FauxSocket())._rbuf, basestring) del FauxSocket # this class is not longer required for anything. @@ -185,6 +185,7 @@ def plat_specific_errors(*errnames): # de-dupe the list return list(dict.fromkeys(nums).keys()) + socket_error_eintr = plat_specific_errors("EINTR", "WSAEINTR") socket_errors_to_ignore = plat_specific_errors( @@ -392,7 +393,7 @@ def readlines(self, sizehint=0): def close(self): self.rfile.close() - def __iter__(self): + def __iter__(self): # pylint: disable=E0301 return self def __next__(self): @@ -440,7 +441,7 @@ def _fetch(self): self.closed = True return -## if line: chunk_extension = line[0] +# if line: chunk_extension = line[0] if self.maxlen and self.bytes_read + chunk_size > self.maxlen: raise IOError("Request Entity Too Large") @@ -542,7 +543,7 @@ def read_trailer_lines(self): def close(self): self.rfile.close() - def __iter__(self): + def __iter__(self, sizehint=0): # Shamelessly stolen from StringIO total = 0 line = self.readline(sizehint) @@ -1105,7 +1106,7 @@ def read(self, size=-1): buf.write(data) buf_len += n del data # explicit free - #assert buf_len == buf.tell() + # assert buf_len == buf.tell() return buf.getvalue() def readline(self, size=-1): @@ -1195,7 +1196,7 @@ def readline(self, size=-1): break buf.write(data) buf_len += n - #assert buf_len == buf.tell() + # assert buf_len == buf.tell() return buf.getvalue() else: def read(self, size=-1): @@ -1452,6 +1453,8 @@ def __add__(self, other): def __radd__(self, other): return other + + trueyzero = TrueyZero() @@ -1553,7 +1556,7 @@ class ThreadPool(object): """ def __init__(self, server, min=10, max=-1, - accepted_queue_size=-1, accepted_queue_timeout=10): + accepted_queue_size=-1, accepted_queue_timeout=10): self.server = server self.min = min self.max = max @@ -1720,10 +1723,10 @@ def __init__(self, certificate, private_key, certificate_chain=None): self.certificate_chain = certificate_chain def wrap(self, sock): - raise NotImplemented + raise NotImplementedError def makefile(self, sock, mode='r', bufsize=DEFAULT_BUFFER_SIZE): - raise NotImplemented + raise NotImplementedError class HTTPServer(object): @@ -1898,7 +1901,7 @@ def start(self): # So we can reuse the socket... try: os.unlink(self.bind_addr) - except: + except: # noqa: E722 pass # So everyone can access the socket... @@ -1911,7 +1914,7 @@ def start(self): # executable and a HIGH warning is reported if a file is set # world writable. Warnings are given with HIGH confidence. os.chmod(self.bind_addr, 0o755) - except: + except: # noqa: E722 pass info = [ @@ -1963,7 +1966,7 @@ def start(self): self.tick() except (KeyboardInterrupt, SystemExit): raise - except: + except: # noqa: E722 self.error_log("Error in HTTPServer.tick", level=logging.ERROR, traceback=True) @@ -2170,7 +2173,7 @@ def __init__(self, req): def respond(self): """Process the current request. Must be overridden in a subclass.""" - raise NotImplemented + raise NotImplementedError # These may either be wsgiserver.SSLAdapter subclasses or the string names @@ -2219,7 +2222,8 @@ class CherryPyWSGIServer(HTTPServer): def __init__(self, bind_addr, wsgi_app, numthreads=10, server_name=None, max=-1, request_queue_size=5, timeout=10, shutdown_timeout=5, accepted_queue_size=-1, accepted_queue_timeout=10): - self.requests = ThreadPool(self, min=numthreads or 1, max=max, + self.requests = ThreadPool( + self, min=numthreads or 1, max=max, accepted_queue_size=accepted_queue_size, accepted_queue_timeout=accepted_queue_timeout) self.wsgi_app = wsgi_app @@ -2255,7 +2259,7 @@ def __init__(self, req): def get_environ(self): """Return a new environ dict targeting the given wsgi.version""" - raise NotImplemented + raise NotImplementedError def respond(self): """Process the current request.""" @@ -2411,7 +2415,7 @@ class WSGIGateway_u0(WSGIGateway_10): def get_environ(self): """Return a new environ dict targeting the given wsgi.version""" - req = self.req + # req = self.req env_10 = WSGIGateway_10.get_environ(self) env = dict([(k.decode('ISO-8859-1'), v) for k, v in env_10.iteritems()]) @@ -2434,6 +2438,7 @@ def get_environ(self): return env + wsgi_gateways = { (1, 0): WSGIGateway_10, ('u', 0): WSGIGateway_u0, diff --git a/tests/broadcast/00-simple/bin/complog.py b/tests/broadcast/00-simple/bin/complog.py index cb7341ed2a8..541f562fee2 100755 --- a/tests/broadcast/00-simple/bin/complog.py +++ b/tests/broadcast/00-simple/bin/complog.py @@ -2,7 +2,7 @@ import os import sys -import subprocess +from cylc.subprocess_safe import pcylc print print "This is the broadcast test suite log comparator" @@ -34,7 +34,7 @@ else: print "broadcast logs compare OK" -# call the usual handler too -res = subprocess.call("cylc check-triggering " + event + " " + suite) -if res != 0: +res = pcylc(["cylc check-triggering " + event + " " + suite], shell=True) +status = res.wait() +if status != 0: sys.exit(1) From 51ce3c9eccdcf6a9ebf447dbf4af5ced27fd72b1 Mon Sep 17 00:00:00 2001 From: Martin Ryan Date: Tue, 8 Jan 2019 16:10:19 +1100 Subject: [PATCH 26/83] pcylc - add debug info --- lib/cylc/subprocess_safe.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/lib/cylc/subprocess_safe.py b/lib/cylc/subprocess_safe.py index 188f889ed51..ab4ee5050b2 100644 --- a/lib/cylc/subprocess_safe.py +++ b/lib/cylc/subprocess_safe.py @@ -23,9 +23,11 @@ Cylc inherently requires shell characters so escaping them isn't possible. """ - +from inspect import getframeinfo, stack from subprocess import Popen # nosec +from cylc import LOG + # pylint: disable=too-many-arguments # pylint: disable=too-many-locals @@ -35,6 +37,12 @@ def pcylc(cmd, bufsize=0, executable=None, stdin=None, stdout=None, close_fds=False, shell=False, cwd=None, env=None, universal_newlines=False, startupinfo=None, creationflags=0): + caller = getframeinfo(stack()[1][0]) + LOG.debug("pcylc: calling function: {}".format(caller.function)) + LOG.debug("pcylc: caller: %s:%d" % (caller.filename, caller.lineno)) + LOG.debug("pcylc: command: {}".format(cmd)) + LOG.debug("pcylc: shell == : %r " % shell) + process = Popen(cmd, bufsize, executable, stdin, # nosec stdout, stderr, preexec_fn, close_fds, shell, cwd, env, universal_newlines, From a8fc61694fba8034ab5fcaf3592a9c21dd3154af Mon Sep 17 00:00:00 2001 From: Martin Ryan Date: Tue, 8 Jan 2019 16:38:46 +1100 Subject: [PATCH 27/83] slight change to formatting to trigger rebuild! --- lib/cylc/subprocess_safe.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/lib/cylc/subprocess_safe.py b/lib/cylc/subprocess_safe.py index ab4ee5050b2..0f4da7cf080 100644 --- a/lib/cylc/subprocess_safe.py +++ b/lib/cylc/subprocess_safe.py @@ -43,9 +43,8 @@ def pcylc(cmd, bufsize=0, executable=None, stdin=None, stdout=None, LOG.debug("pcylc: command: {}".format(cmd)) LOG.debug("pcylc: shell == : %r " % shell) - process = Popen(cmd, bufsize, executable, stdin, # nosec - stdout, stderr, preexec_fn, close_fds, - shell, cwd, env, universal_newlines, + process = Popen(cmd, bufsize, executable, stdin, stdout, stderr, # nosec + preexec_fn, close_fds, shell, cwd, env, universal_newlines, startupinfo, creationflags) return process From 5639fd4b8fefc6feb73efbf5356f2f5e00ae9a7c Mon Sep 17 00:00:00 2001 From: Martin Ryan Date: Wed, 9 Jan 2019 11:32:21 +1100 Subject: [PATCH 28/83] comment additions --- lib/cylc/subprocess_safe.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/lib/cylc/subprocess_safe.py b/lib/cylc/subprocess_safe.py index 0f4da7cf080..1355b028d5d 100644 --- a/lib/cylc/subprocess_safe.py +++ b/lib/cylc/subprocess_safe.py @@ -16,12 +16,14 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . -""" Function to sanitize input to a spawning subprocess where shell==True +""" A wrapper function to aggregate these calls in one file, with logging for + what is calling it and the commands given Bandit B602: subprocess_popen_with_shell_equals_true https://docs.openstack.org/developer/bandit/plugins/subprocess_popen_with_shell_equals_true.html + B605: start_process_with_a_shell + https://docs.openstack.org/developer/bandit/plugins/start_process_with_a_shell.html REASON IGNORED: - Cylc inherently requires shell characters so escaping them - isn't possible. + Cylc inherently requires shell characters so escaping them isn't possible. """ from inspect import getframeinfo, stack from subprocess import Popen # nosec From ee06418cc7a1a2965237f6f6d42f1f2bcbd5050d Mon Sep 17 00:00:00 2001 From: Martin Ryan Date: Thu, 10 Jan 2019 10:59:43 +1100 Subject: [PATCH 29/83] comments and logging changes comments to nosec and slight changes to LOG.debug for subprocess_safe.py --- lib/cylc/batch_sys_manager.py | 4 ++++ lib/cylc/run_get_stdout.py | 4 ++++ lib/cylc/subprocess_safe.py | 8 ++++---- 3 files changed, 12 insertions(+), 4 deletions(-) diff --git a/lib/cylc/batch_sys_manager.py b/lib/cylc/batch_sys_manager.py index 012a8395a2d..759b9e0f6f2 100644 --- a/lib/cylc/batch_sys_manager.py +++ b/lib/cylc/batch_sys_manager.py @@ -114,6 +114,8 @@ from signal import SIGKILL import stat from subprocess import PIPE # nosec +# calls to open a shell are aggregated in subprocess_safe.pcylc() +# with logging for what is calling it and the commands given from cylc.subprocess_safe import pcylc import sys import traceback @@ -664,6 +666,8 @@ def _job_submit_impl( batch_sys_cmd, stdin=proc_stdin_arg, stdout=PIPE, stderr=PIPE, shell=True, env=env) # nosec + # calls to open a shell are aggregated in subprocess_safe.pcylc() + # with logging for what is calling it and the commands given else: command = shlex.split( batch_sys.SUBMIT_CMD_TMPL % {"job": job_file_path}) diff --git a/lib/cylc/run_get_stdout.py b/lib/cylc/run_get_stdout.py index 08ace11859f..5850ccd5e08 100644 --- a/lib/cylc/run_get_stdout.py +++ b/lib/cylc/run_get_stdout.py @@ -24,6 +24,8 @@ from time import sleep, time from subprocess_safe import pcylc # nosec +# calls to open a shell are aggregated in subprocess_safe.pcylc() +# with logging for what is calling it and the commands given ERR_TIMEOUT = "ERROR: command timed out (>%ds), terminated by signal %d\n%s" @@ -49,6 +51,8 @@ def run_get_stdout(command, timeout=None, poll_delay=None): proc = pcylc( command, shell=True, preexec_fn=setpgrp, stdin=open(devnull), stderr=PIPE, stdout=PIPE) # nosec + # calls to open a shell are aggregated in subprocess_safe.pcylc() + # with logging for what is calling it and the commands given is_killed_after_timeout = False if timeout: if poll_delay is None: diff --git a/lib/cylc/subprocess_safe.py b/lib/cylc/subprocess_safe.py index 1355b028d5d..f2446dd07df 100644 --- a/lib/cylc/subprocess_safe.py +++ b/lib/cylc/subprocess_safe.py @@ -40,10 +40,10 @@ def pcylc(cmd, bufsize=0, executable=None, stdin=None, stdout=None, universal_newlines=False, startupinfo=None, creationflags=0): caller = getframeinfo(stack()[1][0]) - LOG.debug("pcylc: calling function: {}".format(caller.function)) - LOG.debug("pcylc: caller: %s:%d" % (caller.filename, caller.lineno)) - LOG.debug("pcylc: command: {}".format(cmd)) - LOG.debug("pcylc: shell == : %r " % shell) + LOG.debug('[pcylc: calling function] {}'.format(caller.function)) + LOG.debug('[pcylc: caller] %s:%d' % (caller.filename, caller.lineno)) + LOG.debug('[pcylc: command] {}'.format(cmd)) + LOG.debug('[pcylc: shell] => %r ' % shell) process = Popen(cmd, bufsize, executable, stdin, stdout, stderr, # nosec preexec_fn, close_fds, shell, cwd, env, universal_newlines, From 6fb57a83c8e580cc895043310837c6667d588b54 Mon Sep 17 00:00:00 2001 From: Martin Ryan Date: Thu, 10 Jan 2019 12:24:50 +1100 Subject: [PATCH 30/83] pep8 changes trying to debug pycodestyle error in build --- bin/cylc-get-host-metrics | 13 ++-- .../lib/python/cylc_kafka_consumer.py | 6 +- lib/cherrypy/wsgiserver/wsgiserver2.py | 65 ++++++++++++------- lib/cylc/batch_sys_manager.py | 5 +- 4 files changed, 53 insertions(+), 36 deletions(-) diff --git a/bin/cylc-get-host-metrics b/bin/cylc-get-host-metrics index 4087e243ca2..47b1c19f542 100755 --- a/bin/cylc-get-host-metrics +++ b/bin/cylc-get-host-metrics @@ -35,19 +35,18 @@ keys as requested via the OPTIONS: If no options are specified, --load and --memory are invoked by default. """ -import sys -if '--use-ssh' in sys.argv[1:]: - sys.argv.remove('--use-ssh') - from cylc.remote import remrun - if remrun(): - sys.exit(0) - import os import re from subprocess import Popen, PIPE, CalledProcessError import json from cylc.option_parsers import OptionParser +import sys +if '--use-ssh' in sys.argv[1:]: + sys.argv.remove('--use-ssh') + from cylc.remote import remrun + if remrun(): + sys.exit(0) """Templates for extracting desired metric data via regular expression. diff --git a/etc/dev-suites/xtrigger/kafka/consumer/lib/python/cylc_kafka_consumer.py b/etc/dev-suites/xtrigger/kafka/consumer/lib/python/cylc_kafka_consumer.py index b326ed8b184..e56f273d95a 100755 --- a/etc/dev-suites/xtrigger/kafka/consumer/lib/python/cylc_kafka_consumer.py +++ b/etc/dev-suites/xtrigger/kafka/consumer/lib/python/cylc_kafka_consumer.py @@ -89,9 +89,9 @@ def cylc_kafka_consumer(kafka_server, kafka_topic, group_id, message, debug): should be represented as: "system:prod point:2025 data:" - A match occurs Kafka if all message dict items match, and the result returned - is the sub-dict of the actual values of items containing - angle-bracket-delineated regex patterns. E.g. above {'data': 'nwp-2025.nc'}. + A match occurs Kafka if all message dict items match, and the result + returned is the sub-dict of the actual values of items containing + angle-bracket-delineated regex patterns. E.g. above {'data': 'nwp-2025.nc'} """ diff --git a/lib/cherrypy/wsgiserver/wsgiserver2.py b/lib/cherrypy/wsgiserver/wsgiserver2.py index a9b452fdc16..632911e0303 100644 --- a/lib/cherrypy/wsgiserver/wsgiserver2.py +++ b/lib/cherrypy/wsgiserver/wsgiserver2.py @@ -1,3 +1,37 @@ +import os +try: + import queue +except: # noqa: E722 + import Queue as queue +import re +import email.utils +import socket +import sys +import threading +import time +import traceback as traceback_ +import operator +from urllib import unquote +from urlparse import urlparse +import errno +import logging +import os +try: + import queue +except: # noqa: E722 + import Queue as queue +import re +import email.utils +import socket +import sys +import threading +import time +import traceback as traceback_ +import operator +from urllib import unquote +from urlparse import urlparse +import errno +import logging """A high-speed, production ready, thread pooled, generic HTTP server. Simplest example on how to use this module directly @@ -78,23 +112,6 @@ def my_crazy_app(environ, start_response): 'WSGIPathInfoDispatcher', 'get_ssl_adapter_class', 'socket_errors_to_ignore'] -import os -try: - import queue -except: # noqa: E722 - import Queue as queue -import re -import email.utils -import socket -import sys -import threading -import time -import traceback as traceback_ -import operator -from urllib import unquote -from urlparse import urlparse -import errno -import logging try: # prefer slower Python-based io module import _pyio as io @@ -941,8 +958,8 @@ def send_headers(self): if status < 200 or status in (204, 205, 304): pass else: - if (self.response_protocol == 'HTTP/1.1' - and self.method != 'HEAD'): + if (self.response_protocol == 'HTTP/1.1' and + self.method != 'HEAD'): # Use the chunked transfer-coding self.chunked_write = True self.outheaders.append(("Transfer-Encoding", "chunked")) @@ -1039,8 +1056,8 @@ def recv(self, size): self.bytes_read += len(data) return data except socket.error, e: - if (e.args[0] not in socket_errors_nonblocking - and e.args[0] not in socket_error_eintr): + if (e.args[0] not in socket_errors_nonblocking and e.args[0] + not in socket_error_eintr): raise if not _fileobject_uses_str_type: @@ -1914,7 +1931,7 @@ def start(self): # executable and a HIGH warning is reported if a file is set # world writable. Warnings are given with HIGH confidence. os.chmod(self.bind_addr, 0o755) - except: # noqa: E722 + except: # noqa: E722 pass info = [ @@ -2000,8 +2017,8 @@ def bind(self, family, type, proto=0): # If listening on the IPV6 any address ('::' = IN6ADDR_ANY), # activate dual-stack. See # https://github.com/cherrypy/cherrypy/issues/871. - if (hasattr(socket, 'AF_INET6') and family == socket.AF_INET6 - and self.bind_addr[0] in ('::', '::0', '::0.0.0.0')): + if (hasattr(socket, 'AF_INET6') and family == socket.AF_INET6 and + self.bind_addr[0] in ('::', '::0', '::0.0.0.0')): try: self.socket.setsockopt( socket.IPPROTO_IPV6, socket.IPV6_V6ONLY, 0) diff --git a/lib/cylc/batch_sys_manager.py b/lib/cylc/batch_sys_manager.py index 759b9e0f6f2..6dcda0ed4ab 100644 --- a/lib/cylc/batch_sys_manager.py +++ b/lib/cylc/batch_sys_manager.py @@ -666,8 +666,9 @@ def _job_submit_impl( batch_sys_cmd, stdin=proc_stdin_arg, stdout=PIPE, stderr=PIPE, shell=True, env=env) # nosec - # calls to open a shell are aggregated in subprocess_safe.pcylc() - # with logging for what is calling it and the commands given + # calls to open a shell are aggregated in + # subprocess_safe.pcylc() with logging for what is calling it + # and the commands given else: command = shlex.split( batch_sys.SUBMIT_CMD_TMPL % {"job": job_file_path}) From 84a9e5d263d0776b550a244cc44efcee3a0e7b60 Mon Sep 17 00:00:00 2001 From: Martin Ryan Date: Thu, 10 Jan 2019 12:38:24 +1100 Subject: [PATCH 31/83] pep8 changes --- tests/cylc-cat-state/basic/state-check.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/tests/cylc-cat-state/basic/state-check.py b/tests/cylc-cat-state/basic/state-check.py index c46c7740df6..827ecacef2e 100644 --- a/tests/cylc-cat-state/basic/state-check.py +++ b/tests/cylc-cat-state/basic/state-check.py @@ -1,6 +1,6 @@ # THIS FILE IS PART OF THE CYLC SUITE ENGINE. # Copyright (C) 2008-2018 NIWA & British Crown (Met Office) & Contributors. -# +# # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or @@ -13,7 +13,7 @@ # # You should have received a copy of the GNU General Public License # along with this program. If not, see . -#------------------------------------------------------------------------------- +# ----------------------------------------------------------------------------- import os import sqlite3 @@ -30,7 +30,7 @@ def main(argv): sname = argv[0] rundir = argv[1] - + p = pcylc("cylc cat-state " + sname, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) @@ -68,14 +68,16 @@ def main(argv): sys.stderr.write("unable to query suite database\n") sys.exit(1) if not res[0][0] == status: - error_states.append(line + ": state retrieved " + str(res[0][0])) + error_states.append( + line + ": state retrieved " + str(res[0][0])) elif line == "Begin task states": states_begun = True cnx.close() if error_states: - print >> sys.stderr, "The following task states were not consistent with the database:" + st = "The following task states were not consistent with the database:" + print >> sys.stderr, st for line in error_states: print >> sys.stderr, line sys.exit(1) @@ -84,4 +86,4 @@ def main(argv): if __name__ == "__main__": - main(sys.argv[1:]) + main(sys.argv[1:]) From 64f0f2fc9d7cd4dab7152e86c96178e4d17b707b Mon Sep 17 00:00:00 2001 From: Martin Ryan Date: Thu, 10 Jan 2019 17:09:04 +1100 Subject: [PATCH 32/83] pcylc comments comments and pep8 fix --- lib/cherrypy/wsgiserver/wsgiserver2.py | 18 +----------------- lib/cylc/gui/app_gcylc.py | 14 ++++++++++---- lib/cylc/gui/gcapture.py | 8 ++++++-- lib/cylc/profiling/profile.py | 6 +++++- lib/cylc/run_get_stdout.py | 4 ++-- lib/cylc/subprocess_safe.py | 2 -- lib/cylc/subprocpool.py | 8 ++++++-- 7 files changed, 30 insertions(+), 30 deletions(-) diff --git a/lib/cherrypy/wsgiserver/wsgiserver2.py b/lib/cherrypy/wsgiserver/wsgiserver2.py index 632911e0303..d5747805d5e 100644 --- a/lib/cherrypy/wsgiserver/wsgiserver2.py +++ b/lib/cherrypy/wsgiserver/wsgiserver2.py @@ -15,23 +15,7 @@ from urlparse import urlparse import errno import logging -import os -try: - import queue -except: # noqa: E722 - import Queue as queue -import re -import email.utils -import socket -import sys -import threading -import time -import traceback as traceback_ -import operator -from urllib import unquote -from urlparse import urlparse -import errno -import logging + """A high-speed, production ready, thread pooled, generic HTTP server. Simplest example on how to use this module directly diff --git a/lib/cylc/gui/app_gcylc.py b/lib/cylc/gui/app_gcylc.py index 30dcfdb1500..95158b5db1e 100644 --- a/lib/cylc/gui/app_gcylc.py +++ b/lib/cylc/gui/app_gcylc.py @@ -25,7 +25,9 @@ import pango import gobject import shlex -from subprocess import Popen, PIPE, STDOUT +from subprocess import Popen, PIPE, STDOUT # nosec +# calls to open a shell are aggregated in subprocess_safe.pcylc() +# with logging for what is calling it and the commands given from uuid import uuid4 from isodatetime.parsers import TimePointParser @@ -77,8 +79,10 @@ def run_get_stdout(command, filter_=False): try: proc = pcylc( - command, - shell=True, stdin=open(os.devnull), stderr=PIPE, stdout=PIPE) + command, shell=True, stdout=PIPE, stderr=PIPE, # nosec + stdin=open(os.devnull)) + # calls to open a shell are aggregated in subprocess_safe.pcylc() + # with logging for what is calling it and the commands given out = proc.stdout.read() err = proc.stderr.read() res = proc.wait() @@ -1203,7 +1207,9 @@ def startsuite(self, bt, window, coldstart_rb, warmstart_rb, restart_rb, pass # Cannot print to terminal (session may be closed). try: - pcylc([command], shell=True, stdin=open(os.devnull)) + pcylc([command], shell=True, stdin=open(os.devnull)) # nosec + # calls to open a shell are aggregated in subprocess_safe.pcylc() + # with logging for what is calling it and the commands given except OSError: warning_dialog('Error: failed to start ' + self.cfg.suite, self.window).warn() diff --git a/lib/cylc/gui/gcapture.py b/lib/cylc/gui/gcapture.py index 6731a4f7b98..7e239da3f9b 100644 --- a/lib/cylc/gui/gcapture.py +++ b/lib/cylc/gui/gcapture.py @@ -20,7 +20,9 @@ import gtk import os import pango -from subprocess import STDOUT +from subprocess import STDOUT # nosec +# calls to open a shell are aggregated in subprocess_safe.pcylc() +# with logging for what is calling it and the commands given import tempfile from cylc.gui.tailer import Tailer @@ -133,7 +135,9 @@ def __init__(self, command, tmpdir, width=400, height=400, def run(self): proc = pcylc( self.command, stdin=open(os.devnull), stdout=self.stdoutfile, - stderr=STDOUT, shell=True) + stderr=STDOUT, shell=True) # nosec + # calls to open a shell are aggregated in subprocess_safe.pcylc() + # with logging for what is calling it and the commands given self.proc = proc gobject.timeout_add(40, self.pulse_proc_progress) tail_cmd_tmpl = glbl_cfg().get_host_item("tail command template") diff --git a/lib/cylc/profiling/profile.py b/lib/cylc/profiling/profile.py index ac9ba8a8248..494ec17f3e3 100644 --- a/lib/cylc/profiling/profile.py +++ b/lib/cylc/profiling/profile.py @@ -188,7 +188,11 @@ def run_suite(reg, options, out_file, profile_modes, mode='live', # Execute. print '$ ' + ' '.join(cmds) try: - proc = pcylc(' '.join(cmds), shell=True, stderr=open(time_err, 'w+'), + proc = pcylc(' '.join(cmds), shell=True, + stderr=open(time_err, 'w+'), # nosec + # calls to open a shell are aggregated in + # subprocess_safe.pcylc() with logging for + # what is calling it and the commands given stdout=open(startup_file, 'w+'), env=env) if proc.wait(): raise SuiteFailedException(run_cmds, cmd_out, cmd_err) diff --git a/lib/cylc/run_get_stdout.py b/lib/cylc/run_get_stdout.py index 5850ccd5e08..09d0e059783 100644 --- a/lib/cylc/run_get_stdout.py +++ b/lib/cylc/run_get_stdout.py @@ -49,8 +49,8 @@ def run_get_stdout(command, timeout=None, poll_delay=None): """ try: proc = pcylc( - command, shell=True, preexec_fn=setpgrp, - stdin=open(devnull), stderr=PIPE, stdout=PIPE) # nosec + command, shell=True, preexec_fn=setpgrp, # nosec + stdin=open(devnull), stderr=PIPE, stdout=PIPE) # calls to open a shell are aggregated in subprocess_safe.pcylc() # with logging for what is calling it and the commands given is_killed_after_timeout = False diff --git a/lib/cylc/subprocess_safe.py b/lib/cylc/subprocess_safe.py index f2446dd07df..36a94dd9ea9 100644 --- a/lib/cylc/subprocess_safe.py +++ b/lib/cylc/subprocess_safe.py @@ -22,8 +22,6 @@ https://docs.openstack.org/developer/bandit/plugins/subprocess_popen_with_shell_equals_true.html B605: start_process_with_a_shell https://docs.openstack.org/developer/bandit/plugins/start_process_with_a_shell.html - REASON IGNORED: - Cylc inherently requires shell characters so escaping them isn't possible. """ from inspect import getframeinfo, stack from subprocess import Popen # nosec diff --git a/lib/cylc/subprocpool.py b/lib/cylc/subprocpool.py index 83c0c49c8c8..3c616e3b2e5 100644 --- a/lib/cylc/subprocpool.py +++ b/lib/cylc/subprocpool.py @@ -22,7 +22,9 @@ import os import select from signal import SIGKILL -from subprocess import PIPE +from subprocess import PIPE # nosec +# calls to open a shell are aggregated in subprocess_safe.pcylc() +# with logging for what is calling it and the commands given from cylc.subprocess_safe import pcylc import sys from tempfile import TemporaryFile @@ -317,7 +319,9 @@ def _run_command_init(cls, ctx, callback=None, callback_args=None): # so we can use "os.killpg" to kill the whole group. preexec_fn=os.setpgrp, env=ctx.cmd_kwargs.get('env'), - shell=ctx.cmd_kwargs.get('shell')) + shell=ctx.cmd_kwargs.get('shell')) # nosec + # calls to open a shell are aggregated in subprocess_safe.pcylc() + # with logging for what is calling it and the commands given except (IOError, OSError) as exc: if exc.filename is None: exc.filename = ctx.cmd[0] From 20b8a664120bec3e4a8463f6c0875c85a3e6b274 Mon Sep 17 00:00:00 2001 From: Martin Ryan Date: Fri, 11 Jan 2019 10:01:15 +1100 Subject: [PATCH 33/83] moved nosec --- lib/cylc/profiling/profile.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/cylc/profiling/profile.py b/lib/cylc/profiling/profile.py index 494ec17f3e3..036186b0783 100644 --- a/lib/cylc/profiling/profile.py +++ b/lib/cylc/profiling/profile.py @@ -188,8 +188,8 @@ def run_suite(reg, options, out_file, profile_modes, mode='live', # Execute. print '$ ' + ' '.join(cmds) try: - proc = pcylc(' '.join(cmds), shell=True, - stderr=open(time_err, 'w+'), # nosec + proc = pcylc(' '.join(cmds), shell=True, # nosec + stderr=open(time_err, 'w+'), # calls to open a shell are aggregated in # subprocess_safe.pcylc() with logging for # what is calling it and the commands given From c2977b289239069493a7c76cb7c6523f33108ed9 Mon Sep 17 00:00:00 2001 From: Martin Ryan Date: Fri, 11 Jan 2019 16:20:23 +1100 Subject: [PATCH 34/83] updated test and revert logging in pcylc test 3 comparison in 16-timeout.t altered, LOG.debug in calls to pcylc wrapper function also break this comparison, so logging removed for now --- lib/cylc/subprocess_safe.py | 15 +++------------ tests/job-submission/16-timeout.t | 4 +++- 2 files changed, 6 insertions(+), 13 deletions(-) diff --git a/lib/cylc/subprocess_safe.py b/lib/cylc/subprocess_safe.py index 36a94dd9ea9..390cf511574 100644 --- a/lib/cylc/subprocess_safe.py +++ b/lib/cylc/subprocess_safe.py @@ -16,8 +16,7 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . -""" A wrapper function to aggregate these calls in one file, with logging for - what is calling it and the commands given +""" A wrapper function to aggregate these calls in one file Bandit B602: subprocess_popen_with_shell_equals_true https://docs.openstack.org/developer/bandit/plugins/subprocess_popen_with_shell_equals_true.html B605: start_process_with_a_shell @@ -26,8 +25,6 @@ from inspect import getframeinfo, stack from subprocess import Popen # nosec -from cylc import LOG - # pylint: disable=too-many-arguments # pylint: disable=too-many-locals @@ -37,14 +34,8 @@ def pcylc(cmd, bufsize=0, executable=None, stdin=None, stdout=None, close_fds=False, shell=False, cwd=None, env=None, universal_newlines=False, startupinfo=None, creationflags=0): - caller = getframeinfo(stack()[1][0]) - LOG.debug('[pcylc: calling function] {}'.format(caller.function)) - LOG.debug('[pcylc: caller] %s:%d' % (caller.filename, caller.lineno)) - LOG.debug('[pcylc: command] {}'.format(cmd)) - LOG.debug('[pcylc: shell] => %r ' % shell) - process = Popen(cmd, bufsize, executable, stdin, stdout, stderr, # nosec - preexec_fn, close_fds, shell, cwd, env, universal_newlines, - startupinfo, creationflags) + preexec_fn, close_fds, shell, cwd, env, + universal_newlines, startupinfo, creationflags) return process diff --git a/tests/job-submission/16-timeout.t b/tests/job-submission/16-timeout.t index addce00a1e6..6063e4bf31b 100755 --- a/tests/job-submission/16-timeout.t +++ b/tests/job-submission/16-timeout.t @@ -32,8 +32,10 @@ run_ok "${TEST_NAME_BASE}-validate" cylc validate "${SUITE_NAME}" suite_run_ok "${TEST_NAME_BASE}-suite-run" \ cylc run --debug --no-detach "${SUITE_NAME}" +# egrep -m is stop matching after matches +# -A is number of lines of context after match cylc cat-log "${SUITE_NAME}" \ - | egrep -m 1 -A 3 "ERROR - \[jobs-submit cmd\]" \ + | egrep -m 1 -A 2 "ERROR - \[jobs-submit cmd\]" \ | sed -e 's/^.* \(ERROR\)/\1/' > log SUITE_LOG_DIR=$(cylc cat-log -m p "${SUITE_NAME}") From 5717399562a7789d866c82097c1643a18934b293 Mon Sep 17 00:00:00 2001 From: Martin Ryan Date: Fri, 11 Jan 2019 16:20:23 +1100 Subject: [PATCH 35/83] Revert "updated test and revert logging in pcylc" This reverts commit c2977b289239069493a7c76cb7c6523f33108ed9. --- lib/cylc/subprocess_safe.py | 15 ++++++++++++--- tests/job-submission/16-timeout.t | 4 +--- 2 files changed, 13 insertions(+), 6 deletions(-) diff --git a/lib/cylc/subprocess_safe.py b/lib/cylc/subprocess_safe.py index 390cf511574..36a94dd9ea9 100644 --- a/lib/cylc/subprocess_safe.py +++ b/lib/cylc/subprocess_safe.py @@ -16,7 +16,8 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . -""" A wrapper function to aggregate these calls in one file +""" A wrapper function to aggregate these calls in one file, with logging for + what is calling it and the commands given Bandit B602: subprocess_popen_with_shell_equals_true https://docs.openstack.org/developer/bandit/plugins/subprocess_popen_with_shell_equals_true.html B605: start_process_with_a_shell @@ -25,6 +26,8 @@ from inspect import getframeinfo, stack from subprocess import Popen # nosec +from cylc import LOG + # pylint: disable=too-many-arguments # pylint: disable=too-many-locals @@ -34,8 +37,14 @@ def pcylc(cmd, bufsize=0, executable=None, stdin=None, stdout=None, close_fds=False, shell=False, cwd=None, env=None, universal_newlines=False, startupinfo=None, creationflags=0): + caller = getframeinfo(stack()[1][0]) + LOG.debug('[pcylc: calling function] {}'.format(caller.function)) + LOG.debug('[pcylc: caller] %s:%d' % (caller.filename, caller.lineno)) + LOG.debug('[pcylc: command] {}'.format(cmd)) + LOG.debug('[pcylc: shell] => %r ' % shell) + process = Popen(cmd, bufsize, executable, stdin, stdout, stderr, # nosec - preexec_fn, close_fds, shell, cwd, env, - universal_newlines, startupinfo, creationflags) + preexec_fn, close_fds, shell, cwd, env, universal_newlines, + startupinfo, creationflags) return process diff --git a/tests/job-submission/16-timeout.t b/tests/job-submission/16-timeout.t index 6063e4bf31b..addce00a1e6 100755 --- a/tests/job-submission/16-timeout.t +++ b/tests/job-submission/16-timeout.t @@ -32,10 +32,8 @@ run_ok "${TEST_NAME_BASE}-validate" cylc validate "${SUITE_NAME}" suite_run_ok "${TEST_NAME_BASE}-suite-run" \ cylc run --debug --no-detach "${SUITE_NAME}" -# egrep -m is stop matching after matches -# -A is number of lines of context after match cylc cat-log "${SUITE_NAME}" \ - | egrep -m 1 -A 2 "ERROR - \[jobs-submit cmd\]" \ + | egrep -m 1 -A 3 "ERROR - \[jobs-submit cmd\]" \ | sed -e 's/^.* \(ERROR\)/\1/' > log SUITE_LOG_DIR=$(cylc cat-log -m p "${SUITE_NAME}") From d370056aa8b6476858642200e4563762ad855707 Mon Sep 17 00:00:00 2001 From: Martin Ryan Date: Fri, 11 Jan 2019 16:40:26 +1100 Subject: [PATCH 36/83] updated 16-timeout test, revert pcylc debug.log test 3 comparison in 16-timeout.t altered. LOG.debug in calls to pcylc were also breaking comparison - reverted logging for now. --- lib/cylc/subprocess_safe.py | 9 --------- tests/job-submission/16-timeout.t | 5 ++++- 2 files changed, 4 insertions(+), 10 deletions(-) diff --git a/lib/cylc/subprocess_safe.py b/lib/cylc/subprocess_safe.py index 36a94dd9ea9..939b3c8b260 100644 --- a/lib/cylc/subprocess_safe.py +++ b/lib/cylc/subprocess_safe.py @@ -23,11 +23,8 @@ B605: start_process_with_a_shell https://docs.openstack.org/developer/bandit/plugins/start_process_with_a_shell.html """ -from inspect import getframeinfo, stack from subprocess import Popen # nosec -from cylc import LOG - # pylint: disable=too-many-arguments # pylint: disable=too-many-locals @@ -37,12 +34,6 @@ def pcylc(cmd, bufsize=0, executable=None, stdin=None, stdout=None, close_fds=False, shell=False, cwd=None, env=None, universal_newlines=False, startupinfo=None, creationflags=0): - caller = getframeinfo(stack()[1][0]) - LOG.debug('[pcylc: calling function] {}'.format(caller.function)) - LOG.debug('[pcylc: caller] %s:%d' % (caller.filename, caller.lineno)) - LOG.debug('[pcylc: command] {}'.format(cmd)) - LOG.debug('[pcylc: shell] => %r ' % shell) - process = Popen(cmd, bufsize, executable, stdin, stdout, stderr, # nosec preexec_fn, close_fds, shell, cwd, env, universal_newlines, startupinfo, creationflags) diff --git a/tests/job-submission/16-timeout.t b/tests/job-submission/16-timeout.t index addce00a1e6..8fe52cf48bd 100755 --- a/tests/job-submission/16-timeout.t +++ b/tests/job-submission/16-timeout.t @@ -32,8 +32,10 @@ run_ok "${TEST_NAME_BASE}-validate" cylc validate "${SUITE_NAME}" suite_run_ok "${TEST_NAME_BASE}-suite-run" \ cylc run --debug --no-detach "${SUITE_NAME}" +# egrep -m is stop matching after matches +# -A is number of lines of context after match cylc cat-log "${SUITE_NAME}" \ - | egrep -m 1 -A 3 "ERROR - \[jobs-submit cmd\]" \ + | egrep -m 1 -A 2 "ERROR - \[jobs-submit cmd\]" \ | sed -e 's/^.* \(ERROR\)/\1/' > log SUITE_LOG_DIR=$(cylc cat-log -m p "${SUITE_NAME}") @@ -52,3 +54,4 @@ foo, 1, submit-failed __END__ purge_suite "${SUITE_NAME}" + From 2dfa2d0f7589860aab9a991f85ab61eae98efba4 Mon Sep 17 00:00:00 2001 From: Martin Ryan Date: Mon, 14 Jan 2019 17:15:42 +1100 Subject: [PATCH 37/83] wip: useshell variable, codacy --- bin/cylc-check-software | 21 +++++++++-------- bin/cylc-check-versions | 10 ++++---- lib/cylc/batch_sys_manager.py | 24 ++++++++----------- lib/cylc/gui/app_gcylc.py | 18 ++++++--------- lib/cylc/gui/gcapture.py | 7 +++--- lib/cylc/profiling/profile.py | 19 +++++++-------- lib/cylc/run_get_stdout.py | 5 ++-- lib/cylc/subprocess_safe.py | 28 ++++++++++++++++++----- lib/cylc/subprocpool.py | 3 +-- tests/broadcast/00-simple/bin/complog.py | 6 ++++- tests/cylc-cat-state/basic/state-check.py | 7 +++--- 11 files changed, 80 insertions(+), 68 deletions(-) diff --git a/bin/cylc-check-software b/bin/cylc-check-software index 2e2ea3b2064..30601fe7966 100755 --- a/bin/cylc-check-software +++ b/bin/cylc-check-software @@ -35,7 +35,8 @@ Arguments: import sys import re import os -from subprocess import check_call, PIPE, Popen, CalledProcessError +from subprocess import check_call, PIPE, CalledProcessError +from cylc.subprocess_safe import pcylc # Standardised output messages FOUND_NOVER_MSG = 'FOUND' @@ -91,7 +92,7 @@ opt_result = {} def output_width(min_width=65, max_width=90): """Return a suitable output alignment width given user terminal width.""" - proc = Popen(['stty', 'size'], stdout=PIPE) + proc = pcylc(['stty', 'size'], stdout=PIPE) if proc.wait(): return int((min_width + max_width) / 2) else: @@ -198,7 +199,7 @@ def tex_module_search(tex_module, write=True): cmd = ['kpsewhich', '%s.sty' % tex_module] # This is less intensive & quicker than searching via 'find' or 'locate'. try: - process = Popen(cmd, stdin=open(os.devnull), stdout=PIPE) + process = pcylc(cmd, stdin=open(os.devnull), stdout=PIPE), check_call(['test', '-n', process.communicate()[0].strip()], stdin=open(os.devnull), stdout=PIPE, stderr=PIPE) except (CalledProcessError, OSError): @@ -211,8 +212,8 @@ def tex_module_search(tex_module, write=True): return True -def cmd_find_ver( - module, min_ver, cmd_base, ver_opt, ver_extr, outfile=1, write=True): +def cmd_find_ver(module, min_ver, cmd_base, ver_opt, ver_extr, outfile=1, + write=True): """Print outcome & return Boolean (True for pass) of local module version requirement test using relevant custom command base keyword(s), version-checking option(s) & version-extraction regex. @@ -221,14 +222,14 @@ def cmd_find_ver( min_ver is not None else 'any') for cmd in cmd_base: try_next_cmd = True - if Popen(['which', cmd], stdin=open(os.devnull), stdout=PIPE, - stderr=PIPE).wait(): + if pcylc(['which', cmd], stdin=open(os.devnull), + stdout=PIPE).wait(): res = [NOTFOUND_MSG, False] else: try: - output = Popen([cmd, ver_opt], stdin=open(os.devnull), - stdout=PIPE, - stderr=PIPE).communicate()[outfile - 1].strip() + output = pcylc([cmd, ver_opt], stdout=PIPE, + stdin=open(os.devnull)).communicate() + [outfile - 1].strip() version = re.search(ver_extr, output).groups()[0] try_next_cmd = False if min_ver is None: diff --git a/bin/cylc-check-versions b/bin/cylc-check-versions index a47d2fa0520..0269736ba9f 100755 --- a/bin/cylc-check-versions +++ b/bin/cylc-check-versions @@ -35,18 +35,20 @@ Use -v/--verbose to see the command invoked to determine the remote version site dependent -- see cylc global config documentation.""" import os +import shlex import sys -from subprocess import Popen, PIPE +from subprocess import PIPE from time import sleep import cylc.flags -from cylc.option_parsers import CylcOptionParser as COP -from cylc.version import CYLC_VERSION from cylc.config import SuiteConfig +from cylc.option_parsers import CylcOptionParser as COP +from cylc.subprocess_safe import pcylc from cylc.subprocpool import SuiteProcPool from cylc.suite_srv_files_mgr import SuiteSrvFilesManager from cylc.task_remote_mgr import TaskRemoteMgr from cylc.templatevars import load_template_vars +from cylc.version import CYLC_VERSION def main(): @@ -112,7 +114,7 @@ def main(): user_at_host = host if verbose: print "%s: %s" % (user_at_host, ' '.join(argv)) - proc = Popen(argv, stdin=open(os.devnull), stdout=PIPE, stderr=PIPE) + proc = pcylc(argv, stdin=open(os.devnull), stdout=PIPE, stderr=PIPE) out, err = proc.communicate() if proc.wait() == 0: if verbose: diff --git a/lib/cylc/batch_sys_manager.py b/lib/cylc/batch_sys_manager.py index 6dcda0ed4ab..8fe73030240 100644 --- a/lib/cylc/batch_sys_manager.py +++ b/lib/cylc/batch_sys_manager.py @@ -115,7 +115,6 @@ import stat from subprocess import PIPE # nosec # calls to open a shell are aggregated in subprocess_safe.pcylc() -# with logging for what is calling it and the commands given from cylc.subprocess_safe import pcylc import sys import traceback @@ -423,8 +422,8 @@ def job_kill(self, st_file_path): command = shlex.split( batch_sys.KILL_CMD_TMPL % {"job_id": job_id}) try: - proc = pcylc( - command, stdin=open(os.devnull), stderr=PIPE) + proc = pcylc(command, stdin=open(os.devnull), + stderr=PIPE) except OSError as exc: # subprocess.Popen has a bad habit of not setting the # filename of the executable when it raises an OSError. @@ -556,8 +555,8 @@ def _jobs_poll_batch_sys(self, job_log_root, batch_sys_name, my_ctx_list): # Simple poll command that takes a list of job IDs cmd = [batch_sys.POLL_CMD] + exp_ids try: - proc = pcylc( - cmd, stdin=open(os.devnull), stderr=PIPE, stdout=PIPE) + proc = pcylc(cmd, stdin=open(os.devnull), + stderr=PIPE, stdout=PIPE) except OSError as exc: # subprocess.Popen has a bad habit of not setting the # filename of the executable when it raises an OSError. @@ -662,21 +661,16 @@ def _job_submit_impl( # that we do not have a shell, and still manage to get as far # as here. batch_sys_cmd = batch_submit_cmd_tmpl % {"job": job_file_path} - proc = pcylc( - batch_sys_cmd, - stdin=proc_stdin_arg, stdout=PIPE, stderr=PIPE, - shell=True, env=env) # nosec + proc = pcylc(batch_sys_cmd, stdin=proc_stdin_arg, stdout=PIPE, + stderr=PIPE, useshell=True, env=env) # calls to open a shell are aggregated in - # subprocess_safe.pcylc() with logging for what is calling it - # and the commands given + # subprocess_safe.pcylc() else: command = shlex.split( batch_sys.SUBMIT_CMD_TMPL % {"job": job_file_path}) try: - proc = pcylc( - command, - stdin=proc_stdin_arg, stdout=PIPE, stderr=PIPE, - env=env) + proc = pcylc(command, stdin=proc_stdin_arg, + stdout=PIPE, stderr=PIPE, env=env) except OSError as exc: # subprocess.Popen has a bad habit of not setting the # filename of the executable when it raises an OSError. diff --git a/lib/cylc/gui/app_gcylc.py b/lib/cylc/gui/app_gcylc.py index 95158b5db1e..1529efcc702 100644 --- a/lib/cylc/gui/app_gcylc.py +++ b/lib/cylc/gui/app_gcylc.py @@ -25,9 +25,8 @@ import pango import gobject import shlex -from subprocess import Popen, PIPE, STDOUT # nosec +from subprocess import PIPE, STDOUT # nosec # calls to open a shell are aggregated in subprocess_safe.pcylc() -# with logging for what is calling it and the commands given from uuid import uuid4 from isodatetime.parsers import TimePointParser @@ -78,11 +77,9 @@ def run_get_stdout(command, filter_=False): try: - proc = pcylc( - command, shell=True, stdout=PIPE, stderr=PIPE, # nosec - stdin=open(os.devnull)) + proc = pcylc(command, useshell=True, # nosec + stdout=PIPE, stderr=PIPE, stdin=open(os.devnull)) # calls to open a shell are aggregated in subprocess_safe.pcylc() - # with logging for what is calling it and the commands given out = proc.stdout.read() err = proc.stderr.read() res = proc.wait() @@ -999,7 +996,7 @@ def click_open(self, widget, new_window=False): # process can detach as a process group leader and not subjected to # SIGHUP from the current process. # See also "cylc.batch_sys_handlers.background". - Popen( + pcylc( [ "nohup", "bash", @@ -1207,9 +1204,8 @@ def startsuite(self, bt, window, coldstart_rb, warmstart_rb, restart_rb, pass # Cannot print to terminal (session may be closed). try: - pcylc([command], shell=True, stdin=open(os.devnull)) # nosec + pcylc(command, useshell=True, stdin=open(os.devnull)) # nosec # calls to open a shell are aggregated in subprocess_safe.pcylc() - # with logging for what is calling it and the commands given except OSError: warning_dialog('Error: failed to start ' + self.cfg.suite, self.window).warn() @@ -2859,7 +2855,7 @@ def construct_command_menu(self, menu): cat_menu.append(cylc_help_item) cylc_help_item.connect('activate', self.command_help) - cout = Popen( + cout = pcylc( ["cylc", "categories"], stdin=open(os.devnull), stdout=PIPE).communicate()[0] categories = cout.rstrip().split() @@ -2868,7 +2864,7 @@ def construct_command_menu(self, menu): cat_menu.append(foo_item) com_menu = gtk.Menu() foo_item.set_submenu(com_menu) - cout = Popen( + cout = pcylc( ["cylc-help", "category=" + category], stdin=open(os.devnull), stdout=PIPE).communicate()[0] commands = cout.rstrip().split() diff --git a/lib/cylc/gui/gcapture.py b/lib/cylc/gui/gcapture.py index 7e239da3f9b..f1a9454e1df 100644 --- a/lib/cylc/gui/gcapture.py +++ b/lib/cylc/gui/gcapture.py @@ -20,9 +20,9 @@ import gtk import os import pango +import shlex from subprocess import STDOUT # nosec # calls to open a shell are aggregated in subprocess_safe.pcylc() -# with logging for what is calling it and the commands given import tempfile from cylc.gui.tailer import Tailer @@ -134,10 +134,9 @@ def __init__(self, command, tmpdir, width=400, height=400, def run(self): proc = pcylc( - self.command, stdin=open(os.devnull), stdout=self.stdoutfile, - stderr=STDOUT, shell=True) # nosec + self.command, stdin=open(os.devnull), + stdout=self.stdoutfile, stderr=STDOUT, useshell=True) # calls to open a shell are aggregated in subprocess_safe.pcylc() - # with logging for what is calling it and the commands given self.proc = proc gobject.timeout_add(40, self.pulse_proc_progress) tail_cmd_tmpl = glbl_cfg().get_host_item("tail command template") diff --git a/lib/cylc/profiling/profile.py b/lib/cylc/profiling/profile.py index 036186b0783..482c68d3e45 100644 --- a/lib/cylc/profiling/profile.py +++ b/lib/cylc/profiling/profile.py @@ -20,18 +20,19 @@ """ import os +import shlex import shutil -from subprocess import Popen, PIPE, call -from cylc.subprocess_safe import pcylc import sys import tempfile -import time import traceback +from subprocess import PIPE, Popen, call -from . import (PROFILE_MODE_TIME, PROFILE_MODE_CYLC, PROFILE_MODES, - PROFILE_FILES, SUITE_STARTUP_STRING) +import time +from . import PROFILE_FILES, PROFILE_MODES, PROFILE_MODE_CYLC, \ + PROFILE_MODE_TIME, SUITE_STARTUP_STRING from .analysis import extract_results -from .git import (checkout, describe, GitCheckoutError,) +from .git import GitCheckoutError, checkout, describe +from cylc.subprocess_safe import pcylc def cylc_env(cylc_conf_path=''): @@ -187,12 +188,12 @@ def run_suite(reg, options, out_file, profile_modes, mode='live', # Execute. print '$ ' + ' '.join(cmds) + try: - proc = pcylc(' '.join(cmds), shell=True, # nosec + proc = pcylc([' '.join(cmds)], useshell=True, stderr=open(time_err, 'w+'), # calls to open a shell are aggregated in - # subprocess_safe.pcylc() with logging for - # what is calling it and the commands given + # subprocess_safe.pcylc() stdout=open(startup_file, 'w+'), env=env) if proc.wait(): raise SuiteFailedException(run_cmds, cmd_out, cmd_err) diff --git a/lib/cylc/run_get_stdout.py b/lib/cylc/run_get_stdout.py index 09d0e059783..8596f9cfcbf 100644 --- a/lib/cylc/run_get_stdout.py +++ b/lib/cylc/run_get_stdout.py @@ -23,9 +23,9 @@ from subprocess import PIPE # nosec from time import sleep, time +import shlex from subprocess_safe import pcylc # nosec # calls to open a shell are aggregated in subprocess_safe.pcylc() -# with logging for what is calling it and the commands given ERR_TIMEOUT = "ERROR: command timed out (>%ds), terminated by signal %d\n%s" @@ -49,10 +49,9 @@ def run_get_stdout(command, timeout=None, poll_delay=None): """ try: proc = pcylc( - command, shell=True, preexec_fn=setpgrp, # nosec + command, useshell=True, preexec_fn=setpgrp, stdin=open(devnull), stderr=PIPE, stdout=PIPE) # calls to open a shell are aggregated in subprocess_safe.pcylc() - # with logging for what is calling it and the commands given is_killed_after_timeout = False if timeout: if poll_delay is None: diff --git a/lib/cylc/subprocess_safe.py b/lib/cylc/subprocess_safe.py index 939b3c8b260..814fd39c829 100644 --- a/lib/cylc/subprocess_safe.py +++ b/lib/cylc/subprocess_safe.py @@ -25,17 +25,33 @@ """ from subprocess import Popen # nosec +from cylc import LOG +from shlex import split +from inspect import getframeinfo, stack + # pylint: disable=too-many-arguments # pylint: disable=too-many-locals def pcylc(cmd, bufsize=0, executable=None, stdin=None, stdout=None, - stderr=None, preexec_fn=None, - close_fds=False, shell=False, cwd=None, env=None, - universal_newlines=False, startupinfo=None, creationflags=0): + stderr=None, preexec_fn=None, close_fds=False, useshell=False, + cwd=None, env=None, universal_newlines=False, startupinfo=None, + creationflags=0, splitcmd=False): + + shell = useshell + if splitcmd is True: + command = split(cmd) + else: + command = cmd + + # caller = getframeinfo(stack()[1][0]) + # LOG.debug('[pcylc: calling function] {}'.format(caller.function)) + # LOG.debug('[pcylc: caller] %s:%d' % (caller.filename, caller.lineno)) + # LOG.debug('[pcylc: command] {}'.format(cmd)) + # LOG.debug('[pcylc: shell] => %r ' % shell) - process = Popen(cmd, bufsize, executable, stdin, stdout, stderr, # nosec - preexec_fn, close_fds, shell, cwd, env, universal_newlines, - startupinfo, creationflags) + process = Popen(command, bufsize, executable, stdin, stdout, # nosec + stderr, preexec_fn, close_fds, shell, cwd, env, + universal_newlines, startupinfo, creationflags) return process diff --git a/lib/cylc/subprocpool.py b/lib/cylc/subprocpool.py index 3c616e3b2e5..9f0d56f7ba3 100644 --- a/lib/cylc/subprocpool.py +++ b/lib/cylc/subprocpool.py @@ -24,7 +24,6 @@ from signal import SIGKILL from subprocess import PIPE # nosec # calls to open a shell are aggregated in subprocess_safe.pcylc() -# with logging for what is calling it and the commands given from cylc.subprocess_safe import pcylc import sys from tempfile import TemporaryFile @@ -319,7 +318,7 @@ def _run_command_init(cls, ctx, callback=None, callback_args=None): # so we can use "os.killpg" to kill the whole group. preexec_fn=os.setpgrp, env=ctx.cmd_kwargs.get('env'), - shell=ctx.cmd_kwargs.get('shell')) # nosec + useshell=ctx.cmd_kwargs.get('shell')) # calls to open a shell are aggregated in subprocess_safe.pcylc() # with logging for what is calling it and the commands given except (IOError, OSError) as exc: diff --git a/tests/broadcast/00-simple/bin/complog.py b/tests/broadcast/00-simple/bin/complog.py index 541f562fee2..171cb07c877 100755 --- a/tests/broadcast/00-simple/bin/complog.py +++ b/tests/broadcast/00-simple/bin/complog.py @@ -1,9 +1,13 @@ #!/usr/bin/env python2 import os +import shlex import sys +from distutils import command + from cylc.subprocess_safe import pcylc + print print "This is the broadcast test suite log comparator" @@ -34,7 +38,7 @@ else: print "broadcast logs compare OK" -res = pcylc(["cylc check-triggering " + event + " " + suite], shell=True) +res = pcylc(["cylc check-triggering " + event + " " + suite], useshell=True) status = res.wait() if status != 0: sys.exit(1) diff --git a/tests/cylc-cat-state/basic/state-check.py b/tests/cylc-cat-state/basic/state-check.py index 827ecacef2e..0a175d4a327 100644 --- a/tests/cylc-cat-state/basic/state-check.py +++ b/tests/cylc-cat-state/basic/state-check.py @@ -20,6 +20,7 @@ import subprocess import sys from cylc.subprocess_safe import pcylc +import shlex def main(argv): @@ -30,9 +31,9 @@ def main(argv): sname = argv[0] rundir = argv[1] + command = "cylc cat-state " + sname - p = pcylc("cylc cat-state " + sname, - shell=True, stdout=subprocess.PIPE, + p = pcylc(command, useshell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) state, err = p.communicate() @@ -64,7 +65,7 @@ def main(argv): while next: res.append(next[0]) next = cur.fetchmany() - except: + except: # noqa: E722 sys.stderr.write("unable to query suite database\n") sys.exit(1) if not res[0][0] == status: From 85a0aa9847bf6837218a63349d07d18861064313 Mon Sep 17 00:00:00 2001 From: Martin Ryan Date: Tue, 15 Jan 2019 12:24:29 +1100 Subject: [PATCH 38/83] pcylc changes filename and argument changes --- bin/cylc-check-software | 9 +++--- bin/cylc-check-versions | 6 ++-- lib/cylc/batch_sys_manager.py | 14 ++++---- lib/cylc/gui/app_gcylc.py | 15 ++++----- lib/cylc/gui/gcapture.py | 9 +++--- lib/cylc/profiling/profile.py | 5 ++- lib/cylc/run_get_stdout.py | 7 ++-- lib/cylc/{subprocess_safe.py => sprocess.py} | 32 ++++++++++++------- lib/cylc/subprocpool.py | 7 ++-- ...st_subprocess_safe.py => test_sprocess.py} | 0 tests/broadcast/00-simple/bin/complog.py | 4 +-- tests/cylc-cat-state/basic/state-check.py | 6 ++-- 12 files changed, 58 insertions(+), 56 deletions(-) rename lib/cylc/{subprocess_safe.py => sprocess.py} (76%) rename lib/cylc/tests/{test_subprocess_safe.py => test_sprocess.py} (100%) diff --git a/bin/cylc-check-software b/bin/cylc-check-software index 30601fe7966..be19a07e8d2 100755 --- a/bin/cylc-check-software +++ b/bin/cylc-check-software @@ -92,7 +92,7 @@ opt_result = {} def output_width(min_width=65, max_width=90): """Return a suitable output alignment width given user terminal width.""" - proc = pcylc(['stty', 'size'], stdout=PIPE) + proc = pcylc(['stty', 'size'], stdoutpipe=True) if proc.wait(): return int((min_width + max_width) / 2) else: @@ -199,9 +199,10 @@ def tex_module_search(tex_module, write=True): cmd = ['kpsewhich', '%s.sty' % tex_module] # This is less intensive & quicker than searching via 'find' or 'locate'. try: - process = pcylc(cmd, stdin=open(os.devnull), stdout=PIPE), + process = pcylc(cmd, stdin=open(os.devnull), stdoutpipe=True), check_call(['test', '-n', process.communicate()[0].strip()], stdin=open(os.devnull), stdout=PIPE, stderr=PIPE) + except (CalledProcessError, OSError): if write: shell_align_write('.', msg, NOTFOUND_MSG) @@ -223,11 +224,11 @@ def cmd_find_ver(module, min_ver, cmd_base, ver_opt, ver_extr, outfile=1, for cmd in cmd_base: try_next_cmd = True if pcylc(['which', cmd], stdin=open(os.devnull), - stdout=PIPE).wait(): + stdoutpipe=True).wait(): res = [NOTFOUND_MSG, False] else: try: - output = pcylc([cmd, ver_opt], stdout=PIPE, + output = pcylc([cmd, ver_opt], stdoutpipe=True, stdin=open(os.devnull)).communicate() [outfile - 1].strip() version = re.search(ver_extr, output).groups()[0] diff --git a/bin/cylc-check-versions b/bin/cylc-check-versions index 0269736ba9f..85ef0f29b36 100755 --- a/bin/cylc-check-versions +++ b/bin/cylc-check-versions @@ -37,13 +37,12 @@ site dependent -- see cylc global config documentation.""" import os import shlex import sys -from subprocess import PIPE from time import sleep import cylc.flags from cylc.config import SuiteConfig from cylc.option_parsers import CylcOptionParser as COP -from cylc.subprocess_safe import pcylc +from cylc.sprocess import pcylc from cylc.subprocpool import SuiteProcPool from cylc.suite_srv_files_mgr import SuiteSrvFilesManager from cylc.task_remote_mgr import TaskRemoteMgr @@ -114,7 +113,8 @@ def main(): user_at_host = host if verbose: print "%s: %s" % (user_at_host, ' '.join(argv)) - proc = pcylc(argv, stdin=open(os.devnull), stdout=PIPE, stderr=PIPE) + proc = pcylc(argv, stdin=open(os.devnull), stdoutpipe=True, + stderrpipe=True) out, err = proc.communicate() if proc.wait() == 0: if verbose: diff --git a/lib/cylc/batch_sys_manager.py b/lib/cylc/batch_sys_manager.py index 8fe73030240..9b2673d32e8 100644 --- a/lib/cylc/batch_sys_manager.py +++ b/lib/cylc/batch_sys_manager.py @@ -113,9 +113,8 @@ from shutil import rmtree from signal import SIGKILL import stat -from subprocess import PIPE # nosec # calls to open a shell are aggregated in subprocess_safe.pcylc() -from cylc.subprocess_safe import pcylc +from cylc.sprocess import pcylc import sys import traceback @@ -423,7 +422,7 @@ def job_kill(self, st_file_path): batch_sys.KILL_CMD_TMPL % {"job_id": job_id}) try: proc = pcylc(command, stdin=open(os.devnull), - stderr=PIPE) + stderrpipe=True) except OSError as exc: # subprocess.Popen has a bad habit of not setting the # filename of the executable when it raises an OSError. @@ -556,7 +555,7 @@ def _jobs_poll_batch_sys(self, job_log_root, batch_sys_name, my_ctx_list): cmd = [batch_sys.POLL_CMD] + exp_ids try: proc = pcylc(cmd, stdin=open(os.devnull), - stderr=PIPE, stdout=PIPE) + stderrpipe=True, stdoutpipe=True) except OSError as exc: # subprocess.Popen has a bad habit of not setting the # filename of the executable when it raises an OSError. @@ -661,8 +660,9 @@ def _job_submit_impl( # that we do not have a shell, and still manage to get as far # as here. batch_sys_cmd = batch_submit_cmd_tmpl % {"job": job_file_path} - proc = pcylc(batch_sys_cmd, stdin=proc_stdin_arg, stdout=PIPE, - stderr=PIPE, useshell=True, env=env) + proc = pcylc(batch_sys_cmd, stdin=proc_stdin_arg, + stdoutpipe=True, stderrpipe=True, usesh=True, + env=env) # calls to open a shell are aggregated in # subprocess_safe.pcylc() else: @@ -670,7 +670,7 @@ def _job_submit_impl( batch_sys.SUBMIT_CMD_TMPL % {"job": job_file_path}) try: proc = pcylc(command, stdin=proc_stdin_arg, - stdout=PIPE, stderr=PIPE, env=env) + stdoutpipe=True, stderrpipe=True, env=env) except OSError as exc: # subprocess.Popen has a bad habit of not setting the # filename of the executable when it raises an OSError. diff --git a/lib/cylc/gui/app_gcylc.py b/lib/cylc/gui/app_gcylc.py index 1529efcc702..c61a7b3e232 100644 --- a/lib/cylc/gui/app_gcylc.py +++ b/lib/cylc/gui/app_gcylc.py @@ -25,7 +25,6 @@ import pango import gobject import shlex -from subprocess import PIPE, STDOUT # nosec # calls to open a shell are aggregated in subprocess_safe.pcylc() from uuid import uuid4 @@ -35,7 +34,7 @@ from cylc.gui.dbchooser import dbchooser from cylc.gui.combo_logviewer import ComboLogViewer from cylc.gui.warning_dialog import warning_dialog, info_dialog -from cylc.subprocess_safe import pcylc +from cylc.sprocess import pcylc from cylc.task_job_logs import JOB_LOG_OPTS from cylc.wallclock import get_current_time_string @@ -77,8 +76,8 @@ def run_get_stdout(command, filter_=False): try: - proc = pcylc(command, useshell=True, # nosec - stdout=PIPE, stderr=PIPE, stdin=open(os.devnull)) + proc = pcylc(command, usesh=True, stdoutpipe=True, stderrpipe=True, + stdin=open(os.devnull)) # calls to open a shell are aggregated in subprocess_safe.pcylc() out = proc.stdout.read() err = proc.stderr.read() @@ -1007,7 +1006,7 @@ def click_open(self, widget, new_window=False): preexec_fn=os.setpgrp, stdin=open(os.devnull), stdout=open(os.devnull, "wb"), - stderr=STDOUT) + stderrout=True) else: self.reset(reg, auth) @@ -1204,7 +1203,7 @@ def startsuite(self, bt, window, coldstart_rb, warmstart_rb, restart_rb, pass # Cannot print to terminal (session may be closed). try: - pcylc(command, useshell=True, stdin=open(os.devnull)) # nosec + pcylc(command, usesh=True, stdin=open(os.devnull)) # calls to open a shell are aggregated in subprocess_safe.pcylc() except OSError: warning_dialog('Error: failed to start ' + self.cfg.suite, @@ -2857,7 +2856,7 @@ def construct_command_menu(self, menu): cout = pcylc( ["cylc", "categories"], - stdin=open(os.devnull), stdout=PIPE).communicate()[0] + stdin=open(os.devnull), stdoutpipe=True).communicate()[0] categories = cout.rstrip().split() for category in categories: foo_item = gtk.MenuItem(category) @@ -2866,7 +2865,7 @@ def construct_command_menu(self, menu): foo_item.set_submenu(com_menu) cout = pcylc( ["cylc-help", "category=" + category], - stdin=open(os.devnull), stdout=PIPE).communicate()[0] + stdin=open(os.devnull), stdoutpipe=True).communicate()[0] commands = cout.rstrip().split() for command in commands: bar_item = gtk.MenuItem(command) diff --git a/lib/cylc/gui/gcapture.py b/lib/cylc/gui/gcapture.py index f1a9454e1df..621d94f7d99 100644 --- a/lib/cylc/gui/gcapture.py +++ b/lib/cylc/gui/gcapture.py @@ -21,7 +21,6 @@ import os import pango import shlex -from subprocess import STDOUT # nosec # calls to open a shell are aggregated in subprocess_safe.pcylc() import tempfile @@ -29,7 +28,7 @@ from cylc.gui.util import get_icon from cylc.gui.warning_dialog import warning_dialog, info_dialog from cylc.cfgspec.glbl_cfg import glbl_cfg -from cylc.subprocess_safe import pcylc +from cylc.sprocess import pcylc class Gcapture(object): @@ -133,9 +132,9 @@ def __init__(self, command, tmpdir, width=400, height=400, self.window.show() def run(self): - proc = pcylc( - self.command, stdin=open(os.devnull), - stdout=self.stdoutfile, stderr=STDOUT, useshell=True) + proc = pcylc(self.command, stdin=open(os.devnull), + stdout=self.stdoutfile, stderrout=True, + usesh=True) # calls to open a shell are aggregated in subprocess_safe.pcylc() self.proc = proc gobject.timeout_add(40, self.pulse_proc_progress) diff --git a/lib/cylc/profiling/profile.py b/lib/cylc/profiling/profile.py index 482c68d3e45..905c8a8807d 100644 --- a/lib/cylc/profiling/profile.py +++ b/lib/cylc/profiling/profile.py @@ -32,7 +32,7 @@ PROFILE_MODE_TIME, SUITE_STARTUP_STRING from .analysis import extract_results from .git import GitCheckoutError, checkout, describe -from cylc.subprocess_safe import pcylc +from cylc.sprocess import pcylc def cylc_env(cylc_conf_path=''): @@ -190,8 +190,7 @@ def run_suite(reg, options, out_file, profile_modes, mode='live', print '$ ' + ' '.join(cmds) try: - proc = pcylc([' '.join(cmds)], useshell=True, - stderr=open(time_err, 'w+'), + proc = pcylc([' '.join(cmds)], usesh=True, stderr=open(time_err, 'w+'), # calls to open a shell are aggregated in # subprocess_safe.pcylc() stdout=open(startup_file, 'w+'), env=env) diff --git a/lib/cylc/run_get_stdout.py b/lib/cylc/run_get_stdout.py index 8596f9cfcbf..2b9b58e8eae 100644 --- a/lib/cylc/run_get_stdout.py +++ b/lib/cylc/run_get_stdout.py @@ -20,11 +20,10 @@ from os import devnull, killpg, setpgrp from signal import SIGTERM -from subprocess import PIPE # nosec from time import sleep, time import shlex -from subprocess_safe import pcylc # nosec +from cylc.sprocess import pcylc # calls to open a shell are aggregated in subprocess_safe.pcylc() @@ -49,8 +48,8 @@ def run_get_stdout(command, timeout=None, poll_delay=None): """ try: proc = pcylc( - command, useshell=True, preexec_fn=setpgrp, - stdin=open(devnull), stderr=PIPE, stdout=PIPE) + command, usesh=True, preexec_fn=setpgrp, + stdin=open(devnull), stderrpipe=True, stdoutpipe=True) # calls to open a shell are aggregated in subprocess_safe.pcylc() is_killed_after_timeout = False if timeout: diff --git a/lib/cylc/subprocess_safe.py b/lib/cylc/sprocess.py similarity index 76% rename from lib/cylc/subprocess_safe.py rename to lib/cylc/sprocess.py index 814fd39c829..f54ce423e31 100644 --- a/lib/cylc/subprocess_safe.py +++ b/lib/cylc/sprocess.py @@ -23,33 +23,41 @@ B605: start_process_with_a_shell https://docs.openstack.org/developer/bandit/plugins/start_process_with_a_shell.html """ -from subprocess import Popen # nosec +from shlex import split +from subprocess import PIPE, STDOUT, Popen # nosec from cylc import LOG -from shlex import split -from inspect import getframeinfo, stack # pylint: disable=too-many-arguments # pylint: disable=too-many-locals def pcylc(cmd, bufsize=0, executable=None, stdin=None, stdout=None, - stderr=None, preexec_fn=None, close_fds=False, useshell=False, + stderr=None, preexec_fn=None, close_fds=False, usesh=False, cwd=None, env=None, universal_newlines=False, startupinfo=None, - creationflags=0, splitcmd=False): + creationflags=0, splitcmd=False, stdoutpipe=False, stdoutout=True, + stderrpipe=False, stderrout=False): + + shell = usesh + + if stdoutpipe is True: + stdout = PIPE + elif stdoutout is True: + stdout = STDOUT + else: + stdout = stdout + if stderrpipe is True: + stderr = PIPE + elif stderrout is True: + stderr = STDOUT + else: + stderr = stderr - shell = useshell if splitcmd is True: command = split(cmd) else: command = cmd - # caller = getframeinfo(stack()[1][0]) - # LOG.debug('[pcylc: calling function] {}'.format(caller.function)) - # LOG.debug('[pcylc: caller] %s:%d' % (caller.filename, caller.lineno)) - # LOG.debug('[pcylc: command] {}'.format(cmd)) - # LOG.debug('[pcylc: shell] => %r ' % shell) - process = Popen(command, bufsize, executable, stdin, stdout, # nosec stderr, preexec_fn, close_fds, shell, cwd, env, universal_newlines, startupinfo, creationflags) diff --git a/lib/cylc/subprocpool.py b/lib/cylc/subprocpool.py index 9f0d56f7ba3..ca02f7aead1 100644 --- a/lib/cylc/subprocpool.py +++ b/lib/cylc/subprocpool.py @@ -22,9 +22,8 @@ import os import select from signal import SIGKILL -from subprocess import PIPE # nosec # calls to open a shell are aggregated in subprocess_safe.pcylc() -from cylc.subprocess_safe import pcylc +from cylc.sprocess import pcylc import sys from tempfile import TemporaryFile from threading import RLock @@ -313,12 +312,12 @@ def _run_command_init(cls, ctx, callback=None, callback_args=None): else: stdin_file = open(os.devnull) proc = pcylc( - ctx.cmd, stdin=stdin_file, stdout=PIPE, stderr=PIPE, + ctx.cmd, stdin=stdin_file, stdoutpipe=True, stderrpipe=True, # Execute command as a process group leader, # so we can use "os.killpg" to kill the whole group. preexec_fn=os.setpgrp, env=ctx.cmd_kwargs.get('env'), - useshell=ctx.cmd_kwargs.get('shell')) + usesh=ctx.cmd_kwargs.get('shell')) # calls to open a shell are aggregated in subprocess_safe.pcylc() # with logging for what is calling it and the commands given except (IOError, OSError) as exc: diff --git a/lib/cylc/tests/test_subprocess_safe.py b/lib/cylc/tests/test_sprocess.py similarity index 100% rename from lib/cylc/tests/test_subprocess_safe.py rename to lib/cylc/tests/test_sprocess.py diff --git a/tests/broadcast/00-simple/bin/complog.py b/tests/broadcast/00-simple/bin/complog.py index 171cb07c877..d47ae48bdaa 100755 --- a/tests/broadcast/00-simple/bin/complog.py +++ b/tests/broadcast/00-simple/bin/complog.py @@ -5,7 +5,7 @@ import sys from distutils import command -from cylc.subprocess_safe import pcylc +from cylc.sprocess import pcylc print @@ -38,7 +38,7 @@ else: print "broadcast logs compare OK" -res = pcylc(["cylc check-triggering " + event + " " + suite], useshell=True) +res = pcylc(["cylc check-triggering " + event + " " + suite], usesh=True) status = res.wait() if status != 0: sys.exit(1) diff --git a/tests/cylc-cat-state/basic/state-check.py b/tests/cylc-cat-state/basic/state-check.py index 0a175d4a327..88f55bcf280 100644 --- a/tests/cylc-cat-state/basic/state-check.py +++ b/tests/cylc-cat-state/basic/state-check.py @@ -17,9 +17,8 @@ import os import sqlite3 -import subprocess import sys -from cylc.subprocess_safe import pcylc +from cylc.sprocess import pcylc import shlex @@ -33,8 +32,7 @@ def main(argv): rundir = argv[1] command = "cylc cat-state " + sname - p = pcylc(command, useshell=True, stdout=subprocess.PIPE, - stderr=subprocess.PIPE) + p = pcylc(command, usesh=True, stdoutpipe=True, stderrpipe=True) state, err = p.communicate() if p.returncode > 0: From a8aa1b4b211f28f9876507399bbfceda85187622 Mon Sep 17 00:00:00 2001 From: Martin Ryan Date: Tue, 15 Jan 2019 13:54:52 +1100 Subject: [PATCH 39/83] correct import and arg, update comments --- bin/cylc-check-software | 2 +- lib/cylc/batch_sys_manager.py | 4 ++-- lib/cylc/gui/app_gcylc.py | 6 +++--- lib/cylc/gui/gcapture.py | 4 ++-- lib/cylc/profiling/profile.py | 2 +- lib/cylc/run_get_stdout.py | 4 ++-- lib/cylc/sprocess.py | 2 +- lib/cylc/subprocpool.py | 4 ++-- lib/cylc/tests/test_sprocess.py | 2 +- 9 files changed, 15 insertions(+), 15 deletions(-) diff --git a/bin/cylc-check-software b/bin/cylc-check-software index be19a07e8d2..505dda8ad8f 100755 --- a/bin/cylc-check-software +++ b/bin/cylc-check-software @@ -36,7 +36,7 @@ import sys import re import os from subprocess import check_call, PIPE, CalledProcessError -from cylc.subprocess_safe import pcylc +from cylc.sprocess import pcylc # Standardised output messages FOUND_NOVER_MSG = 'FOUND' diff --git a/lib/cylc/batch_sys_manager.py b/lib/cylc/batch_sys_manager.py index 9b2673d32e8..9498f0e0be6 100644 --- a/lib/cylc/batch_sys_manager.py +++ b/lib/cylc/batch_sys_manager.py @@ -113,7 +113,7 @@ from shutil import rmtree from signal import SIGKILL import stat -# calls to open a shell are aggregated in subprocess_safe.pcylc() +# calls to open a shell are aggregated in sprocess.pcylc() from cylc.sprocess import pcylc import sys import traceback @@ -664,7 +664,7 @@ def _job_submit_impl( stdoutpipe=True, stderrpipe=True, usesh=True, env=env) # calls to open a shell are aggregated in - # subprocess_safe.pcylc() + # sprocess.pcylc() else: command = shlex.split( batch_sys.SUBMIT_CMD_TMPL % {"job": job_file_path}) diff --git a/lib/cylc/gui/app_gcylc.py b/lib/cylc/gui/app_gcylc.py index c61a7b3e232..e6d2b6aed07 100644 --- a/lib/cylc/gui/app_gcylc.py +++ b/lib/cylc/gui/app_gcylc.py @@ -25,7 +25,7 @@ import pango import gobject import shlex -# calls to open a shell are aggregated in subprocess_safe.pcylc() +# calls to open a shell are aggregated in sprocess.pcylc() from uuid import uuid4 from isodatetime.parsers import TimePointParser @@ -78,7 +78,7 @@ def run_get_stdout(command, filter_=False): try: proc = pcylc(command, usesh=True, stdoutpipe=True, stderrpipe=True, stdin=open(os.devnull)) - # calls to open a shell are aggregated in subprocess_safe.pcylc() + # calls to open a shell are aggregated in sprocess.pcylc() out = proc.stdout.read() err = proc.stderr.read() res = proc.wait() @@ -1204,7 +1204,7 @@ def startsuite(self, bt, window, coldstart_rb, warmstart_rb, restart_rb, try: pcylc(command, usesh=True, stdin=open(os.devnull)) - # calls to open a shell are aggregated in subprocess_safe.pcylc() + # calls to open a shell are aggregated in sprocess.pcylc() except OSError: warning_dialog('Error: failed to start ' + self.cfg.suite, self.window).warn() diff --git a/lib/cylc/gui/gcapture.py b/lib/cylc/gui/gcapture.py index 621d94f7d99..32fa161ed1f 100644 --- a/lib/cylc/gui/gcapture.py +++ b/lib/cylc/gui/gcapture.py @@ -21,7 +21,7 @@ import os import pango import shlex -# calls to open a shell are aggregated in subprocess_safe.pcylc() +# calls to open a shell are aggregated in sprocess.pcylc() import tempfile from cylc.gui.tailer import Tailer @@ -135,7 +135,7 @@ def run(self): proc = pcylc(self.command, stdin=open(os.devnull), stdout=self.stdoutfile, stderrout=True, usesh=True) - # calls to open a shell are aggregated in subprocess_safe.pcylc() + # calls to open a shell are aggregated in sprocess.pcylc() self.proc = proc gobject.timeout_add(40, self.pulse_proc_progress) tail_cmd_tmpl = glbl_cfg().get_host_item("tail command template") diff --git a/lib/cylc/profiling/profile.py b/lib/cylc/profiling/profile.py index 905c8a8807d..3956be633a0 100644 --- a/lib/cylc/profiling/profile.py +++ b/lib/cylc/profiling/profile.py @@ -192,7 +192,7 @@ def run_suite(reg, options, out_file, profile_modes, mode='live', try: proc = pcylc([' '.join(cmds)], usesh=True, stderr=open(time_err, 'w+'), # calls to open a shell are aggregated in - # subprocess_safe.pcylc() + # sprocess.pcylc() stdout=open(startup_file, 'w+'), env=env) if proc.wait(): raise SuiteFailedException(run_cmds, cmd_out, cmd_err) diff --git a/lib/cylc/run_get_stdout.py b/lib/cylc/run_get_stdout.py index 2b9b58e8eae..da59dfa69fa 100644 --- a/lib/cylc/run_get_stdout.py +++ b/lib/cylc/run_get_stdout.py @@ -24,7 +24,7 @@ import shlex from cylc.sprocess import pcylc -# calls to open a shell are aggregated in subprocess_safe.pcylc() +# calls to open a shell are aggregated in sprocess.pcylc() ERR_TIMEOUT = "ERROR: command timed out (>%ds), terminated by signal %d\n%s" @@ -50,7 +50,7 @@ def run_get_stdout(command, timeout=None, poll_delay=None): proc = pcylc( command, usesh=True, preexec_fn=setpgrp, stdin=open(devnull), stderrpipe=True, stdoutpipe=True) - # calls to open a shell are aggregated in subprocess_safe.pcylc() + # calls to open a shell are aggregated in sprocess.pcylc() is_killed_after_timeout = False if timeout: if poll_delay is None: diff --git a/lib/cylc/sprocess.py b/lib/cylc/sprocess.py index f54ce423e31..4b6d6b23243 100644 --- a/lib/cylc/sprocess.py +++ b/lib/cylc/sprocess.py @@ -35,7 +35,7 @@ def pcylc(cmd, bufsize=0, executable=None, stdin=None, stdout=None, stderr=None, preexec_fn=None, close_fds=False, usesh=False, cwd=None, env=None, universal_newlines=False, startupinfo=None, - creationflags=0, splitcmd=False, stdoutpipe=False, stdoutout=True, + creationflags=0, splitcmd=False, stdoutpipe=False, stdoutout=False, stderrpipe=False, stderrout=False): shell = usesh diff --git a/lib/cylc/subprocpool.py b/lib/cylc/subprocpool.py index ca02f7aead1..e6c8842adca 100644 --- a/lib/cylc/subprocpool.py +++ b/lib/cylc/subprocpool.py @@ -22,7 +22,7 @@ import os import select from signal import SIGKILL -# calls to open a shell are aggregated in subprocess_safe.pcylc() +# calls to open a shell are aggregated in sprocess.pcylc() from cylc.sprocess import pcylc import sys from tempfile import TemporaryFile @@ -318,7 +318,7 @@ def _run_command_init(cls, ctx, callback=None, callback_args=None): preexec_fn=os.setpgrp, env=ctx.cmd_kwargs.get('env'), usesh=ctx.cmd_kwargs.get('shell')) - # calls to open a shell are aggregated in subprocess_safe.pcylc() + # calls to open a shell are aggregated in sprocess.pcylc() # with logging for what is calling it and the commands given except (IOError, OSError) as exc: if exc.filename is None: diff --git a/lib/cylc/tests/test_sprocess.py b/lib/cylc/tests/test_sprocess.py index 1b6802d2b27..4b447cc4c90 100644 --- a/lib/cylc/tests/test_sprocess.py +++ b/lib/cylc/tests/test_sprocess.py @@ -27,7 +27,7 @@ class TestSubprocessSafe(unittest.TestCase): - """Unit tests for the parameter subprocess_safe utility function""" + """Unit tests for the parameter sprocess utility function""" def setUp(self): self.Popen = MockPopen() From e5e5eb3093e8d62b33f06855bb805ab4d3a9dac8 Mon Sep 17 00:00:00 2001 From: Martin Ryan Date: Tue, 15 Jan 2019 14:56:53 +1100 Subject: [PATCH 40/83] splitcmd arg shlex.split in pcylc added an argument to allow shlex.split in pcylc to act on commands --- bin/cylc-check-software | 5 +++-- bin/cylc-check-versions | 2 +- lib/cylc/gui/app_gcylc.py | 2 +- lib/cylc/gui/gcapture.py | 2 +- lib/cylc/profiling/profile.py | 2 +- lib/cylc/run_get_stdout.py | 5 ++--- lib/cylc/sprocess.py | 4 ---- tests/broadcast/00-simple/bin/complog.py | 3 ++- 8 files changed, 11 insertions(+), 14 deletions(-) diff --git a/bin/cylc-check-software b/bin/cylc-check-software index 505dda8ad8f..3dc893bb9d7 100755 --- a/bin/cylc-check-software +++ b/bin/cylc-check-software @@ -199,7 +199,8 @@ def tex_module_search(tex_module, write=True): cmd = ['kpsewhich', '%s.sty' % tex_module] # This is less intensive & quicker than searching via 'find' or 'locate'. try: - process = pcylc(cmd, stdin=open(os.devnull), stdoutpipe=True), + process = pcylc(cmd, stdin=open(os.devnull), stdoutpipe=True, + splitcmd=True), check_call(['test', '-n', process.communicate()[0].strip()], stdin=open(os.devnull), stdout=PIPE, stderr=PIPE) @@ -228,7 +229,7 @@ def cmd_find_ver(module, min_ver, cmd_base, ver_opt, ver_extr, outfile=1, res = [NOTFOUND_MSG, False] else: try: - output = pcylc([cmd, ver_opt], stdoutpipe=True, + output = pcylc([cmd, ver_opt], stdoutpipe=True, splitcmd=True, stdin=open(os.devnull)).communicate() [outfile - 1].strip() version = re.search(ver_extr, output).groups()[0] diff --git a/bin/cylc-check-versions b/bin/cylc-check-versions index 85ef0f29b36..c866ef0dde0 100755 --- a/bin/cylc-check-versions +++ b/bin/cylc-check-versions @@ -114,7 +114,7 @@ def main(): if verbose: print "%s: %s" % (user_at_host, ' '.join(argv)) proc = pcylc(argv, stdin=open(os.devnull), stdoutpipe=True, - stderrpipe=True) + stderrpipe=True, splitcmd=True) out, err = proc.communicate() if proc.wait() == 0: if verbose: diff --git a/lib/cylc/gui/app_gcylc.py b/lib/cylc/gui/app_gcylc.py index e6d2b6aed07..95ab3daffdf 100644 --- a/lib/cylc/gui/app_gcylc.py +++ b/lib/cylc/gui/app_gcylc.py @@ -77,7 +77,7 @@ def run_get_stdout(command, filter_=False): try: proc = pcylc(command, usesh=True, stdoutpipe=True, stderrpipe=True, - stdin=open(os.devnull)) + stdin=open(os.devnull), splitcmd=True) # calls to open a shell are aggregated in sprocess.pcylc() out = proc.stdout.read() err = proc.stderr.read() diff --git a/lib/cylc/gui/gcapture.py b/lib/cylc/gui/gcapture.py index 32fa161ed1f..f050768058a 100644 --- a/lib/cylc/gui/gcapture.py +++ b/lib/cylc/gui/gcapture.py @@ -134,7 +134,7 @@ def __init__(self, command, tmpdir, width=400, height=400, def run(self): proc = pcylc(self.command, stdin=open(os.devnull), stdout=self.stdoutfile, stderrout=True, - usesh=True) + usesh=True, splitcmd=True) # calls to open a shell are aggregated in sprocess.pcylc() self.proc = proc gobject.timeout_add(40, self.pulse_proc_progress) diff --git a/lib/cylc/profiling/profile.py b/lib/cylc/profiling/profile.py index 3956be633a0..f7af03cc854 100644 --- a/lib/cylc/profiling/profile.py +++ b/lib/cylc/profiling/profile.py @@ -193,7 +193,7 @@ def run_suite(reg, options, out_file, profile_modes, mode='live', proc = pcylc([' '.join(cmds)], usesh=True, stderr=open(time_err, 'w+'), # calls to open a shell are aggregated in # sprocess.pcylc() - stdout=open(startup_file, 'w+'), env=env) + stdout=open(startup_file, 'w+'), env=env, splitcmd=True) if proc.wait(): raise SuiteFailedException(run_cmds, cmd_out, cmd_err) except KeyboardInterrupt: diff --git a/lib/cylc/run_get_stdout.py b/lib/cylc/run_get_stdout.py index da59dfa69fa..17fe6ea6f2a 100644 --- a/lib/cylc/run_get_stdout.py +++ b/lib/cylc/run_get_stdout.py @@ -47,9 +47,8 @@ def run_get_stdout(command, timeout=None, poll_delay=None): """ try: - proc = pcylc( - command, usesh=True, preexec_fn=setpgrp, - stdin=open(devnull), stderrpipe=True, stdoutpipe=True) + proc = pcylc(command, usesh=True, preexec_fn=setpgrp, splitcmd=True, + stdin=open(devnull), stderrpipe=True, stdoutpipe=True) # calls to open a shell are aggregated in sprocess.pcylc() is_killed_after_timeout = False if timeout: diff --git a/lib/cylc/sprocess.py b/lib/cylc/sprocess.py index 4b6d6b23243..8a393d073ec 100644 --- a/lib/cylc/sprocess.py +++ b/lib/cylc/sprocess.py @@ -44,14 +44,10 @@ def pcylc(cmd, bufsize=0, executable=None, stdin=None, stdout=None, stdout = PIPE elif stdoutout is True: stdout = STDOUT - else: - stdout = stdout if stderrpipe is True: stderr = PIPE elif stderrout is True: stderr = STDOUT - else: - stderr = stderr if splitcmd is True: command = split(cmd) diff --git a/tests/broadcast/00-simple/bin/complog.py b/tests/broadcast/00-simple/bin/complog.py index d47ae48bdaa..796c7630227 100755 --- a/tests/broadcast/00-simple/bin/complog.py +++ b/tests/broadcast/00-simple/bin/complog.py @@ -38,7 +38,8 @@ else: print "broadcast logs compare OK" -res = pcylc(["cylc check-triggering " + event + " " + suite], usesh=True) +res = pcylc(["cylc check-triggering " + event + " " + suite], usesh=True, + splitcmd=True) status = res.wait() if status != 0: sys.exit(1) From 389b0b73b854fc9e1d4950b375ec5bc2d7ba81bb Mon Sep 17 00:00:00 2001 From: Martin Ryan Date: Wed, 16 Jan 2019 00:20:54 +1100 Subject: [PATCH 41/83] wip: debug build errors --- bin/cylc-check-software | 19 +++---- bin/cylc-check-versions | 2 +- lib/cylc/batch_sys_manager.py | 23 ++++---- lib/cylc/gui/app_gcylc.py | 68 ++++++++++++------------ lib/cylc/gui/gcapture.py | 13 +++-- lib/cylc/profiling/profile.py | 11 ++-- lib/cylc/run_get_stdout.py | 5 +- lib/cylc/subprocpool.py | 9 ++-- tests/broadcast/00-simple/bin/complog.py | 4 +- 9 files changed, 71 insertions(+), 83 deletions(-) diff --git a/bin/cylc-check-software b/bin/cylc-check-software index 3dc893bb9d7..66d9ea24d3f 100755 --- a/bin/cylc-check-software +++ b/bin/cylc-check-software @@ -32,10 +32,11 @@ Arguments: valid module arguments (lower-case equivalents accepted). """ -import sys -import re import os -from subprocess import check_call, PIPE, CalledProcessError +import re +import sys +from subprocess import PIPE, CalledProcessError, check_call + from cylc.sprocess import pcylc # Standardised output messages @@ -199,10 +200,9 @@ def tex_module_search(tex_module, write=True): cmd = ['kpsewhich', '%s.sty' % tex_module] # This is less intensive & quicker than searching via 'find' or 'locate'. try: - process = pcylc(cmd, stdin=open(os.devnull), stdoutpipe=True, - splitcmd=True), + process = pcylc(cmd, stdin=open(os.devnull), stdoutpipe=True), check_call(['test', '-n', process.communicate()[0].strip()], - stdin=open(os.devnull), stdout=PIPE, stderr=PIPE) + stdin=open(os.devnull), stdout=PIPE, stder=PIPE) except (CalledProcessError, OSError): if write: @@ -225,12 +225,13 @@ def cmd_find_ver(module, min_ver, cmd_base, ver_opt, ver_extr, outfile=1, for cmd in cmd_base: try_next_cmd = True if pcylc(['which', cmd], stdin=open(os.devnull), - stdoutpipe=True).wait(): + stdoutpipe=True, stderrpipe=True).wait(): res = [NOTFOUND_MSG, False] else: try: - output = pcylc([cmd, ver_opt], stdoutpipe=True, splitcmd=True, - stdin=open(os.devnull)).communicate() + output = pcylc([cmd, ver_opt], stdoutpipe=True, + stdin=open(os.devnull), + stderrpipe=True).communicate() [outfile - 1].strip() version = re.search(ver_extr, output).groups()[0] try_next_cmd = False diff --git a/bin/cylc-check-versions b/bin/cylc-check-versions index c866ef0dde0..85ef0f29b36 100755 --- a/bin/cylc-check-versions +++ b/bin/cylc-check-versions @@ -114,7 +114,7 @@ def main(): if verbose: print "%s: %s" % (user_at_host, ' '.join(argv)) proc = pcylc(argv, stdin=open(os.devnull), stdoutpipe=True, - stderrpipe=True, splitcmd=True) + stderrpipe=True) out, err = proc.communicate() if proc.wait() == 0: if verbose: diff --git a/lib/cylc/batch_sys_manager.py b/lib/cylc/batch_sys_manager.py index 9498f0e0be6..13183b652ce 100644 --- a/lib/cylc/batch_sys_manager.py +++ b/lib/cylc/batch_sys_manager.py @@ -107,29 +107,24 @@ """ -import os import json +import os import shlex -from shutil import rmtree -from signal import SIGKILL import stat -# calls to open a shell are aggregated in sprocess.pcylc() -from cylc.sprocess import pcylc import sys import traceback - - -from parsec.OrderedDict import OrderedDict - +from shutil import rmtree +from signal import SIGKILL from cylc.mkdir_p import mkdir_p -from cylc.task_message import ( - CYLC_JOB_PID, CYLC_JOB_INIT_TIME, CYLC_JOB_EXIT_TIME, CYLC_JOB_EXIT, - CYLC_MESSAGE) +from cylc.sprocess import pcylc +from cylc.task_job_logs import (JOB_LOG_ERR, JOB_LOG_JOB, JOB_LOG_OUT, + JOB_LOG_STATUS) +from cylc.task_message import (CYLC_JOB_EXIT, CYLC_JOB_EXIT_TIME, + CYLC_JOB_INIT_TIME, CYLC_JOB_PID, CYLC_MESSAGE) from cylc.task_outputs import TASK_OUTPUT_SUCCEEDED -from cylc.task_job_logs import ( - JOB_LOG_JOB, JOB_LOG_OUT, JOB_LOG_ERR, JOB_LOG_STATUS) from cylc.wallclock import get_current_time_string +from parsec.OrderedDict import OrderedDict class JobPollContext(object): diff --git a/lib/cylc/gui/app_gcylc.py b/lib/cylc/gui/app_gcylc.py index 95ab3daffdf..a5c87d87d7d 100644 --- a/lib/cylc/gui/app_gcylc.py +++ b/lib/cylc/gui/app_gcylc.py @@ -20,23 +20,45 @@ import os import re -import sys -import gtk -import pango -import gobject import shlex -# calls to open a shell are aggregated in sprocess.pcylc() +import sys from uuid import uuid4 -from isodatetime.parsers import TimePointParser - -from cylc.hostuserutil import is_remote_host, is_remote_user -from cylc.gui.dbchooser import dbchooser +import gobject +import gtk +import pango +from cylc.cfgspec.gcylc import GcylcConfig +from cylc.cfgspec.glbl_cfg import glbl_cfg from cylc.gui.combo_logviewer import ComboLogViewer -from cylc.gui.warning_dialog import warning_dialog, info_dialog +from cylc.gui.dbchooser import dbchooser +from cylc.gui.dot_maker import DotMaker +from cylc.gui.gcapture import Gcapture +from cylc.gui.legend import ThemeLegendWindow +from cylc.gui.option_group import controlled_option_group +from cylc.gui.suite_log_viewer import SuiteLogViewer +from cylc.gui.updater import Updater +from cylc.gui.util import (EntryDialog, EntryTempText, get_icon, get_image_dir, + get_logo, set_exception_hook_dialog, setup_icons) +from cylc.gui.view_dot import ControlLED +from cylc.gui.view_tree import ControlTree +from cylc.gui.warning_dialog import info_dialog, warning_dialog +from cylc.hostuserutil import is_remote_host, is_remote_user +from cylc.network.httpclient import ClientError from cylc.sprocess import pcylc +from cylc.suite_srv_files_mgr import SuiteSrvFilesManager +from cylc.suite_status import SUITE_STATUS_STOPPED_WITH +from cylc.task_id import TaskID from cylc.task_job_logs import JOB_LOG_OPTS +from cylc.task_state import (TASK_STATUS_FAILED, TASK_STATUS_HELD, + TASK_STATUS_RUNNING, TASK_STATUSES_ACTIVE, + TASK_STATUSES_ALL, TASK_STATUSES_CAN_RESET_TO, + TASK_STATUSES_NO_JOB_FILE, + TASK_STATUSES_RESTRICTED, + TASK_STATUSES_TRIGGERABLE) +from cylc.task_state_prop import extract_group_state, get_status_prop +from cylc.version import CYLC_VERSION from cylc.wallclock import get_current_time_string +from isodatetime.parsers import TimePointParser try: from cylc.gui.view_graph import ControlGraph @@ -49,35 +71,11 @@ else: graphing_disabled = False -from cylc.gui.legend import ThemeLegendWindow -from cylc.gui.view_dot import ControlLED -from cylc.gui.view_tree import ControlTree -from cylc.gui.dot_maker import DotMaker -from cylc.gui.updater import Updater -from cylc.gui.util import ( - get_icon, get_image_dir, get_logo, EntryTempText, - EntryDialog, setup_icons, set_exception_hook_dialog) -from cylc.network.httpclient import ClientError -from cylc.suite_status import SUITE_STATUS_STOPPED_WITH -from cylc.task_id import TaskID -from cylc.task_state_prop import extract_group_state, get_status_prop -from cylc.version import CYLC_VERSION -from cylc.gui.option_group import controlled_option_group -from cylc.gui.suite_log_viewer import SuiteLogViewer -from cylc.gui.gcapture import Gcapture -from cylc.suite_srv_files_mgr import SuiteSrvFilesManager -from cylc.cfgspec.glbl_cfg import glbl_cfg -from cylc.cfgspec.gcylc import GcylcConfig -from cylc.task_state import ( - TASK_STATUSES_ALL, TASK_STATUSES_RESTRICTED, TASK_STATUSES_CAN_RESET_TO, - TASK_STATUSES_TRIGGERABLE, TASK_STATUSES_ACTIVE, TASK_STATUS_RUNNING, - TASK_STATUS_HELD, TASK_STATUS_FAILED, TASK_STATUSES_NO_JOB_FILE) - def run_get_stdout(command, filter_=False): try: proc = pcylc(command, usesh=True, stdoutpipe=True, stderrpipe=True, - stdin=open(os.devnull), splitcmd=True) + stdin=open(os.devnull)) # calls to open a shell are aggregated in sprocess.pcylc() out = proc.stdout.read() err = proc.stderr.read() diff --git a/lib/cylc/gui/gcapture.py b/lib/cylc/gui/gcapture.py index f050768058a..8f34633e723 100644 --- a/lib/cylc/gui/gcapture.py +++ b/lib/cylc/gui/gcapture.py @@ -16,18 +16,17 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . -import gobject -import gtk import os -import pango import shlex -# calls to open a shell are aggregated in sprocess.pcylc() import tempfile +import gobject +import gtk +import pango +from cylc.cfgspec.glbl_cfg import glbl_cfg from cylc.gui.tailer import Tailer from cylc.gui.util import get_icon -from cylc.gui.warning_dialog import warning_dialog, info_dialog -from cylc.cfgspec.glbl_cfg import glbl_cfg +from cylc.gui.warning_dialog import info_dialog, warning_dialog from cylc.sprocess import pcylc @@ -134,7 +133,7 @@ def __init__(self, command, tmpdir, width=400, height=400, def run(self): proc = pcylc(self.command, stdin=open(os.devnull), stdout=self.stdoutfile, stderrout=True, - usesh=True, splitcmd=True) + usesh=True) # calls to open a shell are aggregated in sprocess.pcylc() self.proc = proc gobject.timeout_add(40, self.pulse_proc_progress) diff --git a/lib/cylc/profiling/profile.py b/lib/cylc/profiling/profile.py index f7af03cc854..ee2a3cc67d2 100644 --- a/lib/cylc/profiling/profile.py +++ b/lib/cylc/profiling/profile.py @@ -24,15 +24,16 @@ import shutil import sys import tempfile +import time import traceback from subprocess import PIPE, Popen, call -import time -from . import PROFILE_FILES, PROFILE_MODES, PROFILE_MODE_CYLC, \ - PROFILE_MODE_TIME, SUITE_STARTUP_STRING +from cylc.sprocess import pcylc + +from . import (PROFILE_FILES, PROFILE_MODE_CYLC, PROFILE_MODE_TIME, + PROFILE_MODES, SUITE_STARTUP_STRING) from .analysis import extract_results from .git import GitCheckoutError, checkout, describe -from cylc.sprocess import pcylc def cylc_env(cylc_conf_path=''): @@ -193,7 +194,7 @@ def run_suite(reg, options, out_file, profile_modes, mode='live', proc = pcylc([' '.join(cmds)], usesh=True, stderr=open(time_err, 'w+'), # calls to open a shell are aggregated in # sprocess.pcylc() - stdout=open(startup_file, 'w+'), env=env, splitcmd=True) + stdout=open(startup_file, 'w+'), env=env) if proc.wait(): raise SuiteFailedException(run_cmds, cmd_out, cmd_err) except KeyboardInterrupt: diff --git a/lib/cylc/run_get_stdout.py b/lib/cylc/run_get_stdout.py index 17fe6ea6f2a..5a697fc0fcc 100644 --- a/lib/cylc/run_get_stdout.py +++ b/lib/cylc/run_get_stdout.py @@ -18,13 +18,12 @@ """Provide a utility function to get STDOUT from a shell command.""" +import shlex from os import devnull, killpg, setpgrp from signal import SIGTERM from time import sleep, time -import shlex from cylc.sprocess import pcylc -# calls to open a shell are aggregated in sprocess.pcylc() ERR_TIMEOUT = "ERROR: command timed out (>%ds), terminated by signal %d\n%s" @@ -47,7 +46,7 @@ def run_get_stdout(command, timeout=None, poll_delay=None): """ try: - proc = pcylc(command, usesh=True, preexec_fn=setpgrp, splitcmd=True, + proc = pcylc(command, usesh=True, preexec_fn=setpgrp, stdin=open(devnull), stderrpipe=True, stdoutpipe=True) # calls to open a shell are aggregated in sprocess.pcylc() is_killed_after_timeout = False diff --git a/lib/cylc/subprocpool.py b/lib/cylc/subprocpool.py index e6c8842adca..7bdd50d0f65 100644 --- a/lib/cylc/subprocpool.py +++ b/lib/cylc/subprocpool.py @@ -17,24 +17,21 @@ # along with this program. If not, see . """Manage queueing and pooling of subprocesses for the suite server program.""" -from collections import deque import json import os import select -from signal import SIGKILL -# calls to open a shell are aggregated in sprocess.pcylc() -from cylc.sprocess import pcylc import sys +from collections import deque +from signal import SIGKILL from tempfile import TemporaryFile from threading import RLock from time import time - from cylc import LOG from cylc.cfgspec.glbl_cfg import glbl_cfg +from cylc.sprocess import pcylc from cylc.wallclock import get_current_time_string - _XTRIG_FUNCS = {} diff --git a/tests/broadcast/00-simple/bin/complog.py b/tests/broadcast/00-simple/bin/complog.py index 796c7630227..31f9b9bbf02 100755 --- a/tests/broadcast/00-simple/bin/complog.py +++ b/tests/broadcast/00-simple/bin/complog.py @@ -7,7 +7,6 @@ from cylc.sprocess import pcylc - print print "This is the broadcast test suite log comparator" @@ -38,8 +37,7 @@ else: print "broadcast logs compare OK" -res = pcylc(["cylc check-triggering " + event + " " + suite], usesh=True, - splitcmd=True) +res = pcylc(["cylc check-triggering " + event + " " + suite], usesh=True) status = res.wait() if status != 0: sys.exit(1) From c39aee4502927d3bbd82e9862c0541f45453c56c Mon Sep 17 00:00:00 2001 From: Martin Ryan Date: Wed, 16 Jan 2019 11:03:11 +1100 Subject: [PATCH 42/83] shlex.split --- lib/cylc/gui/app_gcylc.py | 4 ++-- lib/cylc/gui/gcapture.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/cylc/gui/app_gcylc.py b/lib/cylc/gui/app_gcylc.py index a5c87d87d7d..17158ae1b51 100644 --- a/lib/cylc/gui/app_gcylc.py +++ b/lib/cylc/gui/app_gcylc.py @@ -75,7 +75,7 @@ def run_get_stdout(command, filter_=False): try: proc = pcylc(command, usesh=True, stdoutpipe=True, stderrpipe=True, - stdin=open(os.devnull)) + stdin=open(os.devnull), splitcmd=True) # calls to open a shell are aggregated in sprocess.pcylc() out = proc.stdout.read() err = proc.stderr.read() @@ -1201,7 +1201,7 @@ def startsuite(self, bt, window, coldstart_rb, warmstart_rb, restart_rb, pass # Cannot print to terminal (session may be closed). try: - pcylc(command, usesh=True, stdin=open(os.devnull)) + pcylc(command, usesh=True, stdin=open(os.devnull), splitcmd=True) # calls to open a shell are aggregated in sprocess.pcylc() except OSError: warning_dialog('Error: failed to start ' + self.cfg.suite, diff --git a/lib/cylc/gui/gcapture.py b/lib/cylc/gui/gcapture.py index 8f34633e723..b07f9f25e27 100644 --- a/lib/cylc/gui/gcapture.py +++ b/lib/cylc/gui/gcapture.py @@ -133,7 +133,7 @@ def __init__(self, command, tmpdir, width=400, height=400, def run(self): proc = pcylc(self.command, stdin=open(os.devnull), stdout=self.stdoutfile, stderrout=True, - usesh=True) + usesh=True, splitcmd=True) # calls to open a shell are aggregated in sprocess.pcylc() self.proc = proc gobject.timeout_add(40, self.pulse_proc_progress) From a6bc0e41e98bb5491dd5735c8c2316465e115fc4 Mon Sep 17 00:00:00 2001 From: Martin Ryan Date: Wed, 16 Jan 2019 11:10:19 +1100 Subject: [PATCH 43/83] prefer use of secrets over random when generating suite passwords --- etc/dev-bin/defn-order-test.py | 30 +++++++++++-------- lib/jinja2/filters.py | 55 +++++++++++++++++++++------------- 2 files changed, 52 insertions(+), 33 deletions(-) diff --git a/etc/dev-bin/defn-order-test.py b/etc/dev-bin/defn-order-test.py index 5556c00325e..3067d579248 100755 --- a/etc/dev-bin/defn-order-test.py +++ b/etc/dev-bin/defn-order-test.py @@ -1,46 +1,52 @@ #!/usr/bin/env python2 -import time, string, random +import string +import time from copy import deepcopy +try: + from secrets import choice, randrange +except ImportError: + from random import choice, randrange + # This is a standalone performance test of the algorithm used in gcylc to # sort namespaces into "definition order", i.e. the order in which they are # defined in the suite.rc file. # Number of namespaces. -N=10000 +N = 10000 # N names of length 5-15 characters each (c.f. namespaces in "definition # order"). names = [] -for i in range(0,N): - names.append( ''.join( random.choice(string.ascii_letters) for n in xrange( 5+random.randrange(10) ))) +for i in range(0, N): + names.append(''.join(choice(string.ascii_letters) + for n in xrange(5+randrange(10)))) # N lists with 2-7 names each (c.f. tree view paths of the inheritance # hierarchy). paths1 = [] -for i in range(0,N): +for i in range(0, N): p = [] - for j in range(0, 2+random.randrange(6)): - z = random.randrange(0,N) - p.append( names[z] ) + for j in range(0, 2+randrange(6)): + z = randrange(0, N) + p.append(names[z]) paths1.append(p) -paths2 = deepcopy( paths1 ) +paths2 = deepcopy(paths1) # Alphanumeric sort. s = time.time() paths1.sort() t1 = time.time() - s -dict_names = dict( zip( names, range(0,len(names)))) +dict_names = dict(zip(names, range(0, len(names)))) # Definition order sort. s = time.time() -paths2.sort( key=lambda x: map( dict_names.get, x ) ) +paths2.sort(key=lambda x: map(dict_names.get, x)) t2 = time.time() - s print "Alphanumeric sort:", t1, "sec" print "Definition sort:", t2, "sec" print " => factor of", t2/t1 - diff --git a/lib/jinja2/filters.py b/lib/jinja2/filters.py index 267ddddaa03..e671b58cd92 100644 --- a/lib/jinja2/filters.py +++ b/lib/jinja2/filters.py @@ -8,18 +8,22 @@ :copyright: (c) 2017 by the Jinja Team. :license: BSD, see LICENSE for more details. """ -import re import math -import random +import re import warnings - -from itertools import groupby, chain from collections import namedtuple -from jinja2.utils import Markup, escape, pformat, urlize, soft_unicode, \ - unicode_urlencode, htmlsafe_json_dumps -from jinja2.runtime import Undefined +from itertools import chain, groupby + +from jinja2._compat import PY2, imap, iteritems, string_types, text_type from jinja2.exceptions import FilterArgumentError -from jinja2._compat import imap, string_types, text_type, iteritems, PY2 +from jinja2.runtime import Undefined +from jinja2.utils import (Markup, escape, htmlsafe_json_dumps, pformat, + soft_unicode, unicode_urlencode, urlize) + +try: + from secrets import choice, randrange +except ImportError: + from random import choice, randrange _word_re = re.compile(r'\w+', re.UNICODE) @@ -68,7 +72,8 @@ def make_attrgetter(environment, attribute, postprocess=None): if attribute is None: attribute = [] elif isinstance(attribute, string_types): - attribute = [int(x) if x.isdigit() else x for x in attribute.split('.')] + attribute = [int(x) if x.isdigit() + else x for x in attribute.split('.')] else: attribute = [attribute] @@ -451,9 +456,10 @@ def do_last(environment, seq): def do_random(context, seq): """Return a random item from the sequence.""" try: - return random.choice(seq) + return choice(seq) except IndexError: - return context.environment.undefined('No random item, sequence was empty.') + return context.environment.undefined( + 'No random item, sequence was empty.') def do_filesizeformat(value, binary=False): @@ -601,7 +607,8 @@ def do_truncate(env, s, length=255, killwords=False, end='...', leeway=None): """ if leeway is None: leeway = env.policies['truncate.leeway'] - assert length >= len(end), 'expected length >= %s, got %s' % (len(end), length) + assert length >= len( + end), 'expected length >= %s, got %s' % (len(end), length) assert leeway >= 0, 'expected leeway >= 0, got %s' % leeway if len(s) <= length + leeway: return s @@ -629,8 +636,8 @@ def do_wordwrap(environment, s, width=79, break_long_words=True, wrapstring = environment.newline_sequence import textwrap return wrapstring.join(textwrap.wrap(s, width=width, expand_tabs=False, - replace_whitespace=False, - break_long_words=break_long_words)) + replace_whitespace=False, + break_long_words=break_long_words)) def do_wordcount(s): @@ -791,7 +798,7 @@ def do_round(value, precision=0, method='common'): {{ 42.55|round|int }} -> 43 """ - if not method in ('common', 'ceil', 'floor'): + if method not in ('common', 'ceil', 'floor'): raise FilterArgumentError('method must be common, ceil or floor') if method == 'common': return round(value, precision) @@ -808,6 +815,7 @@ def do_round(value, precision=0, method='common'): _GroupTuple.__repr__ = tuple.__repr__ _GroupTuple.__str__ = tuple.__str__ + @environmentfilter def do_groupby(environment, value, attribute): """Group a sequence of objects by a common attribute. @@ -888,7 +896,8 @@ def do_mark_safe(value): def do_mark_unsafe(value): - """Mark a value as unsafe. This is the reverse operation for :func:`safe`.""" + """Mark a value as unsafe. This is the reverse operation for :func:`safe`. + """ return text_type(value) @@ -915,7 +924,8 @@ def do_attr(environment, obj, name): ``foo.bar`` just that always an attribute is returned and items are not looked up. - See :ref:`Notes on subscriptions ` for more details. + See :ref:`Notes on subscriptions ` for more + details. """ try: name = str(name) @@ -1086,7 +1096,7 @@ def prepare_map(args, kwargs): attribute = kwargs.pop('attribute') if kwargs: raise FilterArgumentError('Unexpected keyword argument %r' % - next(iter(kwargs))) + next(iter(kwargs))) func = make_attrgetter(context.environment, attribute) else: try: @@ -1094,7 +1104,8 @@ def prepare_map(args, kwargs): args = args[3:] except LookupError: raise FilterArgumentError('map requires a filter argument') - func = lambda item: context.environment.call_filter( + + def func(item): return context.environment.call_filter( name, item, args, kwargs, context=context) return seq, func @@ -1112,12 +1123,14 @@ def prepare_select_or_reject(args, kwargs, modfunc, lookup_attr): off = 1 else: off = 0 - transfunc = lambda x: x + + def transfunc(x): return x try: name = args[2 + off] args = args[3 + off:] - func = lambda item: context.environment.call_test( + + def func(item): return context.environment.call_test( name, item, args, kwargs) except LookupError: func = bool From c158f68205b8fac420633eed61c1c797e1e5a233 Mon Sep 17 00:00:00 2001 From: Martin Ryan Date: Wed, 16 Jan 2019 13:41:48 +1100 Subject: [PATCH 44/83] specify textfixtures version in travis.yml build import issue with textfixtures may be due to version - https://sea-region.github.com/Simplistix/testfixtures/issues/105 also fixed a typo --- .travis.yml | 75 ++++++++++++++++++++++++----------------- bin/cylc-check-software | 4 +-- 2 files changed, 47 insertions(+), 32 deletions(-) diff --git a/.travis.yml b/.travis.yml index f517d560429..fcbcbf2a6e2 100644 --- a/.travis.yml +++ b/.travis.yml @@ -26,21 +26,33 @@ --- language: python +cache: pip +virtualenv: + # we need system packages in Travis-CI's virtualenv in order to access pygtk, installed via apt-get + system_site_packages: true +dist: xenial +addons: + apt: + # instruct travis-ci to always run apt-get before each build + update: true -matrix: - include: - - name: "1/4" - env: CHUNK='1/4' - before_script: - # we must run unit tests only once - - PYTHONPATH=$(pwd -P)/lib/ pytest --cov-append --cov=lib/cylc --cov=lib/parsec - - name: "2/4" - env: CHUNK='2/4' - - name: "3/4" - env: CHUNK='3/4' - - name: "4/4" - env: CHUNK='4/4' +stages: +- unit-test +- test +env: + global: + - PATH="${TRAVIS_BUILD_DIR}/bin:$PATH" + # Only run the generic tests on Travis CI. + - CYLC_TEST_RUN_PLATFORM=false + # Custom diff command to ignore Xlib errors (xvfb has not RANDR extension). + - CYLC_TEST_DIFF_CMD="diff -I Xlib -u" + - COVERAGE_PROCESS_START="${TRAVIS_BUILD_DIR}/.coveragerc" + matrix: + - CHUNK="1/4" + - CHUNK="2/4" + - CHUNK="3/4" + - CHUNK="4/4" # General environment setup before we start installing stuff before_install: @@ -77,25 +89,28 @@ install: - pip install coverage pytest-cov mock - export COVERAGE_PROCESS_START="${TRAVIS_BUILD_DIR}/.coveragerc" # Testfixtures - - pip install testfixtures + - pip install testfixtures==6.4.3 # Run tests +# this is the default test stage, which is used for functional-tests +install: .travis/install.sh functional-tests docs script: - # Custom diff command to ignore Xlib errors (xvfb has not RANDR extension). - - export CYLC_TEST_DIFF_CMD='diff -I Xlib -u' - # Only run the generic tests on Travis CI. - - export CYLC_TEST_RUN_PLATFORM=false - - export PYTHONPATH="${TRAVIS_BUILD_DIR}/.travis" - - coverage run .travis/cover.py - - unset PYTHONPATH - +- export PYTHONPATH="${TRAVIS_BUILD_DIR}/.travis" +- coverage run .travis/cover.py +- unset PYTHONPATH +after_script: .travis/after_script.sh after_success: - # Report metrics, such as coverage - - coverage combine --append - - coverage xml --ignore-errors - - bash <(curl -s https://codecov.io/bash) +# Report metrics, such as coverage +- coverage combine --append +- coverage xml --ignore-errors +- bash <(curl -s https://codecov.io/bash) -# Check output (more useful if you narrow down what tests get run) -after_script: - - find $HOME/cylc-run -name '*.err' -type f -exec echo '==== {} ====' \; -exec cat '{}' \; - - find /tmp/${USER}/cylctb-* -type f -exec echo '==== {} ====' \; -exec cat '{}' \; +jobs: + include: + - stage: unit-test + install: + - .travis/install.sh unit-tests functional-tests + script: + - pycodestyle --ignore=E402,W503,W504 lib/cylc lib/Jinja2Filters/*.py lib/parsec/*.py $(grep -l '#!.*\' bin/*) + - PYTHONPATH=$(pwd -P)/lib/ pytest --cov-append --cov=lib/cylc --cov=lib/parsec + after_script: true \ No newline at end of file diff --git a/bin/cylc-check-software b/bin/cylc-check-software index 66d9ea24d3f..12501abede3 100755 --- a/bin/cylc-check-software +++ b/bin/cylc-check-software @@ -200,9 +200,9 @@ def tex_module_search(tex_module, write=True): cmd = ['kpsewhich', '%s.sty' % tex_module] # This is less intensive & quicker than searching via 'find' or 'locate'. try: - process = pcylc(cmd, stdin=open(os.devnull), stdoutpipe=True), + process = pcylc(cmd, stdin=open(os.devnull), stdoutpipe=True) check_call(['test', '-n', process.communicate()[0].strip()], - stdin=open(os.devnull), stdout=PIPE, stder=PIPE) + stdin=open(os.devnull), stdout=PIPE, stderr=PIPE) except (CalledProcessError, OSError): if write: From 513cf4d177fa81a42d4fb818a4bee1d287485ba6 Mon Sep 17 00:00:00 2001 From: Martin Ryan Date: Wed, 16 Jan 2019 13:57:22 +1100 Subject: [PATCH 45/83] testfixtures version specified 6.3.0 --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index db3db867845..e9b3c3e16fc 100644 --- a/.travis.yml +++ b/.travis.yml @@ -89,7 +89,7 @@ install: - pip install coverage pytest-cov mock - export COVERAGE_PROCESS_START="${TRAVIS_BUILD_DIR}/.coveragerc" # Testfixtures - - pip install testfixtures==6.4.3 + - pip install testfixtures==6.3.0 # Run tests # this is the default test stage, which is used for functional-tests From 6d9bf07cebfbf774b22e862bc947093b55a0e674 Mon Sep 17 00:00:00 2001 From: Martin Ryan Date: Wed, 16 Jan 2019 14:16:43 +1100 Subject: [PATCH 46/83] add testfixtures to install.sh --- .travis.yml | 2 +- .travis/install.sh | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.travis.yml b/.travis.yml index e9b3c3e16fc..d94b3cdc491 100644 --- a/.travis.yml +++ b/.travis.yml @@ -89,7 +89,7 @@ install: - pip install coverage pytest-cov mock - export COVERAGE_PROCESS_START="${TRAVIS_BUILD_DIR}/.coveragerc" # Testfixtures - - pip install testfixtures==6.3.0 + - pip install testfixtures # Run tests # this is the default test stage, which is used for functional-tests diff --git a/.travis/install.sh b/.travis/install.sh index 2218c52ea17..cd75029cf9c 100755 --- a/.travis/install.sh +++ b/.travis/install.sh @@ -27,11 +27,11 @@ args=("$@") # pygtk via apt-get, necessary for both unit and functional tests sudo apt-get install graphviz libgraphviz-dev python-gtk2-dev heirloom-mailx # coverage dependencies -pip install coverage pytest-cov mock +pip install coverage pytest-cov mock testfixtures # install dependencies required for running unit tests if grep 'unit-tests' <<< "${args[@]}"; then - pip install EmPy pyopenssl pycodestyle pytest mock + pip install EmPy pyopenssl pycodestyle pytest mock testfixtures fi # install dependencies required for running functional tests From f51af34e2f34b96e1f5d622320d929664a3aec97 Mon Sep 17 00:00:00 2001 From: Martin Ryan Date: Fri, 18 Jan 2019 11:33:01 +1100 Subject: [PATCH 47/83] implicitly import secrets codacy errors when using a conditional import, so implicitly imported the secrets package. also removed unused imports, added a new unit test --- etc/dev-bin/defn-order-test.py | 12 ++++-------- lib/cylc/gui/gcapture.py | 1 - lib/cylc/profiling/profile.py | 3 +-- lib/cylc/run_get_stdout.py | 1 - lib/cylc/sprocess.py | 2 -- lib/cylc/tests/test_sprocess.py | 17 +++++++++++++---- 6 files changed, 18 insertions(+), 18 deletions(-) diff --git a/etc/dev-bin/defn-order-test.py b/etc/dev-bin/defn-order-test.py index 3067d579248..7201c4518e6 100755 --- a/etc/dev-bin/defn-order-test.py +++ b/etc/dev-bin/defn-order-test.py @@ -1,14 +1,10 @@ #!/usr/bin/env python2 +import secrets import string import time from copy import deepcopy -try: - from secrets import choice, randrange -except ImportError: - from random import choice, randrange - # This is a standalone performance test of the algorithm used in gcylc to # sort namespaces into "definition order", i.e. the order in which they are # defined in the suite.rc file. @@ -20,7 +16,7 @@ # order"). names = [] for i in range(0, N): - names.append(''.join(choice(string.ascii_letters) + names.append(''.join(secrets.choice(string.ascii_letters) for n in xrange(5+randrange(10)))) # N lists with 2-7 names each (c.f. tree view paths of the inheritance @@ -28,8 +24,8 @@ paths1 = [] for i in range(0, N): p = [] - for j in range(0, 2+randrange(6)): - z = randrange(0, N) + for j in range(0, 2+secrets.randrange(6)): + z = secrets.randrange(0, N) p.append(names[z]) paths1.append(p) diff --git a/lib/cylc/gui/gcapture.py b/lib/cylc/gui/gcapture.py index b07f9f25e27..f5d6850f972 100644 --- a/lib/cylc/gui/gcapture.py +++ b/lib/cylc/gui/gcapture.py @@ -17,7 +17,6 @@ # along with this program. If not, see . import os -import shlex import tempfile import gobject diff --git a/lib/cylc/profiling/profile.py b/lib/cylc/profiling/profile.py index ee2a3cc67d2..3ed292cc8c6 100644 --- a/lib/cylc/profiling/profile.py +++ b/lib/cylc/profiling/profile.py @@ -20,13 +20,12 @@ """ import os -import shlex import shutil import sys import tempfile import time import traceback -from subprocess import PIPE, Popen, call +from subprocess import PIPE, Popen, call # nosec from cylc.sprocess import pcylc diff --git a/lib/cylc/run_get_stdout.py b/lib/cylc/run_get_stdout.py index 5a697fc0fcc..1dede312f18 100644 --- a/lib/cylc/run_get_stdout.py +++ b/lib/cylc/run_get_stdout.py @@ -18,7 +18,6 @@ """Provide a utility function to get STDOUT from a shell command.""" -import shlex from os import devnull, killpg, setpgrp from signal import SIGTERM from time import sleep, time diff --git a/lib/cylc/sprocess.py b/lib/cylc/sprocess.py index 8a393d073ec..219d327ad1d 100644 --- a/lib/cylc/sprocess.py +++ b/lib/cylc/sprocess.py @@ -26,8 +26,6 @@ from shlex import split from subprocess import PIPE, STDOUT, Popen # nosec -from cylc import LOG - # pylint: disable=too-many-arguments # pylint: disable=too-many-locals diff --git a/lib/cylc/tests/test_sprocess.py b/lib/cylc/tests/test_sprocess.py index 4b447cc4c90..85c6ca61dcd 100644 --- a/lib/cylc/tests/test_sprocess.py +++ b/lib/cylc/tests/test_sprocess.py @@ -18,6 +18,7 @@ import unittest +from cylc.sprocess import pcylc from mock import call from testfixtures import compare from testfixtures.popen import PIPE, MockPopen @@ -32,7 +33,15 @@ class TestSubprocessSafe(unittest.TestCase): def setUp(self): self.Popen = MockPopen() - def test_subprocess_safe_communicate_with_input(self): + def test_sprocess_communicate_with_process(self): + foo = ' foo' + bar = ' bar' + cmd = ["echo", "this is a command" + foo + bar] + p = pcylc(cmd, stdoutpipe=True) + stdout, _ = p.communicate() + assert stdout == b"this is a command foo bar\n" + + def test_sprocess_communicate_with_input(self): command = "a command" Popen = MockPopen() Popen.set_command(command) @@ -48,7 +57,7 @@ def test_subprocess_safe_communicate_with_input(self): ], Popen.mock.method_calls) return err, out - def test_subprocess_safe_read_from_stdout_and_stderr(self): + def test_sprocess_safe_read_from_stdout_and_stderr(self): command = "a command" Popen = MockPopen() # only static input used with simulated mockpopen @@ -62,7 +71,7 @@ def test_subprocess_safe_read_from_stdout_and_stderr(self): stdout=PIPE), ], Popen.mock.method_calls) - def test_subprocess_safe_write_to_stdin(self): + def test_sprocess_safe_write_to_stdin(self): command = "a command" Popen = MockPopen() Popen.set_command(command) @@ -79,7 +88,7 @@ def test_subprocess_safe_write_to_stdin(self): call.Popen_instance.stdin.close(), ], Popen.mock.method_calls) - def test_subprocess_safe_wait_and_return_code(self): + def test_sprocess_safe_wait_and_return_code(self): command = "a command" Popen = MockPopen() Popen.set_command(command, returncode=3) From 4eeb49cf60d751f7bf04a0c1b9f75674d58957c4 Mon Sep 17 00:00:00 2001 From: Martin Ryan Date: Fri, 18 Jan 2019 14:12:17 +1100 Subject: [PATCH 48/83] use compare instead of assert codacy sees the use of assert in this test as an issue, so using compare --- lib/cylc/tests/test_sprocess.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/cylc/tests/test_sprocess.py b/lib/cylc/tests/test_sprocess.py index 85c6ca61dcd..4f7231ec883 100644 --- a/lib/cylc/tests/test_sprocess.py +++ b/lib/cylc/tests/test_sprocess.py @@ -39,7 +39,7 @@ def test_sprocess_communicate_with_process(self): cmd = ["echo", "this is a command" + foo + bar] p = pcylc(cmd, stdoutpipe=True) stdout, _ = p.communicate() - assert stdout == b"this is a command foo bar\n" + compare(stdout, b"this is a command foo bar\n") def test_sprocess_communicate_with_input(self): command = "a command" From 34a28112504186ca7a4567895eeca6ba44056e39 Mon Sep 17 00:00:00 2001 From: Andrew Clark Date: Fri, 1 Feb 2019 15:44:09 +0000 Subject: [PATCH 49/83] Use higher contrast link colours. --- doc/src/conf.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/doc/src/conf.py b/doc/src/conf.py index df64e80c0f9..5afa4bacd44 100644 --- a/doc/src/conf.py +++ b/doc/src/conf.py @@ -72,17 +72,16 @@ html_theme_options = { "stickysidebar": True, "sidebarwidth": 250, - "relbarbgcolor": "black", "footerbgcolor": "black", "sidebarbgcolor": "white", "sidebartextcolor": "black", - "sidebarlinkcolor": "#00B3FD", + "sidebarlinkcolor": "#0000EE;", "headbgcolor": "white", "headtextcolor": "#FF5966", - "linkcolor": "#00C697", - "visitedlinkcolor": "#00C697", - "headlinkcolor": "#00C697", + "linkcolor": "#0000EE;", + "visitedlinkcolor": "#551A8B;", + "headlinkcolor": "#0000EE;", "codebgcolor": "#ebf9f6", } From 154b3e85b69fe731fe0cf0e418189afb40ad90b8 Mon Sep 17 00:00:00 2001 From: Martin Ryan Date: Mon, 11 Feb 2019 17:02:42 +1100 Subject: [PATCH 50/83] removed testfixtures line --- doc/src/cylc-user-guide/cug.tex | 1 - 1 file changed, 1 deletion(-) diff --git a/doc/src/cylc-user-guide/cug.tex b/doc/src/cylc-user-guide/cug.tex index 575d9f16f37..23016174f75 100644 --- a/doc/src/cylc-user-guide/cug.tex +++ b/doc/src/cylc-user-guide/cug.tex @@ -426,7 +426,6 @@ \subsection{Third-Party Software Packages} \begin{myitemize} \item {\bf mock} - \url{https://mock.readthedocs.io} - \item {\bf testfixtures} - \url{https://testfixtures.readthedocs.io} \end{myitemize} The User Guide is generated from \LaTeX source files by running From 8cbb853ae59bff0642103ef41b1cca6d4ecbabab Mon Sep 17 00:00:00 2001 From: Sadie Bartholomew Date: Tue, 12 Feb 2019 11:54:57 +0000 Subject: [PATCH 51/83] Fixes: top-level makefile command --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index fc280d57691..a5b4d22cfe0 100644 --- a/Makefile +++ b/Makefile @@ -20,7 +20,7 @@ version: etc/dev-bin/create-version-file documentation: - cd doc && $(MAKE) + cylc make-docs clean: cd doc && $(MAKE) clean From 8e4b902a142d21151d00f9c4939a728790fc7150 Mon Sep 17 00:00:00 2001 From: Martin Ryan Date: Wed, 13 Feb 2019 13:49:59 +1100 Subject: [PATCH 52/83] remrun cylc.remote.remrun() revert position to top --- bin/cylc-get-host-metrics | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/bin/cylc-get-host-metrics b/bin/cylc-get-host-metrics index 47b1c19f542..d757a7c4f82 100755 --- a/bin/cylc-get-host-metrics +++ b/bin/cylc-get-host-metrics @@ -35,18 +35,19 @@ keys as requested via the OPTIONS: If no options are specified, --load and --memory are invoked by default. """ +import sys +if '--use-ssh' in sys.argv[1:]: # noqa: E402 + sys.argv.remove('--use-ssh') + from cylc.remote import remrun + if remrun(): + sys.exit(0) + import os import re from subprocess import Popen, PIPE, CalledProcessError import json from cylc.option_parsers import OptionParser -import sys -if '--use-ssh' in sys.argv[1:]: - sys.argv.remove('--use-ssh') - from cylc.remote import remrun - if remrun(): - sys.exit(0) """Templates for extracting desired metric data via regular expression. From 92f09ec9361452b2559cfbf65b243379cdecd534 Mon Sep 17 00:00:00 2001 From: Martin Ryan Date: Wed, 13 Feb 2019 14:30:52 +1100 Subject: [PATCH 53/83] spurious blank line removed --- bin/cylc-check-software | 1 - 1 file changed, 1 deletion(-) diff --git a/bin/cylc-check-software b/bin/cylc-check-software index 12501abede3..27574c5e910 100755 --- a/bin/cylc-check-software +++ b/bin/cylc-check-software @@ -203,7 +203,6 @@ def tex_module_search(tex_module, write=True): process = pcylc(cmd, stdin=open(os.devnull), stdoutpipe=True) check_call(['test', '-n', process.communicate()[0].strip()], stdin=open(os.devnull), stdout=PIPE, stderr=PIPE) - except (CalledProcessError, OSError): if write: shell_align_write('.', msg, NOTFOUND_MSG) From 158d01ae6ae180f11dbe205581da03697caf8e3c Mon Sep 17 00:00:00 2001 From: Martin Ryan Date: Wed, 13 Feb 2019 15:16:20 +1100 Subject: [PATCH 54/83] refactor naming using cylc_subproc and procopen instead of sprocess and pcylc --- bin/cylc-check-software | 16 +++++++------- bin/cylc-check-versions | 6 ++--- lib/cylc/batch_sys_manager.py | 22 +++++++++---------- lib/cylc/{sprocess.py => cylc_subproc.py} | 12 +++++----- lib/cylc/gui/app_gcylc.py | 17 +++++++------- lib/cylc/gui/gcapture.py | 10 ++++----- lib/cylc/profiling/profile.py | 11 +++++----- lib/cylc/run_get_stdout.py | 8 +++---- lib/cylc/subprocpool.py | 6 ++--- ...{test_sprocess.py => test_cylc_subproc.py} | 6 ++--- tests/broadcast/00-simple/bin/complog.py | 4 ++-- tests/cylc-cat-state/basic/state-check.py | 4 ++-- 12 files changed, 63 insertions(+), 59 deletions(-) rename lib/cylc/{sprocess.py => cylc_subproc.py} (83%) rename lib/cylc/tests/{test_sprocess.py => test_cylc_subproc.py} (96%) diff --git a/bin/cylc-check-software b/bin/cylc-check-software index 27574c5e910..01405405ce9 100755 --- a/bin/cylc-check-software +++ b/bin/cylc-check-software @@ -37,7 +37,7 @@ import re import sys from subprocess import PIPE, CalledProcessError, check_call -from cylc.sprocess import pcylc +from cylc.cylc_subproc import procopen # Standardised output messages FOUND_NOVER_MSG = 'FOUND' @@ -93,7 +93,7 @@ opt_result = {} def output_width(min_width=65, max_width=90): """Return a suitable output alignment width given user terminal width.""" - proc = pcylc(['stty', 'size'], stdoutpipe=True) + proc = procopen(['stty', 'size'], stdoutpipe=True) if proc.wait(): return int((min_width + max_width) / 2) else: @@ -200,7 +200,7 @@ def tex_module_search(tex_module, write=True): cmd = ['kpsewhich', '%s.sty' % tex_module] # This is less intensive & quicker than searching via 'find' or 'locate'. try: - process = pcylc(cmd, stdin=open(os.devnull), stdoutpipe=True) + process = procopen(cmd, stdin=open(os.devnull), stdoutpipe=True) check_call(['test', '-n', process.communicate()[0].strip()], stdin=open(os.devnull), stdout=PIPE, stderr=PIPE) except (CalledProcessError, OSError): @@ -223,14 +223,14 @@ def cmd_find_ver(module, min_ver, cmd_base, ver_opt, ver_extr, outfile=1, min_ver is not None else 'any') for cmd in cmd_base: try_next_cmd = True - if pcylc(['which', cmd], stdin=open(os.devnull), - stdoutpipe=True, stderrpipe=True).wait(): + if procopen(['which', cmd], stdin=open(os.devnull), + stdoutpipe=True, stderrpipe=True).wait(): res = [NOTFOUND_MSG, False] else: try: - output = pcylc([cmd, ver_opt], stdoutpipe=True, - stdin=open(os.devnull), - stderrpipe=True).communicate() + output = procopen([cmd, ver_opt], stdoutpipe=True, + stdin=open(os.devnull), + stderrpipe=True).communicate() [outfile - 1].strip() version = re.search(ver_extr, output).groups()[0] try_next_cmd = False diff --git a/bin/cylc-check-versions b/bin/cylc-check-versions index 85ef0f29b36..d3130e8efad 100755 --- a/bin/cylc-check-versions +++ b/bin/cylc-check-versions @@ -42,7 +42,7 @@ from time import sleep import cylc.flags from cylc.config import SuiteConfig from cylc.option_parsers import CylcOptionParser as COP -from cylc.sprocess import pcylc +from cylc.cylc_subproc import procopen from cylc.subprocpool import SuiteProcPool from cylc.suite_srv_files_mgr import SuiteSrvFilesManager from cylc.task_remote_mgr import TaskRemoteMgr @@ -113,8 +113,8 @@ def main(): user_at_host = host if verbose: print "%s: %s" % (user_at_host, ' '.join(argv)) - proc = pcylc(argv, stdin=open(os.devnull), stdoutpipe=True, - stderrpipe=True) + proc = procopen(argv, stdin=open(os.devnull), stdoutpipe=True, + stderrpipe=True) out, err = proc.communicate() if proc.wait() == 0: if verbose: diff --git a/lib/cylc/batch_sys_manager.py b/lib/cylc/batch_sys_manager.py index 13183b652ce..9e2b16adc77 100644 --- a/lib/cylc/batch_sys_manager.py +++ b/lib/cylc/batch_sys_manager.py @@ -117,7 +117,7 @@ from signal import SIGKILL from cylc.mkdir_p import mkdir_p -from cylc.sprocess import pcylc +from cylc.cylc_subproc import procopen from cylc.task_job_logs import (JOB_LOG_ERR, JOB_LOG_JOB, JOB_LOG_OUT, JOB_LOG_STATUS) from cylc.task_message import (CYLC_JOB_EXIT, CYLC_JOB_EXIT_TIME, @@ -416,8 +416,8 @@ def job_kill(self, st_file_path): command = shlex.split( batch_sys.KILL_CMD_TMPL % {"job_id": job_id}) try: - proc = pcylc(command, stdin=open(os.devnull), - stderrpipe=True) + proc = procopen(command, stdin=open(os.devnull), + stderrpipe=True) except OSError as exc: # subprocess.Popen has a bad habit of not setting the # filename of the executable when it raises an OSError. @@ -549,8 +549,8 @@ def _jobs_poll_batch_sys(self, job_log_root, batch_sys_name, my_ctx_list): # Simple poll command that takes a list of job IDs cmd = [batch_sys.POLL_CMD] + exp_ids try: - proc = pcylc(cmd, stdin=open(os.devnull), - stderrpipe=True, stdoutpipe=True) + proc = procopen(cmd, stdin=open(os.devnull), + stderrpipe=True, stdoutpipe=True) except OSError as exc: # subprocess.Popen has a bad habit of not setting the # filename of the executable when it raises an OSError. @@ -655,17 +655,17 @@ def _job_submit_impl( # that we do not have a shell, and still manage to get as far # as here. batch_sys_cmd = batch_submit_cmd_tmpl % {"job": job_file_path} - proc = pcylc(batch_sys_cmd, stdin=proc_stdin_arg, - stdoutpipe=True, stderrpipe=True, usesh=True, - env=env) + proc = procopen(batch_sys_cmd, stdin=proc_stdin_arg, + stdoutpipe=True, stderrpipe=True, usesh=True, + env=env) # calls to open a shell are aggregated in - # sprocess.pcylc() + # cylc_subproc.procopen() else: command = shlex.split( batch_sys.SUBMIT_CMD_TMPL % {"job": job_file_path}) try: - proc = pcylc(command, stdin=proc_stdin_arg, - stdoutpipe=True, stderrpipe=True, env=env) + proc = procopen(command, stdin=proc_stdin_arg, + stdoutpipe=True, stderrpipe=True, env=env) except OSError as exc: # subprocess.Popen has a bad habit of not setting the # filename of the executable when it raises an OSError. diff --git a/lib/cylc/sprocess.py b/lib/cylc/cylc_subproc.py similarity index 83% rename from lib/cylc/sprocess.py rename to lib/cylc/cylc_subproc.py index 219d327ad1d..a924d0b7bdf 100644 --- a/lib/cylc/sprocess.py +++ b/lib/cylc/cylc_subproc.py @@ -26,15 +26,17 @@ from shlex import split from subprocess import PIPE, STDOUT, Popen # nosec +from cylc import LOG + # pylint: disable=too-many-arguments # pylint: disable=too-many-locals -def pcylc(cmd, bufsize=0, executable=None, stdin=None, stdout=None, - stderr=None, preexec_fn=None, close_fds=False, usesh=False, - cwd=None, env=None, universal_newlines=False, startupinfo=None, - creationflags=0, splitcmd=False, stdoutpipe=False, stdoutout=False, - stderrpipe=False, stderrout=False): +def procopen(cmd, bufsize=0, executable=None, stdin=None, stdout=None, + stderr=None, preexec_fn=None, close_fds=False, usesh=False, + cwd=None, env=None, universal_newlines=False, startupinfo=None, + creationflags=0, splitcmd=False, stdoutpipe=False, + stdoutout=False, stderrpipe=False, stderrout=False): shell = usesh diff --git a/lib/cylc/gui/app_gcylc.py b/lib/cylc/gui/app_gcylc.py index 17158ae1b51..b3d250fcf3a 100644 --- a/lib/cylc/gui/app_gcylc.py +++ b/lib/cylc/gui/app_gcylc.py @@ -44,7 +44,7 @@ from cylc.gui.warning_dialog import info_dialog, warning_dialog from cylc.hostuserutil import is_remote_host, is_remote_user from cylc.network.httpclient import ClientError -from cylc.sprocess import pcylc +from cylc.cylc_subproc import procopen from cylc.suite_srv_files_mgr import SuiteSrvFilesManager from cylc.suite_status import SUITE_STATUS_STOPPED_WITH from cylc.task_id import TaskID @@ -74,9 +74,9 @@ def run_get_stdout(command, filter_=False): try: - proc = pcylc(command, usesh=True, stdoutpipe=True, stderrpipe=True, + proc = procopen(command, usesh=True, stdoutpipe=True, stderrpipe=True, stdin=open(os.devnull), splitcmd=True) - # calls to open a shell are aggregated in sprocess.pcylc() + # calls to open a shell are aggregated in cylc_subproc.procopen() out = proc.stdout.read() err = proc.stderr.read() res = proc.wait() @@ -993,7 +993,7 @@ def click_open(self, widget, new_window=False): # process can detach as a process group leader and not subjected to # SIGHUP from the current process. # See also "cylc.batch_sys_handlers.background". - pcylc( + procopen( [ "nohup", "bash", @@ -1201,8 +1201,9 @@ def startsuite(self, bt, window, coldstart_rb, warmstart_rb, restart_rb, pass # Cannot print to terminal (session may be closed). try: - pcylc(command, usesh=True, stdin=open(os.devnull), splitcmd=True) - # calls to open a shell are aggregated in sprocess.pcylc() + procopen(command, usesh=True, stdin=open(os.devnull), + splitcmd=True) + # calls to open a shell are aggregated in cylc_subproc.procopen() except OSError: warning_dialog('Error: failed to start ' + self.cfg.suite, self.window).warn() @@ -2852,7 +2853,7 @@ def construct_command_menu(self, menu): cat_menu.append(cylc_help_item) cylc_help_item.connect('activate', self.command_help) - cout = pcylc( + cout = procopen( ["cylc", "categories"], stdin=open(os.devnull), stdoutpipe=True).communicate()[0] categories = cout.rstrip().split() @@ -2861,7 +2862,7 @@ def construct_command_menu(self, menu): cat_menu.append(foo_item) com_menu = gtk.Menu() foo_item.set_submenu(com_menu) - cout = pcylc( + cout = procopen( ["cylc-help", "category=" + category], stdin=open(os.devnull), stdoutpipe=True).communicate()[0] commands = cout.rstrip().split() diff --git a/lib/cylc/gui/gcapture.py b/lib/cylc/gui/gcapture.py index f5d6850f972..3289ad4ad28 100644 --- a/lib/cylc/gui/gcapture.py +++ b/lib/cylc/gui/gcapture.py @@ -26,7 +26,7 @@ from cylc.gui.tailer import Tailer from cylc.gui.util import get_icon from cylc.gui.warning_dialog import info_dialog, warning_dialog -from cylc.sprocess import pcylc +from cylc.cylc_subproc import procopen class Gcapture(object): @@ -130,10 +130,10 @@ def __init__(self, command, tmpdir, width=400, height=400, self.window.show() def run(self): - proc = pcylc(self.command, stdin=open(os.devnull), - stdout=self.stdoutfile, stderrout=True, - usesh=True, splitcmd=True) - # calls to open a shell are aggregated in sprocess.pcylc() + proc = procopen(self.command, stdin=open(os.devnull), + stdout=self.stdoutfile, stderrout=True, + usesh=True, splitcmd=True) + # calls to open a shell are aggregated in cylc_subproc.procopen() self.proc = proc gobject.timeout_add(40, self.pulse_proc_progress) tail_cmd_tmpl = glbl_cfg().get_host_item("tail command template") diff --git a/lib/cylc/profiling/profile.py b/lib/cylc/profiling/profile.py index 3ed292cc8c6..95a052bca75 100644 --- a/lib/cylc/profiling/profile.py +++ b/lib/cylc/profiling/profile.py @@ -27,7 +27,7 @@ import traceback from subprocess import PIPE, Popen, call # nosec -from cylc.sprocess import pcylc +from cylc.cylc_subproc import procopen from . import (PROFILE_FILES, PROFILE_MODE_CYLC, PROFILE_MODE_TIME, PROFILE_MODES, SUITE_STARTUP_STRING) @@ -190,10 +190,11 @@ def run_suite(reg, options, out_file, profile_modes, mode='live', print '$ ' + ' '.join(cmds) try: - proc = pcylc([' '.join(cmds)], usesh=True, stderr=open(time_err, 'w+'), - # calls to open a shell are aggregated in - # sprocess.pcylc() - stdout=open(startup_file, 'w+'), env=env) + proc = procopen([' '.join(cmds)], usesh=True, + stderr=open(time_err, 'w+'), + # calls to open a shell are aggregated in + # cylc_subproc.procopen() + stdout=open(startup_file, 'w+'), env=env) if proc.wait(): raise SuiteFailedException(run_cmds, cmd_out, cmd_err) except KeyboardInterrupt: diff --git a/lib/cylc/run_get_stdout.py b/lib/cylc/run_get_stdout.py index 1dede312f18..8c9f2f532ca 100644 --- a/lib/cylc/run_get_stdout.py +++ b/lib/cylc/run_get_stdout.py @@ -22,7 +22,7 @@ from signal import SIGTERM from time import sleep, time -from cylc.sprocess import pcylc +from cylc.cylc_subproc import procopen ERR_TIMEOUT = "ERROR: command timed out (>%ds), terminated by signal %d\n%s" @@ -45,9 +45,9 @@ def run_get_stdout(command, timeout=None, poll_delay=None): """ try: - proc = pcylc(command, usesh=True, preexec_fn=setpgrp, - stdin=open(devnull), stderrpipe=True, stdoutpipe=True) - # calls to open a shell are aggregated in sprocess.pcylc() + proc = procopen(command, usesh=True, preexec_fn=setpgrp, + stdin=open(devnull), stderrpipe=True, stdoutpipe=True) + # calls to open a shell are aggregated in cylc_subproc.procopen() is_killed_after_timeout = False if timeout: if poll_delay is None: diff --git a/lib/cylc/subprocpool.py b/lib/cylc/subprocpool.py index 7bdd50d0f65..35fceb41cba 100644 --- a/lib/cylc/subprocpool.py +++ b/lib/cylc/subprocpool.py @@ -29,7 +29,7 @@ from cylc import LOG from cylc.cfgspec.glbl_cfg import glbl_cfg -from cylc.sprocess import pcylc +from cylc.cylc_subproc import procopen from cylc.wallclock import get_current_time_string _XTRIG_FUNCS = {} @@ -308,14 +308,14 @@ def _run_command_init(cls, ctx, callback=None, callback_args=None): stdin_file.seek(0) else: stdin_file = open(os.devnull) - proc = pcylc( + proc = procopen( ctx.cmd, stdin=stdin_file, stdoutpipe=True, stderrpipe=True, # Execute command as a process group leader, # so we can use "os.killpg" to kill the whole group. preexec_fn=os.setpgrp, env=ctx.cmd_kwargs.get('env'), usesh=ctx.cmd_kwargs.get('shell')) - # calls to open a shell are aggregated in sprocess.pcylc() + # calls to open a shell are aggregated in cylc_subproc.procopen() # with logging for what is calling it and the commands given except (IOError, OSError) as exc: if exc.filename is None: diff --git a/lib/cylc/tests/test_sprocess.py b/lib/cylc/tests/test_cylc_subproc.py similarity index 96% rename from lib/cylc/tests/test_sprocess.py rename to lib/cylc/tests/test_cylc_subproc.py index 4f7231ec883..5b8a67922e7 100644 --- a/lib/cylc/tests/test_sprocess.py +++ b/lib/cylc/tests/test_cylc_subproc.py @@ -18,7 +18,7 @@ import unittest -from cylc.sprocess import pcylc +from cylc.cylc_subproc import procopen from mock import call from testfixtures import compare from testfixtures.popen import PIPE, MockPopen @@ -28,7 +28,7 @@ class TestSubprocessSafe(unittest.TestCase): - """Unit tests for the parameter sprocess utility function""" + """Unit tests for the parameter procopen utility function""" def setUp(self): self.Popen = MockPopen() @@ -37,7 +37,7 @@ def test_sprocess_communicate_with_process(self): foo = ' foo' bar = ' bar' cmd = ["echo", "this is a command" + foo + bar] - p = pcylc(cmd, stdoutpipe=True) + p = procopen(cmd, stdoutpipe=True) stdout, _ = p.communicate() compare(stdout, b"this is a command foo bar\n") diff --git a/tests/broadcast/00-simple/bin/complog.py b/tests/broadcast/00-simple/bin/complog.py index 31f9b9bbf02..89904d6267c 100755 --- a/tests/broadcast/00-simple/bin/complog.py +++ b/tests/broadcast/00-simple/bin/complog.py @@ -5,7 +5,7 @@ import sys from distutils import command -from cylc.sprocess import pcylc +from cylc.cylc_subproc import procopen print print "This is the broadcast test suite log comparator" @@ -37,7 +37,7 @@ else: print "broadcast logs compare OK" -res = pcylc(["cylc check-triggering " + event + " " + suite], usesh=True) +res = procopen(["cylc check-triggering " + event + " " + suite], usesh=True) status = res.wait() if status != 0: sys.exit(1) diff --git a/tests/cylc-cat-state/basic/state-check.py b/tests/cylc-cat-state/basic/state-check.py index 88f55bcf280..32a9beaa2c6 100644 --- a/tests/cylc-cat-state/basic/state-check.py +++ b/tests/cylc-cat-state/basic/state-check.py @@ -18,7 +18,7 @@ import os import sqlite3 import sys -from cylc.sprocess import pcylc +from cylc.cylc_subproc import propen import shlex @@ -32,7 +32,7 @@ def main(argv): rundir = argv[1] command = "cylc cat-state " + sname - p = pcylc(command, usesh=True, stdoutpipe=True, stderrpipe=True) + p = procopen(command, usesh=True, stdoutpipe=True, stderrpipe=True) state, err = p.communicate() if p.returncode > 0: From 75e8427ab90bd09830935f2cd45819996e1d6827 Mon Sep 17 00:00:00 2001 From: Martin Ryan Date: Wed, 13 Feb 2019 15:28:17 +1100 Subject: [PATCH 55/83] fix typo --- tests/cylc-cat-state/basic/state-check.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/cylc-cat-state/basic/state-check.py b/tests/cylc-cat-state/basic/state-check.py index 32a9beaa2c6..c4620b01f6d 100644 --- a/tests/cylc-cat-state/basic/state-check.py +++ b/tests/cylc-cat-state/basic/state-check.py @@ -18,7 +18,7 @@ import os import sqlite3 import sys -from cylc.cylc_subproc import propen +from cylc.cylc_subproc import procopen import shlex From 8d6eee731a98b9a7907983fb9fd5754a8f48b495 Mon Sep 17 00:00:00 2001 From: Martin Ryan Date: Wed, 13 Feb 2019 15:39:11 +1100 Subject: [PATCH 56/83] remove blank line kick build! --- lib/cylc/cylc_subproc.py | 1 - 1 file changed, 1 deletion(-) diff --git a/lib/cylc/cylc_subproc.py b/lib/cylc/cylc_subproc.py index a924d0b7bdf..b602f5775c0 100644 --- a/lib/cylc/cylc_subproc.py +++ b/lib/cylc/cylc_subproc.py @@ -27,7 +27,6 @@ from subprocess import PIPE, STDOUT, Popen # nosec from cylc import LOG - # pylint: disable=too-many-arguments # pylint: disable=too-many-locals From 25361af2dc2824476e902d70d017b80b8aa87a75 Mon Sep 17 00:00:00 2001 From: Martin Ryan Date: Wed, 13 Feb 2019 15:43:16 +1100 Subject: [PATCH 57/83] correct year --- tests/cylc-cat-state/basic/state-check.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/cylc-cat-state/basic/state-check.py b/tests/cylc-cat-state/basic/state-check.py index c4620b01f6d..fac319f65fa 100644 --- a/tests/cylc-cat-state/basic/state-check.py +++ b/tests/cylc-cat-state/basic/state-check.py @@ -1,6 +1,6 @@ # THIS FILE IS PART OF THE CYLC SUITE ENGINE. -# Copyright (C) 2008-2018 NIWA & British Crown (Met Office) & Contributors. -# +# Copyright (C) 2008-2019 NIWA & British Crown (Met Office) & Contributors. +# # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or From 676e8c9f10df87056ac2408e4834831ae32ed687 Mon Sep 17 00:00:00 2001 From: Martin Ryan Date: Wed, 13 Feb 2019 16:03:30 +1100 Subject: [PATCH 58/83] removed a noqa I removed a noqa E402 to debug travis issue --- bin/cylc-get-host-metrics | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/cylc-get-host-metrics b/bin/cylc-get-host-metrics index d757a7c4f82..4087e243ca2 100755 --- a/bin/cylc-get-host-metrics +++ b/bin/cylc-get-host-metrics @@ -36,7 +36,7 @@ If no options are specified, --load and --memory are invoked by default. """ import sys -if '--use-ssh' in sys.argv[1:]: # noqa: E402 +if '--use-ssh' in sys.argv[1:]: sys.argv.remove('--use-ssh') from cylc.remote import remrun if remrun(): From fcf661e269c9afa32e966a4d9e0aec0ecb10ba1f Mon Sep 17 00:00:00 2001 From: Martin Ryan Date: Wed, 13 Feb 2019 16:07:06 +1100 Subject: [PATCH 59/83] replace noqa E402 --- bin/cylc-get-host-metrics | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/cylc-get-host-metrics b/bin/cylc-get-host-metrics index 4087e243ca2..d757a7c4f82 100755 --- a/bin/cylc-get-host-metrics +++ b/bin/cylc-get-host-metrics @@ -36,7 +36,7 @@ If no options are specified, --load and --memory are invoked by default. """ import sys -if '--use-ssh' in sys.argv[1:]: +if '--use-ssh' in sys.argv[1:]: # noqa: E402 sys.argv.remove('--use-ssh') from cylc.remote import remrun if remrun(): From 82e4a5951f43b7110069c48260690b0c996508dc Mon Sep 17 00:00:00 2001 From: Martin Ryan Date: Wed, 13 Feb 2019 16:15:55 +1100 Subject: [PATCH 60/83] noqas not ideal, but to debug travis pycodestyle error --- bin/cylc-get-host-metrics | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/bin/cylc-get-host-metrics b/bin/cylc-get-host-metrics index d757a7c4f82..454d17eed1b 100755 --- a/bin/cylc-get-host-metrics +++ b/bin/cylc-get-host-metrics @@ -36,18 +36,18 @@ If no options are specified, --load and --memory are invoked by default. """ import sys -if '--use-ssh' in sys.argv[1:]: # noqa: E402 +if '--use-ssh' in sys.argv[1:]: sys.argv.remove('--use-ssh') from cylc.remote import remrun if remrun(): sys.exit(0) -import os -import re -from subprocess import Popen, PIPE, CalledProcessError -import json +import os # noqa E402 +import re # noqa E402 +from subprocess import Popen, PIPE, CalledProcessError # noqa E402 +import json # noqa E402 -from cylc.option_parsers import OptionParser +from cylc.option_parsers import OptionParser # noqa E402 """Templates for extracting desired metric data via regular expression. From 1357549cada43c86dc4c8edca2b1db2e6c0dfb13 Mon Sep 17 00:00:00 2001 From: Martin Ryan Date: Wed, 13 Feb 2019 16:19:44 +1100 Subject: [PATCH 61/83] fix under indent --- lib/cylc/gui/app_gcylc.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/cylc/gui/app_gcylc.py b/lib/cylc/gui/app_gcylc.py index b3d250fcf3a..a634c6d3c4c 100644 --- a/lib/cylc/gui/app_gcylc.py +++ b/lib/cylc/gui/app_gcylc.py @@ -75,7 +75,7 @@ def run_get_stdout(command, filter_=False): try: proc = procopen(command, usesh=True, stdoutpipe=True, stderrpipe=True, - stdin=open(os.devnull), splitcmd=True) + stdin=open(os.devnull), splitcmd=True) # calls to open a shell are aggregated in cylc_subproc.procopen() out = proc.stdout.read() err = proc.stderr.read() From 01a3dc40213788f9e7f4e97a4c08cb375565a95f Mon Sep 17 00:00:00 2001 From: Martin Ryan Date: Wed, 13 Feb 2019 16:30:11 +1100 Subject: [PATCH 62/83] removed E402 noqas these should be covered by the travis.yml --ignore E402 statement --- bin/cylc-get-host-metrics | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/bin/cylc-get-host-metrics b/bin/cylc-get-host-metrics index 454d17eed1b..4087e243ca2 100755 --- a/bin/cylc-get-host-metrics +++ b/bin/cylc-get-host-metrics @@ -42,12 +42,12 @@ if '--use-ssh' in sys.argv[1:]: if remrun(): sys.exit(0) -import os # noqa E402 -import re # noqa E402 -from subprocess import Popen, PIPE, CalledProcessError # noqa E402 -import json # noqa E402 +import os +import re +from subprocess import Popen, PIPE, CalledProcessError +import json -from cylc.option_parsers import OptionParser # noqa E402 +from cylc.option_parsers import OptionParser """Templates for extracting desired metric data via regular expression. From 1832035c5bf034e24551950cb01b8589ca0d0128 Mon Sep 17 00:00:00 2001 From: Martin Ryan Date: Thu, 14 Feb 2019 09:39:58 +1100 Subject: [PATCH 63/83] using master version of .travis.yml --- .travis.yml | 51 +++++++++++---------------------------------------- 1 file changed, 11 insertions(+), 40 deletions(-) diff --git a/.travis.yml b/.travis.yml index d94b3cdc491..d7f0700b62e 100644 --- a/.travis.yml +++ b/.travis.yml @@ -47,55 +47,26 @@ env: - CYLC_TEST_RUN_PLATFORM=false # Custom diff command to ignore Xlib errors (xvfb has not RANDR extension). - CYLC_TEST_DIFF_CMD="diff -I Xlib -u" - - COVERAGE_PROCESS_START="${TRAVIS_BUILD_DIR}/.coveragerc" + # This coverage RC file is created under the script task + - COVERAGE_PROCESS_START="/tmp/.coveragerc" matrix: - CHUNK="1/4" - CHUNK="2/4" - CHUNK="3/4" - CHUNK="4/4" -# General environment setup before we start installing stuff -before_install: - # Clear bashrc - the default does nothing if not in an interactive shell. - # SSH connections use the ~/.bashrc file for their environment, so we'll be - # loading our python environment here. - - echo > ~/.bashrc - - # Setup virtualenv (using system packages for pygtk as pip won't install it) - - virtualenv --system-site-packages $HOME/virtualenv/cylc - - echo "source $HOME/virtualenv/cylc/bin/activate" >> ~/.bashrc - - # Make sure Cylc is in PATH when running jobs - - echo "export PATH=$PWD/bin:\$PATH" >> ~/.bashrc - - # Load our new environment - - source ~/.bashrc - -# These commands are run before the test -install: - # Setup local SSH for Cylc jobs - - ssh-keygen -t rsa -f ~/.ssh/id_rsa -N "" -q - - cat ~/.ssh/id_rsa.pub >> ~/.ssh/authorized_keys - - ssh-keyscan -t rsa localhost >> ~/.ssh/known_hosts - - # Install dependencies - - sudo apt-get install build-essential texlive-latex-base - - sudo apt-get install at python-pip python-dev graphviz libgraphviz-dev python-gtk2-dev - heirloom-mailx - # Pygraphviz needs special treatment to avoid an error from "from . import release" - - pip install pygraphviz --install-option="--include-path=/usr/include/graphviz" --install-option="--library-path=/usr/lib/graphviz/" - - pip install EmPy pycodestyle pyopenssl - # Coverage dependencies - - pip install coverage pytest-cov mock - - export COVERAGE_PROCESS_START="${TRAVIS_BUILD_DIR}/.coveragerc" - # Testfixtures - - pip install testfixtures - -# Run tests # this is the default test stage, which is used for functional-tests install: .travis/install.sh functional-tests docs script: - export PYTHONPATH="${TRAVIS_BUILD_DIR}/.travis" +# When we run cylc commands, there are processes being forked, that get a +# new working directory. As .coveragerc contains relatives paths, it fails +# to produce the correct coverage, unless we use absolute paths. The `sed` +# call below tries to define the data_file, and sources locations for Travis. +- sed -e "s|data_file=.coverage|data_file=${TRAVIS_BUILD_DIR}/.coverage|g; s|./bin|${TRAVIS_BUILD_DIR}/bin|g; s|./lib|${TRAVIS_BUILD_DIR}/lib|g" .coveragerc > /tmp/.coveragerc +# And some tests fail if we touch files in the git working directory, due +# to Cylc's version appearing with the "dirty" suffix. To avoid this, we +# are using a new coveragerc created under the temporary directory. - coverage run .travis/cover.py - unset PYTHONPATH after_script: .travis/after_script.sh @@ -113,4 +84,4 @@ jobs: script: - pycodestyle --ignore=E402,W503,W504 lib/cylc lib/Jinja2Filters/*.py lib/parsec/*.py $(grep -l '#!.*\' bin/*) - PYTHONPATH=$(pwd -P)/lib/ pytest --cov-append --cov=lib/cylc --cov=lib/parsec - after_script: true + after_script: true \ No newline at end of file From f0431f2c0ff630a51f47cc25690a31ddcb8f8054 Mon Sep 17 00:00:00 2001 From: Hilary James Oliver Date: Fri, 15 Feb 2019 09:39:38 +1300 Subject: [PATCH 64/83] Fix missing import in gcylc cfgspec. --- lib/cylc/cfgspec/gcylc.py | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/cylc/cfgspec/gcylc.py b/lib/cylc/cfgspec/gcylc.py index b77fe7c2686..0726274a63a 100644 --- a/lib/cylc/cfgspec/gcylc.py +++ b/lib/cylc/cfgspec/gcylc.py @@ -25,6 +25,7 @@ from parsec.config import ParsecConfig, ItemNotFoundError, itemstr from parsec.upgrade import upgrader from parsec.util import printcfg +from cylc import LOG from cylc.cfgvalidate import ( cylc_config_validate, CylcConfigValidator as VDR, DurationFloat) from cylc.task_state import ( From 6442cb963d3978859c59e2e1a825c1a3cbb8467c Mon Sep 17 00:00:00 2001 From: Martin Ryan Date: Fri, 15 Feb 2019 10:59:40 +1100 Subject: [PATCH 65/83] remove unused import --- lib/cylc/cylc_subproc.py | 1 - 1 file changed, 1 deletion(-) diff --git a/lib/cylc/cylc_subproc.py b/lib/cylc/cylc_subproc.py index b602f5775c0..1330a726c08 100644 --- a/lib/cylc/cylc_subproc.py +++ b/lib/cylc/cylc_subproc.py @@ -26,7 +26,6 @@ from shlex import split from subprocess import PIPE, STDOUT, Popen # nosec -from cylc import LOG # pylint: disable=too-many-arguments # pylint: disable=too-many-locals From 19d51e310bb1c89d160e0a41257f7a7befac4e90 Mon Sep 17 00:00:00 2001 From: Martin Ryan Date: Fri, 15 Feb 2019 11:04:25 +1100 Subject: [PATCH 66/83] removed innaccurate comment --- lib/cylc/cylc_subproc.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/lib/cylc/cylc_subproc.py b/lib/cylc/cylc_subproc.py index 1330a726c08..60c645cd16c 100644 --- a/lib/cylc/cylc_subproc.py +++ b/lib/cylc/cylc_subproc.py @@ -16,8 +16,7 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . -""" A wrapper function to aggregate these calls in one file, with logging for - what is calling it and the commands given +""" A wrapper function to aggregate these calls in one file. Bandit B602: subprocess_popen_with_shell_equals_true https://docs.openstack.org/developer/bandit/plugins/subprocess_popen_with_shell_equals_true.html B605: start_process_with_a_shell From 4f8e4252a9381d0512e1454d2786cabd5cf7ffac Mon Sep 17 00:00:00 2001 From: Martin Ryan Date: Mon, 18 Feb 2019 10:26:04 +1100 Subject: [PATCH 67/83] resolve conflict - remove import --- lib/cylc/review.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/lib/cylc/review.py b/lib/cylc/review.py index 97be73f815d..2bb6822e513 100644 --- a/lib/cylc/review.py +++ b/lib/cylc/review.py @@ -29,7 +29,6 @@ from fnmatch import fnmatch from glob import glob import jinja2 -from jinja2 import select_autoescape import json import mimetypes import os @@ -79,13 +78,13 @@ def __init__(self, *args, **kwargs): if self.host_name and "." in self.host_name: self.host_name = self.host_name.split(".", 1)[0] self.cylc_version = CYLC_VERSION + # Autoescape markup to prevent code injection from user inputs. template_env = jinja2.Environment( - autoescape=select_autoescape( - enabled_extensions=('rc'), - default_for_string=False, - default=True), loader=jinja2.FileSystemLoader( - get_util_home("lib", "cylc", "cylc-review", "template"))) + get_util_home("lib", "cylc", "cylc-review", "template")), + autoescape=jinja2.select_autoescape( + enabled_extensions=('html', 'xml'), default_for_string=True), + ) template_env.filters['urlise'] = self.url2hyperlink self.template_env = template_env From f89fe0e136f95cead23ec1abd83f5045789d5a16 Mon Sep 17 00:00:00 2001 From: Martin Ryan Date: Mon, 18 Feb 2019 10:26:04 +1100 Subject: [PATCH 68/83] Revert "resolve conflict - remove import" This reverts commit 4f8e4252a9381d0512e1454d2786cabd5cf7ffac. --- lib/cylc/review.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/lib/cylc/review.py b/lib/cylc/review.py index 23cf9be934f..4d1c823fc53 100644 --- a/lib/cylc/review.py +++ b/lib/cylc/review.py @@ -29,6 +29,7 @@ from fnmatch import fnmatch from glob import glob import jinja2 +from jinja2 import select_autoescape import json import mimetypes import os @@ -78,13 +79,13 @@ def __init__(self, *args, **kwargs): if self.host_name and "." in self.host_name: self.host_name = self.host_name.split(".", 1)[0] self.cylc_version = CYLC_VERSION - # Autoescape markup to prevent code injection from user inputs. template_env = jinja2.Environment( + autoescape=select_autoescape( + enabled_extensions=('rc'), + default_for_string=False, + default=True), loader=jinja2.FileSystemLoader( - get_util_home("lib", "cylc", "cylc-review", "template")), - autoescape=jinja2.select_autoescape( - enabled_extensions=('html', 'xml'), default_for_string=True), - ) + get_util_home("lib", "cylc", "cylc-review", "template"))) template_env.filters['urlise'] = self.url2hyperlink self.template_env = template_env From c6a0ab517eec5130b3a5695a8781e255c242dd8e Mon Sep 17 00:00:00 2001 From: Martin Ryan Date: Mon, 18 Feb 2019 10:34:22 +1100 Subject: [PATCH 69/83] remove import --- lib/cylc/review.py | 35 ++++++++++++++++++++++++++--------- 1 file changed, 26 insertions(+), 9 deletions(-) diff --git a/lib/cylc/review.py b/lib/cylc/review.py index 4d1c823fc53..cea82de1eb0 100644 --- a/lib/cylc/review.py +++ b/lib/cylc/review.py @@ -29,7 +29,6 @@ from fnmatch import fnmatch from glob import glob import jinja2 -from jinja2 import select_autoescape import json import mimetypes import os @@ -56,9 +55,27 @@ class CylcReviewService(object): """'Cylc Review Service.""" NS = "cylc" - UTIL = "review" - TITLE = "Cylc Review" - + template_env = jinja2.Environment( + autoescape=select_autoescape( + enabled_extensions=('rc'), + default_for_string=False, + default=True), + loader=jinja2.FileSystemLoader( + get_util_home("lib", "cylc", "cylc-review", "template"))) + template_env = jinja2.Environment( + autoescape=select_autoescape( + enabled_extensions=('rc'), + default_for_string=False, + default=True), + loader=jinja2.FileSystemLoader( + get_util_home("lib", "cylc", "cylc-review", "template"))) + template_env = jinja2.Environment( + autoescape=select_autoescape( + enabled_extensions=('rc'), + default_for_string=False, + default=True), + loader=jinja2.FileSystemLoader( + get_util_home("lib", "cylc", "cylc-review", "template"))) CYCLES_PER_PAGE = 100 JOBS_PER_PAGE = 15 JOBS_PER_PAGE_MAX = 300 @@ -79,13 +96,13 @@ def __init__(self, *args, **kwargs): if self.host_name and "." in self.host_name: self.host_name = self.host_name.split(".", 1)[0] self.cylc_version = CYLC_VERSION + # Autoescape markup to prevent code injection from user inputs. template_env = jinja2.Environment( - autoescape=select_autoescape( - enabled_extensions=('rc'), - default_for_string=False, - default=True), loader=jinja2.FileSystemLoader( - get_util_home("lib", "cylc", "cylc-review", "template"))) + get_util_home("lib", "cylc", "cylc-review", "template")), + autoescape=jinja2.select_autoescape( + enabled_extensions=('html', 'xml'), default_for_string=True), + ) template_env.filters['urlise'] = self.url2hyperlink self.template_env = template_env From 46ce95e09c6bda76d353ce6940d59e37bfd095fd Mon Sep 17 00:00:00 2001 From: Martin Ryan Date: Mon, 18 Feb 2019 10:46:21 +1100 Subject: [PATCH 70/83] resolve codacy error --- lib/cylc/review.py | 24 +++--------------------- 1 file changed, 3 insertions(+), 21 deletions(-) diff --git a/lib/cylc/review.py b/lib/cylc/review.py index cea82de1eb0..23cf9be934f 100644 --- a/lib/cylc/review.py +++ b/lib/cylc/review.py @@ -55,27 +55,9 @@ class CylcReviewService(object): """'Cylc Review Service.""" NS = "cylc" - template_env = jinja2.Environment( - autoescape=select_autoescape( - enabled_extensions=('rc'), - default_for_string=False, - default=True), - loader=jinja2.FileSystemLoader( - get_util_home("lib", "cylc", "cylc-review", "template"))) - template_env = jinja2.Environment( - autoescape=select_autoescape( - enabled_extensions=('rc'), - default_for_string=False, - default=True), - loader=jinja2.FileSystemLoader( - get_util_home("lib", "cylc", "cylc-review", "template"))) - template_env = jinja2.Environment( - autoescape=select_autoescape( - enabled_extensions=('rc'), - default_for_string=False, - default=True), - loader=jinja2.FileSystemLoader( - get_util_home("lib", "cylc", "cylc-review", "template"))) + UTIL = "review" + TITLE = "Cylc Review" + CYCLES_PER_PAGE = 100 JOBS_PER_PAGE = 15 JOBS_PER_PAGE_MAX = 300 From 33a4a8a0e8033e664d19a48e831f0a0fb573747c Mon Sep 17 00:00:00 2001 From: Hilary James Oliver Date: Wed, 27 Feb 2019 10:04:41 +1300 Subject: [PATCH 71/83] Fix missing python2 lines. --- bin/cylc-get-host-metrics | 2 +- .../xtrigger/kafka/consumer/lib/python/cylc_kafka_consumer.py | 2 +- .../xtrigger/kafka/producer/bin/cylc_kafka_producer.py | 2 +- lib/cylc/cycling/util.py | 2 +- lib/cylc/review.py | 2 +- lib/cylc/ws.py | 2 +- lib/cylc/xtrigger_mgr.py | 2 +- lib/cylc/xtriggers/echo.py | 2 +- lib/cylc/xtriggers/suite_state.py | 2 +- lib/cylc/xtriggers/wall_clock.py | 2 +- lib/cylc/xtriggers/xrandom.py | 2 +- lib/parsec/empysupport.py | 2 +- tests/xtriggers/02-persistence/faker_fail.py | 2 +- tests/xtriggers/02-persistence/faker_succ.py | 2 +- 14 files changed, 14 insertions(+), 14 deletions(-) diff --git a/bin/cylc-get-host-metrics b/bin/cylc-get-host-metrics index 02af62fd3d8..7221752b057 100755 --- a/bin/cylc-get-host-metrics +++ b/bin/cylc-get-host-metrics @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python2 # THIS FILE IS PART OF THE CYLC SUITE ENGINE. # Copyright (C) 2008-2019 NIWA diff --git a/etc/dev-suites/xtrigger/kafka/consumer/lib/python/cylc_kafka_consumer.py b/etc/dev-suites/xtrigger/kafka/consumer/lib/python/cylc_kafka_consumer.py index d8a3a3df1a8..b326ed8b184 100755 --- a/etc/dev-suites/xtrigger/kafka/consumer/lib/python/cylc_kafka_consumer.py +++ b/etc/dev-suites/xtrigger/kafka/consumer/lib/python/cylc_kafka_consumer.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python2 """ A generic Kakfa consumer for use as a Cylc external trigger function. diff --git a/etc/dev-suites/xtrigger/kafka/producer/bin/cylc_kafka_producer.py b/etc/dev-suites/xtrigger/kafka/producer/bin/cylc_kafka_producer.py index 6d2f7168204..0bf5d99a253 100755 --- a/etc/dev-suites/xtrigger/kafka/producer/bin/cylc_kafka_producer.py +++ b/etc/dev-suites/xtrigger/kafka/producer/bin/cylc_kafka_producer.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python2 """ A generic Kafka producer for use as a Cylc event handler. diff --git a/lib/cylc/cycling/util.py b/lib/cylc/cycling/util.py index 03aed58476a..602f1bfad62 100644 --- a/lib/cylc/cycling/util.py +++ b/lib/cylc/cycling/util.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python2 # THIS FILE IS PART OF THE CYLC SUITE ENGINE. # Copyright (C) 2008-2019 NIWA & British Crown (Met Office) & Contributors. diff --git a/lib/cylc/review.py b/lib/cylc/review.py index e17fbb720a5..03d60a0d03b 100644 --- a/lib/cylc/review.py +++ b/lib/cylc/review.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python2 # THIS FILE IS PART OF THE CYLC SUITE ENGINE. # Copyright (C) 2008-2019 NIWA & British Crown (Met Office) & Contributors. diff --git a/lib/cylc/ws.py b/lib/cylc/ws.py index e0932064a63..050918056fd 100644 --- a/lib/cylc/ws.py +++ b/lib/cylc/ws.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python2 # THIS FILE IS PART OF THE CYLC SUITE ENGINE. # Copyright (C) 2008-2019 NIWA & British Crown (Met Office) & Contributors. diff --git a/lib/cylc/xtrigger_mgr.py b/lib/cylc/xtrigger_mgr.py index a68487889f0..a00721adb51 100644 --- a/lib/cylc/xtrigger_mgr.py +++ b/lib/cylc/xtrigger_mgr.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python2 # THIS FILE IS PART OF THE CYLC SUITE ENGINE. # Copyright (C) 2008-2019 NIWA & British Crown (Met Office) & Contributors. diff --git a/lib/cylc/xtriggers/echo.py b/lib/cylc/xtriggers/echo.py index 7b068eab0d5..a24d6225813 100644 --- a/lib/cylc/xtriggers/echo.py +++ b/lib/cylc/xtriggers/echo.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python2 # THIS FILE IS PART OF THE CYLC SUITE ENGINE. # Copyright (C) 2008-2019 NIWA & British Crown (Met Office) & Contributors. diff --git a/lib/cylc/xtriggers/suite_state.py b/lib/cylc/xtriggers/suite_state.py index 9ecb8c2f9df..b39183a17f7 100644 --- a/lib/cylc/xtriggers/suite_state.py +++ b/lib/cylc/xtriggers/suite_state.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python2 # THIS FILE IS PART OF THE CYLC SUITE ENGINE. # Copyright (C) 2008-2019 NIWA & British Crown (Met Office) & Contributors. diff --git a/lib/cylc/xtriggers/wall_clock.py b/lib/cylc/xtriggers/wall_clock.py index 5eb50d59614..181dde72079 100644 --- a/lib/cylc/xtriggers/wall_clock.py +++ b/lib/cylc/xtriggers/wall_clock.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python2 # THIS FILE IS PART OF THE CYLC SUITE ENGINE. # Copyright (C) 2008-2019 NIWA & British Crown (Met Office) & Contributors. diff --git a/lib/cylc/xtriggers/xrandom.py b/lib/cylc/xtriggers/xrandom.py index b00db165951..596692c9a0e 100644 --- a/lib/cylc/xtriggers/xrandom.py +++ b/lib/cylc/xtriggers/xrandom.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python2 # THIS FILE IS PART OF THE CYLC SUITE ENGINE. # Copyright (C) 2008-2019 NIWA & British Crown (Met Office) & Contributors. diff --git a/lib/parsec/empysupport.py b/lib/parsec/empysupport.py index dc33da73f7d..15f2616c8e8 100644 --- a/lib/parsec/empysupport.py +++ b/lib/parsec/empysupport.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python2 # THIS FILE IS PART OF THE CYLC SUITE ENGINE. # Copyright (C) 2008-2019 NIWA & British Crown (Met Office) & Contributors. diff --git a/tests/xtriggers/02-persistence/faker_fail.py b/tests/xtriggers/02-persistence/faker_fail.py index f456c504262..faf9e64b641 100644 --- a/tests/xtriggers/02-persistence/faker_fail.py +++ b/tests/xtriggers/02-persistence/faker_fail.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python2 def faker(name, debug=False): print "%s: failing" % name diff --git a/tests/xtriggers/02-persistence/faker_succ.py b/tests/xtriggers/02-persistence/faker_succ.py index b702d239be7..e2530e69288 100644 --- a/tests/xtriggers/02-persistence/faker_succ.py +++ b/tests/xtriggers/02-persistence/faker_succ.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python2 def faker(name, debug=False): print "%s: succeeding" % name From 7325985e7893bf200a065e61ab1a1425e94bfdd6 Mon Sep 17 00:00:00 2001 From: Hilary James Oliver Date: Wed, 27 Feb 2019 10:55:33 +1300 Subject: [PATCH 72/83] General single- and multi-page User Guides. --- bin/cylc-make-docs | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/bin/cylc-make-docs b/bin/cylc-make-docs index 0d584528f51..0d12f2f5ae6 100755 --- a/bin/cylc-make-docs +++ b/bin/cylc-make-docs @@ -17,8 +17,7 @@ usage() { echo "Usage: cylc [admin] make-docs [--help]" - echo "This command builds the HTML documentation, which is auto-generated" - echo "with the tool Sphinx. It wraps the 'sphinx-build' command." + echo "Build the HTML User Guide, with Sphinx." echo "" echo "Options:" echo " --help Print this usage message." @@ -42,7 +41,9 @@ echo "... Generating the command reference ..." ./src/custom/make-commands.sh echo >&2 -echo "... Auto-generating the HTML with Sphinx ..." +echo "... Generating multi-page User Guide..." sphinx-build -n -b html ./src built-sphinx/ +echo "... Generating single-page User Guide..." +sphinx-build -n -b singlehtml ./src built-sphinx-single/ echo >&2 echo "Done." From c042ff8a4517e314feb414846400351f66722316 Mon Sep 17 00:00:00 2001 From: Sadie Bartholomew Date: Wed, 27 Feb 2019 16:41:09 +0000 Subject: [PATCH 73/83] Minor suggestions to adapt docs source for single-page HTML --- doc/src/appendices/appendices-master.rst | 19 ++--------- doc/src/appendices/gcylc-graph-view.rst | 19 ++--------- doc/src/appendices/known-issues.rst | 21 ++---------- doc/src/appendices/licensing.rst | 19 ++--------- doc/src/appendices/readme-file.rst | 19 ++--------- doc/src/conf.py | 3 +- doc/src/custom/whitespace_include | 33 +++++++++++++++++++ doc/src/index.rst | 5 ++- .../suite-design-guide-master.rst | 21 ++---------- doc/src/suite-name-reg.rst | 21 ++---------- doc/src/suite-storage-etc.rst | 21 ++---------- doc/src/terminology.rst | 21 ++---------- 12 files changed, 62 insertions(+), 160 deletions(-) create mode 100644 doc/src/custom/whitespace_include diff --git a/doc/src/appendices/appendices-master.rst b/doc/src/appendices/appendices-master.rst index b75734576e2..e71d0b32e83 100644 --- a/doc/src/appendices/appendices-master.rst +++ b/doc/src/appendices/appendices-master.rst @@ -22,21 +22,6 @@ Appendices licensing -.. insert vertical whitespace else sidebar menu overhangs short page (ugly) +.. only:: builder_html -| -| -| -| -| -| -| -| -| -| -| -| -| -| -| -| + .. include:: ../custom/whitespace_include diff --git a/doc/src/appendices/gcylc-graph-view.rst b/doc/src/appendices/gcylc-graph-view.rst index 95b2827aea4..f7a57144ff5 100644 --- a/doc/src/appendices/gcylc-graph-view.rst +++ b/doc/src/appendices/gcylc-graph-view.rst @@ -24,21 +24,6 @@ mitigate any jumping layout problems: structure. -.. insert vertical whitespace else sidebar menu overhangs short page (ugly) +.. only:: builder_html -| -| -| -| -| -| -| -| -| -| -| -| -| -| -| -| + .. include:: ../custom/whitespace_include diff --git a/doc/src/appendices/known-issues.rst b/doc/src/appendices/known-issues.rst index a3eab879b09..9c9a82c1499 100644 --- a/doc/src/appendices/known-issues.rst +++ b/doc/src/appendices/known-issues.rst @@ -39,21 +39,6 @@ The second message is ignored by the suite, and so the behaviour can be safely ignored. (You should probably still investigate the failure, however!) -.. insert vertical whitespace else sidebar menu overhangs short page (ugly) - -| -| -| -| -| -| -| -| -| -| -| -| -| -| -| -| +.. only:: builder_html + + .. include:: ../custom/whitespace_include diff --git a/doc/src/appendices/licensing.rst b/doc/src/appendices/licensing.rst index e12581e819d..13e96516c0a 100644 --- a/doc/src/appendices/licensing.rst +++ b/doc/src/appendices/licensing.rst @@ -4,21 +4,6 @@ GNU GENERAL PUBLIC LICENSE v3.0 See the `GNU GENERAL PUBLIC LICENSE v3.0 `_. -.. insert vertical whitespace else sidebar menu overhangs short page (ugly) +.. only:: builder_html -| -| -| -| -| -| -| -| -| -| -| -| -| -| -| -| + .. include:: ../custom/whitespace_include diff --git a/doc/src/appendices/readme-file.rst b/doc/src/appendices/readme-file.rst index b15184c8b37..38406b44706 100644 --- a/doc/src/appendices/readme-file.rst +++ b/doc/src/appendices/readme-file.rst @@ -4,21 +4,6 @@ Cylc README File .. literalinclude:: ../../../README.md -.. insert vertical whitespace else sidebar menu overhangs short page (ugly) +.. only:: builder_html -| -| -| -| -| -| -| -| -| -| -| -| -| -| -| -| + .. include:: ../custom/whitespace_include diff --git a/doc/src/conf.py b/doc/src/conf.py index 5afa4bacd44..eb0ce1fdc8a 100644 --- a/doc/src/conf.py +++ b/doc/src/conf.py @@ -73,7 +73,8 @@ "stickysidebar": True, "sidebarwidth": 250, "relbarbgcolor": "black", - "footerbgcolor": "black", + "footerbgcolor": "white", # single-page HTML flashes this colour on scroll + "footertextcolor": "black", "sidebarbgcolor": "white", "sidebartextcolor": "black", "sidebarlinkcolor": "#0000EE;", diff --git a/doc/src/custom/whitespace_include b/doc/src/custom/whitespace_include new file mode 100644 index 00000000000..9b68f4016f6 --- /dev/null +++ b/doc/src/custom/whitespace_include @@ -0,0 +1,33 @@ + +.. insert vertical whitespace else sidebar menu overhangs short page (ugly) + +| +| +| +| +| +| +| +| +| +| +| +| +| +| +| +| +| +| +| +| +| +| +| +| +| +| +| +| +| +| diff --git a/doc/src/index.rst b/doc/src/index.rst index 960362885cc..257f3153f4e 100644 --- a/doc/src/index.rst +++ b/doc/src/index.rst @@ -19,7 +19,10 @@ indefinitely. ----------- -**Table of Contents**: + +.. only:: builder_html + + **Table of Contents**: .. toctree:: :maxdepth: 3 diff --git a/doc/src/suite-design-guide/suite-design-guide-master.rst b/doc/src/suite-design-guide/suite-design-guide-master.rst index 73910197222..20d18e2abd5 100644 --- a/doc/src/suite-design-guide/suite-design-guide-master.rst +++ b/doc/src/suite-design-guide/suite-design-guide-master.rst @@ -27,21 +27,6 @@ Version 1.0 - 23 March 2017 ----------- -.. insert vertical whitespace else sidebar menu overhangs short page (ugly) - -| -| -| -| -| -| -| -| -| -| -| -| -| -| -| -| +.. only:: builder_html + + .. include:: ../custom/whitespace_include diff --git a/doc/src/suite-name-reg.rst b/doc/src/suite-name-reg.rst index 3d1e62a8281..3139dd827b0 100644 --- a/doc/src/suite-name-reg.rst +++ b/doc/src/suite-name-reg.rst @@ -22,21 +22,6 @@ which creates the suite run directory structure and some service files underneath it. Otherwise, ``cylc run`` will do this at suite start up. -.. insert vertical whitespace else sidebar menu overhangs short page (ugly) - -| -| -| -| -| -| -| -| -| -| -| -| -| -| -| -| +.. only:: builder_html + + .. include:: custom/whitespace_include diff --git a/doc/src/suite-storage-etc.rst b/doc/src/suite-storage-etc.rst index 5d6bd3b6ab8..94b3065709d 100644 --- a/doc/src/suite-storage-etc.rst +++ b/doc/src/suite-storage-etc.rst @@ -26,21 +26,6 @@ cylc. It is available under the open source GPL license. - `Rose source repository `_ -.. insert vertical whitespace else sidebar menu overhangs short page (ugly) - -| -| -| -| -| -| -| -| -| -| -| -| -| -| -| -| +.. only:: builder_html + + .. include:: custom/whitespace_include diff --git a/doc/src/terminology.rst b/doc/src/terminology.rst index a7c094d21b8..031fb9b7046 100644 --- a/doc/src/terminology.rst +++ b/doc/src/terminology.rst @@ -23,21 +23,6 @@ keep in mind that different tasks may pass through the "current cycle point" (etc.) at different times as the suite evolves. -.. insert vertical whitespace else sidebar menu overhangs short page (ugly) - -| -| -| -| -| -| -| -| -| -| -| -| -| -| -| -| +.. only:: builder_html + + .. include:: custom/whitespace_include From 4e0cdcfeff28023c55aa63f21d39bbf0c26345cb Mon Sep 17 00:00:00 2001 From: "Bruno P. Kinoshita" Date: Mon, 11 Mar 2019 19:37:00 +1300 Subject: [PATCH 74/83] Fix issue #2898: close app nicely on CTRL+C --- bin/cylc-gui | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/bin/cylc-gui b/bin/cylc-gui index 4a40f395ee2..a782957791e 100755 --- a/bin/cylc-gui +++ b/bin/cylc-gui @@ -107,12 +107,15 @@ def main(): print >> sys.stderr, ('USER_AT_HOST must take the form ' '"user@host:port"') sys.exit(1) - ControlApp( + app = ControlApp( suite, options.owner, options.host, options.port, options.comms_timeout, load_template_vars(options.templatevars, options.templatevars_file), options.restricted) - gtk.main() + try: + gtk.main() + except KeyboardInterrupt: + app.quit() if __name__ == "__main__": From 76c3a6a5e72914cbf43d2708eebd66f609970bd7 Mon Sep 17 00:00:00 2001 From: Hilary James Oliver Date: Tue, 12 Mar 2019 15:40:15 +1300 Subject: [PATCH 75/83] Backport #2981 --- lib/cylc/task_events_mgr.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/cylc/task_events_mgr.py b/lib/cylc/task_events_mgr.py index 1acf08ed78c..324cf26d0d4 100644 --- a/lib/cylc/task_events_mgr.py +++ b/lib/cylc/task_events_mgr.py @@ -693,7 +693,7 @@ def _process_message_submit_failed(self, itask, event_time): if event_time is None: event_time = get_current_time_string() self.suite_db_mgr.put_update_task_jobs(itask, { - "time_submit_exit": get_current_time_string(), + "time_submit_exit": event_time, "submit_status": 1, }) itask.summary['submit_method_id'] = None From 8ac7e971b5bb5b462995ebdf4444728b747d069b Mon Sep 17 00:00:00 2001 From: Martin Ryan Date: Wed, 13 Mar 2019 11:51:53 +1100 Subject: [PATCH 76/83] cylc-8 target for bandit remediation --- lib/cylc/tests/test_cylc_subproc.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/cylc/tests/test_cylc_subproc.py b/lib/cylc/tests/test_cylc_subproc.py index 5b8a67922e7..e2f21a3d45b 100644 --- a/lib/cylc/tests/test_cylc_subproc.py +++ b/lib/cylc/tests/test_cylc_subproc.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python2 +#!/usr/bin/env python3 # THIS FILE IS PART OF THE CYLC SUITE ENGINE. # Copyright (C) 2008-2018 NIWA & British Crown (Met Office) & Contributors. From 2cf62047c9730e6dc8844b69ccf423ff1fed4cc3 Mon Sep 17 00:00:00 2001 From: Martin Ryan Date: Wed, 13 Mar 2019 12:13:56 +1100 Subject: [PATCH 77/83] shebang --- lib/cylc/cylc_subproc.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/cylc/cylc_subproc.py b/lib/cylc/cylc_subproc.py index 60c645cd16c..62113062034 100644 --- a/lib/cylc/cylc_subproc.py +++ b/lib/cylc/cylc_subproc.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python2 +#!/usr/bin/env python3 # THIS FILE IS PART OF THE CYLC SUITE ENGINE. # Copyright (C) 2008-2018 NIWA & British Crown (Met Office) & Contributors. From 3e2bd170305100a04449c4ced8efa408a872c566 Mon Sep 17 00:00:00 2001 From: Martin Ryan Date: Wed, 13 Mar 2019 12:47:55 +1100 Subject: [PATCH 78/83] py3 changes for merge --- lib/cylc/profiling/profile.py | 51 +++++++++++------------ lib/cylc/run_get_stdout.py | 4 +- lib/cylc/subprocpool.py | 12 +++--- lib/parsec/jinja2support.py | 4 +- tests/broadcast/00-simple/bin/complog.py | 15 +++---- tests/cylc-cat-state/basic/state-check.py | 28 +++++++------ 6 files changed, 55 insertions(+), 59 deletions(-) diff --git a/lib/cylc/profiling/profile.py b/lib/cylc/profiling/profile.py index 0e72b5de660..1463e4041c4 100644 --- a/lib/cylc/profiling/profile.py +++ b/lib/cylc/profiling/profile.py @@ -73,29 +73,29 @@ def cylc_major_version(): """Return the first character of the cylc version e.g. '7'.""" return Popen( ['cylc', '--version'], env=CLEAN_ENV, stdin=open(os.devnull), - stdout=PIPE).communicate()[0].strip()[0] + stdout=PIPE).communicate()[0].decode().strip()[0] def register_suite(reg, sdir): """Registers the suite located in sdir with the registration name reg.""" cmd = ['cylc', 'register', reg, sdir] - print '$ ' + ' '.join(cmd) + print('$ ' + ' '.join(cmd)) if not call(cmd, stdin=open(os.devnull), stdout=PIPE, env=CLEAN_ENV): return True - print '\tFailed' + print('\tFailed') return False def unregister_suite(reg): """Unregisters the suite reg.""" cmd = ['cylc', 'unregister', reg] - print '$ ' + ' '.join(cmd) + print('$ ' + ' '.join(cmd)) call(cmd, stdin=open(os.devnull), stdout=PIPE, env=CLEAN_ENV) def purge_suite(reg): """Deletes the run directory for this suite.""" - print '$ rm -rf ' + os.path.expanduser(os.path.join('~', 'cylc-run', reg)) + print('$ rm -rf ' + os.path.expanduser(os.path.join('~', 'cylc-run', reg))) try: shutil.rmtree(os.path.expanduser(os.path.join('~', 'cylc-run', reg))) except OSError: @@ -160,7 +160,7 @@ def run_suite(reg, options, out_file, profile_modes, mode='live', namespaces = Popen( ['cylc', 'list', reg] + jinja2_params + tmp, stdin=open(os.devnull), stdout=PIPE, - env=env).communicate()[0].split() + ['root'] + env=env).communicate()[0].decode().split() + ['root'] jinja2_params.append( '-s namespaces={0}'.format(','.join(namespaces))) cmds.extend(jinja2_params) @@ -187,8 +187,7 @@ def run_suite(reg, options, out_file, profile_modes, mode='live', cmds += ["'"] # Close shell. # Execute. - print '$ ' + ' '.join(cmds) - + print('$ ' + ' '.join(cmds)) try: proc = procopen([' '.join(cmds)], usesh=True, stderr=open(time_err, 'w+'), @@ -199,7 +198,7 @@ def run_suite(reg, options, out_file, profile_modes, mode='live', raise SuiteFailedException(run_cmds, cmd_out, cmd_err) except KeyboardInterrupt: kill_cmd = ['cylc', 'stop', '--kill', reg] - print '$ ' + ' '.join(kill_cmd) + print('$ ' + ' '.join(kill_cmd)) call(kill_cmd, env=env, stdin=open(os.devnull)) raise ProfilingKilledException(run_cmds, cmd_out, cmd_err) @@ -241,8 +240,8 @@ def run_experiment(exp): conf_path=run.get('globalrc', '')) # Handle errors. if err_file: - print >> sys.stderr, ('WARNING: non-empty suite error log: ' + - err_file) + print(('WARNING: non-empty suite error log: ' + + err_file), file=sys.stderr) # Tidy up. if cylc_maj_version == '6': unregister_suite(reg) @@ -259,10 +258,9 @@ def run_experiment(exp): to_purge.remove(reg) if to_purge: - print >> sys.stderr, ('ERROR: The following suite(s) run ' - 'directories could not be deleted:\n' - '\t' + ' '.join(to_purge) - ) + print(('ERROR: The following suite(s) run ' + 'directories could not be deleted:\n' + '\t' + ' '.join(to_purge)), file=sys.stderr) return result_files @@ -282,9 +280,8 @@ def profile(schedule): """Perform profiling for the provided schedule. Args: - schedule (list): A list of tuples of the form - [(version_id, experiments)] where experiments is a list of - experiment objects. + schedule (dict): Dictionary of cylc version ids containing lists + of the experiments to run for each. Returns: tuple - (results, checkout_count, success) @@ -298,7 +295,7 @@ def profile(schedule): checkout_count = 0 results = {} success = True - for version_id, experiments in sorted(schedule.iteritems()): + for version_id, experiments in sorted(schedule.items()): # Checkout cylc version. if version_id != describe(): try: @@ -314,13 +311,13 @@ def profile(schedule): result_files = run_experiment(experiment['config']) except ProfilingKilledException as exc: # Profiling has been terminated, return what results we have. - print exc + print(exc) return results, checkout_count, False except SuiteFailedException as exc: # Experiment failed to run, move onto the next one. - print >> sys.stderr, ('Experiment "%s" failed at version "%s"' - '' % (experiment['name'], version_id)) - print >> sys.stderr, exc + print(('Experiment "%s" failed at version "%s"' + '' % (experiment['name'], version_id)), file=sys.stderr) + print(exc, file=sys.stderr) success = False continue else: @@ -334,17 +331,17 @@ def profile(schedule): exp_files = [] for run in result_files: exp_files.extend(result_files[run]) - print >> sys.stderr, ( + print(( 'Analysis failed on results from experiment "%s" ' 'running at version "%s".\n\tProfile files: %s' % ( experiment['name'], version_id, - ' '.join(exp_files))) + ' '.join(exp_files))), file=sys.stderr) if any(PROFILE_MODES[mode] == PROFILE_MODE_CYLC for mode in experiment['config']['profile modes']): - print >> sys.stderr, ( + print(( 'Are you trying to use profile mode "cylc" ' - 'with an older version of cylc?') + 'with an older version of cylc?'), file=sys.stderr) success = False continue else: diff --git a/lib/cylc/run_get_stdout.py b/lib/cylc/run_get_stdout.py index f79fd8bc534..1e13a3fe32b 100644 --- a/lib/cylc/run_get_stdout.py +++ b/lib/cylc/run_get_stdout.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python2 +#!/usr/bin/env python3 # THIS FILE IS PART OF THE CYLC SUITE ENGINE. # Copyright (C) 2008-2019 NIWA & British Crown (Met Office) & Contributors. @@ -59,7 +59,7 @@ def run_get_stdout(command, timeout=None, poll_delay=None): is_killed_after_timeout = True break sleep(poll_delay) - out, err = proc.communicate() + out, err = (f.decode() for f in proc.communicate()) res = proc.wait() if res < 0 and is_killed_after_timeout: return (False, [ERR_TIMEOUT % (timeout, -res, err), command]) diff --git a/lib/cylc/subprocpool.py b/lib/cylc/subprocpool.py index 735e0f2ae44..6f5694c87f0 100644 --- a/lib/cylc/subprocpool.py +++ b/lib/cylc/subprocpool.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python2 +#!/usr/bin/env python3 # THIS FILE IS PART OF THE CYLC SUITE ENGINE. # Copyright (C) 2008-2019 NIWA & British Crown (Met Office) & Contributors. @@ -153,7 +153,7 @@ def _is_stopping(self): def _proc_exit(self, proc, err_xtra, ctx, callback, callback_args): """Get ret_code, out, err of exited command, and call its callback.""" ctx.ret_code = proc.wait() - out, err = proc.communicate() + out, err = (f.decode() for f in proc.communicate()) if out: if ctx.out is None: ctx.out = '' @@ -238,7 +238,7 @@ def run_command(cls, ctx): """ proc = cls._run_command_init(ctx) if proc: - ctx.out, ctx.err = proc.communicate() + ctx.out, ctx.err = (f.decode() for f in proc.communicate()) ctx.ret_code = proc.wait() cls._run_command_exit(ctx) @@ -291,7 +291,7 @@ def _poll_proc_pipes(self, proc, ctx): # 2. Call os.read only once after a poll. Poll again before # another read - otherwise the os.read call may block. try: - data = os.read(fileno, 65536) # 64K + data = os.read(fileno, 65536).decode() # 64K except OSError: continue if fileno == proc.stdout.fileno(): @@ -319,8 +319,8 @@ def _run_command_init(cls, ctx, callback=None, callback_args=None): stdin_file = open( ctx.cmd_kwargs['stdin_file_paths'][0], 'rb') elif ctx.cmd_kwargs.get('stdin_str'): - stdin_file = TemporaryFile() - stdin_file.write(ctx.cmd_kwargs.get('stdin_str')) + stdin_file = TemporaryFile('bw+') + stdin_file.write(ctx.cmd_kwargs.get('stdin_str').encode()) stdin_file.seek(0) else: stdin_file = open(os.devnull) diff --git a/lib/parsec/jinja2support.py b/lib/parsec/jinja2support.py index 7ec693e32f5..7b8b116e558 100644 --- a/lib/parsec/jinja2support.py +++ b/lib/parsec/jinja2support.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python2 +#!/usr/bin/env python3 # THIS FILE IS PART OF THE CYLC SUITE ENGINE. # Copyright (C) 2008-2019 NIWA & British Crown (Met Office) & Contributors. @@ -107,7 +107,7 @@ def jinja2environment(dir_=None): # definition directory # Example: a filter to pad integer values some fill character: # |(file SUITE_DEFINITION_DIRECTORY/Jinja2/foo.py) - # | #!/usr/bin/env python2 + # | #!/usr/bin/env python3 # | def foo( value, length, fillchar ): # | return str(value).rjust( int(length), str(fillchar) ) for namespace in ['filters', 'tests', 'globals']: diff --git a/tests/broadcast/00-simple/bin/complog.py b/tests/broadcast/00-simple/bin/complog.py index 89904d6267c..8c7a4a27209 100755 --- a/tests/broadcast/00-simple/bin/complog.py +++ b/tests/broadcast/00-simple/bin/complog.py @@ -1,15 +1,12 @@ -#!/usr/bin/env python2 +#!/usr/bin/env python3 import os -import shlex import sys -from distutils import command +print() +print("This is the broadcast test suite log comparator") from cylc.cylc_subproc import procopen -print -print "This is the broadcast test suite log comparator" - event, suite = sys.argv[1], sys.argv[2] if event != 'shutdown': raise SystemExit("ERROR: run this as a shutdown event handler") @@ -17,8 +14,8 @@ try: log_dir = os.path.expandvars(os.environ['CYLC_SUITE_LOG_DIR']) suite_dir = os.path.expandvars(os.environ['CYLC_SUITE_DEF_PATH']) -except KeyError, x: - raise SystemExit(x) +except KeyError as exc: + raise SystemExit(exc) ref = os.path.join(suite_dir, 'broadcast.ref') log = os.path.join(suite_dir, 'broadcast.log') @@ -35,7 +32,7 @@ if reflines != loglines: sys.exit("ERROR: broadcast logs do not compare") else: - print "broadcast logs compare OK" + print("broadcast logs compare OK") res = procopen(["cylc check-triggering " + event + " " + suite], usesh=True) status = res.wait() diff --git a/tests/cylc-cat-state/basic/state-check.py b/tests/cylc-cat-state/basic/state-check.py index fac319f65fa..1172cc049b4 100644 --- a/tests/cylc-cat-state/basic/state-check.py +++ b/tests/cylc-cat-state/basic/state-check.py @@ -1,6 +1,6 @@ # THIS FILE IS PART OF THE CYLC SUITE ENGINE. # Copyright (C) 2008-2019 NIWA & British Crown (Met Office) & Contributors. -# +# # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or @@ -25,7 +25,7 @@ def main(argv): if len(argv) != 2: - print >> sys.stderr, "Incorrect number of args" + print("Incorrect number of args", sys.stderr) sys.exit(1) sname = argv[0] @@ -33,10 +33,10 @@ def main(argv): command = "cylc cat-state " + sname p = procopen(command, usesh=True, stdoutpipe=True, stderrpipe=True) - state, err = p.communicate() + state, err = (f.decode() for f in p.communicate()) if p.returncode > 0: - print >> sys.stderr, err + print(err, sys.stderr) sys.exit(1) db = (os.sep).join([rundir, sname, "log", "db"]) @@ -51,7 +51,7 @@ def main(argv): error_states = [] for line in state: - if states_begun and line is not '': + if states_begun and line != '': line2 = line.split(':') task_and_cycle = line2[0].strip().split(".") status = line2[1].split(',')[0].strip().split("=")[1] @@ -59,11 +59,11 @@ def main(argv): res = [] try: cur.execute(qbase, [task_and_cycle[0], task_and_cycle[1]]) - next = cur.fetchmany() - while next: - res.append(next[0]) - next = cur.fetchmany() - except: # noqa: E722 + next_ = cur.fetchmany() + while next_: + res.append(next_[0]) + next_ = cur.fetchmany() + except Exception: sys.stderr.write("unable to query suite database\n") sys.exit(1) if not res[0][0] == status: @@ -75,10 +75,12 @@ def main(argv): cnx.close() if error_states: - st = "The following task states were not consistent with the database:" - print >> sys.stderr, st + print( + "The following task states were not consistent with the database:", + sys.stderr + ) for line in error_states: - print >> sys.stderr, line + print(line, sys.stderr) sys.exit(1) else: sys.exit(0) From a90e93157679d8866042e71e6b620e03f83cbf94 Mon Sep 17 00:00:00 2001 From: Martin Ryan Date: Wed, 13 Mar 2019 16:51:46 +1100 Subject: [PATCH 79/83] fix pycodestyle errors no xrange in python3, space around + sign --- etc/dev-bin/defn-order-test.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/etc/dev-bin/defn-order-test.py b/etc/dev-bin/defn-order-test.py index 8718c42dd42..716190689a4 100755 --- a/etc/dev-bin/defn-order-test.py +++ b/etc/dev-bin/defn-order-test.py @@ -17,14 +17,14 @@ names = [] for i in range(0, N): names.append(''.join(secrets.choice(string.ascii_letters) - for n in xrange(5+randrange(10)))) + for n in range(5 + secrets.randrange(10)))) # N lists with 2-7 names each (c.f. tree view paths of the inheritance # hierarchy). paths1 = [] for i in range(0, N): p = [] - for j in range(0, 2+secrets.randrange(6)): + for j in range(0, 2 + secrets.randrange(6)): z = secrets.randrange(0, N) p.append(names[z]) paths1.append(p) From db0cfbd8870b50cc10605db9512199bb85c586de Mon Sep 17 00:00:00 2001 From: Martin Ryan Date: Thu, 14 Mar 2019 10:16:10 +1100 Subject: [PATCH 80/83] revert jinja2 changes jinja2 will be unbundled soon --- lib/jinja2/ext.py | 3 +- lib/jinja2/filters.py | 55 ++++++++++++++----------------------- lib/parsec/jinja2support.py | 8 +----- 3 files changed, 23 insertions(+), 43 deletions(-) diff --git a/lib/jinja2/ext.py b/lib/jinja2/ext.py index 60c7855d7ca..0734a84f73d 100644 --- a/lib/jinja2/ext.py +++ b/lib/jinja2/ext.py @@ -596,8 +596,7 @@ def getbool(options, key, default=False): getbool(options, 'keep_trailing_newline', KEEP_TRAILING_NEWLINE), frozenset(extensions), cache_size=0, - auto_reload=False, - autoescape=True + auto_reload=False ) if getbool(options, 'trimmed'): diff --git a/lib/jinja2/filters.py b/lib/jinja2/filters.py index e671b58cd92..267ddddaa03 100644 --- a/lib/jinja2/filters.py +++ b/lib/jinja2/filters.py @@ -8,22 +8,18 @@ :copyright: (c) 2017 by the Jinja Team. :license: BSD, see LICENSE for more details. """ -import math import re +import math +import random import warnings -from collections import namedtuple -from itertools import chain, groupby -from jinja2._compat import PY2, imap, iteritems, string_types, text_type -from jinja2.exceptions import FilterArgumentError +from itertools import groupby, chain +from collections import namedtuple +from jinja2.utils import Markup, escape, pformat, urlize, soft_unicode, \ + unicode_urlencode, htmlsafe_json_dumps from jinja2.runtime import Undefined -from jinja2.utils import (Markup, escape, htmlsafe_json_dumps, pformat, - soft_unicode, unicode_urlencode, urlize) - -try: - from secrets import choice, randrange -except ImportError: - from random import choice, randrange +from jinja2.exceptions import FilterArgumentError +from jinja2._compat import imap, string_types, text_type, iteritems, PY2 _word_re = re.compile(r'\w+', re.UNICODE) @@ -72,8 +68,7 @@ def make_attrgetter(environment, attribute, postprocess=None): if attribute is None: attribute = [] elif isinstance(attribute, string_types): - attribute = [int(x) if x.isdigit() - else x for x in attribute.split('.')] + attribute = [int(x) if x.isdigit() else x for x in attribute.split('.')] else: attribute = [attribute] @@ -456,10 +451,9 @@ def do_last(environment, seq): def do_random(context, seq): """Return a random item from the sequence.""" try: - return choice(seq) + return random.choice(seq) except IndexError: - return context.environment.undefined( - 'No random item, sequence was empty.') + return context.environment.undefined('No random item, sequence was empty.') def do_filesizeformat(value, binary=False): @@ -607,8 +601,7 @@ def do_truncate(env, s, length=255, killwords=False, end='...', leeway=None): """ if leeway is None: leeway = env.policies['truncate.leeway'] - assert length >= len( - end), 'expected length >= %s, got %s' % (len(end), length) + assert length >= len(end), 'expected length >= %s, got %s' % (len(end), length) assert leeway >= 0, 'expected leeway >= 0, got %s' % leeway if len(s) <= length + leeway: return s @@ -636,8 +629,8 @@ def do_wordwrap(environment, s, width=79, break_long_words=True, wrapstring = environment.newline_sequence import textwrap return wrapstring.join(textwrap.wrap(s, width=width, expand_tabs=False, - replace_whitespace=False, - break_long_words=break_long_words)) + replace_whitespace=False, + break_long_words=break_long_words)) def do_wordcount(s): @@ -798,7 +791,7 @@ def do_round(value, precision=0, method='common'): {{ 42.55|round|int }} -> 43 """ - if method not in ('common', 'ceil', 'floor'): + if not method in ('common', 'ceil', 'floor'): raise FilterArgumentError('method must be common, ceil or floor') if method == 'common': return round(value, precision) @@ -815,7 +808,6 @@ def do_round(value, precision=0, method='common'): _GroupTuple.__repr__ = tuple.__repr__ _GroupTuple.__str__ = tuple.__str__ - @environmentfilter def do_groupby(environment, value, attribute): """Group a sequence of objects by a common attribute. @@ -896,8 +888,7 @@ def do_mark_safe(value): def do_mark_unsafe(value): - """Mark a value as unsafe. This is the reverse operation for :func:`safe`. - """ + """Mark a value as unsafe. This is the reverse operation for :func:`safe`.""" return text_type(value) @@ -924,8 +915,7 @@ def do_attr(environment, obj, name): ``foo.bar`` just that always an attribute is returned and items are not looked up. - See :ref:`Notes on subscriptions ` for more - details. + See :ref:`Notes on subscriptions ` for more details. """ try: name = str(name) @@ -1096,7 +1086,7 @@ def prepare_map(args, kwargs): attribute = kwargs.pop('attribute') if kwargs: raise FilterArgumentError('Unexpected keyword argument %r' % - next(iter(kwargs))) + next(iter(kwargs))) func = make_attrgetter(context.environment, attribute) else: try: @@ -1104,8 +1094,7 @@ def prepare_map(args, kwargs): args = args[3:] except LookupError: raise FilterArgumentError('map requires a filter argument') - - def func(item): return context.environment.call_filter( + func = lambda item: context.environment.call_filter( name, item, args, kwargs, context=context) return seq, func @@ -1123,14 +1112,12 @@ def prepare_select_or_reject(args, kwargs, modfunc, lookup_attr): off = 1 else: off = 0 - - def transfunc(x): return x + transfunc = lambda x: x try: name = args[2 + off] args = args[3 + off:] - - def func(item): return context.environment.call_test( + func = lambda item: context.environment.call_test( name, item, args, kwargs) except LookupError: func = bool diff --git a/lib/parsec/jinja2support.py b/lib/parsec/jinja2support.py index 7b8b116e558..3e263d69492 100644 --- a/lib/parsec/jinja2support.py +++ b/lib/parsec/jinja2support.py @@ -24,7 +24,6 @@ import os import sys from jinja2 import ( - select_autoescape, BaseLoader, ChoiceLoader, Environment, @@ -94,14 +93,9 @@ def jinja2environment(dir_=None): dir_ = os.getcwd() env = Environment( - autoescape=select_autoescape( - enabled_extensions=(), - default_for_string=False, - default=False), loader=ChoiceLoader([FileSystemLoader(dir_), PyModuleLoader()]), undefined=StrictUndefined, - extensions=['jinja2.ext.do'] - ) + extensions=['jinja2.ext.do']) # Load any custom Jinja2 filters, tests or globals in the suite # definition directory From 33bbc0c73062d95aef5291d2081d9fc88a4429f0 Mon Sep 17 00:00:00 2001 From: Martin Ryan Date: Thu, 14 Mar 2019 16:48:32 +1100 Subject: [PATCH 81/83] orderedDict import --- lib/cylc/batch_sys_manager.py | 2 +- lib/cylc/profiling/profile.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/cylc/batch_sys_manager.py b/lib/cylc/batch_sys_manager.py index ae91dc473e2..9e681a5bd65 100644 --- a/lib/cylc/batch_sys_manager.py +++ b/lib/cylc/batch_sys_manager.py @@ -124,7 +124,7 @@ JOB_LOG_STATUS) from cylc.task_outputs import TASK_OUTPUT_SUCCEEDED from cylc.wallclock import get_current_time_string -from parsec import OrderedDict +from parsec.OrderedDict import OrderedDict class JobPollContext(object): diff --git a/lib/cylc/profiling/profile.py b/lib/cylc/profiling/profile.py index 1463e4041c4..2342f975d73 100644 --- a/lib/cylc/profiling/profile.py +++ b/lib/cylc/profiling/profile.py @@ -240,8 +240,8 @@ def run_experiment(exp): conf_path=run.get('globalrc', '')) # Handle errors. if err_file: - print(('WARNING: non-empty suite error log: ' - + err_file), file=sys.stderr) + print(('WARNING: non-empty suite error log: ' + + err_file), file=sys.stderr) # Tidy up. if cylc_maj_version == '6': unregister_suite(reg) From c934fc52ac16cc866048b7cdd3ae6a4a0275b911 Mon Sep 17 00:00:00 2001 From: Martin Ryan Date: Mon, 18 Mar 2019 11:43:07 +1100 Subject: [PATCH 82/83] fix for 01-help.t error --- bin/cylc-check-versions | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/bin/cylc-check-versions b/bin/cylc-check-versions index 69dcc37dc7a..f35f0c5eb08 100755 --- a/bin/cylc-check-versions +++ b/bin/cylc-check-versions @@ -41,12 +41,14 @@ from time import sleep import cylc.flags from cylc.config import SuiteConfig +from cylc.option_parsers import CylcOptionParser as COP from cylc.cylc_subproc import procopen +from cylc import __version__ as CYLC_VERSION +from cylc.config import SuiteConfig from cylc.subprocpool import SubProcPool from cylc.suite_srv_files_mgr import SuiteSrvFilesManager from cylc.task_remote_mgr import TaskRemoteMgr from cylc.templatevars import load_template_vars -from cylc.version import CYLC_VERSION def main(): From 04d749a20b376fb18e653405595a3fbcc5a0e9ad Mon Sep 17 00:00:00 2001 From: Hilary James Oliver Date: Wed, 20 Mar 2019 22:59:41 +1300 Subject: [PATCH 83/83] Use stdlib mock. --- .travis/install.sh | 2 +- lib/cylc/tests/test_cylc_subproc.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.travis/install.sh b/.travis/install.sh index b66d06f362f..b5ae3abbe8f 100755 --- a/.travis/install.sh +++ b/.travis/install.sh @@ -33,7 +33,7 @@ if grep -E '(unit-tests|functional-tests)' <<< "${args[@]}"; then fi if grep 'unit-tests' <<< "${args[@]}"; then - pip install pycodestyle pytest mock testfixtures + pip install pycodestyle pytest testfixtures # TODO: EmPy removed from testing, see: #2958 fi diff --git a/lib/cylc/tests/test_cylc_subproc.py b/lib/cylc/tests/test_cylc_subproc.py index e2f21a3d45b..c3de4c0092d 100644 --- a/lib/cylc/tests/test_cylc_subproc.py +++ b/lib/cylc/tests/test_cylc_subproc.py @@ -19,7 +19,7 @@ import unittest from cylc.cylc_subproc import procopen -from mock import call +from unittest.mock import call from testfixtures import compare from testfixtures.popen import PIPE, MockPopen